starred/pkl
1
0
Fork 0
mirror of https://github.com/apple/pkl.git synced 2026-04-22 16:28:34 +02:00
Code Issues 198 Packages Projects Releases 10 Wiki Activity

Implement SPICE-0009 External Readers (#660)

Browse Source 
This adds a new feature, which allows Pkl to read resources and modules from external processes.

Follows the design laid out in SPICE-0009.

Also, this moves most of the messaging API into pkl-core
This commit is contained in:
Josh B
2024-10-28 18:22:14 -07:00
committed by GitHub
parent 466ae6fd4c
commit 666f8c3939
110 changed files with 4368 additions and 1810 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
pkl-core/src/main/java/org/pkl/core/EvaluatorBuilder.java
View File

@@ -20,6 +20,9 @@ import java.nio.file.Path;
import java.util.*;
import java.util.regex.Pattern;
import org.pkl.core.SecurityManagers.StandardBuilder;
import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader;
import org.pkl.core.externalreader.ExternalReaderProcess;
import org.pkl.core.externalreader.ExternalReaderProcessImpl;
import org.pkl.core.http.HttpClient;
import org.pkl.core.module.ModuleKeyFactories;
import org.pkl.core.module.ModuleKeyFactory;
@@ -478,6 +481,25 @@ public final class EvaluatorBuilder {
} else if (settings.moduleCacheDir() != null) {
setModuleCacheDir(settings.moduleCacheDir());
}
// this isn't ideal as project and non-project ExternalProcessImpl instances can be dupes
var procs = new HashMap<ExternalReader, ExternalReaderProcess>();
if (settings.externalModuleReaders() != null) {
for (var entry : settings.externalModuleReaders().entrySet()) {
addModuleKeyFactory(
ModuleKeyFactories.externalProcess(
entry.getKey(),
procs.computeIfAbsent(entry.getValue(), ExternalReaderProcessImpl::new)));
}
}
if (settings.externalResourceReaders() != null) {
for (var entry : settings.externalResourceReaders().entrySet()) {
addResourceReader(
ResourceReaders.externalProcess(
entry.getKey(),
procs.computeIfAbsent(entry.getValue(), ExternalReaderProcessImpl::new)));
}
}
return this;
}

7
pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java
View File

@@ -55,6 +55,7 @@ import org.pkl.core.ast.internal.ToStringNodeGen;
import org.pkl.core.ast.lambda.ApplyVmFunction1NodeGen;
import org.pkl.core.ast.member.*;
import org.pkl.core.ast.type.*;
import org.pkl.core.externalreader.ExternalReaderProcessException;
import org.pkl.core.module.ModuleKey;
import org.pkl.core.module.ModuleKeys;
import org.pkl.core.module.ResolvedModuleKey;
@@ -1847,6 +1848,12 @@ public final class AstBuilder extends AbstractAstBuilder<Object> {
.withHint(e.getHint())
.withSourceSection(createSourceSection(importUriCtx))
.build();
} catch (ExternalReaderProcessException e) {
throw exceptionBuilder()
.evalError("externalReaderFailure")
.withCause(e.getCause())
.withSourceSection(createSourceSection(importUriCtx))
.build();
}
if (!resolvedUri.isAbsolute()) {

3
pkl-core/src/main/java/org/pkl/core/ast/expression/unary/AbstractReadNode.java
View File

@@ -23,6 +23,7 @@ import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import org.pkl.core.SecurityManagerException;
import org.pkl.core.externalreader.ExternalReaderProcessException;
import org.pkl.core.module.ModuleKey;
import org.pkl.core.packages.PackageLoadError;
import org.pkl.core.runtime.VmContext;
@@ -75,6 +76,8 @@ public abstract class AbstractReadNode extends UnaryExpressionNode {
.build();
} catch (PackageLoadError | SecurityManagerException e) {
throw exceptionBuilder().withCause(e).build();
} catch (ExternalReaderProcessException e) {
throw exceptionBuilder().evalError("externalReaderFailure").withCause(e).build();
}
if (!resolvedUri.isAbsolute()) {

3
pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ImportGlobNode.java
View File

@@ -25,6 +25,7 @@ import java.io.IOException;
import java.net.URI;
import org.pkl.core.SecurityManagerException;
import org.pkl.core.ast.member.SharedMemberNode;
import org.pkl.core.externalreader.ExternalReaderProcessException;
import org.pkl.core.http.HttpClientInitException;
import org.pkl.core.module.ResolvedModuleKey;
import org.pkl.core.packages.PackageLoadError;
@@ -104,6 +105,8 @@ public class ImportGlobNode extends AbstractImportNode {
.evalError("invalidGlobPattern", globPattern)
.withHint(e.getMessage())
.build();
} catch (ExternalReaderProcessException e) {
throw exceptionBuilder().evalError("externalReaderFailure").withCause(e).build();
}
}
}

3
pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ReadGlobNode.java
View File

@@ -26,6 +26,7 @@ import java.net.URISyntaxException;
import org.graalvm.collections.EconomicMap;
import org.pkl.core.SecurityManagerException;
import org.pkl.core.ast.member.SharedMemberNode;
import org.pkl.core.externalreader.ExternalReaderProcessException;
import org.pkl.core.http.HttpClientInitException;
import org.pkl.core.module.ModuleKey;
import org.pkl.core.runtime.VmContext;
@@ -103,6 +104,8 @@ public abstract class ReadGlobNode extends AbstractReadNode {
.evalError("invalidGlobPattern", globPattern)
.withHint(e.getMessage())
.build();
} catch (ExternalReaderProcessException e) {
throw exceptionBuilder().evalError("externalReaderFailure").withCause(e).build();
}
}
}

40
pkl-core/src/main/java/org/pkl/core/evaluatorSettings/PklEvaluatorSettings.java
View File

@@ -20,9 +20,11 @@ import java.net.URISyntaxException;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.function.BiFunction;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.pkl.core.Duration;
import org.pkl.core.PNull;
import org.pkl.core.PObject;
@@ -43,7 +45,9 @@ public record PklEvaluatorSettings(
@Nullable List<Path> modulePath,
@Nullable Duration timeout,
@Nullable Path rootDir,
@Nullable Http http) {
@Nullable Http http,
@Nullable Map<String, ExternalReader> externalModuleReaders,
@Nullable Map<String, ExternalReader> externalResourceReaders) {
/** Initializes a {@link PklEvaluatorSettings} from a raw object representation. */
@SuppressWarnings("unchecked")
@@ -80,6 +84,24 @@ public record PklEvaluatorSettings(
var rootDirStr = (String) pSettings.get("rootDir");
var rootDir = rootDirStr == null ? null : pathNormalizer.apply(rootDirStr, "rootDir");
var externalModuleReadersRaw = (Map<String, Value>) pSettings.get("externalModuleReaders");
var externalModuleReaders =
externalModuleReadersRaw == null
? null
: externalModuleReadersRaw.entrySet().stream()
.collect(
Collectors.toMap(
Entry::getKey, entry -> ExternalReader.parse(entry.getValue())));
var externalResourceReadersRaw = (Map<String, Value>) pSettings.get("externalResourceReaders");
var externalResourceReaders =
externalResourceReadersRaw == null
? null
: externalResourceReadersRaw.entrySet().stream()
.collect(
Collectors.toMap(
Entry::getKey, entry -> ExternalReader.parse(entry.getValue())));
return new PklEvaluatorSettings(
(Map<String, String>) pSettings.get("externalProperties"),
(Map<String, String>) pSettings.get("env"),
@@ -90,7 +112,9 @@ public record PklEvaluatorSettings(
modulePath,
(Duration) pSettings.get("timeout"),
rootDir,
Http.parse((Value) pSettings.get("http")));
Http.parse((Value) pSettings.get("http")),
externalModuleReaders,
externalResourceReaders);
}
public record Http(@Nullable Proxy proxy) {
@@ -133,6 +157,18 @@ public record PklEvaluatorSettings(
}
}
public record ExternalReader(String executable, @Nullable List<String> arguments) {
@SuppressWarnings("unchecked")
public static ExternalReader parse(Value input) {
if (input instanceof PObject externalReader) {
var executable = (String) externalReader.getProperty("executable");
var arguments = (List<String>) externalReader.get("arguments");
return new ExternalReader(executable, arguments);
}
throw PklBugException.unreachableCode();
}
}
private boolean arePatternsEqual(
@Nullable List<Pattern> thesePatterns, @Nullable List<Pattern> thosePatterns) {
if (thesePatterns == null) {

61
pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderMessagePackDecoder.java Normal file
View File

@@ -0,0 +1,61 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.externalreader;
import java.io.InputStream;
import java.net.URISyntaxException;
import java.util.Map;
import org.msgpack.core.MessagePack;
import org.msgpack.core.MessageUnpacker;
import org.msgpack.value.Value;
import org.pkl.core.externalreader.ExternalReaderMessages.*;
import org.pkl.core.messaging.BaseMessagePackDecoder;
import org.pkl.core.messaging.DecodeException;
import org.pkl.core.messaging.Message;
import org.pkl.core.messaging.Message.Type;
import org.pkl.core.util.Nullable;
public class ExternalReaderMessagePackDecoder extends BaseMessagePackDecoder {
public ExternalReaderMessagePackDecoder(MessageUnpacker unpacker) {
super(unpacker);
}
public ExternalReaderMessagePackDecoder(InputStream inputStream) {
this(MessagePack.newDefaultUnpacker(inputStream));
}
@Override
protected @Nullable Message decodeMessage(Type msgType, Map<Value, Value> map)
throws DecodeException, URISyntaxException {
return switch (msgType) {
case INITIALIZE_MODULE_READER_REQUEST ->
new InitializeModuleReaderRequest(
unpackLong(map, "requestId"), unpackString(map, "scheme"));
case INITIALIZE_RESOURCE_READER_REQUEST ->
new InitializeResourceReaderRequest(
unpackLong(map, "requestId"), unpackString(map, "scheme"));
case INITIALIZE_MODULE_READER_RESPONSE ->
new InitializeModuleReaderResponse(
unpackLong(map, "requestId"), unpackModuleReaderSpec(getNullable(map, "spec")));
case INITIALIZE_RESOURCE_READER_RESPONSE ->
new InitializeResourceReaderResponse(
unpackLong(map, "requestId"), unpackResourceReaderSpec(getNullable(map, "spec")));
case CLOSE_EXTERNAL_PROCESS -> new CloseExternalProcess();
default -> super.decodeMessage(msgType, map);
};
}
}

75
pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderMessagePackEncoder.java Normal file
View File

@@ -0,0 +1,75 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.externalreader;
import java.io.IOException;
import java.io.OutputStream;
import org.msgpack.core.MessagePack;
import org.msgpack.core.MessagePacker;
import org.pkl.core.externalreader.ExternalReaderMessages.*;
import org.pkl.core.messaging.BaseMessagePackEncoder;
import org.pkl.core.messaging.Message;
import org.pkl.core.messaging.ProtocolException;
import org.pkl.core.util.Nullable;
public class ExternalReaderMessagePackEncoder extends BaseMessagePackEncoder {
public ExternalReaderMessagePackEncoder(MessagePacker packer) {
super(packer);
}
public ExternalReaderMessagePackEncoder(OutputStream outputStream) {
this(MessagePack.newDefaultPacker(outputStream));
}
@Override
protected @Nullable void encodeMessage(Message msg) throws ProtocolException, IOException {
switch (msg.type()) {
case INITIALIZE_MODULE_READER_REQUEST -> {
var m = (InitializeModuleReaderRequest) msg;
packer.packMapHeader(2);
packKeyValue("requestId", m.requestId());
packKeyValue("scheme", m.scheme());
}
case INITIALIZE_RESOURCE_READER_REQUEST -> {
var m = (InitializeResourceReaderRequest) msg;
packer.packMapHeader(2);
packKeyValue("requestId", m.requestId());
packKeyValue("scheme", m.scheme());
}
case INITIALIZE_MODULE_READER_RESPONSE -> {
var m = (InitializeModuleReaderResponse) msg;
packMapHeader(1, m.spec());
packKeyValue("requestId", m.requestId());
if (m.spec() != null) {
packer.packString("spec");
packModuleReaderSpec(m.spec());
}
}
case INITIALIZE_RESOURCE_READER_RESPONSE -> {
var m = (InitializeResourceReaderResponse) msg;
packMapHeader(1, m.spec());
packKeyValue("requestId", m.requestId());
if (m.spec() != null) {
packer.packString("spec");
packResourceReaderSpec(m.spec());
}
}
case CLOSE_EXTERNAL_PROCESS -> packer.packMapHeader(0);
default -> super.encodeMessage(msg);
}
}
}

58
pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderMessages.java Normal file
View File

@@ -0,0 +1,58 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.externalreader;
import org.pkl.core.messaging.Message.*;
import org.pkl.core.messaging.Messages.ModuleReaderSpec;
import org.pkl.core.messaging.Messages.ResourceReaderSpec;
import org.pkl.core.util.Nullable;
public class ExternalReaderMessages {
public record InitializeModuleReaderRequest(long requestId, String scheme)
implements Server.Request {
public Type type() {
return Type.INITIALIZE_MODULE_READER_REQUEST;
}
}
public record InitializeResourceReaderRequest(long requestId, String scheme)
implements Server.Request {
public Type type() {
return Type.INITIALIZE_RESOURCE_READER_REQUEST;
}
}
public record InitializeModuleReaderResponse(long requestId, @Nullable ModuleReaderSpec spec)
implements Client.Response {
public Type type() {
return Type.INITIALIZE_MODULE_READER_RESPONSE;
}
}
public record InitializeResourceReaderResponse(long requestId, @Nullable ResourceReaderSpec spec)
implements Client.Response {
public Type type() {
return Type.INITIALIZE_RESOURCE_READER_RESPONSE;
}
}
public record CloseExternalProcess() implements Server.OneWay {
public Type type() {
return Type.CLOSE_EXTERNAL_PROCESS;
}
}
}

54
pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderProcess.java Normal file
View File

@@ -0,0 +1,54 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.externalreader;
import java.io.IOException;
import org.pkl.core.messaging.MessageTransport;
import org.pkl.core.messaging.Messages.ModuleReaderSpec;
import org.pkl.core.messaging.Messages.ResourceReaderSpec;
import org.pkl.core.util.Nullable;
/** An interface for interacting with external module/resource processes. */
public interface ExternalReaderProcess extends AutoCloseable {
/**
* Obtain the process's underlying {@link MessageTransport} for sending reader-specific message
*
* <p>May allocate resources upon first call, including spawning a child process. Must not be
* called after {@link ExternalReaderProcess#close} has been called.
*/
MessageTransport getTransport() throws ExternalReaderProcessException;
/** Retrieve the spec, if available, of the process's module reader with the given scheme. */
@Nullable
ModuleReaderSpec getModuleReaderSpec(String scheme) throws IOException;
/** Retrieve the spec, if available, of the process's resource reader with the given scheme. */
@Nullable
ResourceReaderSpec getResourceReaderSpec(String scheme) throws IOException;
/**
* Close the external process, cleaning up any resources.
*
* <p>The {@link MessageTransport} is sent the {@link ExternalReaderMessages.CloseExternalProcess}
* message to request a graceful stop. A bespoke (empty) message type is used here instead of an
* OS mechanism like signals to avoid forcing external reader implementers needing to handle many
* OS-specific mechanisms. Implementations may then forcibly clean up resources after a timeout.
* Must be safe to call multiple times.
*/
@Override
void close();
}

26
pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderProcessException.java Normal file
View File

@@ -0,0 +1,26 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.externalreader;
public final class ExternalReaderProcessException extends Exception {
public ExternalReaderProcessException(String msg) {
super(msg);
}
public ExternalReaderProcessException(Throwable cause) {
super(cause);
}
}

226
pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderProcessImpl.java Normal file
View File

@@ -0,0 +1,226 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.externalreader;
import java.io.IOException;
import java.lang.ProcessBuilder.Redirect;
import java.util.ArrayList;
import java.util.Map;
import java.util.Objects;
import java.util.Random;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Future;
import javax.annotation.concurrent.GuardedBy;
import org.pkl.core.Duration;
import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader;
import org.pkl.core.externalreader.ExternalReaderMessages.*;
import org.pkl.core.messaging.MessageTransport;
import org.pkl.core.messaging.MessageTransports;
import org.pkl.core.messaging.Messages.ModuleReaderSpec;
import org.pkl.core.messaging.Messages.ResourceReaderSpec;
import org.pkl.core.messaging.ProtocolException;
import org.pkl.core.util.LateInit;
import org.pkl.core.util.Nullable;
public class ExternalReaderProcessImpl implements ExternalReaderProcess {
private static final Duration CLOSE_TIMEOUT = Duration.ofSeconds(3);
private final ExternalReader spec;
private final @Nullable String logPrefix;
private final Map<String, Future<@Nullable ModuleReaderSpec>> initializeModuleReaderResponses =
new ConcurrentHashMap<>();
private final Map<String, Future<@Nullable ResourceReaderSpec>>
initializeResourceReaderResponses = new ConcurrentHashMap<>();
private @GuardedBy("this") boolean closed = false;
@LateInit
@GuardedBy("this")
private Process process;
@LateInit
@GuardedBy("this")
private MessageTransport transport;
private void log(String msg) {
if (logPrefix != null) {
System.err.println(logPrefix + msg);
}
}
public ExternalReaderProcessImpl(ExternalReader spec) {
this.spec = spec;
logPrefix =
Objects.equals(System.getenv("PKL_DEBUG"), "1")
? "[pkl-core][external-process][" + spec.executable() + "] "
: null;
}
@Override
public synchronized MessageTransport getTransport() throws ExternalReaderProcessException {
if (closed) {
throw new ExternalReaderProcessException("ExternalProcessImpl has already been closed");
}
if (process != null) {
if (!process.isAlive()) {
throw new ExternalReaderProcessException("ExternalProcessImpl process is no longer alive");
}
return transport;
}
// This relies on Java/OS behavior around PATH resolution, absolute/relative paths, etc.
var command = new ArrayList<String>();
command.add(spec.executable());
command.addAll(spec.arguments());
var builder = new ProcessBuilder(command);
builder.redirectError(Redirect.INHERIT); // inherit stderr from this pkl process
try {
process = builder.start();
} catch (IOException e) {
throw new ExternalReaderProcessException(e);
}
transport =
MessageTransports.stream(
new ExternalReaderMessagePackDecoder(process.getInputStream()),
new ExternalReaderMessagePackEncoder(process.getOutputStream()),
this::log);
var rxThread = new Thread(this::runTransport, "ExternalProcessImpl rxThread for " + spec);
rxThread.setDaemon(true);
rxThread.start();
return transport;
}
/**
* Runs the underlying message transport so it can receive responses from the child process.
*
* <p>Blocks until the underlying transport is closed.
*/
private void runTransport() {
try {
transport.start(
(msg) -> {
throw new ProtocolException("Unexpected incoming one-way message: " + msg);
},
(msg) -> {
throw new ProtocolException("Unexpected incoming request message: " + msg);
});
} catch (ProtocolException | IOException e) {
throw new RuntimeException(e);
}
}
@Override
public synchronized void close() {
closed = true;
if (process == null || !process.isAlive()) {
return;
}
try {
if (transport != null) {
transport.send(new CloseExternalProcess());
transport.close();
}
// forcefully stop the process after the timeout
// note that both transport.close() and process.destroy() are safe to call multiple times
new Timer()
.schedule(
new TimerTask() {
@Override
public void run() {
if (process != null) {
transport.close();
process.destroyForcibly();
}
}
},
CLOSE_TIMEOUT.inWholeMillis());
// block on process exit
process.onExit().get();
} catch (Exception e) {
transport.close();
process.destroyForcibly();
} finally {
process = null;
transport = null;
}
}
@Override
public @Nullable ModuleReaderSpec getModuleReaderSpec(String uriScheme) throws IOException {
return MessageTransports.resolveFuture(
initializeModuleReaderResponses.computeIfAbsent(
uriScheme,
(scheme) -> {
var future = new CompletableFuture<@Nullable ModuleReaderSpec>();
var request = new InitializeModuleReaderRequest(new Random().nextLong(), scheme);
try {
getTransport()
.send(
request,
(response) -> {
if (response instanceof InitializeModuleReaderResponse resp) {
future.complete(resp.spec());
} else {
future.completeExceptionally(
new ProtocolException("unexpected response"));
}
});
} catch (ProtocolException | IOException | ExternalReaderProcessException e) {
future.completeExceptionally(e);
}
return future;
}));
}
@Override
public @Nullable ResourceReaderSpec getResourceReaderSpec(String uriScheme) throws IOException {
return MessageTransports.resolveFuture(
initializeResourceReaderResponses.computeIfAbsent(
uriScheme,
(scheme) -> {
var future = new CompletableFuture<@Nullable ResourceReaderSpec>();
var request = new InitializeResourceReaderRequest(new Random().nextLong(), scheme);
try {
getTransport()
.send(
request,
(response) -> {
log(response.toString());
if (response instanceof InitializeResourceReaderResponse resp) {
future.complete(resp.spec());
} else {
future.completeExceptionally(
new ProtocolException("unexpected response"));
}
});
} catch (ProtocolException | IOException | ExternalReaderProcessException e) {
future.completeExceptionally(e);
}
return future;
}));
}
}

4
pkl-core/src/main/java/org/pkl/core/externalreader/package-info.java Normal file
View File

@@ -0,0 +1,4 @@
@NonnullByDefault
package org.pkl.core.externalreader;
import org.pkl.core.util.NonnullByDefault;

220
pkl-core/src/main/java/org/pkl/core/messaging/AbstractMessagePackDecoder.java Normal file
View File

@@ -0,0 +1,220 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.messaging;
import java.io.IOException;
import java.io.InputStream;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.msgpack.core.MessagePack;
import org.msgpack.core.MessageTypeException;
import org.msgpack.core.MessageUnpacker;
import org.msgpack.value.Value;
import org.msgpack.value.impl.ImmutableStringValueImpl;
import org.pkl.core.messaging.Message.Type;
import org.pkl.core.util.ErrorMessages;
import org.pkl.core.util.Nullable;
public abstract class AbstractMessagePackDecoder implements MessageDecoder {
protected final MessageUnpacker unpacker;
public AbstractMessagePackDecoder(MessageUnpacker unpacker) {
this.unpacker = unpacker;
}
public AbstractMessagePackDecoder(InputStream stream) {
this(MessagePack.newDefaultUnpacker(stream));
}
protected abstract @Nullable Message decodeMessage(Type msgType, Map<Value, Value> map)
throws DecodeException, URISyntaxException;
@Override
public @Nullable Message decode() throws IOException, DecodeException {
if (!unpacker.hasNext()) {
return null;
}
int code;
try {
var arraySize = unpacker.unpackArrayHeader();
if (arraySize != 2) {
throw new DecodeException(ErrorMessages.create("malformedMessageHeaderLength", arraySize));
}
code = unpacker.unpackInt();
} catch (MessageTypeException e) {
throw new DecodeException(ErrorMessages.create("malformedMessageHeaderException"), e);
}
Type msgType;
try {
msgType = Type.fromInt(code);
} catch (IllegalArgumentException e) {
throw new DecodeException(
ErrorMessages.create("malformedMessageHeaderUnrecognizedCode", Integer.toHexString(code)),
e);
}
try {
var map = unpacker.unpackValue().asMapValue().map();
var decoded = decodeMessage(msgType, map);
if (decoded != null) {
return decoded;
}
throw new DecodeException(
ErrorMessages.create("unhandledMessageCode", Integer.toHexString(code)));
} catch (MessageTypeException | URISyntaxException e) {
throw new DecodeException(ErrorMessages.create("malformedMessageBody", code), e);
}
}
protected static @Nullable Value getNullable(Map<Value, Value> map, String key) {
return map.get(new ImmutableStringValueImpl(key));
}
protected static Value get(Map<Value, Value> map, String key) throws DecodeException {
var value = map.get(new ImmutableStringValueImpl(key));
if (value == null) {
throw new DecodeException(ErrorMessages.create("missingMessageParameter", key));
}
return value;
}
protected static String unpackString(Map<Value, Value> map, String key) throws DecodeException {
return get(map, key).asStringValue().asString();
}
protected static @Nullable String unpackStringOrNull(Map<Value, Value> map, String key) {
var value = getNullable(map, key);
if (value == null) {
return null;
}
return value.asStringValue().asString();
}
protected static <T> @Nullable T unpackStringOrNull(
Map<Value, Value> map, String key, Function<String, T> mapper) {
var value = getNullable(map, key);
if (value == null) {
return null;
}
return mapper.apply(value.asStringValue().asString());
}
protected static byte @Nullable [] unpackByteArray(Map<Value, Value> map, String key) {
var value = getNullable(map, key);
if (value == null) {
return null;
}
return value.asBinaryValue().asByteArray();
}
protected static boolean unpackBoolean(Map<Value, Value> map, String key) throws DecodeException {
return get(map, key).asBooleanValue().getBoolean();
}
protected static int unpackInt(Map<Value, Value> map, String key) throws DecodeException {
return get(map, key).asIntegerValue().asInt();
}
protected static long unpackLong(Map<Value, Value> map, String key) throws DecodeException {
return get(map, key).asIntegerValue().asLong();
}
protected static @Nullable Long unpackLongOrNull(Map<Value, Value> map, String key) {
var value = getNullable(map, key);
if (value == null) {
return null;
}
return value.asIntegerValue().asLong();
}
protected static <T> @Nullable T unpackLongOrNull(
Map<Value, Value> map, String key, Function<Long, T> mapper) {
var value = unpackLongOrNull(map, key);
if (value == null) {
return null;
}
return mapper.apply(value);
}
protected static @Nullable List<String> unpackStringListOrNull(
Map<Value, Value> map, String key) {
var value = getNullable(map, key);
if (value == null) {
return null;
}
return value.asArrayValue().list().stream().map((it) -> it.asStringValue().asString()).toList();
}
protected static @Nullable Map<String, String> unpackStringMapOrNull(
Map<Value, Value> map, String key) {
var value = getNullable(map, key);
if (value == null) {
return null;
}
return value.asMapValue().entrySet().stream()
.collect(
Collectors.toMap(
(e) -> e.getKey().asStringValue().asString(),
(e) -> e.getValue().asStringValue().asString()));
}
protected static <T> @Nullable List<T> unpackStringListOrNull(
Map<Value, Value> map, String key, Function<String, T> mapper) {
var value = unpackStringListOrNull(map, key);
if (value == null) {
return null;
}
return value.stream().map(mapper).toList();
}
protected static <T> @Nullable List<T> unpackListOrNull(
Map<Value, Value> map, String key, Function<Value, T> mapper) {
var keys = getNullable(map, key);
if (keys == null) {
return null;
}
var result = new ArrayList<T>(keys.asArrayValue().size());
for (Value value : keys.asArrayValue()) {
result.add(mapper.apply(value));
}
return result;
}
protected static <T> @Nullable Map<String, T> unpackStringMapOrNull(
Map<Value, Value> map, String key, Function<Map<Value, Value>, T> mapper) {
var value = getNullable(map, key);
if (value == null) {
return null;
}
return value.asMapValue().entrySet().stream()
.collect(
Collectors.toMap(
(e) -> e.getKey().asStringValue().asString(),
(e) -> mapper.apply(e.getValue().asMapValue().map())));
}
}

184
pkl-core/src/main/java/org/pkl/core/messaging/AbstractMessagePackEncoder.java Normal file
View File

@@ -0,0 +1,184 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.messaging;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Collection;
import java.util.Map;
import java.util.function.Function;
import org.msgpack.core.MessagePack;
import org.msgpack.core.MessagePacker;
import org.pkl.core.util.Nullable;
public abstract class AbstractMessagePackEncoder implements MessageEncoder {
protected final MessagePacker packer;
public AbstractMessagePackEncoder(MessagePacker packer) {
this.packer = packer;
}
public AbstractMessagePackEncoder(OutputStream stream) {
this(MessagePack.newDefaultPacker(stream));
}
protected abstract @Nullable void encodeMessage(Message msg)
throws ProtocolException, IOException;
@Override
public final void encode(Message msg) throws IOException, ProtocolException {
packer.packArrayHeader(2);
packer.packInt(msg.type().getCode());
encodeMessage(msg);
packer.flush();
}
protected void packMapHeader(int size, @Nullable Object value1) throws IOException {
packer.packMapHeader(size + (value1 != null ? 1 : 0));
}
protected void packMapHeader(int size, @Nullable Object value1, @Nullable Object value2)
throws IOException {
packer.packMapHeader(size + (value1 != null ? 1 : 0) + (value2 != null ? 1 : 0));
}
protected void packMapHeader(
int size,
@Nullable Object value1,
@Nullable Object value2,
@Nullable Object value3,
@Nullable Object value4,
@Nullable Object value5,
@Nullable Object value6,
@Nullable Object value7,
@Nullable Object value8,
@Nullable Object value9,
@Nullable Object valueA,
@Nullable Object valueB,
@Nullable Object valueC,
@Nullable Object valueD,
@Nullable Object valueE,
@Nullable Object valueF)
throws IOException {
packer.packMapHeader(
size
+ (value1 != null ? 1 : 0)
+ (value2 != null ? 1 : 0)
+ (value3 != null ? 1 : 0)
+ (value4 != null ? 1 : 0)
+ (value5 != null ? 1 : 0)
+ (value6 != null ? 1 : 0)
+ (value7 != null ? 1 : 0)
+ (value8 != null ? 1 : 0)
+ (value9 != null ? 1 : 0)
+ (valueA != null ? 1 : 0)
+ (valueB != null ? 1 : 0)
+ (valueC != null ? 1 : 0)
+ (valueD != null ? 1 : 0)
+ (valueE != null ? 1 : 0)
+ (valueF != null ? 1 : 0));
}
protected void packKeyValue(String name, @Nullable Integer value) throws IOException {
if (value == null) {
return;
}
packer.packString(name);
packer.packInt(value);
}
protected void packKeyValue(String name, @Nullable Long value) throws IOException {
if (value == null) {
return;
}
packer.packString(name);
packer.packLong(value);
}
protected <T> void packKeyValueLong(String name, @Nullable T value, Function<T, Long> mapper)
throws IOException {
if (value == null) {
return;
}
packKeyValue(name, mapper.apply(value));
}
protected void packKeyValue(String name, @Nullable String value) throws IOException {
if (value == null) {
return;
}
packer.packString(name);
packer.packString(value);
}
protected <T> void packKeyValueString(String name, @Nullable T value, Function<T, String> mapper)
throws IOException {
if (value == null) {
return;
}
packKeyValue(name, mapper.apply(value));
}
protected void packKeyValue(String name, @Nullable Collection<String> value) throws IOException {
if (value == null) {
return;
}
packer.packString(name);
packer.packArrayHeader(value.size());
for (String elem : value) {
packer.packString(elem);
}
}
protected <T> void packKeyValue(
String name, @Nullable Collection<T> value, Function<T, String> mapper) throws IOException {
if (value == null) {
return;
}
packer.packString(name);
packer.packArrayHeader(value.size());
for (T elem : value) {
packer.packString(mapper.apply(elem));
}
}
protected void packKeyValue(String name, @Nullable Map<String, String> value) throws IOException {
if (value == null) {
return;
}
packer.packString(name);
packer.packMapHeader(value.size());
for (Map.Entry<String, String> e : value.entrySet()) {
packer.packString(e.getKey());
packer.packString(e.getValue());
}
}
protected void packKeyValue(String name, byte @Nullable [] value) throws IOException {
if (value == null) {
return;
}
packer.packString(name);
packer.packBinaryHeader(value.length);
packer.writePayload(value);
}
protected void packKeyValue(String name, boolean value) throws IOException {
packer.packString(name);
packer.packBoolean(value);
}
}

131
pkl-core/src/main/java/org/pkl/core/messaging/BaseMessagePackDecoder.java Normal file
View File

@@ -0,0 +1,131 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.messaging;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.*;
import org.msgpack.core.MessageUnpacker;
import org.msgpack.value.Value;
import org.pkl.core.messaging.Message.Type;
import org.pkl.core.messaging.Messages.*;
import org.pkl.core.module.PathElement;
import org.pkl.core.util.Nullable;
public class BaseMessagePackDecoder extends AbstractMessagePackDecoder {
public BaseMessagePackDecoder(MessageUnpacker unpacker) {
super(unpacker);
}
public BaseMessagePackDecoder(InputStream stream) {
super(stream);
}
protected @Nullable Message decodeMessage(Type msgType, Map<Value, Value> map)
throws DecodeException, URISyntaxException {
return switch (msgType) {
case READ_RESOURCE_REQUEST ->
new ReadResourceRequest(
unpackLong(map, "requestId"),
unpackLong(map, "evaluatorId"),
new URI(unpackString(map, "uri")));
case READ_RESOURCE_RESPONSE ->
new ReadResourceResponse(
unpackLong(map, "requestId"),
unpackLong(map, "evaluatorId"),
unpackByteArray(map, "contents"),
unpackStringOrNull(map, "error"));
case READ_MODULE_REQUEST ->
new ReadModuleRequest(
unpackLong(map, "requestId"),
unpackLong(map, "evaluatorId"),
new URI(unpackString(map, "uri")));
case READ_MODULE_RESPONSE ->
new ReadModuleResponse(
unpackLong(map, "requestId"),
unpackLong(map, "evaluatorId"),
unpackStringOrNull(map, "contents"),
unpackStringOrNull(map, "error"));
case LIST_RESOURCES_REQUEST ->
new ListResourcesRequest(
unpackLong(map, "requestId"),
unpackLong(map, "evaluatorId"),
new URI(unpackString(map, "uri")));
case LIST_RESOURCES_RESPONSE ->
new ListResourcesResponse(
unpackLong(map, "requestId"),
unpackLong(map, "evaluatorId"),
unpackPathElements(map, "pathElements"),
unpackStringOrNull(map, "error"));
case LIST_MODULES_REQUEST ->
new ListModulesRequest(
unpackLong(map, "requestId"),
unpackLong(map, "evaluatorId"),
new URI(unpackString(map, "uri")));
case LIST_MODULES_RESPONSE ->
new ListModulesResponse(
unpackLong(map, "requestId"),
unpackLong(map, "evaluatorId"),
unpackPathElements(map, "pathElements"),
unpackStringOrNull(map, "error"));
default -> null;
};
}
protected static @Nullable ModuleReaderSpec unpackModuleReaderSpec(@Nullable Value value)
throws DecodeException {
if (value == null) {
return null;
}
var map = value.asMapValue().map();
return new ModuleReaderSpec(
unpackString(map, "scheme"),
unpackBoolean(map, "hasHierarchicalUris"),
unpackBoolean(map, "isLocal"),
unpackBoolean(map, "isGlobbable"));
}
protected static @Nullable ResourceReaderSpec unpackResourceReaderSpec(@Nullable Value value)
throws DecodeException {
if (value == null) {
return null;
}
var map = value.asMapValue().map();
return new ResourceReaderSpec(
unpackString(map, "scheme"),
unpackBoolean(map, "hasHierarchicalUris"),
unpackBoolean(map, "isGlobbable"));
}
protected static @Nullable List<PathElement> unpackPathElements(Map<Value, Value> map, String key)
throws DecodeException {
var value = getNullable(map, key);
if (value == null) {
return null;
}
var result = new ArrayList<PathElement>(value.asArrayValue().size());
for (Value pathElement : value.asArrayValue()) {
var pathElementMap = pathElement.asMapValue().map();
result.add(
new PathElement(
unpackString(pathElementMap, "name"), unpackBoolean(pathElementMap, "isDirectory")));
}
return result;
}
}

136
pkl-core/src/main/java/org/pkl/core/messaging/BaseMessagePackEncoder.java Normal file
View File

@@ -0,0 +1,136 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.messaging;
import java.io.IOException;
import java.io.OutputStream;
import org.msgpack.core.MessagePacker;
import org.pkl.core.messaging.Messages.*;
import org.pkl.core.module.PathElement;
import org.pkl.core.util.ErrorMessages;
import org.pkl.core.util.Nullable;
public class BaseMessagePackEncoder extends AbstractMessagePackEncoder {
public BaseMessagePackEncoder(MessagePacker packer) {
super(packer);
}
public BaseMessagePackEncoder(OutputStream stream) {
super(stream);
}
protected void packModuleReaderSpec(ModuleReaderSpec reader) throws IOException {
packer.packMapHeader(4);
packKeyValue("scheme", reader.scheme());
packKeyValue("hasHierarchicalUris", reader.hasHierarchicalUris());
packKeyValue("isLocal", reader.isLocal());
packKeyValue("isGlobbable", reader.isGlobbable());
}
protected void packResourceReaderSpec(ResourceReaderSpec reader) throws IOException {
packer.packMapHeader(3);
packKeyValue("scheme", reader.scheme());
packKeyValue("hasHierarchicalUris", reader.hasHierarchicalUris());
packKeyValue("isGlobbable", reader.isGlobbable());
}
protected void packPathElement(PathElement pathElement) throws IOException {
packer.packMapHeader(2);
packKeyValue("name", pathElement.getName());
packKeyValue("isDirectory", pathElement.isDirectory());
}
protected @Nullable void encodeMessage(Message msg) throws ProtocolException, IOException {
switch (msg.type()) {
case READ_RESOURCE_REQUEST -> {
var m = (ReadResourceRequest) msg;
packer.packMapHeader(3);
packKeyValue("requestId", m.requestId());
packKeyValue("evaluatorId", m.evaluatorId());
packKeyValue("uri", m.uri().toString());
}
case READ_RESOURCE_RESPONSE -> {
var m = (ReadResourceResponse) msg;
packMapHeader(2, m.contents(), m.error());
packKeyValue("requestId", m.requestId());
packKeyValue("evaluatorId", m.evaluatorId());
packKeyValue("contents", m.contents());
packKeyValue("error", m.error());
}
case READ_MODULE_REQUEST -> {
var m = (ReadModuleRequest) msg;
packer.packMapHeader(3);
packKeyValue("requestId", m.requestId());
packKeyValue("evaluatorId", m.evaluatorId());
packKeyValue("uri", m.uri().toString());
}
case READ_MODULE_RESPONSE -> {
var m = (ReadModuleResponse) msg;
packMapHeader(2, m.contents(), m.error());
packKeyValue("requestId", m.requestId());
packKeyValue("evaluatorId", m.evaluatorId());
packKeyValue("contents", m.contents());
packKeyValue("error", m.error());
}
case LIST_RESOURCES_REQUEST -> {
var m = (ListResourcesRequest) msg;
packer.packMapHeader(3);
packKeyValue("requestId", m.requestId());
packKeyValue("evaluatorId", m.evaluatorId());
packKeyValue("uri", m.uri().toString());
}
case LIST_RESOURCES_RESPONSE -> {
var m = (ListResourcesResponse) msg;
packMapHeader(2, m.pathElements(), m.error());
packKeyValue("requestId", m.requestId());
packKeyValue("evaluatorId", m.evaluatorId());
if (m.pathElements() != null) {
packer.packString("pathElements");
packer.packArrayHeader(m.pathElements().size());
for (var pathElement : m.pathElements()) {
packPathElement(pathElement);
}
}
packKeyValue("error", m.error());
}
case LIST_MODULES_REQUEST -> {
var m = (ListModulesRequest) msg;
packer.packMapHeader(3);
packKeyValue("requestId", m.requestId());
packKeyValue("evaluatorId", m.evaluatorId());
packKeyValue("uri", m.uri().toString());
}
case LIST_MODULES_RESPONSE -> {
var m = (ListModulesResponse) msg;
packMapHeader(2, m.pathElements(), m.error());
packKeyValue("requestId", m.requestId());
packKeyValue("evaluatorId", m.evaluatorId());
if (m.pathElements() != null) {
packer.packString("pathElements");
packer.packArrayHeader(m.pathElements().size());
for (var pathElement : m.pathElements()) {
packPathElement(pathElement);
}
}
packKeyValue("error", m.error());
}
default ->
throw new ProtocolException(
ErrorMessages.create("unhandledMessageType", msg.type().toString()));
}
}
}

27
pkl-core/src/main/java/org/pkl/core/messaging/DecodeException.java Normal file
View File

@@ -0,0 +1,27 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.messaging;
public final class DecodeException extends ProtocolException {
public DecodeException(String msg, Throwable cause) {
super(msg, cause);
}
public DecodeException(String msg) {
super(msg);
}
}

89
pkl-core/src/main/java/org/pkl/core/messaging/Message.java Normal file
View File

@@ -0,0 +1,89 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.messaging;
public interface Message {
Type type();
enum Type {
CREATE_EVALUATOR_REQUEST(0x20),
CREATE_EVALUATOR_RESPONSE(0x21),
CLOSE_EVALUATOR(0x22),
EVALUATE_REQUEST(0x23),
EVALUATE_RESPONSE(0x24),
LOG_MESSAGE(0x25),
READ_RESOURCE_REQUEST(0x26),
READ_RESOURCE_RESPONSE(0x27),
READ_MODULE_REQUEST(0x28),
READ_MODULE_RESPONSE(0x29),
LIST_RESOURCES_REQUEST(0x2a),
LIST_RESOURCES_RESPONSE(0x2b),
LIST_MODULES_REQUEST(0x2c),
LIST_MODULES_RESPONSE(0x2d),
INITIALIZE_MODULE_READER_REQUEST(0x2e),
INITIALIZE_MODULE_READER_RESPONSE(0x2f),
INITIALIZE_RESOURCE_READER_REQUEST(0x30),
INITIALIZE_RESOURCE_READER_RESPONSE(0x31),
CLOSE_EXTERNAL_PROCESS(0x32);
private final int code;
Type(int code) {
this.code = code;
}
public static Type fromInt(int val) throws IllegalArgumentException {
for (Type t : Type.values()) {
if (t.code == val) {
return t;
}
}
throw new IllegalArgumentException("Unknown Message.Type code");
}
public int getCode() {
return code;
}
}
interface OneWay extends Message {}
interface Request extends Message {
long requestId();
}
interface Response extends Message {
long requestId();
}
interface Client extends Message {
interface Request extends Client, Message.Request {}
interface Response extends Client, Message.Response {}
interface OneWay extends Client, Message.OneWay {}
}
interface Server extends Message {
interface Request extends Server, Message.Request {}
interface Response extends Server, Message.Response {}
interface OneWay extends Server, Message.OneWay {}
}
}

25
pkl-core/src/main/java/org/pkl/core/messaging/MessageDecoder.java Normal file
View File

@@ -0,0 +1,25 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.messaging;
import java.io.IOException;
import org.pkl.core.util.Nullable;
/** Decodes a stream of messages. */
public interface MessageDecoder {
@Nullable
Message decode() throws IOException, DecodeException;
}

23
pkl-core/src/main/java/org/pkl/core/messaging/MessageEncoder.java Normal file
View File

@@ -0,0 +1,23 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.messaging;
import java.io.IOException;
/** Encodes a stream of messages. */
public interface MessageEncoder {
void encode(Message msg) throws IOException, ProtocolException;
}

46
pkl-core/src/main/java/org/pkl/core/messaging/MessageTransport.java Normal file
View File

@@ -0,0 +1,46 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.messaging;
import java.io.IOException;
/** A bidirectional transport for sending and receiving messages. */
public interface MessageTransport extends AutoCloseable {
interface OneWayHandler {
void handleOneWay(Message.OneWay msg) throws ProtocolException;
}
interface RequestHandler {
void handleRequest(Message.Request msg) throws ProtocolException, IOException;
}
interface ResponseHandler {
void handleResponse(Message.Response msg) throws ProtocolException;
}
void start(OneWayHandler oneWayHandler, RequestHandler requestHandler)
throws ProtocolException, IOException;
void send(Message.OneWay message) throws ProtocolException, IOException;
void send(Message.Request message, ResponseHandler responseHandler)
throws ProtocolException, IOException;
void send(Message.Response message) throws ProtocolException, IOException;
@Override
void close();
}

197
pkl-core/src/main/java/org/pkl/core/messaging/MessageTransports.java Normal file
View File

@@ -0,0 +1,197 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.messaging;
import java.io.IOException;
import java.text.MessageFormat;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import org.pkl.core.messaging.Message.OneWay;
import org.pkl.core.messaging.Message.Response;
import org.pkl.core.util.ErrorMessages;
import org.pkl.core.util.Pair;
/** Factory methods for creating [MessageTransport]s. */
public class MessageTransports {
public interface Logger {
void log(String msg);
}
/** Creates a message transport that reads from [inputStream] and writes to [outputStream]. */
public static MessageTransport stream(
MessageDecoder decoder, MessageEncoder encoder, Logger logger) {
return new EncodingMessageTransport(decoder, encoder, logger);
}
/** Creates "client" and "server" transports that are directly connected to each other. */
public static Pair<MessageTransport, MessageTransport> direct(Logger logger) {
var transport1 = new DirectMessageTransport(logger);
var transport2 = new DirectMessageTransport(logger);
transport1.setOther(transport2);
transport2.setOther(transport1);
return Pair.of(transport1, transport2);
}
public static <T> T resolveFuture(Future<T> future) throws IOException {
try {
return future.get();
} catch (ExecutionException | InterruptedException e) {
if (e.getCause() instanceof IOException ioExc) {
throw ioExc;
} else {
throw new IOException("external read failure: " + e.getMessage(), e.getCause());
}
}
}
protected static class EncodingMessageTransport extends AbstractMessageTransport {
private final MessageDecoder decoder;
private final MessageEncoder encoder;
private volatile boolean isClosed = false;
protected EncodingMessageTransport(
MessageDecoder decoder, MessageEncoder encoder, Logger logger) {
super(logger);
this.decoder = decoder;
this.encoder = encoder;
}
@Override
protected void doStart() throws ProtocolException, IOException {
while (!isClosed) {
var message = decoder.decode();
if (message == null) {
return;
}
accept(message);
}
}
@Override
protected void doClose() {
isClosed = true;
}
@Override
protected void doSend(Message message) throws ProtocolException, IOException {
encoder.encode(message);
}
}
protected static class DirectMessageTransport extends AbstractMessageTransport {
private DirectMessageTransport other;
protected DirectMessageTransport(Logger logger) {
super(logger);
}
@Override
protected void doStart() {}
@Override
protected void doClose() {}
@Override
protected void doSend(Message message) throws ProtocolException, IOException {
other.accept(message);
}
public void setOther(DirectMessageTransport other) {
this.other = other;
}
}
protected abstract static class AbstractMessageTransport implements MessageTransport {
private final Logger logger;
private MessageTransport.OneWayHandler oneWayHandler;
private MessageTransport.RequestHandler requestHandler;
private final Map<Long, ResponseHandler> responseHandlers = new ConcurrentHashMap<>();
protected AbstractMessageTransport(Logger logger) {
this.logger = logger;
}
protected void log(String message, Object... args) {
var formatter = new MessageFormat(message);
logger.log(formatter.format(args));
}
protected abstract void doStart() throws ProtocolException, IOException;
protected abstract void doClose();
protected abstract void doSend(Message message) throws ProtocolException, IOException;
protected void accept(Message message) throws ProtocolException, IOException {
log("Received message: {0}", message);
if (message instanceof Message.OneWay msg) {
oneWayHandler.handleOneWay(msg);
} else if (message instanceof Message.Request msg) {
requestHandler.handleRequest(msg);
} else if (message instanceof Message.Response msg) {
var handler = responseHandlers.remove(msg.requestId());
if (handler == null) {
throw new ProtocolException(
ErrorMessages.create(
"unknownRequestId", message.getClass().getSimpleName(), msg.requestId()));
}
handler.handleResponse(msg);
}
}
@Override
public final void start(OneWayHandler oneWayHandler, RequestHandler requestHandler)
throws ProtocolException, IOException {
log("Starting transport: {0}", this);
this.oneWayHandler = oneWayHandler;
this.requestHandler = requestHandler;
doStart();
}
@Override
public final void close() {
log("Closing transport: {0}", this);
doClose();
responseHandlers.clear();
}
@Override
public void send(OneWay message) throws ProtocolException, IOException {
log("Sending message: {0}", message);
doSend(message);
}
@Override
public void send(Message.Request message, ResponseHandler responseHandler)
throws ProtocolException, IOException {
log("Sending message: {0}", message);
responseHandlers.put(message.requestId(), responseHandler);
doSend(message);
}
@Override
public void send(Response message) throws ProtocolException, IOException {
log("Sending message: {0}", message);
doSend(message);
}
}
}

126
pkl-core/src/main/java/org/pkl/core/messaging/Messages.java Normal file
View File

@@ -0,0 +1,126 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.messaging;
import java.net.URI;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import org.pkl.core.messaging.Message.*;
import org.pkl.core.module.PathElement;
import org.pkl.core.util.Nullable;
public class Messages {
public record ModuleReaderSpec(
String scheme, boolean hasHierarchicalUris, boolean isLocal, boolean isGlobbable) {}
public record ResourceReaderSpec(
String scheme, boolean hasHierarchicalUris, boolean isGlobbable) {}
public record ListResourcesRequest(long requestId, long evaluatorId, URI uri)
implements Server.Request {
public Type type() {
return Type.LIST_RESOURCES_REQUEST;
}
}
public record ListResourcesResponse(
long requestId,
long evaluatorId,
@Nullable List<PathElement> pathElements,
@Nullable String error)
implements Client.Response {
public Type type() {
return Type.LIST_RESOURCES_RESPONSE;
}
}
public record ListModulesRequest(long requestId, long evaluatorId, URI uri)
implements Server.Request {
public Type type() {
return Type.LIST_MODULES_REQUEST;
}
}
public record ListModulesResponse(
long requestId,
long evaluatorId,
@Nullable List<PathElement> pathElements,
@Nullable String error)
implements Client.Response {
public Type type() {
return Type.LIST_MODULES_RESPONSE;
}
}
public record ReadResourceRequest(long requestId, long evaluatorId, URI uri)
implements Message.Request {
public Type type() {
return Type.READ_RESOURCE_REQUEST;
}
}
public record ReadResourceResponse(
long requestId, long evaluatorId, byte @Nullable [] contents, @Nullable String error)
implements Client.Response {
// workaround for kotlin bridging issue where `byte @Nullable [] contents` isn't detected as
// nullable
// public ReadResourceResponse(long requestId, long evaluatorId, @Nullable String error) {
// this(requestId, evaluatorId, null, error);
// }
public Type type() {
return Type.READ_RESOURCE_RESPONSE;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
ReadResourceResponse that = (ReadResourceResponse) o;
return requestId == that.requestId
&& evaluatorId == that.evaluatorId
&& Objects.equals(error, that.error)
&& Arrays.equals(contents, that.contents);
}
@Override
public int hashCode() {
return Objects.hash(requestId, evaluatorId, Arrays.hashCode(contents), error);
}
}
public record ReadModuleRequest(long requestId, long evaluatorId, URI uri)
implements Message.Request {
public Type type() {
return Type.READ_MODULE_REQUEST;
}
}
public record ReadModuleResponse(
long requestId, long evaluatorId, @Nullable String contents, @Nullable String error)
implements Client.Response {
public Type type() {
return Type.READ_MODULE_RESPONSE;
}
}
}

26
pkl-core/src/main/java/org/pkl/core/messaging/ProtocolException.java Normal file
View File

@@ -0,0 +1,26 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.messaging;
public class ProtocolException extends Exception {
public ProtocolException(String msg, Throwable cause) {
super(msg, cause);
}
public ProtocolException(String msg) {
super(msg);
}
}

4
pkl-core/src/main/java/org/pkl/core/messaging/package-info.java Normal file
View File

@@ -0,0 +1,4 @@
@NonnullByDefault
package org.pkl.core.messaging;
import org.pkl.core.util.NonnullByDefault;

129
pkl-core/src/main/java/org/pkl/core/module/ExternalModuleResolver.java Normal file
View File

@@ -0,0 +1,129 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.module;
import java.io.IOException;
import java.net.URI;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Random;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Future;
import org.pkl.core.SecurityManager;
import org.pkl.core.SecurityManagerException;
import org.pkl.core.messaging.MessageTransport;
import org.pkl.core.messaging.MessageTransports;
import org.pkl.core.messaging.Messages.ListModulesRequest;
import org.pkl.core.messaging.Messages.ListModulesResponse;
import org.pkl.core.messaging.Messages.ReadModuleRequest;
import org.pkl.core.messaging.Messages.ReadModuleResponse;
import org.pkl.core.messaging.ProtocolException;
public class ExternalModuleResolver {
private final MessageTransport transport;
private final long evaluatorId;
private final Map<URI, Future<String>> readResponses = new ConcurrentHashMap<>();
private final Map<URI, Future<List<PathElement>>> listResponses = new ConcurrentHashMap<>();
public ExternalModuleResolver(MessageTransport transport, long evaluatorId) {
this.transport = transport;
this.evaluatorId = evaluatorId;
}
public List<PathElement> listElements(SecurityManager securityManager, URI uri)
throws IOException, SecurityManagerException {
securityManager.checkResolveModule(uri);
return doListElements(uri);
}
public boolean hasElement(SecurityManager securityManager, URI uri)
throws SecurityManagerException {
securityManager.checkResolveModule(uri);
try {
doReadModule(uri);
return true;
} catch (IOException e) {
return false;
}
}
public String resolveModule(SecurityManager securityManager, URI uri)
throws IOException, SecurityManagerException {
securityManager.checkResolveModule(uri);
return doReadModule(uri);
}
private String doReadModule(URI moduleUri) throws IOException {
return MessageTransports.resolveFuture(
readResponses.computeIfAbsent(
moduleUri,
(uri) -> {
var future = new CompletableFuture<String>();
var request = new ReadModuleRequest(new Random().nextLong(), evaluatorId, uri);
try {
transport.send(
request,
(response) -> {
if (response instanceof ReadModuleResponse resp) {
if (resp.error() != null) {
future.completeExceptionally(new IOException(resp.error()));
} else if (resp.contents() != null) {
future.complete(resp.contents());
} else {
future.complete("");
}
} else {
future.completeExceptionally(new ProtocolException("unexpected response"));
}
});
} catch (ProtocolException | IOException e) {
future.completeExceptionally(e);
}
return future;
}));
}
private List<PathElement> doListElements(URI baseUri) throws IOException {
return MessageTransports.resolveFuture(
listResponses.computeIfAbsent(
baseUri,
(uri) -> {
var future = new CompletableFuture<List<PathElement>>();
var request = new ListModulesRequest(new Random().nextLong(), evaluatorId, uri);
try {
transport.send(
request,
(response) -> {
if (response instanceof ListModulesResponse resp) {
if (resp.error() != null) {
future.completeExceptionally(new IOException(resp.error()));
} else {
future.complete(
Objects.requireNonNullElseGet(resp.pathElements(), List::of));
}
} else {
future.completeExceptionally(new ProtocolException("unexpected response"));
}
});
} catch (ProtocolException | IOException e) {
future.completeExceptionally(e);
}
return future;
}));
}
}

71
pkl-core/src/main/java/org/pkl/core/module/ModuleKeyFactories.java
View File

@@ -15,6 +15,7 @@
*/
package org.pkl.core.module;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.FileSystemNotFoundException;
@@ -25,6 +26,10 @@ import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.ServiceLoader;
import javax.annotation.concurrent.GuardedBy;
import org.pkl.core.externalreader.ExternalReaderProcess;
import org.pkl.core.externalreader.ExternalReaderProcessException;
import org.pkl.core.util.ErrorMessages;
import org.pkl.core.util.IoUtils;
/** Utilities for obtaining and using module key factories. */
@@ -72,7 +77,27 @@ public final class ModuleKeyFactories {
return new ClassPath(classLoader);
}
/** Closes the given factories, ignoring any exceptions. */
/**
* Returns a factory for external reader module keys
*
* <p>NOTE: {@code process} needs to be {@link ExternalReaderProcess#close closed} to avoid
* resource leaks.
*/
public static ModuleKeyFactory externalProcess(String scheme, ExternalReaderProcess process) {
return new ExternalProcess(scheme, process, 0);
}
public static ModuleKeyFactory externalProcess(
String scheme, ExternalReaderProcess process, long evaluatorId) {
return new ExternalProcess(scheme, process, evaluatorId);
}
/**
* Closes the given factories, ignoring any exceptions.
*
* @deprecated Replaced by {@link org.pkl.core.util.Readers#closeQuietly}.
*/
@Deprecated(since = "0.27.0", forRemoval = true)
public static void closeQuietly(Iterable<ModuleKeyFactory> factories) {
for (ModuleKeyFactory factory : factories) {
try {
@@ -225,4 +250,48 @@ public final class ModuleKeyFactories {
INSTANCE = Collections.unmodifiableList(factories);
}
}
/** Represents a module from an external reader process. */
private static final class ExternalProcess implements ModuleKeyFactory {
private final String scheme;
private final ExternalReaderProcess process;
private final long evaluatorId;
@GuardedBy("this")
private ExternalModuleResolver resolver;
public ExternalProcess(String scheme, ExternalReaderProcess process, long evaluatorId) {
this.scheme = scheme;
this.process = process;
this.evaluatorId = evaluatorId;
}
private synchronized ExternalModuleResolver getResolver()
throws ExternalReaderProcessException {
if (resolver != null) {
return resolver;
}
resolver = new ExternalModuleResolver(process.getTransport(), evaluatorId);
return resolver;
}
public Optional<ModuleKey> create(URI uri)
throws URISyntaxException, ExternalReaderProcessException, IOException {
if (!scheme.equalsIgnoreCase(uri.getScheme())) return Optional.empty();
var spec = process.getModuleReaderSpec(scheme);
if (spec == null) {
throw new ExternalReaderProcessException(
ErrorMessages.create("externalReaderDoesNotSupportScheme", "module", scheme));
}
return Optional.of(ModuleKeys.externalResolver(uri, spec, getResolver()));
}
@Override
public void close() {
process.close();
}
}
}

5
pkl-core/src/main/java/org/pkl/core/module/ModuleKeyFactory.java
View File

@@ -15,9 +15,11 @@
*/
package org.pkl.core.module;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Optional;
import org.pkl.core.externalreader.ExternalReaderProcessException;
/** A factory for {@link ModuleKey}s. */
public interface ModuleKeyFactory extends AutoCloseable {
@@ -35,7 +37,8 @@ public interface ModuleKeyFactory extends AutoCloseable {
* @param uri an absolute normalized URI
* @return a module key for the given URI
*/
Optional<ModuleKey> create(URI uri) throws URISyntaxException;
Optional<ModuleKey> create(URI uri)
throws URISyntaxException, ExternalReaderProcessException, IOException;
/**
* Closes this factory, releasing any resources held. See the documentation of factory methods in

76
pkl-core/src/main/java/org/pkl/core/module/ModuleKeys.java
View File

@@ -29,6 +29,8 @@ import java.util.List;
import java.util.Map;
import org.pkl.core.SecurityManager;
import org.pkl.core.SecurityManagerException;
import org.pkl.core.externalreader.ExternalReaderProcessException;
import org.pkl.core.messaging.Messages.*;
import org.pkl.core.packages.Dependency;
import org.pkl.core.packages.Dependency.LocalDependency;
import org.pkl.core.packages.PackageAssetUri;
@@ -127,6 +129,12 @@ public final class ModuleKeys {
return new ProjectPackage(assetUri);
}
/** Creates a module key for an externally read module. */
public static ModuleKey externalResolver(
URI uri, ModuleReaderSpec spec, ExternalModuleResolver resolver) throws URISyntaxException {
return new ExternalResolver(uri, spec, resolver);
}
/**
* Creates a module key that behaves like {@code delegate}, except that it returns {@code text} as
* its loaded source.
@@ -165,7 +173,7 @@ public final class ModuleKeys {
}
@Override
public boolean hasHierarchicalUris() {
public boolean hasHierarchicalUris() throws IOException, ExternalReaderProcessException {
return delegate.hasHierarchicalUris();
}
@@ -175,19 +183,19 @@ public final class ModuleKeys {
}
@Override
public boolean isGlobbable() {
public boolean isGlobbable() throws IOException, ExternalReaderProcessException {
return delegate.isGlobbable();
}
@Override
public boolean hasElement(SecurityManager securityManager, URI uri)
throws IOException, SecurityManagerException {
throws IOException, SecurityManagerException, ExternalReaderProcessException {
return delegate.hasElement(securityManager, uri);
}
@Override
public List<PathElement> listElements(SecurityManager securityManager, URI baseUri)
throws IOException, SecurityManagerException {
throws IOException, SecurityManagerException, ExternalReaderProcessException {
return delegate.listElements(securityManager, baseUri);
}
}
@@ -397,7 +405,6 @@ public final class ModuleKeys {
}
private static final class ClassPath implements ModuleKey {
final URI uri;
final ClassLoader classLoader;
@@ -460,7 +467,6 @@ public final class ModuleKeys {
}
private static class Http implements ModuleKey {
private final URI uri;
Http(URI uri) {
@@ -550,7 +556,6 @@ public final class ModuleKeys {
}
private abstract static class AbstractPackage implements ModuleKey {
protected final PackageAssetUri packageAssetUri;
AbstractPackage(PackageAssetUri packageAssetUri) {
@@ -663,6 +668,7 @@ public final class ModuleKeys {
* an internal implementation detail, and we do not expect a module to declare this.
*/
public static class ProjectPackage extends AbstractPackage {
ProjectPackage(PackageAssetUri packageAssetUri) {
super(packageAssetUri);
}
@@ -712,7 +718,7 @@ public final class ModuleKeys {
@Override
public List<PathElement> listElements(SecurityManager securityManager, URI baseUri)
throws IOException, SecurityManagerException {
throws IOException, SecurityManagerException, ExternalReaderProcessException {
securityManager.checkResolveModule(baseUri);
var packageAssetUri = PackageAssetUri.create(baseUri);
var dependency =
@@ -733,7 +739,7 @@ public final class ModuleKeys {
@Override
public boolean hasElement(SecurityManager securityManager, URI elementUri)
throws IOException, SecurityManagerException {
throws IOException, SecurityManagerException, ExternalReaderProcessException {
securityManager.checkResolveModule(elementUri);
var packageAssetUri = PackageAssetUri.create(elementUri);
var dependency =
@@ -769,4 +775,56 @@ public final class ModuleKeys {
return projectResolver.getResolvedDependenciesForPackage(packageUri, dependencyMetadata);
}
}
public static class ExternalResolver implements ModuleKey {
private final URI uri;
private final ModuleReaderSpec spec;
private final ExternalModuleResolver resolver;
public ExternalResolver(URI uri, ModuleReaderSpec spec, ExternalModuleResolver resolver) {
this.uri = uri;
this.spec = spec;
this.resolver = resolver;
}
@Override
public boolean isLocal() {
return spec.isLocal();
}
@Override
public boolean hasHierarchicalUris() {
return spec.hasHierarchicalUris();
}
@Override
public boolean isGlobbable() {
return spec.isGlobbable();
}
@Override
public URI getUri() {
return uri;
}
@Override
public List<PathElement> listElements(SecurityManager securityManager, URI baseUri)
throws IOException, SecurityManagerException {
return resolver.listElements(securityManager, baseUri);
}
@Override
public ResolvedModuleKey resolve(SecurityManager securityManager)
throws IOException, SecurityManagerException {
var contents = resolver.resolveModule(securityManager, uri);
return ResolvedModuleKeys.virtual(this, uri, contents, true);
}
@Override
public boolean hasElement(SecurityManager securityManager, URI elementUri)
throws IOException, SecurityManagerException {
return resolver.hasElement(securityManager, elementUri);
}
}
}

3
pkl-core/src/main/java/org/pkl/core/packages/Checksums.java
View File

@@ -16,6 +16,7 @@
package org.pkl.core.packages;
import java.util.Objects;
import org.pkl.core.util.Nullable;
public final class Checksums {
private final String sha256;
@@ -34,7 +35,7 @@ public final class Checksums {
}
@Override
public boolean equals(Object o) {
public boolean equals(@Nullable Object o) {
if (this == o) {
return true;
}

2
pkl-core/src/main/java/org/pkl/core/project/Project.java
View File

@@ -522,6 +522,8 @@ public final class Project {
modulePath,
timeout,
rootDir,
null,
null,
null);
}

127
pkl-core/src/main/java/org/pkl/core/resource/ExternalResourceResolver.java Normal file
View File

@@ -0,0 +1,127 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.resource;
import java.io.IOException;
import java.net.URI;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Random;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Future;
import org.pkl.core.SecurityManager;
import org.pkl.core.SecurityManagerException;
import org.pkl.core.messaging.MessageTransport;
import org.pkl.core.messaging.MessageTransports;
import org.pkl.core.messaging.Messages.*;
import org.pkl.core.messaging.ProtocolException;
import org.pkl.core.module.PathElement;
public class ExternalResourceResolver {
private final MessageTransport transport;
private final long evaluatorId;
private final Map<URI, Future<byte[]>> readResponses = new ConcurrentHashMap<>();
private final Map<URI, Future<List<PathElement>>> listResponses = new ConcurrentHashMap<>();
public ExternalResourceResolver(MessageTransport transport, long evaluatorId) {
this.transport = transport;
this.evaluatorId = evaluatorId;
}
public Optional<Object> read(URI uri) throws IOException {
var result = doRead(uri);
return Optional.of(new Resource(uri, result));
}
public boolean hasElement(org.pkl.core.SecurityManager securityManager, URI elementUri)
throws SecurityManagerException {
securityManager.checkResolveResource(elementUri);
try {
doRead(elementUri);
return true;
} catch (IOException e) {
return false;
}
}
public List<PathElement> listElements(SecurityManager securityManager, URI baseUri)
throws IOException, SecurityManagerException {
securityManager.checkResolveResource(baseUri);
return doListElements(baseUri);
}
public List<PathElement> doListElements(URI baseUri) throws IOException {
return MessageTransports.resolveFuture(
listResponses.computeIfAbsent(
baseUri,
(uri) -> {
var future = new CompletableFuture<List<PathElement>>();
var request = new ListResourcesRequest(new Random().nextLong(), evaluatorId, uri);
try {
transport.send(
request,
(response) -> {
if (response instanceof ListResourcesResponse resp) {
if (resp.error() != null) {
future.completeExceptionally(new IOException(resp.error()));
} else {
future.complete(
Objects.requireNonNullElseGet(resp.pathElements(), List::of));
}
} else {
future.completeExceptionally(new ProtocolException("unexpected response"));
}
});
} catch (ProtocolException | IOException e) {
future.completeExceptionally(e);
}
return future;
}));
}
public byte[] doRead(URI baseUri) throws IOException {
return MessageTransports.resolveFuture(
readResponses.computeIfAbsent(
baseUri,
(uri) -> {
var future = new CompletableFuture<byte[]>();
var request = new ReadResourceRequest(new Random().nextLong(), evaluatorId, uri);
try {
transport.send(
request,
(response) -> {
if (response instanceof ReadResourceResponse resp) {
if (resp.error() != null) {
future.completeExceptionally(new IOException(resp.error()));
} else if (resp.contents() != null) {
future.complete(resp.contents());
} else {
future.complete(new byte[0]);
}
} else {
future.completeExceptionally(new ProtocolException("unexpected response"));
}
});
} catch (ProtocolException | IOException e) {
future.completeExceptionally(e);
}
return future;
}));
}
}

16
pkl-core/src/main/java/org/pkl/core/resource/ResourceReader.java
View File

@@ -20,6 +20,7 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.util.Optional;
import org.pkl.core.SecurityManagerException;
import org.pkl.core.externalreader.ExternalReaderProcessException;
import org.pkl.core.runtime.ReaderBase;
/**
@@ -29,7 +30,7 @@ import org.pkl.core.runtime.ReaderBase;
*
* <p>See {@link ResourceReaders} for predefined resource readers.
*/
public interface ResourceReader extends ReaderBase {
public interface ResourceReader extends ReaderBase, AutoCloseable {
/** The URI scheme associated with resources read by this resource reader. */
String getUriScheme();
@@ -54,5 +55,16 @@ public interface ResourceReader extends ReaderBase {
* manager.
* </ul>
*/
Optional<Object> read(URI uri) throws IOException, URISyntaxException, SecurityManagerException;
Optional<Object> read(URI uri)
throws IOException,
URISyntaxException,
SecurityManagerException,
ExternalReaderProcessException;
/**
* Closes this reader, releasing any resources held. See the documentation of factory methods in
* {@link ResourceReaders} for which factories need to be closed.
*/
@Override
default void close() {}
}

132
pkl-core/src/main/java/org/pkl/core/resource/ResourceReaders.java
View File

@@ -29,6 +29,9 @@ import java.util.Optional;
import java.util.ServiceLoader;
import org.pkl.core.SecurityManager;
import org.pkl.core.SecurityManagerException;
import org.pkl.core.externalreader.ExternalReaderProcess;
import org.pkl.core.externalreader.ExternalReaderProcessException;
import org.pkl.core.messaging.Messages.*;
import org.pkl.core.module.FileResolver;
import org.pkl.core.module.ModulePathResolver;
import org.pkl.core.module.PathElement;
@@ -137,6 +140,21 @@ public final class ResourceReaders {
return FromServiceProviders.INSTANCE;
}
public static ResourceReader externalProcess(
String scheme, ExternalReaderProcess externalReaderProcess) {
return new ExternalProcess(scheme, externalReaderProcess, 0);
}
public static ResourceReader externalProcess(
String scheme, ExternalReaderProcess externalReaderProcess, long evaluatorId) {
return new ExternalProcess(scheme, externalReaderProcess, evaluatorId);
}
public static ResourceReader externalResolver(
ResourceReaderSpec spec, ExternalResourceResolver resolver) {
return new ExternalResolver(spec, resolver);
}
private static final class EnvironmentVariable implements ResourceReader {
static final ResourceReader INSTANCE = new EnvironmentVariable();
@@ -521,7 +539,7 @@ public final class ResourceReaders {
@Override
public List<PathElement> listElements(SecurityManager securityManager, URI baseUri)
throws IOException, SecurityManagerException {
throws IOException, SecurityManagerException, ExternalReaderProcessException {
securityManager.checkResolveResource(baseUri);
var packageAssetUri = PackageAssetUri.create(baseUri);
var dependency =
@@ -543,7 +561,7 @@ public final class ResourceReaders {
@Override
public boolean hasElement(SecurityManager securityManager, URI elementUri)
throws IOException, SecurityManagerException {
throws IOException, SecurityManagerException, ExternalReaderProcessException {
securityManager.checkResolveResource(elementUri);
var packageAssetUri = PackageAssetUri.create(elementUri);
var dependency =
@@ -585,6 +603,7 @@ public final class ResourceReaders {
}
private static class FromServiceProviders {
private static final List<ResourceReader> INSTANCE;
static {
@@ -594,4 +613,113 @@ public final class ResourceReaders {
INSTANCE = Collections.unmodifiableList(readers);
}
}
private static final class ExternalProcess implements ResourceReader {
private final String scheme;
private final ExternalReaderProcess process;
private final long evaluatorId;
private ExternalResolver underlying;
public ExternalProcess(String scheme, ExternalReaderProcess process, long evaluatorId) {
this.scheme = scheme;
this.process = process;
this.evaluatorId = evaluatorId;
}
private ExternalResolver getUnderlyingReader()
throws ExternalReaderProcessException, IOException {
if (underlying != null) {
return underlying;
}
var spec = process.getResourceReaderSpec(scheme);
if (spec == null) {
throw new ExternalReaderProcessException(
ErrorMessages.create("externalReaderDoesNotSupportScheme", "resource", scheme));
}
underlying =
new ExternalResolver(
spec, new ExternalResourceResolver(process.getTransport(), evaluatorId));
return underlying;
}
@Override
public String getUriScheme() {
return scheme;
}
@Override
public boolean hasHierarchicalUris() throws ExternalReaderProcessException, IOException {
return getUnderlyingReader().hasHierarchicalUris();
}
@Override
public boolean isGlobbable() throws ExternalReaderProcessException, IOException {
return getUnderlyingReader().isGlobbable();
}
@Override
public Optional<Object> read(URI uri) throws IOException, ExternalReaderProcessException {
return getUnderlyingReader().read(uri);
}
@Override
public boolean hasElement(SecurityManager securityManager, URI elementUri)
throws IOException, SecurityManagerException, ExternalReaderProcessException {
return getUnderlyingReader().hasElement(securityManager, elementUri);
}
@Override
public List<PathElement> listElements(SecurityManager securityManager, URI baseUri)
throws IOException, SecurityManagerException, ExternalReaderProcessException {
return getUnderlyingReader().listElements(securityManager, baseUri);
}
@Override
public void close() {
process.close();
}
}
private static final class ExternalResolver implements ResourceReader {
private final ResourceReaderSpec readerSpec;
private final ExternalResourceResolver resolver;
public ExternalResolver(ResourceReaderSpec readerSpec, ExternalResourceResolver resolver) {
this.readerSpec = readerSpec;
this.resolver = resolver;
}
@Override
public boolean hasHierarchicalUris() {
return readerSpec.hasHierarchicalUris();
}
@Override
public boolean isGlobbable() {
return readerSpec.isGlobbable();
}
@Override
public String getUriScheme() {
return readerSpec.scheme();
}
@Override
public Optional<Object> read(URI uri) throws IOException {
return resolver.read(uri);
}
@Override
public boolean hasElement(org.pkl.core.SecurityManager securityManager, URI elementUri)
throws SecurityManagerException {
return resolver.hasElement(securityManager, elementUri);
}
@Override
public List<PathElement> listElements(SecurityManager securityManager, URI baseUri)
throws IOException, SecurityManagerException {
return resolver.listElements(securityManager, baseUri);
}
}
}

14
pkl-core/src/main/java/org/pkl/core/runtime/ModuleResolver.java
View File

@@ -16,11 +16,13 @@
package org.pkl.core.runtime;
import com.oracle.truffle.api.nodes.Node;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Collection;
import java.util.Optional;
import org.pkl.core.ModuleSource;
import org.pkl.core.externalreader.ExternalReaderProcessException;
import org.pkl.core.module.ModuleKey;
import org.pkl.core.module.ModuleKeyFactory;
import org.pkl.core.module.ModuleKeys;
@@ -82,6 +84,18 @@ public final class ModuleResolver {
.evalError("invalidModuleUri", moduleUri)
.withHint(e.getReason())
.build();
} catch (ExternalReaderProcessException e) {
throw new VmExceptionBuilder()
.withOptionalLocation(importNode)
.evalError("externalReaderFailure")
.withCause(e)
.build();
} catch (IOException e) {
throw new VmExceptionBuilder()
.withOptionalLocation(importNode)
.evalError("ioErrorLoadingModule")
.withCause(e)
.build();
}
if (key.isPresent()) return key.get();
}

9
pkl-core/src/main/java/org/pkl/core/runtime/ReaderBase.java
View File

@@ -20,6 +20,7 @@ import java.net.URI;
import java.util.List;
import org.pkl.core.SecurityManager;
import org.pkl.core.SecurityManagerException;
import org.pkl.core.externalreader.ExternalReaderProcessException;
import org.pkl.core.module.ModuleKey;
import org.pkl.core.module.PathElement;
import org.pkl.core.util.IoUtils;
@@ -29,10 +30,10 @@ public interface ReaderBase {
* Tells if the URIs represented by this module key or resource reader should be interpreted as <a
* href="https://www.rfc-editor.org/rfc/rfc3986#section-1.2.3">hierarchical</a>.
*/
boolean hasHierarchicalUris();
boolean hasHierarchicalUris() throws ExternalReaderProcessException, IOException;
/** Tells if this module key or resource reader supports globbing. */
boolean isGlobbable();
boolean isGlobbable() throws ExternalReaderProcessException, IOException;
/**
* Tells if relative paths of this URI should be resolved from {@link URI#getFragment()}, rather
@@ -49,7 +50,7 @@ public interface ReaderBase {
* if either {@link #isGlobbable()} or {@link ModuleKey#isLocal()} returns true.
*/
default boolean hasElement(SecurityManager securityManager, URI elementUri)
throws IOException, SecurityManagerException {
throws IOException, SecurityManagerException, ExternalReaderProcessException {
throw new UnsupportedOperationException();
}
@@ -66,7 +67,7 @@ public interface ReaderBase {
* this reader.
*/
default List<PathElement> listElements(SecurityManager securityManager, URI baseUri)
throws IOException, SecurityManagerException {
throws IOException, SecurityManagerException, ExternalReaderProcessException {
throw new UnsupportedOperationException();
}

6
pkl-core/src/main/java/org/pkl/core/runtime/ResourceManager.java
View File

@@ -26,6 +26,7 @@ import java.util.Map;
import java.util.Optional;
import org.pkl.core.SecurityManager;
import org.pkl.core.SecurityManagerException;
import org.pkl.core.externalreader.ExternalReaderProcessException;
import org.pkl.core.http.HttpClientInitException;
import org.pkl.core.packages.PackageLoadError;
import org.pkl.core.resource.Resource;
@@ -83,7 +84,10 @@ public final class ResourceManager {
.withHint(e.getReason())
.withOptionalLocation(readNode)
.build();
} catch (SecurityManagerException | PackageLoadError | HttpClientInitException e) {
} catch (SecurityManagerException
| PackageLoadError
| HttpClientInitException
| ExternalReaderProcessException e) {
throw new VmExceptionBuilder().withCause(e).withOptionalLocation(readNode).build();
}
return resource;

16
pkl-core/src/main/java/org/pkl/core/runtime/VmImportAnalyzer.java
View File

@@ -30,6 +30,7 @@ import org.pkl.core.SecurityManager;
import org.pkl.core.SecurityManagerException;
import org.pkl.core.ast.builder.ImportsAndReadsParser;
import org.pkl.core.ast.builder.ImportsAndReadsParser.Entry;
import org.pkl.core.externalreader.ExternalReaderProcessException;
import org.pkl.core.util.GlobResolver;
import org.pkl.core.util.GlobResolver.InvalidGlobPatternException;
import org.pkl.core.util.GlobResolver.ResolvedGlobElement;
@@ -38,7 +39,10 @@ import org.pkl.core.util.IoUtils;
public class VmImportAnalyzer {
@TruffleBoundary
public static ImportGraph analyze(URI[] moduleUris, VmContext context)
throws IOException, URISyntaxException, SecurityManagerException {
throws IOException,
URISyntaxException,
SecurityManagerException,
ExternalReaderProcessException {
var imports = new TreeMap<URI, Set<ImportGraph.Import>>();
var resolvedImports = new TreeMap<URI, URI>();
for (var moduleUri : moduleUris) {
@@ -53,7 +57,10 @@ public class VmImportAnalyzer {
VmContext context,
Map<URI, Set<ImportGraph.Import>> imports,
Map<URI, URI> resolvedImports)
throws IOException, URISyntaxException, SecurityManagerException {
throws IOException,
URISyntaxException,
SecurityManagerException,
ExternalReaderProcessException {
var moduleResolver = context.getModuleResolver();
var securityManager = context.getSecurityManager();
var importsInModule = collectImports(moduleUri, moduleResolver, securityManager);
@@ -71,7 +78,10 @@ public class VmImportAnalyzer {
private static Set<ImportGraph.Import> collectImports(
URI moduleUri, ModuleResolver moduleResolver, SecurityManager securityManager)
throws IOException, URISyntaxException, SecurityManagerException {
throws IOException,
URISyntaxException,
SecurityManagerException,
ExternalReaderProcessException {
var moduleKey = moduleResolver.resolve(moduleUri);
var resolvedModuleKey = moduleKey.resolve(securityManager);
List<Entry> importsAndReads;

4
pkl-core/src/main/java/org/pkl/core/service/ExecutorSpiImpl.java
View File

@@ -33,6 +33,7 @@ import org.pkl.core.module.ModuleKeyFactories;
import org.pkl.core.module.ModulePathResolver;
import org.pkl.core.project.Project;
import org.pkl.core.resource.ResourceReaders;
import org.pkl.core.util.Readers;
import org.pkl.executor.spi.v1.ExecutorSpi;
import org.pkl.executor.spi.v1.ExecutorSpiException;
import org.pkl.executor.spi.v1.ExecutorSpiOptions;
@@ -125,7 +126,8 @@ public final class ExecutorSpiImpl implements ExecutorSpi {
} catch (PklException e) {
throw new ExecutorSpiException(e.getMessage(), e.getCause());
} finally {
ModuleKeyFactories.closeQuietly(builder.getModuleKeyFactories());
Readers.closeQuietly(builder.getModuleKeyFactories());
Readers.closeQuietly(builder.getResourceReaders());
}
}

7
pkl-core/src/main/java/org/pkl/core/stdlib/analyze/AnalyzeNodes.java
View File

@@ -23,6 +23,7 @@ import java.net.URISyntaxException;
import org.pkl.core.ImportGraph;
import org.pkl.core.ImportGraph.Import;
import org.pkl.core.SecurityManagerException;
import org.pkl.core.externalreader.ExternalReaderProcessException;
import org.pkl.core.packages.PackageLoadError;
import org.pkl.core.runtime.AnalyzeModule;
import org.pkl.core.runtime.VmContext;
@@ -91,7 +92,11 @@ public final class AnalyzeNodes {
try {
var results = VmImportAnalyzer.analyze(uris, context);
return importGraphFactory.create(results);
} catch (IOException | URISyntaxException | SecurityManagerException | PackageLoadError e) {
} catch (IOException
| URISyntaxException
| SecurityManagerException
| PackageLoadError
| ExternalReaderProcessException e) {
throw exceptionBuilder().withCause(e).build();
}
}

9
pkl-core/src/main/java/org/pkl/core/stdlib/benchmark/OutputBenchmarkNodes.java
View File

@@ -25,6 +25,7 @@ import java.nio.file.Path;
import java.util.List;
import org.pkl.core.SecurityManager;
import org.pkl.core.SecurityManagerException;
import org.pkl.core.externalreader.ExternalReaderProcessException;
import org.pkl.core.module.ModuleKey;
import org.pkl.core.module.PathElement;
import org.pkl.core.module.ResolvedModuleKey;
@@ -108,7 +109,7 @@ public final class OutputBenchmarkNodes {
}
@Override
public boolean hasHierarchicalUris() {
public boolean hasHierarchicalUris() throws IOException, ExternalReaderProcessException {
return delegate.hasHierarchicalUris();
}
@@ -118,19 +119,19 @@ public final class OutputBenchmarkNodes {
}
@Override
public boolean isGlobbable() {
public boolean isGlobbable() throws IOException, ExternalReaderProcessException {
return delegate.isGlobbable();
}
@Override
public boolean hasElement(SecurityManager securityManager, URI uri)
throws IOException, SecurityManagerException {
throws IOException, SecurityManagerException, ExternalReaderProcessException {
return delegate.hasElement(securityManager, uri);
}
@Override
public List<PathElement> listElements(SecurityManager securityManager, URI baseUri)
throws IOException, SecurityManagerException {
throws IOException, SecurityManagerException, ExternalReaderProcessException {
return delegate.listElements(securityManager, baseUri);
}
}

23
pkl-core/src/main/java/org/pkl/core/util/GlobResolver.java
View File

@@ -31,6 +31,7 @@ import java.util.stream.Collectors;
import org.pkl.core.PklBugException;
import org.pkl.core.SecurityManager;
import org.pkl.core.SecurityManagerException;
import org.pkl.core.externalreader.ExternalReaderProcessException;
import org.pkl.core.module.ModuleKey;
import org.pkl.core.module.PathElement;
import org.pkl.core.runtime.ReaderBase;
@@ -260,7 +261,7 @@ public final class GlobResolver {
URI globUri,
Pattern pattern,
Map<String, ResolvedGlobElement> result)
throws IOException, SecurityManagerException {
throws IOException, SecurityManagerException, ExternalReaderProcessException {
var elements = reader.listElements(securityManager, globUri);
for (var elem : sorted(elements)) {
URI resolvedUri;
@@ -318,7 +319,10 @@ public final class GlobResolver {
boolean isGlobStar,
boolean hasAbsoluteGlob,
MutableLong listElementCallCount)
throws IOException, SecurityManagerException, InvalidGlobPatternException {
throws IOException,
SecurityManagerException,
InvalidGlobPatternException,
ExternalReaderProcessException {
var result = new ArrayList<ResolvedGlobElement>();
doExpandHierarchicalGlobPart(
securityManager,
@@ -343,7 +347,10 @@ public final class GlobResolver {
boolean hasAbsoluteGlob,
MutableLong listElementCallCount,
List<ResolvedGlobElement> result)
throws IOException, SecurityManagerException, InvalidGlobPatternException {
throws IOException,
SecurityManagerException,
InvalidGlobPatternException,
ExternalReaderProcessException {
if (listElementCallCount.getAndIncrement() > maxListElements()) {
throw new InvalidGlobPatternException(ErrorMessages.create("invalidGlobTooComplex"));
@@ -384,7 +391,10 @@ public final class GlobResolver {
boolean hasAbsoluteGlob,
Map<String, ResolvedGlobElement> result,
MutableLong listElementCallCount)
throws IOException, SecurityManagerException, InvalidGlobPatternException {
throws IOException,
SecurityManagerException,
InvalidGlobPatternException,
ExternalReaderProcessException {
var isLeaf = idx == globPatternParts.length - 1;
var patternPart = globPatternParts[idx];
if (isRegularPathPart(patternPart)) {
@@ -481,7 +491,10 @@ public final class GlobResolver {
ModuleKey enclosingModuleKey,
URI enclosingUri,
String globPattern)
throws IOException, SecurityManagerException, InvalidGlobPatternException {
throws IOException,
SecurityManagerException,
InvalidGlobPatternException,
ExternalReaderProcessException {
var result = new LinkedHashMap<String, ResolvedGlobElement>();
var hasAbsoluteGlob = globPattern.matches("\\w+:.*");

11
pkl-core/src/main/java/org/pkl/core/util/IoUtils.java
View File

@@ -38,6 +38,7 @@ import org.pkl.core.PklBugException;
import org.pkl.core.Platform;
import org.pkl.core.SecurityManager;
import org.pkl.core.SecurityManagerException;
import org.pkl.core.externalreader.ExternalReaderProcessException;
import org.pkl.core.module.ModuleKey;
import org.pkl.core.packages.PackageLoadError;
import org.pkl.core.runtime.ReaderBase;
@@ -317,7 +318,7 @@ public final class IoUtils {
private static URI resolveTripleDotImport(
SecurityManager securityManager, ModuleKey moduleKey, String tripleDotPath)
throws IOException, SecurityManagerException {
throws IOException, SecurityManagerException, ExternalReaderProcessException {
var moduleKeyUri = moduleKey.getUri();
if (!moduleKey.isLocal() || !moduleKey.hasHierarchicalUris()) {
throw new VmExceptionBuilder()
@@ -363,7 +364,8 @@ public final class IoUtils {
return Pair.of(importPath.substring(1, idx), importPath.substring(idx));
}
private static URI resolveProjectDependency(ModuleKey moduleKey, String notation) {
private static URI resolveProjectDependency(ModuleKey moduleKey, String notation)
throws IOException, ExternalReaderProcessException {
var parsed = parseDependencyNotation(notation);
var name = parsed.getFirst();
var path = parsed.getSecond();
@@ -395,7 +397,10 @@ public final class IoUtils {
* dependency notation ()
*/
public static URI resolve(SecurityManager securityManager, ModuleKey moduleKey, URI importUri)
throws URISyntaxException, IOException, SecurityManagerException {
throws URISyntaxException,
IOException,
SecurityManagerException,
ExternalReaderProcessException {
if (importUri.isAbsolute()) {
return moduleKey.resolveUri(importUri);
}

28
pkl-core/src/main/java/org/pkl/core/util/Readers.java Normal file
View File

@@ -0,0 +1,28 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.util;
public class Readers {
/** Closes the given readers, ignoring any exceptions. */
public static void closeQuietly(Iterable<? extends AutoCloseable> readers) {
for (var reader : readers) {
try {
reader.close();
} catch (Exception ignored) {
}
}
}
}

30
pkl-core/src/main/resources/org/pkl/core/errorMessages.properties
View File

@@ -1081,3 +1081,33 @@ Malformed proxy URI (expecting `http://<host>[:<port>]`): `{0}`.
cannotAnalyzeBecauseSyntaxError=\
Found a syntax error when parsing module `{0}`.
malformedMessageHeaderLength=\
Malformed message header (expected size 2, but got {0}).
malformedMessageHeaderException=\
Malformed message header.
malformedMessageHeaderUnrecognizedCode=\
Malformed message header (unrecognized code `{0}`).
unhandledMessageCode=\
Unhandled decoding message code `{0}`.
unhandledMessageType=\
Unhandled encoding message type `{0}`.
malformedMessageBody=\
Malformed message body for message with code `{0}`.
missingMessageParameter=\
Missing message parameter `{0}`
unknownRequestId=\
Received response {0} for unknown request ID `{1}`.
externalReaderFailure=\
Failed to communicate with external reader process.
externalReaderDoesNotSupportScheme=\
External {0} reader does not support scheme `{1}`.

10
pkl-core/src/test/kotlin/org/pkl/core/EvaluatorBuilderTest.kt
View File

@@ -71,8 +71,10 @@ class EvaluatorBuilderTest {
fun `sets evaluator settings from project`() {
val projectPath = Path.of(javaClass.getResource("project/project1/PklProject")!!.toURI())
val project = Project.loadFromPath(projectPath, SecurityManagers.defaultManager, null)
val projectDir = Path.of(javaClass.getResource("project/project1/PklProject")!!.toURI()).parent
val builder = EvaluatorBuilder.unconfigured().applyFromProject(project)
val projectDir = projectPath.parent
val builder = EvaluatorBuilder.unconfigured()
val moduleKeyFactoryCount = builder.moduleKeyFactories.size
builder.applyFromProject(project)
assertThat(builder.allowedResources.map { it.pattern() }).isEqualTo(listOf("foo:", "bar:"))
assertThat(builder.allowedModules.map { it.pattern() }).isEqualTo(listOf("baz:", "biz:"))
assertThat(builder.externalProperties).isEqualTo(mapOf("one" to "1"))
@@ -80,5 +82,9 @@ class EvaluatorBuilderTest {
assertThat(builder.moduleCacheDir).isEqualTo(projectDir.resolve("my-cache-dir/"))
assertThat(builder.rootDir).isEqualTo(projectDir.resolve("my-root-dir/"))
assertThat(builder.timeout).isEqualTo(Duration.ofMinutes(5L))
assertThat(builder.moduleKeyFactories.size - moduleKeyFactoryCount)
.isEqualTo(3) // two external readers, one module path
assertThat(builder.resourceReaders.find { it.uriScheme == "scheme3" }).isNotNull
assertThat(builder.resourceReaders.find { it.uriScheme == "scheme4" }).isNotNull
}
}

29
pkl-core/src/test/kotlin/org/pkl/core/externalreader/ExternalModuleReader.kt Normal file
View File

@@ -0,0 +1,29 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.externalreader
import java.net.URI
import org.pkl.core.messaging.Messages.ModuleReaderSpec
/** An external module reader, to be used with [ExternalReaderRuntime]. */
interface ExternalModuleReader : ExternalReaderBase {
val isLocal: Boolean
fun read(uri: URI): String
val spec: ModuleReaderSpec
get() = ModuleReaderSpec(scheme, hasHierarchicalUris, isLocal, isGlobbable)
}

73
pkl-core/src/test/kotlin/org/pkl/core/externalreader/ExternalProcessProcessReaderMessagePackCodecTest.kt Normal file
View File

@@ -0,0 +1,73 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.externalreader
import java.io.PipedInputStream
import java.io.PipedOutputStream
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test
import org.msgpack.core.MessagePack
import org.pkl.core.externalreader.ExternalReaderMessages.*
import org.pkl.core.messaging.*
class ExternalProcessProcessReaderMessagePackCodecTest {
private val encoder: MessageEncoder
private val decoder: MessageDecoder
init {
val inputStream = PipedInputStream()
val outputStream = PipedOutputStream(inputStream)
encoder = ExternalReaderMessagePackEncoder(MessagePack.newDefaultPacker(outputStream))
decoder = ExternalReaderMessagePackDecoder(MessagePack.newDefaultUnpacker(inputStream))
}
private fun roundtrip(message: Message) {
encoder.encode(message)
val decoded = decoder.decode()
assertThat(decoded).isEqualTo(message)
}
@Test
fun `round-trip InitializeModuleReaderRequest`() {
roundtrip(InitializeModuleReaderRequest(123, "my-scheme"))
}
@Test
fun `round-trip InitializeResourceReaderRequest`() {
roundtrip(InitializeResourceReaderRequest(123, "my-scheme"))
}
@Test
fun `round-trip InitializeModuleReaderResponse`() {
roundtrip(InitializeModuleReaderResponse(123, null))
roundtrip(
InitializeModuleReaderResponse(123, Messages.ModuleReaderSpec("my-scheme", true, true, true))
)
}
@Test
fun `round-trip InitializeResourceReaderResponse`() {
roundtrip(InitializeResourceReaderResponse(123, null))
roundtrip(
InitializeResourceReaderResponse(123, Messages.ResourceReaderSpec("my-scheme", true, true))
)
}
@Test
fun `round-trip CloseExternalProcess`() {
roundtrip(CloseExternalProcess())
}
}

30
pkl-core/src/test/kotlin/org/pkl/core/externalreader/ExternalReaderBase.kt Normal file
View File

@@ -0,0 +1,30 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.externalreader
import java.net.URI
import org.pkl.core.module.PathElement
/** Base interface for external module and resource readers. */
interface ExternalReaderBase {
val scheme: String
val hasHierarchicalUris: Boolean
val isGlobbable: Boolean
fun listElements(uri: URI): List<PathElement>
}

199
pkl-core/src/test/kotlin/org/pkl/core/externalreader/ExternalReaderRuntime.kt Normal file
View File

@@ -0,0 +1,199 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.externalreader
import java.io.IOException
import org.pkl.core.externalreader.ExternalReaderMessages.*
import org.pkl.core.messaging.Message
import org.pkl.core.messaging.MessageTransport
import org.pkl.core.messaging.Messages.*
import org.pkl.core.messaging.ProtocolException
import org.pkl.core.util.Nullable
/** An implementation of the client side of the external reader flow */
class ExternalReaderRuntime(
private val moduleReaders: List<ExternalModuleReader>,
private val resourceReaders: List<ExternalResourceReader>,
private val transport: MessageTransport
) {
/** Close the runtime and its transport. */
fun close() {
transport.close()
}
private fun findModuleReader(scheme: String): @Nullable ExternalModuleReader? {
for (moduleReader in moduleReaders) {
if (moduleReader.scheme.equals(scheme, ignoreCase = true)) {
return moduleReader
}
}
return null
}
private fun findResourceReader(scheme: String): @Nullable ExternalResourceReader? {
for (resourceReader in resourceReaders) {
if (resourceReader.scheme.equals(scheme, ignoreCase = true)) {
return resourceReader
}
}
return null
}
/**
* Start the runtime so it can respond to incoming messages on its transport.
*
* Blocks until the underlying transport is closed.
*/
@Throws(ProtocolException::class, IOException::class)
fun run() {
transport.start(
{ msg: Message.OneWay ->
if (msg.type() == Message.Type.CLOSE_EXTERNAL_PROCESS) {
close()
} else {
throw ProtocolException("Unexpected incoming one-way message: $msg")
}
},
{ msg: Message.Request ->
when (msg.type()) {
Message.Type.INITIALIZE_MODULE_READER_REQUEST -> {
val req = msg as InitializeModuleReaderRequest
val reader = findModuleReader(req.scheme)
var spec: @Nullable ModuleReaderSpec? = null
if (reader != null) {
spec = reader.spec
}
transport.send(InitializeModuleReaderResponse(req.requestId, spec))
}
Message.Type.INITIALIZE_RESOURCE_READER_REQUEST -> {
val req = msg as InitializeResourceReaderRequest
val reader = findResourceReader(req.scheme)
var spec: @Nullable ResourceReaderSpec? = null
if (reader != null) {
spec = reader.spec
}
transport.send(InitializeResourceReaderResponse(req.requestId, spec))
}
Message.Type.LIST_MODULES_REQUEST -> {
val req = msg as ListModulesRequest
val reader = findModuleReader(req.uri.scheme)
if (reader == null) {
transport.send(
ListModulesResponse(
req.requestId,
req.evaluatorId,
null,
"No module reader found for scheme " + req.uri.scheme
)
)
return@start
}
try {
transport.send(
ListModulesResponse(
req.requestId,
req.evaluatorId,
reader.listElements(req.uri),
null
)
)
} catch (e: Exception) {
transport.send(
ListModulesResponse(req.requestId, req.evaluatorId, null, e.toString())
)
}
}
Message.Type.LIST_RESOURCES_REQUEST -> {
val req = msg as ListResourcesRequest
val reader = findModuleReader(req.uri.scheme)
if (reader == null) {
transport.send(
ListResourcesResponse(
req.requestId,
req.evaluatorId,
null,
"No resource reader found for scheme " + req.uri.scheme
)
)
return@start
}
try {
transport.send(
ListResourcesResponse(
req.requestId,
req.evaluatorId,
reader.listElements(req.uri),
null
)
)
} catch (e: Exception) {
transport.send(
ListResourcesResponse(req.requestId, req.evaluatorId, null, e.toString())
)
}
}
Message.Type.READ_MODULE_REQUEST -> {
val req = msg as ReadModuleRequest
val reader = findModuleReader(req.uri.scheme)
if (reader == null) {
transport.send(
ReadModuleResponse(
req.requestId,
req.evaluatorId,
null,
"No module reader found for scheme " + req.uri.scheme
)
)
return@start
}
try {
transport.send(
ReadModuleResponse(req.requestId, req.evaluatorId, reader.read(req.uri), null)
)
} catch (e: Exception) {
transport.send(ReadModuleResponse(req.requestId, req.evaluatorId, null, e.toString()))
}
}
Message.Type.READ_RESOURCE_REQUEST -> {
val req = msg as ReadResourceRequest
val reader = findResourceReader(req.uri.scheme)
if (reader == null) {
transport.send(
ReadResourceResponse(
req.requestId,
req.evaluatorId,
byteArrayOf(),
"No resource reader found for scheme " + req.uri.scheme
)
)
return@start
}
try {
transport.send(
ReadResourceResponse(req.requestId, req.evaluatorId, reader.read(req.uri), null)
)
} catch (e: Exception) {
transport.send(
ReadResourceResponse(req.requestId, req.evaluatorId, byteArrayOf(), e.toString())
)
}
}
else -> throw ProtocolException("Unexpected incoming request message: $msg")
}
}
)
}
}

27
pkl-core/src/test/kotlin/org/pkl/core/externalreader/ExternalResourceReader.kt Normal file
View File

@@ -0,0 +1,27 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.externalreader
import java.net.URI
import org.pkl.core.messaging.Messages.ResourceReaderSpec
/** An external resource reader, to be used with [ExternalReaderRuntime]. */
interface ExternalResourceReader : ExternalReaderBase {
fun read(uri: URI): ByteArray
val spec: ResourceReaderSpec
get() = ResourceReaderSpec(scheme, hasHierarchicalUris, isGlobbable)
}

38
pkl-core/src/test/kotlin/org/pkl/core/externalreader/TestExternalModuleReader.kt Normal file
View File

@@ -0,0 +1,38 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.externalreader
import java.net.URI
import org.pkl.core.module.PathElement
class TestExternalModuleReader : ExternalModuleReader {
override val scheme: String = "test"
override val hasHierarchicalUris: Boolean = false
override val isLocal: Boolean = true
override val isGlobbable: Boolean = false
override fun read(uri: URI): String =
"""
name = "Pigeon"
age = 40
"""
.trimIndent()
override fun listElements(uri: URI): List<PathElement> = emptyList()
}

130
pkl-core/src/test/kotlin/org/pkl/core/externalreader/TestExternalReaderProcess.kt Normal file
View File

@@ -0,0 +1,130 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.externalreader
import java.io.IOException
import java.io.PipedInputStream
import java.io.PipedOutputStream
import java.util.concurrent.CompletableFuture
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.ExecutionException
import java.util.concurrent.Future
import kotlin.random.Random
import org.pkl.core.externalreader.ExternalReaderMessages.*
import org.pkl.core.messaging.MessageTransport
import org.pkl.core.messaging.MessageTransports
import org.pkl.core.messaging.Messages.*
import org.pkl.core.messaging.ProtocolException
class TestExternalReaderProcess(private val transport: MessageTransport) : ExternalReaderProcess {
private val initializeModuleReaderResponses: MutableMap<String, Future<ModuleReaderSpec?>> =
ConcurrentHashMap()
private val initializeResourceReaderResponses: MutableMap<String, Future<ResourceReaderSpec?>> =
ConcurrentHashMap()
override fun close() {
transport.send(CloseExternalProcess())
transport.close()
}
override fun getTransport(): MessageTransport = transport
fun run() {
try {
transport.start(
{ throw ProtocolException("Unexpected incoming one-way message: $it") },
{ throw ProtocolException("Unexpected incoming request message: $it") },
)
} catch (e: ProtocolException) {
throw RuntimeException(e)
} catch (e: IOException) {
throw RuntimeException(e)
}
}
override fun getModuleReaderSpec(scheme: String): ModuleReaderSpec? =
initializeModuleReaderResponses
.computeIfAbsent(scheme) {
CompletableFuture<ModuleReaderSpec?>().apply {
val request = InitializeModuleReaderRequest(Random.nextLong(), scheme)
transport.send(request) { response ->
when (response) {
is InitializeModuleReaderResponse -> {
complete(response.spec)
}
else -> completeExceptionally(ProtocolException("unexpected response"))
}
}
}
}
.getUnderlying()
override fun getResourceReaderSpec(scheme: String): ResourceReaderSpec? =
initializeResourceReaderResponses
.computeIfAbsent(scheme) {
CompletableFuture<ResourceReaderSpec?>().apply {
val request = InitializeResourceReaderRequest(Random.nextLong(), scheme)
transport.send(request) { response ->
when (response) {
is InitializeResourceReaderResponse -> {
complete(response.spec)
}
else -> completeExceptionally(ProtocolException("unexpected response"))
}
}
}
}
.getUnderlying()
companion object {
fun initializeTestHarness(
moduleReaders: List<ExternalModuleReader>,
resourceReaders: List<ExternalResourceReader>
): Pair<TestExternalReaderProcess, ExternalReaderRuntime> {
val rxIn = PipedInputStream(10240)
val rxOut = PipedOutputStream(rxIn)
val txIn = PipedInputStream(10240)
val txOut = PipedOutputStream(txIn)
val serverTransport =
MessageTransports.stream(
ExternalReaderMessagePackDecoder(rxIn),
ExternalReaderMessagePackEncoder(txOut),
{}
)
val clientTransport =
MessageTransports.stream(
ExternalReaderMessagePackDecoder(txIn),
ExternalReaderMessagePackEncoder(rxOut),
{}
)
val runtime = ExternalReaderRuntime(moduleReaders, resourceReaders, clientTransport)
val proc = TestExternalReaderProcess(serverTransport)
Thread(runtime::run).start()
Thread(proc::run).start()
return proc to runtime
}
}
}
fun <T> Future<T>.getUnderlying(): T =
try {
get()
} catch (e: ExecutionException) {
throw e.cause!!
}

31
pkl-core/src/test/kotlin/org/pkl/core/externalreader/TestExternalResourceReader.kt Normal file
View File

@@ -0,0 +1,31 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.externalreader
import java.net.URI
import org.pkl.core.module.PathElement
class TestExternalResourceReader : ExternalResourceReader {
override val scheme: String = "test"
override val hasHierarchicalUris: Boolean = false
override val isGlobbable: Boolean = false
override fun read(uri: URI): ByteArray = "success".toByteArray(Charsets.UTF_8)
override fun listElements(uri: URI): List<PathElement> = emptyList()
}

128
pkl-core/src/test/kotlin/org/pkl/core/messaging/BaseMessagePackCodecTest.kt Normal file
View File

@@ -0,0 +1,128 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.messaging
import java.io.PipedInputStream
import java.io.PipedOutputStream
import java.net.URI
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.assertThrows
import org.msgpack.core.MessagePack
import org.pkl.core.messaging.Messages.*
import org.pkl.core.module.PathElement
class BaseMessagePackCodecTest {
private val encoder: MessageEncoder
private val decoder: MessageDecoder
init {
val inputStream = PipedInputStream()
val outputStream = PipedOutputStream(inputStream)
encoder = BaseMessagePackEncoder(MessagePack.newDefaultPacker(outputStream))
decoder = BaseMessagePackDecoder(MessagePack.newDefaultUnpacker(inputStream))
}
private fun roundtrip(message: Message) {
encoder.encode(message)
val decoded = decoder.decode()
assertThat(decoded).isEqualTo(message)
}
@Test
fun `round-trip ReadResourceRequest`() {
roundtrip(ReadResourceRequest(123, 456, URI("some/resource.json")))
}
@Test
fun `round-trip ReadResourceResponse`() {
roundtrip(ReadResourceResponse(123, 456, byteArrayOf(1, 2, 3, 4, 5), null))
}
@Test
fun `round-trip ReadModuleRequest`() {
roundtrip(ReadModuleRequest(123, 456, URI("some/module.pkl")))
}
@Test
fun `round-trip ReadModuleResponse`() {
roundtrip(ReadModuleResponse(123, 456, "x = 42", null))
}
@Test
fun `round-trip ListModulesRequest`() {
roundtrip(ListModulesRequest(135, 246, URI("foo:/bar/baz/biz")))
}
@Test
fun `round-trip ListModulesResponse`() {
roundtrip(
ListModulesResponse(
123,
234,
listOf(PathElement("foo", true), PathElement("bar", false)),
null
)
)
roundtrip(ListModulesResponse(123, 234, null, "Something dun went wrong"))
}
@Test
fun `round-trip ListResourcesRequest`() {
roundtrip(ListResourcesRequest(987, 1359, URI("bar:/bazzy")))
}
@Test
fun `round-trip ListResourcesResponse`() {
roundtrip(
ListResourcesResponse(
3851,
3019,
listOf(PathElement("foo", true), PathElement("bar", false)),
null
)
)
roundtrip(ListResourcesResponse(3851, 3019, null, "something went wrong"))
}
@Test
fun `decode request with missing request ID`() {
val bytes =
MessagePack.newDefaultBufferPacker()
.apply {
packArrayHeader(2)
packInt(Message.Type.LIST_RESOURCES_REQUEST.code)
packMapHeader(1)
packString("uri")
packString("file:/test")
}
.toByteArray()
val decoder = BaseMessagePackDecoder(MessagePack.newDefaultUnpacker(bytes))
val exception = assertThrows<DecodeException> { decoder.decode() }
assertThat(exception.message).contains("requestId")
}
@Test
fun `decode invalid message header`() {
val bytes = MessagePack.newDefaultBufferPacker().apply { packInt(2) }.toByteArray()
val decoder = BaseMessagePackDecoder(MessagePack.newDefaultUnpacker(bytes))
val exception = assertThrows<DecodeException> { decoder.decode() }
assertThat(exception).hasMessage("Malformed message header.")
assertThat(exception).hasRootCauseMessage("Expected Array, but got Integer (02)")
}
}

20
pkl-core/src/test/kotlin/org/pkl/core/module/ModuleKeyFactoriesTest.kt
View File

@@ -26,6 +26,7 @@ import org.pkl.commons.createParentDirectories
import org.pkl.commons.toPath
import org.pkl.commons.writeString
import org.pkl.core.SecurityManagers
import org.pkl.core.externalreader.*
class ModuleKeyFactoriesTest {
@Test
@@ -126,4 +127,23 @@ class ModuleKeyFactoriesTest {
val module2 = factory.create(URI("other"))
assertThat(module2).isNotPresent
}
@Test
fun externalProcess() {
val extReader = TestExternalModuleReader()
val (proc, runtime) =
TestExternalReaderProcess.initializeTestHarness(listOf(extReader), emptyList())
val factory = ModuleKeyFactories.externalProcess(extReader.scheme, proc)
val module = factory.create(URI("test:foo"))
assertThat(module).isPresent
assertThat(module.get().uri.scheme).isEqualTo("test")
val module2 = factory.create(URI("other"))
assertThat(module2).isNotPresent
proc.close()
runtime.close()
}
}

2
pkl-core/src/test/kotlin/org/pkl/core/project/ProjectTest.kt
View File

@@ -70,6 +70,8 @@ class ProjectTest {
listOf(path.resolve("modulepath1/"), path.resolve("modulepath2/")),
Duration.ofMinutes(5.0),
path,
null,
null,
null
)
val expectedAnnotations =

19
pkl-core/src/test/kotlin/org/pkl/core/resource/ResourceReadersTest.kt
View File

@@ -23,6 +23,8 @@ import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.assertThrows
import org.junit.jupiter.api.io.TempDir
import org.pkl.core.externalreader.TestExternalReaderProcess
import org.pkl.core.externalreader.TestExternalResourceReader
import org.pkl.core.module.ModulePathResolver
class ResourceReadersTest {
@@ -132,4 +134,21 @@ class ResourceReadersTest {
assertThat(resource).contains("success")
}
@Test
fun externalProcess() {
val extReader = TestExternalResourceReader()
val (proc, runtime) =
TestExternalReaderProcess.initializeTestHarness(emptyList(), listOf(extReader))
val reader = ResourceReaders.externalProcess(extReader.scheme, proc)
val resource = reader.read(URI("test:foo"))
assertThat(resource).isPresent
assertThat(resource.get()).isInstanceOf(Resource::class.java)
assertThat((resource.get() as Resource).text).contains("success")
proc.close()
runtime.close()
}
}

18
pkl-core/src/test/resources/org/pkl/core/project/project1/PklProject
View File

@@ -22,4 +22,22 @@ evaluatorSettings {
noCache = false
rootDir = "my-root-dir/"
timeout = 5.min
externalModuleReaders {
["scheme1"] {
executable = "reader1"
}
["scheme2"] {
executable = "reader2"
arguments { "with"; "args" }
}
}
externalResourceReaders {
["scheme3"] {
executable = "reader3"
}
["scheme4"] {
executable = "reader4"
arguments { "with"; "args" }
}
}
}