Untangle external reader code (#776)

- move the following classes into package externalreader:
  - ExternalModuleResolver
  - ExternalResourceResolver
  - MessageTransportModuleResolver (renamed to ExternalModuleResolverImpl, made package-private)
  - MessageTransportResourceResolver (renamed to ExternalResourceResolverImpl, made package-private)
- replace interface ExternalModuleResolver.Spec with record ExternalModuleReaderSpec
- replace interface ExternalResourceResolver.Spec with record ExternalResourceReaderSpec
- translate between messaging.ResourceReaderSpec and ExternalResourceReaderSpec (eliminates dependency from messaging on higher layer)
- translate between messaging.ResourceResolverSpec and ExternalResourceResolverSpec (eliminates dependency from messaging on higher layer)
- add ServerMessages.ExternalReader and translate between this message component and the PklEvaluatorSettings.ExternalReader API
- add ServerMessages.Proxy and translate between this message component and the PklEvaluatorSettings.Proxy API
- change type of CreateEvaluatorRequest.allowedModules/allowedResources from List<Pattern>? to List<String>?
  - removes a lot of code
  - should not need to create a Pattern object to send a message
- deprecate method evaluatorSettings.PklEvaluatorSettings.Proxy.create()
  - only seems useful internally, inlined

Co-authored-by: Dan Chao <dan.chao@apple.com>
This commit is contained in:
odenix
2025-02-20 22:38:51 -08:00
committed by GitHub
parent 31c80e792e
commit 52a86d3f32
31 changed files with 219 additions and 213 deletions
@@ -1,5 +1,5 @@
/* /*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -137,6 +137,7 @@ public record PklEvaluatorSettings(
} }
public record Proxy(@Nullable URI address, @Nullable List<String> noProxy) { public record Proxy(@Nullable URI address, @Nullable List<String> noProxy) {
@Deprecated(forRemoval = true)
public static Proxy create(@Nullable String address, @Nullable List<String> noProxy) { public static Proxy create(@Nullable String address, @Nullable List<String> noProxy) {
URI addressUri; URI addressUri;
try { try {
@@ -147,14 +148,19 @@ public record PklEvaluatorSettings(
return new Proxy(addressUri, noProxy); return new Proxy(addressUri, noProxy);
} }
@SuppressWarnings("unchecked")
public static @Nullable Proxy parse(Value input) { public static @Nullable Proxy parse(Value input) {
if (input instanceof PNull) { if (input instanceof PNull) {
return null; return null;
} else if (input instanceof PObject proxy) { } else if (input instanceof PObject proxy) {
var address = (String) proxy.get("address"); var address = (String) proxy.get("address");
@SuppressWarnings("unchecked")
var noProxy = (List<String>) proxy.get("noProxy"); var noProxy = (List<String>) proxy.get("noProxy");
return create(address, noProxy); try {
var addressUri = address == null ? null : new URI(address);
return new Proxy(addressUri, noProxy);
} catch (URISyntaxException e) {
throw new PklException(ErrorMessages.create("invalidUri", address));
}
} else { } else {
throw PklBugException.unreachableCode(); throw PklBugException.unreachableCode();
} }
@@ -1,5 +1,5 @@
/* /*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -13,24 +13,19 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.pkl.core.module; package org.pkl.core.externalreader;
import java.io.IOException; import java.io.IOException;
import java.net.URI; import java.net.URI;
import java.util.List; import java.util.List;
import org.pkl.core.SecurityManager; import org.pkl.core.SecurityManager;
import org.pkl.core.SecurityManagerException; import org.pkl.core.SecurityManagerException;
import org.pkl.core.messaging.MessageTransport;
import org.pkl.core.module.PathElement;
public interface ExternalModuleResolver { public interface ExternalModuleResolver {
static ExternalModuleResolver of(MessageTransport transport, long evaluatorId) {
interface Spec { return new ExternalModuleResolverImpl(transport, evaluatorId);
boolean hasHierarchicalUris();
boolean isGlobbable();
boolean isLocal();
String scheme();
} }
String resolveModule(SecurityManager securityManager, URI uri) String resolveModule(SecurityManager securityManager, URI uri)
@@ -1,5 +1,5 @@
/* /*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.pkl.core.messaging; package org.pkl.core.externalreader;
import java.io.IOException; import java.io.IOException;
import java.net.URI; import java.net.URI;
@@ -26,21 +26,23 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Future; import java.util.concurrent.Future;
import org.pkl.core.SecurityManager; import org.pkl.core.SecurityManager;
import org.pkl.core.SecurityManagerException; import org.pkl.core.SecurityManagerException;
import org.pkl.core.messaging.MessageTransport;
import org.pkl.core.messaging.MessageTransports;
import org.pkl.core.messaging.Messages.ListModulesRequest; import org.pkl.core.messaging.Messages.ListModulesRequest;
import org.pkl.core.messaging.Messages.ListModulesResponse; import org.pkl.core.messaging.Messages.ListModulesResponse;
import org.pkl.core.messaging.Messages.ReadModuleRequest; import org.pkl.core.messaging.Messages.ReadModuleRequest;
import org.pkl.core.messaging.Messages.ReadModuleResponse; import org.pkl.core.messaging.Messages.ReadModuleResponse;
import org.pkl.core.module.ExternalModuleResolver; import org.pkl.core.messaging.ProtocolException;
import org.pkl.core.module.PathElement; import org.pkl.core.module.PathElement;
public class MessageTransportModuleResolver implements ExternalModuleResolver { final class ExternalModuleResolverImpl implements ExternalModuleResolver {
private final MessageTransport transport; private final MessageTransport transport;
private final long evaluatorId; private final long evaluatorId;
private final Map<URI, Future<String>> readResponses = new ConcurrentHashMap<>(); private final Map<URI, Future<String>> readResponses = new ConcurrentHashMap<>();
private final Map<URI, Future<List<PathElement>>> listResponses = new ConcurrentHashMap<>(); private final Map<URI, Future<List<PathElement>>> listResponses = new ConcurrentHashMap<>();
private final Random requestIdGenerator = new Random(); private final Random requestIdGenerator = new Random();
public MessageTransportModuleResolver(MessageTransport transport, long evaluatorId) { ExternalModuleResolverImpl(MessageTransport transport, long evaluatorId) {
this.transport = transport; this.transport = transport;
this.evaluatorId = evaluatorId; this.evaluatorId = evaluatorId;
} }
@@ -1,5 +1,5 @@
/* /*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -1,5 +1,5 @@
/* /*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -23,35 +23,33 @@ import org.pkl.core.util.Nullable;
final class ExternalReaderMessages { final class ExternalReaderMessages {
private ExternalReaderMessages() {} private ExternalReaderMessages() {}
public record InitializeModuleReaderRequest(long requestId, String scheme) record InitializeModuleReaderRequest(long requestId, String scheme) implements Server.Request {
implements Server.Request {
public Type type() { public Type type() {
return Type.INITIALIZE_MODULE_READER_REQUEST; return Type.INITIALIZE_MODULE_READER_REQUEST;
} }
} }
public record InitializeResourceReaderRequest(long requestId, String scheme) record InitializeResourceReaderRequest(long requestId, String scheme) implements Server.Request {
implements Server.Request {
public Type type() { public Type type() {
return Type.INITIALIZE_RESOURCE_READER_REQUEST; return Type.INITIALIZE_RESOURCE_READER_REQUEST;
} }
} }
public record InitializeModuleReaderResponse(long requestId, @Nullable ModuleReaderSpec spec) record InitializeModuleReaderResponse(long requestId, @Nullable ModuleReaderSpec spec)
implements Client.Response { implements Client.Response {
public Type type() { public Type type() {
return Type.INITIALIZE_MODULE_READER_RESPONSE; return Type.INITIALIZE_MODULE_READER_RESPONSE;
} }
} }
public record InitializeResourceReaderResponse(long requestId, @Nullable ResourceReaderSpec spec) record InitializeResourceReaderResponse(long requestId, @Nullable ResourceReaderSpec spec)
implements Client.Response { implements Client.Response {
public Type type() { public Type type() {
return Type.INITIALIZE_RESOURCE_READER_RESPONSE; return Type.INITIALIZE_RESOURCE_READER_RESPONSE;
} }
} }
public record CloseExternalProcess() implements Server.OneWay { record CloseExternalProcess() implements Server.OneWay {
public Type type() { public Type type() {
return Type.CLOSE_EXTERNAL_PROCESS; return Type.CLOSE_EXTERNAL_PROCESS;
} }
@@ -1,5 +1,5 @@
/* /*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -17,8 +17,6 @@ package org.pkl.core.externalreader;
import java.io.IOException; import java.io.IOException;
import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader; import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader;
import org.pkl.core.module.ExternalModuleResolver;
import org.pkl.core.resource.ExternalResourceResolver;
import org.pkl.core.util.Nullable; import org.pkl.core.util.Nullable;
/** An external process that reads Pkl modules and resources. */ /** An external process that reads Pkl modules and resources. */
@@ -56,7 +54,8 @@ public interface ExternalReaderProcess extends AutoCloseable {
* @throws IllegalStateException if this process has already been {@linkplain #close closed} * @throws IllegalStateException if this process has already been {@linkplain #close closed}
* @throws IOException if an I/O error occurs * @throws IOException if an I/O error occurs
*/ */
ExternalModuleResolver.@Nullable Spec getModuleReaderSpec(String scheme) throws IOException; @Nullable
ModuleReaderSpec getModuleReaderSpec(String scheme) throws IOException;
/** /**
* Returns the spec, if available, of this process's resource reader with the given scheme. * Returns the spec, if available, of this process's resource reader with the given scheme.
@@ -64,7 +63,8 @@ public interface ExternalReaderProcess extends AutoCloseable {
* @throws IllegalStateException if this process has already been {@linkplain #close closed} * @throws IllegalStateException if this process has already been {@linkplain #close closed}
* @throws IOException if an I/O error occurs * @throws IOException if an I/O error occurs
*/ */
ExternalResourceResolver.@Nullable Spec getResourceReaderSpec(String scheme) throws IOException; @Nullable
ResourceReaderSpec getResourceReaderSpec(String scheme) throws IOException;
/** /**
* Closes this process, releasing any associated resources. * Closes this process, releasing any associated resources.
@@ -30,12 +30,8 @@ import javax.annotation.concurrent.GuardedBy;
import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader; import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader;
import org.pkl.core.externalreader.ExternalReaderMessages.*; import org.pkl.core.externalreader.ExternalReaderMessages.*;
import org.pkl.core.messaging.MessageTransport; import org.pkl.core.messaging.MessageTransport;
import org.pkl.core.messaging.MessageTransportModuleResolver;
import org.pkl.core.messaging.MessageTransportResourceResolver;
import org.pkl.core.messaging.MessageTransports; import org.pkl.core.messaging.MessageTransports;
import org.pkl.core.messaging.ProtocolException; import org.pkl.core.messaging.ProtocolException;
import org.pkl.core.module.ExternalModuleResolver;
import org.pkl.core.resource.ExternalResourceResolver;
import org.pkl.core.util.ErrorMessages; import org.pkl.core.util.ErrorMessages;
import org.pkl.core.util.LateInit; import org.pkl.core.util.LateInit;
import org.pkl.core.util.Nullable; import org.pkl.core.util.Nullable;
@@ -46,9 +42,9 @@ final class ExternalReaderProcessImpl implements ExternalReaderProcess {
private final ExternalReader spec; private final ExternalReader spec;
private final @Nullable String logPrefix; private final @Nullable String logPrefix;
private final Map<String, Future<ExternalModuleResolver.@Nullable Spec>> private final Map<String, Future<@Nullable ModuleReaderSpec>> initializeModuleReaderResponses =
initializeModuleReaderResponses = new ConcurrentHashMap<>(); new ConcurrentHashMap<>();
private final Map<String, Future<ExternalResourceResolver.@Nullable Spec>> private final Map<String, Future<@Nullable ResourceReaderSpec>>
initializeResourceReaderResponses = new ConcurrentHashMap<>(); initializeResourceReaderResponses = new ConcurrentHashMap<>();
private final Random requestIdGenerator = new Random(); private final Random requestIdGenerator = new Random();
@@ -80,13 +76,13 @@ final class ExternalReaderProcessImpl implements ExternalReaderProcess {
@Override @Override
public ExternalModuleResolver getModuleResolver(long evaluatorId) public ExternalModuleResolver getModuleResolver(long evaluatorId)
throws ExternalReaderProcessException { throws ExternalReaderProcessException {
return new MessageTransportModuleResolver(getTransport(), evaluatorId); return ExternalModuleResolver.of(getTransport(), evaluatorId);
} }
@Override @Override
public ExternalResourceResolver getResourceResolver(long evaluatorId) public ExternalResourceResolver getResourceResolver(long evaluatorId)
throws ExternalReaderProcessException { throws ExternalReaderProcessException {
return new MessageTransportResourceResolver(getTransport(), evaluatorId); return ExternalResourceResolver.of(getTransport(), evaluatorId);
} }
private MessageTransport getTransport() throws ExternalReaderProcessException { private MessageTransport getTransport() throws ExternalReaderProcessException {
@@ -175,13 +171,12 @@ final class ExternalReaderProcessImpl implements ExternalReaderProcess {
} }
@Override @Override
public ExternalModuleResolver.@Nullable Spec getModuleReaderSpec(String uriScheme) public ModuleReaderSpec getModuleReaderSpec(String uriScheme) throws IOException {
throws IOException {
return MessageTransports.resolveFuture( return MessageTransports.resolveFuture(
initializeModuleReaderResponses.computeIfAbsent( initializeModuleReaderResponses.computeIfAbsent(
uriScheme, uriScheme,
(scheme) -> { (scheme) -> {
var future = new CompletableFuture<ExternalModuleResolver.@Nullable Spec>(); var future = new CompletableFuture<@Nullable ModuleReaderSpec>();
var request = var request =
new InitializeModuleReaderRequest(requestIdGenerator.nextLong(), scheme); new InitializeModuleReaderRequest(requestIdGenerator.nextLong(), scheme);
try { try {
@@ -190,7 +185,15 @@ final class ExternalReaderProcessImpl implements ExternalReaderProcess {
request, request,
(response) -> { (response) -> {
if (response instanceof InitializeModuleReaderResponse resp) { if (response instanceof InitializeModuleReaderResponse resp) {
future.complete(resp.spec()); var spec =
resp.spec() == null
? null
: new ModuleReaderSpec(
resp.spec().scheme(),
resp.spec().hasHierarchicalUris(),
resp.spec().isLocal(),
resp.spec().isGlobbable());
future.complete(spec);
} else { } else {
future.completeExceptionally( future.completeExceptionally(
new ProtocolException("unexpected response")); new ProtocolException("unexpected response"));
@@ -204,13 +207,12 @@ final class ExternalReaderProcessImpl implements ExternalReaderProcess {
} }
@Override @Override
public ExternalResourceResolver.@Nullable Spec getResourceReaderSpec(String uriScheme) public ResourceReaderSpec getResourceReaderSpec(String uriScheme) throws IOException {
throws IOException {
return MessageTransports.resolveFuture( return MessageTransports.resolveFuture(
initializeResourceReaderResponses.computeIfAbsent( initializeResourceReaderResponses.computeIfAbsent(
uriScheme, uriScheme,
(scheme) -> { (scheme) -> {
var future = new CompletableFuture<ExternalResourceResolver.@Nullable Spec>(); var future = new CompletableFuture<@Nullable ResourceReaderSpec>();
var request = var request =
new InitializeResourceReaderRequest(requestIdGenerator.nextLong(), scheme); new InitializeResourceReaderRequest(requestIdGenerator.nextLong(), scheme);
try { try {
@@ -220,7 +222,14 @@ final class ExternalReaderProcessImpl implements ExternalReaderProcess {
(response) -> { (response) -> {
log(response.toString()); log(response.toString());
if (response instanceof InitializeResourceReaderResponse resp) { if (response instanceof InitializeResourceReaderResponse resp) {
future.complete(resp.spec()); var spec =
resp.spec() == null
? null
: new ResourceReaderSpec(
resp.spec().scheme(),
resp.spec().hasHierarchicalUris(),
resp.spec().isGlobbable());
future.complete(spec);
} else { } else {
future.completeExceptionally( future.completeExceptionally(
new ProtocolException("unexpected response")); new ProtocolException("unexpected response"));
@@ -1,5 +1,5 @@
/* /*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.pkl.core.resource; package org.pkl.core.externalreader;
import java.io.IOException; import java.io.IOException;
import java.net.URI; import java.net.URI;
@@ -21,16 +21,12 @@ import java.util.List;
import java.util.Optional; import java.util.Optional;
import org.pkl.core.SecurityManager; import org.pkl.core.SecurityManager;
import org.pkl.core.SecurityManagerException; import org.pkl.core.SecurityManagerException;
import org.pkl.core.messaging.MessageTransport;
import org.pkl.core.module.PathElement; import org.pkl.core.module.PathElement;
public interface ExternalResourceResolver { public interface ExternalResourceResolver {
static ExternalResourceResolver of(MessageTransport transport, long evaluatorId) {
interface Spec { return new ExternalResourceResolverImpl(transport, evaluatorId);
boolean hasHierarchicalUris();
boolean isGlobbable();
String scheme();
} }
Optional<Object> read(URI uri) throws IOException; Optional<Object> read(URI uri) throws IOException;
@@ -1,5 +1,5 @@
/* /*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.pkl.core.messaging; package org.pkl.core.externalreader;
import java.io.IOException; import java.io.IOException;
import java.net.URI; import java.net.URI;
@@ -27,19 +27,21 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Future; import java.util.concurrent.Future;
import org.pkl.core.SecurityManager; import org.pkl.core.SecurityManager;
import org.pkl.core.SecurityManagerException; import org.pkl.core.SecurityManagerException;
import org.pkl.core.messaging.MessageTransport;
import org.pkl.core.messaging.MessageTransports;
import org.pkl.core.messaging.Messages.*; import org.pkl.core.messaging.Messages.*;
import org.pkl.core.messaging.ProtocolException;
import org.pkl.core.module.PathElement; import org.pkl.core.module.PathElement;
import org.pkl.core.resource.ExternalResourceResolver;
import org.pkl.core.resource.Resource; import org.pkl.core.resource.Resource;
public class MessageTransportResourceResolver implements ExternalResourceResolver { final class ExternalResourceResolverImpl implements ExternalResourceResolver {
private final MessageTransport transport; private final MessageTransport transport;
private final long evaluatorId; private final long evaluatorId;
private final Map<URI, Future<byte[]>> readResponses = new ConcurrentHashMap<>(); private final Map<URI, Future<byte[]>> readResponses = new ConcurrentHashMap<>();
private final Map<URI, Future<List<PathElement>>> listResponses = new ConcurrentHashMap<>(); private final Map<URI, Future<List<PathElement>>> listResponses = new ConcurrentHashMap<>();
private final Random requestIdGenerator = new Random(); private final Random requestIdGenerator = new Random();
public MessageTransportResourceResolver(MessageTransport transport, long evaluatorId) { ExternalResourceResolverImpl(MessageTransport transport, long evaluatorId) {
this.transport = transport; this.transport = transport;
this.evaluatorId = evaluatorId; this.evaluatorId = evaluatorId;
} }
@@ -0,0 +1,19 @@
/*
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.externalreader;
public record ModuleReaderSpec(
String scheme, boolean hasHierarchicalUris, boolean isLocal, boolean isGlobbable) {}
@@ -0,0 +1,18 @@
/*
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.externalreader;
public record ResourceReaderSpec(String scheme, boolean hasHierarchicalUris, boolean isGlobbable) {}
@@ -1,5 +1,5 @@
/* /*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -20,20 +20,17 @@ import java.util.Arrays;
import java.util.List; import java.util.List;
import java.util.Objects; import java.util.Objects;
import org.pkl.core.messaging.Message.*; import org.pkl.core.messaging.Message.*;
import org.pkl.core.module.ExternalModuleResolver;
import org.pkl.core.module.PathElement; import org.pkl.core.module.PathElement;
import org.pkl.core.resource.ExternalResourceResolver;
import org.pkl.core.util.Nullable; import org.pkl.core.util.Nullable;
public final class Messages { public final class Messages {
private Messages() {} private Messages() {}
public record ModuleReaderSpec( public record ModuleReaderSpec(
String scheme, boolean hasHierarchicalUris, boolean isLocal, boolean isGlobbable) String scheme, boolean hasHierarchicalUris, boolean isLocal, boolean isGlobbable) {}
implements ExternalModuleResolver.Spec {}
public record ResourceReaderSpec(String scheme, boolean hasHierarchicalUris, boolean isGlobbable) public record ResourceReaderSpec(
implements ExternalResourceResolver.Spec {} String scheme, boolean hasHierarchicalUris, boolean isGlobbable) {}
public record ListResourcesRequest(long requestId, long evaluatorId, URI uri) public record ListResourcesRequest(long requestId, long evaluatorId, URI uri)
implements Server.Request { implements Server.Request {
@@ -1,5 +1,5 @@
/* /*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -28,6 +28,7 @@ import java.util.Optional;
import java.util.ServiceLoader; import java.util.ServiceLoader;
import javax.annotation.concurrent.GuardedBy; import javax.annotation.concurrent.GuardedBy;
import org.pkl.core.Closeables; import org.pkl.core.Closeables;
import org.pkl.core.externalreader.ExternalModuleResolver;
import org.pkl.core.externalreader.ExternalReaderProcess; import org.pkl.core.externalreader.ExternalReaderProcess;
import org.pkl.core.externalreader.ExternalReaderProcessException; import org.pkl.core.externalreader.ExternalReaderProcessException;
import org.pkl.core.util.ErrorMessages; import org.pkl.core.util.ErrorMessages;
@@ -1,5 +1,5 @@
/* /*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -29,7 +29,9 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import org.pkl.core.SecurityManager; import org.pkl.core.SecurityManager;
import org.pkl.core.SecurityManagerException; import org.pkl.core.SecurityManagerException;
import org.pkl.core.externalreader.ExternalModuleResolver;
import org.pkl.core.externalreader.ExternalReaderProcessException; import org.pkl.core.externalreader.ExternalReaderProcessException;
import org.pkl.core.externalreader.ModuleReaderSpec;
import org.pkl.core.packages.Dependency; import org.pkl.core.packages.Dependency;
import org.pkl.core.packages.Dependency.LocalDependency; import org.pkl.core.packages.Dependency.LocalDependency;
import org.pkl.core.packages.PackageAssetUri; import org.pkl.core.packages.PackageAssetUri;
@@ -130,7 +132,7 @@ public final class ModuleKeys {
/** Creates a module key for an externally read module. */ /** Creates a module key for an externally read module. */
public static ModuleKey externalResolver( public static ModuleKey externalResolver(
URI uri, ExternalModuleResolver.Spec spec, ExternalModuleResolver resolver) { URI uri, ModuleReaderSpec spec, ExternalModuleResolver resolver) {
return new ExternalResolver(uri, spec, resolver); return new ExternalResolver(uri, spec, resolver);
} }
@@ -778,10 +780,10 @@ public final class ModuleKeys {
public static class ExternalResolver implements ModuleKey { public static class ExternalResolver implements ModuleKey {
private final URI uri; private final URI uri;
private final ExternalModuleResolver.Spec spec; private final ModuleReaderSpec spec;
private final ExternalModuleResolver resolver; private final ExternalModuleResolver resolver;
ExternalResolver(URI uri, ExternalModuleResolver.Spec spec, ExternalModuleResolver resolver) { ExternalResolver(URI uri, ModuleReaderSpec spec, ExternalModuleResolver resolver) {
this.uri = uri; this.uri = uri;
this.spec = spec; this.spec = spec;
this.resolver = resolver; this.resolver = resolver;
@@ -1,5 +1,5 @@
/* /*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -31,6 +31,8 @@ import org.pkl.core.SecurityManager;
import org.pkl.core.SecurityManagerException; import org.pkl.core.SecurityManagerException;
import org.pkl.core.externalreader.ExternalReaderProcess; import org.pkl.core.externalreader.ExternalReaderProcess;
import org.pkl.core.externalreader.ExternalReaderProcessException; import org.pkl.core.externalreader.ExternalReaderProcessException;
import org.pkl.core.externalreader.ExternalResourceResolver;
import org.pkl.core.externalreader.ResourceReaderSpec;
import org.pkl.core.module.FileResolver; import org.pkl.core.module.FileResolver;
import org.pkl.core.module.ModulePathResolver; import org.pkl.core.module.ModulePathResolver;
import org.pkl.core.module.PathElement; import org.pkl.core.module.PathElement;
@@ -162,7 +164,7 @@ public final class ResourceReaders {
/** Returns a reader for external and client reader resources. */ /** Returns a reader for external and client reader resources. */
public static ResourceReader externalResolver( public static ResourceReader externalResolver(
ExternalResourceResolver.Spec spec, ExternalResourceResolver resolver) { ResourceReaderSpec spec, ExternalResourceResolver resolver) {
return new ExternalResolver(spec, resolver); return new ExternalResolver(spec, resolver);
} }
@@ -691,11 +693,10 @@ public final class ResourceReaders {
} }
private static final class ExternalResolver implements ResourceReader { private static final class ExternalResolver implements ResourceReader {
private final ExternalResourceResolver.Spec readerSpec; private final ResourceReaderSpec readerSpec;
private final ExternalResourceResolver resolver; private final ExternalResourceResolver resolver;
public ExternalResolver( public ExternalResolver(ResourceReaderSpec readerSpec, ExternalResourceResolver resolver) {
ExternalResourceResolver.Spec readerSpec, ExternalResourceResolver resolver) {
this.readerSpec = readerSpec; this.readerSpec = readerSpec;
this.resolver = resolver; this.resolver = resolver;
} }
@@ -43,7 +43,7 @@ import org.pkl.core.util.IoUtils;
public class VmImportAnalyzer { public class VmImportAnalyzer {
@TruffleBoundary @TruffleBoundary
public static ImportGraph analyze(URI[] moduleUris, VmContext context) public static ImportGraph analyze(URI[] moduleUris, VmContext context)
throws IOException, SecurityManagerException { throws IOException, SecurityManagerException, ExternalReaderProcessException {
var imports = new TreeMap<URI, Set<ImportGraph.Import>>(); var imports = new TreeMap<URI, Set<ImportGraph.Import>>();
var resolvedImports = new TreeMap<URI, URI>(); var resolvedImports = new TreeMap<URI, URI>();
for (var moduleUri : moduleUris) { for (var moduleUri : moduleUris) {
@@ -23,6 +23,7 @@ import java.net.URISyntaxException;
import org.pkl.core.ImportGraph; import org.pkl.core.ImportGraph;
import org.pkl.core.ImportGraph.Import; import org.pkl.core.ImportGraph.Import;
import org.pkl.core.SecurityManagerException; import org.pkl.core.SecurityManagerException;
import org.pkl.core.externalreader.ExternalReaderProcessException;
import org.pkl.core.packages.PackageLoadError; import org.pkl.core.packages.PackageLoadError;
import org.pkl.core.runtime.AnalyzeModule; import org.pkl.core.runtime.AnalyzeModule;
import org.pkl.core.runtime.VmContext; import org.pkl.core.runtime.VmContext;
@@ -91,7 +92,10 @@ public final class AnalyzeNodes {
try { try {
var results = VmImportAnalyzer.analyze(uris, context); var results = VmImportAnalyzer.analyze(uris, context);
return importGraphFactory.create(results); return importGraphFactory.create(results);
} catch (IOException | SecurityManagerException | PackageLoadError e) { } catch (IOException
| SecurityManagerException
| PackageLoadError
| ExternalReaderProcessException e) {
throw exceptionBuilder().withCause(e).build(); throw exceptionBuilder().withCause(e).build();
} }
} }
@@ -1,5 +1,5 @@
/* /*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -18,7 +18,7 @@ package org.pkl.core.externalreader
import java.net.URI import java.net.URI
import org.pkl.core.messaging.Messages.ModuleReaderSpec import org.pkl.core.messaging.Messages.ModuleReaderSpec
/** An external module reader, to be used with [ExternalReaderRuntime]. */ /** An external module reader, to be used with [ExternalReaderClient]. */
interface ExternalModuleReader : ExternalReaderBase { interface ExternalModuleReader : ExternalReaderBase {
val isLocal: Boolean val isLocal: Boolean
@@ -19,14 +19,15 @@ import java.io.IOException
import org.pkl.core.externalreader.ExternalReaderMessages.* import org.pkl.core.externalreader.ExternalReaderMessages.*
import org.pkl.core.messaging.Message import org.pkl.core.messaging.Message
import org.pkl.core.messaging.MessageTransport import org.pkl.core.messaging.MessageTransport
import org.pkl.core.messaging.Messages
import org.pkl.core.messaging.Messages.* import org.pkl.core.messaging.Messages.*
import org.pkl.core.messaging.ProtocolException import org.pkl.core.messaging.ProtocolException
import org.pkl.core.util.Nullable import org.pkl.core.util.Nullable
/** An implementation of the client side of the external reader flow */ /** An implementation of the client side of the external reader flow */
class ExternalReaderRuntime( class ExternalReaderClient(
private val moduleReaders: List<ExternalModuleReader>, private val externalModuleReaders: List<ExternalModuleReader>,
private val resourceReaders: List<ExternalResourceReader>, private val externalResourceReaders: List<ExternalResourceReader>,
private val transport: MessageTransport, private val transport: MessageTransport,
) { ) {
/** Close the runtime and its transport. */ /** Close the runtime and its transport. */
@@ -35,7 +36,7 @@ class ExternalReaderRuntime(
} }
private fun findModuleReader(scheme: String): @Nullable ExternalModuleReader? { private fun findModuleReader(scheme: String): @Nullable ExternalModuleReader? {
for (moduleReader in moduleReaders) { for (moduleReader in externalModuleReaders) {
if (moduleReader.scheme.equals(scheme, ignoreCase = true)) { if (moduleReader.scheme.equals(scheme, ignoreCase = true)) {
return moduleReader return moduleReader
} }
@@ -44,7 +45,7 @@ class ExternalReaderRuntime(
} }
private fun findResourceReader(scheme: String): @Nullable ExternalResourceReader? { private fun findResourceReader(scheme: String): @Nullable ExternalResourceReader? {
for (resourceReader in resourceReaders) { for (resourceReader in externalResourceReaders) {
if (resourceReader.scheme.equals(scheme, ignoreCase = true)) { if (resourceReader.scheme.equals(scheme, ignoreCase = true)) {
return resourceReader return resourceReader
} }
@@ -72,7 +73,7 @@ class ExternalReaderRuntime(
Message.Type.INITIALIZE_MODULE_READER_REQUEST -> { Message.Type.INITIALIZE_MODULE_READER_REQUEST -> {
val req = msg as InitializeModuleReaderRequest val req = msg as InitializeModuleReaderRequest
val reader = findModuleReader(req.scheme) val reader = findModuleReader(req.scheme)
var spec: @Nullable ModuleReaderSpec? = null var spec: Messages.ModuleReaderSpec? = null
if (reader != null) { if (reader != null) {
spec = reader.spec spec = reader.spec
} }
@@ -81,7 +82,7 @@ class ExternalReaderRuntime(
Message.Type.INITIALIZE_RESOURCE_READER_REQUEST -> { Message.Type.INITIALIZE_RESOURCE_READER_REQUEST -> {
val req = msg as InitializeResourceReaderRequest val req = msg as InitializeResourceReaderRequest
val reader = findResourceReader(req.scheme) val reader = findResourceReader(req.scheme)
var spec: @Nullable ResourceReaderSpec? = null var spec: Messages.ResourceReaderSpec? = null
if (reader != null) { if (reader != null) {
spec = reader.spec spec = reader.spec
} }
@@ -1,5 +1,5 @@
/* /*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -18,7 +18,7 @@ package org.pkl.core.externalreader
import java.net.URI import java.net.URI
import org.pkl.core.messaging.Messages.ResourceReaderSpec import org.pkl.core.messaging.Messages.ResourceReaderSpec
/** An external resource reader, to be used with [ExternalReaderRuntime]. */ /** An external resource reader, to be used with [ExternalReaderClient]. */
interface ExternalResourceReader : ExternalReaderBase { interface ExternalResourceReader : ExternalReaderBase {
fun read(uri: URI): ByteArray fun read(uri: URI): ByteArray
@@ -1,5 +1,5 @@
/* /*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -23,7 +23,7 @@ import org.msgpack.core.MessagePack
import org.pkl.core.externalreader.ExternalReaderMessages.* import org.pkl.core.externalreader.ExternalReaderMessages.*
import org.pkl.core.messaging.* import org.pkl.core.messaging.*
class ExternalProcessProcessReaderMessagePackCodecTest { class MessagePackCodecTest {
private val encoder: MessageEncoder private val encoder: MessageEncoder
private val decoder: MessageDecoder private val decoder: MessageDecoder
@@ -1,5 +1,5 @@
/* /*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -25,10 +25,7 @@ import java.util.concurrent.Future
import kotlin.random.Random import kotlin.random.Random
import org.pkl.core.externalreader.ExternalReaderMessages.* import org.pkl.core.externalreader.ExternalReaderMessages.*
import org.pkl.core.messaging.MessageTransport import org.pkl.core.messaging.MessageTransport
import org.pkl.core.messaging.MessageTransportModuleResolver
import org.pkl.core.messaging.MessageTransportResourceResolver
import org.pkl.core.messaging.MessageTransports import org.pkl.core.messaging.MessageTransports
import org.pkl.core.messaging.Messages.*
import org.pkl.core.messaging.ProtocolException import org.pkl.core.messaging.ProtocolException
class TestExternalReaderProcess(private val transport: MessageTransport) : ExternalReaderProcess { class TestExternalReaderProcess(private val transport: MessageTransport) : ExternalReaderProcess {
@@ -42,11 +39,11 @@ class TestExternalReaderProcess(private val transport: MessageTransport) : Exter
transport.close() transport.close()
} }
override fun getModuleResolver(evaluatorId: Long): MessageTransportModuleResolver = override fun getModuleResolver(evaluatorId: Long): ExternalModuleResolver =
MessageTransportModuleResolver(transport, evaluatorId) ExternalModuleResolver.of(transport, evaluatorId)
override fun getResourceResolver(evaluatorId: Long): MessageTransportResourceResolver = override fun getResourceResolver(evaluatorId: Long): ExternalResourceResolver =
MessageTransportResourceResolver(transport, evaluatorId) ExternalResourceResolver.of(transport, evaluatorId)
fun run() { fun run() {
try { try {
@@ -69,7 +66,11 @@ class TestExternalReaderProcess(private val transport: MessageTransport) : Exter
transport.send(request) { response -> transport.send(request) { response ->
when (response) { when (response) {
is InitializeModuleReaderResponse -> { is InitializeModuleReaderResponse -> {
complete(response.spec) val spec =
response.spec?.let {
ModuleReaderSpec(it.scheme, it.hasHierarchicalUris, it.isLocal, it.isGlobbable)
}
complete(spec)
} }
else -> completeExceptionally(ProtocolException("unexpected response")) else -> completeExceptionally(ProtocolException("unexpected response"))
} }
@@ -86,7 +87,11 @@ class TestExternalReaderProcess(private val transport: MessageTransport) : Exter
transport.send(request) { response -> transport.send(request) { response ->
when (response) { when (response) {
is InitializeResourceReaderResponse -> { is InitializeResourceReaderResponse -> {
complete(response.spec) val spec =
response.spec?.let {
ResourceReaderSpec(it.scheme, it.hasHierarchicalUris, it.isGlobbable)
}
complete(spec)
} }
else -> completeExceptionally(ProtocolException("unexpected response")) else -> completeExceptionally(ProtocolException("unexpected response"))
} }
@@ -97,9 +102,9 @@ class TestExternalReaderProcess(private val transport: MessageTransport) : Exter
companion object { companion object {
fun initializeTestHarness( fun initializeTestHarness(
moduleReaders: List<ExternalModuleReader>, externalModuleReaders: List<ExternalModuleReader>,
resourceReaders: List<ExternalResourceReader>, externalResourceReaders: List<ExternalResourceReader>,
): Pair<TestExternalReaderProcess, ExternalReaderRuntime> { ): Pair<TestExternalReaderProcess, ExternalReaderClient> {
val rxIn = PipedInputStream(10240) val rxIn = PipedInputStream(10240)
val rxOut = PipedOutputStream(rxIn) val rxOut = PipedOutputStream(rxIn)
val txIn = PipedInputStream(10240) val txIn = PipedInputStream(10240)
@@ -115,13 +120,14 @@ class TestExternalReaderProcess(private val transport: MessageTransport) : Exter
ExternalReaderMessagePackEncoder(rxOut), ExternalReaderMessagePackEncoder(rxOut),
) {} ) {}
val runtime = ExternalReaderRuntime(moduleReaders, resourceReaders, clientTransport) val client =
ExternalReaderClient(externalModuleReaders, externalResourceReaders, clientTransport)
val proc = TestExternalReaderProcess(serverTransport) val proc = TestExternalReaderProcess(serverTransport)
Thread(runtime::run).start() Thread(client::run).start()
Thread(proc::run).start() Thread(proc::run).start()
return proc to runtime return proc to client
} }
} }
} }
@@ -1,5 +1,5 @@
/* /*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -17,9 +17,9 @@ package org.pkl.server
import java.net.URI import java.net.URI
import java.util.Optional import java.util.Optional
import org.pkl.core.externalreader.ExternalModuleResolver
import org.pkl.core.externalreader.ModuleReaderSpec
import org.pkl.core.messaging.* import org.pkl.core.messaging.*
import org.pkl.core.messaging.MessageTransportModuleResolver
import org.pkl.core.messaging.Messages.*
import org.pkl.core.module.* import org.pkl.core.module.*
internal class ClientModuleKeyFactory( internal class ClientModuleKeyFactory(
@@ -29,8 +29,7 @@ internal class ClientModuleKeyFactory(
) : ModuleKeyFactory { ) : ModuleKeyFactory {
private val schemes = readerSpecs.map { it.scheme } private val schemes = readerSpecs.map { it.scheme }
private val resolver: MessageTransportModuleResolver = private val resolver: ExternalModuleResolver = ExternalModuleResolver.of(transport, evaluatorId)
MessageTransportModuleResolver(transport, evaluatorId)
override fun create(uri: URI): Optional<ModuleKey> = override fun create(uri: URI): Optional<ModuleKey> =
when (uri.scheme) { when (uri.scheme) {
@@ -21,13 +21,16 @@ import java.net.URI
import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.ExecutorService import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors import java.util.concurrent.Executors
import java.util.regex.Pattern
import kotlin.random.Random import kotlin.random.Random
import org.pkl.core.* import org.pkl.core.*
import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader import org.pkl.core.evaluatorSettings.PklEvaluatorSettings
import org.pkl.core.externalreader.ExternalReaderProcess import org.pkl.core.externalreader.ExternalReaderProcess
import org.pkl.core.externalreader.ExternalResourceResolver
import org.pkl.core.externalreader.ModuleReaderSpec
import org.pkl.core.externalreader.ResourceReaderSpec
import org.pkl.core.http.HttpClient import org.pkl.core.http.HttpClient
import org.pkl.core.messaging.MessageTransport import org.pkl.core.messaging.MessageTransport
import org.pkl.core.messaging.MessageTransportResourceResolver
import org.pkl.core.messaging.MessageTransports import org.pkl.core.messaging.MessageTransports
import org.pkl.core.messaging.ProtocolException import org.pkl.core.messaging.ProtocolException
import org.pkl.core.module.ModuleKeyFactories import org.pkl.core.module.ModuleKeyFactories
@@ -182,8 +185,8 @@ class Server(private val transport: MessageTransport) : AutoCloseable {
private fun createEvaluator(message: CreateEvaluatorRequest, evaluatorId: Long): BinaryEvaluator { private fun createEvaluator(message: CreateEvaluatorRequest, evaluatorId: Long): BinaryEvaluator {
val modulePaths = message.modulePaths ?: emptyList() val modulePaths = message.modulePaths ?: emptyList()
val resolver = ModulePathResolver(modulePaths) val resolver = ModulePathResolver(modulePaths)
val allowedModules = message.allowedModules ?: emptyList() val allowedModules = message.allowedModules?.map { Pattern.compile(it) } ?: emptyList()
val allowedResources = message.allowedResources ?: emptyList() val allowedResources = message.allowedResources?.map { Pattern.compile(it) } ?: emptyList()
val rootDir = message.rootDir val rootDir = message.rootDir
val env = message.env ?: emptyMap() val env = message.env ?: emptyMap()
val properties = message.properties ?: emptyMap() val properties = message.properties ?: emptyMap()
@@ -247,8 +250,12 @@ class Server(private val transport: MessageTransport) : AutoCloseable {
for (readerSpec in message.clientResourceReaders ?: emptyList()) { for (readerSpec in message.clientResourceReaders ?: emptyList()) {
add( add(
ResourceReaders.externalResolver( ResourceReaders.externalResolver(
readerSpec, ResourceReaderSpec(
MessageTransportResourceResolver(transport, evaluatorId), readerSpec.scheme,
readerSpec.hasHierarchicalUris,
readerSpec.isGlobbable,
),
ExternalResourceResolver.of(transport, evaluatorId),
) )
) )
} }
@@ -261,7 +268,11 @@ class Server(private val transport: MessageTransport) : AutoCloseable {
): List<ModuleKeyFactory> = buildList { ): List<ModuleKeyFactory> = buildList {
// add client-side module key factory first to ensure it wins over builtin ones // add client-side module key factory first to ensure it wins over builtin ones
if (message.clientModuleReaders?.isNotEmpty() == true) { if (message.clientModuleReaders?.isNotEmpty() == true) {
add(ClientModuleKeyFactory(message.clientModuleReaders, transport, evaluatorId)) val readerSpecs =
message.clientModuleReaders.map {
ModuleReaderSpec(it.scheme, it.hasHierarchicalUris, it.isLocal, it.isGlobbable)
}
add(ClientModuleKeyFactory(readerSpecs, transport, evaluatorId))
} }
for ((scheme, spec) in message.externalModuleReaders ?: emptyMap()) { for ((scheme, spec) in message.externalModuleReaders ?: emptyMap()) {
add( add(
@@ -285,5 +296,7 @@ class Server(private val transport: MessageTransport) : AutoCloseable {
private fun getExternalProcess(evaluatorId: Long, spec: ExternalReader): ExternalReaderProcess = private fun getExternalProcess(evaluatorId: Long, spec: ExternalReader): ExternalReaderProcess =
externalReaderProcesses externalReaderProcesses
.computeIfAbsent(evaluatorId) { ConcurrentHashMap() } .computeIfAbsent(evaluatorId) { ConcurrentHashMap() }
.computeIfAbsent(spec) { ExternalReaderProcess.of(it) } .computeIfAbsent(spec) {
ExternalReaderProcess.of(PklEvaluatorSettings.ExternalReader(it.executable, it.arguments))
}
} }
@@ -19,12 +19,9 @@ import java.io.InputStream
import java.net.URI import java.net.URI
import java.nio.file.Path import java.nio.file.Path
import java.time.Duration import java.time.Duration
import java.util.regex.Pattern
import org.msgpack.core.MessagePack import org.msgpack.core.MessagePack
import org.msgpack.core.MessageUnpacker import org.msgpack.core.MessageUnpacker
import org.msgpack.value.Value import org.msgpack.value.Value
import org.pkl.core.evaluatorSettings.PklEvaluatorSettings
import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader
import org.pkl.core.messaging.BaseMessagePackDecoder import org.pkl.core.messaging.BaseMessagePackDecoder
import org.pkl.core.messaging.Message import org.pkl.core.messaging.Message
import org.pkl.core.packages.Checksums import org.pkl.core.packages.Checksums
@@ -38,8 +35,8 @@ class ServerMessagePackDecoder(unpacker: MessageUnpacker) : BaseMessagePackDecod
Message.Type.CREATE_EVALUATOR_REQUEST -> Message.Type.CREATE_EVALUATOR_REQUEST ->
CreateEvaluatorRequest( CreateEvaluatorRequest(
get(map, "requestId").asIntegerValue().asLong(), get(map, "requestId").asIntegerValue().asLong(),
unpackStringListOrNull(map, "allowedModules", Pattern::compile), unpackStringListOrNull(map, "allowedModules"),
unpackStringListOrNull(map, "allowedResources", Pattern::compile), unpackStringListOrNull(map, "allowedResources"),
unpackListOrNull(map, "clientModuleReaders") { unpackModuleReaderSpec(it)!! }, unpackListOrNull(map, "clientModuleReaders") { unpackModuleReaderSpec(it)!! },
unpackListOrNull(map, "clientResourceReaders") { unpackResourceReaderSpec(it)!! }, unpackListOrNull(map, "clientResourceReaders") { unpackResourceReaderSpec(it)!! },
unpackStringListOrNull(map, "modulePaths", Path::of), unpackStringListOrNull(map, "modulePaths", Path::of),
@@ -101,11 +98,11 @@ class ServerMessagePackDecoder(unpacker: MessageUnpacker) : BaseMessagePackDecod
return Http(caCertificates, proxy) return Http(caCertificates, proxy)
} }
private fun Map<Value, Value>.unpackProxy(): PklEvaluatorSettings.Proxy? { private fun Map<Value, Value>.unpackProxy(): Proxy? {
val proxyMap = getNullable(this, "proxy")?.asMapValue()?.map() ?: return null val proxyMap = getNullable(this, "proxy")?.asMapValue()?.map() ?: return null
val address = unpackString(proxyMap, "address") val address = unpackString(proxyMap, "address")
val noProxy = unpackStringListOrNull(proxyMap, "noProxy") val noProxy = unpackStringListOrNull(proxyMap, "noProxy")
return PklEvaluatorSettings.Proxy.create(address, noProxy) return Proxy(URI(address), noProxy)
} }
private fun Map<Value, Value>.unpackDependencies(name: String): Map<String, Dependency> { private fun Map<Value, Value>.unpackDependencies(name: String): Map<String, Dependency> {
@@ -20,7 +20,6 @@ import java.nio.file.Path
import kotlin.io.path.pathString import kotlin.io.path.pathString
import org.msgpack.core.MessagePack import org.msgpack.core.MessagePack
import org.msgpack.core.MessagePacker import org.msgpack.core.MessagePacker
import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader
import org.pkl.core.messaging.BaseMessagePackEncoder import org.pkl.core.messaging.BaseMessagePackEncoder
import org.pkl.core.messaging.Message import org.pkl.core.messaging.Message
import org.pkl.core.packages.Checksums import org.pkl.core.packages.Checksums
@@ -105,8 +104,8 @@ class ServerMessagePackEncoder(packer: MessagePacker) : BaseMessagePackEncoder(p
msg.externalResourceReaders, msg.externalResourceReaders,
) )
packKeyValue("requestId", msg.requestId()) packKeyValue("requestId", msg.requestId())
packKeyValue("allowedModules", msg.allowedModules?.map { it.toString() }) packKeyValue("allowedModules", msg.allowedModules)
packKeyValue("allowedResources", msg.allowedResources?.map { it.toString() }) packKeyValue("allowedResources", msg.allowedResources)
if (msg.clientModuleReaders != null) { if (msg.clientModuleReaders != null) {
packer.packString("clientModuleReaders") packer.packString("clientModuleReaders")
packer.packArrayHeader(msg.clientModuleReaders.size) packer.packArrayHeader(msg.clientModuleReaders.size)
@@ -19,23 +19,16 @@ import java.net.URI
import java.nio.file.Path import java.nio.file.Path
import java.time.Duration import java.time.Duration
import java.util.* import java.util.*
import java.util.regex.Pattern
import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader
import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.Proxy
import org.pkl.core.messaging.Message import org.pkl.core.messaging.Message
import org.pkl.core.messaging.Messages.* import org.pkl.core.messaging.Messages
import org.pkl.core.packages.Checksums import org.pkl.core.packages.Checksums
private fun <T> T?.equalsNullable(other: Any?): Boolean {
return Objects.equals(this, other)
}
data class CreateEvaluatorRequest( data class CreateEvaluatorRequest(
private val requestId: Long, private val requestId: Long,
val allowedModules: List<Pattern>?, val allowedModules: List<String>?,
val allowedResources: List<Pattern>?, val allowedResources: List<String>?,
val clientModuleReaders: List<ModuleReaderSpec>?, val clientModuleReaders: List<Messages.ModuleReaderSpec>?,
val clientResourceReaders: List<ResourceReaderSpec>?, val clientResourceReaders: List<Messages.ResourceReaderSpec>?,
val modulePaths: List<Path>?, val modulePaths: List<Path>?,
val env: Map<String, String>?, val env: Map<String, String>?,
val properties: Map<String, String>?, val properties: Map<String, String>?,
@@ -52,58 +45,12 @@ data class CreateEvaluatorRequest(
override fun type(): Message.Type = Message.Type.CREATE_EVALUATOR_REQUEST override fun type(): Message.Type = Message.Type.CREATE_EVALUATOR_REQUEST
override fun requestId(): Long = requestId override fun requestId(): Long = requestId
// need to implement this manually because [Pattern.equals] returns false for two patterns
// that have the same underlying pattern string.
override fun equals(other: Any?): Boolean {
if (other == null) return false
if (other !is CreateEvaluatorRequest) return false
return requestId == other.requestId &&
Objects.equals(
allowedModules?.map { it.pattern() },
other.allowedModules?.map { it.pattern() },
) &&
Objects.equals(
allowedResources?.map { it.pattern() },
other.allowedResources?.map { it.pattern() },
) &&
clientModuleReaders.equalsNullable(other.clientModuleReaders) &&
clientResourceReaders.equalsNullable(other.clientResourceReaders) &&
modulePaths.equalsNullable(other.modulePaths) &&
env.equalsNullable(other.env) &&
properties.equalsNullable(other.properties) &&
timeout.equalsNullable(other.timeout) &&
rootDir.equalsNullable(other.rootDir) &&
cacheDir.equalsNullable(other.cacheDir) &&
outputFormat.equalsNullable(other.outputFormat) &&
project.equalsNullable(other.project) &&
http.equalsNullable(other.http) &&
externalModuleReaders.equalsNullable(other.externalModuleReaders) &&
externalResourceReaders.equalsNullable(other.externalResourceReaders)
}
@Suppress("DuplicatedCode") // false duplicate within method
override fun hashCode(): Int {
var result = requestId.hashCode()
result = 31 * result + allowedModules?.map { it.pattern() }.hashCode()
result = 31 * result + allowedResources?.map { it.pattern() }.hashCode()
result = 31 * result + clientModuleReaders.hashCode()
result = 31 * result + clientResourceReaders.hashCode()
result = 31 * result + modulePaths.hashCode()
result = 31 * result + env.hashCode()
result = 31 * result + properties.hashCode()
result = 31 * result + timeout.hashCode()
result = 31 * result + rootDir.hashCode()
result = 31 * result + cacheDir.hashCode()
result = 31 * result + outputFormat.hashCode()
result = 31 * result + project.hashCode()
result = 31 * result + http.hashCode()
result = 31 * result + externalModuleReaders.hashCode()
result = 31 * result + externalResourceReaders.hashCode()
return result
}
} }
data class ExternalReader(val executable: String, val arguments: List<String>?)
data class Proxy(val address: URI?, val noProxy: List<String>?)
data class Http( data class Http(
/** PEM-format CA certificates as raw bytes. */ /** PEM-format CA certificates as raw bytes. */
val caCertificates: ByteArray?, val caCertificates: ByteArray?,
@@ -19,7 +19,6 @@ import java.net.URI
import java.nio.file.Path import java.nio.file.Path
import java.util.concurrent.ExecutorService import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors import java.util.concurrent.Executors
import java.util.regex.Pattern
import kotlin.io.path.createDirectories import kotlin.io.path.createDirectories
import kotlin.io.path.outputStream import kotlin.io.path.outputStream
import kotlin.io.path.writeText import kotlin.io.path.writeText
@@ -947,8 +946,8 @@ abstract class AbstractServerTest {
val message = val message =
CreateEvaluatorRequest( CreateEvaluatorRequest(
123, 123,
listOf(Pattern.compile(".*")), listOf(".*"),
listOf(Pattern.compile(".*")), listOf(".*"),
moduleReaders, moduleReaders,
resourceReaders, resourceReaders,
modulePaths, modulePaths,
@@ -20,16 +20,13 @@ import java.io.PipedOutputStream
import java.net.URI import java.net.URI
import java.nio.file.Path import java.nio.file.Path
import java.time.Duration import java.time.Duration
import java.util.regex.Pattern
import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test import org.junit.jupiter.api.Test
import org.msgpack.core.MessagePack import org.msgpack.core.MessagePack
import org.pkl.core.evaluatorSettings.PklEvaluatorSettings
import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader
import org.pkl.core.messaging.Message import org.pkl.core.messaging.Message
import org.pkl.core.messaging.MessageDecoder import org.pkl.core.messaging.MessageDecoder
import org.pkl.core.messaging.MessageEncoder import org.pkl.core.messaging.MessageEncoder
import org.pkl.core.messaging.Messages.* import org.pkl.core.messaging.Messages
import org.pkl.core.packages.Checksums import org.pkl.core.packages.Checksums
class ServerMessagePackCodecTest { class ServerMessagePackCodecTest {
@@ -52,18 +49,16 @@ class ServerMessagePackCodecTest {
@Test @Test
fun `round-trip CreateEvaluatorRequest`() { fun `round-trip CreateEvaluatorRequest`() {
val resourceReader1 = ResourceReaderSpec("resourceReader1", true, true) val resourceReader1 = Messages.ResourceReaderSpec("resourceReader1", true, true)
val resourceReader2 = ResourceReaderSpec("resourceReader2", true, false) val resourceReader2 = Messages.ResourceReaderSpec("resourceReader2", true, false)
val moduleReader1 = ModuleReaderSpec("moduleReader1", true, true, true) val moduleReader1 = Messages.ModuleReaderSpec("moduleReader1", true, true, true)
val moduleReader2 = ModuleReaderSpec("moduleReader2", true, false, false) val moduleReader2 = Messages.ModuleReaderSpec("moduleReader2", true, false, false)
val externalReader = ExternalReader("external-cmd", listOf("arg1", "arg2")) val externalReader = ExternalReader("external-cmd", listOf("arg1", "arg2"))
roundtrip( roundtrip(
CreateEvaluatorRequest( CreateEvaluatorRequest(
requestId = 123, requestId = 123,
allowedModules = listOf("pkl", "file", "https").map(Pattern::compile), allowedModules = listOf("pkl", "file", "https"),
allowedResources = allowedResources = listOf("pkl", "file", "https", "resourceReader1", "resourceReader2"),
listOf("pkl", "file", "https", "resourceReader1", "resourceReader2")
.map(Pattern::compile),
clientResourceReaders = listOf(resourceReader1, resourceReader2), clientResourceReaders = listOf(resourceReader1, resourceReader2),
clientModuleReaders = listOf(moduleReader1, moduleReader2), clientModuleReaders = listOf(moduleReader1, moduleReader2),
modulePaths = listOf(Path.of("some/path.zip"), Path.of("other/path.zip")), modulePaths = listOf(Path.of("some/path.zip"), Path.of("other/path.zip")),
@@ -99,7 +94,7 @@ class ServerMessagePackCodecTest {
), ),
http = http =
Http( Http(
proxy = PklEvaluatorSettings.Proxy(URI("http://foo.com:1234"), listOf("bar", "baz")), proxy = Proxy(URI("http://foo.com:1234"), listOf("bar", "baz")),
caCertificates = byteArrayOf(1, 2, 3, 4), caCertificates = byteArrayOf(1, 2, 3, 4),
), ),
externalModuleReaders = mapOf("external" to externalReader, "external2" to externalReader), externalModuleReaders = mapOf("external" to externalReader, "external2" to externalReader),