Implement SPICE-0009 External Readers (#660)

This adds a new feature, which allows Pkl to read resources and modules from external processes.

Follows the design laid out in SPICE-0009.

Also, this moves most of the messaging API into pkl-core

pkl-core/src/test/kotlin/org/pkl/core/EvaluatorBuilderTest.kt

@@ -71,8 +71,10 @@ class EvaluatorBuilderTest {
   fun `sets evaluator settings from project`() {
     val projectPath = Path.of(javaClass.getResource("project/project1/PklProject")!!.toURI())
     val project = Project.loadFromPath(projectPath, SecurityManagers.defaultManager, null)
-    val projectDir = Path.of(javaClass.getResource("project/project1/PklProject")!!.toURI()).parent
-    val builder = EvaluatorBuilder.unconfigured().applyFromProject(project)
+    val projectDir = projectPath.parent
+    val builder = EvaluatorBuilder.unconfigured()
+    val moduleKeyFactoryCount = builder.moduleKeyFactories.size
+    builder.applyFromProject(project)
     assertThat(builder.allowedResources.map { it.pattern() }).isEqualTo(listOf("foo:", "bar:"))
     assertThat(builder.allowedModules.map { it.pattern() }).isEqualTo(listOf("baz:", "biz:"))
     assertThat(builder.externalProperties).isEqualTo(mapOf("one" to "1"))
@@ -80,5 +82,9 @@ class EvaluatorBuilderTest {
     assertThat(builder.moduleCacheDir).isEqualTo(projectDir.resolve("my-cache-dir/"))
     assertThat(builder.rootDir).isEqualTo(projectDir.resolve("my-root-dir/"))
     assertThat(builder.timeout).isEqualTo(Duration.ofMinutes(5L))
+    assertThat(builder.moduleKeyFactories.size - moduleKeyFactoryCount)
+      .isEqualTo(3) // two external readers, one module path
+    assertThat(builder.resourceReaders.find { it.uriScheme == "scheme3" }).isNotNull
+    assertThat(builder.resourceReaders.find { it.uriScheme == "scheme4" }).isNotNull
   }
 }

pkl-core/src/test/kotlin/org/pkl/core/externalreader/ExternalModuleReader.kt

@@ -0,0 +1,29 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.externalreader
+
+import java.net.URI
+import org.pkl.core.messaging.Messages.ModuleReaderSpec
+
+/** An external module reader, to be used with [ExternalReaderRuntime]. */
+interface ExternalModuleReader : ExternalReaderBase {
+  val isLocal: Boolean
+
+  fun read(uri: URI): String
+
+  val spec: ModuleReaderSpec
+    get() = ModuleReaderSpec(scheme, hasHierarchicalUris, isLocal, isGlobbable)
+}

pkl-core/src/test/kotlin/org/pkl/core/externalreader/ExternalProcessProcessReaderMessagePackCodecTest.kt

@@ -0,0 +1,73 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.externalreader
+
+import java.io.PipedInputStream
+import java.io.PipedOutputStream
+import org.assertj.core.api.Assertions.assertThat
+import org.junit.jupiter.api.Test
+import org.msgpack.core.MessagePack
+import org.pkl.core.externalreader.ExternalReaderMessages.*
+import org.pkl.core.messaging.*
+
+class ExternalProcessProcessReaderMessagePackCodecTest {
+  private val encoder: MessageEncoder
+  private val decoder: MessageDecoder
+
+  init {
+    val inputStream = PipedInputStream()
+    val outputStream = PipedOutputStream(inputStream)
+    encoder = ExternalReaderMessagePackEncoder(MessagePack.newDefaultPacker(outputStream))
+    decoder = ExternalReaderMessagePackDecoder(MessagePack.newDefaultUnpacker(inputStream))
+  }
+
+  private fun roundtrip(message: Message) {
+    encoder.encode(message)
+    val decoded = decoder.decode()
+    assertThat(decoded).isEqualTo(message)
+  }
+
+  @Test
+  fun `round-trip InitializeModuleReaderRequest`() {
+    roundtrip(InitializeModuleReaderRequest(123, "my-scheme"))
+  }
+
+  @Test
+  fun `round-trip InitializeResourceReaderRequest`() {
+    roundtrip(InitializeResourceReaderRequest(123, "my-scheme"))
+  }
+
+  @Test
+  fun `round-trip InitializeModuleReaderResponse`() {
+    roundtrip(InitializeModuleReaderResponse(123, null))
+    roundtrip(
+      InitializeModuleReaderResponse(123, Messages.ModuleReaderSpec("my-scheme", true, true, true))
+    )
+  }
+
+  @Test
+  fun `round-trip InitializeResourceReaderResponse`() {
+    roundtrip(InitializeResourceReaderResponse(123, null))
+    roundtrip(
+      InitializeResourceReaderResponse(123, Messages.ResourceReaderSpec("my-scheme", true, true))
+    )
+  }
+
+  @Test
+  fun `round-trip CloseExternalProcess`() {
+    roundtrip(CloseExternalProcess())
+  }
+}

pkl-core/src/test/kotlin/org/pkl/core/externalreader/ExternalReaderBase.kt

@@ -0,0 +1,30 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.externalreader
+
+import java.net.URI
+import org.pkl.core.module.PathElement
+
+/** Base interface for external module and resource readers. */
+interface ExternalReaderBase {
+  val scheme: String
+
+  val hasHierarchicalUris: Boolean
+
+  val isGlobbable: Boolean
+
+  fun listElements(uri: URI): List<PathElement>
+}

pkl-core/src/test/kotlin/org/pkl/core/externalreader/ExternalReaderRuntime.kt

@@ -0,0 +1,199 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.externalreader
+
+import java.io.IOException
+import org.pkl.core.externalreader.ExternalReaderMessages.*
+import org.pkl.core.messaging.Message
+import org.pkl.core.messaging.MessageTransport
+import org.pkl.core.messaging.Messages.*
+import org.pkl.core.messaging.ProtocolException
+import org.pkl.core.util.Nullable
+
+/** An implementation of the client side of the external reader flow */
+class ExternalReaderRuntime(
+  private val moduleReaders: List<ExternalModuleReader>,
+  private val resourceReaders: List<ExternalResourceReader>,
+  private val transport: MessageTransport
+) {
+  /** Close the runtime and its transport. */
+  fun close() {
+    transport.close()
+  }
+
+  private fun findModuleReader(scheme: String): @Nullable ExternalModuleReader? {
+    for (moduleReader in moduleReaders) {
+      if (moduleReader.scheme.equals(scheme, ignoreCase = true)) {
+        return moduleReader
+      }
+    }
+    return null
+  }
+
+  private fun findResourceReader(scheme: String): @Nullable ExternalResourceReader? {
+    for (resourceReader in resourceReaders) {
+      if (resourceReader.scheme.equals(scheme, ignoreCase = true)) {
+        return resourceReader
+      }
+    }
+    return null
+  }
+
+  /**
+   * Start the runtime so it can respond to incoming messages on its transport.
+   *
+   * Blocks until the underlying transport is closed.
+   */
+  @Throws(ProtocolException::class, IOException::class)
+  fun run() {
+    transport.start(
+      { msg: Message.OneWay ->
+        if (msg.type() == Message.Type.CLOSE_EXTERNAL_PROCESS) {
+          close()
+        } else {
+          throw ProtocolException("Unexpected incoming one-way message: $msg")
+        }
+      },
+      { msg: Message.Request ->
+        when (msg.type()) {
+          Message.Type.INITIALIZE_MODULE_READER_REQUEST -> {
+            val req = msg as InitializeModuleReaderRequest
+            val reader = findModuleReader(req.scheme)
+            var spec: @Nullable ModuleReaderSpec? = null
+            if (reader != null) {
+              spec = reader.spec
+            }
+            transport.send(InitializeModuleReaderResponse(req.requestId, spec))
+          }
+          Message.Type.INITIALIZE_RESOURCE_READER_REQUEST -> {
+            val req = msg as InitializeResourceReaderRequest
+            val reader = findResourceReader(req.scheme)
+            var spec: @Nullable ResourceReaderSpec? = null
+            if (reader != null) {
+              spec = reader.spec
+            }
+            transport.send(InitializeResourceReaderResponse(req.requestId, spec))
+          }
+          Message.Type.LIST_MODULES_REQUEST -> {
+            val req = msg as ListModulesRequest
+            val reader = findModuleReader(req.uri.scheme)
+            if (reader == null) {
+              transport.send(
+                ListModulesResponse(
+                  req.requestId,
+                  req.evaluatorId,
+                  null,
+                  "No module reader found for scheme " + req.uri.scheme
+                )
+              )
+              return@start
+            }
+            try {
+              transport.send(
+                ListModulesResponse(
+                  req.requestId,
+                  req.evaluatorId,
+                  reader.listElements(req.uri),
+                  null
+                )
+              )
+            } catch (e: Exception) {
+              transport.send(
+                ListModulesResponse(req.requestId, req.evaluatorId, null, e.toString())
+              )
+            }
+          }
+          Message.Type.LIST_RESOURCES_REQUEST -> {
+            val req = msg as ListResourcesRequest
+            val reader = findModuleReader(req.uri.scheme)
+            if (reader == null) {
+              transport.send(
+                ListResourcesResponse(
+                  req.requestId,
+                  req.evaluatorId,
+                  null,
+                  "No resource reader found for scheme " + req.uri.scheme
+                )
+              )
+              return@start
+            }
+            try {
+              transport.send(
+                ListResourcesResponse(
+                  req.requestId,
+                  req.evaluatorId,
+                  reader.listElements(req.uri),
+                  null
+                )
+              )
+            } catch (e: Exception) {
+              transport.send(
+                ListResourcesResponse(req.requestId, req.evaluatorId, null, e.toString())
+              )
+            }
+          }
+          Message.Type.READ_MODULE_REQUEST -> {
+            val req = msg as ReadModuleRequest
+            val reader = findModuleReader(req.uri.scheme)
+            if (reader == null) {
+              transport.send(
+                ReadModuleResponse(
+                  req.requestId,
+                  req.evaluatorId,
+                  null,
+                  "No module reader found for scheme " + req.uri.scheme
+                )
+              )
+              return@start
+            }
+            try {
+              transport.send(
+                ReadModuleResponse(req.requestId, req.evaluatorId, reader.read(req.uri), null)
+              )
+            } catch (e: Exception) {
+              transport.send(ReadModuleResponse(req.requestId, req.evaluatorId, null, e.toString()))
+            }
+          }
+          Message.Type.READ_RESOURCE_REQUEST -> {
+            val req = msg as ReadResourceRequest
+            val reader = findResourceReader(req.uri.scheme)
+            if (reader == null) {
+              transport.send(
+                ReadResourceResponse(
+                  req.requestId,
+                  req.evaluatorId,
+                  byteArrayOf(),
+                  "No resource reader found for scheme " + req.uri.scheme
+                )
+              )
+              return@start
+            }
+            try {
+              transport.send(
+                ReadResourceResponse(req.requestId, req.evaluatorId, reader.read(req.uri), null)
+              )
+            } catch (e: Exception) {
+              transport.send(
+                ReadResourceResponse(req.requestId, req.evaluatorId, byteArrayOf(), e.toString())
+              )
+            }
+          }
+          else -> throw ProtocolException("Unexpected incoming request message: $msg")
+        }
+      }
+    )
+  }
+}

pkl-core/src/test/kotlin/org/pkl/core/externalreader/ExternalResourceReader.kt

@@ -0,0 +1,27 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.externalreader
+
+import java.net.URI
+import org.pkl.core.messaging.Messages.ResourceReaderSpec
+
+/** An external resource reader, to be used with [ExternalReaderRuntime]. */
+interface ExternalResourceReader : ExternalReaderBase {
+  fun read(uri: URI): ByteArray
+
+  val spec: ResourceReaderSpec
+    get() = ResourceReaderSpec(scheme, hasHierarchicalUris, isGlobbable)
+}

pkl-core/src/test/kotlin/org/pkl/core/externalreader/TestExternalModuleReader.kt

@@ -0,0 +1,38 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.externalreader
+
+import java.net.URI
+import org.pkl.core.module.PathElement
+
+class TestExternalModuleReader : ExternalModuleReader {
+  override val scheme: String = "test"
+
+  override val hasHierarchicalUris: Boolean = false
+
+  override val isLocal: Boolean = true
+
+  override val isGlobbable: Boolean = false
+
+  override fun read(uri: URI): String =
+    """
+    name = "Pigeon"
+    age = 40
+  """
+      .trimIndent()
+
+  override fun listElements(uri: URI): List<PathElement> = emptyList()
+}

pkl-core/src/test/kotlin/org/pkl/core/externalreader/TestExternalReaderProcess.kt

@@ -0,0 +1,130 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.externalreader
+
+import java.io.IOException
+import java.io.PipedInputStream
+import java.io.PipedOutputStream
+import java.util.concurrent.CompletableFuture
+import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.ExecutionException
+import java.util.concurrent.Future
+import kotlin.random.Random
+import org.pkl.core.externalreader.ExternalReaderMessages.*
+import org.pkl.core.messaging.MessageTransport
+import org.pkl.core.messaging.MessageTransports
+import org.pkl.core.messaging.Messages.*
+import org.pkl.core.messaging.ProtocolException
+
+class TestExternalReaderProcess(private val transport: MessageTransport) : ExternalReaderProcess {
+  private val initializeModuleReaderResponses: MutableMap<String, Future<ModuleReaderSpec?>> =
+    ConcurrentHashMap()
+  private val initializeResourceReaderResponses: MutableMap<String, Future<ResourceReaderSpec?>> =
+    ConcurrentHashMap()
+
+  override fun close() {
+    transport.send(CloseExternalProcess())
+    transport.close()
+  }
+
+  override fun getTransport(): MessageTransport = transport
+
+  fun run() {
+    try {
+      transport.start(
+        { throw ProtocolException("Unexpected incoming one-way message: $it") },
+        { throw ProtocolException("Unexpected incoming request message: $it") },
+      )
+    } catch (e: ProtocolException) {
+      throw RuntimeException(e)
+    } catch (e: IOException) {
+      throw RuntimeException(e)
+    }
+  }
+
+  override fun getModuleReaderSpec(scheme: String): ModuleReaderSpec? =
+    initializeModuleReaderResponses
+      .computeIfAbsent(scheme) {
+        CompletableFuture<ModuleReaderSpec?>().apply {
+          val request = InitializeModuleReaderRequest(Random.nextLong(), scheme)
+          transport.send(request) { response ->
+            when (response) {
+              is InitializeModuleReaderResponse -> {
+                complete(response.spec)
+              }
+              else -> completeExceptionally(ProtocolException("unexpected response"))
+            }
+          }
+        }
+      }
+      .getUnderlying()
+
+  override fun getResourceReaderSpec(scheme: String): ResourceReaderSpec? =
+    initializeResourceReaderResponses
+      .computeIfAbsent(scheme) {
+        CompletableFuture<ResourceReaderSpec?>().apply {
+          val request = InitializeResourceReaderRequest(Random.nextLong(), scheme)
+          transport.send(request) { response ->
+            when (response) {
+              is InitializeResourceReaderResponse -> {
+                complete(response.spec)
+              }
+              else -> completeExceptionally(ProtocolException("unexpected response"))
+            }
+          }
+        }
+      }
+      .getUnderlying()
+
+  companion object {
+    fun initializeTestHarness(
+      moduleReaders: List<ExternalModuleReader>,
+      resourceReaders: List<ExternalResourceReader>
+    ): Pair<TestExternalReaderProcess, ExternalReaderRuntime> {
+      val rxIn = PipedInputStream(10240)
+      val rxOut = PipedOutputStream(rxIn)
+      val txIn = PipedInputStream(10240)
+      val txOut = PipedOutputStream(txIn)
+      val serverTransport =
+        MessageTransports.stream(
+          ExternalReaderMessagePackDecoder(rxIn),
+          ExternalReaderMessagePackEncoder(txOut),
+          {}
+        )
+      val clientTransport =
+        MessageTransports.stream(
+          ExternalReaderMessagePackDecoder(txIn),
+          ExternalReaderMessagePackEncoder(rxOut),
+          {}
+        )
+
+      val runtime = ExternalReaderRuntime(moduleReaders, resourceReaders, clientTransport)
+      val proc = TestExternalReaderProcess(serverTransport)
+
+      Thread(runtime::run).start()
+      Thread(proc::run).start()
+
+      return proc to runtime
+    }
+  }
+}
+
+fun <T> Future<T>.getUnderlying(): T =
+  try {
+    get()
+  } catch (e: ExecutionException) {
+    throw e.cause!!
+  }

pkl-core/src/test/kotlin/org/pkl/core/externalreader/TestExternalResourceReader.kt

@@ -0,0 +1,31 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.externalreader
+
+import java.net.URI
+import org.pkl.core.module.PathElement
+
+class TestExternalResourceReader : ExternalResourceReader {
+  override val scheme: String = "test"
+
+  override val hasHierarchicalUris: Boolean = false
+
+  override val isGlobbable: Boolean = false
+
+  override fun read(uri: URI): ByteArray = "success".toByteArray(Charsets.UTF_8)
+
+  override fun listElements(uri: URI): List<PathElement> = emptyList()
+}

pkl-core/src/test/kotlin/org/pkl/core/messaging/BaseMessagePackCodecTest.kt

@@ -0,0 +1,128 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.messaging
+
+import java.io.PipedInputStream
+import java.io.PipedOutputStream
+import java.net.URI
+import org.assertj.core.api.Assertions.assertThat
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.assertThrows
+import org.msgpack.core.MessagePack
+import org.pkl.core.messaging.Messages.*
+import org.pkl.core.module.PathElement
+
+class BaseMessagePackCodecTest {
+  private val encoder: MessageEncoder
+  private val decoder: MessageDecoder
+
+  init {
+    val inputStream = PipedInputStream()
+    val outputStream = PipedOutputStream(inputStream)
+    encoder = BaseMessagePackEncoder(MessagePack.newDefaultPacker(outputStream))
+    decoder = BaseMessagePackDecoder(MessagePack.newDefaultUnpacker(inputStream))
+  }
+
+  private fun roundtrip(message: Message) {
+    encoder.encode(message)
+    val decoded = decoder.decode()
+    assertThat(decoded).isEqualTo(message)
+  }
+
+  @Test
+  fun `round-trip ReadResourceRequest`() {
+    roundtrip(ReadResourceRequest(123, 456, URI("some/resource.json")))
+  }
+
+  @Test
+  fun `round-trip ReadResourceResponse`() {
+    roundtrip(ReadResourceResponse(123, 456, byteArrayOf(1, 2, 3, 4, 5), null))
+  }
+
+  @Test
+  fun `round-trip ReadModuleRequest`() {
+    roundtrip(ReadModuleRequest(123, 456, URI("some/module.pkl")))
+  }
+
+  @Test
+  fun `round-trip ReadModuleResponse`() {
+    roundtrip(ReadModuleResponse(123, 456, "x = 42", null))
+  }
+
+  @Test
+  fun `round-trip ListModulesRequest`() {
+    roundtrip(ListModulesRequest(135, 246, URI("foo:/bar/baz/biz")))
+  }
+
+  @Test
+  fun `round-trip ListModulesResponse`() {
+    roundtrip(
+      ListModulesResponse(
+        123,
+        234,
+        listOf(PathElement("foo", true), PathElement("bar", false)),
+        null
+      )
+    )
+    roundtrip(ListModulesResponse(123, 234, null, "Something dun went wrong"))
+  }
+
+  @Test
+  fun `round-trip ListResourcesRequest`() {
+    roundtrip(ListResourcesRequest(987, 1359, URI("bar:/bazzy")))
+  }
+
+  @Test
+  fun `round-trip ListResourcesResponse`() {
+    roundtrip(
+      ListResourcesResponse(
+        3851,
+        3019,
+        listOf(PathElement("foo", true), PathElement("bar", false)),
+        null
+      )
+    )
+    roundtrip(ListResourcesResponse(3851, 3019, null, "something went wrong"))
+  }
+
+  @Test
+  fun `decode request with missing request ID`() {
+    val bytes =
+      MessagePack.newDefaultBufferPacker()
+        .apply {
+          packArrayHeader(2)
+          packInt(Message.Type.LIST_RESOURCES_REQUEST.code)
+          packMapHeader(1)
+          packString("uri")
+          packString("file:/test")
+        }
+        .toByteArray()
+
+    val decoder = BaseMessagePackDecoder(MessagePack.newDefaultUnpacker(bytes))
+    val exception = assertThrows<DecodeException> { decoder.decode() }
+    assertThat(exception.message).contains("requestId")
+  }
+
+  @Test
+  fun `decode invalid message header`() {
+    val bytes = MessagePack.newDefaultBufferPacker().apply { packInt(2) }.toByteArray()
+
+    val decoder = BaseMessagePackDecoder(MessagePack.newDefaultUnpacker(bytes))
+    val exception = assertThrows<DecodeException> { decoder.decode() }
+    assertThat(exception).hasMessage("Malformed message header.")
+    assertThat(exception).hasRootCauseMessage("Expected Array, but got Integer (02)")
+  }
+}

pkl-core/src/test/kotlin/org/pkl/core/module/ModuleKeyFactoriesTest.kt

@@ -26,6 +26,7 @@ import org.pkl.commons.createParentDirectories
 import org.pkl.commons.toPath
 import org.pkl.commons.writeString
 import org.pkl.core.SecurityManagers
+import org.pkl.core.externalreader.*
 
 class ModuleKeyFactoriesTest {
   @Test
@@ -126,4 +127,23 @@ class ModuleKeyFactoriesTest {
     val module2 = factory.create(URI("other"))
     assertThat(module2).isNotPresent
   }
+
+  @Test
+  fun externalProcess() {
+    val extReader = TestExternalModuleReader()
+    val (proc, runtime) =
+      TestExternalReaderProcess.initializeTestHarness(listOf(extReader), emptyList())
+
+    val factory = ModuleKeyFactories.externalProcess(extReader.scheme, proc)
+
+    val module = factory.create(URI("test:foo"))
+    assertThat(module).isPresent
+    assertThat(module.get().uri.scheme).isEqualTo("test")
+
+    val module2 = factory.create(URI("other"))
+    assertThat(module2).isNotPresent
+
+    proc.close()
+    runtime.close()
+  }
 }

pkl-core/src/test/kotlin/org/pkl/core/project/ProjectTest.kt

@@ -70,6 +70,8 @@ class ProjectTest {
         listOf(path.resolve("modulepath1/"), path.resolve("modulepath2/")),
         Duration.ofMinutes(5.0),
         path,
+        null,
+        null,
         null
       )
     val expectedAnnotations =

pkl-core/src/test/kotlin/org/pkl/core/resource/ResourceReadersTest.kt

@@ -23,6 +23,8 @@ import org.assertj.core.api.Assertions.assertThat
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertThrows
 import org.junit.jupiter.api.io.TempDir
+import org.pkl.core.externalreader.TestExternalReaderProcess
+import org.pkl.core.externalreader.TestExternalResourceReader
 import org.pkl.core.module.ModulePathResolver
 
 class ResourceReadersTest {
@@ -132,4 +134,21 @@ class ResourceReadersTest {
 
     assertThat(resource).contains("success")
   }
+
+  @Test
+  fun externalProcess() {
+    val extReader = TestExternalResourceReader()
+    val (proc, runtime) =
+      TestExternalReaderProcess.initializeTestHarness(emptyList(), listOf(extReader))
+
+    val reader = ResourceReaders.externalProcess(extReader.scheme, proc)
+    val resource = reader.read(URI("test:foo"))
+
+    assertThat(resource).isPresent
+    assertThat(resource.get()).isInstanceOf(Resource::class.java)
+    assertThat((resource.get() as Resource).text).contains("success")
+
+    proc.close()
+    runtime.close()
+  }
 }

pkl-core/src/test/resources/org/pkl/core/project/project1/PklProject

@@ -22,4 +22,22 @@ evaluatorSettings {
   noCache = false
   rootDir = "my-root-dir/"
   timeout = 5.min
+  externalModuleReaders {
+    ["scheme1"] {
+      executable = "reader1"
+    }
+    ["scheme2"] {
+      executable = "reader2"
+      arguments { "with"; "args" }
+    }
+  }
+  externalResourceReaders {
+    ["scheme3"] {
+      executable = "reader3"
+    }
+    ["scheme4"] {
+      executable = "reader4"
+      arguments { "with"; "args" }
+    }
+  }
 }