Implement SPICE-0009 External Readers (#660)

This adds a new feature, which allows Pkl to read resources and modules from external processes. Follows the design laid out in SPICE-0009. Also, this moves most of the messaging API into pkl-core
2026-04-20 23:41:27 +02:00 · 2024-10-28 18:22:14 -07:00
parent 466ae6fd4c
commit 666f8c3939
110 changed files with 4368 additions and 1810 deletions
--- a/pkl-server/src/test/kotlin/org/pkl/server/AbstractServerTest.kt
+++ b/pkl-server/src/test/kotlin/org/pkl/server/AbstractServerTest.kt
@@ -25,10 +25,12 @@ import kotlin.io.path.outputStream
 import kotlin.io.path.writeText
 import org.assertj.core.api.Assertions.assertThat
 import org.junit.jupiter.api.AfterAll
+import org.junit.jupiter.api.Disabled
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.io.TempDir
 import org.msgpack.core.MessagePack
 import org.pkl.commons.test.PackageServer
+import org.pkl.core.messaging.Messages.*
 import org.pkl.core.module.PathElement

 abstract class AbstractServerTest {
@@ -55,8 +57,8 @@ abstract class AbstractServerTest {

  @Test
  fun `create and close evaluator`() {
-    val evaluatorId = client.sendCreateEvaluatorRequest(requestId = 123)
-    client.send(CloseEvaluator(evaluatorId = evaluatorId))
+    val evaluatorId = client.sendCreateEvaluatorRequest(123)
+    client.send(CloseEvaluator(evaluatorId))
  }

  @Test
@@ -66,24 +68,23 @@ abstract class AbstractServerTest {

    client.send(
      EvaluateRequest(
-        requestId = requestId,
-        evaluatorId = evaluatorId,
-        moduleUri = URI("repl:text"),
-        moduleText =
-          """
+        requestId,
+        evaluatorId,
+        URI("repl:text"),
+        """
        foo {
          bar = "bar"
        }
      """
-            .trimIndent(),
-        expr = null
+          .trimIndent(),
+        null
      )
    )

    val response = client.receive<EvaluateResponse>()
    assertThat(response.error).isNull()
    assertThat(response.result).isNotNull
-    assertThat(response.requestId).isEqualTo(requestId)
+    assertThat(response.requestId()).isEqualTo(requestId)

    val unpacker = MessagePack.newDefaultUnpacker(response.result)
    val value = unpacker.unpackValue()
@@ -96,15 +97,14 @@ abstract class AbstractServerTest {

    client.send(
      EvaluateRequest(
-        requestId = 1,
-        evaluatorId = evaluatorId,
-        moduleUri = URI("repl:text"),
-        moduleText =
-          """
+        1,
+        evaluatorId,
+        URI("repl:text"),
+        """
        foo = trace(1 + 2 + 3)
      """
-            .trimIndent(),
-        expr = null
+          .trimIndent(),
+        null
      )
    )

@@ -121,18 +121,17 @@ abstract class AbstractServerTest {

    client.send(
      EvaluateRequest(
-        requestId = 1,
-        evaluatorId = evaluatorId,
-        moduleUri = URI("repl:text"),
-        moduleText =
-          """
+        1,
+        evaluatorId,
+        URI("repl:text"),
+        """
        @Deprecated { message = "use bar instead" }
        function foo() = 5

        result = foo()
      """
-            .trimIndent(),
-        expr = null
+          .trimIndent(),
+        null
      )
    )

@@ -145,17 +144,16 @@ abstract class AbstractServerTest {

  @Test
  fun `read resource`() {
-    val reader =
-      ResourceReaderSpec(scheme = "bahumbug", hasHierarchicalUris = true, isGlobbable = false)
+    val reader = ResourceReaderSpec("bahumbug", true, false)
    val evaluatorId = client.sendCreateEvaluatorRequest(resourceReaders = listOf(reader))

    client.send(
      EvaluateRequest(
-        requestId = 1,
-        evaluatorId = evaluatorId,
-        moduleUri = URI("repl:text"),
-        moduleText = """res = read("bahumbug:/foo.pkl").text""",
-        expr = "res"
+        1,
+        evaluatorId,
+        URI("repl:text"),
+        """res = read("bahumbug:/foo.pkl").text""",
+        "res"
      )
    )

@@ -165,10 +163,10 @@ abstract class AbstractServerTest {

    client.send(
      ReadResourceResponse(
-        requestId = readResourceMsg.requestId,
-        evaluatorId = evaluatorId,
-        contents = "my bahumbug".toByteArray(),
-        error = null
+        readResourceMsg.requestId,
+        evaluatorId,
+        "my bahumbug".toByteArray(),
+        null
      )
    )

@@ -180,10 +178,12 @@ abstract class AbstractServerTest {
    assertThat(value.asStringValue().asString()).isEqualTo("my bahumbug")
  }

+  @Disabled(
+    "Unable to construct ReadResourceResponse with null contents due to Kotlin compiler bug"
+  )
  @Test
  fun `read resource -- null contents and null error`() {
-    val reader =
-      ResourceReaderSpec(scheme = "bahumbug", hasHierarchicalUris = true, isGlobbable = false)
+    val reader = ResourceReaderSpec("bahumbug", true, false)
    val evaluatorId = client.sendCreateEvaluatorRequest(resourceReaders = listOf(reader))

    client.send(
@@ -200,14 +200,11 @@ abstract class AbstractServerTest {
    assertThat(readResourceMsg.uri.toString()).isEqualTo("bahumbug:/foo.pkl")
    assertThat(readResourceMsg.evaluatorId).isEqualTo(evaluatorId)

-    client.send(
-      ReadResourceResponse(
-        requestId = readResourceMsg.requestId,
-        evaluatorId = evaluatorId,
-        contents = null,
-        error = null
-      )
-    )
+    client.send(ReadResourceResponse(readResourceMsg.requestId, evaluatorId, byteArrayOf(), null))
+    // for this test to be correct this should actually be:
+    // client.send(ReadResourceResponse(readResourceMsg.requestId, evaluatorId, null, null))
+    // this should be evaluated again once https://github.com/apple/pkl/issues/698 is addressed
+    // see conversation here https://github.com/apple/pkl/pull/660#discussion_r1819545811

    val evaluateResponse = client.receive<EvaluateResponse>()
    assertThat(evaluateResponse.error).isNull()
@@ -219,17 +216,16 @@ abstract class AbstractServerTest {

  @Test
  fun `read resource error`() {
-    val reader =
-      ResourceReaderSpec(scheme = "bahumbug", hasHierarchicalUris = true, isGlobbable = false)
+    val reader = ResourceReaderSpec("bahumbug", true, false)
    val evaluatorId = client.sendCreateEvaluatorRequest(resourceReaders = listOf(reader))

    client.send(
      EvaluateRequest(
-        requestId = 1,
-        evaluatorId = evaluatorId,
-        moduleUri = URI("repl:text"),
-        moduleText = """res = read("bahumbug:/foo.txt").text""",
-        expr = "res"
+        1,
+        evaluatorId,
+        URI("repl:text"),
+        """res = read("bahumbug:/foo.txt").text""",
+        "res"
      )
    )

@@ -237,10 +233,10 @@ abstract class AbstractServerTest {

    client.send(
      ReadResourceResponse(
-        requestId = readResourceMsg.requestId,
-        evaluatorId = evaluatorId,
-        contents = null,
-        error = "cannot read my bahumbug"
+        readResourceMsg.requestId,
+        evaluatorId,
+        byteArrayOf(),
+        "cannot read my bahumbug"
      )
    )

@@ -251,46 +247,44 @@ abstract class AbstractServerTest {

  @Test
  fun `glob resource`() {
-    val reader = ResourceReaderSpec(scheme = "bird", hasHierarchicalUris = true, isGlobbable = true)
+    val reader = ResourceReaderSpec("bird", true, true)
    val evaluatorId = client.sendCreateEvaluatorRequest(resourceReaders = listOf(reader))
    client.send(
      EvaluateRequest(
-        requestId = 1,
-        evaluatorId = evaluatorId,
-        moduleUri = URI("repl:text"),
-        moduleText =
-          """
+        1,
+        evaluatorId,
+        URI("repl:text"),
+        """
        res = read*("bird:/**.txt").keys
      """
-            .trimIndent(),
-        expr = "res"
+          .trimIndent(),
+        "res"
      )
    )
    val listResourcesRequest = client.receive<ListResourcesRequest>()
    assertThat(listResourcesRequest.uri.toString()).isEqualTo("bird:/")
    client.send(
      ListResourcesResponse(
-        requestId = listResourcesRequest.requestId,
-        evaluatorId = listResourcesRequest.evaluatorId,
-        pathElements = listOf(PathElement("foo.txt", false), PathElement("subdir", true)),
-        error = null
+        listResourcesRequest.requestId,
+        listResourcesRequest.evaluatorId,
+        listOf(PathElement("foo.txt", false), PathElement("subdir", true)),
+        null
      )
    )
    val listResourcesRequest2 = client.receive<ListResourcesRequest>()
    assertThat(listResourcesRequest2.uri.toString()).isEqualTo("bird:/subdir/")
    client.send(
      ListResourcesResponse(
-        requestId = listResourcesRequest2.requestId,
-        evaluatorId = listResourcesRequest2.evaluatorId,
-        pathElements =
-          listOf(
-            PathElement("bar.txt", false),
-          ),
-        error = null
+        listResourcesRequest2.requestId,
+        listResourcesRequest2.evaluatorId,
+        listOf(
+          PathElement("bar.txt", false),
+        ),
+        null
      )
    )
    val evaluateResponse = client.receive<EvaluateResponse>()
-    assertThat(evaluateResponse.result!!.debugYaml)
+    assertThat(evaluateResponse.result?.debugYaml)
      .isEqualTo(
        """
      - 6
@@ -304,32 +298,31 @@ abstract class AbstractServerTest {

  @Test
  fun `glob resources -- null pathElements and null error`() {
-    val reader = ResourceReaderSpec(scheme = "bird", hasHierarchicalUris = true, isGlobbable = true)
+    val reader = ResourceReaderSpec("bird", true, true)
    val evaluatorId = client.sendCreateEvaluatorRequest(resourceReaders = listOf(reader))
    client.send(
      EvaluateRequest(
-        requestId = 1,
-        evaluatorId = evaluatorId,
-        moduleUri = URI("repl:text"),
-        moduleText =
-          """
+        1,
+        evaluatorId,
+        URI("repl:text"),
+        """
        res = read*("bird:/**.txt").keys
      """
-            .trimIndent(),
-        expr = "res"
+          .trimIndent(),
+        "res"
      )
    )
    val listResourcesRequest = client.receive<ListResourcesRequest>()
    client.send(
      ListResourcesResponse(
-        requestId = listResourcesRequest.requestId,
-        evaluatorId = listResourcesRequest.evaluatorId,
-        pathElements = null,
-        error = null
+        listResourcesRequest.requestId,
+        listResourcesRequest.evaluatorId,
+        null,
+        null
      )
    )
    val evaluateResponse = client.receive<EvaluateResponse>()
-    assertThat(evaluateResponse.result!!.debugYaml)
+    assertThat(evaluateResponse.result?.debugYaml)
      .isEqualTo(
        """
        - 6
@@ -341,29 +334,28 @@ abstract class AbstractServerTest {

  @Test
  fun `glob resource error`() {
-    val reader = ResourceReaderSpec(scheme = "bird", hasHierarchicalUris = true, isGlobbable = true)
+    val reader = ResourceReaderSpec("bird", true, true)
    val evaluatorId = client.sendCreateEvaluatorRequest(resourceReaders = listOf(reader))
    client.send(
      EvaluateRequest(
-        requestId = 1,
-        evaluatorId = evaluatorId,
-        moduleUri = URI("repl:text"),
-        moduleText =
-          """
+        1,
+        evaluatorId,
+        URI("repl:text"),
+        """
        res = read*("bird:/**.txt").keys
      """
-            .trimIndent(),
-        expr = "res"
+          .trimIndent(),
+        "res"
      )
    )
    val listResourcesRequest = client.receive<ListResourcesRequest>()
    assertThat(listResourcesRequest.uri.toString()).isEqualTo("bird:/")
    client.send(
      ListResourcesResponse(
-        requestId = listResourcesRequest.requestId,
-        evaluatorId = listResourcesRequest.evaluatorId,
-        pathElements = null,
-        error = "didnt work"
+        listResourcesRequest.requestId,
+        listResourcesRequest.evaluatorId,
+        null,
+        "didnt work"
      )
    )
    val evaluateResponse = client.receive<EvaluateResponse>()
@@ -389,22 +381,16 @@ abstract class AbstractServerTest {

  @Test
  fun `read module`() {
-    val reader =
-      ModuleReaderSpec(
-        scheme = "bird",
-        hasHierarchicalUris = true,
-        isLocal = true,
-        isGlobbable = false
-      )
+    val reader = ModuleReaderSpec("bird", true, true, false)
    val evaluatorId = client.sendCreateEvaluatorRequest(moduleReaders = listOf(reader))

    client.send(
      EvaluateRequest(
-        requestId = 1,
-        evaluatorId = evaluatorId,
-        moduleUri = URI("repl:text"),
-        moduleText = """res = import("bird:/pigeon.pkl").value""",
-        expr = "res"
+        1,
+        evaluatorId,
+        URI("repl:text"),
+        """res = import("bird:/pigeon.pkl").value""",
+        "res"
      )
    )

@@ -412,14 +398,7 @@ abstract class AbstractServerTest {
    assertThat(readModuleMsg.uri.toString()).isEqualTo("bird:/pigeon.pkl")
    assertThat(readModuleMsg.evaluatorId).isEqualTo(evaluatorId)

-    client.send(
-      ReadModuleResponse(
-        requestId = readModuleMsg.requestId,
-        evaluatorId = evaluatorId,
-        contents = "value = 5",
-        error = null
-      )
-    )
+    client.send(ReadModuleResponse(readModuleMsg.requestId, evaluatorId, "value = 5", null))

    val evaluateResponse = client.receive<EvaluateResponse>()
    assertThat(evaluateResponse.error).isNull()
@@ -430,13 +409,7 @@ abstract class AbstractServerTest {

  @Test
  fun `read module -- null contents and null error`() {
-    val reader =
-      ModuleReaderSpec(
-        scheme = "bird",
-        hasHierarchicalUris = true,
-        isLocal = true,
-        isGlobbable = false
-      )
+    val reader = ModuleReaderSpec("bird", true, true, false)
    val evaluatorId = client.sendCreateEvaluatorRequest(moduleReaders = listOf(reader))

    client.send(
@@ -453,14 +426,7 @@ abstract class AbstractServerTest {
    assertThat(readModuleMsg.uri.toString()).isEqualTo("bird:/pigeon.pkl")
    assertThat(readModuleMsg.evaluatorId).isEqualTo(evaluatorId)

-    client.send(
-      ReadModuleResponse(
-        requestId = readModuleMsg.requestId,
-        evaluatorId = evaluatorId,
-        contents = null,
-        error = null
-      )
-    )
+    client.send(ReadModuleResponse(readModuleMsg.requestId, evaluatorId, null, null))

    val evaluateResponse = client.receive<EvaluateResponse>()
    assertThat(evaluateResponse.error).isNull()
@@ -473,22 +439,16 @@ abstract class AbstractServerTest {

  @Test
  fun `read module error`() {
-    val reader =
-      ModuleReaderSpec(
-        scheme = "bird",
-        hasHierarchicalUris = true,
-        isLocal = true,
-        isGlobbable = false
-      )
+    val reader = ModuleReaderSpec("bird", true, true, false)
    val evaluatorId = client.sendCreateEvaluatorRequest(moduleReaders = listOf(reader))

    client.send(
      EvaluateRequest(
-        requestId = 1,
-        evaluatorId = evaluatorId,
-        moduleUri = URI("repl:text"),
-        moduleText = """res = import("bird:/pigeon.pkl").value""",
-        expr = "res"
+        1,
+        evaluatorId,
+        URI("repl:text"),
+        """res = import("bird:/pigeon.pkl").value""",
+        "res"
      )
    )

@@ -497,12 +457,7 @@ abstract class AbstractServerTest {
    assertThat(readModuleMsg.evaluatorId).isEqualTo(evaluatorId)

    client.send(
-      ReadModuleResponse(
-        requestId = readModuleMsg.requestId,
-        evaluatorId = evaluatorId,
-        contents = null,
-        error = "Don't know where Pigeon is"
-      )
+      ReadModuleResponse(readModuleMsg.requestId, evaluatorId, null, "Don't know where Pigeon is")
    )

    val evaluateResponse = client.receive<EvaluateResponse>()
@@ -511,22 +466,16 @@ abstract class AbstractServerTest {

  @Test
  fun `glob module`() {
-    val reader =
-      ModuleReaderSpec(
-        scheme = "bird",
-        hasHierarchicalUris = true,
-        isLocal = true,
-        isGlobbable = true
-      )
+    val reader = ModuleReaderSpec("bird", true, true, true)
    val evaluatorId = client.sendCreateEvaluatorRequest(moduleReaders = listOf(reader))

    client.send(
      EvaluateRequest(
-        requestId = 1,
-        evaluatorId = evaluatorId,
-        moduleUri = URI("repl:text"),
-        moduleText = """res = import*("bird:/**.pkl").keys""",
-        expr = "res"
+        1,
+        evaluatorId,
+        URI("repl:text"),
+        """res = import*("bird:/**.pkl").keys""",
+        "res"
      )
    )

@@ -535,15 +484,14 @@ abstract class AbstractServerTest {
    assertThat(listModulesMsg.uri.path).isEqualTo("/")
    client.send(
      ListModulesResponse(
-        requestId = listModulesMsg.requestId,
-        evaluatorId = evaluatorId,
-        pathElements =
-          listOf(
-            PathElement("birds", true),
-            PathElement("majesticBirds", true),
-            PathElement("Person.pkl", false)
-          ),
-        error = null
+        listModulesMsg.requestId,
+        evaluatorId,
+        listOf(
+          PathElement("birds", true),
+          PathElement("majesticBirds", true),
+          PathElement("Person.pkl", false)
+        ),
+        null
      )
    )
    val listModulesMsg2 = client.receive<ListModulesRequest>()
@@ -551,14 +499,13 @@ abstract class AbstractServerTest {
    assertThat(listModulesMsg2.uri.path).isEqualTo("/birds/")
    client.send(
      ListModulesResponse(
-        requestId = listModulesMsg2.requestId,
-        evaluatorId = listModulesMsg2.evaluatorId,
-        pathElements =
-          listOf(
-            PathElement("pigeon.pkl", false),
-            PathElement("parrot.pkl", false),
-          ),
-        error = null
+        listModulesMsg2.requestId,
+        listModulesMsg2.evaluatorId,
+        listOf(
+          PathElement("pigeon.pkl", false),
+          PathElement("parrot.pkl", false),
+        ),
+        null
      )
    )
    val listModulesMsg3 = client.receive<ListModulesRequest>()
@@ -566,19 +513,18 @@ abstract class AbstractServerTest {
    assertThat(listModulesMsg3.uri.path).isEqualTo("/majesticBirds/")
    client.send(
      ListModulesResponse(
-        requestId = listModulesMsg3.requestId,
-        evaluatorId = listModulesMsg3.evaluatorId,
-        pathElements =
-          listOf(
-            PathElement("barnOwl.pkl", false),
-            PathElement("elfOwl.pkl", false),
-          ),
-        error = null
+        listModulesMsg3.requestId,
+        listModulesMsg3.evaluatorId,
+        listOf(
+          PathElement("barnOwl.pkl", false),
+          PathElement("elfOwl.pkl", false),
+        ),
+        null
      )
    )

    val evaluateResponse = client.receive<EvaluateResponse>()
-    assertThat(evaluateResponse.result!!.debugRendering)
+    assertThat(evaluateResponse.result?.debugRendering)
      .isEqualTo(
        """
      - 6
@@ -595,36 +541,23 @@ abstract class AbstractServerTest {

  @Test
  fun `glob module -- null pathElements and null error`() {
-    val reader =
-      ModuleReaderSpec(
-        scheme = "bird",
-        hasHierarchicalUris = true,
-        isLocal = true,
-        isGlobbable = true
-      )
+    val reader = ModuleReaderSpec("bird", true, true, true)
    val evaluatorId = client.sendCreateEvaluatorRequest(moduleReaders = listOf(reader))

    client.send(
      EvaluateRequest(
-        requestId = 1,
-        evaluatorId = evaluatorId,
-        moduleUri = URI("repl:text"),
-        moduleText = """res = import*("bird:/**.pkl").keys""",
-        expr = "res"
+        1,
+        evaluatorId,
+        URI("repl:text"),
+        """res = import*("bird:/**.pkl").keys""",
+        "res"
      )
    )
    val listModulesMsg = client.receive<ListModulesRequest>()
-    client.send(
-      ListModulesResponse(
-        requestId = listModulesMsg.requestId,
-        evaluatorId = evaluatorId,
-        pathElements = null,
-        error = null
-      )
-    )
+    client.send(ListModulesResponse(listModulesMsg.requestId, evaluatorId, null, null))

    val evaluateResponse = client.receive<EvaluateResponse>()
-    assertThat(evaluateResponse.result!!.debugRendering)
+    assertThat(evaluateResponse.result?.debugRendering)
      .isEqualTo(
        """
      - 6
@@ -636,36 +569,23 @@ abstract class AbstractServerTest {

  @Test
  fun `glob module error`() {
-    val reader =
-      ModuleReaderSpec(
-        scheme = "bird",
-        hasHierarchicalUris = true,
-        isLocal = true,
-        isGlobbable = true
-      )
+    val reader = ModuleReaderSpec("bird", true, true, true)
    val evaluatorId = client.sendCreateEvaluatorRequest(moduleReaders = listOf(reader))

    client.send(
      EvaluateRequest(
-        requestId = 1,
-        evaluatorId = evaluatorId,
-        moduleUri = URI("repl:text"),
-        moduleText = """res = import*("bird:/**.pkl").keys""",
-        expr = "res"
+        1,
+        evaluatorId,
+        URI("repl:text"),
+        """res = import*("bird:/**.pkl").keys""",
+        "res"
      )
    )

    val listModulesMsg = client.receive<ListModulesRequest>()
    assertThat(listModulesMsg.uri.scheme).isEqualTo("bird")
    assertThat(listModulesMsg.uri.path).isEqualTo("/")
-    client.send(
-      ListModulesResponse(
-        requestId = listModulesMsg.requestId,
-        evaluatorId = evaluatorId,
-        pathElements = null,
-        error = "nope"
-      )
-    )
+    client.send(ListModulesResponse(listModulesMsg.requestId, evaluatorId, null, "nope"))
    val evaluateResponse = client.receive<EvaluateResponse>()
    assertThat(evaluateResponse.error)
      .isEqualTo(
@@ -699,19 +619,13 @@ abstract class AbstractServerTest {
    val evaluatorId = client.sendCreateEvaluatorRequest(modulePaths = listOf(jarFile))

    client.send(
-      EvaluateRequest(
-        requestId = 1,
-        evaluatorId = evaluatorId,
-        moduleUri = URI("modulepath:/dir1/module.pkl"),
-        moduleText = null,
-        expr = "output.text"
-      )
+      EvaluateRequest(1, evaluatorId, URI("modulepath:/dir1/module.pkl"), null, "output.text")
    )

    val response = client.receive<EvaluateResponse>()
    assertThat(response.error).isNull()
    val tripleQuote = "\"\"\""
-    assertThat(response.result!!.debugYaml)
+    assertThat(response.result?.debugYaml)
      .isEqualTo(
        """
      |
@@ -741,38 +655,31 @@ abstract class AbstractServerTest {

  @Test
  fun `import triple-dot path`() {
-    val reader =
-      ModuleReaderSpec(
-        scheme = "bird",
-        hasHierarchicalUris = true,
-        isLocal = true,
-        isGlobbable = true
-      )
+    val reader = ModuleReaderSpec("bird", true, true, true)
    val evaluatorId = client.sendCreateEvaluatorRequest(moduleReaders = listOf(reader))

    client.send(
      EvaluateRequest(
-        requestId = 1,
-        evaluatorId = evaluatorId,
-        moduleUri = URI("bird:/foo/bar/baz.pkl"),
-        moduleText =
-          """
+        1,
+        evaluatorId,
+        URI("bird:/foo/bar/baz.pkl"),
+        """
        import ".../buz.pkl"
        
        res = buz.res
      """
-            .trimIndent(),
-        expr = "res"
+          .trimIndent(),
+        "res"
      )
    )
    val readModuleRequest = client.receive<ReadModuleRequest>()
    assertThat(readModuleRequest.uri).isEqualTo(URI("bird:/foo/buz.pkl"))
    client.send(
      ReadModuleResponse(
-        requestId = readModuleRequest.requestId,
-        evaluatorId = readModuleRequest.evaluatorId,
-        contents = null,
-        error = "not here"
+        readModuleRequest.requestId,
+        readModuleRequest.evaluatorId,
+        null,
+        "not here"
      )
    )

@@ -780,54 +687,40 @@ abstract class AbstractServerTest {
    assertThat(readModuleRequest2.uri).isEqualTo(URI("bird:/buz.pkl"))
    client.send(
      ReadModuleResponse(
-        requestId = readModuleRequest2.requestId,
-        evaluatorId = readModuleRequest2.evaluatorId,
-        contents = "res = 1",
-        error = null
+        readModuleRequest2.requestId,
+        readModuleRequest2.evaluatorId,
+        "res = 1",
+        null
      )
    )

    val evaluatorResponse = client.receive<EvaluateResponse>()
-    assertThat(evaluatorResponse.result!!.debugYaml).isEqualTo("1")
+    assertThat(evaluatorResponse.result?.debugYaml).isEqualTo("1")
  }

  @Test
  fun `evaluate error`() {
    val evaluatorId = client.sendCreateEvaluatorRequest()

-    client.send(
-      EvaluateRequest(
-        requestId = 1,
-        evaluatorId = evaluatorId,
-        moduleUri = URI("repl:text"),
-        moduleText = """foo = 1""",
-        expr = "foo as String"
-      )
-    )
+    client.send(EvaluateRequest(1, evaluatorId, URI("repl:text"), """foo = 1""", "foo as String"))

    val evaluateResponse = client.receive<EvaluateResponse>()
-    assertThat(evaluateResponse.requestId).isEqualTo(1)
+    assertThat(evaluateResponse.requestId()).isEqualTo(1)
    assertThat(evaluateResponse.error).contains("Expected value of type")
  }

  @Test
  fun `evaluate client-provided module reader`() {
-    val reader =
-      ModuleReaderSpec(
-        scheme = "bird",
-        hasHierarchicalUris = true,
-        isLocal = false,
-        isGlobbable = false
-      )
+    val reader = ModuleReaderSpec("bird", true, false, false)
    val evaluatorId = client.sendCreateEvaluatorRequest(moduleReaders = listOf(reader))

    client.send(
      EvaluateRequest(
-        requestId = 1,
-        evaluatorId = evaluatorId,
-        moduleUri = URI("bird:/pigeon.pkl"),
-        moduleText = null,
-        expr = "output.text",
+        1,
+        evaluatorId,
+        URI("bird:/pigeon.pkl"),
+        null,
+        "output.text",
      )
    )

@@ -836,22 +729,21 @@ abstract class AbstractServerTest {

    client.send(
      ReadModuleResponse(
-        requestId = readModuleRequest.requestId,
-        evaluatorId = evaluatorId,
-        contents =
-          """
+        readModuleRequest.requestId,
+        evaluatorId,
+        """
          firstName = "Pigeon"
          lastName = "Bird"
          fullName = firstName + " " + lastName
        """
-            .trimIndent(),
-        error = null
+          .trimIndent(),
+        null
      )
    )

    val evaluateResponse = client.receive<EvaluateResponse>()
    assertThat(evaluateResponse.result).isNotNull
-    assertThat(evaluateResponse.result!!.debugYaml)
+    assertThat(evaluateResponse.result?.debugYaml)
      .isEqualTo(
        """
        |
@@ -865,33 +757,19 @@ abstract class AbstractServerTest {

  @Test
  fun `concurrent evaluations`() {
-    val reader =
-      ModuleReaderSpec(
-        scheme = "bird",
-        hasHierarchicalUris = true,
-        isLocal = false,
-        isGlobbable = false
-      )
+    val reader = ModuleReaderSpec("bird", true, false, false)
    val evaluatorId = client.sendCreateEvaluatorRequest(moduleReaders = listOf(reader))
    client.send(
      EvaluateRequest(
-        requestId = 1,
-        evaluatorId = evaluatorId,
-        moduleUri = URI("bird:/pigeon.pkl"),
-        moduleText = null,
-        expr = "output.text",
+        1,
+        evaluatorId,
+        URI("bird:/pigeon.pkl"),
+        null,
+        "output.text",
      )
    )

-    client.send(
-      EvaluateRequest(
-        requestId = 2,
-        evaluatorId = evaluatorId,
-        moduleUri = URI("bird:/parrot.pkl"),
-        moduleText = null,
-        expr = "output.text"
-      )
-    )
+    client.send(EvaluateRequest(2, evaluatorId, URI("bird:/parrot.pkl"), null, "output.text"))

    // evaluation is single-threaded; `parrot.pkl` gets evaluated after `pigeon.pkl` completes.
    val response11 = client.receive<ReadModuleRequest>()
@@ -901,20 +779,19 @@ abstract class AbstractServerTest {
      ReadModuleResponse(
        response11.requestId,
        evaluatorId,
-        contents =
-          """
+        """
          firstName = "Pigeon"
          lastName = "Bird"
          fullName = firstName + " " + lastName
        """
-            .trimIndent(),
-        error = null
+          .trimIndent(),
+        null
      )
    )

    val response12 = client.receive<EvaluateResponse>()
    assertThat(response12.result).isNotNull
-    assertThat(response12.result!!.debugYaml)
+    assertThat(response12.result?.debugYaml)
      .isEqualTo(
        """
        |
@@ -932,20 +809,19 @@ abstract class AbstractServerTest {
      ReadModuleResponse(
        response21.requestId,
        evaluatorId,
-        contents =
-          """
+        """
          firstName = "Parrot"
          lastName = "Bird"
          fullName = firstName + " " + lastName
        """
-            .trimIndent(),
-        error = null
+          .trimIndent(),
+        null
      )
    )

    val response22 = client.receive<EvaluateResponse>()
    assertThat(response22.result).isNotNull
-    assertThat(response22.result!!.debugYaml)
+    assertThat(response22.result?.debugYaml)
      .isEqualTo(
        """
        |
@@ -1039,34 +915,32 @@ abstract class AbstractServerTest {
        cacheDir = cacheDir,
        project =
          Project(
-            projectFileUri = projectDir.resolve("PklProject").toUri(),
-            packageUri = null,
-            dependencies =
-              mapOf(
-                "birds" to
-                  RemoteDependency(packageUri = URI("package://localhost:0/birds@0.5.0"), null),
-                "lib" to
-                  Project(
-                    projectFileUri = libDir.toUri().resolve("PklProject"),
-                    packageUri = URI("package://localhost:0/lib@5.0.0"),
-                    dependencies = emptyMap()
-                  )
-              )
+            projectDir.resolve("PklProject").toUri(),
+            null,
+            mapOf(
+              "birds" to RemoteDependency(URI("package://localhost:0/birds@0.5.0"), null),
+              "lib" to
+                Project(
+                  libDir.toUri().resolve("PklProject"),
+                  URI("package://localhost:0/lib@5.0.0"),
+                  emptyMap()
+                )
+            )
          )
      )
    client.send(
      EvaluateRequest(
-        requestId = 1,
-        evaluatorId = evaluatorId,
-        moduleUri = module.toUri(),
-        moduleText = null,
-        expr = "output.text",
+        1,
+        evaluatorId,
+        module.toUri(),
+        null,
+        "output.text",
      )
    )
    val resp2 = client.receive<EvaluateResponse>()
    assertThat(resp2.error).isNull()
    assertThat(resp2.result).isNotNull()
-    assertThat(resp2.result!!.debugRendering.trim())
+    assertThat(resp2.result?.debugRendering?.trim())
      .isEqualTo(
        """
        |
@@ -1098,26 +972,28 @@ abstract class AbstractServerTest {
  ): Long {
    val message =
      CreateEvaluatorRequest(
-        requestId = 123,
-        allowedResources = listOf(Pattern.compile(".*")),
-        allowedModules = listOf(Pattern.compile(".*")),
-        clientResourceReaders = resourceReaders,
-        clientModuleReaders = moduleReaders,
-        modulePaths = modulePaths,
-        env = mapOf(),
-        properties = mapOf(),
-        timeout = null,
-        rootDir = null,
-        cacheDir = cacheDir,
-        outputFormat = null,
-        project = project,
-        http = http
+        123,
+        listOf(Pattern.compile(".*")),
+        listOf(Pattern.compile(".*")),
+        moduleReaders,
+        resourceReaders,
+        modulePaths,
+        mapOf(),
+        mapOf(),
+        null,
+        null,
+        cacheDir,
+        null,
+        project,
+        http,
+        null,
+        null
      )

    send(message)

    val response = receive<CreateEvaluatorResponse>()
-    assertThat(response.requestId).isEqualTo(requestId)
+    assertThat(response.requestId()).isEqualTo(requestId)
    assertThat(response.evaluatorId).isNotNull
    assertThat(response.error).isNull()

--- a/pkl-server/src/test/kotlin/org/pkl/server/BinaryEvaluatorSnippetTests.kt
+++ b/pkl-server/src/test/kotlin/org/pkl/server/BinaryEvaluatorSnippetTests.kt
@@ -66,3 +66,6 @@ class BinaryEvaluatorSnippetTestEngine : InputOutputTestEngine() {
    return true to bytes.debugRendering.stripFilePaths()
  }
 }
+
+val ByteArray.debugRendering
+  get() = MessagePackDebugRenderer(this).output
--- a/pkl-server/src/test/kotlin/org/pkl/server/JvmServerTest.kt
+++ b/pkl-server/src/test/kotlin/org/pkl/server/JvmServerTest.kt
@@ -19,17 +19,32 @@ import java.io.PipedInputStream
 import java.io.PipedOutputStream
 import org.junit.jupiter.api.AfterEach
 import org.junit.jupiter.api.BeforeEach
+import org.pkl.core.messaging.MessageTransport
+import org.pkl.core.messaging.MessageTransports
+import org.pkl.core.util.Pair

 class JvmServerTest : AbstractServerTest() {
  private val transports: Pair<MessageTransport, MessageTransport> = run {
    if (USE_DIRECT_TRANSPORT) {
-      MessageTransports.direct()
+      MessageTransports.direct(::log)
    } else {
-      val in1 = PipedInputStream()
+      val in1 =
+        PipedInputStream(10240) // use larger pipe size since large messages can be >1024 bytes
      val out1 = PipedOutputStream(in1)
      val in2 = PipedInputStream()
      val out2 = PipedOutputStream(in2)
-      MessageTransports.stream(in1, out2) to MessageTransports.stream(in2, out1)
+      Pair.of(
+        MessageTransports.stream(
+          ServerMessagePackDecoder(in1),
+          ServerMessagePackEncoder(out2),
+          ::log
+        ),
+        MessageTransports.stream(
+          ServerMessagePackDecoder(in2),
+          ServerMessagePackEncoder(out1),
+          ::log
+        )
+      )
    }
  }

--- a/pkl-server/src/test/kotlin/org/pkl/server/MessagePackDebugRenderer.kt
+++ b/pkl-server/src/test/kotlin/org/pkl/server/MessagePackDebugRenderer.kt
@@ -105,6 +105,3 @@ class MessagePackDebugRenderer(bytes: ByteArray) {
    sb.toString().removePrefix("\n")
  }
 }
-
-val ByteArray.debugRendering
-  get() = MessagePackDebugRenderer(this).output
--- a/pkl-server/src/test/kotlin/org/pkl/server/NativeServerTest.kt
+++ b/pkl-server/src/test/kotlin/org/pkl/server/NativeServerTest.kt
@@ -18,6 +18,7 @@ package org.pkl.server
 import org.junit.jupiter.api.AfterEach
 import org.junit.jupiter.api.BeforeEach
 import org.pkl.commons.test.PklExecutablePaths
+import org.pkl.core.messaging.MessageTransports

 class NativeServerTest : AbstractServerTest() {
  private lateinit var server: Process
@@ -27,7 +28,14 @@ class NativeServerTest : AbstractServerTest() {
  fun beforeEach() {
    val executable = PklExecutablePaths.firstExisting.toString()
    server = ProcessBuilder(executable, "server").start()
-    client = TestTransport(MessageTransports.stream(server.inputStream, server.outputStream))
+    client =
+      TestTransport(
+        MessageTransports.stream(
+          ServerMessagePackDecoder(server.inputStream),
+          ServerMessagePackEncoder(server.outputStream)
+        ) { _ ->
+        }
+      )
    executor.execute { client.start() }
  }

--- a/pkl-server/src/test/kotlin/org/pkl/server/ServerMessagePackCodecTest.kt
+++ b/pkl-server/src/test/kotlin/org/pkl/server/ServerMessagePackCodecTest.kt
@@ -23,21 +23,25 @@ import java.time.Duration
 import java.util.regex.Pattern
 import org.assertj.core.api.Assertions.assertThat
 import org.junit.jupiter.api.Test
-import org.junit.jupiter.api.assertThrows
 import org.msgpack.core.MessagePack
 import org.pkl.core.evaluatorSettings.PklEvaluatorSettings
-import org.pkl.core.module.PathElement
+import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader
+import org.pkl.core.messaging.Message
+import org.pkl.core.messaging.MessageDecoder
+import org.pkl.core.messaging.MessageEncoder
+import org.pkl.core.messaging.Messages.*
 import org.pkl.core.packages.Checksums

-class MessagePackCodecTest {
+class ServerMessagePackCodecTest {
  private val encoder: MessageEncoder
  private val decoder: MessageDecoder

  init {
-    val inputStream = PipedInputStream()
+    val inputStream =
+      PipedInputStream(10240) // use larger pipe size since large messages can be >1024 bytes
    val outputStream = PipedOutputStream(inputStream)
-    encoder = MessagePackEncoder(MessagePack.newDefaultPacker(outputStream))
-    decoder = MessagePackDecoder(MessagePack.newDefaultUnpacker(inputStream))
+    encoder = ServerMessagePackEncoder(MessagePack.newDefaultPacker(outputStream))
+    decoder = ServerMessagePackDecoder(MessagePack.newDefaultUnpacker(inputStream))
  }

  private fun roundtrip(message: Message) {
@@ -50,31 +54,19 @@ class MessagePackCodecTest {
  fun `round-trip CreateEvaluatorRequest`() {
    val resourceReader1 =
      ResourceReaderSpec(
-        scheme = "resourceReader1",
-        hasHierarchicalUris = true,
-        isGlobbable = true,
+        "resourceReader1",
+        true,
+        true,
      )
    val resourceReader2 =
      ResourceReaderSpec(
-        scheme = "resourceReader2",
-        hasHierarchicalUris = true,
-        isGlobbable = false,
+        "resourceReader2",
+        true,
+        false,
      )
-    val moduleReader1 =
-      ModuleReaderSpec(
-        scheme = "moduleReader1",
-        hasHierarchicalUris = true,
-        isGlobbable = true,
-        isLocal = true
-      )
-    val moduleReader2 =
-      ModuleReaderSpec(
-        scheme = "moduleReader2",
-        hasHierarchicalUris = true,
-        isGlobbable = false,
-        isLocal = false
-      )
-    @Suppress("HttpUrlsUsage")
+    val moduleReader1 = ModuleReaderSpec("moduleReader1", true, true, true)
+    val moduleReader2 = ModuleReaderSpec("moduleReader2", true, false, false)
+    val externalReader = ExternalReader("external-cmd", listOf("arg1", "arg2"))
    roundtrip(
      CreateEvaluatorRequest(
        requestId = 123,
@@ -119,7 +111,9 @@ class MessagePackCodecTest {
          Http(
            proxy = PklEvaluatorSettings.Proxy(URI("http://foo.com:1234"), listOf("bar", "baz")),
            caCertificates = byteArrayOf(1, 2, 3, 4)
-          )
+          ),
+        externalModuleReaders = mapOf("external" to externalReader, "external2" to externalReader),
+        externalResourceReaders = mapOf("external" to externalReader),
      )
    )
  }
@@ -170,113 +164,4 @@ class MessagePackCodecTest {
      )
    )
  }
-
-  @Test
-  fun `round-trip ReadResourceRequest`() {
-    roundtrip(
-      ReadResourceRequest(requestId = 123, evaluatorId = 456, uri = URI("some/resource.json"))
-    )
-  }
-
-  @Test
-  fun `round-trip ReadResourceResponse`() {
-    roundtrip(
-      ReadResourceResponse(
-        requestId = 123,
-        evaluatorId = 456,
-        contents = byteArrayOf(1, 2, 3, 4, 5),
-        error = null
-      )
-    )
-  }
-
-  @Test
-  fun `round-trip ReadModuleRequest`() {
-    roundtrip(ReadModuleRequest(requestId = 123, evaluatorId = 456, uri = URI("some/module.pkl")))
-  }
-
-  @Test
-  fun `round-trip ReadModuleResponse`() {
-    roundtrip(
-      ReadModuleResponse(requestId = 123, evaluatorId = 456, contents = "x = 42", error = null)
-    )
-  }
-
-  @Test
-  fun `round-trip ListModulesRequest`() {
-    roundtrip(ListModulesRequest(requestId = 135, evaluatorId = 246, uri = URI("foo:/bar/baz/biz")))
-  }
-
-  @Test
-  fun `round-trip ListModulesResponse`() {
-    roundtrip(
-      ListModulesResponse(
-        requestId = 123,
-        evaluatorId = 234,
-        pathElements = listOf(PathElement("foo", true), PathElement("bar", false)),
-        error = null
-      )
-    )
-    roundtrip(
-      ListModulesResponse(
-        requestId = 123,
-        evaluatorId = 234,
-        pathElements = null,
-        error = "Something dun went wrong"
-      )
-    )
-  }
-
-  @Test
-  fun `round-trip ListResourcesRequest`() {
-    roundtrip(ListResourcesRequest(requestId = 987, evaluatorId = 1359, uri = URI("bar:/bazzy")))
-  }
-
-  @Test
-  fun `round-trip ListResourcesResponse`() {
-    roundtrip(
-      ListResourcesResponse(
-        requestId = 3851,
-        evaluatorId = 3019,
-        pathElements = listOf(PathElement("foo", true), PathElement("bar", false)),
-        error = null
-      )
-    )
-    roundtrip(
-      ListResourcesResponse(
-        requestId = 3851,
-        evaluatorId = 3019,
-        pathElements = null,
-        error = "something went wrong"
-      )
-    )
-  }
-
-  @Test
-  fun `decode request with missing request ID`() {
-    val bytes =
-      MessagePack.newDefaultBufferPacker()
-        .apply {
-          packArrayHeader(2)
-          packInt(MessageType.CREATE_EVALUATOR_REQUEST.code)
-          packMapHeader(1)
-          packString("clientResourceSchemes")
-          packArrayHeader(0)
-        }
-        .toByteArray()
-
-    val decoder = MessagePackDecoder(MessagePack.newDefaultUnpacker(bytes))
-    val exception = assertThrows<DecodeException> { decoder.decode() }
-    assertThat(exception.message).contains("requestId")
-  }
-
-  @Test
-  fun `decode invalid message header`() {
-    val bytes = MessagePack.newDefaultBufferPacker().apply { packInt(2) }.toByteArray()
-
-    val decoder = MessagePackDecoder(MessagePack.newDefaultUnpacker(bytes))
-    val exception = assertThrows<DecodeException> { decoder.decode() }
-    assertThat(exception).hasMessage("Malformed message header.")
-    assertThat(exception).hasRootCauseMessage("Expected Array, but got Integer (02)")
-  }
 }
--- a/pkl-server/src/test/kotlin/org/pkl/server/TestTransport.kt
+++ b/pkl-server/src/test/kotlin/org/pkl/server/TestTransport.kt
@@ -17,7 +17,9 @@ package org.pkl.server

 import java.util.concurrent.ArrayBlockingQueue
 import java.util.concurrent.BlockingQueue
-import org.assertj.core.api.Assertions.assertThat
+import org.assertj.core.api.Assertions.*
+import org.pkl.core.messaging.Message
+import org.pkl.core.messaging.MessageTransport

 class TestTransport(private val delegate: MessageTransport) : AutoCloseable {
  val incomingMessages: BlockingQueue<Message> = ArrayBlockingQueue(10)
@@ -30,15 +32,15 @@ class TestTransport(private val delegate: MessageTransport) : AutoCloseable {
    delegate.close()
  }

-  fun send(message: ClientOneWayMessage) {
+  fun send(message: Message.Client.OneWay) {
    delegate.send(message)
  }

-  fun send(message: ClientRequestMessage) {
+  fun send(message: Message.Client.Request) {
    delegate.send(message) { incomingMessages.put(it) }
  }

-  fun send(message: ClientResponseMessage) {
+  fun send(message: Message.Client.Response) {
    delegate.send(message)
  }