Implement SPICE-0009 External Readers (#660)

This adds a new feature, which allows Pkl to read resources and modules from external processes.

Follows the design laid out in SPICE-0009.

Also, this moves most of the messaging API into pkl-core

bench/gradle.lockfile

@@ -32,6 +32,7 @@ org.junit.jupiter:junit-jupiter-params:5.11.2=testCompileClasspath,testImplement
 org.junit.platform:junit-platform-commons:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit.platform:junit-platform-engine:1.11.2=testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit:junit-bom:5.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
+org.msgpack:msgpack-core:0.9.8=jmh,jmhRuntimeClasspath
 org.openjdk.jmh:jmh-core:1.37=jmh,jmhCompileClasspath,jmhImplementationDependenciesMetadata,jmhRuntimeClasspath
 org.openjdk.jmh:jmh-generator-asm:1.37=jmh,jmhCompileClasspath,jmhImplementationDependenciesMetadata,jmhRuntimeClasspath
 org.openjdk.jmh:jmh-generator-bytecode:1.37=jmh,jmhCompileClasspath,jmhImplementationDependenciesMetadata,jmhRuntimeClasspath

docs/gradle.lockfile

@@ -30,6 +30,7 @@ org.junit.jupiter:junit-jupiter-params:5.11.2=testCompileClasspath,testImplement
 org.junit.platform:junit-platform-commons:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit.platform:junit-platform-engine:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit:junit-bom:5.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
+org.msgpack:msgpack-core:0.9.8=testRuntimeClasspath
 org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.organicdesign:Paguro:3.10.3=testRuntimeClasspath
 org.snakeyaml:snakeyaml-engine:2.5=testRuntimeClasspath

docs/modules/bindings-specification/pages/message-passing-api.adoc

@@ -9,8 +9,8 @@ The first element of the array is a code that designates the message's type, enc
 The second element of the array is the message body, encoded as a map.
 
 Messages are passed between the _client_ and the _server_.
-The _client_ is the host language (for example, the Swift application when using pkl-swift).
-The _server_ is the entity that provides controls for interacting with Pkl.
+When hosting Pkl (for example, the Swift application when using pkl-swift), the _client_ is the host program and the _server_ is the entity that provides controls for interacting with Pkl.
+When implementing an xref:language-reference:index.adoc#external-readers[external reader], the _client_ is the external reader process and the _server_ is the Pkl evaluator.
 
 For example, in JSON representation:
 
@@ -597,3 +597,96 @@ class PathElement {
   isDirectory: Boolean
 }
 ----
+
+[[initialize-module-reader-request]]
+=== Initialize Module Reader Request
+
+Code: `0x2e` +
+Type: <<server-message,Server>> <<request-message,Request>>
+
+Initialize an xref:language-reference:index.adoc#external-readers[External Module Reader].
+This message is sent to external reader processes the first time a module scheme it is registered for is read.
+
+[source,pkl]
+----
+/// A number identifying this request.
+requestId: Int
+
+/// The module scheme to initialize.
+scheme: String
+----
+
+[[initialize-module-reader-response]]
+=== Initialize Module Reader Response
+
+Code: `0x2f` +
+Type: <<client-message,Client>> <<response-message,Response>>
+
+Return the requested external module reader specification.
+The `spec` field should be set to `null` when the external process does not implement the requested module scheme.
+
+[source,pkl]
+----
+/// A number identifying this request.
+requestId: Int
+
+/// Client-side module reader spec.
+///
+/// Null when the external process does not implement the requested module scheme.
+spec: ClientModuleReader?
+----
+
+`ClientModuleReader` is defined above by <<create-evaluator-request,Create Evaluator Request>>.
+
+[[initialize-resource-reader-request]]
+=== Initialize Resource Reader Request
+
+Code: `0x30` +
+Type: <<server-message,Server>> <<request-message,Request>>
+
+Initialize an xref:language-reference:index.adoc#external-readers[External Resource Reader].
+This message is sent to external reader processes the first time a resource scheme it is registered for is read.
+
+[source,pkl]
+----
+/// A number identifying this request.
+requestId: Int
+
+/// The resource scheme to initialize.
+scheme: String
+----
+
+[[initialize-resource-reader-response]]
+=== Initialize Resource Reader Response
+
+Code: `0x31` +
+Type: <<client-message,Client>> <<response-message,Response>>
+
+Return the requested external resource reader specification.
+The `spec` field should be set to `null` when the external process does not implement the requested resource scheme.
+
+[source,pkl]
+----
+/// A number identifying this request.
+requestId: Int
+
+/// Client-side resource reader spec.
+///
+/// Null when the external process does not implement the requested resource scheme.
+spec: ClientResourceReader?
+----
+
+`ClientResourceReader` is defined above by <<create-evaluator-request,Create Evaluator Request>>. 
+
+[[close-external-process]]
+=== Close External Process
+
+Code: `0x32` +
+Type: <<server-message,Server>> <<one-way-message,One Way>> 
+
+Initiate graceful shutdown of the external reader process.
+
+[source,pkl]
+----
+/// This message has no properties.
+----

docs/modules/language-reference/pages/index.adoc

@@ -3199,7 +3199,8 @@ This section discusses language features that are generally more relevant to tem
 <<grammar-definition,Grammar Definition>> +
 <<reserved-keywords,Reserved Keywords>> +
 <<blank-identifiers,Blank Identifiers>> +
-<<projects,Projects>>
+<<projects,Projects>> +
+<<external-readers,External Readers>>
 
 [[meaning-of-new]]
 === Meaning of `new`
@@ -5465,3 +5466,68 @@ It can be imported using dependency notation, i.e. `import "@fruit/Pear.pkl"`.
 At runtime, it will resolve to relative path `../fruit/Pear.pkl`.
 
 When packaging projects with local dependencies, both the project and its dependent project must be passed to the xref:pkl-cli:index.adoc#command-project-package[`pkl project package`] command.
+
+[[external-readers]]
+=== External Readers
+
+External readers are a mechanism to extend the <<modules,module>> and <<resources,resource>> URI schemes that Pkl supports.
+Readers are implemented as ordinary executables and use Pkl's xref:bindings-specification:message-passing-api.adoc[message passing API] to communicate with the hosting Pkl evaluator.
+The xref:swift:ROOT:index.adoc[Swift] and xref:go:ROOT:index.adoc[Go] language binding libraries provide an `ExternalReaderRuntime` type to facilitate implementing external readers.
+
+External readers are configured separately for modules and resources.
+They are registered by mapping their URI scheme to the executable to run and additonal arguments to pass.
+This is done on the command line by passing `--external-resource-reader` and `--external-module-reader` flags, which may both be passed multiple times.
+
+[source,text]
+----
+$ pkl eval <module> --external-resource-reader <scheme>=<executable> --external-module-reader <scheme>='<executable> <argument> <argument>'
+----
+
+External readers may also be configured in a <<projects, Project's>> `PklProject` file.
+[source,{pkl}]
+----
+evaluatorSettings {
+  externalResourceReaders {
+    ["<scheme>"] {
+      executable = "<executable>"
+    }
+  }
+  externalModuleReaders {
+    ["<scheme>"] {
+      executable = "<executable>"
+      arguments { "<arg>"; "<arg>" }
+    }
+  }
+}
+----
+
+Registering an external reader for a scheme automatically adds that scheme to the default allowed modules/resources.
+As with Pkl's built-in module and resource schemes, setting explicit allowed modules or resources overrides this behavior and appropriate patterns must be specified to allow use of external readers.
+
+==== Example
+
+Consider this module:
+
+[source,{pkl}]
+----
+username = "pigeon"
+
+email = read("ldap://ds.example.com:389/dc=example,dc=com?mail?sub?(uid=\(username))").text
+----
+
+Pkl doesn't implement the `ldap:` resource URI scheme natively, but an external reader can provide it.
+Assuming a hypothetical `pkl-ldap` executable implementing the external reader protocol and the `ldap:` scheme is in the `$PATH`, this module can be evaluated as:
+
+[source,text]
+----
+$ pkl eval <module> --external-resource-reader ldap=pkl-ldap
+username = "pigeon"
+email = "pigeon@example.com"
+----
+
+In this example, the external reader may provide both `ldap:` and `ldaps:` schemes.
+To support both schemes during evaluation, both would need to be registered explicitly:
+[source,text]
+----
+$ pkl eval <module> --external-resource-reader ldap=pkl-ldap --external-resource-reader ldaps=pkl-ldap
+----

docs/src/test/kotlin/DocSnippetTests.kt

@@ -22,10 +22,10 @@ import org.pkl.core.parser.antlr.PklParser
 import org.pkl.core.repl.ReplRequest
 import org.pkl.core.repl.ReplResponse
 import org.pkl.core.repl.ReplServer
-import org.pkl.core.resource.ResourceReaders
 import org.pkl.core.util.IoUtils
 import org.antlr.v4.runtime.ParserRuleContext
 import org.pkl.core.http.HttpClient
+import org.pkl.core.resource.ResourceReaders
 import java.nio.file.Files
 import kotlin.io.path.isDirectory
 import kotlin.io.path.isRegularFile

pkl-cli/gradle.lockfile

@@ -99,4 +99,4 @@ org.xmlunit:xmlunit-core:2.10.0=testCompileClasspath,testImplementationDependenc
 org.xmlunit:xmlunit-legacy:2.10.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.xmlunit:xmlunit-placeholders:2.10.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.yaml:snakeyaml:2.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
-empty=annotationProcessor,intransitiveDependenciesMetadata,javaExecutable,kotlinCompilerPluginClasspath,kotlinNativeCompilerPluginClasspath,kotlinScriptDef,kotlinScriptDefExtensions,runtimeOnlyDependenciesMetadata,shadow,signatures,sourcesJar,stagedAlpineLinuxAmd64Executable,stagedLinuxAarch64Executable,stagedLinuxAmd64Executable,stagedMacAarch64Executable,stagedMacAmd64Executable,stagedWindowsAmd64Executable,testAnnotationProcessor,testApiDependenciesMetadata,testCompileOnlyDependenciesMetadata,testIntransitiveDependenciesMetadata,testKotlinScriptDef,testKotlinScriptDefExtensions
+empty=annotationProcessor,archives,compile,intransitiveDependenciesMetadata,javaExecutable,kotlinCompilerPluginClasspath,kotlinNativeCompilerPluginClasspath,kotlinScriptDef,kotlinScriptDefExtensions,runtime,runtimeOnlyDependenciesMetadata,shadow,signatures,sourcesJar,stagedAlpineLinuxAmd64Executable,stagedLinuxAarch64Executable,stagedLinuxAmd64Executable,stagedMacAarch64Executable,stagedMacAmd64Executable,stagedWindowsAmd64Executable,testAnnotationProcessor,testApiDependenciesMetadata,testCompile,testCompileOnly,testCompileOnlyDependenciesMetadata,testIntransitiveDependenciesMetadata,testKotlinScriptDef,testKotlinScriptDefExtensions,testRuntime

pkl-cli/src/main/kotlin/org/pkl/cli/CliEvaluator.kt

@@ -31,12 +31,12 @@ import org.pkl.commons.writeString
 import org.pkl.core.EvaluatorBuilder
 import org.pkl.core.ModuleSource
 import org.pkl.core.PklException
-import org.pkl.core.module.ModuleKeyFactories
 import org.pkl.core.module.ModulePathResolver
 import org.pkl.core.runtime.ModuleResolver
 import org.pkl.core.runtime.VmException
 import org.pkl.core.runtime.VmUtils
 import org.pkl.core.util.IoUtils
+import org.pkl.core.util.Readers
 
 private data class OutputFile(val pathSpec: String, val moduleUri: URI)
 
@@ -100,7 +100,8 @@ constructor(
         writeOutput(builder)
       }
     } finally {
-      ModuleKeyFactories.closeQuietly(builder.moduleKeyFactories)
+      Readers.closeQuietly(builder.moduleKeyFactories)
+      Readers.closeQuietly(builder.resourceReaders)
     }
   }
 

pkl-cli/src/main/kotlin/org/pkl/cli/CliImportAnalyzer.kt

@@ -20,7 +20,7 @@ import org.pkl.commons.cli.CliCommand
 import org.pkl.commons.createParentDirectories
 import org.pkl.commons.writeString
 import org.pkl.core.ModuleSource
-import org.pkl.core.module.ModuleKeyFactories
+import org.pkl.core.util.Readers
 
 class CliImportAnalyzer
 @JvmOverloads
@@ -73,7 +73,8 @@ constructor(
         .build()
         .use { it.evaluateOutputText(sourceModule) }
     } finally {
-      ModuleKeyFactories.closeQuietly(builder.moduleKeyFactories)
+      Readers.closeQuietly(builder.moduleKeyFactories)
+      Readers.closeQuietly(builder.resourceReaders)
     }
   }
 }

pkl-cli/src/main/kotlin/org/pkl/cli/CliServer.kt

@@ -18,14 +18,13 @@ package org.pkl.cli
 import org.pkl.commons.cli.CliBaseOptions
 import org.pkl.commons.cli.CliCommand
 import org.pkl.commons.cli.CliException
-import org.pkl.server.MessageTransports
-import org.pkl.server.ProtocolException
+import org.pkl.core.messaging.ProtocolException
 import org.pkl.server.Server
 
 class CliServer(options: CliBaseOptions) : CliCommand(options) {
   override fun doRun() =
     try {
-      val server = Server(MessageTransports.stream(System.`in`, System.out))
+      val server = Server.stream(System.`in`, System.out)
       server.use { it.start() }
     } catch (e: ProtocolException) {
       throw CliException(e.message!!)

pkl-cli/src/main/kotlin/org/pkl/cli/CliTestRunner.kt

@@ -19,10 +19,10 @@ import java.io.Writer
 import org.pkl.commons.cli.*
 import org.pkl.core.EvaluatorBuilder
 import org.pkl.core.ModuleSource.uri
-import org.pkl.core.module.ModuleKeyFactories
 import org.pkl.core.stdlib.test.report.JUnitReport
 import org.pkl.core.stdlib.test.report.SimpleReport
 import org.pkl.core.util.ErrorMessages
+import org.pkl.core.util.Readers
 
 class CliTestRunner
 @JvmOverloads
@@ -38,7 +38,8 @@ constructor(
     try {
       evalTest(builder)
     } finally {
-      ModuleKeyFactories.closeQuietly(builder.moduleKeyFactories)
+      Readers.closeQuietly(builder.moduleKeyFactories)
+      Readers.closeQuietly(builder.resourceReaders)
     }
   }
 

pkl-codegen-java/gradle.lockfile

@@ -33,6 +33,7 @@ org.junit.jupiter:junit-jupiter-params:5.11.2=testCompileClasspath,testImplement
 org.junit.platform:junit-platform-commons:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit.platform:junit-platform-engine:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit:junit-bom:5.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
+org.msgpack:msgpack-core:0.9.8=runtimeClasspath,testRuntimeClasspath
 org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.organicdesign:Paguro:3.10.3=runtimeClasspath,testRuntimeClasspath
 org.snakeyaml:snakeyaml-engine:2.5=runtimeClasspath,testRuntimeClasspath

pkl-codegen-java/src/main/kotlin/org/pkl/codegen/java/CliJavaCodeGenerator.kt

@@ -21,7 +21,7 @@ import org.pkl.commons.cli.CliException
 import org.pkl.commons.createParentDirectories
 import org.pkl.commons.writeString
 import org.pkl.core.ModuleSource
-import org.pkl.core.module.ModuleKeyFactories
+import org.pkl.core.util.Readers
 
 /** API for the Java code generator CLI. */
 class CliJavaCodeGenerator(private val options: CliJavaCodeGeneratorOptions) :
@@ -49,7 +49,8 @@ class CliJavaCodeGenerator(private val options: CliJavaCodeGeneratorOptions) :
         }
       }
     } finally {
-      ModuleKeyFactories.closeQuietly(builder.moduleKeyFactories)
+      Readers.closeQuietly(builder.moduleKeyFactories)
+      Readers.closeQuietly(builder.resourceReaders)
     }
   }
 }

pkl-codegen-kotlin/gradle.lockfile

@@ -36,6 +36,7 @@ org.junit.jupiter:junit-jupiter-params:5.11.2=testCompileClasspath,testImplement
 org.junit.platform:junit-platform-commons:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit.platform:junit-platform-engine:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit:junit-bom:5.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
+org.msgpack:msgpack-core:0.9.8=runtimeClasspath,testRuntimeClasspath
 org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.organicdesign:Paguro:3.10.3=runtimeClasspath,testRuntimeClasspath
 org.snakeyaml:snakeyaml-engine:2.5=runtimeClasspath,testRuntimeClasspath

pkl-codegen-kotlin/src/main/kotlin/org/pkl/codegen/kotlin/CliKotlinCodeGenerator.kt

@@ -21,7 +21,7 @@ import org.pkl.commons.cli.CliException
 import org.pkl.commons.createParentDirectories
 import org.pkl.commons.writeString
 import org.pkl.core.ModuleSource
-import org.pkl.core.module.ModuleKeyFactories
+import org.pkl.core.util.Readers
 
 /** API for the Kotlin code generator CLI. */
 class CliKotlinCodeGenerator(private val options: CliKotlinCodeGeneratorOptions) :
@@ -50,7 +50,8 @@ class CliKotlinCodeGenerator(private val options: CliKotlinCodeGeneratorOptions)
         }
       }
     } finally {
-      ModuleKeyFactories.closeQuietly(builder.moduleKeyFactories)
+      Readers.closeQuietly(builder.moduleKeyFactories)
+      Readers.closeQuietly(builder.resourceReaders)
     }
   }
 }

pkl-commons-cli/gradle.lockfile

@@ -31,6 +31,7 @@ org.junit.jupiter:junit-jupiter-params:5.11.2=testCompileClasspath,testImplement
 org.junit.platform:junit-platform-commons:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit.platform:junit-platform-engine:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit:junit-bom:5.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
+org.msgpack:msgpack-core:0.9.8=runtimeClasspath,testRuntimeClasspath
 org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.organicdesign:Paguro:3.10.3=runtimeClasspath,testRuntimeClasspath
 org.snakeyaml:snakeyaml-engine:2.5=runtimeClasspath,testRuntimeClasspath

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliBaseOptions.kt

@@ -20,6 +20,7 @@ import java.nio.file.Files
 import java.nio.file.Path
 import java.time.Duration
 import java.util.regex.Pattern
+import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader
 import org.pkl.core.module.ProjectDependenciesManager
 import org.pkl.core.util.IoUtils
 
@@ -134,6 +135,12 @@ data class CliBaseOptions(
 
   /** Hostnames, IP addresses, or CIDR blocks to not proxy. */
   val httpNoProxy: List<String>? = null,
+
+  /** External module reader process specs */
+  val externalModuleReaders: Map<String, ExternalReader> = mapOf(),
+
+  /** External resource reader process specs */
+  val externalResourceReaders: Map<String, ExternalReader> = mapOf(),
 ) {
 
   companion object {

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliCommand.kt

@@ -21,6 +21,7 @@ import java.util.regex.Pattern
 import kotlin.io.path.isRegularFile
 import org.pkl.core.*
 import org.pkl.core.evaluatorSettings.PklEvaluatorSettings
+import org.pkl.core.externalreader.ExternalReaderProcessImpl
 import org.pkl.core.http.HttpClient
 import org.pkl.core.module.ModuleKeyFactories
 import org.pkl.core.module.ModuleKeyFactory
@@ -108,12 +109,16 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
 
   protected val allowedModules: List<Pattern> by lazy {
     cliOptions.allowedModules
-      ?: evaluatorSettings?.allowedModules ?: SecurityManagers.defaultAllowedModules
+      ?: evaluatorSettings?.allowedModules
+        ?: (SecurityManagers.defaultAllowedModules +
+        externalModuleReaders.keys.map { Pattern.compile("$it:") }.toList())
   }
 
   protected val allowedResources: List<Pattern> by lazy {
     cliOptions.allowedResources
-      ?: evaluatorSettings?.allowedResources ?: SecurityManagers.defaultAllowedResources
+      ?: evaluatorSettings?.allowedResources
+        ?: (SecurityManagers.defaultAllowedResources +
+        externalResourceReaders.keys.map { Pattern.compile("$it:") }.toList())
   }
 
   protected val rootDir: Path? by lazy {
@@ -169,6 +174,26 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
       ?: project?.evaluatorSettings?.http?.proxy?.noProxy ?: settings.http?.proxy?.noProxy
   }
 
+  private val externalModuleReaders by lazy {
+    (project?.evaluatorSettings?.externalModuleReaders
+      ?: emptyMap()) + cliOptions.externalModuleReaders
+  }
+
+  private val externalResourceReaders by lazy {
+    (project?.evaluatorSettings?.externalResourceReaders
+      ?: emptyMap()) + cliOptions.externalResourceReaders
+  }
+
+  private val externalProcesses by lazy {
+    // share ExternalProcessImpl instances between configured external resource/module readers with
+    // the same spec
+    // this avoids spawning multiple subprocesses if the same reader implements both reader types
+    // and/or multiple schemes
+    (externalModuleReaders + externalResourceReaders).values.toSet().associateWith {
+      ExternalReaderProcessImpl(it)
+    }
+  }
+
   private fun HttpClient.Builder.addDefaultCliCertificates() {
     val caCertsDir = IoUtils.getPklHomeDir().resolve("cacerts")
     var certsAdded = false
@@ -213,6 +238,9 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
 
   protected fun moduleKeyFactories(modulePathResolver: ModulePathResolver): List<ModuleKeyFactory> {
     return buildList {
+      externalModuleReaders.forEach { (key, value) ->
+        add(ModuleKeyFactories.externalProcess(key, externalProcesses[value]!!))
+      }
       add(ModuleKeyFactories.standardLibrary)
       add(ModuleKeyFactories.modulePath(modulePathResolver))
       add(ModuleKeyFactories.pkg)
@@ -234,6 +262,9 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
       add(ResourceReaders.file())
       add(ResourceReaders.http())
       add(ResourceReaders.https())
+      externalResourceReaders.forEach { (key, value) ->
+        add(ResourceReaders.externalProcess(key, externalProcesses[value]!!))
+      }
     }
   }
 

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseOptions.kt

@@ -28,6 +28,8 @@ import java.time.Duration
 import java.util.regex.Pattern
 import org.pkl.commons.cli.CliBaseOptions
 import org.pkl.commons.cli.CliException
+import org.pkl.commons.shlex
+import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader
 import org.pkl.core.runtime.VmUtils
 import org.pkl.core.util.IoUtils
 
@@ -74,6 +76,17 @@ class BaseOptions : OptionGroup() {
         .multiple()
         .toMap()
     }
+
+    fun OptionWithValues<String?, String, String>.parseExternalReader(
+      delimiter: String
+    ): OptionWithValues<
+      Pair<String, ExternalReader>?, Pair<String, ExternalReader>, Pair<String, ExternalReader>
+    > {
+      return splitPair(delimiter).convert {
+        val cmd = shlex(it.second)
+        Pair(it.first, ExternalReader(cmd.first(), cmd.drop(1)))
+      }
+    }
   }
 
   private val defaults = CliBaseOptions()
@@ -207,6 +220,26 @@ class BaseOptions : OptionGroup() {
       .single()
       .split(",")
 
+  val externalModuleReaders: Map<String, ExternalReader> by
+    option(
+        names = arrayOf("--external-module-reader"),
+        metavar = "<scheme>='<executable>[ <arguments>]'",
+        help = "External reader registrations for module URI schemes"
+      )
+      .parseExternalReader("=")
+      .multiple()
+      .toMap()
+
+  val externalResourceReaders: Map<String, ExternalReader> by
+    option(
+        names = arrayOf("--external-resource-reader"),
+        metavar = "<scheme>='<executable>[ <arguments>]'",
+        help = "External reader registrations for resource URI schemes"
+      )
+      .parseExternalReader("=")
+      .multiple()
+      .toMap()
+
   // hidden option used by native tests
   private val testPort: Int by
     option(names = arrayOf("--test-port"), help = "Internal test option", hidden = true)
@@ -239,7 +272,9 @@ class BaseOptions : OptionGroup() {
       noProject = projectOptions?.noProject ?: false,
       caCertificates = caCertificates,
       httpProxy = proxy,
-      httpNoProxy = noProxy ?: emptyList()
+      httpNoProxy = noProxy ?: emptyList(),
+      externalModuleReaders = externalModuleReaders,
+      externalResourceReaders = externalResourceReaders,
     )
   }
 }

pkl-commons-cli/src/test/kotlin/org/pkl/commons/cli/BaseCommandTest.kt

@@ -24,6 +24,7 @@ import org.assertj.core.api.Assertions.assertThat
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertThrows
 import org.pkl.commons.cli.commands.BaseCommand
+import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader
 
 class BaseCommandTest {
 
@@ -72,4 +73,34 @@ class BaseCommandTest {
 
     assertThat(cmd.baseOptions.allowedResources).isEmpty()
   }
+
+  @Test
+  fun `--external-resource-reader and --external-module-reader are parsed correctly`() {
+    cmd.parse(
+      arrayOf(
+        "--external-module-reader",
+        "scheme3=reader3",
+        "--external-module-reader",
+        "scheme4=reader4 with args",
+        "--external-resource-reader",
+        "scheme1=reader1",
+        "--external-resource-reader",
+        "scheme2=reader2 with args"
+      )
+    )
+    assertThat(cmd.baseOptions.externalModuleReaders)
+      .isEqualTo(
+        mapOf(
+          "scheme3" to ExternalReader("reader3", emptyList()),
+          "scheme4" to ExternalReader("reader4", listOf("with", "args"))
+        )
+      )
+    assertThat(cmd.baseOptions.externalResourceReaders)
+      .isEqualTo(
+        mapOf(
+          "scheme1" to ExternalReader("reader1", emptyList()),
+          "scheme2" to ExternalReader("reader2", listOf("with", "args"))
+        )
+      )
+  }
 }

pkl-commons/src/main/kotlin/org/pkl/commons/Strings.kt

@@ -18,6 +18,7 @@ package org.pkl.commons
 import java.io.File
 import java.net.URI
 import java.nio.file.Path
+import java.util.*
 import java.util.regex.Pattern
 
 fun String.toPath(): Path = Path.of(this)
@@ -36,3 +37,53 @@ fun String.toUri(): URI {
   }
   return URI(null, null, this, null)
 }
+
+/** Lex a string into tokens similar to how a shell would */
+fun shlex(input: String): List<String> {
+  val result = mutableListOf<String>()
+  var inEscape = false
+  var quote: Char? = null
+  var lastCloseQuoteIndex = Int.MIN_VALUE
+  val current = StringBuilder()
+
+  for ((idx, char) in input.withIndex()) {
+    when {
+      // if in an escape always append the next character
+      inEscape -> {
+        inEscape = false
+        current.append(char)
+      }
+      // enter an escape on \ if not in a quote or in a non-single quote
+      char == '\\' && quote != '\'' -> inEscape = true
+      // if in a quote and encounter the delimiter, tentatively exit the quote
+      // this handles cases with adjoining quotes e.g. `abc'123''xyz'`
+      quote == char -> {
+        quote = null
+        lastCloseQuoteIndex = idx
+      }
+      // if not in a quote and encounter a quote charater, enter a quote
+      quote == null && (char == '\'' || char == '"') -> {
+        quote = char
+      }
+      // if not in a quote and whitespace is encountered
+      quote == null && char.isWhitespace() -> {
+        // if the current token isn't empty or if a quote has just ended, finalize the current token
+        // otherwise do nothing, which handles multiple whitespace cases e.g. `abc     123`
+        if (current.isNotEmpty() || lastCloseQuoteIndex == (idx - 1)) {
+          result.add(current.toString())
+          current.clear()
+        }
+      }
+      // in other cases, append to the current token
+      else -> current.append(char)
+    }
+  }
+  // clean up last token
+  // if the current token isn't empty or if a quote has just ended, finalize the token
+  // if this condition is false, the input likely ended in whitespace
+  if (current.isNotEmpty() || lastCloseQuoteIndex == (input.length - 1)) {
+    result.add(current.toString())
+  }
+
+  return result
+}

pkl-commons/src/test/kotlin/org/pkl/commons/ShlexTest.kt

@@ -0,0 +1,78 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.commons
+
+import org.assertj.core.api.Assertions.assertThat
+import org.junit.jupiter.api.Test
+
+class ShlexTest {
+
+  @Test
+  fun `empty input produces empty output`() {
+    assertThat(shlex("")).isEqualTo(emptyList<String>())
+  }
+
+  @Test
+  fun `whitespace input produces empty output`() {
+    assertThat(shlex("  \n \t  ")).isEqualTo(emptyList<String>())
+  }
+
+  @Test
+  fun `regular token parsing`() {
+    assertThat(shlex("\nabc  def\tghi ")).isEqualTo(listOf("abc", "def", "ghi"))
+  }
+
+  @Test
+  fun `single quoted token parsing`() {
+    assertThat(shlex("'this is a single token'")).isEqualTo(listOf("this is a single token"))
+  }
+
+  @Test
+  fun `double quoted token parsing`() {
+    assertThat(shlex("\"this is a single token\"")).isEqualTo(listOf("this is a single token"))
+  }
+
+  @Test
+  fun `escaping handles double quotes`() {
+    assertThat(shlex(""""\"this is a single double quoted token\"""""))
+      .isEqualTo(listOf("\"this is a single double quoted token\""))
+  }
+
+  @Test
+  fun `escaping does not apply within single quotes`() {
+    assertThat(shlex("""'this is a single \" token'"""))
+      .isEqualTo(listOf("""this is a single \" token"""))
+  }
+
+  @Test
+  fun `adjacent quoted strings are one token`() {
+    assertThat(shlex(""""single"' joined 'token""")).isEqualTo(listOf("single joined token"))
+    assertThat(shlex(""""single"' 'token""")).isEqualTo(listOf("single token"))
+  }
+
+  @Test
+  fun `space escapes do not split tokens`() {
+    assertThat(shlex("""single\ token""")).isEqualTo(listOf("single token"))
+  }
+
+  @Test
+  fun `empty quotes produce a single empty token`() {
+    assertThat(shlex("\"\"")).isEqualTo(listOf(""))
+    assertThat(shlex("''")).isEqualTo(listOf(""))
+    assertThat(shlex("'' ''")).isEqualTo(listOf("", ""))
+    assertThat(shlex("''''")).isEqualTo(listOf(""))
+  }
+}

pkl-config-java/gradle.lockfile

@@ -34,6 +34,7 @@ org.junit.jupiter:junit-jupiter-params:5.11.2=testCompileClasspath,testImplement
 org.junit.platform:junit-platform-commons:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit.platform:junit-platform-engine:1.11.2=testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit:junit-bom:5.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
+org.msgpack:msgpack-core:0.9.8=pklCodegenJava,runtimeClasspath,testRuntimeClasspath
 org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.organicdesign:Paguro:3.10.3=pklCodegenJava,runtimeClasspath,testRuntimeClasspath
 org.snakeyaml:snakeyaml-engine:2.5=pklCodegenJava,runtimeClasspath,testRuntimeClasspath

pkl-config-kotlin/gradle.lockfile

@@ -33,6 +33,7 @@ org.junit.jupiter:junit-jupiter-params:5.11.2=testCompileClasspath,testImplement
 org.junit.platform:junit-platform-commons:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit.platform:junit-platform-engine:1.11.2=testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit:junit-bom:5.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
+org.msgpack:msgpack-core:0.9.8=pklCodegenKotlin,pklConfigJava,runtimeClasspath,testRuntimeClasspath
 org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.organicdesign:Paguro:3.10.3=pklCodegenKotlin,pklConfigJava,runtimeClasspath,testRuntimeClasspath
 org.snakeyaml:snakeyaml-engine:2.5=pklCodegenKotlin,pklConfigJava,runtimeClasspath,testRuntimeClasspath

pkl-core/gradle.lockfile

@@ -37,6 +37,7 @@ org.junit.jupiter:junit-jupiter-params:5.11.2=testCompileClasspath,testImplement
 org.junit.platform:junit-platform-commons:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit.platform:junit-platform-engine:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit:junit-bom:5.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
+org.msgpack:msgpack-core:0.9.8=compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.organicdesign:Paguro:3.10.3=compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.snakeyaml:snakeyaml-engine:2.5=compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath

pkl-core/pkl-core.gradle.kts

@@ -57,6 +57,7 @@ dependencies {
   compileOnly(projects.pklExecutor)
 
   implementation(libs.antlrRuntime)
+  implementation(libs.msgpack)
   implementation(libs.truffleApi)
   implementation(libs.graalSdk)
 

pkl-core/src/main/java/org/pkl/core/EvaluatorBuilder.java

@@ -20,6 +20,9 @@ import java.nio.file.Path;
 import java.util.*;
 import java.util.regex.Pattern;
 import org.pkl.core.SecurityManagers.StandardBuilder;
+import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader;
+import org.pkl.core.externalreader.ExternalReaderProcess;
+import org.pkl.core.externalreader.ExternalReaderProcessImpl;
 import org.pkl.core.http.HttpClient;
 import org.pkl.core.module.ModuleKeyFactories;
 import org.pkl.core.module.ModuleKeyFactory;
@@ -478,6 +481,25 @@ public final class EvaluatorBuilder {
     } else if (settings.moduleCacheDir() != null) {
       setModuleCacheDir(settings.moduleCacheDir());
     }
+
+    // this isn't ideal as project and non-project ExternalProcessImpl instances can be dupes
+    var procs = new HashMap<ExternalReader, ExternalReaderProcess>();
+    if (settings.externalModuleReaders() != null) {
+      for (var entry : settings.externalModuleReaders().entrySet()) {
+        addModuleKeyFactory(
+            ModuleKeyFactories.externalProcess(
+                entry.getKey(),
+                procs.computeIfAbsent(entry.getValue(), ExternalReaderProcessImpl::new)));
+      }
+    }
+    if (settings.externalResourceReaders() != null) {
+      for (var entry : settings.externalResourceReaders().entrySet()) {
+        addResourceReader(
+            ResourceReaders.externalProcess(
+                entry.getKey(),
+                procs.computeIfAbsent(entry.getValue(), ExternalReaderProcessImpl::new)));
+      }
+    }
     return this;
   }
 

pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java

@@ -55,6 +55,7 @@ import org.pkl.core.ast.internal.ToStringNodeGen;
 import org.pkl.core.ast.lambda.ApplyVmFunction1NodeGen;
 import org.pkl.core.ast.member.*;
 import org.pkl.core.ast.type.*;
+import org.pkl.core.externalreader.ExternalReaderProcessException;
 import org.pkl.core.module.ModuleKey;
 import org.pkl.core.module.ModuleKeys;
 import org.pkl.core.module.ResolvedModuleKey;
@@ -1847,6 +1848,12 @@ public final class AstBuilder extends AbstractAstBuilder<Object> {
           .withHint(e.getHint())
           .withSourceSection(createSourceSection(importUriCtx))
           .build();
+    } catch (ExternalReaderProcessException e) {
+      throw exceptionBuilder()
+          .evalError("externalReaderFailure")
+          .withCause(e.getCause())
+          .withSourceSection(createSourceSection(importUriCtx))
+          .build();
     }
 
     if (!resolvedUri.isAbsolute()) {

pkl-core/src/main/java/org/pkl/core/ast/expression/unary/AbstractReadNode.java

@@ -23,6 +23,7 @@ import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import org.pkl.core.SecurityManagerException;
+import org.pkl.core.externalreader.ExternalReaderProcessException;
 import org.pkl.core.module.ModuleKey;
 import org.pkl.core.packages.PackageLoadError;
 import org.pkl.core.runtime.VmContext;
@@ -75,6 +76,8 @@ public abstract class AbstractReadNode extends UnaryExpressionNode {
           .build();
     } catch (PackageLoadError | SecurityManagerException e) {
       throw exceptionBuilder().withCause(e).build();
+    } catch (ExternalReaderProcessException e) {
+      throw exceptionBuilder().evalError("externalReaderFailure").withCause(e).build();
     }
 
     if (!resolvedUri.isAbsolute()) {

pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ImportGlobNode.java

@@ -25,6 +25,7 @@ import java.io.IOException;
 import java.net.URI;
 import org.pkl.core.SecurityManagerException;
 import org.pkl.core.ast.member.SharedMemberNode;
+import org.pkl.core.externalreader.ExternalReaderProcessException;
 import org.pkl.core.http.HttpClientInitException;
 import org.pkl.core.module.ResolvedModuleKey;
 import org.pkl.core.packages.PackageLoadError;
@@ -104,6 +105,8 @@ public class ImportGlobNode extends AbstractImportNode {
           .evalError("invalidGlobPattern", globPattern)
           .withHint(e.getMessage())
           .build();
+    } catch (ExternalReaderProcessException e) {
+      throw exceptionBuilder().evalError("externalReaderFailure").withCause(e).build();
     }
   }
 }

pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ReadGlobNode.java

@@ -26,6 +26,7 @@ import java.net.URISyntaxException;
 import org.graalvm.collections.EconomicMap;
 import org.pkl.core.SecurityManagerException;
 import org.pkl.core.ast.member.SharedMemberNode;
+import org.pkl.core.externalreader.ExternalReaderProcessException;
 import org.pkl.core.http.HttpClientInitException;
 import org.pkl.core.module.ModuleKey;
 import org.pkl.core.runtime.VmContext;
@@ -103,6 +104,8 @@ public abstract class ReadGlobNode extends AbstractReadNode {
           .evalError("invalidGlobPattern", globPattern)
           .withHint(e.getMessage())
           .build();
+    } catch (ExternalReaderProcessException e) {
+      throw exceptionBuilder().evalError("externalReaderFailure").withCause(e).build();
     }
   }
 }

pkl-core/src/main/java/org/pkl/core/evaluatorSettings/PklEvaluatorSettings.java

@@ -20,9 +20,11 @@ import java.net.URISyntaxException;
 import java.nio.file.Path;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Objects;
 import java.util.function.BiFunction;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 import org.pkl.core.Duration;
 import org.pkl.core.PNull;
 import org.pkl.core.PObject;
@@ -43,7 +45,9 @@ public record PklEvaluatorSettings(
     @Nullable List<Path> modulePath,
     @Nullable Duration timeout,
     @Nullable Path rootDir,
-    @Nullable Http http) {
+    @Nullable Http http,
+    @Nullable Map<String, ExternalReader> externalModuleReaders,
+    @Nullable Map<String, ExternalReader> externalResourceReaders) {
 
   /** Initializes a {@link PklEvaluatorSettings} from a raw object representation. */
   @SuppressWarnings("unchecked")
@@ -80,6 +84,24 @@ public record PklEvaluatorSettings(
     var rootDirStr = (String) pSettings.get("rootDir");
     var rootDir = rootDirStr == null ? null : pathNormalizer.apply(rootDirStr, "rootDir");
 
+    var externalModuleReadersRaw = (Map<String, Value>) pSettings.get("externalModuleReaders");
+    var externalModuleReaders =
+        externalModuleReadersRaw == null
+            ? null
+            : externalModuleReadersRaw.entrySet().stream()
+                .collect(
+                    Collectors.toMap(
+                        Entry::getKey, entry -> ExternalReader.parse(entry.getValue())));
+
+    var externalResourceReadersRaw = (Map<String, Value>) pSettings.get("externalResourceReaders");
+    var externalResourceReaders =
+        externalResourceReadersRaw == null
+            ? null
+            : externalResourceReadersRaw.entrySet().stream()
+                .collect(
+                    Collectors.toMap(
+                        Entry::getKey, entry -> ExternalReader.parse(entry.getValue())));
+
     return new PklEvaluatorSettings(
         (Map<String, String>) pSettings.get("externalProperties"),
         (Map<String, String>) pSettings.get("env"),
@@ -90,7 +112,9 @@ public record PklEvaluatorSettings(
         modulePath,
         (Duration) pSettings.get("timeout"),
         rootDir,
-        Http.parse((Value) pSettings.get("http")));
+        Http.parse((Value) pSettings.get("http")),
+        externalModuleReaders,
+        externalResourceReaders);
   }
 
   public record Http(@Nullable Proxy proxy) {
@@ -133,6 +157,18 @@ public record PklEvaluatorSettings(
     }
   }
 
+  public record ExternalReader(String executable, @Nullable List<String> arguments) {
+    @SuppressWarnings("unchecked")
+    public static ExternalReader parse(Value input) {
+      if (input instanceof PObject externalReader) {
+        var executable = (String) externalReader.getProperty("executable");
+        var arguments = (List<String>) externalReader.get("arguments");
+        return new ExternalReader(executable, arguments);
+      }
+      throw PklBugException.unreachableCode();
+    }
+  }
+
   private boolean arePatternsEqual(
       @Nullable List<Pattern> thesePatterns, @Nullable List<Pattern> thosePatterns) {
     if (thesePatterns == null) {

pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderMessagePackDecoder.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.externalreader;
+
+import java.io.InputStream;
+import java.net.URISyntaxException;
+import java.util.Map;
+import org.msgpack.core.MessagePack;
+import org.msgpack.core.MessageUnpacker;
+import org.msgpack.value.Value;
+import org.pkl.core.externalreader.ExternalReaderMessages.*;
+import org.pkl.core.messaging.BaseMessagePackDecoder;
+import org.pkl.core.messaging.DecodeException;
+import org.pkl.core.messaging.Message;
+import org.pkl.core.messaging.Message.Type;
+import org.pkl.core.util.Nullable;
+
+public class ExternalReaderMessagePackDecoder extends BaseMessagePackDecoder {
+
+  public ExternalReaderMessagePackDecoder(MessageUnpacker unpacker) {
+    super(unpacker);
+  }
+
+  public ExternalReaderMessagePackDecoder(InputStream inputStream) {
+    this(MessagePack.newDefaultUnpacker(inputStream));
+  }
+
+  @Override
+  protected @Nullable Message decodeMessage(Type msgType, Map<Value, Value> map)
+      throws DecodeException, URISyntaxException {
+    return switch (msgType) {
+      case INITIALIZE_MODULE_READER_REQUEST ->
+          new InitializeModuleReaderRequest(
+              unpackLong(map, "requestId"), unpackString(map, "scheme"));
+      case INITIALIZE_RESOURCE_READER_REQUEST ->
+          new InitializeResourceReaderRequest(
+              unpackLong(map, "requestId"), unpackString(map, "scheme"));
+      case INITIALIZE_MODULE_READER_RESPONSE ->
+          new InitializeModuleReaderResponse(
+              unpackLong(map, "requestId"), unpackModuleReaderSpec(getNullable(map, "spec")));
+      case INITIALIZE_RESOURCE_READER_RESPONSE ->
+          new InitializeResourceReaderResponse(
+              unpackLong(map, "requestId"), unpackResourceReaderSpec(getNullable(map, "spec")));
+      case CLOSE_EXTERNAL_PROCESS -> new CloseExternalProcess();
+      default -> super.decodeMessage(msgType, map);
+    };
+  }
+}

pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderMessagePackEncoder.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.externalreader;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import org.msgpack.core.MessagePack;
+import org.msgpack.core.MessagePacker;
+import org.pkl.core.externalreader.ExternalReaderMessages.*;
+import org.pkl.core.messaging.BaseMessagePackEncoder;
+import org.pkl.core.messaging.Message;
+import org.pkl.core.messaging.ProtocolException;
+import org.pkl.core.util.Nullable;
+
+public class ExternalReaderMessagePackEncoder extends BaseMessagePackEncoder {
+
+  public ExternalReaderMessagePackEncoder(MessagePacker packer) {
+    super(packer);
+  }
+
+  public ExternalReaderMessagePackEncoder(OutputStream outputStream) {
+    this(MessagePack.newDefaultPacker(outputStream));
+  }
+
+  @Override
+  protected @Nullable void encodeMessage(Message msg) throws ProtocolException, IOException {
+    switch (msg.type()) {
+      case INITIALIZE_MODULE_READER_REQUEST -> {
+        var m = (InitializeModuleReaderRequest) msg;
+        packer.packMapHeader(2);
+        packKeyValue("requestId", m.requestId());
+        packKeyValue("scheme", m.scheme());
+      }
+      case INITIALIZE_RESOURCE_READER_REQUEST -> {
+        var m = (InitializeResourceReaderRequest) msg;
+        packer.packMapHeader(2);
+        packKeyValue("requestId", m.requestId());
+        packKeyValue("scheme", m.scheme());
+      }
+      case INITIALIZE_MODULE_READER_RESPONSE -> {
+        var m = (InitializeModuleReaderResponse) msg;
+        packMapHeader(1, m.spec());
+        packKeyValue("requestId", m.requestId());
+        if (m.spec() != null) {
+          packer.packString("spec");
+          packModuleReaderSpec(m.spec());
+        }
+      }
+      case INITIALIZE_RESOURCE_READER_RESPONSE -> {
+        var m = (InitializeResourceReaderResponse) msg;
+        packMapHeader(1, m.spec());
+        packKeyValue("requestId", m.requestId());
+        if (m.spec() != null) {
+          packer.packString("spec");
+          packResourceReaderSpec(m.spec());
+        }
+      }
+      case CLOSE_EXTERNAL_PROCESS -> packer.packMapHeader(0);
+      default -> super.encodeMessage(msg);
+    }
+  }
+}

pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderMessages.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.externalreader;
+
+import org.pkl.core.messaging.Message.*;
+import org.pkl.core.messaging.Messages.ModuleReaderSpec;
+import org.pkl.core.messaging.Messages.ResourceReaderSpec;
+import org.pkl.core.util.Nullable;
+
+public class ExternalReaderMessages {
+
+  public record InitializeModuleReaderRequest(long requestId, String scheme)
+      implements Server.Request {
+    public Type type() {
+      return Type.INITIALIZE_MODULE_READER_REQUEST;
+    }
+  }
+
+  public record InitializeResourceReaderRequest(long requestId, String scheme)
+      implements Server.Request {
+    public Type type() {
+      return Type.INITIALIZE_RESOURCE_READER_REQUEST;
+    }
+  }
+
+  public record InitializeModuleReaderResponse(long requestId, @Nullable ModuleReaderSpec spec)
+      implements Client.Response {
+    public Type type() {
+      return Type.INITIALIZE_MODULE_READER_RESPONSE;
+    }
+  }
+
+  public record InitializeResourceReaderResponse(long requestId, @Nullable ResourceReaderSpec spec)
+      implements Client.Response {
+    public Type type() {
+      return Type.INITIALIZE_RESOURCE_READER_RESPONSE;
+    }
+  }
+
+  public record CloseExternalProcess() implements Server.OneWay {
+    public Type type() {
+      return Type.CLOSE_EXTERNAL_PROCESS;
+    }
+  }
+}

pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderProcess.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.externalreader;
+
+import java.io.IOException;
+import org.pkl.core.messaging.MessageTransport;
+import org.pkl.core.messaging.Messages.ModuleReaderSpec;
+import org.pkl.core.messaging.Messages.ResourceReaderSpec;
+import org.pkl.core.util.Nullable;
+
+/** An interface for interacting with external module/resource processes. */
+public interface ExternalReaderProcess extends AutoCloseable {
+
+  /**
+   * Obtain the process's underlying {@link MessageTransport} for sending reader-specific message
+   *
+   * <p>May allocate resources upon first call, including spawning a child process. Must not be
+   * called after {@link ExternalReaderProcess#close} has been called.
+   */
+  MessageTransport getTransport() throws ExternalReaderProcessException;
+
+  /** Retrieve the spec, if available, of the process's module reader with the given scheme. */
+  @Nullable
+  ModuleReaderSpec getModuleReaderSpec(String scheme) throws IOException;
+
+  /** Retrieve the spec, if available, of the process's resource reader with the given scheme. */
+  @Nullable
+  ResourceReaderSpec getResourceReaderSpec(String scheme) throws IOException;
+
+  /**
+   * Close the external process, cleaning up any resources.
+   *
+   * <p>The {@link MessageTransport} is sent the {@link ExternalReaderMessages.CloseExternalProcess}
+   * message to request a graceful stop. A bespoke (empty) message type is used here instead of an
+   * OS mechanism like signals to avoid forcing external reader implementers needing to handle many
+   * OS-specific mechanisms. Implementations may then forcibly clean up resources after a timeout.
+   * Must be safe to call multiple times.
+   */
+  @Override
+  void close();
+}

pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderProcessException.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.externalreader;
+
+public final class ExternalReaderProcessException extends Exception {
+  public ExternalReaderProcessException(String msg) {
+    super(msg);
+  }
+
+  public ExternalReaderProcessException(Throwable cause) {
+    super(cause);
+  }
+}

pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderProcessImpl.java

@@ -0,0 +1,226 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.externalreader;
+
+import java.io.IOException;
+import java.lang.ProcessBuilder.Redirect;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Random;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Future;
+import javax.annotation.concurrent.GuardedBy;
+import org.pkl.core.Duration;
+import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader;
+import org.pkl.core.externalreader.ExternalReaderMessages.*;
+import org.pkl.core.messaging.MessageTransport;
+import org.pkl.core.messaging.MessageTransports;
+import org.pkl.core.messaging.Messages.ModuleReaderSpec;
+import org.pkl.core.messaging.Messages.ResourceReaderSpec;
+import org.pkl.core.messaging.ProtocolException;
+import org.pkl.core.util.LateInit;
+import org.pkl.core.util.Nullable;
+
+public class ExternalReaderProcessImpl implements ExternalReaderProcess {
+
+  private static final Duration CLOSE_TIMEOUT = Duration.ofSeconds(3);
+
+  private final ExternalReader spec;
+  private final @Nullable String logPrefix;
+  private final Map<String, Future<@Nullable ModuleReaderSpec>> initializeModuleReaderResponses =
+      new ConcurrentHashMap<>();
+  private final Map<String, Future<@Nullable ResourceReaderSpec>>
+      initializeResourceReaderResponses = new ConcurrentHashMap<>();
+
+  private @GuardedBy("this") boolean closed = false;
+
+  @LateInit
+  @GuardedBy("this")
+  private Process process;
+
+  @LateInit
+  @GuardedBy("this")
+  private MessageTransport transport;
+
+  private void log(String msg) {
+    if (logPrefix != null) {
+      System.err.println(logPrefix + msg);
+    }
+  }
+
+  public ExternalReaderProcessImpl(ExternalReader spec) {
+    this.spec = spec;
+    logPrefix =
+        Objects.equals(System.getenv("PKL_DEBUG"), "1")
+            ? "[pkl-core][external-process][" + spec.executable() + "] "
+            : null;
+  }
+
+  @Override
+  public synchronized MessageTransport getTransport() throws ExternalReaderProcessException {
+    if (closed) {
+      throw new ExternalReaderProcessException("ExternalProcessImpl has already been closed");
+    }
+    if (process != null) {
+      if (!process.isAlive()) {
+        throw new ExternalReaderProcessException("ExternalProcessImpl process is no longer alive");
+      }
+
+      return transport;
+    }
+
+    // This relies on Java/OS behavior around PATH resolution, absolute/relative paths, etc.
+    var command = new ArrayList<String>();
+    command.add(spec.executable());
+    command.addAll(spec.arguments());
+
+    var builder = new ProcessBuilder(command);
+    builder.redirectError(Redirect.INHERIT); // inherit stderr from this pkl process
+    try {
+      process = builder.start();
+    } catch (IOException e) {
+      throw new ExternalReaderProcessException(e);
+    }
+    transport =
+        MessageTransports.stream(
+            new ExternalReaderMessagePackDecoder(process.getInputStream()),
+            new ExternalReaderMessagePackEncoder(process.getOutputStream()),
+            this::log);
+
+    var rxThread = new Thread(this::runTransport, "ExternalProcessImpl rxThread for " + spec);
+    rxThread.setDaemon(true);
+    rxThread.start();
+
+    return transport;
+  }
+
+  /**
+   * Runs the underlying message transport so it can receive responses from the child process.
+   *
+   * <p>Blocks until the underlying transport is closed.
+   */
+  private void runTransport() {
+    try {
+      transport.start(
+          (msg) -> {
+            throw new ProtocolException("Unexpected incoming one-way message: " + msg);
+          },
+          (msg) -> {
+            throw new ProtocolException("Unexpected incoming request message: " + msg);
+          });
+    } catch (ProtocolException | IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Override
+  public synchronized void close() {
+    closed = true;
+    if (process == null || !process.isAlive()) {
+      return;
+    }
+
+    try {
+      if (transport != null) {
+        transport.send(new CloseExternalProcess());
+        transport.close();
+      }
+
+      // forcefully stop the process after the timeout
+      // note that both transport.close() and process.destroy() are safe to call multiple times
+      new Timer()
+          .schedule(
+              new TimerTask() {
+                @Override
+                public void run() {
+                  if (process != null) {
+                    transport.close();
+                    process.destroyForcibly();
+                  }
+                }
+              },
+              CLOSE_TIMEOUT.inWholeMillis());
+
+      // block on process exit
+      process.onExit().get();
+    } catch (Exception e) {
+      transport.close();
+      process.destroyForcibly();
+    } finally {
+      process = null;
+      transport = null;
+    }
+  }
+
+  @Override
+  public @Nullable ModuleReaderSpec getModuleReaderSpec(String uriScheme) throws IOException {
+    return MessageTransports.resolveFuture(
+        initializeModuleReaderResponses.computeIfAbsent(
+            uriScheme,
+            (scheme) -> {
+              var future = new CompletableFuture<@Nullable ModuleReaderSpec>();
+              var request = new InitializeModuleReaderRequest(new Random().nextLong(), scheme);
+              try {
+                getTransport()
+                    .send(
+                        request,
+                        (response) -> {
+                          if (response instanceof InitializeModuleReaderResponse resp) {
+                            future.complete(resp.spec());
+                          } else {
+                            future.completeExceptionally(
+                                new ProtocolException("unexpected response"));
+                          }
+                        });
+              } catch (ProtocolException | IOException | ExternalReaderProcessException e) {
+                future.completeExceptionally(e);
+              }
+              return future;
+            }));
+  }
+
+  @Override
+  public @Nullable ResourceReaderSpec getResourceReaderSpec(String uriScheme) throws IOException {
+    return MessageTransports.resolveFuture(
+        initializeResourceReaderResponses.computeIfAbsent(
+            uriScheme,
+            (scheme) -> {
+              var future = new CompletableFuture<@Nullable ResourceReaderSpec>();
+              var request = new InitializeResourceReaderRequest(new Random().nextLong(), scheme);
+              try {
+                getTransport()
+                    .send(
+                        request,
+                        (response) -> {
+                          log(response.toString());
+                          if (response instanceof InitializeResourceReaderResponse resp) {
+                            future.complete(resp.spec());
+                          } else {
+                            future.completeExceptionally(
+                                new ProtocolException("unexpected response"));
+                          }
+                        });
+              } catch (ProtocolException | IOException | ExternalReaderProcessException e) {
+                future.completeExceptionally(e);
+              }
+              return future;
+            }));
+  }
+}

pkl-core/src/main/java/org/pkl/core/externalreader/package-info.java

@@ -0,0 +1,4 @@
+@NonnullByDefault
+package org.pkl.core.externalreader;
+
+import org.pkl.core.util.NonnullByDefault;

pkl-core/src/main/java/org/pkl/core/messaging/AbstractMessagePackDecoder.java

@@ -0,0 +1,220 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.messaging;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import org.msgpack.core.MessagePack;
+import org.msgpack.core.MessageTypeException;
+import org.msgpack.core.MessageUnpacker;
+import org.msgpack.value.Value;
+import org.msgpack.value.impl.ImmutableStringValueImpl;
+import org.pkl.core.messaging.Message.Type;
+import org.pkl.core.util.ErrorMessages;
+import org.pkl.core.util.Nullable;
+
+public abstract class AbstractMessagePackDecoder implements MessageDecoder {
+
+  protected final MessageUnpacker unpacker;
+
+  public AbstractMessagePackDecoder(MessageUnpacker unpacker) {
+    this.unpacker = unpacker;
+  }
+
+  public AbstractMessagePackDecoder(InputStream stream) {
+    this(MessagePack.newDefaultUnpacker(stream));
+  }
+
+  protected abstract @Nullable Message decodeMessage(Type msgType, Map<Value, Value> map)
+      throws DecodeException, URISyntaxException;
+
+  @Override
+  public @Nullable Message decode() throws IOException, DecodeException {
+    if (!unpacker.hasNext()) {
+      return null;
+    }
+
+    int code;
+    try {
+      var arraySize = unpacker.unpackArrayHeader();
+      if (arraySize != 2) {
+        throw new DecodeException(ErrorMessages.create("malformedMessageHeaderLength", arraySize));
+      }
+      code = unpacker.unpackInt();
+    } catch (MessageTypeException e) {
+      throw new DecodeException(ErrorMessages.create("malformedMessageHeaderException"), e);
+    }
+
+    Type msgType;
+    try {
+      msgType = Type.fromInt(code);
+    } catch (IllegalArgumentException e) {
+      throw new DecodeException(
+          ErrorMessages.create("malformedMessageHeaderUnrecognizedCode", Integer.toHexString(code)),
+          e);
+    }
+
+    try {
+      var map = unpacker.unpackValue().asMapValue().map();
+      var decoded = decodeMessage(msgType, map);
+      if (decoded != null) {
+        return decoded;
+      }
+      throw new DecodeException(
+          ErrorMessages.create("unhandledMessageCode", Integer.toHexString(code)));
+    } catch (MessageTypeException | URISyntaxException e) {
+      throw new DecodeException(ErrorMessages.create("malformedMessageBody", code), e);
+    }
+  }
+
+  protected static @Nullable Value getNullable(Map<Value, Value> map, String key) {
+    return map.get(new ImmutableStringValueImpl(key));
+  }
+
+  protected static Value get(Map<Value, Value> map, String key) throws DecodeException {
+    var value = map.get(new ImmutableStringValueImpl(key));
+    if (value == null) {
+      throw new DecodeException(ErrorMessages.create("missingMessageParameter", key));
+    }
+    return value;
+  }
+
+  protected static String unpackString(Map<Value, Value> map, String key) throws DecodeException {
+    return get(map, key).asStringValue().asString();
+  }
+
+  protected static @Nullable String unpackStringOrNull(Map<Value, Value> map, String key) {
+    var value = getNullable(map, key);
+    if (value == null) {
+      return null;
+    }
+    return value.asStringValue().asString();
+  }
+
+  protected static <T> @Nullable T unpackStringOrNull(
+      Map<Value, Value> map, String key, Function<String, T> mapper) {
+    var value = getNullable(map, key);
+    if (value == null) {
+      return null;
+    }
+    return mapper.apply(value.asStringValue().asString());
+  }
+
+  protected static byte @Nullable [] unpackByteArray(Map<Value, Value> map, String key) {
+    var value = getNullable(map, key);
+    if (value == null) {
+      return null;
+    }
+    return value.asBinaryValue().asByteArray();
+  }
+
+  protected static boolean unpackBoolean(Map<Value, Value> map, String key) throws DecodeException {
+    return get(map, key).asBooleanValue().getBoolean();
+  }
+
+  protected static int unpackInt(Map<Value, Value> map, String key) throws DecodeException {
+    return get(map, key).asIntegerValue().asInt();
+  }
+
+  protected static long unpackLong(Map<Value, Value> map, String key) throws DecodeException {
+    return get(map, key).asIntegerValue().asLong();
+  }
+
+  protected static @Nullable Long unpackLongOrNull(Map<Value, Value> map, String key) {
+    var value = getNullable(map, key);
+    if (value == null) {
+      return null;
+    }
+    return value.asIntegerValue().asLong();
+  }
+
+  protected static <T> @Nullable T unpackLongOrNull(
+      Map<Value, Value> map, String key, Function<Long, T> mapper) {
+    var value = unpackLongOrNull(map, key);
+    if (value == null) {
+      return null;
+    }
+    return mapper.apply(value);
+  }
+
+  protected static @Nullable List<String> unpackStringListOrNull(
+      Map<Value, Value> map, String key) {
+    var value = getNullable(map, key);
+    if (value == null) {
+      return null;
+    }
+
+    return value.asArrayValue().list().stream().map((it) -> it.asStringValue().asString()).toList();
+  }
+
+  protected static @Nullable Map<String, String> unpackStringMapOrNull(
+      Map<Value, Value> map, String key) {
+    var value = getNullable(map, key);
+    if (value == null) {
+      return null;
+    }
+
+    return value.asMapValue().entrySet().stream()
+        .collect(
+            Collectors.toMap(
+                (e) -> e.getKey().asStringValue().asString(),
+                (e) -> e.getValue().asStringValue().asString()));
+  }
+
+  protected static <T> @Nullable List<T> unpackStringListOrNull(
+      Map<Value, Value> map, String key, Function<String, T> mapper) {
+    var value = unpackStringListOrNull(map, key);
+    if (value == null) {
+      return null;
+    }
+
+    return value.stream().map(mapper).toList();
+  }
+
+  protected static <T> @Nullable List<T> unpackListOrNull(
+      Map<Value, Value> map, String key, Function<Value, T> mapper) {
+    var keys = getNullable(map, key);
+    if (keys == null) {
+      return null;
+    }
+
+    var result = new ArrayList<T>(keys.asArrayValue().size());
+    for (Value value : keys.asArrayValue()) {
+      result.add(mapper.apply(value));
+    }
+    return result;
+  }
+
+  protected static <T> @Nullable Map<String, T> unpackStringMapOrNull(
+      Map<Value, Value> map, String key, Function<Map<Value, Value>, T> mapper) {
+    var value = getNullable(map, key);
+    if (value == null) {
+      return null;
+    }
+
+    return value.asMapValue().entrySet().stream()
+        .collect(
+            Collectors.toMap(
+                (e) -> e.getKey().asStringValue().asString(),
+                (e) -> mapper.apply(e.getValue().asMapValue().map())));
+  }
+}

pkl-core/src/main/java/org/pkl/core/messaging/AbstractMessagePackEncoder.java

@@ -0,0 +1,184 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.messaging;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Collection;
+import java.util.Map;
+import java.util.function.Function;
+import org.msgpack.core.MessagePack;
+import org.msgpack.core.MessagePacker;
+import org.pkl.core.util.Nullable;
+
+public abstract class AbstractMessagePackEncoder implements MessageEncoder {
+
+  protected final MessagePacker packer;
+
+  public AbstractMessagePackEncoder(MessagePacker packer) {
+    this.packer = packer;
+  }
+
+  public AbstractMessagePackEncoder(OutputStream stream) {
+    this(MessagePack.newDefaultPacker(stream));
+  }
+
+  protected abstract @Nullable void encodeMessage(Message msg)
+      throws ProtocolException, IOException;
+
+  @Override
+  public final void encode(Message msg) throws IOException, ProtocolException {
+    packer.packArrayHeader(2);
+    packer.packInt(msg.type().getCode());
+    encodeMessage(msg);
+    packer.flush();
+  }
+
+  protected void packMapHeader(int size, @Nullable Object value1) throws IOException {
+    packer.packMapHeader(size + (value1 != null ? 1 : 0));
+  }
+
+  protected void packMapHeader(int size, @Nullable Object value1, @Nullable Object value2)
+      throws IOException {
+    packer.packMapHeader(size + (value1 != null ? 1 : 0) + (value2 != null ? 1 : 0));
+  }
+
+  protected void packMapHeader(
+      int size,
+      @Nullable Object value1,
+      @Nullable Object value2,
+      @Nullable Object value3,
+      @Nullable Object value4,
+      @Nullable Object value5,
+      @Nullable Object value6,
+      @Nullable Object value7,
+      @Nullable Object value8,
+      @Nullable Object value9,
+      @Nullable Object valueA,
+      @Nullable Object valueB,
+      @Nullable Object valueC,
+      @Nullable Object valueD,
+      @Nullable Object valueE,
+      @Nullable Object valueF)
+      throws IOException {
+    packer.packMapHeader(
+        size
+            + (value1 != null ? 1 : 0)
+            + (value2 != null ? 1 : 0)
+            + (value3 != null ? 1 : 0)
+            + (value4 != null ? 1 : 0)
+            + (value5 != null ? 1 : 0)
+            + (value6 != null ? 1 : 0)
+            + (value7 != null ? 1 : 0)
+            + (value8 != null ? 1 : 0)
+            + (value9 != null ? 1 : 0)
+            + (valueA != null ? 1 : 0)
+            + (valueB != null ? 1 : 0)
+            + (valueC != null ? 1 : 0)
+            + (valueD != null ? 1 : 0)
+            + (valueE != null ? 1 : 0)
+            + (valueF != null ? 1 : 0));
+  }
+
+  protected void packKeyValue(String name, @Nullable Integer value) throws IOException {
+    if (value == null) {
+      return;
+    }
+    packer.packString(name);
+    packer.packInt(value);
+  }
+
+  protected void packKeyValue(String name, @Nullable Long value) throws IOException {
+    if (value == null) {
+      return;
+    }
+    packer.packString(name);
+    packer.packLong(value);
+  }
+
+  protected <T> void packKeyValueLong(String name, @Nullable T value, Function<T, Long> mapper)
+      throws IOException {
+    if (value == null) {
+      return;
+    }
+    packKeyValue(name, mapper.apply(value));
+  }
+
+  protected void packKeyValue(String name, @Nullable String value) throws IOException {
+    if (value == null) {
+      return;
+    }
+    packer.packString(name);
+    packer.packString(value);
+  }
+
+  protected <T> void packKeyValueString(String name, @Nullable T value, Function<T, String> mapper)
+      throws IOException {
+    if (value == null) {
+      return;
+    }
+    packKeyValue(name, mapper.apply(value));
+  }
+
+  protected void packKeyValue(String name, @Nullable Collection<String> value) throws IOException {
+    if (value == null) {
+      return;
+    }
+    packer.packString(name);
+    packer.packArrayHeader(value.size());
+    for (String elem : value) {
+      packer.packString(elem);
+    }
+  }
+
+  protected <T> void packKeyValue(
+      String name, @Nullable Collection<T> value, Function<T, String> mapper) throws IOException {
+    if (value == null) {
+      return;
+    }
+    packer.packString(name);
+    packer.packArrayHeader(value.size());
+    for (T elem : value) {
+      packer.packString(mapper.apply(elem));
+    }
+  }
+
+  protected void packKeyValue(String name, @Nullable Map<String, String> value) throws IOException {
+    if (value == null) {
+      return;
+    }
+    packer.packString(name);
+    packer.packMapHeader(value.size());
+    for (Map.Entry<String, String> e : value.entrySet()) {
+      packer.packString(e.getKey());
+      packer.packString(e.getValue());
+    }
+  }
+
+  protected void packKeyValue(String name, byte @Nullable [] value) throws IOException {
+    if (value == null) {
+      return;
+    }
+    packer.packString(name);
+    packer.packBinaryHeader(value.length);
+    packer.writePayload(value);
+  }
+
+  protected void packKeyValue(String name, boolean value) throws IOException {
+    packer.packString(name);
+    packer.packBoolean(value);
+  }
+}

pkl-core/src/main/java/org/pkl/core/messaging/BaseMessagePackDecoder.java

@@ -0,0 +1,131 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.messaging;
+
+import java.io.InputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.*;
+import org.msgpack.core.MessageUnpacker;
+import org.msgpack.value.Value;
+import org.pkl.core.messaging.Message.Type;
+import org.pkl.core.messaging.Messages.*;
+import org.pkl.core.module.PathElement;
+import org.pkl.core.util.Nullable;
+
+public class BaseMessagePackDecoder extends AbstractMessagePackDecoder {
+
+  public BaseMessagePackDecoder(MessageUnpacker unpacker) {
+    super(unpacker);
+  }
+
+  public BaseMessagePackDecoder(InputStream stream) {
+    super(stream);
+  }
+
+  protected @Nullable Message decodeMessage(Type msgType, Map<Value, Value> map)
+      throws DecodeException, URISyntaxException {
+    return switch (msgType) {
+      case READ_RESOURCE_REQUEST ->
+          new ReadResourceRequest(
+              unpackLong(map, "requestId"),
+              unpackLong(map, "evaluatorId"),
+              new URI(unpackString(map, "uri")));
+      case READ_RESOURCE_RESPONSE ->
+          new ReadResourceResponse(
+              unpackLong(map, "requestId"),
+              unpackLong(map, "evaluatorId"),
+              unpackByteArray(map, "contents"),
+              unpackStringOrNull(map, "error"));
+      case READ_MODULE_REQUEST ->
+          new ReadModuleRequest(
+              unpackLong(map, "requestId"),
+              unpackLong(map, "evaluatorId"),
+              new URI(unpackString(map, "uri")));
+      case READ_MODULE_RESPONSE ->
+          new ReadModuleResponse(
+              unpackLong(map, "requestId"),
+              unpackLong(map, "evaluatorId"),
+              unpackStringOrNull(map, "contents"),
+              unpackStringOrNull(map, "error"));
+      case LIST_RESOURCES_REQUEST ->
+          new ListResourcesRequest(
+              unpackLong(map, "requestId"),
+              unpackLong(map, "evaluatorId"),
+              new URI(unpackString(map, "uri")));
+      case LIST_RESOURCES_RESPONSE ->
+          new ListResourcesResponse(
+              unpackLong(map, "requestId"),
+              unpackLong(map, "evaluatorId"),
+              unpackPathElements(map, "pathElements"),
+              unpackStringOrNull(map, "error"));
+      case LIST_MODULES_REQUEST ->
+          new ListModulesRequest(
+              unpackLong(map, "requestId"),
+              unpackLong(map, "evaluatorId"),
+              new URI(unpackString(map, "uri")));
+      case LIST_MODULES_RESPONSE ->
+          new ListModulesResponse(
+              unpackLong(map, "requestId"),
+              unpackLong(map, "evaluatorId"),
+              unpackPathElements(map, "pathElements"),
+              unpackStringOrNull(map, "error"));
+      default -> null;
+    };
+  }
+
+  protected static @Nullable ModuleReaderSpec unpackModuleReaderSpec(@Nullable Value value)
+      throws DecodeException {
+    if (value == null) {
+      return null;
+    }
+    var map = value.asMapValue().map();
+    return new ModuleReaderSpec(
+        unpackString(map, "scheme"),
+        unpackBoolean(map, "hasHierarchicalUris"),
+        unpackBoolean(map, "isLocal"),
+        unpackBoolean(map, "isGlobbable"));
+  }
+
+  protected static @Nullable ResourceReaderSpec unpackResourceReaderSpec(@Nullable Value value)
+      throws DecodeException {
+    if (value == null) {
+      return null;
+    }
+    var map = value.asMapValue().map();
+    return new ResourceReaderSpec(
+        unpackString(map, "scheme"),
+        unpackBoolean(map, "hasHierarchicalUris"),
+        unpackBoolean(map, "isGlobbable"));
+  }
+
+  protected static @Nullable List<PathElement> unpackPathElements(Map<Value, Value> map, String key)
+      throws DecodeException {
+    var value = getNullable(map, key);
+    if (value == null) {
+      return null;
+    }
+
+    var result = new ArrayList<PathElement>(value.asArrayValue().size());
+    for (Value pathElement : value.asArrayValue()) {
+      var pathElementMap = pathElement.asMapValue().map();
+      result.add(
+          new PathElement(
+              unpackString(pathElementMap, "name"), unpackBoolean(pathElementMap, "isDirectory")));
+    }
+    return result;
+  }
+}

pkl-core/src/main/java/org/pkl/core/messaging/BaseMessagePackEncoder.java

@@ -0,0 +1,136 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.messaging;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import org.msgpack.core.MessagePacker;
+import org.pkl.core.messaging.Messages.*;
+import org.pkl.core.module.PathElement;
+import org.pkl.core.util.ErrorMessages;
+import org.pkl.core.util.Nullable;
+
+public class BaseMessagePackEncoder extends AbstractMessagePackEncoder {
+
+  public BaseMessagePackEncoder(MessagePacker packer) {
+    super(packer);
+  }
+
+  public BaseMessagePackEncoder(OutputStream stream) {
+    super(stream);
+  }
+
+  protected void packModuleReaderSpec(ModuleReaderSpec reader) throws IOException {
+    packer.packMapHeader(4);
+    packKeyValue("scheme", reader.scheme());
+    packKeyValue("hasHierarchicalUris", reader.hasHierarchicalUris());
+    packKeyValue("isLocal", reader.isLocal());
+    packKeyValue("isGlobbable", reader.isGlobbable());
+  }
+
+  protected void packResourceReaderSpec(ResourceReaderSpec reader) throws IOException {
+    packer.packMapHeader(3);
+    packKeyValue("scheme", reader.scheme());
+    packKeyValue("hasHierarchicalUris", reader.hasHierarchicalUris());
+    packKeyValue("isGlobbable", reader.isGlobbable());
+  }
+
+  protected void packPathElement(PathElement pathElement) throws IOException {
+    packer.packMapHeader(2);
+    packKeyValue("name", pathElement.getName());
+    packKeyValue("isDirectory", pathElement.isDirectory());
+  }
+
+  protected @Nullable void encodeMessage(Message msg) throws ProtocolException, IOException {
+    switch (msg.type()) {
+      case READ_RESOURCE_REQUEST -> {
+        var m = (ReadResourceRequest) msg;
+        packer.packMapHeader(3);
+        packKeyValue("requestId", m.requestId());
+        packKeyValue("evaluatorId", m.evaluatorId());
+        packKeyValue("uri", m.uri().toString());
+      }
+      case READ_RESOURCE_RESPONSE -> {
+        var m = (ReadResourceResponse) msg;
+        packMapHeader(2, m.contents(), m.error());
+        packKeyValue("requestId", m.requestId());
+        packKeyValue("evaluatorId", m.evaluatorId());
+        packKeyValue("contents", m.contents());
+        packKeyValue("error", m.error());
+      }
+      case READ_MODULE_REQUEST -> {
+        var m = (ReadModuleRequest) msg;
+        packer.packMapHeader(3);
+        packKeyValue("requestId", m.requestId());
+        packKeyValue("evaluatorId", m.evaluatorId());
+        packKeyValue("uri", m.uri().toString());
+      }
+      case READ_MODULE_RESPONSE -> {
+        var m = (ReadModuleResponse) msg;
+        packMapHeader(2, m.contents(), m.error());
+        packKeyValue("requestId", m.requestId());
+        packKeyValue("evaluatorId", m.evaluatorId());
+        packKeyValue("contents", m.contents());
+        packKeyValue("error", m.error());
+      }
+      case LIST_RESOURCES_REQUEST -> {
+        var m = (ListResourcesRequest) msg;
+        packer.packMapHeader(3);
+        packKeyValue("requestId", m.requestId());
+        packKeyValue("evaluatorId", m.evaluatorId());
+        packKeyValue("uri", m.uri().toString());
+      }
+      case LIST_RESOURCES_RESPONSE -> {
+        var m = (ListResourcesResponse) msg;
+        packMapHeader(2, m.pathElements(), m.error());
+        packKeyValue("requestId", m.requestId());
+        packKeyValue("evaluatorId", m.evaluatorId());
+        if (m.pathElements() != null) {
+          packer.packString("pathElements");
+          packer.packArrayHeader(m.pathElements().size());
+          for (var pathElement : m.pathElements()) {
+            packPathElement(pathElement);
+          }
+        }
+        packKeyValue("error", m.error());
+      }
+      case LIST_MODULES_REQUEST -> {
+        var m = (ListModulesRequest) msg;
+        packer.packMapHeader(3);
+        packKeyValue("requestId", m.requestId());
+        packKeyValue("evaluatorId", m.evaluatorId());
+        packKeyValue("uri", m.uri().toString());
+      }
+      case LIST_MODULES_RESPONSE -> {
+        var m = (ListModulesResponse) msg;
+        packMapHeader(2, m.pathElements(), m.error());
+        packKeyValue("requestId", m.requestId());
+        packKeyValue("evaluatorId", m.evaluatorId());
+        if (m.pathElements() != null) {
+          packer.packString("pathElements");
+          packer.packArrayHeader(m.pathElements().size());
+          for (var pathElement : m.pathElements()) {
+            packPathElement(pathElement);
+          }
+        }
+        packKeyValue("error", m.error());
+      }
+      default ->
+          throw new ProtocolException(
+              ErrorMessages.create("unhandledMessageType", msg.type().toString()));
+    }
+  }
+}

pkl-core/src/main/java/org/pkl/core/messaging/DecodeException.java

@@ -13,12 +13,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.pkl.server
+package org.pkl.core.messaging;
 
-sealed class ServerException(msg: String, cause: Throwable?) : Exception(msg, cause)
+public final class DecodeException extends ProtocolException {
 
-open class ProtocolException(msg: String, cause: Throwable? = null) : ServerException(msg, cause)
+  public DecodeException(String msg, Throwable cause) {
+    super(msg, cause);
+  }
 
-class InvalidCommandException(msg: String, cause: Throwable? = null) : ServerException(msg, cause)
-
-class DecodeException(msg: String, cause: Throwable? = null) : ProtocolException(msg, cause)
+  public DecodeException(String msg) {
+    super(msg);
+  }
+}

pkl-core/src/main/java/org/pkl/core/messaging/Message.java

@@ -0,0 +1,89 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.messaging;
+
+public interface Message {
+
+  Type type();
+
+  enum Type {
+    CREATE_EVALUATOR_REQUEST(0x20),
+    CREATE_EVALUATOR_RESPONSE(0x21),
+    CLOSE_EVALUATOR(0x22),
+    EVALUATE_REQUEST(0x23),
+    EVALUATE_RESPONSE(0x24),
+    LOG_MESSAGE(0x25),
+    READ_RESOURCE_REQUEST(0x26),
+    READ_RESOURCE_RESPONSE(0x27),
+    READ_MODULE_REQUEST(0x28),
+    READ_MODULE_RESPONSE(0x29),
+    LIST_RESOURCES_REQUEST(0x2a),
+    LIST_RESOURCES_RESPONSE(0x2b),
+    LIST_MODULES_REQUEST(0x2c),
+    LIST_MODULES_RESPONSE(0x2d),
+    INITIALIZE_MODULE_READER_REQUEST(0x2e),
+    INITIALIZE_MODULE_READER_RESPONSE(0x2f),
+    INITIALIZE_RESOURCE_READER_REQUEST(0x30),
+    INITIALIZE_RESOURCE_READER_RESPONSE(0x31),
+    CLOSE_EXTERNAL_PROCESS(0x32);
+
+    private final int code;
+
+    Type(int code) {
+      this.code = code;
+    }
+
+    public static Type fromInt(int val) throws IllegalArgumentException {
+      for (Type t : Type.values()) {
+        if (t.code == val) {
+          return t;
+        }
+      }
+
+      throw new IllegalArgumentException("Unknown Message.Type code");
+    }
+
+    public int getCode() {
+      return code;
+    }
+  }
+
+  interface OneWay extends Message {}
+
+  interface Request extends Message {
+    long requestId();
+  }
+
+  interface Response extends Message {
+    long requestId();
+  }
+
+  interface Client extends Message {
+    interface Request extends Client, Message.Request {}
+
+    interface Response extends Client, Message.Response {}
+
+    interface OneWay extends Client, Message.OneWay {}
+  }
+
+  interface Server extends Message {
+    interface Request extends Server, Message.Request {}
+
+    interface Response extends Server, Message.Response {}
+
+    interface OneWay extends Server, Message.OneWay {}
+  }
+}

pkl-core/src/main/java/org/pkl/core/messaging/MessageDecoder.java

@@ -13,9 +13,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.pkl.server
+package org.pkl.core.messaging;
+
+import java.io.IOException;
+import org.pkl.core.util.Nullable;
 
 /** Decodes a stream of messages. */
-internal interface MessageDecoder {
-  fun decode(): Message?
+public interface MessageDecoder {
+  @Nullable
+  Message decode() throws IOException, DecodeException;
 }

pkl-core/src/main/java/org/pkl/core/messaging/MessageEncoder.java

@@ -13,9 +13,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.pkl.server
+package org.pkl.core.messaging;
+
+import java.io.IOException;
 
 /** Encodes a stream of messages. */
-internal interface MessageEncoder {
-  fun encode(msg: Message)
+public interface MessageEncoder {
+  void encode(Message msg) throws IOException, ProtocolException;
 }

pkl-core/src/main/java/org/pkl/core/messaging/MessageTransport.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.messaging;
+
+import java.io.IOException;
+
+/** A bidirectional transport for sending and receiving messages. */
+public interface MessageTransport extends AutoCloseable {
+  interface OneWayHandler {
+    void handleOneWay(Message.OneWay msg) throws ProtocolException;
+  }
+
+  interface RequestHandler {
+    void handleRequest(Message.Request msg) throws ProtocolException, IOException;
+  }
+
+  interface ResponseHandler {
+    void handleResponse(Message.Response msg) throws ProtocolException;
+  }
+
+  void start(OneWayHandler oneWayHandler, RequestHandler requestHandler)
+      throws ProtocolException, IOException;
+
+  void send(Message.OneWay message) throws ProtocolException, IOException;
+
+  void send(Message.Request message, ResponseHandler responseHandler)
+      throws ProtocolException, IOException;
+
+  void send(Message.Response message) throws ProtocolException, IOException;
+
+  @Override
+  void close();
+}

pkl-core/src/main/java/org/pkl/core/messaging/MessageTransports.java

@@ -0,0 +1,197 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.messaging;
+
+import java.io.IOException;
+import java.text.MessageFormat;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import org.pkl.core.messaging.Message.OneWay;
+import org.pkl.core.messaging.Message.Response;
+import org.pkl.core.util.ErrorMessages;
+import org.pkl.core.util.Pair;
+
+/** Factory methods for creating [MessageTransport]s. */
+public class MessageTransports {
+
+  public interface Logger {
+    void log(String msg);
+  }
+
+  /** Creates a message transport that reads from [inputStream] and writes to [outputStream]. */
+  public static MessageTransport stream(
+      MessageDecoder decoder, MessageEncoder encoder, Logger logger) {
+    return new EncodingMessageTransport(decoder, encoder, logger);
+  }
+
+  /** Creates "client" and "server" transports that are directly connected to each other. */
+  public static Pair<MessageTransport, MessageTransport> direct(Logger logger) {
+    var transport1 = new DirectMessageTransport(logger);
+    var transport2 = new DirectMessageTransport(logger);
+    transport1.setOther(transport2);
+    transport2.setOther(transport1);
+    return Pair.of(transport1, transport2);
+  }
+
+  public static <T> T resolveFuture(Future<T> future) throws IOException {
+    try {
+      return future.get();
+    } catch (ExecutionException | InterruptedException e) {
+      if (e.getCause() instanceof IOException ioExc) {
+        throw ioExc;
+      } else {
+        throw new IOException("external read failure: " + e.getMessage(), e.getCause());
+      }
+    }
+  }
+
+  protected static class EncodingMessageTransport extends AbstractMessageTransport {
+
+    private final MessageDecoder decoder;
+    private final MessageEncoder encoder;
+    private volatile boolean isClosed = false;
+
+    protected EncodingMessageTransport(
+        MessageDecoder decoder, MessageEncoder encoder, Logger logger) {
+      super(logger);
+      this.decoder = decoder;
+      this.encoder = encoder;
+    }
+
+    @Override
+    protected void doStart() throws ProtocolException, IOException {
+      while (!isClosed) {
+        var message = decoder.decode();
+        if (message == null) {
+          return;
+        }
+        accept(message);
+      }
+    }
+
+    @Override
+    protected void doClose() {
+      isClosed = true;
+    }
+
+    @Override
+    protected void doSend(Message message) throws ProtocolException, IOException {
+      encoder.encode(message);
+    }
+  }
+
+  protected static class DirectMessageTransport extends AbstractMessageTransport {
+
+    private DirectMessageTransport other;
+
+    protected DirectMessageTransport(Logger logger) {
+      super(logger);
+    }
+
+    @Override
+    protected void doStart() {}
+
+    @Override
+    protected void doClose() {}
+
+    @Override
+    protected void doSend(Message message) throws ProtocolException, IOException {
+      other.accept(message);
+    }
+
+    public void setOther(DirectMessageTransport other) {
+      this.other = other;
+    }
+  }
+
+  protected abstract static class AbstractMessageTransport implements MessageTransport {
+
+    private final Logger logger;
+    private MessageTransport.OneWayHandler oneWayHandler;
+    private MessageTransport.RequestHandler requestHandler;
+    private final Map<Long, ResponseHandler> responseHandlers = new ConcurrentHashMap<>();
+
+    protected AbstractMessageTransport(Logger logger) {
+      this.logger = logger;
+    }
+
+    protected void log(String message, Object... args) {
+      var formatter = new MessageFormat(message);
+      logger.log(formatter.format(args));
+    }
+
+    protected abstract void doStart() throws ProtocolException, IOException;
+
+    protected abstract void doClose();
+
+    protected abstract void doSend(Message message) throws ProtocolException, IOException;
+
+    protected void accept(Message message) throws ProtocolException, IOException {
+      log("Received message: {0}", message);
+      if (message instanceof Message.OneWay msg) {
+        oneWayHandler.handleOneWay(msg);
+      } else if (message instanceof Message.Request msg) {
+        requestHandler.handleRequest(msg);
+      } else if (message instanceof Message.Response msg) {
+        var handler = responseHandlers.remove(msg.requestId());
+        if (handler == null) {
+          throw new ProtocolException(
+              ErrorMessages.create(
+                  "unknownRequestId", message.getClass().getSimpleName(), msg.requestId()));
+        }
+        handler.handleResponse(msg);
+      }
+    }
+
+    @Override
+    public final void start(OneWayHandler oneWayHandler, RequestHandler requestHandler)
+        throws ProtocolException, IOException {
+      log("Starting transport: {0}", this);
+      this.oneWayHandler = oneWayHandler;
+      this.requestHandler = requestHandler;
+      doStart();
+    }
+
+    @Override
+    public final void close() {
+      log("Closing transport: {0}", this);
+      doClose();
+      responseHandlers.clear();
+    }
+
+    @Override
+    public void send(OneWay message) throws ProtocolException, IOException {
+      log("Sending message: {0}", message);
+      doSend(message);
+    }
+
+    @Override
+    public void send(Message.Request message, ResponseHandler responseHandler)
+        throws ProtocolException, IOException {
+      log("Sending message: {0}", message);
+      responseHandlers.put(message.requestId(), responseHandler);
+      doSend(message);
+    }
+
+    @Override
+    public void send(Response message) throws ProtocolException, IOException {
+      log("Sending message: {0}", message);
+      doSend(message);
+    }
+  }
+}

pkl-core/src/main/java/org/pkl/core/messaging/Messages.java

@@ -0,0 +1,126 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.messaging;
+
+import java.net.URI;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import org.pkl.core.messaging.Message.*;
+import org.pkl.core.module.PathElement;
+import org.pkl.core.util.Nullable;
+
+public class Messages {
+
+  public record ModuleReaderSpec(
+      String scheme, boolean hasHierarchicalUris, boolean isLocal, boolean isGlobbable) {}
+
+  public record ResourceReaderSpec(
+      String scheme, boolean hasHierarchicalUris, boolean isGlobbable) {}
+
+  public record ListResourcesRequest(long requestId, long evaluatorId, URI uri)
+      implements Server.Request {
+    public Type type() {
+      return Type.LIST_RESOURCES_REQUEST;
+    }
+  }
+
+  public record ListResourcesResponse(
+      long requestId,
+      long evaluatorId,
+      @Nullable List<PathElement> pathElements,
+      @Nullable String error)
+      implements Client.Response {
+    public Type type() {
+      return Type.LIST_RESOURCES_RESPONSE;
+    }
+  }
+
+  public record ListModulesRequest(long requestId, long evaluatorId, URI uri)
+      implements Server.Request {
+    public Type type() {
+      return Type.LIST_MODULES_REQUEST;
+    }
+  }
+
+  public record ListModulesResponse(
+      long requestId,
+      long evaluatorId,
+      @Nullable List<PathElement> pathElements,
+      @Nullable String error)
+      implements Client.Response {
+    public Type type() {
+      return Type.LIST_MODULES_RESPONSE;
+    }
+  }
+
+  public record ReadResourceRequest(long requestId, long evaluatorId, URI uri)
+      implements Message.Request {
+    public Type type() {
+      return Type.READ_RESOURCE_REQUEST;
+    }
+  }
+
+  public record ReadResourceResponse(
+      long requestId, long evaluatorId, byte @Nullable [] contents, @Nullable String error)
+      implements Client.Response {
+
+    // workaround for kotlin bridging issue where `byte @Nullable [] contents` isn't detected as
+    // nullable
+    //    public ReadResourceResponse(long requestId, long evaluatorId, @Nullable String error) {
+    //      this(requestId, evaluatorId, null, error);
+    //    }
+
+    public Type type() {
+      return Type.READ_RESOURCE_RESPONSE;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (this == o) {
+        return true;
+      }
+      if (o == null || getClass() != o.getClass()) {
+        return false;
+      }
+      ReadResourceResponse that = (ReadResourceResponse) o;
+      return requestId == that.requestId
+          && evaluatorId == that.evaluatorId
+          && Objects.equals(error, that.error)
+          && Arrays.equals(contents, that.contents);
+    }
+
+    @Override
+    public int hashCode() {
+      return Objects.hash(requestId, evaluatorId, Arrays.hashCode(contents), error);
+    }
+  }
+
+  public record ReadModuleRequest(long requestId, long evaluatorId, URI uri)
+      implements Message.Request {
+    public Type type() {
+      return Type.READ_MODULE_REQUEST;
+    }
+  }
+
+  public record ReadModuleResponse(
+      long requestId, long evaluatorId, @Nullable String contents, @Nullable String error)
+      implements Client.Response {
+    public Type type() {
+      return Type.READ_MODULE_RESPONSE;
+    }
+  }
+}

pkl-core/src/main/java/org/pkl/core/messaging/ProtocolException.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.messaging;
+
+public class ProtocolException extends Exception {
+  public ProtocolException(String msg, Throwable cause) {
+    super(msg, cause);
+  }
+
+  public ProtocolException(String msg) {
+    super(msg);
+  }
+}

pkl-core/src/main/java/org/pkl/core/messaging/package-info.java

@@ -0,0 +1,4 @@
+@NonnullByDefault
+package org.pkl.core.messaging;
+
+import org.pkl.core.util.NonnullByDefault;

pkl-core/src/main/java/org/pkl/core/module/ExternalModuleResolver.java

@@ -0,0 +1,129 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.module;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Random;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Future;
+import org.pkl.core.SecurityManager;
+import org.pkl.core.SecurityManagerException;
+import org.pkl.core.messaging.MessageTransport;
+import org.pkl.core.messaging.MessageTransports;
+import org.pkl.core.messaging.Messages.ListModulesRequest;
+import org.pkl.core.messaging.Messages.ListModulesResponse;
+import org.pkl.core.messaging.Messages.ReadModuleRequest;
+import org.pkl.core.messaging.Messages.ReadModuleResponse;
+import org.pkl.core.messaging.ProtocolException;
+
+public class ExternalModuleResolver {
+  private final MessageTransport transport;
+  private final long evaluatorId;
+  private final Map<URI, Future<String>> readResponses = new ConcurrentHashMap<>();
+  private final Map<URI, Future<List<PathElement>>> listResponses = new ConcurrentHashMap<>();
+
+  public ExternalModuleResolver(MessageTransport transport, long evaluatorId) {
+    this.transport = transport;
+    this.evaluatorId = evaluatorId;
+  }
+
+  public List<PathElement> listElements(SecurityManager securityManager, URI uri)
+      throws IOException, SecurityManagerException {
+    securityManager.checkResolveModule(uri);
+    return doListElements(uri);
+  }
+
+  public boolean hasElement(SecurityManager securityManager, URI uri)
+      throws SecurityManagerException {
+    securityManager.checkResolveModule(uri);
+    try {
+      doReadModule(uri);
+      return true;
+    } catch (IOException e) {
+      return false;
+    }
+  }
+
+  public String resolveModule(SecurityManager securityManager, URI uri)
+      throws IOException, SecurityManagerException {
+    securityManager.checkResolveModule(uri);
+    return doReadModule(uri);
+  }
+
+  private String doReadModule(URI moduleUri) throws IOException {
+    return MessageTransports.resolveFuture(
+        readResponses.computeIfAbsent(
+            moduleUri,
+            (uri) -> {
+              var future = new CompletableFuture<String>();
+              var request = new ReadModuleRequest(new Random().nextLong(), evaluatorId, uri);
+              try {
+                transport.send(
+                    request,
+                    (response) -> {
+                      if (response instanceof ReadModuleResponse resp) {
+                        if (resp.error() != null) {
+                          future.completeExceptionally(new IOException(resp.error()));
+                        } else if (resp.contents() != null) {
+                          future.complete(resp.contents());
+                        } else {
+                          future.complete("");
+                        }
+                      } else {
+                        future.completeExceptionally(new ProtocolException("unexpected response"));
+                      }
+                    });
+              } catch (ProtocolException | IOException e) {
+                future.completeExceptionally(e);
+              }
+              return future;
+            }));
+  }
+
+  private List<PathElement> doListElements(URI baseUri) throws IOException {
+    return MessageTransports.resolveFuture(
+        listResponses.computeIfAbsent(
+            baseUri,
+            (uri) -> {
+              var future = new CompletableFuture<List<PathElement>>();
+              var request = new ListModulesRequest(new Random().nextLong(), evaluatorId, uri);
+              try {
+                transport.send(
+                    request,
+                    (response) -> {
+                      if (response instanceof ListModulesResponse resp) {
+                        if (resp.error() != null) {
+                          future.completeExceptionally(new IOException(resp.error()));
+                        } else {
+                          future.complete(
+                              Objects.requireNonNullElseGet(resp.pathElements(), List::of));
+                        }
+                      } else {
+                        future.completeExceptionally(new ProtocolException("unexpected response"));
+                      }
+                    });
+              } catch (ProtocolException | IOException e) {
+                future.completeExceptionally(e);
+              }
+              return future;
+            }));
+  }
+}

pkl-core/src/main/java/org/pkl/core/module/ModuleKeyFactories.java

@@ -15,6 +15,7 @@
  */
 package org.pkl.core.module;
 
+import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.file.FileSystemNotFoundException;
@@ -25,6 +26,10 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 import java.util.ServiceLoader;
+import javax.annotation.concurrent.GuardedBy;
+import org.pkl.core.externalreader.ExternalReaderProcess;
+import org.pkl.core.externalreader.ExternalReaderProcessException;
+import org.pkl.core.util.ErrorMessages;
 import org.pkl.core.util.IoUtils;
 
 /** Utilities for obtaining and using module key factories. */
@@ -72,7 +77,27 @@ public final class ModuleKeyFactories {
     return new ClassPath(classLoader);
   }
 
-  /** Closes the given factories, ignoring any exceptions. */
+  /**
+   * Returns a factory for external reader module keys
+   *
+   * <p>NOTE: {@code process} needs to be {@link ExternalReaderProcess#close closed} to avoid
+   * resource leaks.
+   */
+  public static ModuleKeyFactory externalProcess(String scheme, ExternalReaderProcess process) {
+    return new ExternalProcess(scheme, process, 0);
+  }
+
+  public static ModuleKeyFactory externalProcess(
+      String scheme, ExternalReaderProcess process, long evaluatorId) {
+    return new ExternalProcess(scheme, process, evaluatorId);
+  }
+
+  /**
+   * Closes the given factories, ignoring any exceptions.
+   *
+   * @deprecated Replaced by {@link org.pkl.core.util.Readers#closeQuietly}.
+   */
+  @Deprecated(since = "0.27.0", forRemoval = true)
   public static void closeQuietly(Iterable<ModuleKeyFactory> factories) {
     for (ModuleKeyFactory factory : factories) {
       try {
@@ -225,4 +250,48 @@ public final class ModuleKeyFactories {
       INSTANCE = Collections.unmodifiableList(factories);
     }
   }
+
+  /** Represents a module from an external reader process. */
+  private static final class ExternalProcess implements ModuleKeyFactory {
+    private final String scheme;
+    private final ExternalReaderProcess process;
+    private final long evaluatorId;
+
+    @GuardedBy("this")
+    private ExternalModuleResolver resolver;
+
+    public ExternalProcess(String scheme, ExternalReaderProcess process, long evaluatorId) {
+      this.scheme = scheme;
+      this.process = process;
+      this.evaluatorId = evaluatorId;
+    }
+
+    private synchronized ExternalModuleResolver getResolver()
+        throws ExternalReaderProcessException {
+      if (resolver != null) {
+        return resolver;
+      }
+
+      resolver = new ExternalModuleResolver(process.getTransport(), evaluatorId);
+      return resolver;
+    }
+
+    public Optional<ModuleKey> create(URI uri)
+        throws URISyntaxException, ExternalReaderProcessException, IOException {
+      if (!scheme.equalsIgnoreCase(uri.getScheme())) return Optional.empty();
+
+      var spec = process.getModuleReaderSpec(scheme);
+      if (spec == null) {
+        throw new ExternalReaderProcessException(
+            ErrorMessages.create("externalReaderDoesNotSupportScheme", "module", scheme));
+      }
+
+      return Optional.of(ModuleKeys.externalResolver(uri, spec, getResolver()));
+    }
+
+    @Override
+    public void close() {
+      process.close();
+    }
+  }
 }

pkl-core/src/main/java/org/pkl/core/module/ModuleKeyFactory.java

@@ -15,9 +15,11 @@
  */
 package org.pkl.core.module;
 
+import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.Optional;
+import org.pkl.core.externalreader.ExternalReaderProcessException;
 
 /** A factory for {@link ModuleKey}s. */
 public interface ModuleKeyFactory extends AutoCloseable {
@@ -35,7 +37,8 @@ public interface ModuleKeyFactory extends AutoCloseable {
    * @param uri an absolute normalized URI
    * @return a module key for the given URI
    */
-  Optional<ModuleKey> create(URI uri) throws URISyntaxException;
+  Optional<ModuleKey> create(URI uri)
+      throws URISyntaxException, ExternalReaderProcessException, IOException;
 
   /**
    * Closes this factory, releasing any resources held. See the documentation of factory methods in

pkl-core/src/main/java/org/pkl/core/module/ModuleKeys.java

@@ -29,6 +29,8 @@ import java.util.List;
 import java.util.Map;
 import org.pkl.core.SecurityManager;
 import org.pkl.core.SecurityManagerException;
+import org.pkl.core.externalreader.ExternalReaderProcessException;
+import org.pkl.core.messaging.Messages.*;
 import org.pkl.core.packages.Dependency;
 import org.pkl.core.packages.Dependency.LocalDependency;
 import org.pkl.core.packages.PackageAssetUri;
@@ -127,6 +129,12 @@ public final class ModuleKeys {
     return new ProjectPackage(assetUri);
   }
 
+  /** Creates a module key for an externally read module. */
+  public static ModuleKey externalResolver(
+      URI uri, ModuleReaderSpec spec, ExternalModuleResolver resolver) throws URISyntaxException {
+    return new ExternalResolver(uri, spec, resolver);
+  }
+
   /**
    * Creates a module key that behaves like {@code delegate}, except that it returns {@code text} as
    * its loaded source.
@@ -165,7 +173,7 @@ public final class ModuleKeys {
     }
 
     @Override
-    public boolean hasHierarchicalUris() {
+    public boolean hasHierarchicalUris() throws IOException, ExternalReaderProcessException {
       return delegate.hasHierarchicalUris();
     }
 
@@ -175,19 +183,19 @@ public final class ModuleKeys {
     }
 
     @Override
-    public boolean isGlobbable() {
+    public boolean isGlobbable() throws IOException, ExternalReaderProcessException {
       return delegate.isGlobbable();
     }
 
     @Override
     public boolean hasElement(SecurityManager securityManager, URI uri)
-        throws IOException, SecurityManagerException {
+        throws IOException, SecurityManagerException, ExternalReaderProcessException {
       return delegate.hasElement(securityManager, uri);
     }
 
     @Override
     public List<PathElement> listElements(SecurityManager securityManager, URI baseUri)
-        throws IOException, SecurityManagerException {
+        throws IOException, SecurityManagerException, ExternalReaderProcessException {
       return delegate.listElements(securityManager, baseUri);
     }
   }
@@ -397,7 +405,6 @@ public final class ModuleKeys {
   }
 
   private static final class ClassPath implements ModuleKey {
-
     final URI uri;
 
     final ClassLoader classLoader;
@@ -460,7 +467,6 @@ public final class ModuleKeys {
   }
 
   private static class Http implements ModuleKey {
-
     private final URI uri;
 
     Http(URI uri) {
@@ -550,7 +556,6 @@ public final class ModuleKeys {
   }
 
   private abstract static class AbstractPackage implements ModuleKey {
-
     protected final PackageAssetUri packageAssetUri;
 
     AbstractPackage(PackageAssetUri packageAssetUri) {
@@ -663,6 +668,7 @@ public final class ModuleKeys {
    * an internal implementation detail, and we do not expect a module to declare this.
    */
   public static class ProjectPackage extends AbstractPackage {
+
     ProjectPackage(PackageAssetUri packageAssetUri) {
       super(packageAssetUri);
     }
@@ -712,7 +718,7 @@ public final class ModuleKeys {
 
     @Override
     public List<PathElement> listElements(SecurityManager securityManager, URI baseUri)
-        throws IOException, SecurityManagerException {
+        throws IOException, SecurityManagerException, ExternalReaderProcessException {
       securityManager.checkResolveModule(baseUri);
       var packageAssetUri = PackageAssetUri.create(baseUri);
       var dependency =
@@ -733,7 +739,7 @@ public final class ModuleKeys {
 
     @Override
     public boolean hasElement(SecurityManager securityManager, URI elementUri)
-        throws IOException, SecurityManagerException {
+        throws IOException, SecurityManagerException, ExternalReaderProcessException {
       securityManager.checkResolveModule(elementUri);
       var packageAssetUri = PackageAssetUri.create(elementUri);
       var dependency =
@@ -769,4 +775,56 @@ public final class ModuleKeys {
       return projectResolver.getResolvedDependenciesForPackage(packageUri, dependencyMetadata);
     }
   }
+
+  public static class ExternalResolver implements ModuleKey {
+
+    private final URI uri;
+    private final ModuleReaderSpec spec;
+    private final ExternalModuleResolver resolver;
+
+    public ExternalResolver(URI uri, ModuleReaderSpec spec, ExternalModuleResolver resolver) {
+      this.uri = uri;
+      this.spec = spec;
+      this.resolver = resolver;
+    }
+
+    @Override
+    public boolean isLocal() {
+      return spec.isLocal();
+    }
+
+    @Override
+    public boolean hasHierarchicalUris() {
+      return spec.hasHierarchicalUris();
+    }
+
+    @Override
+    public boolean isGlobbable() {
+      return spec.isGlobbable();
+    }
+
+    @Override
+    public URI getUri() {
+      return uri;
+    }
+
+    @Override
+    public List<PathElement> listElements(SecurityManager securityManager, URI baseUri)
+        throws IOException, SecurityManagerException {
+      return resolver.listElements(securityManager, baseUri);
+    }
+
+    @Override
+    public ResolvedModuleKey resolve(SecurityManager securityManager)
+        throws IOException, SecurityManagerException {
+      var contents = resolver.resolveModule(securityManager, uri);
+      return ResolvedModuleKeys.virtual(this, uri, contents, true);
+    }
+
+    @Override
+    public boolean hasElement(SecurityManager securityManager, URI elementUri)
+        throws IOException, SecurityManagerException {
+      return resolver.hasElement(securityManager, elementUri);
+    }
+  }
 }

pkl-core/src/main/java/org/pkl/core/packages/Checksums.java

@@ -16,6 +16,7 @@
 package org.pkl.core.packages;
 
 import java.util.Objects;
+import org.pkl.core.util.Nullable;
 
 public final class Checksums {
   private final String sha256;
@@ -34,7 +35,7 @@ public final class Checksums {
   }
 
   @Override
-  public boolean equals(Object o) {
+  public boolean equals(@Nullable Object o) {
     if (this == o) {
       return true;
     }

pkl-core/src/main/java/org/pkl/core/project/Project.java

@@ -522,6 +522,8 @@ public final class Project {
               modulePath,
               timeout,
               rootDir,
+              null,
+              null,
               null);
     }
 

pkl-core/src/main/java/org/pkl/core/resource/ExternalResourceResolver.java

@@ -0,0 +1,127 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.resource;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Random;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Future;
+import org.pkl.core.SecurityManager;
+import org.pkl.core.SecurityManagerException;
+import org.pkl.core.messaging.MessageTransport;
+import org.pkl.core.messaging.MessageTransports;
+import org.pkl.core.messaging.Messages.*;
+import org.pkl.core.messaging.ProtocolException;
+import org.pkl.core.module.PathElement;
+
+public class ExternalResourceResolver {
+  private final MessageTransport transport;
+  private final long evaluatorId;
+  private final Map<URI, Future<byte[]>> readResponses = new ConcurrentHashMap<>();
+  private final Map<URI, Future<List<PathElement>>> listResponses = new ConcurrentHashMap<>();
+
+  public ExternalResourceResolver(MessageTransport transport, long evaluatorId) {
+    this.transport = transport;
+    this.evaluatorId = evaluatorId;
+  }
+
+  public Optional<Object> read(URI uri) throws IOException {
+    var result = doRead(uri);
+    return Optional.of(new Resource(uri, result));
+  }
+
+  public boolean hasElement(org.pkl.core.SecurityManager securityManager, URI elementUri)
+      throws SecurityManagerException {
+    securityManager.checkResolveResource(elementUri);
+    try {
+      doRead(elementUri);
+      return true;
+    } catch (IOException e) {
+      return false;
+    }
+  }
+
+  public List<PathElement> listElements(SecurityManager securityManager, URI baseUri)
+      throws IOException, SecurityManagerException {
+    securityManager.checkResolveResource(baseUri);
+    return doListElements(baseUri);
+  }
+
+  public List<PathElement> doListElements(URI baseUri) throws IOException {
+    return MessageTransports.resolveFuture(
+        listResponses.computeIfAbsent(
+            baseUri,
+            (uri) -> {
+              var future = new CompletableFuture<List<PathElement>>();
+              var request = new ListResourcesRequest(new Random().nextLong(), evaluatorId, uri);
+              try {
+                transport.send(
+                    request,
+                    (response) -> {
+                      if (response instanceof ListResourcesResponse resp) {
+                        if (resp.error() != null) {
+                          future.completeExceptionally(new IOException(resp.error()));
+                        } else {
+                          future.complete(
+                              Objects.requireNonNullElseGet(resp.pathElements(), List::of));
+                        }
+                      } else {
+                        future.completeExceptionally(new ProtocolException("unexpected response"));
+                      }
+                    });
+              } catch (ProtocolException | IOException e) {
+                future.completeExceptionally(e);
+              }
+              return future;
+            }));
+  }
+
+  public byte[] doRead(URI baseUri) throws IOException {
+    return MessageTransports.resolveFuture(
+        readResponses.computeIfAbsent(
+            baseUri,
+            (uri) -> {
+              var future = new CompletableFuture<byte[]>();
+              var request = new ReadResourceRequest(new Random().nextLong(), evaluatorId, uri);
+              try {
+                transport.send(
+                    request,
+                    (response) -> {
+                      if (response instanceof ReadResourceResponse resp) {
+                        if (resp.error() != null) {
+                          future.completeExceptionally(new IOException(resp.error()));
+                        } else if (resp.contents() != null) {
+                          future.complete(resp.contents());
+                        } else {
+                          future.complete(new byte[0]);
+                        }
+                      } else {
+                        future.completeExceptionally(new ProtocolException("unexpected response"));
+                      }
+                    });
+              } catch (ProtocolException | IOException e) {
+                future.completeExceptionally(e);
+              }
+              return future;
+            }));
+  }
+}

pkl-core/src/main/java/org/pkl/core/resource/ResourceReader.java

@@ -20,6 +20,7 @@ import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.Optional;
 import org.pkl.core.SecurityManagerException;
+import org.pkl.core.externalreader.ExternalReaderProcessException;
 import org.pkl.core.runtime.ReaderBase;
 
 /**
@@ -29,7 +30,7 @@ import org.pkl.core.runtime.ReaderBase;
  *
  * <p>See {@link ResourceReaders} for predefined resource readers.
  */
-public interface ResourceReader extends ReaderBase {
+public interface ResourceReader extends ReaderBase, AutoCloseable {
   /** The URI scheme associated with resources read by this resource reader. */
   String getUriScheme();
 
@@ -54,5 +55,16 @@ public interface ResourceReader extends ReaderBase {
    *       manager.
    * </ul>
    */
-  Optional<Object> read(URI uri) throws IOException, URISyntaxException, SecurityManagerException;
+  Optional<Object> read(URI uri)
+      throws IOException,
+          URISyntaxException,
+          SecurityManagerException,
+          ExternalReaderProcessException;
+
+  /**
+   * Closes this reader, releasing any resources held. See the documentation of factory methods in
+   * {@link ResourceReaders} for which factories need to be closed.
+   */
+  @Override
+  default void close() {}
 }

pkl-core/src/main/java/org/pkl/core/resource/ResourceReaders.java

@@ -29,6 +29,9 @@ import java.util.Optional;
 import java.util.ServiceLoader;
 import org.pkl.core.SecurityManager;
 import org.pkl.core.SecurityManagerException;
+import org.pkl.core.externalreader.ExternalReaderProcess;
+import org.pkl.core.externalreader.ExternalReaderProcessException;
+import org.pkl.core.messaging.Messages.*;
 import org.pkl.core.module.FileResolver;
 import org.pkl.core.module.ModulePathResolver;
 import org.pkl.core.module.PathElement;
@@ -137,6 +140,21 @@ public final class ResourceReaders {
     return FromServiceProviders.INSTANCE;
   }
 
+  public static ResourceReader externalProcess(
+      String scheme, ExternalReaderProcess externalReaderProcess) {
+    return new ExternalProcess(scheme, externalReaderProcess, 0);
+  }
+
+  public static ResourceReader externalProcess(
+      String scheme, ExternalReaderProcess externalReaderProcess, long evaluatorId) {
+    return new ExternalProcess(scheme, externalReaderProcess, evaluatorId);
+  }
+
+  public static ResourceReader externalResolver(
+      ResourceReaderSpec spec, ExternalResourceResolver resolver) {
+    return new ExternalResolver(spec, resolver);
+  }
+
   private static final class EnvironmentVariable implements ResourceReader {
     static final ResourceReader INSTANCE = new EnvironmentVariable();
 
@@ -521,7 +539,7 @@ public final class ResourceReaders {
 
     @Override
     public List<PathElement> listElements(SecurityManager securityManager, URI baseUri)
-        throws IOException, SecurityManagerException {
+        throws IOException, SecurityManagerException, ExternalReaderProcessException {
       securityManager.checkResolveResource(baseUri);
       var packageAssetUri = PackageAssetUri.create(baseUri);
       var dependency =
@@ -543,7 +561,7 @@ public final class ResourceReaders {
 
     @Override
     public boolean hasElement(SecurityManager securityManager, URI elementUri)
-        throws IOException, SecurityManagerException {
+        throws IOException, SecurityManagerException, ExternalReaderProcessException {
       securityManager.checkResolveResource(elementUri);
       var packageAssetUri = PackageAssetUri.create(elementUri);
       var dependency =
@@ -585,6 +603,7 @@ public final class ResourceReaders {
   }
 
   private static class FromServiceProviders {
+
     private static final List<ResourceReader> INSTANCE;
 
     static {
@@ -594,4 +613,113 @@ public final class ResourceReaders {
       INSTANCE = Collections.unmodifiableList(readers);
     }
   }
+
+  private static final class ExternalProcess implements ResourceReader {
+    private final String scheme;
+    private final ExternalReaderProcess process;
+    private final long evaluatorId;
+    private ExternalResolver underlying;
+
+    public ExternalProcess(String scheme, ExternalReaderProcess process, long evaluatorId) {
+      this.scheme = scheme;
+      this.process = process;
+      this.evaluatorId = evaluatorId;
+    }
+
+    private ExternalResolver getUnderlyingReader()
+        throws ExternalReaderProcessException, IOException {
+      if (underlying != null) {
+        return underlying;
+      }
+
+      var spec = process.getResourceReaderSpec(scheme);
+      if (spec == null) {
+        throw new ExternalReaderProcessException(
+            ErrorMessages.create("externalReaderDoesNotSupportScheme", "resource", scheme));
+      }
+      underlying =
+          new ExternalResolver(
+              spec, new ExternalResourceResolver(process.getTransport(), evaluatorId));
+      return underlying;
+    }
+
+    @Override
+    public String getUriScheme() {
+      return scheme;
+    }
+
+    @Override
+    public boolean hasHierarchicalUris() throws ExternalReaderProcessException, IOException {
+      return getUnderlyingReader().hasHierarchicalUris();
+    }
+
+    @Override
+    public boolean isGlobbable() throws ExternalReaderProcessException, IOException {
+      return getUnderlyingReader().isGlobbable();
+    }
+
+    @Override
+    public Optional<Object> read(URI uri) throws IOException, ExternalReaderProcessException {
+      return getUnderlyingReader().read(uri);
+    }
+
+    @Override
+    public boolean hasElement(SecurityManager securityManager, URI elementUri)
+        throws IOException, SecurityManagerException, ExternalReaderProcessException {
+      return getUnderlyingReader().hasElement(securityManager, elementUri);
+    }
+
+    @Override
+    public List<PathElement> listElements(SecurityManager securityManager, URI baseUri)
+        throws IOException, SecurityManagerException, ExternalReaderProcessException {
+      return getUnderlyingReader().listElements(securityManager, baseUri);
+    }
+
+    @Override
+    public void close() {
+      process.close();
+    }
+  }
+
+  private static final class ExternalResolver implements ResourceReader {
+    private final ResourceReaderSpec readerSpec;
+    private final ExternalResourceResolver resolver;
+
+    public ExternalResolver(ResourceReaderSpec readerSpec, ExternalResourceResolver resolver) {
+      this.readerSpec = readerSpec;
+      this.resolver = resolver;
+    }
+
+    @Override
+    public boolean hasHierarchicalUris() {
+      return readerSpec.hasHierarchicalUris();
+    }
+
+    @Override
+    public boolean isGlobbable() {
+      return readerSpec.isGlobbable();
+    }
+
+    @Override
+    public String getUriScheme() {
+      return readerSpec.scheme();
+    }
+
+    @Override
+    public Optional<Object> read(URI uri) throws IOException {
+      return resolver.read(uri);
+    }
+
+    @Override
+    public boolean hasElement(org.pkl.core.SecurityManager securityManager, URI elementUri)
+        throws SecurityManagerException {
+      return resolver.hasElement(securityManager, elementUri);
+    }
+
+    @Override
+    public List<PathElement> listElements(SecurityManager securityManager, URI baseUri)
+        throws IOException, SecurityManagerException {
+      return resolver.listElements(securityManager, baseUri);
+    }
+  }
 }

pkl-core/src/main/java/org/pkl/core/runtime/ModuleResolver.java

@@ -16,11 +16,13 @@
 package org.pkl.core.runtime;
 
 import com.oracle.truffle.api.nodes.Node;
+import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.Collection;
 import java.util.Optional;
 import org.pkl.core.ModuleSource;
+import org.pkl.core.externalreader.ExternalReaderProcessException;
 import org.pkl.core.module.ModuleKey;
 import org.pkl.core.module.ModuleKeyFactory;
 import org.pkl.core.module.ModuleKeys;
@@ -82,6 +84,18 @@ public final class ModuleResolver {
             .evalError("invalidModuleUri", moduleUri)
             .withHint(e.getReason())
             .build();
+      } catch (ExternalReaderProcessException e) {
+        throw new VmExceptionBuilder()
+            .withOptionalLocation(importNode)
+            .evalError("externalReaderFailure")
+            .withCause(e)
+            .build();
+      } catch (IOException e) {
+        throw new VmExceptionBuilder()
+            .withOptionalLocation(importNode)
+            .evalError("ioErrorLoadingModule")
+            .withCause(e)
+            .build();
       }
       if (key.isPresent()) return key.get();
     }

pkl-core/src/main/java/org/pkl/core/runtime/ReaderBase.java

@@ -20,6 +20,7 @@ import java.net.URI;
 import java.util.List;
 import org.pkl.core.SecurityManager;
 import org.pkl.core.SecurityManagerException;
+import org.pkl.core.externalreader.ExternalReaderProcessException;
 import org.pkl.core.module.ModuleKey;
 import org.pkl.core.module.PathElement;
 import org.pkl.core.util.IoUtils;
@@ -29,10 +30,10 @@ public interface ReaderBase {
    * Tells if the URIs represented by this module key or resource reader should be interpreted as <a
    * href="https://www.rfc-editor.org/rfc/rfc3986#section-1.2.3">hierarchical</a>.
    */
-  boolean hasHierarchicalUris();
+  boolean hasHierarchicalUris() throws ExternalReaderProcessException, IOException;
 
   /** Tells if this module key or resource reader supports globbing. */
-  boolean isGlobbable();
+  boolean isGlobbable() throws ExternalReaderProcessException, IOException;
 
   /**
    * Tells if relative paths of this URI should be resolved from {@link URI#getFragment()}, rather
@@ -49,7 +50,7 @@ public interface ReaderBase {
    * if either {@link #isGlobbable()} or {@link ModuleKey#isLocal()} returns true.
    */
   default boolean hasElement(SecurityManager securityManager, URI elementUri)
-      throws IOException, SecurityManagerException {
+      throws IOException, SecurityManagerException, ExternalReaderProcessException {
     throw new UnsupportedOperationException();
   }
 
@@ -66,7 +67,7 @@ public interface ReaderBase {
    * this reader.
    */
   default List<PathElement> listElements(SecurityManager securityManager, URI baseUri)
-      throws IOException, SecurityManagerException {
+      throws IOException, SecurityManagerException, ExternalReaderProcessException {
     throw new UnsupportedOperationException();
   }
 

pkl-core/src/main/java/org/pkl/core/runtime/ResourceManager.java

@@ -26,6 +26,7 @@ import java.util.Map;
 import java.util.Optional;
 import org.pkl.core.SecurityManager;
 import org.pkl.core.SecurityManagerException;
+import org.pkl.core.externalreader.ExternalReaderProcessException;
 import org.pkl.core.http.HttpClientInitException;
 import org.pkl.core.packages.PackageLoadError;
 import org.pkl.core.resource.Resource;
@@ -83,7 +84,10 @@ public final class ResourceManager {
           .withHint(e.getReason())
           .withOptionalLocation(readNode)
           .build();
-    } catch (SecurityManagerException | PackageLoadError | HttpClientInitException e) {
+    } catch (SecurityManagerException
+        | PackageLoadError
+        | HttpClientInitException
+        | ExternalReaderProcessException e) {
       throw new VmExceptionBuilder().withCause(e).withOptionalLocation(readNode).build();
     }
     return resource;

pkl-core/src/main/java/org/pkl/core/runtime/VmImportAnalyzer.java

@@ -30,6 +30,7 @@ import org.pkl.core.SecurityManager;
 import org.pkl.core.SecurityManagerException;
 import org.pkl.core.ast.builder.ImportsAndReadsParser;
 import org.pkl.core.ast.builder.ImportsAndReadsParser.Entry;
+import org.pkl.core.externalreader.ExternalReaderProcessException;
 import org.pkl.core.util.GlobResolver;
 import org.pkl.core.util.GlobResolver.InvalidGlobPatternException;
 import org.pkl.core.util.GlobResolver.ResolvedGlobElement;
@@ -38,7 +39,10 @@ import org.pkl.core.util.IoUtils;
 public class VmImportAnalyzer {
   @TruffleBoundary
   public static ImportGraph analyze(URI[] moduleUris, VmContext context)
-      throws IOException, URISyntaxException, SecurityManagerException {
+      throws IOException,
+          URISyntaxException,
+          SecurityManagerException,
+          ExternalReaderProcessException {
     var imports = new TreeMap<URI, Set<ImportGraph.Import>>();
     var resolvedImports = new TreeMap<URI, URI>();
     for (var moduleUri : moduleUris) {
@@ -53,7 +57,10 @@ public class VmImportAnalyzer {
       VmContext context,
       Map<URI, Set<ImportGraph.Import>> imports,
       Map<URI, URI> resolvedImports)
-      throws IOException, URISyntaxException, SecurityManagerException {
+      throws IOException,
+          URISyntaxException,
+          SecurityManagerException,
+          ExternalReaderProcessException {
     var moduleResolver = context.getModuleResolver();
     var securityManager = context.getSecurityManager();
     var importsInModule = collectImports(moduleUri, moduleResolver, securityManager);
@@ -71,7 +78,10 @@ public class VmImportAnalyzer {
 
   private static Set<ImportGraph.Import> collectImports(
       URI moduleUri, ModuleResolver moduleResolver, SecurityManager securityManager)
-      throws IOException, URISyntaxException, SecurityManagerException {
+      throws IOException,
+          URISyntaxException,
+          SecurityManagerException,
+          ExternalReaderProcessException {
     var moduleKey = moduleResolver.resolve(moduleUri);
     var resolvedModuleKey = moduleKey.resolve(securityManager);
     List<Entry> importsAndReads;

pkl-core/src/main/java/org/pkl/core/service/ExecutorSpiImpl.java

@@ -33,6 +33,7 @@ import org.pkl.core.module.ModuleKeyFactories;
 import org.pkl.core.module.ModulePathResolver;
 import org.pkl.core.project.Project;
 import org.pkl.core.resource.ResourceReaders;
+import org.pkl.core.util.Readers;
 import org.pkl.executor.spi.v1.ExecutorSpi;
 import org.pkl.executor.spi.v1.ExecutorSpiException;
 import org.pkl.executor.spi.v1.ExecutorSpiOptions;
@@ -125,7 +126,8 @@ public final class ExecutorSpiImpl implements ExecutorSpi {
     } catch (PklException e) {
       throw new ExecutorSpiException(e.getMessage(), e.getCause());
     } finally {
-      ModuleKeyFactories.closeQuietly(builder.getModuleKeyFactories());
+      Readers.closeQuietly(builder.getModuleKeyFactories());
+      Readers.closeQuietly(builder.getResourceReaders());
     }
   }
 

pkl-core/src/main/java/org/pkl/core/stdlib/analyze/AnalyzeNodes.java

@@ -23,6 +23,7 @@ import java.net.URISyntaxException;
 import org.pkl.core.ImportGraph;
 import org.pkl.core.ImportGraph.Import;
 import org.pkl.core.SecurityManagerException;
+import org.pkl.core.externalreader.ExternalReaderProcessException;
 import org.pkl.core.packages.PackageLoadError;
 import org.pkl.core.runtime.AnalyzeModule;
 import org.pkl.core.runtime.VmContext;
@@ -91,7 +92,11 @@ public final class AnalyzeNodes {
       try {
         var results = VmImportAnalyzer.analyze(uris, context);
         return importGraphFactory.create(results);
-      } catch (IOException | URISyntaxException | SecurityManagerException | PackageLoadError e) {
+      } catch (IOException
+          | URISyntaxException
+          | SecurityManagerException
+          | PackageLoadError
+          | ExternalReaderProcessException e) {
         throw exceptionBuilder().withCause(e).build();
       }
     }

pkl-core/src/main/java/org/pkl/core/stdlib/benchmark/OutputBenchmarkNodes.java

@@ -25,6 +25,7 @@ import java.nio.file.Path;
 import java.util.List;
 import org.pkl.core.SecurityManager;
 import org.pkl.core.SecurityManagerException;
+import org.pkl.core.externalreader.ExternalReaderProcessException;
 import org.pkl.core.module.ModuleKey;
 import org.pkl.core.module.PathElement;
 import org.pkl.core.module.ResolvedModuleKey;
@@ -108,7 +109,7 @@ public final class OutputBenchmarkNodes {
     }
 
     @Override
-    public boolean hasHierarchicalUris() {
+    public boolean hasHierarchicalUris() throws IOException, ExternalReaderProcessException {
       return delegate.hasHierarchicalUris();
     }
 
@@ -118,19 +119,19 @@ public final class OutputBenchmarkNodes {
     }
 
     @Override
-    public boolean isGlobbable() {
+    public boolean isGlobbable() throws IOException, ExternalReaderProcessException {
       return delegate.isGlobbable();
     }
 
     @Override
     public boolean hasElement(SecurityManager securityManager, URI uri)
-        throws IOException, SecurityManagerException {
+        throws IOException, SecurityManagerException, ExternalReaderProcessException {
       return delegate.hasElement(securityManager, uri);
     }
 
     @Override
     public List<PathElement> listElements(SecurityManager securityManager, URI baseUri)
-        throws IOException, SecurityManagerException {
+        throws IOException, SecurityManagerException, ExternalReaderProcessException {
       return delegate.listElements(securityManager, baseUri);
     }
   }

pkl-core/src/main/java/org/pkl/core/util/GlobResolver.java

@@ -31,6 +31,7 @@ import java.util.stream.Collectors;
 import org.pkl.core.PklBugException;
 import org.pkl.core.SecurityManager;
 import org.pkl.core.SecurityManagerException;
+import org.pkl.core.externalreader.ExternalReaderProcessException;
 import org.pkl.core.module.ModuleKey;
 import org.pkl.core.module.PathElement;
 import org.pkl.core.runtime.ReaderBase;
@@ -260,7 +261,7 @@ public final class GlobResolver {
       URI globUri,
       Pattern pattern,
       Map<String, ResolvedGlobElement> result)
-      throws IOException, SecurityManagerException {
+      throws IOException, SecurityManagerException, ExternalReaderProcessException {
     var elements = reader.listElements(securityManager, globUri);
     for (var elem : sorted(elements)) {
       URI resolvedUri;
@@ -318,7 +319,10 @@ public final class GlobResolver {
       boolean isGlobStar,
       boolean hasAbsoluteGlob,
       MutableLong listElementCallCount)
-      throws IOException, SecurityManagerException, InvalidGlobPatternException {
+      throws IOException,
+          SecurityManagerException,
+          InvalidGlobPatternException,
+          ExternalReaderProcessException {
     var result = new ArrayList<ResolvedGlobElement>();
     doExpandHierarchicalGlobPart(
         securityManager,
@@ -343,7 +347,10 @@ public final class GlobResolver {
       boolean hasAbsoluteGlob,
       MutableLong listElementCallCount,
       List<ResolvedGlobElement> result)
-      throws IOException, SecurityManagerException, InvalidGlobPatternException {
+      throws IOException,
+          SecurityManagerException,
+          InvalidGlobPatternException,
+          ExternalReaderProcessException {
 
     if (listElementCallCount.getAndIncrement() > maxListElements()) {
       throw new InvalidGlobPatternException(ErrorMessages.create("invalidGlobTooComplex"));
@@ -384,7 +391,10 @@ public final class GlobResolver {
       boolean hasAbsoluteGlob,
       Map<String, ResolvedGlobElement> result,
       MutableLong listElementCallCount)
-      throws IOException, SecurityManagerException, InvalidGlobPatternException {
+      throws IOException,
+          SecurityManagerException,
+          InvalidGlobPatternException,
+          ExternalReaderProcessException {
     var isLeaf = idx == globPatternParts.length - 1;
     var patternPart = globPatternParts[idx];
     if (isRegularPathPart(patternPart)) {
@@ -481,7 +491,10 @@ public final class GlobResolver {
       ModuleKey enclosingModuleKey,
       URI enclosingUri,
       String globPattern)
-      throws IOException, SecurityManagerException, InvalidGlobPatternException {
+      throws IOException,
+          SecurityManagerException,
+          InvalidGlobPatternException,
+          ExternalReaderProcessException {
 
     var result = new LinkedHashMap<String, ResolvedGlobElement>();
     var hasAbsoluteGlob = globPattern.matches("\\w+:.*");

pkl-core/src/main/java/org/pkl/core/util/IoUtils.java

@@ -38,6 +38,7 @@ import org.pkl.core.PklBugException;
 import org.pkl.core.Platform;
 import org.pkl.core.SecurityManager;
 import org.pkl.core.SecurityManagerException;
+import org.pkl.core.externalreader.ExternalReaderProcessException;
 import org.pkl.core.module.ModuleKey;
 import org.pkl.core.packages.PackageLoadError;
 import org.pkl.core.runtime.ReaderBase;
@@ -317,7 +318,7 @@ public final class IoUtils {
 
   private static URI resolveTripleDotImport(
       SecurityManager securityManager, ModuleKey moduleKey, String tripleDotPath)
-      throws IOException, SecurityManagerException {
+      throws IOException, SecurityManagerException, ExternalReaderProcessException {
     var moduleKeyUri = moduleKey.getUri();
     if (!moduleKey.isLocal() || !moduleKey.hasHierarchicalUris()) {
       throw new VmExceptionBuilder()
@@ -363,7 +364,8 @@ public final class IoUtils {
     return Pair.of(importPath.substring(1, idx), importPath.substring(idx));
   }
 
-  private static URI resolveProjectDependency(ModuleKey moduleKey, String notation) {
+  private static URI resolveProjectDependency(ModuleKey moduleKey, String notation)
+      throws IOException, ExternalReaderProcessException {
     var parsed = parseDependencyNotation(notation);
     var name = parsed.getFirst();
     var path = parsed.getSecond();
@@ -395,7 +397,10 @@ public final class IoUtils {
    * dependency notation ()
    */
   public static URI resolve(SecurityManager securityManager, ModuleKey moduleKey, URI importUri)
-      throws URISyntaxException, IOException, SecurityManagerException {
+      throws URISyntaxException,
+          IOException,
+          SecurityManagerException,
+          ExternalReaderProcessException {
     if (importUri.isAbsolute()) {
       return moduleKey.resolveUri(importUri);
     }

pkl-core/src/main/java/org/pkl/core/util/Readers.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.util;
+
+public class Readers {
+  /** Closes the given readers, ignoring any exceptions. */
+  public static void closeQuietly(Iterable<? extends AutoCloseable> readers) {
+    for (var reader : readers) {
+      try {
+        reader.close();
+      } catch (Exception ignored) {
+      }
+    }
+  }
+}

pkl-core/src/main/resources/org/pkl/core/errorMessages.properties

@@ -1081,3 +1081,33 @@ Malformed proxy URI (expecting `http://<host>[:<port>]`): `{0}`.
 
 cannotAnalyzeBecauseSyntaxError=\
 Found a syntax error when parsing module `{0}`.
+
+malformedMessageHeaderLength=\
+Malformed message header (expected size 2, but got {0}).
+
+malformedMessageHeaderException=\
+Malformed message header.
+
+malformedMessageHeaderUnrecognizedCode=\
+Malformed message header (unrecognized code `{0}`).
+
+unhandledMessageCode=\
+Unhandled decoding message code `{0}`.
+
+unhandledMessageType=\
+Unhandled encoding message type `{0}`.
+
+malformedMessageBody=\
+Malformed message body for message with code `{0}`.
+
+missingMessageParameter=\
+Missing message parameter `{0}`
+
+unknownRequestId=\
+Received response {0} for unknown request ID `{1}`.
+
+externalReaderFailure=\
+Failed to communicate with external reader process.
+
+externalReaderDoesNotSupportScheme=\
+External {0} reader does not support scheme `{1}`.

pkl-core/src/test/kotlin/org/pkl/core/EvaluatorBuilderTest.kt

@@ -71,8 +71,10 @@ class EvaluatorBuilderTest {
   fun `sets evaluator settings from project`() {
     val projectPath = Path.of(javaClass.getResource("project/project1/PklProject")!!.toURI())
     val project = Project.loadFromPath(projectPath, SecurityManagers.defaultManager, null)
-    val projectDir = Path.of(javaClass.getResource("project/project1/PklProject")!!.toURI()).parent
-    val builder = EvaluatorBuilder.unconfigured().applyFromProject(project)
+    val projectDir = projectPath.parent
+    val builder = EvaluatorBuilder.unconfigured()
+    val moduleKeyFactoryCount = builder.moduleKeyFactories.size
+    builder.applyFromProject(project)
     assertThat(builder.allowedResources.map { it.pattern() }).isEqualTo(listOf("foo:", "bar:"))
     assertThat(builder.allowedModules.map { it.pattern() }).isEqualTo(listOf("baz:", "biz:"))
     assertThat(builder.externalProperties).isEqualTo(mapOf("one" to "1"))
@@ -80,5 +82,9 @@ class EvaluatorBuilderTest {
     assertThat(builder.moduleCacheDir).isEqualTo(projectDir.resolve("my-cache-dir/"))
     assertThat(builder.rootDir).isEqualTo(projectDir.resolve("my-root-dir/"))
     assertThat(builder.timeout).isEqualTo(Duration.ofMinutes(5L))
+    assertThat(builder.moduleKeyFactories.size - moduleKeyFactoryCount)
+      .isEqualTo(3) // two external readers, one module path
+    assertThat(builder.resourceReaders.find { it.uriScheme == "scheme3" }).isNotNull
+    assertThat(builder.resourceReaders.find { it.uriScheme == "scheme4" }).isNotNull
   }
 }

pkl-core/src/test/kotlin/org/pkl/core/externalreader/ExternalModuleReader.kt

@@ -13,16 +13,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.pkl.server
+package org.pkl.core.externalreader
 
-import java.io.OutputStream
-import org.msgpack.core.MessagePack
-import org.msgpack.core.MessagePacker
+import java.net.URI
+import org.pkl.core.messaging.Messages.ModuleReaderSpec
 
-/** Factory methods for creating [MessageEncoder]s. */
-internal object MessageEncoders {
-  fun into(stream: OutputStream): MessageEncoder =
-    MessagePackEncoder(MessagePack.newDefaultPacker(stream))
+/** An external module reader, to be used with [ExternalReaderRuntime]. */
+interface ExternalModuleReader : ExternalReaderBase {
+  val isLocal: Boolean
 
-  fun into(packer: MessagePacker): MessageEncoder = MessagePackEncoder(packer)
+  fun read(uri: URI): String
+
+  val spec: ModuleReaderSpec
+    get() = ModuleReaderSpec(scheme, hasHierarchicalUris, isLocal, isGlobbable)
 }

pkl-core/src/test/kotlin/org/pkl/core/externalreader/ExternalProcessProcessReaderMessagePackCodecTest.kt

@@ -0,0 +1,73 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.externalreader
+
+import java.io.PipedInputStream
+import java.io.PipedOutputStream
+import org.assertj.core.api.Assertions.assertThat
+import org.junit.jupiter.api.Test
+import org.msgpack.core.MessagePack
+import org.pkl.core.externalreader.ExternalReaderMessages.*
+import org.pkl.core.messaging.*
+
+class ExternalProcessProcessReaderMessagePackCodecTest {
+  private val encoder: MessageEncoder
+  private val decoder: MessageDecoder
+
+  init {
+    val inputStream = PipedInputStream()
+    val outputStream = PipedOutputStream(inputStream)
+    encoder = ExternalReaderMessagePackEncoder(MessagePack.newDefaultPacker(outputStream))
+    decoder = ExternalReaderMessagePackDecoder(MessagePack.newDefaultUnpacker(inputStream))
+  }
+
+  private fun roundtrip(message: Message) {
+    encoder.encode(message)
+    val decoded = decoder.decode()
+    assertThat(decoded).isEqualTo(message)
+  }
+
+  @Test
+  fun `round-trip InitializeModuleReaderRequest`() {
+    roundtrip(InitializeModuleReaderRequest(123, "my-scheme"))
+  }
+
+  @Test
+  fun `round-trip InitializeResourceReaderRequest`() {
+    roundtrip(InitializeResourceReaderRequest(123, "my-scheme"))
+  }
+
+  @Test
+  fun `round-trip InitializeModuleReaderResponse`() {
+    roundtrip(InitializeModuleReaderResponse(123, null))
+    roundtrip(
+      InitializeModuleReaderResponse(123, Messages.ModuleReaderSpec("my-scheme", true, true, true))
+    )
+  }
+
+  @Test
+  fun `round-trip InitializeResourceReaderResponse`() {
+    roundtrip(InitializeResourceReaderResponse(123, null))
+    roundtrip(
+      InitializeResourceReaderResponse(123, Messages.ResourceReaderSpec("my-scheme", true, true))
+    )
+  }
+
+  @Test
+  fun `round-trip CloseExternalProcess`() {
+    roundtrip(CloseExternalProcess())
+  }
+}

pkl-core/src/test/kotlin/org/pkl/core/externalreader/ExternalReaderBase.kt

@@ -13,15 +13,18 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.pkl.server
+package org.pkl.core.externalreader
 
-/** A bidirectional transport for sending and receiving messages. */
-interface MessageTransport : AutoCloseable {
-  fun start(oneWayHandler: (OneWayMessage) -> Unit, requestHandler: (RequestMessage) -> Unit)
+import java.net.URI
+import org.pkl.core.module.PathElement
 
-  fun send(message: OneWayMessage)
+/** Base interface for external module and resource readers. */
+interface ExternalReaderBase {
+  val scheme: String
 
-  fun send(message: RequestMessage, responseHandler: (ResponseMessage) -> Unit)
+  val hasHierarchicalUris: Boolean
 
-  fun send(message: ResponseMessage)
+  val isGlobbable: Boolean
+
+  fun listElements(uri: URI): List<PathElement>
 }

pkl-core/src/test/kotlin/org/pkl/core/externalreader/ExternalReaderRuntime.kt

@@ -0,0 +1,199 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.externalreader
+
+import java.io.IOException
+import org.pkl.core.externalreader.ExternalReaderMessages.*
+import org.pkl.core.messaging.Message
+import org.pkl.core.messaging.MessageTransport
+import org.pkl.core.messaging.Messages.*
+import org.pkl.core.messaging.ProtocolException
+import org.pkl.core.util.Nullable
+
+/** An implementation of the client side of the external reader flow */
+class ExternalReaderRuntime(
+  private val moduleReaders: List<ExternalModuleReader>,
+  private val resourceReaders: List<ExternalResourceReader>,
+  private val transport: MessageTransport
+) {
+  /** Close the runtime and its transport. */
+  fun close() {
+    transport.close()
+  }
+
+  private fun findModuleReader(scheme: String): @Nullable ExternalModuleReader? {
+    for (moduleReader in moduleReaders) {
+      if (moduleReader.scheme.equals(scheme, ignoreCase = true)) {
+        return moduleReader
+      }
+    }
+    return null
+  }
+
+  private fun findResourceReader(scheme: String): @Nullable ExternalResourceReader? {
+    for (resourceReader in resourceReaders) {
+      if (resourceReader.scheme.equals(scheme, ignoreCase = true)) {
+        return resourceReader
+      }
+    }
+    return null
+  }
+
+  /**
+   * Start the runtime so it can respond to incoming messages on its transport.
+   *
+   * Blocks until the underlying transport is closed.
+   */
+  @Throws(ProtocolException::class, IOException::class)
+  fun run() {
+    transport.start(
+      { msg: Message.OneWay ->
+        if (msg.type() == Message.Type.CLOSE_EXTERNAL_PROCESS) {
+          close()
+        } else {
+          throw ProtocolException("Unexpected incoming one-way message: $msg")
+        }
+      },
+      { msg: Message.Request ->
+        when (msg.type()) {
+          Message.Type.INITIALIZE_MODULE_READER_REQUEST -> {
+            val req = msg as InitializeModuleReaderRequest
+            val reader = findModuleReader(req.scheme)
+            var spec: @Nullable ModuleReaderSpec? = null
+            if (reader != null) {
+              spec = reader.spec
+            }
+            transport.send(InitializeModuleReaderResponse(req.requestId, spec))
+          }
+          Message.Type.INITIALIZE_RESOURCE_READER_REQUEST -> {
+            val req = msg as InitializeResourceReaderRequest
+            val reader = findResourceReader(req.scheme)
+            var spec: @Nullable ResourceReaderSpec? = null
+            if (reader != null) {
+              spec = reader.spec
+            }
+            transport.send(InitializeResourceReaderResponse(req.requestId, spec))
+          }
+          Message.Type.LIST_MODULES_REQUEST -> {
+            val req = msg as ListModulesRequest
+            val reader = findModuleReader(req.uri.scheme)
+            if (reader == null) {
+              transport.send(
+                ListModulesResponse(
+                  req.requestId,
+                  req.evaluatorId,
+                  null,
+                  "No module reader found for scheme " + req.uri.scheme
+                )
+              )
+              return@start
+            }
+            try {
+              transport.send(
+                ListModulesResponse(
+                  req.requestId,
+                  req.evaluatorId,
+                  reader.listElements(req.uri),
+                  null
+                )
+              )
+            } catch (e: Exception) {
+              transport.send(
+                ListModulesResponse(req.requestId, req.evaluatorId, null, e.toString())
+              )
+            }
+          }
+          Message.Type.LIST_RESOURCES_REQUEST -> {
+            val req = msg as ListResourcesRequest
+            val reader = findModuleReader(req.uri.scheme)
+            if (reader == null) {
+              transport.send(
+                ListResourcesResponse(
+                  req.requestId,
+                  req.evaluatorId,
+                  null,
+                  "No resource reader found for scheme " + req.uri.scheme
+                )
+              )
+              return@start
+            }
+            try {
+              transport.send(
+                ListResourcesResponse(
+                  req.requestId,
+                  req.evaluatorId,
+                  reader.listElements(req.uri),
+                  null
+                )
+              )
+            } catch (e: Exception) {
+              transport.send(
+                ListResourcesResponse(req.requestId, req.evaluatorId, null, e.toString())
+              )
+            }
+          }
+          Message.Type.READ_MODULE_REQUEST -> {
+            val req = msg as ReadModuleRequest
+            val reader = findModuleReader(req.uri.scheme)
+            if (reader == null) {
+              transport.send(
+                ReadModuleResponse(
+                  req.requestId,
+                  req.evaluatorId,
+                  null,
+                  "No module reader found for scheme " + req.uri.scheme
+                )
+              )
+              return@start
+            }
+            try {
+              transport.send(
+                ReadModuleResponse(req.requestId, req.evaluatorId, reader.read(req.uri), null)
+              )
+            } catch (e: Exception) {
+              transport.send(ReadModuleResponse(req.requestId, req.evaluatorId, null, e.toString()))
+            }
+          }
+          Message.Type.READ_RESOURCE_REQUEST -> {
+            val req = msg as ReadResourceRequest
+            val reader = findResourceReader(req.uri.scheme)
+            if (reader == null) {
+              transport.send(
+                ReadResourceResponse(
+                  req.requestId,
+                  req.evaluatorId,
+                  byteArrayOf(),
+                  "No resource reader found for scheme " + req.uri.scheme
+                )
+              )
+              return@start
+            }
+            try {
+              transport.send(
+                ReadResourceResponse(req.requestId, req.evaluatorId, reader.read(req.uri), null)
+              )
+            } catch (e: Exception) {
+              transport.send(
+                ReadResourceResponse(req.requestId, req.evaluatorId, byteArrayOf(), e.toString())
+              )
+            }
+          }
+          else -> throw ProtocolException("Unexpected incoming request message: $msg")
+        }
+      }
+    )
+  }
+}

pkl-core/src/test/kotlin/org/pkl/core/externalreader/ExternalResourceReader.kt

@@ -0,0 +1,27 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.externalreader
+
+import java.net.URI
+import org.pkl.core.messaging.Messages.ResourceReaderSpec
+
+/** An external resource reader, to be used with [ExternalReaderRuntime]. */
+interface ExternalResourceReader : ExternalReaderBase {
+  fun read(uri: URI): ByteArray
+
+  val spec: ResourceReaderSpec
+    get() = ResourceReaderSpec(scheme, hasHierarchicalUris, isGlobbable)
+}

pkl-core/src/test/kotlin/org/pkl/core/externalreader/TestExternalModuleReader.kt

@@ -0,0 +1,38 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.externalreader
+
+import java.net.URI
+import org.pkl.core.module.PathElement
+
+class TestExternalModuleReader : ExternalModuleReader {
+  override val scheme: String = "test"
+
+  override val hasHierarchicalUris: Boolean = false
+
+  override val isLocal: Boolean = true
+
+  override val isGlobbable: Boolean = false
+
+  override fun read(uri: URI): String =
+    """
+    name = "Pigeon"
+    age = 40
+  """
+      .trimIndent()
+
+  override fun listElements(uri: URI): List<PathElement> = emptyList()
+}

pkl-core/src/test/kotlin/org/pkl/core/externalreader/TestExternalReaderProcess.kt

@@ -0,0 +1,130 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.externalreader
+
+import java.io.IOException
+import java.io.PipedInputStream
+import java.io.PipedOutputStream
+import java.util.concurrent.CompletableFuture
+import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.ExecutionException
+import java.util.concurrent.Future
+import kotlin.random.Random
+import org.pkl.core.externalreader.ExternalReaderMessages.*
+import org.pkl.core.messaging.MessageTransport
+import org.pkl.core.messaging.MessageTransports
+import org.pkl.core.messaging.Messages.*
+import org.pkl.core.messaging.ProtocolException
+
+class TestExternalReaderProcess(private val transport: MessageTransport) : ExternalReaderProcess {
+  private val initializeModuleReaderResponses: MutableMap<String, Future<ModuleReaderSpec?>> =
+    ConcurrentHashMap()
+  private val initializeResourceReaderResponses: MutableMap<String, Future<ResourceReaderSpec?>> =
+    ConcurrentHashMap()
+
+  override fun close() {
+    transport.send(CloseExternalProcess())
+    transport.close()
+  }
+
+  override fun getTransport(): MessageTransport = transport
+
+  fun run() {
+    try {
+      transport.start(
+        { throw ProtocolException("Unexpected incoming one-way message: $it") },
+        { throw ProtocolException("Unexpected incoming request message: $it") },
+      )
+    } catch (e: ProtocolException) {
+      throw RuntimeException(e)
+    } catch (e: IOException) {
+      throw RuntimeException(e)
+    }
+  }
+
+  override fun getModuleReaderSpec(scheme: String): ModuleReaderSpec? =
+    initializeModuleReaderResponses
+      .computeIfAbsent(scheme) {
+        CompletableFuture<ModuleReaderSpec?>().apply {
+          val request = InitializeModuleReaderRequest(Random.nextLong(), scheme)
+          transport.send(request) { response ->
+            when (response) {
+              is InitializeModuleReaderResponse -> {
+                complete(response.spec)
+              }
+              else -> completeExceptionally(ProtocolException("unexpected response"))
+            }
+          }
+        }
+      }
+      .getUnderlying()
+
+  override fun getResourceReaderSpec(scheme: String): ResourceReaderSpec? =
+    initializeResourceReaderResponses
+      .computeIfAbsent(scheme) {
+        CompletableFuture<ResourceReaderSpec?>().apply {
+          val request = InitializeResourceReaderRequest(Random.nextLong(), scheme)
+          transport.send(request) { response ->
+            when (response) {
+              is InitializeResourceReaderResponse -> {
+                complete(response.spec)
+              }
+              else -> completeExceptionally(ProtocolException("unexpected response"))
+            }
+          }
+        }
+      }
+      .getUnderlying()
+
+  companion object {
+    fun initializeTestHarness(
+      moduleReaders: List<ExternalModuleReader>,
+      resourceReaders: List<ExternalResourceReader>
+    ): Pair<TestExternalReaderProcess, ExternalReaderRuntime> {
+      val rxIn = PipedInputStream(10240)
+      val rxOut = PipedOutputStream(rxIn)
+      val txIn = PipedInputStream(10240)
+      val txOut = PipedOutputStream(txIn)
+      val serverTransport =
+        MessageTransports.stream(
+          ExternalReaderMessagePackDecoder(rxIn),
+          ExternalReaderMessagePackEncoder(txOut),
+          {}
+        )
+      val clientTransport =
+        MessageTransports.stream(
+          ExternalReaderMessagePackDecoder(txIn),
+          ExternalReaderMessagePackEncoder(rxOut),
+          {}
+        )
+
+      val runtime = ExternalReaderRuntime(moduleReaders, resourceReaders, clientTransport)
+      val proc = TestExternalReaderProcess(serverTransport)
+
+      Thread(runtime::run).start()
+      Thread(proc::run).start()
+
+      return proc to runtime
+    }
+  }
+}
+
+fun <T> Future<T>.getUnderlying(): T =
+  try {
+    get()
+  } catch (e: ExecutionException) {
+    throw e.cause!!
+  }

pkl-core/src/test/kotlin/org/pkl/core/externalreader/TestExternalResourceReader.kt

@@ -0,0 +1,31 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.externalreader
+
+import java.net.URI
+import org.pkl.core.module.PathElement
+
+class TestExternalResourceReader : ExternalResourceReader {
+  override val scheme: String = "test"
+
+  override val hasHierarchicalUris: Boolean = false
+
+  override val isGlobbable: Boolean = false
+
+  override fun read(uri: URI): ByteArray = "success".toByteArray(Charsets.UTF_8)
+
+  override fun listElements(uri: URI): List<PathElement> = emptyList()
+}

pkl-core/src/test/kotlin/org/pkl/core/messaging/BaseMessagePackCodecTest.kt

@@ -0,0 +1,128 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.messaging
+
+import java.io.PipedInputStream
+import java.io.PipedOutputStream
+import java.net.URI
+import org.assertj.core.api.Assertions.assertThat
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.assertThrows
+import org.msgpack.core.MessagePack
+import org.pkl.core.messaging.Messages.*
+import org.pkl.core.module.PathElement
+
+class BaseMessagePackCodecTest {
+  private val encoder: MessageEncoder
+  private val decoder: MessageDecoder
+
+  init {
+    val inputStream = PipedInputStream()
+    val outputStream = PipedOutputStream(inputStream)
+    encoder = BaseMessagePackEncoder(MessagePack.newDefaultPacker(outputStream))
+    decoder = BaseMessagePackDecoder(MessagePack.newDefaultUnpacker(inputStream))
+  }
+
+  private fun roundtrip(message: Message) {
+    encoder.encode(message)
+    val decoded = decoder.decode()
+    assertThat(decoded).isEqualTo(message)
+  }
+
+  @Test
+  fun `round-trip ReadResourceRequest`() {
+    roundtrip(ReadResourceRequest(123, 456, URI("some/resource.json")))
+  }
+
+  @Test
+  fun `round-trip ReadResourceResponse`() {
+    roundtrip(ReadResourceResponse(123, 456, byteArrayOf(1, 2, 3, 4, 5), null))
+  }
+
+  @Test
+  fun `round-trip ReadModuleRequest`() {
+    roundtrip(ReadModuleRequest(123, 456, URI("some/module.pkl")))
+  }
+
+  @Test
+  fun `round-trip ReadModuleResponse`() {
+    roundtrip(ReadModuleResponse(123, 456, "x = 42", null))
+  }
+
+  @Test
+  fun `round-trip ListModulesRequest`() {
+    roundtrip(ListModulesRequest(135, 246, URI("foo:/bar/baz/biz")))
+  }
+
+  @Test
+  fun `round-trip ListModulesResponse`() {
+    roundtrip(
+      ListModulesResponse(
+        123,
+        234,
+        listOf(PathElement("foo", true), PathElement("bar", false)),
+        null
+      )
+    )
+    roundtrip(ListModulesResponse(123, 234, null, "Something dun went wrong"))
+  }
+
+  @Test
+  fun `round-trip ListResourcesRequest`() {
+    roundtrip(ListResourcesRequest(987, 1359, URI("bar:/bazzy")))
+  }
+
+  @Test
+  fun `round-trip ListResourcesResponse`() {
+    roundtrip(
+      ListResourcesResponse(
+        3851,
+        3019,
+        listOf(PathElement("foo", true), PathElement("bar", false)),
+        null
+      )
+    )
+    roundtrip(ListResourcesResponse(3851, 3019, null, "something went wrong"))
+  }
+
+  @Test
+  fun `decode request with missing request ID`() {
+    val bytes =
+      MessagePack.newDefaultBufferPacker()
+        .apply {
+          packArrayHeader(2)
+          packInt(Message.Type.LIST_RESOURCES_REQUEST.code)
+          packMapHeader(1)
+          packString("uri")
+          packString("file:/test")
+        }
+        .toByteArray()
+
+    val decoder = BaseMessagePackDecoder(MessagePack.newDefaultUnpacker(bytes))
+    val exception = assertThrows<DecodeException> { decoder.decode() }
+    assertThat(exception.message).contains("requestId")
+  }
+
+  @Test
+  fun `decode invalid message header`() {
+    val bytes = MessagePack.newDefaultBufferPacker().apply { packInt(2) }.toByteArray()
+
+    val decoder = BaseMessagePackDecoder(MessagePack.newDefaultUnpacker(bytes))
+    val exception = assertThrows<DecodeException> { decoder.decode() }
+    assertThat(exception).hasMessage("Malformed message header.")
+    assertThat(exception).hasRootCauseMessage("Expected Array, but got Integer (02)")
+  }
+}

pkl-core/src/test/kotlin/org/pkl/core/module/ModuleKeyFactoriesTest.kt

@@ -26,6 +26,7 @@ import org.pkl.commons.createParentDirectories
 import org.pkl.commons.toPath
 import org.pkl.commons.writeString
 import org.pkl.core.SecurityManagers
+import org.pkl.core.externalreader.*
 
 class ModuleKeyFactoriesTest {
   @Test
@@ -126,4 +127,23 @@ class ModuleKeyFactoriesTest {
     val module2 = factory.create(URI("other"))
     assertThat(module2).isNotPresent
   }
+
+  @Test
+  fun externalProcess() {
+    val extReader = TestExternalModuleReader()
+    val (proc, runtime) =
+      TestExternalReaderProcess.initializeTestHarness(listOf(extReader), emptyList())
+
+    val factory = ModuleKeyFactories.externalProcess(extReader.scheme, proc)
+
+    val module = factory.create(URI("test:foo"))
+    assertThat(module).isPresent
+    assertThat(module.get().uri.scheme).isEqualTo("test")
+
+    val module2 = factory.create(URI("other"))
+    assertThat(module2).isNotPresent
+
+    proc.close()
+    runtime.close()
+  }
 }

pkl-core/src/test/kotlin/org/pkl/core/project/ProjectTest.kt

@@ -70,6 +70,8 @@ class ProjectTest {
         listOf(path.resolve("modulepath1/"), path.resolve("modulepath2/")),
         Duration.ofMinutes(5.0),
         path,
+        null,
+        null,
         null
       )
     val expectedAnnotations =

pkl-core/src/test/kotlin/org/pkl/core/resource/ResourceReadersTest.kt

@@ -23,6 +23,8 @@ import org.assertj.core.api.Assertions.assertThat
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertThrows
 import org.junit.jupiter.api.io.TempDir
+import org.pkl.core.externalreader.TestExternalReaderProcess
+import org.pkl.core.externalreader.TestExternalResourceReader
 import org.pkl.core.module.ModulePathResolver
 
 class ResourceReadersTest {
@@ -132,4 +134,21 @@ class ResourceReadersTest {
 
     assertThat(resource).contains("success")
   }
+
+  @Test
+  fun externalProcess() {
+    val extReader = TestExternalResourceReader()
+    val (proc, runtime) =
+      TestExternalReaderProcess.initializeTestHarness(emptyList(), listOf(extReader))
+
+    val reader = ResourceReaders.externalProcess(extReader.scheme, proc)
+    val resource = reader.read(URI("test:foo"))
+
+    assertThat(resource).isPresent
+    assertThat(resource.get()).isInstanceOf(Resource::class.java)
+    assertThat((resource.get() as Resource).text).contains("success")
+
+    proc.close()
+    runtime.close()
+  }
 }

pkl-core/src/test/resources/org/pkl/core/project/project1/PklProject

@@ -22,4 +22,22 @@ evaluatorSettings {
   noCache = false
   rootDir = "my-root-dir/"
   timeout = 5.min
+  externalModuleReaders {
+    ["scheme1"] {
+      executable = "reader1"
+    }
+    ["scheme2"] {
+      executable = "reader2"
+      arguments { "with"; "args" }
+    }
+  }
+  externalResourceReaders {
+    ["scheme3"] {
+      executable = "reader3"
+    }
+    ["scheme4"] {
+      executable = "reader4"
+      arguments { "with"; "args" }
+    }
+  }
 }

pkl-doc/gradle.lockfile

@@ -75,6 +75,7 @@ org.junit.jupiter:junit-jupiter-params:5.11.2=testCompileClasspath,testImplement
 org.junit.platform:junit-platform-commons:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit.platform:junit-platform-engine:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit:junit-bom:5.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
+org.msgpack:msgpack-core:0.9.8=runtimeClasspath,testRuntimeClasspath
 org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.organicdesign:Paguro:3.10.3=runtimeClasspath,testRuntimeClasspath
 org.snakeyaml:snakeyaml-engine:2.5=runtimeClasspath,testRuntimeClasspath

pkl-doc/src/main/kotlin/org/pkl/doc/CliDocGenerator.kt

@@ -24,8 +24,8 @@ import org.pkl.commons.cli.CliCommand
 import org.pkl.commons.cli.CliException
 import org.pkl.commons.toPath
 import org.pkl.core.*
-import org.pkl.core.module.ModuleKeyFactories
 import org.pkl.core.packages.*
+import org.pkl.core.util.Readers
 
 /**
  * Entry point for the high-level Pkldoc API.
@@ -250,7 +250,8 @@ class CliDocGenerator(private val options: CliDocGeneratorOptions) : CliCommand(
         importedModules[pklBaseUri] = evaluator.evaluateSchema(ModuleSource.uri(pklBaseUri))
       }
     } finally {
-      ModuleKeyFactories.closeQuietly(builder.moduleKeyFactories)
+      Readers.closeQuietly(builder.moduleKeyFactories)
+      Readers.closeQuietly(builder.resourceReaders)
     }
 
     val versions = mutableMapOf<String, Version>()

pkl-executor/gradle.lockfile

@@ -29,6 +29,7 @@ org.junit.jupiter:junit-jupiter-params:5.11.2=testCompileClasspath,testImplement
 org.junit.platform:junit-platform-commons:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit.platform:junit-platform-engine:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit:junit-bom:5.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
+org.msgpack:msgpack-core:0.9.8=testRuntimeClasspath
 org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.organicdesign:Paguro:3.10.3=testRuntimeClasspath
 org.pkl-lang:pkl-config-java-all:0.25.0=pklHistoricalDistributions

pkl-gradle/src/main/java/org/pkl/gradle/task/BasePklTask.java

@@ -25,6 +25,7 @@ import java.nio.file.Paths;
 import java.time.Duration;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.function.Function;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -170,7 +171,9 @@ public abstract class BasePklTask extends DefaultTask {
               getTestPort().getOrElse(-1),
               Collections.emptyList(),
               getHttpProxy().getOrNull(),
-              getHttpNoProxy().getOrElse(List.of()));
+              getHttpNoProxy().getOrElse(List.of()),
+              Map.of(),
+              Map.of());
     }
     return cachedOptions;
   }

pkl-gradle/src/main/java/org/pkl/gradle/task/ModulesTask.java

@@ -182,7 +182,9 @@ public abstract class ModulesTask extends BasePklTask {
               getTestPort().getOrElse(-1),
               Collections.emptyList(),
               null,
-              List.of());
+              List.of(),
+              Map.of(),
+              Map.of());
     }
     return cachedOptions;
   }

pkl-server/src/main/kotlin/org/pkl/server/ClientLogger.kt

@@ -17,16 +17,17 @@ package org.pkl.server
 
 import org.pkl.core.Logger
 import org.pkl.core.StackFrame
+import org.pkl.core.messaging.MessageTransport
 
 internal class ClientLogger(
   private val evaluatorId: Long,
   private val transport: MessageTransport
 ) : Logger {
   override fun trace(message: String, frame: StackFrame) {
-    transport.send(LogMessage(evaluatorId, level = 0, message, frame.moduleUri))
+    transport.send(LogMessage(evaluatorId, 0, message, frame.moduleUri))
   }
 
   override fun warn(message: String, frame: StackFrame) {
-    transport.send(LogMessage(evaluatorId, level = 1, message, frame.moduleUri))
+    transport.send(LogMessage(evaluatorId, 1, message, frame.moduleUri))
   }
 }

pkl-server/src/main/kotlin/org/pkl/server/ClientModuleKeyFactory.kt

@@ -15,136 +15,27 @@
  */
 package org.pkl.server
 
-import java.io.IOException
 import java.net.URI
 import java.util.Optional
-import java.util.concurrent.CompletableFuture
-import java.util.concurrent.ConcurrentHashMap
-import java.util.concurrent.Future
-import kotlin.random.Random
-import org.pkl.core.SecurityManager
-import org.pkl.core.module.ModuleKey
-import org.pkl.core.module.ModuleKeyFactory
-import org.pkl.core.module.PathElement
-import org.pkl.core.module.ResolvedModuleKey
-import org.pkl.core.module.ResolvedModuleKeys
+import org.pkl.core.messaging.*
+import org.pkl.core.messaging.Messages.*
+import org.pkl.core.module.*
+import org.pkl.core.module.ExternalModuleResolver
 
 internal class ClientModuleKeyFactory(
   private val readerSpecs: Collection<ModuleReaderSpec>,
   transport: MessageTransport,
   evaluatorId: Long
 ) : ModuleKeyFactory {
-  companion object {
-    private class ClientModuleKeyResolver(
-      private val transport: MessageTransport,
-      private val evaluatorId: Long,
-    ) {
-      private val readResponses: MutableMap<URI, Future<String>> = ConcurrentHashMap()
-
-      private val listResponses: MutableMap<URI, Future<List<PathElement>>> = ConcurrentHashMap()
-
-      fun listElements(securityManager: SecurityManager, uri: URI): List<PathElement> {
-        securityManager.checkResolveModule(uri)
-        return doListElements(uri)
-      }
-
-      fun hasElement(securityManager: SecurityManager, uri: URI): Boolean {
-        securityManager.checkResolveModule(uri)
-        return try {
-          doReadModule(uri)
-          true
-        } catch (e: IOException) {
-          false
-        }
-      }
-
-      fun resolveModule(securityManager: SecurityManager, uri: URI): String {
-        securityManager.checkResolveModule(uri)
-        return doReadModule(uri)
-      }
-
-      private fun doReadModule(uri: URI): String =
-        readResponses
-          .computeIfAbsent(uri) {
-            CompletableFuture<String>().apply {
-              val request = ReadModuleRequest(Random.nextLong(), evaluatorId, uri)
-              transport.send(request) { response ->
-                when (response) {
-                  is ReadModuleResponse -> {
-                    if (response.error != null) {
-                      completeExceptionally(IOException(response.error))
-                    } else {
-                      complete(response.contents ?: "")
-                    }
-                  }
-                  else -> {
-                    completeExceptionally(ProtocolException("unexpected response"))
-                  }
-                }
-              }
-            }
-          }
-          .getUnderlying()
-
-      private fun doListElements(uri: URI): List<PathElement> =
-        listResponses
-          .computeIfAbsent(uri) {
-            CompletableFuture<List<PathElement>>().apply {
-              val request = ListModulesRequest(Random.nextLong(), evaluatorId, uri)
-              transport.send(request) { response ->
-                when (response) {
-                  is ListModulesResponse -> {
-                    if (response.error != null) {
-                      completeExceptionally(IOException(response.error))
-                    } else {
-                      complete(response.pathElements ?: emptyList())
-                    }
-                  }
-                  else -> completeExceptionally(ProtocolException("unexpected response"))
-                }
-              }
-            }
-          }
-          .getUnderlying()
-    }
-
-    /** [ModuleKey] that delegates module reads to the client. */
-    private class ClientModuleKey(
-      private val uri: URI,
-      private val spec: ModuleReaderSpec,
-      private val resolver: ClientModuleKeyResolver,
-    ) : ModuleKey {
-      override fun isLocal(): Boolean = spec.isLocal
-
-      override fun hasHierarchicalUris(): Boolean = spec.hasHierarchicalUris
-
-      override fun isGlobbable(): Boolean = spec.isGlobbable
-
-      override fun getUri(): URI = uri
-
-      override fun listElements(securityManager: SecurityManager, baseUri: URI): List<PathElement> =
-        resolver.listElements(securityManager, baseUri)
-
-      override fun resolve(securityManager: SecurityManager): ResolvedModuleKey {
-        val contents = resolver.resolveModule(securityManager, uri)
-        return ResolvedModuleKeys.virtual(this, uri, contents, true)
-      }
-
-      override fun hasElement(securityManager: SecurityManager, uri: URI): Boolean {
-        return resolver.hasElement(securityManager, uri)
-      }
-    }
-  }
-
   private val schemes = readerSpecs.map { it.scheme }
 
-  private val resolver: ClientModuleKeyResolver = ClientModuleKeyResolver(transport, evaluatorId)
+  private val resolver: ExternalModuleResolver = ExternalModuleResolver(transport, evaluatorId)
 
   override fun create(uri: URI): Optional<ModuleKey> =
     when (uri.scheme) {
       in schemes -> {
         val readerSpec = readerSpecs.find { it.scheme == uri.scheme }!!
-        val moduleKey = ClientModuleKey(uri, readerSpec, resolver)
+        val moduleKey = ModuleKeys.externalResolver(uri, readerSpec, resolver)
         Optional.of(moduleKey)
       }
       else -> Optional.empty()

pkl-server/src/main/kotlin/org/pkl/server/ClientResourceReader.kt

@@ -1,105 +0,0 @@
-/*
- * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.pkl.server
-
-import java.io.IOException
-import java.net.URI
-import java.util.Optional
-import java.util.concurrent.CompletableFuture
-import java.util.concurrent.ConcurrentHashMap
-import java.util.concurrent.Future
-import kotlin.random.Random
-import org.pkl.core.SecurityManager
-import org.pkl.core.module.PathElement
-import org.pkl.core.resource.Resource
-import org.pkl.core.resource.ResourceReader
-
-/** Resource reader that delegates read logic to the client. */
-internal class ClientResourceReader(
-  private val transport: MessageTransport,
-  private val evaluatorId: Long,
-  private val readerSpec: ResourceReaderSpec,
-) : ResourceReader {
-  private val readResponses: MutableMap<URI, Future<ByteArray>> = ConcurrentHashMap()
-
-  private val listResources: MutableMap<URI, Future<List<PathElement>>> = ConcurrentHashMap()
-
-  override fun hasHierarchicalUris(): Boolean = readerSpec.hasHierarchicalUris
-
-  override fun isGlobbable(): Boolean = readerSpec.isGlobbable
-
-  override fun getUriScheme() = readerSpec.scheme
-
-  override fun read(uri: URI): Optional<Any> = Optional.of(Resource(uri, doRead(uri)))
-
-  override fun hasElement(securityManager: SecurityManager, elementUri: URI): Boolean {
-    securityManager.checkResolveResource(elementUri)
-    return try {
-      doRead(elementUri)
-      true
-    } catch (e: IOException) {
-      false
-    }
-  }
-
-  override fun listElements(securityManager: SecurityManager, baseUri: URI): List<PathElement> {
-    securityManager.checkResolveResource(baseUri)
-    return doListElements(baseUri)
-  }
-
-  private fun doListElements(baseUri: URI): List<PathElement> =
-    listResources
-      .computeIfAbsent(baseUri) {
-        CompletableFuture<List<PathElement>>().apply {
-          val request = ListResourcesRequest(Random.nextLong(), evaluatorId, baseUri)
-          transport.send(request) { response ->
-            when (response) {
-              is ListResourcesResponse ->
-                if (response.error != null) {
-                  completeExceptionally(IOException(response.error))
-                } else {
-                  complete(response.pathElements ?: emptyList())
-                }
-              else -> completeExceptionally(ProtocolException("Unexpected response"))
-            }
-          }
-        }
-      }
-      .getUnderlying()
-
-  private fun doRead(uri: URI): ByteArray =
-    readResponses
-      .computeIfAbsent(uri) {
-        CompletableFuture<ByteArray>().apply {
-          val request = ReadResourceRequest(Random.nextLong(), evaluatorId, uri)
-          transport.send(request) { response ->
-            when (response) {
-              is ReadResourceResponse -> {
-                if (response.error != null) {
-                  completeExceptionally(IOException(response.error))
-                } else {
-                  complete(response.contents ?: ByteArray(0))
-                }
-              }
-              else -> {
-                completeExceptionally(ProtocolException("Unexpected response: $response"))
-              }
-            }
-          }
-        }
-      }
-      .getUnderlying()
-}

pkl-server/src/main/kotlin/org/pkl/server/MessageDecoders.kt

@@ -1,31 +0,0 @@
-/*
- * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.pkl.server
-
-import java.io.InputStream
-import org.msgpack.core.MessagePack
-import org.msgpack.core.MessageUnpacker
-
-/** Factory methods for creating [MessageDecoder]s. */
-internal object MessageDecoders {
-  fun from(stream: InputStream): MessageDecoder =
-    MessagePackDecoder(MessagePack.newDefaultUnpacker(stream))
-
-  fun from(unpacker: MessageUnpacker): MessageDecoder = MessagePackDecoder(unpacker)
-
-  fun from(array: ByteArray): MessageDecoder =
-    MessagePackDecoder(MessagePack.newDefaultUnpacker(array))
-}

pkl-server/src/main/kotlin/org/pkl/server/MessagePackDecoder.kt

@@ -1,292 +0,0 @@
-/*
- * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.pkl.server
-
-import java.net.URI
-import java.nio.file.Path
-import java.time.Duration
-import java.util.regex.Pattern
-import org.msgpack.core.MessageTypeException
-import org.msgpack.core.MessageUnpacker
-import org.msgpack.value.Value
-import org.msgpack.value.impl.ImmutableStringValueImpl
-import org.pkl.core.evaluatorSettings.PklEvaluatorSettings
-import org.pkl.core.module.PathElement
-import org.pkl.core.packages.Checksums
-
-internal class MessagePackDecoder(private val unpacker: MessageUnpacker) : MessageDecoder {
-  override fun decode(): Message? {
-    if (!unpacker.hasNext()) return null
-
-    val code =
-      try {
-        val arraySize = unpacker.unpackArrayHeader()
-        if (arraySize != 2) {
-          throw DecodeException("Malformed message header (expected size 2, but got $arraySize).")
-        }
-        unpacker.unpackInt()
-      } catch (e: MessageTypeException) {
-        throw DecodeException("Malformed message header.", e)
-      }
-
-    return try {
-      val map = unpacker.unpackValue().asMapValue().map()
-      when (code) {
-        MessageType.CREATE_EVALUATOR_REQUEST.code -> {
-          CreateEvaluatorRequest(
-            requestId = map.get("requestId").asIntegerValue().asLong(),
-            allowedModules = map.unpackStringListOrNull("allowedModules")?.map(Pattern::compile),
-            allowedResources =
-              map.unpackStringListOrNull("allowedResources")?.map(Pattern::compile),
-            clientModuleReaders = map.unpackModuleReaderSpec(),
-            clientResourceReaders = map.unpackResourceReaderSpec(),
-            modulePaths = map.unpackStringListOrNull("modulePaths")?.map(Path::of),
-            env = map.unpackStringMapOrNull("env"),
-            properties = map.unpackStringMapOrNull("properties"),
-            timeout = map.unpackLongOrNull("timeoutSeconds")?.let(Duration::ofSeconds),
-            rootDir = map.unpackStringOrNull("rootDir")?.let(Path::of),
-            cacheDir = map.unpackStringOrNull("cacheDir")?.let(Path::of),
-            outputFormat = map.unpackStringOrNull("outputFormat"),
-            project = map.unpackProject(),
-            http = map.unpackHttp(),
-          )
-        }
-        MessageType.CREATE_EVALUATOR_RESPONSE.code -> {
-          CreateEvaluatorResponse(
-            requestId = map.unpackLong("requestId"),
-            evaluatorId = map.unpackLongOrNull("evaluatorId"),
-            error = map.unpackStringOrNull("error")
-          )
-        }
-        MessageType.CLOSE_EVALUATOR.code -> {
-          CloseEvaluator(evaluatorId = map.unpackLong("evaluatorId"))
-        }
-        MessageType.EVALUATE_REQUEST.code -> {
-          EvaluateRequest(
-            requestId = map.unpackLong("requestId"),
-            evaluatorId = map.unpackLong("evaluatorId"),
-            moduleUri = map.unpackString("moduleUri").let(::URI),
-            moduleText = map.unpackStringOrNull("moduleText"),
-            expr = map.unpackStringOrNull("expr")
-          )
-        }
-        MessageType.EVALUATE_RESPONSE.code -> {
-          EvaluateResponse(
-            requestId = map.unpackLong("requestId"),
-            evaluatorId = map.unpackLong("evaluatorId"),
-            result = map.unpackByteArrayOrNull("result"),
-            error = map.unpackStringOrNull("error")
-          )
-        }
-        MessageType.LOG_MESSAGE.code -> {
-          LogMessage(
-            evaluatorId = map.unpackLong("evaluatorId"),
-            level = map.unpackIntValue("level"),
-            message = map.unpackString("message"),
-            frameUri = map.unpackString("frameUri")
-          )
-        }
-        MessageType.READ_RESOURCE_REQUEST.code -> {
-          ReadResourceRequest(
-            requestId = map.unpackLong("requestId"),
-            evaluatorId = map.unpackLong("evaluatorId"),
-            uri = map.unpackString("uri").let(::URI)
-          )
-        }
-        MessageType.READ_RESOURCE_RESPONSE.code -> {
-          ReadResourceResponse(
-            requestId = map.unpackLong("requestId"),
-            evaluatorId = map.unpackLong("evaluatorId"),
-            contents = map.unpackByteArrayOrNull("contents"),
-            error = map.unpackStringOrNull("error")
-          )
-        }
-        MessageType.READ_MODULE_REQUEST.code -> {
-          ReadModuleRequest(
-            requestId = map.unpackLong("requestId"),
-            evaluatorId = map.unpackLong("evaluatorId"),
-            uri = map.unpackString("uri").let(::URI)
-          )
-        }
-        MessageType.READ_MODULE_RESPONSE.code -> {
-          ReadModuleResponse(
-            requestId = map.unpackLong("requestId"),
-            evaluatorId = map.unpackLong("evaluatorId"),
-            contents = map.unpackStringOrNull("contents"),
-            error = map.unpackStringOrNull("error")
-          )
-        }
-        MessageType.LIST_MODULES_REQUEST.code -> {
-          ListModulesRequest(
-            requestId = map.unpackLong("requestId"),
-            evaluatorId = map.unpackLong("evaluatorId"),
-            uri = map.unpackString("uri").let(::URI)
-          )
-        }
-        MessageType.LIST_MODULES_RESPONSE.code -> {
-          ListModulesResponse(
-            requestId = map.unpackLong("requestId"),
-            evaluatorId = map.unpackLong("evaluatorId"),
-            pathElements = map.unpackPathElements("pathElements"),
-            error = map.unpackStringOrNull("error")
-          )
-        }
-        MessageType.LIST_RESOURCES_REQUEST.code -> {
-          ListResourcesRequest(
-            requestId = map.unpackLong("requestId"),
-            evaluatorId = map.unpackLong("evaluatorId"),
-            uri = map.unpackString("uri").let(::URI)
-          )
-        }
-        MessageType.LIST_RESOURCES_RESPONSE.code -> {
-          ListResourcesResponse(
-            requestId = map.unpackLong("requestId"),
-            evaluatorId = map.unpackLong("evaluatorId"),
-            pathElements = map.unpackPathElements("pathElements"),
-            error = map.unpackStringOrNull("error")
-          )
-        }
-        else -> throw ProtocolException("Invalid message code: $code")
-      }
-    } catch (e: MessageTypeException) {
-      throw DecodeException("Malformed message body for message with code `$code`.", e)
-    }
-  }
-
-  private fun Array<Value>.unpackValueOrNull(key: String): Value? {
-    for (i in indices.step(2)) {
-      val currKey = this[i].asStringValue().asString()
-      if (currKey == key) return this[i + 1]
-    }
-    return null
-  }
-
-  private fun Map<Value, Value>.getNullable(key: String): Value? =
-    this[ImmutableStringValueImpl(key)]
-
-  private fun Map<Value, Value>.get(key: String): Value =
-    getNullable(key) ?: throw DecodeException("Missing message parameter `$key`")
-
-  private fun Array<Value>.unpackValue(key: String): Value =
-    unpackValueOrNull(key) ?: throw DecodeException("Missing message parameter `$key`.")
-
-  private fun Map<Value, Value>.unpackStringListOrNull(key: String): List<String>? {
-    val value = getNullable(key) ?: return null
-    return value.asArrayValue().map { it.asStringValue().asString() }
-  }
-
-  private fun Map<Value, Value>.unpackStringMapOrNull(key: String): Map<String, String>? {
-    val value = getNullable(key) ?: return null
-    return value.asMapValue().entrySet().associate { (k, v) ->
-      k.asStringValue().asString() to v.asStringValue().asString()
-    }
-  }
-
-  private fun Map<Value, Value>.unpackLong(key: String): Long = get(key).asIntegerValue().asLong()
-
-  private fun Map<Value, Value>.unpackBoolean(key: String): Boolean =
-    get(key).asBooleanValue().boolean
-
-  private fun Map<Value, Value>.unpackBooleanOrNull(key: String): Boolean? =
-    getNullable(key)?.asBooleanValue()?.boolean
-
-  private fun Map<Value, Value>.unpackLongOrNull(key: String): Long? =
-    getNullable(key)?.asIntegerValue()?.asLong()
-
-  private fun Map<Value, Value>.unpackIntValue(key: String): Int = get(key).asIntegerValue().asInt()
-
-  private fun Map<Value, Value>.unpackString(key: String): String =
-    get(key).asStringValue().asString()
-
-  private fun Map<Value, Value>.unpackStringOrNull(key: String): String? =
-    getNullable(key)?.asStringValue()?.asString()
-
-  private fun Map<Value, Value>.unpackByteArrayOrNull(key: String): ByteArray? =
-    getNullable(key)?.asBinaryValue()?.asByteArray()
-
-  private fun Map<Value, Value>.unpackPathElements(key: String): List<PathElement>? =
-    getNullable(key)?.asArrayValue()?.map { pathElement ->
-      val map = pathElement.asMapValue().map()
-      PathElement(map.unpackString("name"), map.unpackBoolean("isDirectory"))
-    }
-
-  private fun Map<Value, Value>.unpackModuleReaderSpec(): List<ModuleReaderSpec>? {
-    val keys = getNullable("clientModuleReaders") ?: return null
-    return keys.asArrayValue().toList().map { value ->
-      val readerMap = value.asMapValue().map()
-      ModuleReaderSpec(
-        scheme = readerMap.unpackString("scheme"),
-        hasHierarchicalUris = readerMap.unpackBoolean("hasHierarchicalUris"),
-        isLocal = readerMap.unpackBoolean("isLocal"),
-        isGlobbable = readerMap.unpackBoolean("isGlobbable")
-      )
-    }
-  }
-
-  private fun Map<Value, Value>.unpackResourceReaderSpec(): List<ResourceReaderSpec> {
-    val keys = getNullable("clientResourceReaders") ?: return emptyList()
-    return keys.asArrayValue().toList().map { value ->
-      val readerMap = value.asMapValue().map()
-      ResourceReaderSpec(
-        scheme = readerMap.unpackString("scheme"),
-        hasHierarchicalUris = readerMap.unpackBoolean("hasHierarchicalUris"),
-        isGlobbable = readerMap.unpackBoolean("isGlobbable")
-      )
-    }
-  }
-
-  private fun Map<Value, Value>.unpackProject(): Project? {
-    val projMap = getNullable("project")?.asMapValue()?.map() ?: return null
-    val projectFileUri = URI(projMap.unpackString("projectFileUri"))
-    val dependencies = projMap.unpackDependencies("dependencies")
-    return Project(projectFileUri, null, dependencies)
-  }
-
-  private fun Map<Value, Value>.unpackHttp(): Http? {
-    val httpMap = getNullable("http")?.asMapValue()?.map() ?: return null
-    val proxy = httpMap.unpackProxy()
-    val caCertificates = httpMap.getNullable("caCertificates")?.asBinaryValue()?.asByteArray()
-    return Http(caCertificates, proxy)
-  }
-
-  private fun Map<Value, Value>.unpackProxy(): PklEvaluatorSettings.Proxy? {
-    val proxyMap = getNullable("proxy")?.asMapValue()?.map() ?: return null
-    val address = proxyMap.unpackString("address")
-    val noProxy = proxyMap.unpackStringListOrNull("noProxy")
-    return PklEvaluatorSettings.Proxy.create(address, noProxy)
-  }
-
-  private fun Map<Value, Value>.unpackDependencies(name: String): Map<String, Dependency> {
-    val mapValue = get(name).asMapValue().map()
-    return mapValue.entries.associate { (key, value) ->
-      val dependencyName = key.asStringValue().asString()
-      val dependencyObj = value.asMapValue().map()
-      val type = dependencyObj.unpackString("type")
-      val packageUri = URI(dependencyObj.unpackString("packageUri"))
-      if (type == DependencyType.REMOTE.value) {
-        val checksums =
-          dependencyObj.getNullable("checksums")?.asMapValue()?.map()?.let { obj ->
-            val sha256 = obj.unpackString("sha256")
-            Checksums(sha256)
-          }
-        return@associate dependencyName to RemoteDependency(packageUri, checksums)
-      }
-      val dependencies = dependencyObj.unpackDependencies("dependencies")
-      val projectFileUri = dependencyObj.unpackString("projectFileUri")
-      dependencyName to Project(URI(projectFileUri), packageUri, dependencies)
-    }
-  }
-}

pkl-server/src/main/kotlin/org/pkl/server/MessagePackEncoder.kt

@@ -1,332 +0,0 @@
-/*
- * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.pkl.server
-
-import kotlin.io.path.pathString
-import org.msgpack.core.MessagePacker
-import org.pkl.core.module.PathElement
-import org.pkl.core.packages.Checksums
-
-internal class MessagePackEncoder(private val packer: MessagePacker) : MessageEncoder {
-  private fun MessagePacker.packModuleReaderSpec(reader: ModuleReaderSpec) {
-    packMapHeader(4)
-    packKeyValue("scheme", reader.scheme)
-    packKeyValue("hasHierarchicalUris", reader.hasHierarchicalUris)
-    packKeyValue("isLocal", reader.isLocal)
-    packKeyValue("isGlobbable", reader.isGlobbable)
-  }
-
-  private fun MessagePacker.packResourceReaderSpec(reader: ResourceReaderSpec) {
-    packMapHeader(3)
-    packKeyValue("scheme", reader.scheme)
-    packKeyValue("hasHierarchicalUris", reader.hasHierarchicalUris)
-    packKeyValue("isGlobbable", reader.isGlobbable)
-  }
-
-  private fun MessagePacker.packPathElement(pathElement: PathElement) {
-    packMapHeader(2)
-    packKeyValue("name", pathElement.name)
-    packKeyValue("isDirectory", pathElement.isDirectory)
-  }
-
-  private fun MessagePacker.packProject(project: Project) {
-    packMapHeader(2)
-    packKeyValue("projectFileUri", project.projectFileUri.toString())
-    packString("dependencies")
-    packDependencies(project.dependencies)
-  }
-
-  private fun MessagePacker.packHttp(http: Http) {
-    if ((http.caCertificates ?: http.proxy) == null) {
-      packMapHeader(0)
-      return
-    }
-    packMapHeader(0, http.caCertificates, http.proxy)
-    packKeyValue("caCertificates", http.caCertificates)
-    http.proxy?.let { proxy ->
-      packString("proxy")
-      packMapHeader(0, proxy.address, proxy.noProxy)
-      packKeyValue("address", proxy.address?.toString())
-      packKeyValue("noProxy", proxy.noProxy)
-    }
-  }
-
-  private fun MessagePacker.packDependencies(dependencies: Map<String, Dependency>) {
-    packMapHeader(dependencies.size)
-    for ((name, dep) in dependencies) {
-      packString(name)
-      if (dep is Project) {
-        packMapHeader(4)
-        packKeyValue("type", dep.type.value)
-        packKeyValue("packageUri", dep.packageUri.toString())
-        packKeyValue("projectFileUri", dep.projectFileUri.toString())
-        packString("dependencies")
-        packDependencies(dep.dependencies)
-      } else {
-        dep as RemoteDependency
-        packMapHeader(dep.checksums?.let { 3 } ?: 2)
-        packKeyValue("type", dep.type.value)
-        packKeyValue("packageUri", dep.packageUri.toString())
-        dep.checksums?.let { checksums ->
-          packString("checksums")
-          packChecksums(checksums)
-        }
-      }
-    }
-  }
-
-  private fun MessagePacker.packChecksums(checksums: Checksums) {
-    packMapHeader(1)
-    packKeyValue("sha256", checksums.sha256)
-  }
-
-  override fun encode(msg: Message) =
-    with(packer) {
-      packArrayHeader(2)
-      packInt(msg.type.code)
-
-      @Suppress("DuplicatedCode")
-      when (msg.type.code) {
-        MessageType.CREATE_EVALUATOR_REQUEST.code -> {
-          msg as CreateEvaluatorRequest
-          packMapHeader(
-            8,
-            msg.timeout,
-            msg.rootDir,
-            msg.cacheDir,
-            msg.outputFormat,
-            msg.project,
-            msg.http
-          )
-          packKeyValue("requestId", msg.requestId)
-          packKeyValue("allowedModules", msg.allowedModules?.map { it.toString() })
-          packKeyValue("allowedResources", msg.allowedResources?.map { it.toString() })
-          if (msg.clientModuleReaders != null) {
-            packString("clientModuleReaders")
-            packArrayHeader(msg.clientModuleReaders.size)
-            for (moduleReader in msg.clientModuleReaders) {
-              packModuleReaderSpec(moduleReader)
-            }
-          }
-          if (msg.clientResourceReaders != null) {
-            packString("clientResourceReaders")
-            packArrayHeader(msg.clientResourceReaders.size)
-            for (resourceReader in msg.clientResourceReaders) {
-              packResourceReaderSpec(resourceReader)
-            }
-          }
-          packKeyValue("modulePaths", msg.modulePaths?.map { it.pathString })
-          packKeyValue("env", msg.env)
-          packKeyValue("properties", msg.properties)
-          packKeyValue("timeoutSeconds", msg.timeout?.toSeconds())
-          packKeyValue("rootDir", msg.rootDir?.pathString)
-          packKeyValue("cacheDir", msg.cacheDir?.pathString)
-          packKeyValue("outputFormat", msg.outputFormat)
-          if (msg.project != null) {
-            packString("project")
-            packProject(msg.project)
-          }
-          if (msg.http != null) {
-            packString("http")
-            packHttp(msg.http)
-          }
-        }
-        MessageType.CREATE_EVALUATOR_RESPONSE.code -> {
-          msg as CreateEvaluatorResponse
-          packMapHeader(1, msg.evaluatorId, msg.error)
-          packKeyValue("requestId", msg.requestId)
-          packKeyValue("evaluatorId", msg.evaluatorId)
-          packKeyValue("error", msg.error)
-        }
-        MessageType.CLOSE_EVALUATOR.code -> {
-          msg as CloseEvaluator
-          packMapHeader(1)
-          packKeyValue("evaluatorId", msg.evaluatorId)
-        }
-        MessageType.EVALUATE_REQUEST.code -> {
-          msg as EvaluateRequest
-          packMapHeader(3, msg.moduleText, msg.expr)
-          packKeyValue("requestId", msg.requestId)
-          packKeyValue("evaluatorId", msg.evaluatorId)
-          packKeyValue("moduleUri", msg.moduleUri.toString())
-          packKeyValue("moduleText", msg.moduleText)
-          packKeyValue("expr", msg.expr)
-        }
-        MessageType.EVALUATE_RESPONSE.code -> {
-          msg as EvaluateResponse
-          packMapHeader(2, msg.result, msg.error)
-          packKeyValue("requestId", msg.requestId)
-          packKeyValue("evaluatorId", msg.evaluatorId)
-          packKeyValue("result", msg.result)
-          packKeyValue("error", msg.error)
-        }
-        MessageType.LOG_MESSAGE.code -> {
-          msg as LogMessage
-          packMapHeader(4)
-          packKeyValue("evaluatorId", msg.evaluatorId)
-          packKeyValue("level", msg.level)
-          packKeyValue("message", msg.message)
-          packKeyValue("frameUri", msg.frameUri)
-        }
-        MessageType.READ_RESOURCE_REQUEST.code -> {
-          msg as ReadResourceRequest
-          packMapHeader(3)
-          packKeyValue("requestId", msg.requestId)
-          packKeyValue("evaluatorId", msg.evaluatorId)
-          packKeyValue("uri", msg.uri.toString())
-        }
-        MessageType.READ_RESOURCE_RESPONSE.code -> {
-          msg as ReadResourceResponse
-          packMapHeader(2, msg.contents, msg.error)
-          packKeyValue("requestId", msg.requestId)
-          packKeyValue("evaluatorId", msg.evaluatorId)
-          packKeyValue("contents", msg.contents)
-          packKeyValue("error", msg.error)
-        }
-        MessageType.READ_MODULE_REQUEST.code -> {
-          msg as ReadModuleRequest
-          packMapHeader(3)
-          packKeyValue("requestId", msg.requestId)
-          packKeyValue("evaluatorId", msg.evaluatorId)
-          packKeyValue("uri", msg.uri.toString())
-        }
-        MessageType.READ_MODULE_RESPONSE.code -> {
-          msg as ReadModuleResponse
-          packMapHeader(2, msg.contents, msg.error)
-          packKeyValue("requestId", msg.requestId)
-          packKeyValue("evaluatorId", msg.evaluatorId)
-          packKeyValue("contents", msg.contents)
-          packKeyValue("error", msg.error)
-        }
-        MessageType.LIST_MODULES_REQUEST.code -> {
-          msg as ListModulesRequest
-          packMapHeader(3)
-          packKeyValue("requestId", msg.requestId)
-          packKeyValue("evaluatorId", msg.evaluatorId)
-          packKeyValue("uri", msg.uri.toString())
-        }
-        MessageType.LIST_MODULES_RESPONSE.code -> {
-          msg as ListModulesResponse
-          packMapHeader(2, msg.pathElements, msg.error)
-          packKeyValue("requestId", msg.requestId)
-          packKeyValue("evaluatorId", msg.evaluatorId)
-          if (msg.pathElements != null) {
-            packString("pathElements")
-            packArrayHeader(msg.pathElements.size)
-            for (pathElement in msg.pathElements) {
-              packPathElement(pathElement)
-            }
-          }
-          packKeyValue("error", msg.error)
-        }
-        MessageType.LIST_RESOURCES_REQUEST.code -> {
-          msg as ListResourcesRequest
-          packMapHeader(3)
-          packKeyValue("requestId", msg.requestId)
-          packKeyValue("evaluatorId", msg.evaluatorId)
-          packKeyValue("uri", msg.uri.toString())
-        }
-        MessageType.LIST_RESOURCES_RESPONSE.code -> {
-          msg as ListResourcesResponse
-          packMapHeader(2, msg.pathElements, msg.error)
-          packKeyValue("requestId", msg.requestId)
-          packKeyValue("evaluatorId", msg.evaluatorId)
-          if (msg.pathElements != null) {
-            packString("pathElements")
-            packArrayHeader(msg.pathElements.size)
-            for (pathElement in msg.pathElements) {
-              packPathElement(pathElement)
-            }
-          }
-          packKeyValue("error", msg.error)
-        }
-        else -> {
-          throw RuntimeException("Missing encoding for ${msg.javaClass.simpleName}")
-        }
-      }
-
-      flush()
-    }
-
-  private fun MessagePacker.packMapHeader(size: Int, value1: Any?, value2: Any?) =
-    packMapHeader(size + (if (value1 != null) 1 else 0) + (if (value2 != null) 1 else 0))
-
-  private fun MessagePacker.packMapHeader(
-    size: Int,
-    value1: Any?,
-    value2: Any?,
-    value3: Any?,
-    value4: Any?,
-    value5: Any?,
-    value6: Any?
-  ) =
-    packMapHeader(
-      size +
-        (if (value1 != null) 1 else 0) +
-        (if (value2 != null) 1 else 0) +
-        (if (value3 != null) 1 else 0) +
-        (if (value4 != null) 1 else 0) +
-        (if (value5 != null) 1 else 0) +
-        (if (value6 != null) 1 else 0)
-    )
-
-  private fun MessagePacker.packKeyValue(name: String, value: Int?) {
-    if (value == null) return
-    packString(name)
-    packInt(value)
-  }
-
-  private fun MessagePacker.packKeyValue(name: String, value: Long?) {
-    if (value == null) return
-    packString(name)
-    packLong(value)
-  }
-
-  private fun MessagePacker.packKeyValue(name: String, value: String?) {
-    if (value == null) return
-    packString(name)
-    packString(value)
-  }
-
-  private fun MessagePacker.packKeyValue(name: String, value: Collection<String>?) {
-    if (value == null) return
-    packString(name)
-    packArrayHeader(value.size)
-    for (elem in value) packString(elem)
-  }
-
-  private fun MessagePacker.packKeyValue(name: String, value: Map<String, String>?) {
-    if (value == null) return
-    packString(name)
-    packMapHeader(value.size)
-    for ((k, v) in value) {
-      packString(k)
-      packString(v)
-    }
-  }
-
-  private fun MessagePacker.packKeyValue(name: String, value: ByteArray?) {
-    if (value == null) return
-    packString(name)
-    packBinaryHeader(value.size)
-    writePayload(value)
-  }
-
-  private fun MessagePacker.packKeyValue(name: String, value: Boolean) {
-    packString(name)
-    packBoolean(value)
-  }
-}

pkl-server/src/main/kotlin/org/pkl/server/MessageTransports.kt

@@ -1,135 +0,0 @@
-/*
- * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.pkl.server
-
-import java.io.InputStream
-import java.io.OutputStream
-import java.util.concurrent.ConcurrentHashMap
-
-/** Factory methods for creating [MessageTransport]s. */
-object MessageTransports {
-  /** Creates a message transport that reads from [inputStream] and writes to [outputStream]. */
-  fun stream(inputStream: InputStream, outputStream: OutputStream): MessageTransport {
-    return EncodingMessageTransport(
-      MessageDecoders.from(inputStream),
-      MessageEncoders.into(outputStream)
-    )
-  }
-
-  /** Creates "client" and "server" transports that are directly connected to each other. */
-  fun direct(): Pair<MessageTransport, MessageTransport> {
-    val transport1 = DirectMessageTransport()
-    val transport2 = DirectMessageTransport()
-    transport1.other = transport2
-    transport2.other = transport1
-    return transport1 to transport2
-  }
-
-  internal class EncodingMessageTransport(
-    private val decoder: MessageDecoder,
-    private val encoder: MessageEncoder,
-  ) : AbstractMessageTransport() {
-    @Volatile private var isClosed: Boolean = false
-
-    override fun doStart() {
-      while (!isClosed) {
-        val message = decoder.decode() ?: return
-        accept(message)
-      }
-    }
-
-    override fun doClose() {
-      isClosed = true
-    }
-
-    override fun doSend(message: Message) {
-      encoder.encode(message)
-    }
-  }
-
-  internal class DirectMessageTransport : AbstractMessageTransport() {
-    lateinit var other: DirectMessageTransport
-
-    override fun doStart() {}
-
-    override fun doClose() {}
-
-    override fun doSend(message: Message) {
-      other.accept(message)
-    }
-  }
-
-  // TODO: clean up callbacks if evaluation fails for some reason (ThreadInterrupt, timeout, etc)
-  internal abstract class AbstractMessageTransport : MessageTransport {
-    private lateinit var oneWayHandler: (OneWayMessage) -> Unit
-    private lateinit var requestHandler: (RequestMessage) -> Unit
-    private val responseHandlers: MutableMap<Long, (ResponseMessage) -> Unit> = ConcurrentHashMap()
-
-    protected abstract fun doStart()
-
-    protected abstract fun doClose()
-
-    protected abstract fun doSend(message: Message)
-
-    protected fun accept(message: Message) {
-      log("Received message: $message")
-      when (message) {
-        is OneWayMessage -> oneWayHandler(message)
-        is RequestMessage -> requestHandler(message)
-        is ResponseMessage -> {
-          val handler =
-            responseHandlers.remove(message.requestId)
-              ?: throw ProtocolException(
-                "Received response ${message.javaClass.simpleName} for unknown request ID `${message.requestId}`."
-              )
-          handler(message)
-        }
-      }
-    }
-
-    final override fun start(
-      oneWayHandler: (OneWayMessage) -> Unit,
-      requestHandler: (RequestMessage) -> Unit
-    ) {
-      log("Starting transport: $this")
-      this.oneWayHandler = oneWayHandler
-      this.requestHandler = requestHandler
-      doStart()
-    }
-
-    final override fun close() {
-      log("Closing transport: $this")
-      doClose()
-      responseHandlers.clear()
-    }
-
-    override fun send(message: OneWayMessage) {
-      log("Sending message: $message")
-      doSend(message)
-    }
-
-    override fun send(message: RequestMessage, responseHandler: (ResponseMessage) -> Unit) {
-      log("Sending message: $message")
-      responseHandlers[message.requestId] = responseHandler
-      return doSend(message)
-    }
-
-    override fun send(message: ResponseMessage) {
-      log("Sending message: $message")
-      doSend(message)
-    }
-  }
-}

pkl-server/src/main/kotlin/org/pkl/server/Server.kt

@@ -15,18 +15,27 @@
  */
 package org.pkl.server
 
+import java.io.InputStream
+import java.io.OutputStream
 import java.net.URI
 import java.util.concurrent.ConcurrentHashMap
 import java.util.concurrent.ExecutorService
 import java.util.concurrent.Executors
 import kotlin.random.Random
 import org.pkl.core.*
+import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader
+import org.pkl.core.externalreader.ExternalReaderProcess
+import org.pkl.core.externalreader.ExternalReaderProcessImpl
 import org.pkl.core.http.HttpClient
+import org.pkl.core.messaging.MessageTransport
+import org.pkl.core.messaging.MessageTransports
+import org.pkl.core.messaging.ProtocolException
 import org.pkl.core.module.ModuleKeyFactories
 import org.pkl.core.module.ModuleKeyFactory
 import org.pkl.core.module.ModulePathResolver
 import org.pkl.core.packages.PackageUri
 import org.pkl.core.project.DeclaredDependencies
+import org.pkl.core.resource.ExternalResourceResolver
 import org.pkl.core.resource.ResourceReader
 import org.pkl.core.resource.ResourceReaders
 import org.pkl.core.util.IoUtils
@@ -37,6 +46,22 @@ class Server(private val transport: MessageTransport) : AutoCloseable {
   // https://github.com/jano7/executor would be the perfect executor here
   private val executor: ExecutorService = Executors.newSingleThreadExecutor()
 
+  // ExternalProcess instances with the same ExternalReader spec are shared per evaluator
+  private val externalReaderProcesses:
+    MutableMap<Long, MutableMap<ExternalReader, ExternalReaderProcess>> =
+    ConcurrentHashMap()
+
+  companion object {
+    fun stream(inputStream: InputStream, outputStream: OutputStream): Server =
+      Server(
+        MessageTransports.stream(
+          ServerMessagePackDecoder(inputStream),
+          ServerMessagePackEncoder(outputStream),
+          ::log
+        )
+      )
+  }
+
   /** Starts listening to incoming messages */
   fun start() {
     transport.start(
@@ -71,13 +96,13 @@ class Server(private val transport: MessageTransport) : AutoCloseable {
 
   private fun handleCreateEvaluator(message: CreateEvaluatorRequest) {
     val evaluatorId = Random.Default.nextLong()
-    val baseResponse = CreateEvaluatorResponse(message.requestId, evaluatorId = null, error = null)
+    val baseResponse = CreateEvaluatorResponse(message.requestId(), null, null)
 
     val evaluator =
       try {
         createEvaluator(message, evaluatorId)
-      } catch (e: ServerException) {
-        transport.send(baseResponse.copy(error = e.message))
+      } catch (e: ProtocolException) {
+        transport.send(baseResponse.copy(error = e.message ?: ""))
         return
       }
 
@@ -86,7 +111,7 @@ class Server(private val transport: MessageTransport) : AutoCloseable {
   }
 
   private fun handleEvaluate(msg: EvaluateRequest) {
-    val baseResponse = EvaluateResponse(msg.requestId, msg.evaluatorId, result = null, error = null)
+    val baseResponse = EvaluateResponse(msg.requestId(), msg.evaluatorId, null, null)
 
     val evaluator = evaluators[msg.evaluatorId]
     if (evaluator == null) {
@@ -103,7 +128,7 @@ class Server(private val transport: MessageTransport) : AutoCloseable {
       } catch (e: PklBugException) {
         transport.send(baseResponse.copy(error = e.toString()))
       } catch (e: PklException) {
-        transport.send(baseResponse.copy(error = e.message))
+        transport.send(baseResponse.copy(error = e.message ?: ""))
       }
     }
   }
@@ -115,6 +140,9 @@ class Server(private val transport: MessageTransport) : AutoCloseable {
       return
     }
     evaluator.close()
+
+    // close any running ExternalProcess instances for the closed evaluator
+    externalReaderProcesses[message.evaluatorId]?.values?.forEach { it.close() }
   }
 
   private fun buildDeclaredDependencies(
@@ -167,8 +195,9 @@ class Server(private val transport: MessageTransport) : AutoCloseable {
         message.http?.proxy?.let { proxy ->
           setProxy(proxy.address, proxy.noProxy ?: listOf())
           proxy.address?.let(IoUtils::setSystemProxy)
+          proxy.noProxy?.let { System.setProperty("http.nonProxyHosts", it.joinToString("|")) }
         }
-        message.http?.caCertificates?.let { caCertificates -> addCertificates(caCertificates) }
+        message.http?.caCertificates?.let(::addCertificates)
         buildLazily()
       }
     val dependencies =
@@ -210,10 +239,19 @@ class Server(private val transport: MessageTransport) : AutoCloseable {
     add(ResourceReaders.pkg())
     add(ResourceReaders.projectpackage())
     add(ResourceReaders.modulePath(modulePathResolver))
+    for ((scheme, spec) in message.externalResourceReaders ?: emptyMap()) {
+      add(
+        ResourceReaders.externalProcess(scheme, getExternalProcess(evaluatorId, spec), evaluatorId)
+      )
+    }
     // add client-side resource readers last to ensure they win over builtin ones
     for (readerSpec in message.clientResourceReaders ?: emptyList()) {
-      val resourceReader = ClientResourceReader(transport, evaluatorId, readerSpec)
-      add(resourceReader)
+      add(
+        ResourceReaders.externalResolver(
+          readerSpec,
+          ExternalResourceResolver(transport, evaluatorId)
+        )
+      )
     }
   }
 
@@ -226,6 +264,15 @@ class Server(private val transport: MessageTransport) : AutoCloseable {
     if (message.clientModuleReaders?.isNotEmpty() == true) {
       add(ClientModuleKeyFactory(message.clientModuleReaders, transport, evaluatorId))
     }
+    for ((scheme, spec) in message.externalModuleReaders ?: emptyMap()) {
+      add(
+        ModuleKeyFactories.externalProcess(
+          scheme,
+          getExternalProcess(evaluatorId, spec),
+          evaluatorId
+        )
+      )
+    }
     add(ModuleKeyFactories.standardLibrary)
     addAll(ModuleKeyFactories.fromServiceProviders())
     add(ModuleKeyFactories.file)
@@ -235,4 +282,9 @@ class Server(private val transport: MessageTransport) : AutoCloseable {
     add(ModuleKeyFactories.http)
     add(ModuleKeyFactories.genericUrl)
   }
+
+  private fun getExternalProcess(evaluatorId: Long, spec: ExternalReader): ExternalReaderProcess =
+    externalReaderProcesses
+      .computeIfAbsent(evaluatorId) { ConcurrentHashMap() }
+      .computeIfAbsent(spec) { ExternalReaderProcessImpl(it) }
 }

pkl-server/src/main/kotlin/org/pkl/server/ServerMessagePackDecoder.kt

@@ -0,0 +1,134 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.server
+
+import java.io.InputStream
+import java.net.URI
+import java.nio.file.Path
+import java.time.Duration
+import java.util.regex.Pattern
+import org.msgpack.core.MessagePack
+import org.msgpack.core.MessageUnpacker
+import org.msgpack.value.Value
+import org.pkl.core.evaluatorSettings.PklEvaluatorSettings
+import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader
+import org.pkl.core.messaging.BaseMessagePackDecoder
+import org.pkl.core.messaging.Message
+import org.pkl.core.packages.Checksums
+
+class ServerMessagePackDecoder(unpacker: MessageUnpacker) : BaseMessagePackDecoder(unpacker) {
+
+  constructor(stream: InputStream) : this(MessagePack.newDefaultUnpacker(stream))
+
+  override fun decodeMessage(msgType: Message.Type, map: Map<Value, Value>): Message? {
+    return when (msgType) {
+      Message.Type.CREATE_EVALUATOR_REQUEST ->
+        CreateEvaluatorRequest(
+          get(map, "requestId").asIntegerValue().asLong(),
+          unpackStringListOrNull(map, "allowedModules", Pattern::compile),
+          unpackStringListOrNull(map, "allowedResources", Pattern::compile),
+          unpackListOrNull(map, "clientModuleReaders") { unpackModuleReaderSpec(it)!! },
+          unpackListOrNull(map, "clientResourceReaders") { unpackResourceReaderSpec(it)!! },
+          unpackStringListOrNull(map, "modulePaths", Path::of),
+          unpackStringMapOrNull(map, "env"),
+          unpackStringMapOrNull(map, "properties"),
+          unpackLongOrNull(map, "timeoutSeconds", Duration::ofSeconds),
+          unpackStringOrNull(map, "rootDir", Path::of),
+          unpackStringOrNull(map, "cacheDir", Path::of),
+          unpackStringOrNull(map, "outputFormat"),
+          map.unpackProject(),
+          map.unpackHttp(),
+          unpackStringMapOrNull(map, "externalModuleReaders", ::unpackExternalReader),
+          unpackStringMapOrNull(map, "externalResourceReaders", ::unpackExternalReader)
+        )
+      Message.Type.CREATE_EVALUATOR_RESPONSE ->
+        CreateEvaluatorResponse(
+          unpackLong(map, "requestId"),
+          unpackLongOrNull(map, "evaluatorId"),
+          unpackStringOrNull(map, "error")
+        )
+      Message.Type.CLOSE_EVALUATOR -> CloseEvaluator(unpackLong(map, "evaluatorId"))
+      Message.Type.EVALUATE_REQUEST ->
+        EvaluateRequest(
+          unpackLong(map, "requestId"),
+          unpackLong(map, "evaluatorId"),
+          URI(unpackString(map, "moduleUri")),
+          unpackStringOrNull(map, "moduleText"),
+          unpackStringOrNull(map, "expr")
+        )
+      Message.Type.EVALUATE_RESPONSE ->
+        EvaluateResponse(
+          unpackLong(map, "requestId"),
+          unpackLong(map, "evaluatorId"),
+          unpackByteArray(map, "result"),
+          unpackStringOrNull(map, "error")
+        )
+      Message.Type.LOG_MESSAGE ->
+        LogMessage(
+          unpackLong(map, "evaluatorId"),
+          unpackInt(map, "level"),
+          unpackString(map, "message"),
+          unpackString(map, "frameUri")
+        )
+      else -> super.decodeMessage(msgType, map)
+    }
+  }
+
+  private fun Map<Value, Value>.unpackProject(): Project? {
+    val projMap = getNullable(this, "project")?.asMapValue()?.map() ?: return null
+    val projectFileUri = URI(unpackString(projMap, "projectFileUri"))
+    val dependencies = projMap.unpackDependencies("dependencies")
+    return Project(projectFileUri, null, dependencies)
+  }
+
+  private fun Map<Value, Value>.unpackHttp(): Http? {
+    val httpMap = getNullable(this, "http")?.asMapValue()?.map() ?: return null
+    val proxy = httpMap.unpackProxy()
+    val caCertificates = getNullable(httpMap, "caCertificates")?.asBinaryValue()?.asByteArray()
+    return Http(caCertificates, proxy)
+  }
+
+  private fun Map<Value, Value>.unpackProxy(): PklEvaluatorSettings.Proxy? {
+    val proxyMap = getNullable(this, "proxy")?.asMapValue()?.map() ?: return null
+    val address = unpackString(proxyMap, "address")
+    val noProxy = unpackStringListOrNull(proxyMap, "noProxy")
+    return PklEvaluatorSettings.Proxy.create(address, noProxy)
+  }
+
+  private fun Map<Value, Value>.unpackDependencies(name: String): Map<String, Dependency> {
+    val mapValue = get(this, name).asMapValue().map()
+    return mapValue.entries.associate { (key, value) ->
+      val dependencyName = key.asStringValue().asString()
+      val dependencyObj = value.asMapValue().map()
+      val type = unpackString(dependencyObj, "type")
+      val packageUri = URI(unpackString(dependencyObj, "packageUri"))
+      if (type == DependencyType.REMOTE.value) {
+        val checksums =
+          getNullable(dependencyObj, "checksums")?.asMapValue()?.map()?.let { obj ->
+            val sha256 = unpackString(obj, "sha256")
+            Checksums(sha256)
+          }
+        return@associate dependencyName to RemoteDependency(packageUri, checksums)
+      }
+      val dependencies = dependencyObj.unpackDependencies("dependencies")
+      val projectFileUri = unpackString(dependencyObj, "projectFileUri")
+      dependencyName to Project(URI(projectFileUri), packageUri, dependencies)
+    }
+  }
+
+  private fun unpackExternalReader(map: Map<Value, Value>): ExternalReader =
+    ExternalReader(unpackString(map, "executable"), unpackStringListOrNull(map, "arguments")!!)
+}

pkl-server/src/main/kotlin/org/pkl/server/ServerMessagePackEncoder.kt

@@ -0,0 +1,197 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.server
+
+import java.io.OutputStream
+import java.nio.file.Path
+import kotlin.io.path.pathString
+import org.msgpack.core.MessagePack
+import org.msgpack.core.MessagePacker
+import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader
+import org.pkl.core.messaging.BaseMessagePackEncoder
+import org.pkl.core.messaging.Message
+import org.pkl.core.packages.Checksums
+
+class ServerMessagePackEncoder(packer: MessagePacker) : BaseMessagePackEncoder(packer) {
+
+  constructor(stream: OutputStream) : this(MessagePack.newDefaultPacker(stream))
+
+  private fun MessagePacker.packProject(project: Project) {
+    packMapHeader(2)
+    packKeyValue("projectFileUri", project.projectFileUri.toString())
+    packString("dependencies")
+    packDependencies(project.dependencies)
+  }
+
+  private fun MessagePacker.packHttp(http: Http) {
+    packMapHeader(0, http.caCertificates, http.proxy)
+    http.caCertificates?.let { packKeyValue("caCertificates", it) }
+    http.proxy?.let { proxy ->
+      packString("proxy")
+      packMapHeader(0, proxy.address, proxy.noProxy)
+      packKeyValue("address", proxy.address?.toString())
+      packKeyValue("noProxy", proxy.noProxy)
+    }
+  }
+
+  private fun MessagePacker.packDependencies(dependencies: Map<String, Dependency>) {
+    packMapHeader(dependencies.size)
+    for ((name, dep) in dependencies) {
+      packString(name)
+      if (dep is Project) {
+        packMapHeader(4)
+        packKeyValue("type", dep.type.value)
+        packKeyValue("packageUri", dep.packageUri.toString())
+        packKeyValue("projectFileUri", dep.projectFileUri.toString())
+        packString("dependencies")
+        packDependencies(dep.dependencies)
+      } else {
+        dep as RemoteDependency
+        packMapHeader(dep.checksums?.let { 3 } ?: 2)
+        packKeyValue("type", dep.type.value)
+        packKeyValue("packageUri", dep.packageUri.toString())
+        dep.checksums?.let { checksums ->
+          packString("checksums")
+          packChecksums(checksums)
+        }
+      }
+    }
+  }
+
+  private fun MessagePacker.packChecksums(checksums: Checksums) {
+    packMapHeader(1)
+    packKeyValue("sha256", checksums.sha256)
+  }
+
+  private fun MessagePacker.packExternalReader(spec: ExternalReader) {
+    packMapHeader(1, spec.arguments)
+    packKeyValue("executable", spec.executable)
+    spec.arguments?.let { packKeyValue("arguments", it) }
+  }
+
+  override fun encodeMessage(msg: Message) {
+    when (msg.type()) {
+      Message.Type.CREATE_EVALUATOR_REQUEST -> {
+        msg as CreateEvaluatorRequest
+        packMapHeader(
+          1,
+          msg.allowedModules,
+          msg.allowedResources,
+          msg.clientModuleReaders,
+          msg.clientResourceReaders,
+          msg.modulePaths,
+          msg.env,
+          msg.properties,
+          msg.timeout,
+          msg.rootDir,
+          msg.cacheDir,
+          msg.outputFormat,
+          msg.project,
+          msg.http,
+          msg.externalModuleReaders,
+          msg.externalResourceReaders,
+        )
+        packKeyValue("requestId", msg.requestId())
+        packKeyValue("allowedModules", msg.allowedModules?.map { it.toString() })
+        packKeyValue("allowedResources", msg.allowedResources?.map { it.toString() })
+        if (msg.clientModuleReaders != null) {
+          packer.packString("clientModuleReaders")
+          packer.packArrayHeader(msg.clientModuleReaders.size)
+          for (moduleReader in msg.clientModuleReaders) {
+            packModuleReaderSpec(moduleReader)
+          }
+        }
+        if (msg.clientResourceReaders != null) {
+          packer.packString("clientResourceReaders")
+          packer.packArrayHeader(msg.clientResourceReaders.size)
+          for (resourceReader in msg.clientResourceReaders) {
+            packResourceReaderSpec(resourceReader)
+          }
+        }
+        packKeyValue("modulePaths", msg.modulePaths, Path::toString)
+        packKeyValue("env", msg.env)
+        packKeyValue("properties", msg.properties)
+        packKeyValue("timeoutSeconds", msg.timeout?.toSeconds())
+        packKeyValue("rootDir", msg.rootDir?.pathString)
+        packKeyValue("cacheDir", msg.cacheDir?.pathString)
+        packKeyValue("outputFormat", msg.outputFormat)
+        if (msg.project != null) {
+          packer.packString("project")
+          packer.packProject(msg.project)
+        }
+        if (msg.http != null) {
+          packer.packString("http")
+          packer.packHttp(msg.http)
+        }
+        if (msg.externalModuleReaders != null) {
+          packer.packString("externalModuleReaders")
+          packer.packMapHeader(msg.externalModuleReaders.size)
+          for ((scheme, spec) in msg.externalModuleReaders) {
+            packer.packString(scheme)
+            packer.packExternalReader(spec)
+          }
+        }
+        if (msg.externalResourceReaders != null) {
+          packer.packString("externalResourceReaders")
+          packer.packMapHeader(msg.externalResourceReaders.size)
+          for ((scheme, spec) in msg.externalResourceReaders) {
+            packer.packString(scheme)
+            packer.packExternalReader(spec)
+          }
+        }
+        return
+      }
+      Message.Type.CREATE_EVALUATOR_RESPONSE -> {
+        msg as CreateEvaluatorResponse
+        packMapHeader(1, msg.evaluatorId, msg.error)
+        packKeyValue("requestId", msg.requestId())
+        packKeyValue("evaluatorId", msg.evaluatorId)
+        packKeyValue("error", msg.error)
+      }
+      Message.Type.CLOSE_EVALUATOR -> {
+        msg as CloseEvaluator
+        packer.packMapHeader(1)
+        packKeyValue("evaluatorId", msg.evaluatorId)
+      }
+      Message.Type.EVALUATE_REQUEST -> {
+        msg as EvaluateRequest
+        packMapHeader(3, msg.moduleText, msg.expr)
+        packKeyValue("requestId", msg.requestId())
+        packKeyValue("evaluatorId", msg.evaluatorId)
+        packKeyValue("moduleUri", msg.moduleUri.toString())
+        packKeyValue("moduleText", msg.moduleText)
+        packKeyValue("expr", msg.expr)
+      }
+      Message.Type.EVALUATE_RESPONSE -> {
+        msg as EvaluateResponse
+        packMapHeader(2, msg.result, msg.error)
+        packKeyValue("requestId", msg.requestId())
+        packKeyValue("evaluatorId", msg.evaluatorId)
+        msg.result?.let { packKeyValue("result", it) }
+        packKeyValue("error", msg.error)
+      }
+      Message.Type.LOG_MESSAGE -> {
+        msg as LogMessage
+        packer.packMapHeader(4)
+        packKeyValue("evaluatorId", msg.evaluatorId)
+        packKeyValue("level", msg.level)
+        packKeyValue("message", msg.message)
+        packKeyValue("frameUri", msg.frameUri)
+      }
+      else -> super.encodeMessage(msg)
+    }
+  }
+}