Implement SPICE-0009 External Readers (#660)

This adds a new feature, which allows Pkl to read resources and modules from external processes.

Follows the design laid out in SPICE-0009.

Also, this moves most of the messaging API into pkl-core

pkl-commons-cli/gradle.lockfile

@@ -31,6 +31,7 @@ org.junit.jupiter:junit-jupiter-params:5.11.2=testCompileClasspath,testImplement
 org.junit.platform:junit-platform-commons:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit.platform:junit-platform-engine:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit:junit-bom:5.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
+org.msgpack:msgpack-core:0.9.8=runtimeClasspath,testRuntimeClasspath
 org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.organicdesign:Paguro:3.10.3=runtimeClasspath,testRuntimeClasspath
 org.snakeyaml:snakeyaml-engine:2.5=runtimeClasspath,testRuntimeClasspath

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliBaseOptions.kt

@@ -20,6 +20,7 @@ import java.nio.file.Files
 import java.nio.file.Path
 import java.time.Duration
 import java.util.regex.Pattern
+import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader
 import org.pkl.core.module.ProjectDependenciesManager
 import org.pkl.core.util.IoUtils
 
@@ -134,6 +135,12 @@ data class CliBaseOptions(
 
   /** Hostnames, IP addresses, or CIDR blocks to not proxy. */
   val httpNoProxy: List<String>? = null,
+
+  /** External module reader process specs */
+  val externalModuleReaders: Map<String, ExternalReader> = mapOf(),
+
+  /** External resource reader process specs */
+  val externalResourceReaders: Map<String, ExternalReader> = mapOf(),
 ) {
 
   companion object {

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliCommand.kt

@@ -21,6 +21,7 @@ import java.util.regex.Pattern
 import kotlin.io.path.isRegularFile
 import org.pkl.core.*
 import org.pkl.core.evaluatorSettings.PklEvaluatorSettings
+import org.pkl.core.externalreader.ExternalReaderProcessImpl
 import org.pkl.core.http.HttpClient
 import org.pkl.core.module.ModuleKeyFactories
 import org.pkl.core.module.ModuleKeyFactory
@@ -108,12 +109,16 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
 
   protected val allowedModules: List<Pattern> by lazy {
     cliOptions.allowedModules
-      ?: evaluatorSettings?.allowedModules ?: SecurityManagers.defaultAllowedModules
+      ?: evaluatorSettings?.allowedModules
+        ?: (SecurityManagers.defaultAllowedModules +
+        externalModuleReaders.keys.map { Pattern.compile("$it:") }.toList())
   }
 
   protected val allowedResources: List<Pattern> by lazy {
     cliOptions.allowedResources
-      ?: evaluatorSettings?.allowedResources ?: SecurityManagers.defaultAllowedResources
+      ?: evaluatorSettings?.allowedResources
+        ?: (SecurityManagers.defaultAllowedResources +
+        externalResourceReaders.keys.map { Pattern.compile("$it:") }.toList())
   }
 
   protected val rootDir: Path? by lazy {
@@ -169,6 +174,26 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
       ?: project?.evaluatorSettings?.http?.proxy?.noProxy ?: settings.http?.proxy?.noProxy
   }
 
+  private val externalModuleReaders by lazy {
+    (project?.evaluatorSettings?.externalModuleReaders
+      ?: emptyMap()) + cliOptions.externalModuleReaders
+  }
+
+  private val externalResourceReaders by lazy {
+    (project?.evaluatorSettings?.externalResourceReaders
+      ?: emptyMap()) + cliOptions.externalResourceReaders
+  }
+
+  private val externalProcesses by lazy {
+    // share ExternalProcessImpl instances between configured external resource/module readers with
+    // the same spec
+    // this avoids spawning multiple subprocesses if the same reader implements both reader types
+    // and/or multiple schemes
+    (externalModuleReaders + externalResourceReaders).values.toSet().associateWith {
+      ExternalReaderProcessImpl(it)
+    }
+  }
+
   private fun HttpClient.Builder.addDefaultCliCertificates() {
     val caCertsDir = IoUtils.getPklHomeDir().resolve("cacerts")
     var certsAdded = false
@@ -213,6 +238,9 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
 
   protected fun moduleKeyFactories(modulePathResolver: ModulePathResolver): List<ModuleKeyFactory> {
     return buildList {
+      externalModuleReaders.forEach { (key, value) ->
+        add(ModuleKeyFactories.externalProcess(key, externalProcesses[value]!!))
+      }
       add(ModuleKeyFactories.standardLibrary)
       add(ModuleKeyFactories.modulePath(modulePathResolver))
       add(ModuleKeyFactories.pkg)
@@ -234,6 +262,9 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
       add(ResourceReaders.file())
       add(ResourceReaders.http())
       add(ResourceReaders.https())
+      externalResourceReaders.forEach { (key, value) ->
+        add(ResourceReaders.externalProcess(key, externalProcesses[value]!!))
+      }
     }
   }
 

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseOptions.kt

@@ -28,6 +28,8 @@ import java.time.Duration
 import java.util.regex.Pattern
 import org.pkl.commons.cli.CliBaseOptions
 import org.pkl.commons.cli.CliException
+import org.pkl.commons.shlex
+import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader
 import org.pkl.core.runtime.VmUtils
 import org.pkl.core.util.IoUtils
 
@@ -74,6 +76,17 @@ class BaseOptions : OptionGroup() {
         .multiple()
         .toMap()
     }
+
+    fun OptionWithValues<String?, String, String>.parseExternalReader(
+      delimiter: String
+    ): OptionWithValues<
+      Pair<String, ExternalReader>?, Pair<String, ExternalReader>, Pair<String, ExternalReader>
+    > {
+      return splitPair(delimiter).convert {
+        val cmd = shlex(it.second)
+        Pair(it.first, ExternalReader(cmd.first(), cmd.drop(1)))
+      }
+    }
   }
 
   private val defaults = CliBaseOptions()
@@ -207,6 +220,26 @@ class BaseOptions : OptionGroup() {
       .single()
       .split(",")
 
+  val externalModuleReaders: Map<String, ExternalReader> by
+    option(
+        names = arrayOf("--external-module-reader"),
+        metavar = "<scheme>='<executable>[ <arguments>]'",
+        help = "External reader registrations for module URI schemes"
+      )
+      .parseExternalReader("=")
+      .multiple()
+      .toMap()
+
+  val externalResourceReaders: Map<String, ExternalReader> by
+    option(
+        names = arrayOf("--external-resource-reader"),
+        metavar = "<scheme>='<executable>[ <arguments>]'",
+        help = "External reader registrations for resource URI schemes"
+      )
+      .parseExternalReader("=")
+      .multiple()
+      .toMap()
+
   // hidden option used by native tests
   private val testPort: Int by
     option(names = arrayOf("--test-port"), help = "Internal test option", hidden = true)
@@ -239,7 +272,9 @@ class BaseOptions : OptionGroup() {
       noProject = projectOptions?.noProject ?: false,
       caCertificates = caCertificates,
       httpProxy = proxy,
-      httpNoProxy = noProxy ?: emptyList()
+      httpNoProxy = noProxy ?: emptyList(),
+      externalModuleReaders = externalModuleReaders,
+      externalResourceReaders = externalResourceReaders,
     )
   }
 }

pkl-commons-cli/src/test/kotlin/org/pkl/commons/cli/BaseCommandTest.kt

@@ -24,6 +24,7 @@ import org.assertj.core.api.Assertions.assertThat
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertThrows
 import org.pkl.commons.cli.commands.BaseCommand
+import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader
 
 class BaseCommandTest {
 
@@ -72,4 +73,34 @@ class BaseCommandTest {
 
     assertThat(cmd.baseOptions.allowedResources).isEmpty()
   }
+
+  @Test
+  fun `--external-resource-reader and --external-module-reader are parsed correctly`() {
+    cmd.parse(
+      arrayOf(
+        "--external-module-reader",
+        "scheme3=reader3",
+        "--external-module-reader",
+        "scheme4=reader4 with args",
+        "--external-resource-reader",
+        "scheme1=reader1",
+        "--external-resource-reader",
+        "scheme2=reader2 with args"
+      )
+    )
+    assertThat(cmd.baseOptions.externalModuleReaders)
+      .isEqualTo(
+        mapOf(
+          "scheme3" to ExternalReader("reader3", emptyList()),
+          "scheme4" to ExternalReader("reader4", listOf("with", "args"))
+        )
+      )
+    assertThat(cmd.baseOptions.externalResourceReaders)
+      .isEqualTo(
+        mapOf(
+          "scheme1" to ExternalReader("reader1", emptyList()),
+          "scheme2" to ExternalReader("reader2", listOf("with", "args"))
+        )
+      )
+  }
 }