Implement SPICE-0009 External Readers (#660)

This adds a new feature, which allows Pkl to read resources and modules from external processes.

Follows the design laid out in SPICE-0009.

Also, this moves most of the messaging API into pkl-core

pkl-commons/src/main/kotlin/org/pkl/commons/Strings.kt

@@ -18,6 +18,7 @@ package org.pkl.commons
 import java.io.File
 import java.net.URI
 import java.nio.file.Path
+import java.util.*
 import java.util.regex.Pattern
 
 fun String.toPath(): Path = Path.of(this)
@@ -36,3 +37,53 @@ fun String.toUri(): URI {
   }
   return URI(null, null, this, null)
 }
+
+/** Lex a string into tokens similar to how a shell would */
+fun shlex(input: String): List<String> {
+  val result = mutableListOf<String>()
+  var inEscape = false
+  var quote: Char? = null
+  var lastCloseQuoteIndex = Int.MIN_VALUE
+  val current = StringBuilder()
+
+  for ((idx, char) in input.withIndex()) {
+    when {
+      // if in an escape always append the next character
+      inEscape -> {
+        inEscape = false
+        current.append(char)
+      }
+      // enter an escape on \ if not in a quote or in a non-single quote
+      char == '\\' && quote != '\'' -> inEscape = true
+      // if in a quote and encounter the delimiter, tentatively exit the quote
+      // this handles cases with adjoining quotes e.g. `abc'123''xyz'`
+      quote == char -> {
+        quote = null
+        lastCloseQuoteIndex = idx
+      }
+      // if not in a quote and encounter a quote charater, enter a quote
+      quote == null && (char == '\'' || char == '"') -> {
+        quote = char
+      }
+      // if not in a quote and whitespace is encountered
+      quote == null && char.isWhitespace() -> {
+        // if the current token isn't empty or if a quote has just ended, finalize the current token
+        // otherwise do nothing, which handles multiple whitespace cases e.g. `abc     123`
+        if (current.isNotEmpty() || lastCloseQuoteIndex == (idx - 1)) {
+          result.add(current.toString())
+          current.clear()
+        }
+      }
+      // in other cases, append to the current token
+      else -> current.append(char)
+    }
+  }
+  // clean up last token
+  // if the current token isn't empty or if a quote has just ended, finalize the token
+  // if this condition is false, the input likely ended in whitespace
+  if (current.isNotEmpty() || lastCloseQuoteIndex == (input.length - 1)) {
+    result.add(current.toString())
+  }
+
+  return result
+}

pkl-commons/src/test/kotlin/org/pkl/commons/ShlexTest.kt

@@ -0,0 +1,78 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.commons
+
+import org.assertj.core.api.Assertions.assertThat
+import org.junit.jupiter.api.Test
+
+class ShlexTest {
+
+  @Test
+  fun `empty input produces empty output`() {
+    assertThat(shlex("")).isEqualTo(emptyList<String>())
+  }
+
+  @Test
+  fun `whitespace input produces empty output`() {
+    assertThat(shlex("  \n \t  ")).isEqualTo(emptyList<String>())
+  }
+
+  @Test
+  fun `regular token parsing`() {
+    assertThat(shlex("\nabc  def\tghi ")).isEqualTo(listOf("abc", "def", "ghi"))
+  }
+
+  @Test
+  fun `single quoted token parsing`() {
+    assertThat(shlex("'this is a single token'")).isEqualTo(listOf("this is a single token"))
+  }
+
+  @Test
+  fun `double quoted token parsing`() {
+    assertThat(shlex("\"this is a single token\"")).isEqualTo(listOf("this is a single token"))
+  }
+
+  @Test
+  fun `escaping handles double quotes`() {
+    assertThat(shlex(""""\"this is a single double quoted token\"""""))
+      .isEqualTo(listOf("\"this is a single double quoted token\""))
+  }
+
+  @Test
+  fun `escaping does not apply within single quotes`() {
+    assertThat(shlex("""'this is a single \" token'"""))
+      .isEqualTo(listOf("""this is a single \" token"""))
+  }
+
+  @Test
+  fun `adjacent quoted strings are one token`() {
+    assertThat(shlex(""""single"' joined 'token""")).isEqualTo(listOf("single joined token"))
+    assertThat(shlex(""""single"' 'token""")).isEqualTo(listOf("single token"))
+  }
+
+  @Test
+  fun `space escapes do not split tokens`() {
+    assertThat(shlex("""single\ token""")).isEqualTo(listOf("single token"))
+  }
+
+  @Test
+  fun `empty quotes produce a single empty token`() {
+    assertThat(shlex("\"\"")).isEqualTo(listOf(""))
+    assertThat(shlex("''")).isEqualTo(listOf(""))
+    assertThat(shlex("'' ''")).isEqualTo(listOf("", ""))
+    assertThat(shlex("''''")).isEqualTo(listOf(""))
+  }
+}