Support scheme-agnostic projects (#486)

This adds changes to support loading project dependencies in non-file based projects.

The design for this feature can be found in SPICE-0005: https://github.com/apple/pkl-evolution/pull/6

Changes:
* Consider all imports prefixed with `@` as dependency notation.
* Bugfix: fix resolution of glob expressions in a local dependency.
* Adjust pkl.Project:
  - Allow local dependencies from a scheme-local paths.
  - Disallow certain evaluator settings if not loaded as a file-based module.
* Breaking API change: `ProjectDependenciesManager` constructor now requires `ModuleResolver` and `SecurityManager`.

pkl-core/src/test/files/LanguageSnippetTests/input/projects/project1/PklProject

@@ -11,4 +11,5 @@ dependencies {
     uri = "package://localhost:0/badImportsWithinPackage@1.0.0"
   }
   ["project2"] = import("../project2/PklProject")
+  ["project6"] = import("../project6/PklProject")
 }

pkl-core/src/test/files/LanguageSnippetTests/input/projects/project1/PklProject.deps.json

@@ -20,6 +20,11 @@
       "uri": "projectpackage://localhost:0/project2@1.0.0",
       "path": "../project2/"
     },
+    "package://localhost:12110/project6@1": {
+      "type": "local",
+      "uri": "projectpackage://localhost:12110/project6@1.0.0",
+      "path": "../project6/"
+    },
     "package://localhost:0/badImportsWithinPackage@1": {
       "type": "remote",
       "uri": "projectpackage://localhost:0/badImportsWithinPackage@1.0.0",

pkl-core/src/test/files/LanguageSnippetTests/input/projects/project1/globbing.pkl

@@ -34,4 +34,8 @@ examples {
   ["glob-read absolute package uri"] {
     read*("package://localhost:0/birds@0.5.0#/catalog/*.pkl")
   }
+  
+  ["glob-import behind local project import"] {
+    import("@project6/children.pkl")
+  }
 }

pkl-core/src/test/files/LanguageSnippetTests/input/projects/project3/PklProject.deps.json

@@ -20,6 +20,11 @@
       "uri": "projectpackage://localhost:0/project2@1.0.0",
       "path": "../project2/"
     },
+    "package://localhost:12110/project6@1": {
+      "type": "local",
+      "uri": "projectpackage://localhost:12110/project6@1.0.0",
+      "path": "../project6/"
+    },
     "package://localhost:0/badImportsWithinPackage@1": {
       "type": "remote",
       "uri": "projectpackage://localhost:0/badImportsWithinPackage@1.0.0",

pkl-core/src/test/files/LanguageSnippetTests/input/projects/project6/PklProject

@@ -0,0 +1,8 @@
+amends "pkl:Project"
+
+package {
+  name = "project6"
+  baseUri = "package://localhost:12110/project6"
+  version = "1.0.0"
+  packageZipUrl = "https://localhost:12110/project6/project6-\(version).zip"
+}

pkl-core/src/test/files/LanguageSnippetTests/input/projects/project6/PklProject.deps.json

@@ -0,0 +1,4 @@
+{
+  "schemaVersion": 1,
+  "resolvedDependencies": {}
+}

pkl-core/src/test/files/LanguageSnippetTests/input/projects/project6/children.pkl

@@ -0,0 +1 @@
+children = import*("children/*.pkl")

pkl-core/src/test/files/LanguageSnippetTests/input/projects/project6/children/a.pkl

@@ -0,0 +1 @@
+name = "a"

pkl-core/src/test/files/LanguageSnippetTests/input/projects/project6/children/b.pkl

@@ -0,0 +1 @@
+name = "b"

pkl-core/src/test/files/LanguageSnippetTests/input/projects/project6/children/c.pkl

@@ -0,0 +1 @@
+name = "c"

pkl-core/src/test/files/LanguageSnippetTests/output/projects/badProjectDeps4/bug.err

@@ -1,13 +1,13 @@
 –– Pkl Error ––
-Expected value of type `*RemoteDependency|LocalDependency`, but got a different `pkl.Project`.
+Expected value of type `*RemoteDependency|Project(isValidLoadDependency)`, but got a different `pkl.Project`.
 Value: new ModuleClass { package = null; tests {}; dependencies {}; evaluatorSetting...
 
-xxx | dependencies: Mapping<String(!contains("/")), *RemoteDependency|LocalDependency>
-                                                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+xxx | dependencies: Mapping<String(!contains("/")), *RemoteDependency|Project(isValidLoadDependency)>
+                                                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 at pkl.Project#dependencies (pkl:Project)
 
-* Value is not of type `LocalDependency` because:
-  Type constraint `this.package != null` violated.
+* Value is not of type `Project(isValidLoadDependency)` because:
+  Type constraint `isValidLoadDependency` violated.
   Value: new ModuleClass { package = null; tests {}; dependencies {}; evaluatorSetti...
 
 x | dependencies {

pkl-core/src/test/files/LanguageSnippetTests/output/projects/missingProjectDeps/bug.err

@@ -1,8 +1,9 @@
 –– Pkl Error ––
-Cannot resolve dependency because file `PklProject.deps.json` is missing in project directory `file:///$snippetsDir/input/projects/missingProjectDeps/`.
+Encountered an error when attempting to load `PklProject.deps.json` at `file:///$snippetsDir/input/projects/missingProjectDeps/PklProject.deps.json`.
+NoSuchFileException: /$snippetsDir/input/projects/missingProjectDeps/PklProject.deps.json
 
 x | import "@birds/Bird.pkl"
            ^^^^^^^^^^^^^^^^^
 at bug (file:///$snippetsDir/input/projects/missingProjectDeps/bug.pkl)
 
-Run `pkl project resolve` to create a new set of dependencies.
+Try running `pkl project resolve` within the project directory to create a new set of dependencies.

pkl-core/src/test/files/LanguageSnippetTests/output/projects/project1/globbing.pcf

@@ -262,4 +262,19 @@ examples {
       }
     }
   }
+  ["glob-import behind local project import"] {
+    new {
+      children {
+        ["children/a.pkl"] {
+          name = "a"
+        }
+        ["children/b.pkl"] {
+          name = "b"
+        }
+        ["children/c.pkl"] {
+          name = "c"
+        }
+      }
+    }
+  }
 }

pkl-core/src/test/files/LanguageSnippetTests/output/projects/project6/children.pcf

@@ -0,0 +1,11 @@
+children {
+  ["children/a.pkl"] {
+    name = "a"
+  }
+  ["children/b.pkl"] {
+    name = "b"
+  }
+  ["children/c.pkl"] {
+    name = "c"
+  }
+}

pkl-core/src/test/files/LanguageSnippetTests/output/projects/project6/children/a.pcf

@@ -0,0 +1 @@
+name = "a"

pkl-core/src/test/files/LanguageSnippetTests/output/projects/project6/children/b.pcf

@@ -0,0 +1 @@
+name = "b"

pkl-core/src/test/files/LanguageSnippetTests/output/projects/project6/children/c.pcf

@@ -0,0 +1 @@
+name = "c"

pkl-core/src/test/kotlin/org/pkl/core/EvaluatorTest.kt

@@ -6,6 +6,7 @@ import java.net.URI
 import java.nio.file.Files
 import java.nio.file.Path
 import org.assertj.core.api.Assertions.assertThat
+import org.assertj.core.api.Assertions.assertThatCode
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertThrows
 import org.junit.jupiter.api.io.TempDir
@@ -15,7 +16,16 @@ import org.pkl.commons.writeString
 import org.pkl.core.ModuleSource.*
 import org.pkl.core.util.IoUtils
 import org.junit.jupiter.api.AfterAll
+import org.pkl.commons.test.PackageServer
+import org.pkl.core.module.ModuleKey
+import org.pkl.core.module.ModuleKeyFactories
+import org.pkl.core.module.ModuleKeyFactory
+import org.pkl.core.module.ResolvedModuleKey
+import org.pkl.core.project.Project
+import java.nio.charset.StandardCharsets
 import java.nio.file.FileSystems
+import java.util.*
+import java.util.regex.Pattern
 import kotlin.io.path.writeText
 
 class EvaluatorTest {
@@ -24,6 +34,28 @@ class EvaluatorTest {
 
     private const val sourceText = "name = \"pigeon\"; age = 10 + 20"
 
+    private object CustomModuleKeyFactory : ModuleKeyFactory {
+      override fun create(uri: URI): Optional<ModuleKey> {
+        return if (uri.scheme == "custom") Optional.of(CustomModuleKey(uri))
+        else Optional.empty<ModuleKey>()
+      }
+    }
+
+    private class CustomModuleKey(private val uri: URI) : ModuleKey, ResolvedModuleKey {
+      override fun hasHierarchicalUris(): Boolean = true
+
+      override fun isGlobbable(): Boolean = false
+
+      override fun getOriginal(): ModuleKey = this
+
+      override fun getUri(): URI = uri
+
+      override fun loadSource(): String = javaClass.classLoader.getResourceAsStream(uri.path.drop(1))!!.use { it.readAllBytes().toString(
+        StandardCharsets.UTF_8) }
+
+      override fun resolve(securityManager: SecurityManager): ResolvedModuleKey = this
+    }
+
     @AfterAll
     @JvmStatic
     fun afterAll() {
@@ -291,6 +323,132 @@ class EvaluatorTest {
     assertThat(output["bar/../bark.yml"]?.text).isEqualTo("bark: bark bark")
   }
 
+  @Test
+  fun `project set from modulepath`(@TempDir cacheDir: Path) {
+    PackageServer.populateCacheDir(cacheDir)
+    val evaluatorBuilder = EvaluatorBuilder.preconfigured().setModuleCacheDir(cacheDir)
+    val project = Project.load(modulePath("/org/pkl/core/project/project5/PklProject"))
+    val result = evaluatorBuilder.setProjectDependencies(project.dependencies).build().use { evaluator ->
+      evaluator.evaluateOutputText(modulePath("/org/pkl/core/project/project5/main.pkl"))
+    }
+    assertThat(result).isEqualTo("""
+      prop1 {
+        name = "Apple"
+      }
+      prop2 {
+        res = 1
+      }
+
+    """.trimIndent())
+  }
+
+  @Test
+  fun `project set from custom ModuleKeyFactory`(@TempDir cacheDir: Path) {
+    PackageServer.populateCacheDir(cacheDir)
+    val evaluatorBuilder = with(EvaluatorBuilder.preconfigured()) {
+      setAllowedModules(SecurityManagers.defaultAllowedModules + Pattern.compile("custom:"))
+      setAllowedResources(SecurityManagers.defaultAllowedResources + Pattern.compile("custom:"))
+      setModuleCacheDir(cacheDir)
+      setModuleKeyFactories(
+        listOf(
+          CustomModuleKeyFactory,
+          ModuleKeyFactories.standardLibrary,
+          ModuleKeyFactories.pkg,
+          ModuleKeyFactories.projectpackage,
+          ModuleKeyFactories.file
+        )
+      )
+    }
+    val project = evaluatorBuilder.build().use { Project.load(it, uri("custom:/org/pkl/core/project/project5/PklProject")) }
+
+    val evaluator = evaluatorBuilder.setProjectDependencies(project.dependencies).build()
+    val output = evaluator.use { it.evaluateOutputText(uri("custom:/org/pkl/core/project/project5/main.pkl")) }
+    assertThat(output)
+      .isEqualTo(
+        """
+        prop1 {
+          name = "Apple"
+        }
+        prop2 {
+          res = 1
+        }
+
+        """
+          .trimIndent()
+      )
+  }
+
+  @Test
+  fun `project base path set to non-hierarchical scheme`() {
+    class FooBarModuleKey(val moduleUri: URI) : ModuleKey, ResolvedModuleKey {
+      override fun hasHierarchicalUris(): Boolean = false
+      override fun isGlobbable(): Boolean = false
+      override fun getOriginal(): ModuleKey = this
+      override fun getUri(): URI = moduleUri
+      override fun loadSource(): String =
+        if (uri.schemeSpecificPart.endsWith("PklProject")) {
+          """
+          amends "pkl:Project"
+          """.trimIndent()
+        } else """
+          birds = import("@birds/catalog/Ostritch.pkl")
+        """.trimIndent()
+      override fun resolve(securityManager: SecurityManager): ResolvedModuleKey {
+        return this
+      }
+    }
+
+    val fooBayModuleKeyFactory = ModuleKeyFactory { uri ->
+      if (uri.scheme == "foobar") Optional.of(FooBarModuleKey(uri))
+      else Optional.empty()
+    }
+
+    val evaluatorBuilder = with(EvaluatorBuilder.preconfigured()) {
+      setAllowedModules(SecurityManagers.defaultAllowedModules + Pattern.compile("foobar:"))
+      setAllowedResources(SecurityManagers.defaultAllowedResources + Pattern.compile("foobar:"))
+      setModuleKeyFactories(
+        listOf(
+          fooBayModuleKeyFactory,
+          ModuleKeyFactories.standardLibrary,
+          ModuleKeyFactories.pkg,
+          ModuleKeyFactories.projectpackage,
+          ModuleKeyFactories.file
+        )
+      )
+    }
+
+    val project = evaluatorBuilder.build().use { Project.load(it, uri("foobar:foo/PklProject")) }
+    val evaluator = evaluatorBuilder.setProjectDependencies(project.dependencies).build()
+    assertThatCode { evaluator.use { it.evaluateOutputText(uri("foobar:baz")) } }
+      .hasMessageContaining("Cannot import dependency because project URI `foobar:foo/PklProject` does not have a hierarchical path.")
+  }
+
+  @Test
+  fun `cannot glob import in local dependency from modulepath`(@TempDir cacheDir: Path) {
+    PackageServer.populateCacheDir(cacheDir)
+    val evaluatorBuilder = EvaluatorBuilder.preconfigured().setModuleCacheDir(cacheDir)
+    val project = Project.load(modulePath("/org/pkl/core/project/project6/PklProject"))
+    evaluatorBuilder.setProjectDependencies(project.dependencies).build().use { evaluator ->
+      assertThatCode { 
+        evaluator.evaluateOutputText(modulePath("/org/pkl/core/project/project6/globWithinDependency.pkl"))
+      }.hasMessageContaining("""
+        Cannot resolve import in local dependency because scheme `modulepath` is not globbable.
+        
+        1 | res = import*("*.pkl")
+                  ^^^^^^^^^^^^^^^^
+      """.trimIndent())
+      assertThatCode {
+        evaluator.evaluateOutputText(modulePath("/org/pkl/core/project/project6/globIntoDependency.pkl"))
+      }.hasMessageContaining("""
+        –– Pkl Error ––
+        Cannot resolve import in local dependency because scheme `modulepath` is not globbable.
+        
+        1 | import* "@project7/*.pkl" as proj7Files
+            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+      """.trimIndent())
+    }
+  }
+
   private fun checkModule(module: PModule) {
     assertThat(module.properties.size).isEqualTo(2)
     assertThat(module.getProperty("name")).isEqualTo("pigeon")

pkl-core/src/test/kotlin/org/pkl/core/LanguageSnippetTestsEngine.kt

@@ -38,10 +38,15 @@ abstract class AbstractLanguageSnippetTestsEngine : InputOutputTestEngine() {
   internal val selection: String = ""
 
   protected val packageServer: PackageServer = PackageServer()
-  
+
   override val includedTests: List<Regex> = listOf(Regex(".*$selection\\.pkl"))
 
-  override val excludedTests: List<Regex> = listOf(Regex(".*/native/.*"))
+  override val excludedTests: List<Regex> = buildList { 
+    add(Regex(".*/native/.*"))
+    if (IoUtils.isWindows()) {
+      addAll(windowsExcludedTests)
+    }
+  }
 
   override val inputDir: Path = snippetsDir.resolve("input")
 
@@ -68,7 +73,12 @@ abstract class AbstractLanguageSnippetTestsEngine : InputOutputTestEngine() {
     packageServer.close()
   }
 
-  protected fun String.stripFilePaths() = replace(snippetsDir.toUri().toString(), "file:///\$snippetsDir/")
+  private val replacement by lazy {
+    if (snippetsDir.root.toString() != "/") "\$snippetsDir" else "/\$snippetsDir"
+  } 
+
+  protected fun String.stripFilePaths(): String =
+    replace(IoUtils.toNormalizedPathString(snippetsDir), replacement)
 
   protected fun String.stripLineNumbers() = replace(lineNumberRegex) { result ->
     // replace line number with equivalent number of 'x' characters to keep formatting intact
@@ -80,7 +90,7 @@ abstract class AbstractLanguageSnippetTestsEngine : InputOutputTestEngine() {
   // can't think of a better solution right now
   protected fun String.stripVersionCheckErrorMessage() =
     replace("Pkl version is ${Release.current().version()}", "Pkl version is xxx")
-  
+
   protected fun String.stripStdlibLocationSha(): String =
     replace("https://github.com/apple/pkl/blob/${Release.current().commitId()}/stdlib/", "https://github.com/apple/pkl/blob/\$commitId/stdlib/")
 
@@ -261,7 +271,12 @@ class AlpineLanguageSnippetTestsEngine : AbstractNativeLanguageSnippetTestsEngin
   override val testClass: KClass<*> = AlpineLanguageSnippetTests::class
 }
 
+// error message contains different file path on Windows
+private val windowsExcludedTests get() = listOf(Regex(".*missingProjectDeps/bug\\.pkl"))
+
 class WindowsLanguageSnippetTestsEngine : AbstractNativeLanguageSnippetTestsEngine() {
   override val pklExecutablePath: Path = PklExecutablePaths.windowsAmd64
   override val testClass: KClass<*> = WindowsLanguageSnippetTests::class
+  override val excludedTests: List<Regex> 
+    get() = super.excludedTests + windowsExcludedTests
 }

pkl-core/src/test/kotlin/org/pkl/core/project/ProjectTest.kt

@@ -1,16 +1,16 @@
 package org.pkl.core.project
 
-import org.pkl.commons.test.PackageServer
-import org.pkl.commons.writeString
-import org.pkl.core.*
-import org.pkl.core.packages.PackageUri
-import org.pkl.core.project.Project.EvaluatorSettings
 import org.assertj.core.api.Assertions.assertThat
 import org.assertj.core.api.Assertions.assertThatCode
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.io.TempDir
 import org.pkl.commons.test.FileTestUtils
+import org.pkl.commons.test.PackageServer
+import org.pkl.commons.writeString
+import org.pkl.core.*
 import org.pkl.core.http.HttpClient
+import org.pkl.core.packages.PackageUri
+import org.pkl.core.project.Project.EvaluatorSettings
 import java.net.URI
 import java.nio.file.Path
 import java.util.regex.Pattern

pkl-core/src/test/resources/org/pkl/core/project/project5/PklProject

@@ -0,0 +1,8 @@
+amends "pkl:Project"
+
+dependencies {
+  ["fruit"] {
+    uri = "package://localhost:0/fruit@1.0.5"
+  }
+  ["project4"] = import("../project4/PklProject")
+}

pkl-core/src/test/resources/org/pkl/core/project/project5/PklProject.deps.json

@@ -0,0 +1,17 @@
+{
+  "schemaVersion": 1,
+  "resolvedDependencies": {
+    "package://localhost:0/fruit@1": {
+      "type": "remote",
+      "uri": "projectpackage://localhost:0/fruit@1.0.5",
+      "checksums": {
+        "sha256": "$skipChecksumVerification"
+      }
+    },
+    "package://localhost:0/project4@1": {
+      "type": "local",
+      "uri": "projectpackage://localhost:0/project4@1.0.0",
+      "path": "../project4"
+    }
+  }
+}

pkl-core/src/test/resources/org/pkl/core/project/project5/main.pkl

@@ -0,0 +1,5 @@
+import "@fruit/catalog/apple.pkl"
+import "@project4/module1.pkl"
+
+prop1 = apple
+prop2 = module1

pkl-core/src/test/resources/org/pkl/core/project/project6/PklProject

@@ -0,0 +1,5 @@
+amends "pkl:Project"
+
+dependencies {
+  ["project7"] = import("../project7/PklProject")
+}

pkl-core/src/test/resources/org/pkl/core/project/project6/PklProject.deps.json

@@ -0,0 +1,10 @@
+{
+  "schemaVersion": 1,
+  "resolvedDependencies": {
+    "package://localhost:0/project7@1": {
+      "type": "local",
+      "uri": "projectpackage://localhost:0/project7@1.0.0",
+      "path": "../project7"
+    }
+  }
+}

pkl-core/src/test/resources/org/pkl/core/project/project6/globIntoDependency.pkl

@@ -0,0 +1,3 @@
+import* "@project7/*.pkl" as proj7Files
+
+res = proj7Files

pkl-core/src/test/resources/org/pkl/core/project/project6/globWithinDependency.pkl

@@ -0,0 +1,3 @@
+import "@project7/main.pkl"
+
+res = main.res

pkl-core/src/test/resources/org/pkl/core/project/project7/PklProject

@@ -0,0 +1,8 @@
+amends "pkl:Project"
+
+package {
+  name = "project7"
+  version = "1.0.0"
+  packageZipUrl = "https://bogus.value"
+  baseUri = "package://localhost:0/project7"
+}

pkl-core/src/test/resources/org/pkl/core/project/project7/main.pkl

@@ -0,0 +1 @@
+res = import*("*.pkl")