Add analyze imports libs (SPICE-0001) (#695)

This adds a new feature to build a dependency graph of Pkl programs, following the SPICE outlined in https://github.com/apple/pkl-evolution/pull/2.

It adds:
* CLI command `pkl analyze imports`
* Java API `org.pkl.core.Analyzer`
* Pkl stdlib module `pkl:analyze`
* pkl-gradle extension `analyze`

In addition, it also changes the Gradle plugin such that `transitiveModules` is by default computed from the import graph.
This commit is contained in:
Daniel Chao
2024-10-23 14:36:57 -07:00
committed by GitHub
parent eb3891b21f
commit ce25cb8ef0
53 changed files with 2054 additions and 53 deletions
@@ -0,0 +1 @@
import "b.pkl"
@@ -0,0 +1 @@
import "cyclicalB.pkl"
@@ -0,0 +1 @@
import "cyclicalA.pkl"
@@ -0,0 +1 @@
import* "[ab].pkl"
@@ -0,0 +1,33 @@
amends "../snippetTest.pkl"
import "pkl:analyze"
import "pkl:reflect"
import ".../input-helper/analyze/a.pkl"
import ".../input-helper/analyze/cyclicalA.pkl"
import ".../input-helper/analyze/globImport.pkl"
examples {
["basic"] {
analyze.importGraph(Set(reflect.Module(a).uri))
}
["cycles"] {
analyze.importGraph(Set(reflect.Module(cyclicalA).uri))
}
["globs"] {
analyze.importGraph(Set(reflect.Module(globImport).uri))
}
["packages"] {
analyze.importGraph(Set("package://localhost:0/birds@0.5.0#/Bird.pkl"))
}
}
output {
renderer {
// mimick result of `pkl analyze imports` CLI command
converters {
[Map] = (it) -> it.toMapping()
[Set] = (it) -> it.toListing()
}
}
}
@@ -0,0 +1,3 @@
import "pkl:analyze"
result = analyze.importGraph(Set("http://localhost:0/foo.pkl"))
@@ -0,0 +1,3 @@
import "pkl:analyze"
result = analyze.importGraph(Set("foo <>"))
@@ -0,0 +1,3 @@
import "pkl:analyze"
result = analyze.importGraph(Set("foo.pkl"))
@@ -0,0 +1,79 @@
examples {
["basic"] {
new {
imports {
["file:///$snippetsDir/input-helper/analyze/a.pkl"] {
new {
uri = "file:///$snippetsDir/input-helper/analyze/b.pkl"
}
}
["file:///$snippetsDir/input-helper/analyze/b.pkl"] {}
}
resolvedImports {
["file:///$snippetsDir/input-helper/analyze/a.pkl"] = "file:///$snippetsDir/input-helper/analyze/a.pkl"
["file:///$snippetsDir/input-helper/analyze/b.pkl"] = "file:///$snippetsDir/input-helper/analyze/b.pkl"
}
}
}
["cycles"] {
new {
imports {
["file:///$snippetsDir/input-helper/analyze/cyclicalA.pkl"] {
new {
uri = "file:///$snippetsDir/input-helper/analyze/cyclicalB.pkl"
}
}
["file:///$snippetsDir/input-helper/analyze/cyclicalB.pkl"] {
new {
uri = "file:///$snippetsDir/input-helper/analyze/cyclicalA.pkl"
}
}
}
resolvedImports {
["file:///$snippetsDir/input-helper/analyze/cyclicalA.pkl"] = "file:///$snippetsDir/input-helper/analyze/cyclicalA.pkl"
["file:///$snippetsDir/input-helper/analyze/cyclicalB.pkl"] = "file:///$snippetsDir/input-helper/analyze/cyclicalB.pkl"
}
}
}
["globs"] {
new {
imports {
["file:///$snippetsDir/input-helper/analyze/a.pkl"] {
new {
uri = "file:///$snippetsDir/input-helper/analyze/b.pkl"
}
}
["file:///$snippetsDir/input-helper/analyze/b.pkl"] {}
["file:///$snippetsDir/input-helper/analyze/globImport.pkl"] {
new {
uri = "file:///$snippetsDir/input-helper/analyze/a.pkl"
}
new {
uri = "file:///$snippetsDir/input-helper/analyze/b.pkl"
}
}
}
resolvedImports {
["file:///$snippetsDir/input-helper/analyze/a.pkl"] = "file:///$snippetsDir/input-helper/analyze/a.pkl"
["file:///$snippetsDir/input-helper/analyze/b.pkl"] = "file:///$snippetsDir/input-helper/analyze/b.pkl"
["file:///$snippetsDir/input-helper/analyze/globImport.pkl"] = "file:///$snippetsDir/input-helper/analyze/globImport.pkl"
}
}
}
["packages"] {
new {
imports {
["package://localhost:0/birds@0.5.0#/Bird.pkl"] {
new {
uri = "package://localhost:0/fruit@1.0.5#/Fruit.pkl"
}
}
["package://localhost:0/fruit@1.0.5#/Fruit.pkl"] {}
}
resolvedImports {
["package://localhost:0/birds@0.5.0#/Bird.pkl"] = "package://localhost:0/birds@0.5.0#/Bird.pkl"
["package://localhost:0/fruit@1.0.5#/Fruit.pkl"] = "package://localhost:0/fruit@1.0.5#/Fruit.pkl"
}
}
}
}
@@ -0,0 +1,10 @@
–– Pkl Error ––
HTTP/1.1 header parser received no bytes
x | result = analyze.importGraph(Set("http://localhost:0/foo.pkl"))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
at analyzeInvalidHttpModule#result (file:///$snippetsDir/input/errors/analyzeInvalidHttpModule.pkl)
xxx | text = renderer.renderDocument(value)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
at pkl.base#Module.output.text (pkl:base)
@@ -0,0 +1,12 @@
–– Pkl Error ––
Module URI `foo <>` has invalid syntax.
x | result = analyze.importGraph(Set("foo <>"))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
at analyzeInvalidModuleUri#result (file:///$snippetsDir/input/errors/analyzeInvalidModuleUri.pkl)
Illegal character in path at index 3: foo <>
xxx | text = renderer.renderDocument(value)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
at pkl.base#Module.output.text (pkl:base)
@@ -0,0 +1,10 @@
–– Pkl Error ––
Cannot analyze relative module URI `foo.pkl`.
x | result = analyze.importGraph(Set("foo.pkl"))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
at analyzeRelativeModuleUri#result (file:///$snippetsDir/input/errors/analyzeRelativeModuleUri.pkl)
xxx | text = renderer.renderDocument(value)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
at pkl.base#Module.output.text (pkl:base)
@@ -6,6 +6,7 @@ x | import "pkl:nonExisting"
at cannotFindStdLibModule#nonExisting (file:///$snippetsDir/input/errors/cannotFindStdLibModule.pkl)
Available standard library modules:
pkl:analyze
pkl:base
pkl:Benchmark
pkl:DocPackageInfo
@@ -0,0 +1,318 @@
/**
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core
import java.net.URI
import java.nio.file.Path
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.io.TempDir
import org.pkl.commons.createParentDirectories
import org.pkl.commons.test.PackageServer
import org.pkl.commons.writeString
import org.pkl.core.http.HttpClient
import org.pkl.core.module.ModuleKeyFactories
import org.pkl.core.project.Project
class AnalyzerTest {
private val simpleAnalyzer =
Analyzer(
StackFrameTransformers.defaultTransformer,
SecurityManagers.defaultManager,
listOf(ModuleKeyFactories.file, ModuleKeyFactories.standardLibrary, ModuleKeyFactories.pkg),
null,
null,
HttpClient.dummyClient()
)
@Test
fun `simple case`(@TempDir tempDir: Path) {
val file =
tempDir
.resolve("test.pkl")
.writeString(
"""
amends "pkl:base"
import "pkl:json"
myProp = import("pkl:xml")
"""
.trimIndent()
)
.toUri()
val result = simpleAnalyzer.importGraph(file)
assertThat(result.imports)
.containsEntry(
file,
setOf(
ImportGraph.Import(URI("pkl:base")),
ImportGraph.Import(URI("pkl:json")),
ImportGraph.Import(URI("pkl:xml"))
)
)
}
@Test
fun `glob imports`(@TempDir tempDir: Path) {
val file1 =
tempDir
.resolve("file1.pkl")
.writeString(
"""
import* "*.pkl"
"""
.trimIndent()
)
.toUri()
val file2 = tempDir.resolve("file2.pkl").writeString("foo = 1").toUri()
val file3 = tempDir.resolve("file3.pkl").writeString("bar = 1").toUri()
val result = simpleAnalyzer.importGraph(file1)
assertThat(result.imports)
.isEqualTo(
mapOf(
file1 to
setOf(ImportGraph.Import(file1), ImportGraph.Import(file2), ImportGraph.Import(file3)),
file2 to emptySet(),
file3 to emptySet()
),
)
}
@Test
fun `cyclical imports`(@TempDir tempDir: Path) {
val file1 = tempDir.resolve("file1.pkl").writeString("import \"file2.pkl\"").toUri()
val file2 = tempDir.resolve("file2.pkl").writeString("import \"file1.pkl\"").toUri()
val result = simpleAnalyzer.importGraph(file1)
assertThat(result.imports)
.isEqualTo(
mapOf(file1 to setOf(ImportGraph.Import(file2)), file2 to setOf(ImportGraph.Import(file1)))
)
}
@Test
fun `package imports`(@TempDir tempDir: Path) {
val analyzer =
Analyzer(
StackFrameTransformers.defaultTransformer,
SecurityManagers.defaultManager,
listOf(ModuleKeyFactories.file, ModuleKeyFactories.standardLibrary, ModuleKeyFactories.pkg),
tempDir.resolve("packages"),
null,
HttpClient.dummyClient(),
)
PackageServer.populateCacheDir(tempDir.resolve("packages"))
val file1 =
tempDir
.resolve("file1.pkl")
.writeString("import \"package://localhost:0/birds@0.5.0#/Bird.pkl\"")
.toUri()
val result = analyzer.importGraph(file1)
assertThat(result.imports)
.isEqualTo(
mapOf(
file1 to setOf(ImportGraph.Import(URI("package://localhost:0/birds@0.5.0#/Bird.pkl"))),
URI("package://localhost:0/birds@0.5.0#/Bird.pkl") to
setOf(ImportGraph.Import(URI("package://localhost:0/fruit@1.0.5#/Fruit.pkl"))),
URI("package://localhost:0/fruit@1.0.5#/Fruit.pkl") to emptySet()
)
)
}
@Test
fun `project dependency imports`(@TempDir tempDir: Path) {
tempDir
.resolve("PklProject")
.writeString(
"""
amends "pkl:Project"
dependencies {
["birds"] { uri = "package://localhost:0/birds@0.5.0" }
}
"""
.trimIndent()
)
tempDir
.resolve("PklProject.deps.json")
.writeString(
"""
{
"schemaVersion": 1,
"resolvedDependencies": {
"package://localhost:0/birds@0": {
"type": "remote",
"uri": "projectpackage://localhost:0/birds@0.5.0",
"checksums": {
"sha256": "${'$'}skipChecksumVerification"
}
},
"package://localhost:0/fruit@1": {
"type": "remote",
"uri": "projectpackage://localhost:0/fruit@1.0.5",
"checksums": {
"sha256": "${'$'}skipChecksumVerification"
}
}
}
}
"""
.trimIndent()
)
val project = Project.loadFromPath(tempDir.resolve("PklProject"))
PackageServer.populateCacheDir(tempDir.resolve("packages"))
val analyzer =
Analyzer(
StackFrameTransformers.defaultTransformer,
SecurityManagers.defaultManager,
listOf(
ModuleKeyFactories.file,
ModuleKeyFactories.standardLibrary,
ModuleKeyFactories.pkg,
ModuleKeyFactories.projectpackage
),
tempDir.resolve("packages"),
project.dependencies,
HttpClient.dummyClient()
)
val file1 =
tempDir
.resolve("file1.pkl")
.writeString(
"""
import "@birds/Bird.pkl"
"""
.trimIndent()
)
.toUri()
val result = analyzer.importGraph(file1)
assertThat(result.imports)
.isEqualTo(
mapOf(
file1 to
setOf(ImportGraph.Import(URI("projectpackage://localhost:0/birds@0.5.0#/Bird.pkl"))),
URI("projectpackage://localhost:0/birds@0.5.0#/Bird.pkl") to
setOf(ImportGraph.Import(URI("projectpackage://localhost:0/fruit@1.0.5#/Fruit.pkl"))),
URI("projectpackage://localhost:0/fruit@1.0.5#/Fruit.pkl") to emptySet()
)
)
assertThat(result.resolvedImports)
.isEqualTo(
mapOf(
file1 to file1.realPath(),
URI("projectpackage://localhost:0/birds@0.5.0#/Bird.pkl") to
URI("projectpackage://localhost:0/birds@0.5.0#/Bird.pkl"),
URI("projectpackage://localhost:0/fruit@1.0.5#/Fruit.pkl") to
URI("projectpackage://localhost:0/fruit@1.0.5#/Fruit.pkl")
)
)
}
@Test
fun `local project dependency import`(@TempDir tempDir: Path) {
val pklProject =
tempDir
.resolve("project1/PklProject")
.createParentDirectories()
.writeString(
"""
amends "pkl:Project"
dependencies {
["birds"] = import("../birds/PklProject")
}
"""
.trimIndent()
)
tempDir
.resolve("birds/PklProject")
.createParentDirectories()
.writeString(
"""
amends "pkl:Project"
package {
name = "birds"
version = "1.0.0"
packageZipUrl = "https://localhost:0/foo.zip"
baseUri = "package://localhost:0/birds"
}
"""
.trimIndent()
)
val birdModule = tempDir.resolve("birds/bird.pkl").writeString("name = \"Warbler\"")
pklProject.parent
.resolve("PklProject.deps.json")
.writeString(
"""
{
"schemaVersion": 1,
"resolvedDependencies": {
"package://localhost:0/birds@1": {
"type": "local",
"uri": "projectpackage://localhost:0/birds@1.0.0",
"path": "../birds"
}
}
}
"""
.trimIndent()
)
val mainPkl =
pklProject.parent
.resolve("main.pkl")
.writeString(
"""
import "@birds/bird.pkl"
"""
.trimIndent()
)
val project = Project.loadFromPath(pklProject)
val analyzer =
Analyzer(
StackFrameTransformers.defaultTransformer,
SecurityManagers.defaultManager,
listOf(
ModuleKeyFactories.file,
ModuleKeyFactories.standardLibrary,
ModuleKeyFactories.pkg,
ModuleKeyFactories.projectpackage
),
tempDir.resolve("packages"),
project.dependencies,
HttpClient.dummyClient()
)
val result = analyzer.importGraph(mainPkl.toUri())
val birdUri = URI("projectpackage://localhost:0/birds@1.0.0#/bird.pkl")
assertThat(result.imports)
.isEqualTo(
mapOf(mainPkl.toUri() to setOf(ImportGraph.Import(birdUri)), birdUri to emptySet()),
)
assertThat(result.resolvedImports)
.isEqualTo(
mapOf(
mainPkl.toUri() to mainPkl.toRealPath().toUri(),
birdUri to birdModule.toRealPath().toUri()
)
)
}
private fun URI.realPath() = Path.of(this).toRealPath().toUri()
}
@@ -18,8 +18,11 @@ package org.pkl.core.ast.builder
import java.net.URI
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.assertThrows
import org.pkl.core.SecurityManagers
import org.pkl.core.StackFrameTransformers
import org.pkl.core.module.ModuleKeys
import org.pkl.core.runtime.VmException
class ImportsAndReadsParserTest {
@Test
@@ -27,13 +30,13 @@ class ImportsAndReadsParserTest {
val moduleText =
"""
amends "foo.pkl"
import "bar.pkl"
import "bazzy/buz.pkl"
res1 = import("qux.pkl")
res2 = import*("qux/*.pkl")
class MyClass {
res3 {
res4 {
@@ -48,7 +51,7 @@ class ImportsAndReadsParserTest {
val moduleKey = ModuleKeys.synthetic(URI("repl:text"), moduleText)
val imports =
ImportsAndReadsParser.parse(moduleKey, moduleKey.resolve(SecurityManagers.defaultManager))
assertThat(imports?.map { it.first })
assertThat(imports?.map { it.stringValue })
.hasSameElementsAs(
listOf(
"foo.pkl",
@@ -62,4 +65,31 @@ class ImportsAndReadsParserTest {
)
)
}
@Test
fun `invalid syntax`() {
val moduleText =
"""
not valid Pkl syntax
"""
.trimIndent()
val moduleKey = ModuleKeys.synthetic(URI("repl:text"), moduleText)
val err =
assertThrows<VmException> {
ImportsAndReadsParser.parse(moduleKey, moduleKey.resolve(SecurityManagers.defaultManager))
}
assertThat(err.toPklException(StackFrameTransformers.defaultTransformer))
.hasMessage(
"""
–– Pkl Error ––
Mismatched input: `<EOF>`. Expected one of: `{`, `=`, `:`
1 | not valid Pkl syntax
^
at text (repl:text)
"""
.trimIndent()
)
}
}