Add analyze imports libs (SPICE-0001) (#695)

This adds a new feature to build a dependency graph of Pkl programs, following the SPICE outlined in https://github.com/apple/pkl-evolution/pull/2.

It adds:
* CLI command `pkl analyze imports`
* Java API `org.pkl.core.Analyzer`
* Pkl stdlib module `pkl:analyze`
* pkl-gradle extension `analyze`

In addition, it also changes the Gradle plugin such that `transitiveModules` is by default computed from the import graph.

pkl-cli/src/main/kotlin/org/pkl/cli/CliImportAnalyzer.kt

@@ -0,0 +1,79 @@
+/**
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.cli
+
+import java.io.Writer
+import org.pkl.commons.cli.CliCommand
+import org.pkl.commons.createParentDirectories
+import org.pkl.commons.writeString
+import org.pkl.core.ModuleSource
+import org.pkl.core.module.ModuleKeyFactories
+
+class CliImportAnalyzer
+@JvmOverloads
+constructor(
+  private val options: CliImportAnalyzerOptions,
+  private val consoleWriter: Writer = System.out.writer()
+) : CliCommand(options.base) {
+
+  override fun doRun() {
+    val rendered = render()
+    if (options.outputPath != null) {
+      options.outputPath.createParentDirectories()
+      options.outputPath.writeString(rendered)
+    } else {
+      consoleWriter.write(rendered)
+      consoleWriter.flush()
+    }
+  }
+
+  // language=pkl
+  private val sourceModule =
+    ModuleSource.text(
+      """
+        import "pkl:analyze"
+
+        local importStrings = read*("prop:pkl.analyzeImports.**").toMap().values.toSet()
+
+        output {
+          value = analyze.importGraph(importStrings)
+          renderer {
+            converters {
+              [Map] = (it) -> it.toMapping()
+              [Set] = (it) -> it.toListing()
+            }
+          }
+        }
+      """
+        .trimIndent()
+    )
+
+  private fun render(): String {
+    val builder = evaluatorBuilder().setOutputFormat(options.outputFormat)
+    try {
+      return builder
+        .apply {
+          for ((idx, sourceModule) in options.base.normalizedSourceModules.withIndex()) {
+            addExternalProperty("pkl.analyzeImports.$idx", sourceModule.toString())
+          }
+        }
+        .build()
+        .use { it.evaluateOutputText(sourceModule) }
+    } finally {
+      ModuleKeyFactories.closeQuietly(builder.moduleKeyFactories)
+    }
+  }
+}

pkl-cli/src/main/kotlin/org/pkl/cli/CliImportAnalyzerOptions.kt

@@ -0,0 +1,34 @@
+/**
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.cli
+
+import java.nio.file.Path
+import org.pkl.commons.cli.CliBaseOptions
+
+data class CliImportAnalyzerOptions(
+  /** Base options shared between CLI commands. */
+  val base: CliBaseOptions,
+
+  /** The file path where the output file is placed. */
+  val outputPath: Path? = null,
+
+  /**
+   * The output format to generate.
+   *
+   * These accept the same options as [CliEvaluatorOptions.outputFormat].
+   */
+  val outputFormat: String? = null,
+)

pkl-cli/src/main/kotlin/org/pkl/cli/Main.kt

@@ -34,7 +34,8 @@ internal fun main(args: Array<String>) {
         ServerCommand(helpLink),
         TestCommand(helpLink),
         ProjectCommand(helpLink),
-        DownloadPackageCommand(helpLink)
+        DownloadPackageCommand(helpLink),
+        AnalyzeCommand(helpLink),
       )
       .main(args)
   }

pkl-cli/src/main/kotlin/org/pkl/cli/commands/AnalyzeCommand.kt

@@ -0,0 +1,66 @@
+/**
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.cli.commands
+
+import com.github.ajalt.clikt.core.NoOpCliktCommand
+import com.github.ajalt.clikt.core.subcommands
+import com.github.ajalt.clikt.parameters.options.option
+import com.github.ajalt.clikt.parameters.types.path
+import java.nio.file.Path
+import org.pkl.cli.CliImportAnalyzer
+import org.pkl.cli.CliImportAnalyzerOptions
+import org.pkl.commons.cli.commands.ModulesCommand
+import org.pkl.commons.cli.commands.single
+
+class AnalyzeCommand(helpLink: String) :
+  NoOpCliktCommand(
+    name = "analyze",
+    help = "Commands related to static analysis",
+    epilog = "For more information, visit $helpLink"
+  ) {
+  init {
+    subcommands(AnalyzeImportsCommand(helpLink))
+  }
+
+  companion object {
+    class AnalyzeImportsCommand(helpLink: String) :
+      ModulesCommand(
+        name = "imports",
+        helpLink = helpLink,
+        help = "Prints the the graph of modules imported by the input module(s)."
+      ) {
+
+      private val outputPath: Path? by
+        option(
+            names = arrayOf("-o", "--output-path"),
+            metavar = "<path>",
+            help = "File path where the output file is placed."
+          )
+          .path()
+          .single()
+
+      override fun run() {
+        val options =
+          CliImportAnalyzerOptions(
+            base = baseOptions.baseOptions(modules, projectOptions),
+            outputFormat = baseOptions.format,
+            outputPath = outputPath
+          )
+        CliImportAnalyzer(options).run()
+      }
+    }
+  }
+}

pkl-cli/src/test/kotlin/org/pkl/cli/CliImportAnalyzerTest.kt

@@ -0,0 +1,142 @@
+/**
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.cli
+
+import java.net.URI
+import java.nio.file.Path
+import org.assertj.core.api.Assertions.assertThat
+import org.assertj.core.api.Assertions.assertThatCode
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.io.TempDir
+import org.pkl.commons.cli.CliBaseOptions
+import org.pkl.commons.writeString
+import org.pkl.core.OutputFormat
+import org.pkl.core.util.StringBuilderWriter
+
+class CliImportAnalyzerTest {
+  @Test
+  fun `write to console writer`(@TempDir tempDir: Path) {
+    val file = tempDir.resolve("test.pkl").writeString("import \"bar.pkl\"")
+    val otherFile = tempDir.resolve("bar.pkl").writeString("")
+    val baseOptions = CliBaseOptions(sourceModules = listOf(file.toUri()))
+    val sb = StringBuilder()
+    val analyzer = CliImportAnalyzer(CliImportAnalyzerOptions(baseOptions), StringBuilderWriter(sb))
+    analyzer.run()
+    assertThat(sb.toString())
+      .isEqualTo(
+        """
+          imports {
+            ["${otherFile.toUri()}"] {}
+            ["${file.toUri()}"] {
+              new {
+                uri = "${otherFile.toUri()}"
+              }
+            }
+          }
+          resolvedImports {
+            ["${otherFile.toUri()}"] = "${otherFile.toRealPath().toUri()}"
+            ["${file.toUri()}"] = "${file.toRealPath().toUri()}"
+          }
+
+        """
+          .trimIndent()
+      )
+  }
+
+  @Test
+  fun `different output format`(@TempDir tempDir: Path) {
+    val file = tempDir.resolve("test.pkl").writeString("import \"bar.pkl\"")
+    val otherFile = tempDir.resolve("bar.pkl").writeString("")
+    val baseOptions = CliBaseOptions(sourceModules = listOf(file.toUri()))
+    val sb = StringBuilder()
+    val analyzer =
+      CliImportAnalyzer(
+        CliImportAnalyzerOptions(baseOptions, outputFormat = OutputFormat.JSON.toString()),
+        StringBuilderWriter(sb)
+      )
+    analyzer.run()
+    assertThat(sb.toString())
+      .isEqualTo(
+        """
+          {
+            "imports": {
+              "${otherFile.toUri()}": [],
+              "${file.toUri()}": [
+                {
+                  "uri": "${otherFile.toUri()}"
+                }
+              ]
+            },
+            "resolvedImports": {
+              "${otherFile.toUri()}": "${otherFile.toRealPath().toUri()}",
+              "${file.toUri()}": "${file.toRealPath().toUri()}"
+            }
+          }
+
+        """
+          .trimIndent()
+      )
+  }
+
+  @Test
+  fun `write to output file`(@TempDir tempDir: Path) {
+    val file = tempDir.resolve("test.pkl").writeString("import \"bar.pkl\"")
+    val otherFile = tempDir.resolve("bar.pkl").writeString("")
+    val outputPath = tempDir.resolve("imports.pcf")
+    val baseOptions = CliBaseOptions(sourceModules = listOf(file.toUri()))
+    val analyzer = CliImportAnalyzer(CliImportAnalyzerOptions(baseOptions, outputPath = outputPath))
+    analyzer.run()
+    assertThat(outputPath)
+      .hasContent(
+        """
+          imports {
+            ["${otherFile.toUri()}"] {}
+            ["${file.toUri()}"] {
+              new {
+                uri = "${otherFile.toUri()}"
+              }
+            }
+          }
+          resolvedImports {
+            ["${otherFile.toUri()}"] = "${otherFile.toRealPath().toUri()}"
+            ["${file.toUri()}"] = "${file.toRealPath().toUri()}"
+          }
+
+        """
+          .trimIndent()
+      )
+  }
+
+  @Test
+  fun `invalid syntax in module`(@TempDir tempDir: Path) {
+    val file = tempDir.resolve("test.pkl").writeString("foo = bar(]")
+    assertThatCode {
+        CliImportAnalyzer(
+            CliImportAnalyzerOptions(
+              CliBaseOptions(sourceModules = listOf(file.toUri()), settings = URI("pkl:settings"))
+            )
+          )
+          .run()
+      }
+      .hasMessageContaining(
+        """
+          –– Pkl Error ––
+          Found a syntax error when parsing module `${file.toUri()}`.
+        """
+          .trimIndent()
+      )
+  }
+}

pkl-cli/src/test/kotlin/org/pkl/cli/CliMainTest.kt

@@ -27,6 +27,7 @@ import org.junit.jupiter.api.assertThrows
 import org.junit.jupiter.api.condition.DisabledOnOs
 import org.junit.jupiter.api.condition.OS
 import org.junit.jupiter.api.io.TempDir
+import org.pkl.cli.commands.AnalyzeCommand
 import org.pkl.cli.commands.EvalCommand
 import org.pkl.cli.commands.RootCommand
 import org.pkl.commons.writeString
@@ -34,7 +35,8 @@ import org.pkl.commons.writeString
 class CliMainTest {
 
   private val evalCmd = EvalCommand("")
-  private val cmd = RootCommand("pkl", "pkl version 1", "").subcommands(evalCmd)
+  private val analyzeCommand = AnalyzeCommand("")
+  private val cmd = RootCommand("pkl", "pkl version 1", "").subcommands(evalCmd, analyzeCommand)
 
   @Test
   fun `duplicate CLI option produces meaningful error message`(@TempDir tempDir: Path) {