Add analyze imports libs (SPICE-0001) (#695)

This adds a new feature to build a dependency graph of Pkl programs, following the SPICE outlined in https://github.com/apple/pkl-evolution/pull/2.

It adds:
* CLI command `pkl analyze imports`
* Java API `org.pkl.core.Analyzer`
* Pkl stdlib module `pkl:analyze`
* pkl-gradle extension `analyze`

In addition, it also changes the Gradle plugin such that `transitiveModules` is by default computed from the import graph.

pkl-gradle/src/main/java/org/pkl/gradle/PklAnalyzerCommands.java

@@ -0,0 +1,28 @@
+/**
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.gradle;
+
+import org.gradle.api.Action;
+import org.gradle.api.NamedDomainObjectContainer;
+import org.pkl.gradle.spec.AnalyzeImportsSpec;
+
+public interface PklAnalyzerCommands {
+  NamedDomainObjectContainer<AnalyzeImportsSpec> getImports();
+
+  default void imports(Action<? super NamedDomainObjectContainer<AnalyzeImportsSpec>> action) {
+    action.execute(getImports());
+  }
+}

pkl-gradle/src/main/java/org/pkl/gradle/PklExtension.java

@@ -39,6 +39,9 @@ public interface PklExtension {
   @Nested
   PklProjectCommands getProject();
 
+  @Nested
+  PklAnalyzerCommands getAnalyzers();
+
   default void evaluators(Action<? super NamedDomainObjectContainer<EvalSpec>> action) {
     action.execute(getEvaluators());
   }
@@ -64,4 +67,8 @@ public interface PklExtension {
   default void project(Action<? super PklProjectCommands> action) {
     action.execute(getProject());
   }
+
+  default void analyzers(Action<? super PklAnalyzerCommands> action) {
+    action.execute(getAnalyzers());
+  }
 }

pkl-gradle/src/main/java/org/pkl/gradle/PklPlugin.java

@@ -17,6 +17,7 @@ package org.pkl.gradle;
 
 import java.io.File;
 import java.lang.reflect.InvocationTargetException;
+import java.nio.file.Files;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Optional;
@@ -36,8 +37,12 @@ import org.gradle.language.base.plugins.LifecycleBasePlugin;
 import org.gradle.plugins.ide.idea.model.IdeaModel;
 import org.gradle.util.GradleVersion;
 import org.pkl.cli.CliEvaluatorOptions;
+import org.pkl.core.ImportGraph;
+import org.pkl.core.OutputFormat;
 import org.pkl.core.util.IoUtils;
 import org.pkl.core.util.LateInit;
+import org.pkl.core.util.Nullable;
+import org.pkl.gradle.spec.AnalyzeImportsSpec;
 import org.pkl.gradle.spec.BasePklSpec;
 import org.pkl.gradle.spec.CodeGenSpec;
 import org.pkl.gradle.spec.EvalSpec;
@@ -48,6 +53,7 @@ import org.pkl.gradle.spec.PkldocSpec;
 import org.pkl.gradle.spec.ProjectPackageSpec;
 import org.pkl.gradle.spec.ProjectResolveSpec;
 import org.pkl.gradle.spec.TestSpec;
+import org.pkl.gradle.task.AnalyzeImportsTask;
 import org.pkl.gradle.task.BasePklTask;
 import org.pkl.gradle.task.CodeGenTask;
 import org.pkl.gradle.task.EvalTask;
@@ -87,6 +93,7 @@ public class PklPlugin implements Plugin<Project> {
     configureTestTasks(extension.getTests());
     configureProjectPackageTasks(extension.getProject().getPackagers());
     configureProjectResolveTasks(extension.getProject().getResolvers());
+    configureAnalyzeImportsTasks(extension.getAnalyzers().getImports());
   }
 
   private void configureProjectPackageTasks(NamedDomainObjectContainer<ProjectPackageSpec> specs) {
@@ -128,6 +135,21 @@ public class PklPlugin implements Plugin<Project> {
         });
   }
 
+  private void configureAnalyzeImportsTasks(NamedDomainObjectContainer<AnalyzeImportsSpec> specs) {
+    specs.all(
+        spec -> {
+          configureBaseSpec(spec);
+          spec.getOutputFormat().convention(OutputFormat.PCF.toString());
+          var analyzeImportsTask = createTask(AnalyzeImportsTask.class, spec);
+          analyzeImportsTask.configure(
+              task -> {
+                task.getOutputFormat().set(spec.getOutputFormat());
+                task.getOutputFile().set(spec.getOutputFile());
+                configureModulesTask(task, spec, null);
+              });
+        });
+  }
+
   private void configureEvalTasks(NamedDomainObjectContainer<EvalSpec> specs) {
     specs.all(
         spec -> {
@@ -141,7 +163,7 @@ public class PklPlugin implements Plugin<Project> {
                       // and the working directory is set to the project directory,
                       // so this path works correctly.
                       .file("%{moduleDir}/%{moduleName}.%{outputFormat}"));
-          spec.getOutputFormat().convention("pcf");
+          spec.getOutputFormat().convention(OutputFormat.PCF.toString());
           spec.getModuleOutputSeparator()
               .convention(CliEvaluatorOptions.Companion.getDefaults().getModuleOutputSeparator());
           spec.getExpression()
@@ -431,20 +453,75 @@ public class PklPlugin implements Plugin<Project> {
     task.getHttpNoProxy().set(spec.getHttpNoProxy());
   }
 
-  private <T extends ModulesTask, S extends ModulesSpec> void configureModulesTask(T task, S spec) {
+  private List<File> getTransitiveModules(AnalyzeImportsTask analyzeTask) {
+    var outputFile = analyzeTask.getOutputFile().get().getAsFile().toPath();
+    try {
+      var contents = Files.readString(outputFile);
+      ImportGraph importGraph = ImportGraph.parseFromJson(contents);
+      var imports = importGraph.resolvedImports().values();
+      return imports.stream()
+          .filter((it) -> it.getScheme().equalsIgnoreCase("file"))
+          .map(File::new)
+          .toList();
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  private <T extends ModulesTask, S extends ModulesSpec> void configureModulesTask(
+      T task, S spec, @Nullable TaskProvider<AnalyzeImportsTask> analyzeImportsTask) {
     configureBaseTask(task, spec);
     task.getSourceModules().set(spec.getSourceModules());
-    task.getTransitiveModules().from(spec.getTransitiveModules());
     task.getNoProject().set(spec.getNoProject());
     task.getProjectDir().set(spec.getProjectDir());
     task.getOmitProjectSettings().set(spec.getOmitProjectSettings());
+    if (!spec.getTransitiveModules().isEmpty()) {
+      task.getTransitiveModules().set(spec.getTransitiveModules());
+    } else if (analyzeImportsTask != null) {
+      task.dependsOn(analyzeImportsTask);
+      task.getTransitiveModules().set(analyzeImportsTask.map(this::getTransitiveModules));
+    }
   }
 
-  private <T extends ModulesTask> TaskProvider<T> createModulesTask(
-      Class<T> taskClass, ModulesSpec spec) {
+  private TaskProvider<AnalyzeImportsTask> createAnalyzeImportsTask(ModulesSpec spec) {
+    var outputFile =
+        project
+            .getLayout()
+            .getBuildDirectory()
+            .file("pkl-gradle/imports/" + spec.getName() + ".json");
     return project
         .getTasks()
-        .register(spec.getName(), taskClass, task -> configureModulesTask(task, spec));
+        .register(
+            spec.getName() + "GatherImports",
+            AnalyzeImportsTask.class,
+            task -> {
+              configureModulesTask(task, spec, null);
+              task.setDescription("Compute the set of imports declared by input modules");
+              task.setGroup("build");
+              task.getOutputFormat().set(OutputFormat.JSON.toString());
+              task.getOutputFile().set(outputFile);
+            });
+  }
+
+  /**
+   * Implicitly also create a task of type {@link AnalyzeImportsTask}, postfixing the spec name with
+   * {@code "GatherImports"}.
+   *
+   * <p>The resulting task depends on the analyze task, and configures its own input files based on
+   * the result of analysis.
+   *
+   * <p>The end result is that the task automatically has correct up-to-date checks without users
+   * needing to manually provide transitive modules.
+   */
+  private <T extends ModulesTask> TaskProvider<T> createModulesTask(
+      Class<T> taskClass, ModulesSpec spec) {
+    var analyzeImportsTask = createAnalyzeImportsTask(spec);
+    return project
+        .getTasks()
+        .register(
+            spec.getName(),
+            taskClass,
+            task -> configureModulesTask(task, spec, analyzeImportsTask));
   }
 
   private <T extends BasePklTask> TaskProvider<T> createTask(Class<T> taskClass, BasePklSpec spec) {

pkl-gradle/src/main/java/org/pkl/gradle/spec/AnalyzeImportsSpec.java

@@ -0,0 +1,26 @@
+/**
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.gradle.spec;
+
+import org.gradle.api.file.RegularFileProperty;
+import org.gradle.api.provider.Property;
+
+/** Configuration options for import analyzers. Documented in user manual. */
+public interface AnalyzeImportsSpec extends ModulesSpec {
+  RegularFileProperty getOutputFile();
+
+  Property<String> getOutputFormat();
+}

pkl-gradle/src/main/java/org/pkl/gradle/task/AnalyzeImportsTask.java

@@ -0,0 +1,52 @@
+/**
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.gradle.task;
+
+import java.io.File;
+import org.gradle.api.file.RegularFileProperty;
+import org.gradle.api.provider.Property;
+import org.gradle.api.provider.Provider;
+import org.gradle.api.tasks.Input;
+import org.gradle.api.tasks.Optional;
+import org.gradle.api.tasks.OutputFile;
+import org.pkl.cli.CliImportAnalyzer;
+import org.pkl.cli.CliImportAnalyzerOptions;
+
+public abstract class AnalyzeImportsTask extends ModulesTask {
+  @OutputFile
+  @Optional
+  public abstract RegularFileProperty getOutputFile();
+
+  @Input
+  public abstract Property<String> getOutputFormat();
+
+  private final Provider<CliImportAnalyzer> cliImportAnalyzerProvider =
+      getProviders()
+          .provider(
+              () ->
+                  new CliImportAnalyzer(
+                      new CliImportAnalyzerOptions(
+                          getCliBaseOptions(),
+                          mapAndGetOrNull(getOutputFile(), it -> it.getAsFile().toPath()),
+                          mapAndGetOrNull(getOutputFormat(), it -> it))));
+
+  @Override
+  protected void doRunTask() {
+    //noinspection ResultOfMethodCallIgnored
+    getOutputs().getPreviousOutputFiles().forEach(File::delete);
+    cliImportAnalyzerProvider.get().run();
+  }
+}

pkl-gradle/src/main/java/org/pkl/gradle/task/ModulesTask.java

@@ -25,7 +25,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 import org.gradle.api.InvalidUserDataException;
-import org.gradle.api.file.ConfigurableFileCollection;
 import org.gradle.api.file.DirectoryProperty;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.provider.ListProperty;
@@ -49,7 +48,7 @@ public abstract class ModulesTask extends BasePklTask {
   public abstract ListProperty<Object> getSourceModules();
 
   @InputFiles
-  public abstract ConfigurableFileCollection getTransitiveModules();
+  public abstract ListProperty<File> getTransitiveModules();
 
   private final Map<List<Object>, Pair<List<File>, List<URI>>> parsedSourceModulesCache =
       new HashMap<>();

pkl-gradle/src/test/kotlin/org/pkl/gradle/AnalyzeImportsTest.kt

@@ -0,0 +1,108 @@
+/**
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.gradle
+
+import org.assertj.core.api.Assertions.assertThat
+import org.junit.jupiter.api.Test
+
+class AnalyzeImportsTest : AbstractTest() {
+  @Test
+  fun `write to console`() {
+    writeFile("input.pkl", "")
+    writeFile(
+      "build.gradle",
+      """
+        plugins {
+          id "org.pkl-lang"
+        }
+
+        pkl {
+          analyzers {
+            imports {
+              analyzeMyImports {
+                sourceModules = ["input.pkl"]
+              }
+            }
+          }
+        }
+      """
+        .trimIndent()
+    )
+    val result = runTask("analyzeMyImports")
+    assertThat(result.output).contains("imports {")
+  }
+
+  @Test
+  fun `output file`() {
+    writeFile("input.pkl", "")
+    writeFile(
+      "build.gradle",
+      """
+        plugins {
+          id "org.pkl-lang"
+        }
+
+        pkl {
+          analyzers {
+            imports {
+              analyzeMyImports {
+                sourceModules = ["input.pkl"]
+                outputFile = file("myFile.pcf")
+              }
+            }
+          }
+        }
+      """
+        .trimIndent()
+    )
+    runTask("analyzeMyImports")
+    assertThat(testProjectDir.resolve("myFile.pcf")).exists()
+  }
+
+  @Test
+  fun `output format`() {
+    writeFile("input.pkl", "")
+    writeFile(
+      "build.gradle",
+      """
+        plugins {
+          id "org.pkl-lang"
+        }
+
+        pkl {
+          analyzers {
+            imports {
+              analyzeMyImports {
+                sourceModules = ["input.pkl"]
+                outputFormat = "json"
+              }
+            }
+          }
+        }
+      """
+        .trimIndent()
+    )
+    val result = runTask("analyzeMyImports")
+    assertThat(result.output)
+      .contains(
+        """
+          {
+            "imports": {
+        """
+          .trimIndent()
+      )
+  }
+}

pkl-gradle/src/test/kotlin/org/pkl/gradle/EvaluatorsTest.kt

@@ -839,6 +839,62 @@ class EvaluatorsTest : AbstractTest() {
       )
   }
 
+  @Test
+  fun `implicit dependency tracking for declared imports`() {
+    writePklFile("import \"shared.pkl\"")
+    writeFile("shared.pkl", "foo = 1")
+    writeBuildFile("json")
+    val result1 = runTask("evalTest")
+    assertThat(result1.task(":evalTest")!!.outcome).isEqualTo(TaskOutcome.SUCCESS)
+
+    // evalTest should be up-to-date now
+    val result2 = runTask("evalTest")
+    assertThat(result2.task(":evalTest")!!.outcome).isEqualTo(TaskOutcome.UP_TO_DATE)
+
+    // update transitive module with new contents
+    writeFile("shared.pkl", "foo = 2")
+
+    // evalTest should be out-of-date and need to run again
+    val result3 = runTask("evalTest")
+    assertThat(result3.task(":evalTest")!!.outcome).isEqualTo(TaskOutcome.SUCCESS)
+
+    // running again should be up-to-date again
+    val result4 = runTask("evalTest")
+    assertThat(result4.task(":evalTest")!!.outcome).isEqualTo(TaskOutcome.UP_TO_DATE)
+  }
+
+  @Test
+  fun `explicit dependency tracking using transitive modules`() {
+    writePklFile("import \"shared.pkl\"")
+    writeFile("shared.pkl", "foo = 1")
+    writeFile("shared2.pkl", "foo = 1")
+    // intentionally use wrong transitive module
+    writeBuildFile(
+      "json",
+      additionalContents =
+        """
+      transitiveModules.from(files("shared2.pkl"))
+    """
+          .trimIndent()
+    )
+    val result1 = runTask("evalTest")
+    assertThat(result1.task(":evalTest")!!.outcome).isEqualTo(TaskOutcome.SUCCESS)
+
+    // evalTest should be up-to-date now
+    val result2 = runTask("evalTest")
+    assertThat(result2.task(":evalTest")!!.outcome).isEqualTo(TaskOutcome.UP_TO_DATE)
+
+    // update transitive module with new contents
+    writeFile("shared2.pkl", "foo = 2")
+
+    // evalTest should be out-of-date and need to run again
+    val result5 = runTask("evalTest")
+    assertThat(result5.task(":evalTest")!!.outcome).isEqualTo(TaskOutcome.SUCCESS)
+
+    // the "GatherImports" task did not run
+    assertThat(result5.task(":evalTestGatherImports")).isNull()
+  }
+
   private fun writeBuildFile(
     // don't use `org.pkl.core.OutputFormat`
     // because test compile class path doesn't contain pkl-core