Store PklProject annotations in the project metadata (#708)

Write annotations to project metadata, and provide them to pkl-doc\

The following annotations have meaning for pkl-doc:
* `@Unlisted`: hide package from documentation site
* `@Deprecated`: add deprecated information

pkl-core/src/main/java/org/pkl/core/packages/DependencyMetadata.java

@@ -19,12 +19,25 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 import java.net.URI;
+import java.net.URISyntaxException;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
+import java.util.regex.Pattern;
+import org.pkl.core.DataSize;
+import org.pkl.core.DataSizeUnit;
+import org.pkl.core.Duration;
+import org.pkl.core.DurationUnit;
+import org.pkl.core.PClassInfo;
+import org.pkl.core.PNull;
+import org.pkl.core.PObject;
+import org.pkl.core.Pair;
+import org.pkl.core.PklException;
 import org.pkl.core.Version;
 import org.pkl.core.packages.Dependency.RemoteDependency;
 import org.pkl.core.util.Nullable;
@@ -33,6 +46,7 @@ import org.pkl.core.util.json.Json.FormatException;
 import org.pkl.core.util.json.Json.JsArray;
 import org.pkl.core.util.json.Json.JsObject;
 import org.pkl.core.util.json.Json.JsonParseException;
+import org.pkl.core.util.json.Json.MissingFieldException;
 import org.pkl.core.util.json.JsonWriter;
 
 /**
@@ -63,7 +77,25 @@ import org.pkl.core.util.json.JsonWriter;
  *             "sha256": "abc123"
  *           }
  *         }
- *       }
+ *       },
+ *       "annotations": [
+ *         {
+ *           "moduleName": "pkl.base",
+ *           "class": "Unlisted",
+ *           "moduleUri": "pkl:base",
+ *           "properties": {}
+ *         },
+ *         {
+ *           "moduleName": "pkl.base",
+ *           "class": "Deprecated",
+ *           "moduleUri": "pkl:base",
+ *           "properties": {
+ *             "since": "0.26.1",
+ *             "message": "don't use",
+ *             "replaceWith": null
+ *           }
+ *         }
+ *       ]
  *     }
  *   </code>
  * </pre>
@@ -88,6 +120,8 @@ public final class DependencyMetadata {
     var authors = parsed.getNullable("authors", DependencyMetadata::parseAuthors);
     var issueTracker = parsed.getURIOrNull("issueTracker");
     var description = parsed.getStringOrNull("description");
+    var annotations = parsed.getNullable("annotations", DependencyMetadata::parseAnnotations);
+    if (annotations == null) annotations = List.of();
     return new DependencyMetadata(
         name,
         packageUri,
@@ -102,7 +136,8 @@ public final class DependencyMetadata {
         licenseText,
         authors,
         issueTracker,
-        description);
+        description,
+        annotations);
   }
 
   private static Map<String, RemoteDependency> parseDependencies(Object deps)
@@ -128,6 +163,122 @@ public final class DependencyMetadata {
     return ret;
   }
 
+  private static List<PObject> parseAnnotations(Object ann)
+      throws JsonParseException, URISyntaxException {
+    if (!(ann instanceof JsArray arr)) {
+      throw new FormatException("array", ann.getClass());
+    }
+    var annotations = new ArrayList<PObject>(arr.size());
+    for (var annotation : arr) {
+      var obj = parsePObject(annotation);
+      if (!(obj instanceof PObject pObject)) {
+        throw new PklException("Could not read annotation. Invalid object: " + obj);
+      }
+      annotations.add(pObject);
+    }
+    return annotations;
+  }
+
+  private static Object parsePObject(@Nullable Object obj)
+      throws JsonParseException, URISyntaxException {
+    if (obj == null) {
+      return PNull.getInstance();
+    } else if (obj instanceof String string) {
+      return string;
+    } else if (obj instanceof Boolean bool) {
+      return bool;
+    } else if (obj instanceof Integer integer) {
+      return integer.longValue();
+    } else if (obj instanceof Long aLong) {
+      return aLong;
+    } else if (obj instanceof Float aFloat) {
+      return aFloat.doubleValue();
+    } else if (obj instanceof Double aDouble) {
+      return aDouble;
+    } else if (obj instanceof JsArray array) {
+      var list = new ArrayList<>(array.size());
+      for (var element : array) {
+        list.add(parsePObject(element));
+      }
+      return list;
+    } else if (obj instanceof JsObject jsObj) {
+      var type = jsObj.getString("type");
+      switch (type) {
+        case "Set" -> {
+          var value = jsObj.getArray("value");
+          var set = new HashSet<>(value.size());
+          for (var element : value) {
+            set.add(parsePObject(element));
+          }
+          return set;
+        }
+        case "Map" -> {
+          var value = jsObj.getArray("value");
+          var map = new HashMap<>();
+          for (var kv : value) {
+            var kvObj = (JsObject) kv;
+            map.put(parsePObject(kvObj.get("key")), parsePObject(kvObj.get("value")));
+          }
+          return map;
+        }
+        case "PObject" -> {
+          var classInfoObj = jsObj.getObject("classInfo");
+          var moduleName = classInfoObj.getString("moduleName");
+          var className = classInfoObj.getString("class");
+          var moduleUri = classInfoObj.getString("moduleUri");
+          var props = jsObj.getObject("properties");
+          var classInfo = PClassInfo.get(moduleName, className, new URI(moduleUri));
+          var properties = new HashMap<String, Object>();
+          for (var kv : props.entrySet()) {
+            properties.put(kv.getKey(), parsePObject(kv.getValue()));
+          }
+          return new PObject(classInfo, properties);
+        }
+        case "Pattern" -> {
+          var value = jsObj.getString("value");
+          return Pattern.compile(value);
+        }
+        case "DataSize" -> {
+          var symbol = jsObj.getString("unit");
+          var value = jsObj.get("value");
+          if (value == null) {
+            throw new MissingFieldException(jsObj, "value");
+          }
+          var unit = DataSizeUnit.parse(symbol);
+          if (unit == null) {
+            throw new PklException("Invalid DataSize unit symbol: " + symbol);
+          }
+          if (!(value instanceof Double num)) {
+            throw new FormatException("double", value.getClass());
+          }
+          return new DataSize(num, unit);
+        }
+        case "Duration" -> {
+          var symbol = jsObj.getString("unit");
+          var value = jsObj.get("value");
+          if (value == null) {
+            throw new MissingFieldException(jsObj, "value");
+          }
+          var unit = DurationUnit.parse(symbol);
+          if (unit == null) {
+            throw new PklException("Invalid Duration unit symbol: " + symbol);
+          }
+          if (!(value instanceof Double num)) {
+            throw new FormatException("double", value.getClass());
+          }
+          return new Duration(num, unit);
+        }
+        case "Pair" -> {
+          var first = parsePObject(jsObj.get("first"));
+          var second = parsePObject(jsObj.get("second"));
+          return new Pair<>(first, second);
+        }
+      }
+    }
+    // should never be reached
+    throw new PklException("Could not read annotation. Invalid object type: " + obj.getClass());
+  }
+
   public static Checksums parseChecksums(Object obj) throws JsonParseException {
     if (!(obj instanceof JsObject jsObj)) {
       throw new FormatException("object", obj.getClass());
@@ -164,6 +315,7 @@ public final class DependencyMetadata {
   private final @Nullable List<String> authors;
   private final @Nullable URI issueTracker;
   private final @Nullable String description;
+  private final List<PObject> annotations;
 
   public DependencyMetadata(
       String name,
@@ -179,7 +331,8 @@ public final class DependencyMetadata {
       @Nullable String licenseText,
       @Nullable List<String> authors,
       @Nullable URI issueTracker,
-      @Nullable String description) {
+      @Nullable String description,
+      List<PObject> annotations) {
     this.name = name;
     this.packageUri = packageUri;
     this.version = version;
@@ -194,6 +347,7 @@ public final class DependencyMetadata {
     this.authors = authors;
     this.issueTracker = issueTracker;
     this.description = description;
+    this.annotations = annotations;
   }
 
   public String getName() {
@@ -250,6 +404,10 @@ public final class DependencyMetadata {
     return description;
   }
 
+  public List<PObject> getAnnotations() {
+    return annotations;
+  }
+
   /** Serializes project dependencies to JSON, and writes it to the provided output stream. */
   public void writeTo(OutputStream out) throws IOException {
     new DependencyMetadataWriter(out, this).write();
@@ -277,7 +435,8 @@ public final class DependencyMetadata {
         && Objects.equals(licenseText, that.licenseText)
         && Objects.equals(authors, that.authors)
         && Objects.equals(issueTracker, that.issueTracker)
-        && Objects.equals(description, that.description);
+        && Objects.equals(description, that.description)
+        && Objects.equals(annotations, that.annotations);
   }
 
   @Override
@@ -296,7 +455,8 @@ public final class DependencyMetadata {
         licenseText,
         authors,
         issueTracker,
-        description);
+        description,
+        annotations);
   }
 
   @Override
@@ -339,6 +499,8 @@ public final class DependencyMetadata {
         + '\''
         + ", description="
         + description
+        + ", annotations="
+        + annotations
         + '}';
   }
 
@@ -423,8 +585,125 @@ public final class DependencyMetadata {
       if (dependencyMetadata.description != null) {
         jsonWriter.name("description").value(dependencyMetadata.description);
       }
+      if (!dependencyMetadata.annotations.isEmpty()) {
+        jsonWriter.name("annotations");
+        writeAnnotations();
+      }
       jsonWriter.endObject();
       jsonWriter.close();
     }
+
+    private void writeAnnotations() throws IOException {
+      jsonWriter.beginArray();
+      for (var annotation : dependencyMetadata.annotations) {
+        writePObject(annotation);
+      }
+      jsonWriter.endArray();
+    }
+
+    private void writePClassInfo(PClassInfo<?> pClassInfo) throws IOException {
+      jsonWriter.beginObject();
+      jsonWriter.name("moduleName").value(pClassInfo.getModuleName());
+      jsonWriter.name("class").value(pClassInfo.getSimpleName());
+      jsonWriter.name("moduleUri").value(pClassInfo.getModuleUri().toString());
+      jsonWriter.endObject();
+    }
+
+    private void writePObject(PObject object) throws IOException {
+      jsonWriter.beginObject();
+      jsonWriter.name("type").value("PObject");
+      jsonWriter.name("classInfo");
+      writePClassInfo(object.getClassInfo());
+      jsonWriter.name("properties");
+      jsonWriter.beginObject();
+      for (var kv : object.getProperties().entrySet()) {
+        jsonWriter.name(kv.getKey());
+        writeGenericObject(kv.getValue());
+      }
+      jsonWriter.endObject();
+      jsonWriter.endObject();
+    }
+
+    private void writeGenericObject(Object value) throws IOException {
+      if (value instanceof PNull) {
+        jsonWriter.nullValue();
+      } else if (value instanceof PObject pObject) {
+        writePObject(pObject);
+      } else if (value instanceof String string) {
+        jsonWriter.value(string);
+      } else if (value instanceof Boolean bool) {
+        jsonWriter.value(bool);
+      } else if (value instanceof Integer num) {
+        jsonWriter.value(num);
+      } else if (value instanceof Long num) {
+        jsonWriter.value(num);
+      } else if (value instanceof Float num) {
+        jsonWriter.value(num);
+      } else if (value instanceof Double num) {
+        jsonWriter.value(num);
+      } else if (value instanceof List<?> list) {
+        jsonWriter.beginArray();
+        for (var v : list) {
+          writeGenericObject(v);
+        }
+        jsonWriter.endArray();
+      } else if (value instanceof Set<?> set) {
+        jsonWriter.beginObject();
+        jsonWriter.name("type").value("Set");
+        jsonWriter.name("value");
+        jsonWriter.beginArray();
+        for (var v : set) {
+          writeGenericObject(v);
+        }
+        jsonWriter.endArray();
+        jsonWriter.endObject();
+      } else if (value instanceof Map<?, ?> map) {
+        jsonWriter.beginObject();
+        jsonWriter.name("type").value("Map");
+        jsonWriter.name("value");
+        jsonWriter.beginArray();
+        for (var kv : map.entrySet()) {
+          jsonWriter.beginObject();
+          jsonWriter.name("key");
+          writeGenericObject(kv.getKey());
+          jsonWriter.name("value");
+          writeGenericObject(kv.getValue());
+          jsonWriter.endObject();
+        }
+        jsonWriter.endArray();
+        jsonWriter.endObject();
+      } else if (value instanceof Pattern pattern) {
+        jsonWriter.beginObject();
+        jsonWriter.name("type").value("Pattern");
+        jsonWriter.name("value").value(pattern.pattern());
+        jsonWriter.endObject();
+      } else if (value instanceof DataSize dataSize) {
+        jsonWriter.beginObject();
+        jsonWriter.name("type").value("DataSize");
+        jsonWriter.name("unit").value(dataSize.getUnit().getSymbol());
+        jsonWriter.name("value").value(dataSize.getValue());
+        jsonWriter.endObject();
+      } else if (value instanceof Duration duration) {
+        jsonWriter.beginObject();
+        jsonWriter.name("type").value("Duration");
+        jsonWriter.name("unit").value(duration.getUnit().getSymbol());
+        jsonWriter.name("value").value(duration.getValue());
+        jsonWriter.endObject();
+      } else if (value instanceof Pair<?, ?> pair) {
+        jsonWriter.beginObject();
+        jsonWriter.name("type").value("Pair");
+        jsonWriter.name("first");
+        writeGenericObject(pair.getFirst());
+        jsonWriter.name("second");
+        writeGenericObject(pair.getSecond());
+        jsonWriter.endObject();
+      } else {
+        // PClass and TypeAlias are not supported
+        throw new PklException(
+            "Error serializing annotation for PklProject:\n:"
+                + "  cannot render value with unexpected type: "
+                + value.getClass());
+      }
+    }
   }
 }

pkl-core/src/main/java/org/pkl/core/project/Project.java

@@ -65,6 +65,7 @@ public final class Project {
   private final URI projectBaseUri;
   private final List<URI> tests;
   private final Map<String, Project> localProjectDependencies;
+  private final List<PObject> annotations;
 
   /**
    * Loads Project data from the given {@link Path}.
@@ -256,6 +257,11 @@ public final class Project {
     return new RemoteDependency(packageUri, checksums);
   }
 
+  @SuppressWarnings("unchecked")
+  private static List<PObject> parseAnnotations(PObject module) {
+    return (List<PObject>) getProperty(module, "annotations");
+  }
+
   public static Project parseProject(PObject module) throws URISyntaxException {
     var pkgObj = getNullableProperty(module, "package");
     var projectFileUri = URI.create((String) module.getProperty("projectFileUri"));
@@ -279,6 +285,7 @@ public final class Project {
             .map((it) -> projectBaseUri.resolve(it).normalize())
             .collect(Collectors.toList());
     var localProjectDependencies = parseLocalProjectDependencies(module);
+    var annotations = parseAnnotations(module);
     return new Project(
         pkg,
         dependencies,
@@ -286,7 +293,8 @@ public final class Project {
         projectFileUri,
         projectBaseUri,
         tests,
-        localProjectDependencies);
+        localProjectDependencies,
+        annotations);
   }
 
   private static Map<String, Project> parseLocalProjectDependencies(PObject module)
@@ -399,7 +407,8 @@ public final class Project {
       URI projectFileUri,
       URI projectBaseUri,
       List<URI> tests,
-      Map<String, Project> localProjectDependencies) {
+      Map<String, Project> localProjectDependencies,
+      List<PObject> annotations) {
     this.pkg = pkg;
     this.dependencies = dependencies;
     this.evaluatorSettings = evaluatorSettings;
@@ -407,6 +416,7 @@ public final class Project {
     this.projectBaseUri = projectBaseUri;
     this.tests = tests;
     this.localProjectDependencies = localProjectDependencies;
+    this.annotations = annotations;
   }
 
   public @Nullable Package getPackage() {
@@ -453,12 +463,13 @@ public final class Project {
         && dependencies.equals(project.dependencies)
         && evaluatorSettings.equals(project.evaluatorSettings)
         && projectFileUri.equals(project.projectFileUri)
-        && tests.equals(project.tests);
+        && tests.equals(project.tests)
+        && annotations.equals(project.annotations);
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(pkg, dependencies, evaluatorSettings, projectFileUri, tests);
+    return Objects.hash(pkg, dependencies, evaluatorSettings, projectFileUri, tests, annotations);
   }
 
   public DeclaredDependencies getDependencies() {
@@ -478,6 +489,10 @@ public final class Project {
     return Path.of(projectBaseUri);
   }
 
+  public List<PObject> getAnnotations() {
+    return annotations;
+  }
+
   @Deprecated(forRemoval = true)
   public static class EvaluatorSettings {
     private final PklEvaluatorSettings delegate;

pkl-core/src/main/java/org/pkl/core/project/ProjectPackager.java

@@ -288,7 +288,8 @@ public final class ProjectPackager {
         pkg.getLicenseText(),
         pkg.getAuthors(),
         pkg.getIssueTracker(),
-        pkg.getDescription());
+        pkg.getDescription(),
+        project.getAnnotations());
   }
 
   private DigestOutputStream newDigestOutputStream(OutputStream outputStream) {