Add parser benchmarks (#953)

The current benchmark is not too useful now that we have a linear parser.

This changes the benchmarks to parse the standard library, and also the language snippet tests.

Co-authored-by: Islon Scherer <islonscherer@gmail.com>

bench/bench.gradle.kts

@@ -22,9 +22,9 @@ plugins {
 val truffle: Configuration by configurations.creating
 val graal: Configuration by configurations.creating
 
-@Suppress("UnstableApiUsage")
 dependencies {
   jmh(projects.pklCore)
+  jmh(projects.pklCommonsTest)
   truffle(libs.truffleApi)
   graal(libs.graalCompiler)
 }

bench/gradle.lockfile

@@ -1,11 +1,11 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
-net.bytebuddy:byte-buddy:1.15.11=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+net.bytebuddy:byte-buddy:1.15.11=jmh,jmhRuntimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 net.sf.jopt-simple:jopt-simple:5.0.4=jmh,jmhCompileClasspath,jmhImplementationDependenciesMetadata,jmhRuntimeClasspath
 org.apache.commons:commons-math3:3.6.1=jmh,jmhCompileClasspath,jmhImplementationDependenciesMetadata,jmhRuntimeClasspath
-org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath,testImplementationDependenciesMetadata,testJdk17CompileClasspath,testJdk17ImplementationDependenciesMetadata,testRuntimeOnlyDependenciesMetadata
+org.apiguardian:apiguardian-api:1.1.2=jmhCompileClasspath,jmhImplementationDependenciesMetadata,testCompileClasspath,testImplementationDependenciesMetadata,testJdk17CompileClasspath,testJdk17ImplementationDependenciesMetadata,testRuntimeOnlyDependenciesMetadata
-org.assertj:assertj-core:3.27.3=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+org.assertj:assertj-core:3.27.3=jmh,jmhRuntimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.graalvm.compiler:compiler:24.1.2=graal
 org.graalvm.polyglot:polyglot:24.1.2=jmh,jmhRuntimeClasspath,truffle
 org.graalvm.sdk:collections:24.1.2=graal,jmh,jmhRuntimeClasspath,truffle
@@ -30,24 +30,24 @@ org.jetbrains.kotlin:kotlin-scripting-common:2.0.21=kotlinBuildToolsApiClasspath
 org.jetbrains.kotlin:kotlin-scripting-compiler-embeddable:2.0.21=kotlinBuildToolsApiClasspath,kotlinCompilerPluginClasspathJmh,kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest,kotlinCompilerPluginClasspathTestJdk17
 org.jetbrains.kotlin:kotlin-scripting-compiler-impl-embeddable:2.0.21=kotlinBuildToolsApiClasspath,kotlinCompilerPluginClasspathJmh,kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest,kotlinCompilerPluginClasspathTestJdk17
 org.jetbrains.kotlin:kotlin-scripting-jvm:2.0.21=kotlinBuildToolsApiClasspath,kotlinCompilerPluginClasspathJmh,kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest,kotlinCompilerPluginClasspathTestJdk17
-org.jetbrains.kotlin:kotlin-stdlib-jdk7:2.0.21=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+org.jetbrains.kotlin:kotlin-stdlib-jdk7:2.0.21=jmh,jmhCompileClasspath,jmhImplementationDependenciesMetadata,jmhRuntimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
-org.jetbrains.kotlin:kotlin-stdlib-jdk8:2.0.21=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+org.jetbrains.kotlin:kotlin-stdlib-jdk8:2.0.21=jmh,jmhCompileClasspath,jmhImplementationDependenciesMetadata,jmhRuntimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
-org.jetbrains.kotlin:kotlin-stdlib:2.0.21=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinCompilerPluginClasspathJmh,kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest,kotlinCompilerPluginClasspathTestJdk17,kotlinKlibCommonizerClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+org.jetbrains.kotlin:kotlin-stdlib:2.0.21=jmh,jmhCompileClasspath,jmhImplementationDependenciesMetadata,jmhRuntimeClasspath,kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinCompilerPluginClasspathJmh,kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest,kotlinCompilerPluginClasspathTestJdk17,kotlinKlibCommonizerClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.jetbrains.kotlinx:kotlinx-coroutines-core-jvm:1.6.4=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinKlibCommonizerClasspath
-org.jetbrains:annotations:13.0=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinCompilerPluginClasspathJmh,kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest,kotlinCompilerPluginClasspathTestJdk17,kotlinKlibCommonizerClasspath,testCompileClasspath,testRuntimeClasspath
+org.jetbrains:annotations:13.0=jmh,jmhCompileClasspath,jmhRuntimeClasspath,kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinCompilerPluginClasspathJmh,kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest,kotlinCompilerPluginClasspathTestJdk17,kotlinKlibCommonizerClasspath,testCompileClasspath,testRuntimeClasspath
-org.junit.jupiter:junit-jupiter-api:5.11.4=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
+org.junit.jupiter:junit-jupiter-api:5.11.4=jmh,jmhCompileClasspath,jmhImplementationDependenciesMetadata,jmhRuntimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit.jupiter:junit-jupiter-api:5.8.2=testJdk17CompileClasspath,testJdk17ImplementationDependenciesMetadata,testJdk17RuntimeClasspath
-org.junit.jupiter:junit-jupiter-engine:5.11.4=testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
+org.junit.jupiter:junit-jupiter-engine:5.11.4=jmh,jmhCompileClasspath,jmhImplementationDependenciesMetadata,jmhRuntimeClasspath,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit.jupiter:junit-jupiter-engine:5.8.2=testJdk17RuntimeClasspath
-org.junit.jupiter:junit-jupiter-params:5.11.4=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+org.junit.jupiter:junit-jupiter-params:5.11.4=jmh,jmhCompileClasspath,jmhImplementationDependenciesMetadata,jmhRuntimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.junit.jupiter:junit-jupiter-params:5.8.2=testJdk17CompileClasspath,testJdk17ImplementationDependenciesMetadata,testJdk17RuntimeClasspath
 org.junit.jupiter:junit-jupiter:5.8.2=testJdk17CompileClasspath,testJdk17ImplementationDependenciesMetadata,testJdk17RuntimeClasspath
-org.junit.platform:junit-platform-commons:1.11.4=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
+org.junit.platform:junit-platform-commons:1.11.4=jmh,jmhCompileClasspath,jmhImplementationDependenciesMetadata,jmhRuntimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit.platform:junit-platform-commons:1.8.2=testJdk17CompileClasspath,testJdk17ImplementationDependenciesMetadata,testJdk17RuntimeClasspath
-org.junit.platform:junit-platform-engine:1.11.4=testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
+org.junit.platform:junit-platform-engine:1.11.4=jmh,jmhCompileClasspath,jmhImplementationDependenciesMetadata,jmhRuntimeClasspath,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit.platform:junit-platform-engine:1.8.2=testJdk17RuntimeClasspath
 org.junit.platform:junit-platform-launcher:1.8.2=testJdk17RuntimeClasspath
-org.junit:junit-bom:5.11.4=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
+org.junit:junit-bom:5.11.4=jmh,jmhCompileClasspath,jmhImplementationDependenciesMetadata,jmhRuntimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit:junit-bom:5.8.2=testJdk17CompileClasspath,testJdk17ImplementationDependenciesMetadata,testJdk17RuntimeClasspath
 org.msgpack:msgpack-core:0.9.8=jmh,jmhRuntimeClasspath
 org.openjdk.jmh:jmh-core:1.37=jmh,jmhCompileClasspath,jmhImplementationDependenciesMetadata,jmhRuntimeClasspath
@@ -55,7 +55,7 @@ org.openjdk.jmh:jmh-generator-asm:1.37=jmh,jmhCompileClasspath,jmhImplementation
 org.openjdk.jmh:jmh-generator-bytecode:1.37=jmh,jmhCompileClasspath,jmhImplementationDependenciesMetadata,jmhRuntimeClasspath
 org.openjdk.jmh:jmh-generator-reflection:1.37=jmh,jmhCompileClasspath,jmhImplementationDependenciesMetadata,jmhRuntimeClasspath
 org.opentest4j:opentest4j:1.2.0=testJdk17CompileClasspath,testJdk17ImplementationDependenciesMetadata,testJdk17RuntimeClasspath
-org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
+org.opentest4j:opentest4j:1.3.0=jmh,jmhCompileClasspath,jmhImplementationDependenciesMetadata,jmhRuntimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.organicdesign:Paguro:3.10.3=jmh,jmhRuntimeClasspath
 org.ow2.asm:asm:9.0=jmh,jmhCompileClasspath,jmhImplementationDependenciesMetadata,jmhRuntimeClasspath
 org.snakeyaml:snakeyaml-engine:2.9=jmh,jmhRuntimeClasspath

bench/src/jmh/java/org/pkl/core/parser/ParserBenchmark.java

@@ -15,8 +15,15 @@
  */
 package org.pkl.core.parser;
 
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
 import java.util.concurrent.TimeUnit;
 import org.openjdk.jmh.annotations.*;
+import org.pkl.commons.test.FileTestUtils;
+import org.pkl.commons.test.FileTestUtilsKt;
+import org.pkl.core.Release;
+import org.pkl.core.util.IoUtils;
 
 @SuppressWarnings("unused")
 @Warmup(iterations = 5, time = 2)
@@ -24,37 +31,33 @@ import org.openjdk.jmh.annotations.*;
 @OutputTimeUnit(TimeUnit.SECONDS)
 @Fork(1)
 public class ParserBenchmark {
-  // One-time execution of this code took ~10s until moving rule alternative
-  // for parenthesized expression after alternative for anonymous function.
   @Benchmark
-  public void run() {
+  public void parseStdlib() {
-    new Parser()
+    for (var stdlibModule : Release.current().standardLibrary().modules()) {
-        .parseModule(
+      try {
-            """
+        var moduleSource =
-            a1 {
+            IoUtils.readClassPathResourceAsString(
-              a2 {
+                getClass(), "/org/pkl/core/stdlib/%s.pkl".formatted(stdlibModule.substring(4)));
-                a3 {
+        new Parser().parseModule(moduleSource);
-                  a4 {
+      } catch (IOException e) {
-                    a5 {
+        throw new UncheckedIOException(e);
-                      a6 {
+      }
-                        a7 {
+    }
-                          a8 {
+  }
-                            a9 {
+
-                              a10 {
+  @Benchmark
-                                a11 {
+  public void parseSnippetTests() {
-                                  a12 {
+    var snippetTestDir =
-                                    a13 = map(map(map((x) -> 1)))
+        FileTestUtils.getRootProjectDir()
-                                  }
+            .resolve("pkl-core/src/test/files/LanguageSnippetTests/input");
-                                }
+    for (var snippet : FileTestUtilsKt.listFilesRecursively(snippetTestDir)) {
-                              }
+      try {
-                            }
+        var moduleSource = Files.readString(snippet);
-                          }
+        new Parser().parseModule(moduleSource);
-                        }
+      } catch (IOException e) {
-                      }
+        throw new UncheckedIOException(e);
-                    }
+      } catch (ParserError ignore) {
-                  }
+      }
-                }
+    }
-              }
-            }""");
   }
 }

pkl-commons-test/src/main/kotlin/org/pkl/commons/test/FileTestUtils.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,12 +23,13 @@ import org.opentest4j.AssertionFailedError
 import org.pkl.commons.*
 
 object FileTestUtils {
+  @JvmStatic
   val rootProjectDir: Path by lazy {
     val workingDir = currentWorkingDir
     workingDir.takeIf { it.resolve("settings.gradle.kts").exists() }
       ?: workingDir.parent.takeIf { it.resolve("settings.gradle.kts").exists() }
-        ?: workingDir.parent.parent.takeIf { it.resolve("settings.gradle.kts").exists() }
+      ?: workingDir.parent.parent.takeIf { it.resolve("settings.gradle.kts").exists() }
-        ?: throw AssertionError("Failed to locate root project directory.")
+      ?: throw AssertionError("Failed to locate root project directory.")
   }
   val selfSignedCertificate: Path by lazy {
     rootProjectDir.resolve("pkl-commons-test/build/keystore/localhost.pem")