Add a CPU profiler (#1823)

This introduces a way to run CPU profiling.

The profiler produces output in pprof format.

CLI users can enable CPU profiling via command line flags.
Profiling is otherwise not offered as an API.

Co-authored-by: Jen Basch <jbasch94@gmail.com>
Co-authored-by: Islon Scherer <islonscherer@gmail.com>
This commit is contained in:
authored and GitHub committed 2026-10-15 15:57:55 +00:00
1 parent d062312df9
commit f96fba00e1
48 files changed
+1480 -38

No files matched your search

@@ -21,6 +21,7 @@ import com.github.ajalt.clikt.core.*
import com.github.ajalt.clikt.parameters.arguments.*
import com.github.ajalt.clikt.parameters.options.*
import com.github.ajalt.clikt.parameters.types.int
import com.oracle.truffle.api.TruffleOptions
import java.io.OutputStream
import java.net.URI
import java.nio.file.Path
@@ -40,6 +41,8 @@ import org.pkl.core.FileOutput
import org.pkl.core.ModuleSource.uri
import org.pkl.core.PklBugException
import org.pkl.core.PklException
import org.pkl.core.ProfilerOptions
import org.pkl.core.runtime.VmUtils
import org.pkl.core.util.IoUtils
class CliCommandRunner
@@ -51,15 +54,18 @@ constructor(
private val args: List<String>,
private val outputStream: OutputStream = System.out,
private val errStream: OutputStream = System.err,
private val profilerOptions: ProfilerOptions = ProfilerOptions.DEFAULT,
) : CliCommand(options) {
override fun doRun() {
val builder = evaluatorBuilder()
try {
evalCmd(builder)
withCpuProfiler(profilerOptions) { evalCmd(builder) }
} finally {
Closeables.closeQuietly(builder.moduleKeyFactories)
Closeables.closeQuietly(builder.resourceReaders)
if (TruffleOptions.AOT) {
VmUtils.closeEngine()
}
}
}
@@ -15,6 +15,7 @@
*/
package org.pkl.cli
import com.oracle.truffle.api.TruffleOptions
import java.io.File
import java.io.InputStream
import java.io.OutputStream
@@ -99,14 +100,19 @@ constructor(
override fun doRun() {
val builder = evaluatorBuilder()
try {
if (options.multipleFileOutputPath != null) {
writeMultipleFileOutput(builder)
} else {
writeOutput(builder)
withCpuProfiler(options.profilerOptions) {
if (options.multipleFileOutputPath != null) {
writeMultipleFileOutput(builder)
} else {
writeOutput(builder)
}
}
} finally {
Closeables.closeQuietly(builder.moduleKeyFactories)
Closeables.closeQuietly(builder.resourceReaders)
if (TruffleOptions.AOT) {
VmUtils.closeEngine()
}
}
}
@@ -16,6 +16,7 @@
package org.pkl.cli
import org.pkl.commons.cli.CliBaseOptions
import org.pkl.core.ProfilerOptions
/** Configuration options for [CliEvaluator]. */
data class CliEvaluatorOptions(
@@ -77,6 +78,13 @@ data class CliEvaluatorOptions(
* If unset, the module's `output.bytes` property is evaluated.
*/
val expression: String? = null,
/**
* CPU/Memory profiling options.
*
* WARN: This is an internal option only. Profiling is currently only supported via CLI flags.
*/
val profilerOptions: ProfilerOptions = ProfilerOptions.DEFAULT,
) {
companion object {
@@ -15,6 +15,7 @@
*/
package org.pkl.cli
import com.oracle.truffle.api.TruffleOptions
import java.io.StringWriter
import java.io.Writer
import org.pkl.commons.cli.*
@@ -22,7 +23,9 @@ import org.pkl.core.Closeables
import org.pkl.core.EvaluatorBuilder
import org.pkl.core.ModuleSource.uri
import org.pkl.core.PklException
import org.pkl.core.ProfilerOptions
import org.pkl.core.TestResults
import org.pkl.core.runtime.VmUtils
import org.pkl.core.stdlib.test.report.JUnitReporter
import org.pkl.core.stdlib.test.report.MinimalReporter
import org.pkl.core.stdlib.test.report.SpecReporter
@@ -35,15 +38,18 @@ constructor(
private val testOptions: CliTestOptions,
private val consoleWriter: Writer = System.out.writer(),
private val errWriter: Writer = System.err.writer(),
private val profilerOptions: ProfilerOptions = ProfilerOptions.DEFAULT,
) : CliCommand(options) {
override fun doRun() {
val builder = evaluatorBuilder()
try {
evalTest(builder)
withCpuProfiler(profilerOptions) { evalTest(builder) }
} finally {
Closeables.closeQuietly(builder.moduleKeyFactories)
Closeables.closeQuietly(builder.resourceReaders)
if (TruffleOptions.AOT) {
VmUtils.closeEngine()
}
}
}
@@ -16,6 +16,7 @@
package org.pkl.cli.commands
import com.github.ajalt.clikt.completion.CompletionCandidates
import com.github.ajalt.clikt.parameters.groups.provideDelegate
import com.github.ajalt.clikt.parameters.options.default
import com.github.ajalt.clikt.parameters.options.flag
import com.github.ajalt.clikt.parameters.options.option
@@ -23,6 +24,7 @@ import com.github.ajalt.clikt.parameters.options.validate
import org.pkl.cli.CliEvaluator
import org.pkl.cli.CliEvaluatorOptions
import org.pkl.commons.cli.commands.ModulesCommand
import org.pkl.commons.cli.commands.ProfileOptions
import org.pkl.commons.cli.commands.single
class EvalCommand : ModulesCommand(name = "eval", helpLink = helpLink) {
@@ -81,21 +83,25 @@ class EvalCommand : ModulesCommand(name = "eval", helpLink = helpLink) {
)
.flag("--no-power-assertions", default = true, defaultForHelp = "enabled")
val profileOptions: ProfileOptions by ProfileOptions()
override fun run() {
val cliBaseOptions =
baseOptions.baseOptions(
modules,
projectOptions,
testMode = testMode,
powerAssertionsEnabled = powerAssertionsEnabled,
)
val options =
CliEvaluatorOptions(
base =
baseOptions.baseOptions(
modules,
projectOptions,
testMode = testMode,
powerAssertionsEnabled = powerAssertionsEnabled,
),
base = cliBaseOptions,
outputPath = outputPath,
outputFormat = baseOptions.format,
moduleOutputSeparator = moduleOutputSeparator,
multipleFileOutputPath = multipleFileOutputPath,
expression = expression ?: CliEvaluatorOptions.defaults.expression,
profilerOptions = profileOptions.toOptions(cliBaseOptions.normalizedWorkingDir),
)
CliEvaluator(options).run()
}
@@ -30,6 +30,7 @@ import java.net.URI
import org.pkl.cli.CliCommandRunner
import org.pkl.commons.cli.commands.BaseCommand
import org.pkl.commons.cli.commands.BaseOptions
import org.pkl.commons.cli.commands.ProfileOptions
import org.pkl.commons.cli.commands.ProjectOptions
class RunCommand : BaseCommand(name = "run", helpLink = helpLink) {
@@ -61,6 +62,8 @@ class RunCommand : BaseCommand(name = "run", helpLink = helpLink) {
private val projectOptions by ProjectOptions()
private val profileOptions by ProfileOptions()
override fun run() {
// if no module is specified but --help is show help, otherwise error becuase module is missing
if (module == null)
@@ -75,11 +78,13 @@ class RunCommand : BaseCommand(name = "run", helpLink = helpLink) {
else reservedFlagShortNames.add(it.trimStart('-'))
}
}
val cliBaseOptions = baseOptions.baseOptions(listOf(module!!), projectOptions)
CliCommandRunner(
baseOptions.baseOptions(listOf(module!!), projectOptions),
cliBaseOptions,
reservedFlagNames,
reservedFlagShortNames,
if (showHelp) args + listOf("--help") else args,
profilerOptions = profileOptions.toOptions(cliBaseOptions.normalizedWorkingDir),
)
.run()
}
@@ -26,6 +26,7 @@ import java.net.URI
import org.pkl.cli.CliTestRunner
import org.pkl.commons.cli.commands.BaseCommand
import org.pkl.commons.cli.commands.BaseOptions
import org.pkl.commons.cli.commands.ProfileOptions
import org.pkl.commons.cli.commands.ProjectOptions
import org.pkl.commons.cli.commands.TestOptions
@@ -45,6 +46,8 @@ class TestCommand : BaseCommand(name = "test", helpLink = helpLink) {
private val testOptions by TestOptions()
private val profileOptions by ProfileOptions()
private val powerAssertionsEnabled: Boolean by
option(
names = arrayOf("--power-assertions"),
@@ -53,14 +56,16 @@ class TestCommand : BaseCommand(name = "test", helpLink = helpLink) {
.flag("--no-power-assertions", default = true, defaultForHelp = "enabled")
override fun run() {
val cliBaseOptions =
baseOptions.baseOptions(
modules,
projectOptions,
powerAssertionsEnabled = powerAssertionsEnabled,
)
CliTestRunner(
options =
baseOptions.baseOptions(
modules,
projectOptions,
powerAssertionsEnabled = powerAssertionsEnabled,
),
options = cliBaseOptions,
testOptions = testOptions.cliTestOptions,
profilerOptions = profileOptions.toOptions(cliBaseOptions.normalizedWorkingDir),
)
.run()
}
@@ -0,0 +1,38 @@
/*
* Copyright © 2026 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.cli
import org.pkl.core.ProfilerOptions
import org.pkl.core.runtime.VmUtils
import org.pkl.profiler.CpuProfiler
import org.pkl.profiler.CpuProfilerInstrument
/**
* Runs [body], sampling it with the CPU profiler if [options] enables CPU profiling.
*
* Profiler output is written when [body] finishes, whether normally or exceptionally.
*/
internal inline fun <T> withCpuProfiler(options: ProfilerOptions, body: () -> T): T {
if (!options.cpu.isEnabled) return body()
val profiler =
VmUtils.getEngine().instruments[CpuProfilerInstrument.ID]?.lookup(CpuProfiler::class.java)!!
profiler.start(options.cpu)
try {
return body()
} finally {
profiler.stop()
}
}
@@ -0,0 +1,199 @@
/*
* Copyright © 2026 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.cli
import com.google.perftools.profiles.ProfileProto.Profile
import java.io.ByteArrayOutputStream
import java.io.StringWriter
import java.net.URI
import java.nio.file.Path
import java.util.zip.GZIPInputStream
import kotlin.io.path.exists
import kotlin.io.path.inputStream
import kotlin.io.path.writeText
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.io.TempDir
import org.pkl.commons.cli.CliBaseOptions
import org.pkl.commons.cli.CliTestOptions
import org.pkl.core.ProfilerOptions
/**
* Exercises the CPU profiler by calling into the CLI classes in-process.
*
* See [SubprocessProfilerOutputTest] for tests that run the packaged executables.
*/
class ProfilerOutputTest {
private fun profilerOptions(outputFile: Path) =
ProfilerOptions(ProfilerOptions.Cpu(outputFile, 1))
private fun fib(name: String) =
"function $name(n) = if (n < 2) n else $name(n - 1) + $name(n - 2)"
private fun evalCode(name: String) =
"""
${fib(name)}
x = $name(30)
"""
.trimIndent()
private fun testCode(name: String) =
"""
amends "pkl:test"
local ${fib(name)}
facts {
["fib"] {
$name(30) > 0
}
}
"""
.trimIndent()
private fun runCode(name: String) =
"""
amends "pkl:Command"
command {
name = "do-run"
}
local ${fib(name)}
output {
text = $name(30).toString()
}
"""
.trimIndent()
private fun baseOptions(source: Path) =
CliBaseOptions(sourceModules = listOf(source.toUri()), settings = URI("pkl:settings"))
private fun eval(tempDir: Path, name: String, outputFile: Path) {
val source = tempDir.resolve("$name.pkl").also { it.writeText(evalCode(name)) }
val options =
CliEvaluatorOptions(baseOptions(source), profilerOptions = profilerOptions(outputFile))
CliEvaluator(options, outputStream = ByteArrayOutputStream()).run()
}
private fun test(tempDir: Path, name: String, outputFile: Path) {
val source = tempDir.resolve("$name.pkl").also { it.writeText(testCode(name)) }
CliTestRunner(
baseOptions(source),
CliTestOptions(),
consoleWriter = StringWriter(),
errWriter = StringWriter(),
profilerOptions = profilerOptions(outputFile),
)
.run()
}
private fun run(tempDir: Path, name: String, outputFile: Path) {
val source = tempDir.resolve("$name.pkl").also { it.writeText(runCode(name)) }
CliCommandRunner(
baseOptions(source),
emptySet(),
emptySet(),
emptyList(),
outputStream = ByteArrayOutputStream(),
errStream = ByteArrayOutputStream(),
profilerOptions = profilerOptions(outputFile),
)
.run()
}
private fun readProfile(outputFile: Path): Profile =
GZIPInputStream(outputFile.inputStream()).use { Profile.parseFrom(it) }
private fun Profile.functionNames() = functionList.map { stringTableList[it.name.toInt()] }
/** Smoke test of the profile's structure, and that it contains samples of [fibName]. */
private fun assertValidProfile(outputFile: Path, fibName: String) {
val profile = readProfile(outputFile)
assertThat(profile.stringTableList.first()).isEmpty() // string_table[0] must be ""
assertThat(profile.sampleTypeList).isNotEmpty
assertThat(profile.functionList).isNotEmpty
assertThat(profile.functionNames()).anyMatch { it.contains(fibName) }
}
@Test
fun `pkl eval`(@TempDir tempDir: Path) {
val outputFile = tempDir.resolve("profile.pb.gz")
eval(tempDir, "evalFib", outputFile)
assertValidProfile(outputFile, "evalFib")
}
@Test
fun `pkl test`(@TempDir tempDir: Path) {
val outputFile = tempDir.resolve("profile.pb.gz")
test(tempDir, "testFib", outputFile)
assertValidProfile(outputFile, "testFib")
}
@Test
fun `pkl run`(@TempDir tempDir: Path) {
val outputFile = tempDir.resolve("profile.pb.gz")
run(tempDir, "runFib", outputFile)
assertValidProfile(outputFile, "runFib")
}
@Test
fun `profiler can be restarted, and runs don't share samples`(@TempDir tempDir: Path) {
val firstOutput = tempDir.resolve("first.pb.gz")
val secondOutput = tempDir.resolve("second.pb.gz")
val thirdOutput = tempDir.resolve("third.pb.gz")
eval(tempDir, "firstFib", firstOutput)
test(tempDir, "secondFib", secondOutput)
run(tempDir, "thirdFib", thirdOutput)
assertValidProfile(firstOutput, "firstFib")
assertValidProfile(secondOutput, "secondFib")
assertValidProfile(thirdOutput, "thirdFib")
// samples from earlier runs must not leak into later ones
assertThat(readProfile(secondOutput).functionNames()).noneMatch { it.contains("firstFib") }
assertThat(readProfile(thirdOutput).functionNames()).noneMatch {
it.contains("firstFib") || it.contains("secondFib")
}
}
@Test
fun `profiler restarts after a failed evaluation`(@TempDir tempDir: Path) {
val failingSource = tempDir.resolve("failing.pkl").also { it.writeText("x = throw(\"boom\")") }
val failingOutput = tempDir.resolve("failing.pb.gz")
val options =
CliEvaluatorOptions(
baseOptions(failingSource),
profilerOptions = profilerOptions(failingOutput),
)
runCatching { CliEvaluator(options, outputStream = ByteArrayOutputStream()).run() }
.also { assertThat(it.isFailure).isTrue() }
val outputFile = tempDir.resolve("profile.pb.gz")
eval(tempDir, "afterFailureFib", outputFile)
assertValidProfile(outputFile, "afterFailureFib")
}
@Test
fun `no output is written if profiling is not enabled`(@TempDir tempDir: Path) {
val source = tempDir.resolve("noProfile.pkl").also { it.writeText(evalCode("noProfileFib")) }
CliEvaluator(CliEvaluatorOptions(baseOptions(source)), outputStream = ByteArrayOutputStream())
.run()
assertThat(tempDir.resolve("profile.pb.gz").exists()).isFalse()
}
}
@@ -0,0 +1,131 @@
/*
* Copyright © 2026 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.cli
import com.google.perftools.profiles.ProfileProto.Profile
import java.nio.file.Path
import java.util.zip.GZIPInputStream
import kotlin.io.path.absolutePathString
import kotlin.io.path.inputStream
import kotlin.io.path.writeText
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.io.TempDir
/**
* Exercises the CPU profiler end-to-end by running the packaged executable jar (or native
* executable) as a subprocess.
*
* Run this test using `./gradlew pkl-cli:testProfilerOutput` (executable jar) or `./gradlew
* pkl-cli:testNativeProfilerOutput` (native executable).
*
* To use an IntelliJ JUnit run configuration, configure the VM options and add
* `-Dorg.pkl.cli.testJar=<path>` (or `-Dorg.pkl.cli.testExecutable=<path>`).
*/
class SubprocessProfilerOutputTest {
/**
* The command that launches Pkl: the native executable if `org.pkl.cli.testExecutable` is set,
* otherwise the executable jar given by `org.pkl.cli.testJar`.
*/
private fun launcher(): Array<String> {
System.getProperty("org.pkl.cli.testExecutable")?.let {
return arrayOf(it)
}
val jar =
checkNotNull(System.getProperty("org.pkl.cli.testJar")) {
"neither system property `org.pkl.cli.testExecutable` nor `org.pkl.cli.testJar` is set"
}
val javaBin = ProcessHandle.current().info().command().orElseThrow()
return arrayOf(javaBin, "-jar", jar)
}
private fun runTest(tempDir: Path, pklCode: String, command: Array<String>) {
val sourceFile = tempDir.resolve("profiled.pkl").also { it.writeText(pklCode) }
val outputFile = tempDir.resolve("profile.pb.gz")
val process =
ProcessBuilder(
*launcher(),
*command,
"--profile-cpu-output",
outputFile.absolutePathString(),
"--profile-cpu-sample-period",
"1",
sourceFile.absolutePathString(),
)
.redirectErrorStream(true)
.start()
val output = process.inputStream.bufferedReader().readText()
val exitCode = process.waitFor()
assertThat(exitCode).withFailMessage { "process failed with output:\n$output" }.isZero()
// smoke test only
val profile = GZIPInputStream(outputFile.inputStream()).use { Profile.parseFrom(it) }
assertThat(profile.stringTableList.first()).isEmpty() // string_table[0] must be ""
assertThat(profile.sampleTypeList).isNotEmpty
assertThat(profile.functionList).isNotEmpty
}
@Test
fun `pkl eval`(@TempDir tempDir: Path) {
val pklCode =
"""
function fib(n) = if (n < 2) n else fib(n - 1) + fib(n - 2)
x = fib(30)
"""
.trimIndent()
runTest(tempDir, pklCode, arrayOf("eval"))
}
@Test
fun `pkl test`(@TempDir tempDir: Path) {
val pklCode =
"""
amends "pkl:test"
local function fib(n) = if (n < 2) n else fib(n - 1) + fib(n - 2)
facts {
["fib"] {
fib(30) > 0
}
}
"""
.trimIndent()
runTest(tempDir, pklCode, arrayOf("eval"))
}
@Test
fun `pkl run`(@TempDir tempDir: Path) {
val pklCode =
"""
amends "pkl:Command"
command {
name = "do-run"
}
local function fib(n) = if (n < 2) n else fib(n - 1) + fib(n - 2)
output {
text = fib(30).toString()
}
"""
.trimIndent()
runTest(tempDir, pklCode, arrayOf("run"))
}
}