Bind PackageServer to ephemeral port to avoid port conflicts (#227)

This is a comprehensive solution to the "flaky PackageServer tests" problem. It rules out port conflicts and imposes no limits on test parallelism. The same solution can be used for other test servers in the future. Major changes: - Turn `PackageServer` from a singleton into a class that is instantiated per test class or test method. - Start the server the first time its `port` property is read. Bind the server to an ephemeral port instead of port 12110. - For every test that uses `PackageServer`, pass the server port to `--test-port`, `HttpClient.Builder.setTestPort`, the `CliBaseOptions` or `ExecutorOptions` constructor, or the Gradle plugin's `testPort` property. Wire all of these to `RequestRewritingClient`'s `testPort` constructor parameter. - Enhance `RequestRewritingClient` to replace port 12110 with `testPort` in request URIs unless `testPort` is -1 (its default). - Introduce `ExecutorOptions.Builder`. This makes executor options more comfortable to create and allows to hide options such as `testPort`. - Deprecate the `ExecutorOptions` constructor to steer users towards the builder. - Get rid of `ExecutorOptions2`, which is no longer needed. - Clean up `EmbeddedExecutorTest` with the help of the builder.
2026-01-11 22:30:54 +01:00 · 2024-03-13 10:40:55 -07:00
parent 1e608b2aae
commit 014b3a8816
26 changed files with 756 additions and 581 deletions
--- a/pkl-cli/src/test/kotlin/org/pkl/cli/CliEvaluatorTest.kt
+++ b/pkl-cli/src/test/kotlin/org/pkl/cli/CliEvaluatorTest.kt
@@ -24,6 +24,7 @@ import java.time.Duration
 import kotlin.io.path.*
 import org.assertj.core.api.Assertions.assertThat
 import org.assertj.core.api.Assertions.assertThatCode
+import org.junit.jupiter.api.AfterAll
 import org.junit.jupiter.api.AfterEach
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertThrows
@@ -40,12 +41,22 @@ import org.pkl.core.util.IoUtils

 class CliEvaluatorTest {
  companion object {
-    const val defaultContents = """
-person {
-  name = "pigeon"
-  age = 20 + 10
-}
+    private val defaultContents =
+      """
+      person {
+        name = "pigeon"
+        age = 20 + 10
+      }
    """
+        .trimIndent()
+
+    private val packageServer = PackageServer()
+
+    @AfterAll
+    @JvmStatic
+    fun afterAll() {
+      packageServer.close()
+    }
  }

  // use manually constructed temp dir instead of @TempDir to work around
@@ -1117,7 +1128,6 @@ result = someLib.x

  @Test
  fun `setting noCache will skip writing to the cache dir`() {
-    PackageServer.ensureStarted()
    val moduleUri =
      writePklFile(
        "test.pkl",
@@ -1136,7 +1146,8 @@ result = someLib.x
          workingDir = tempDir,
          moduleCacheDir = tempDir,
          noCache = true,
-          caCertificates = listOf(FileTestUtils.selfSignedCertificate)
+          caCertificates = listOf(FileTestUtils.selfSignedCertificate),
+          testPort = packageServer.port
        ),
      )
    CliEvaluator(options, consoleWriter = buffer).run()
@@ -1185,10 +1196,10 @@ result = someLib.x

  @Test
  fun `gives decent error message if CLI doesn't have the required CA certificate`() {
-    PackageServer.ensureStarted()
    // provide SOME certs to prevent CliEvaluator from falling back to ~/.pkl/cacerts
    val builtInCerts = FileTestUtils.writePklBuiltInCertificates(tempDir)
-    val err = assertThrows<CliException> { evalModuleThatImportsPackage(builtInCerts) }
+    val err =
+      assertThrows<CliException> { evalModuleThatImportsPackage(builtInCerts, packageServer.port) }
    assertThat(err)
      // on some JDK11's this doesn't cause SSLHandshakeException but some other SSLException
      // .hasMessageContaining("Error during SSL handshake with host `localhost`:")
@@ -1196,7 +1207,7 @@ result = someLib.x
      .hasMessageNotContainingAny("java.", "sun.") // class names have been filtered out
  }

-  private fun evalModuleThatImportsPackage(certsFile: Path) {
+  private fun evalModuleThatImportsPackage(certsFile: Path, testPort: Int = -1) {
    val moduleUri =
      writePklFile(
        "test.pkl",
@@ -1212,7 +1223,9 @@ result = someLib.x
        CliBaseOptions(
          sourceModules = listOf(moduleUri),
          caCertificates = listOf(certsFile),
-          noCache = true
+          workingDir = tempDir,
+          noCache = true,
+          testPort = testPort
        ),
      )
    CliEvaluator(options).run()
--- a/pkl-cli/src/test/kotlin/org/pkl/cli/CliPackageDownloaderTest.kt
+++ b/pkl-cli/src/test/kotlin/org/pkl/cli/CliPackageDownloaderTest.kt
@@ -18,7 +18,7 @@ package org.pkl.cli
 import java.nio.file.Path
 import org.assertj.core.api.Assertions.assertThat
 import org.assertj.core.api.Assertions.assertThatCode
-import org.junit.jupiter.api.BeforeAll
+import org.junit.jupiter.api.AfterAll
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.io.TempDir
 import org.pkl.commons.cli.CliBaseOptions
@@ -28,10 +28,12 @@ import org.pkl.core.packages.PackageUri

 class CliPackageDownloaderTest {
  companion object {
-    @BeforeAll
+    val server = PackageServer()
+
+    @AfterAll
    @JvmStatic
-    fun beforeAll() {
-      PackageServer.ensureStarted()
+    fun afterAll() {
+      server.close()
    }
  }

@@ -42,7 +44,8 @@ class CliPackageDownloaderTest {
        baseOptions =
          CliBaseOptions(
            moduleCacheDir = tempDir,
-            caCertificates = listOf(FileTestUtils.selfSignedCertificate)
+            caCertificates = listOf(FileTestUtils.selfSignedCertificate),
+            testPort = server.port
          ),
        packageUris =
          listOf(
@@ -80,7 +83,8 @@ class CliPackageDownloaderTest {
        baseOptions =
          CliBaseOptions(
            workingDir = tempDir,
-            caCertificates = listOf(FileTestUtils.selfSignedCertificate)
+            caCertificates = listOf(FileTestUtils.selfSignedCertificate),
+            testPort = server.port
          ),
        packageUris = listOf(PackageUri("package://localhost:12110/birds@0.5.0")),
        noTransitive = true
@@ -99,7 +103,8 @@ class CliPackageDownloaderTest {
        baseOptions =
          CliBaseOptions(
            moduleCacheDir = tempDir,
-            caCertificates = listOf(FileTestUtils.selfSignedCertificate)
+            caCertificates = listOf(FileTestUtils.selfSignedCertificate),
+            testPort = server.port
          ),
        packageUris =
          listOf(
@@ -121,7 +126,8 @@ class CliPackageDownloaderTest {
        baseOptions =
          CliBaseOptions(
            moduleCacheDir = tempDir,
-            caCertificates = listOf(FileTestUtils.selfSignedCertificate)
+            caCertificates = listOf(FileTestUtils.selfSignedCertificate),
+            testPort = server.port
          ),
        packageUris =
          listOf(
@@ -161,7 +167,8 @@ class CliPackageDownloaderTest {
        baseOptions =
          CliBaseOptions(
            moduleCacheDir = tempDir,
-            caCertificates = listOf(FileTestUtils.selfSignedCertificate)
+            caCertificates = listOf(FileTestUtils.selfSignedCertificate),
+            testPort = server.port
          ),
        packageUris = listOf(PackageUri("package://localhost:12110/badChecksum@1.0.0")),
        noTransitive = true
@@ -179,7 +186,8 @@ class CliPackageDownloaderTest {
        baseOptions =
          CliBaseOptions(
            moduleCacheDir = tempDir,
-            caCertificates = listOf(FileTestUtils.selfSignedCertificate)
+            caCertificates = listOf(FileTestUtils.selfSignedCertificate),
+            testPort = server.port
          ),
        packageUris =
          listOf(
@@ -215,7 +223,8 @@ class CliPackageDownloaderTest {
        baseOptions =
          CliBaseOptions(
            moduleCacheDir = tempDir,
-            caCertificates = listOf(FileTestUtils.selfSignedCertificate)
+            caCertificates = listOf(FileTestUtils.selfSignedCertificate),
+            testPort = server.port
          ),
        packageUris = listOf(PackageUri("package://localhost:12110/birds@0.5.0")),
        noTransitive = false
--- a/pkl-cli/src/test/kotlin/org/pkl/cli/CliProjectPackagerTest.kt
+++ b/pkl-cli/src/test/kotlin/org/pkl/cli/CliProjectPackagerTest.kt
@@ -24,6 +24,7 @@ import java.util.stream.Collectors
 import kotlin.io.path.createDirectories
 import org.assertj.core.api.Assertions.assertThat
 import org.assertj.core.api.Assertions.assertThatCode
+import org.junit.jupiter.api.AfterAll
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertThrows
 import org.junit.jupiter.api.io.TempDir
@@ -36,6 +37,16 @@ import org.pkl.commons.test.PackageServer
 import org.pkl.commons.writeString

 class CliProjectPackagerTest {
+  companion object {
+    private val packageServer = PackageServer()
+
+    @AfterAll
+    @JvmStatic
+    fun afterAll() {
+      packageServer.close()
+    }
+  }
+
  @Test
  fun `missing PklProject when inferring a project dir`(@TempDir tempDir: Path) {
    val packager =
@@ -866,7 +877,6 @@ class CliProjectPackagerTest {

  @Test
  fun `publish checks`(@TempDir tempDir: Path) {
-    PackageServer.ensureStarted()
    tempDir.writeFile("project/main.pkl", "res = 1")
    tempDir.writeFile(
      "project/PklProject",
@@ -888,7 +898,8 @@ class CliProjectPackagerTest {
        CliProjectPackager(
            CliBaseOptions(
              workingDir = tempDir,
-              caCertificates = listOf(FileTestUtils.selfSignedCertificate)
+              caCertificates = listOf(FileTestUtils.selfSignedCertificate),
+              testPort = packageServer.port
            ),
            listOf(tempDir.resolve("project")),
            CliTestOptions(),
@@ -912,7 +923,6 @@ class CliProjectPackagerTest {

  @Test
  fun `publish check when package is not yet published`(@TempDir tempDir: Path) {
-    PackageServer.ensureStarted()
    tempDir.writeFile("project/main.pkl", "res = 1")
    tempDir.writeFile(
      "project/PklProject",
@@ -932,7 +942,8 @@ class CliProjectPackagerTest {
    CliProjectPackager(
        CliBaseOptions(
          workingDir = tempDir,
-          caCertificates = listOf(FileTestUtils.selfSignedCertificate)
+          caCertificates = listOf(FileTestUtils.selfSignedCertificate),
+          testPort = packageServer.port
        ),
        listOf(tempDir.resolve("project")),
        CliTestOptions(),
--- a/pkl-cli/src/test/kotlin/org/pkl/cli/CliProjectResolverTest.kt
+++ b/pkl-cli/src/test/kotlin/org/pkl/cli/CliProjectResolverTest.kt
@@ -18,7 +18,7 @@ package org.pkl.cli
 import java.io.StringWriter
 import java.nio.file.Path
 import org.assertj.core.api.Assertions.assertThat
-import org.junit.jupiter.api.BeforeAll
+import org.junit.jupiter.api.AfterAll
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertThrows
 import org.junit.jupiter.api.io.TempDir
@@ -29,10 +29,12 @@ import org.pkl.commons.test.PackageServer

 class CliProjectResolverTest {
  companion object {
-    @BeforeAll
+    private val packageServer = PackageServer()
+
+    @AfterAll
    @JvmStatic
-    fun beforeAll() {
-      PackageServer.ensureStarted()
+    fun afterAll() {
+      packageServer.close()
    }
  }

@@ -40,7 +42,7 @@ class CliProjectResolverTest {
  fun `missing PklProject when inferring a project dir`(@TempDir tempDir: Path) {
    val packager =
      CliProjectResolver(
-        CliBaseOptions(workingDir = tempDir),
+        CliBaseOptions(workingDir = tempDir, noCache = true),
        emptyList(),
        consoleWriter = StringWriter(),
        errWriter = StringWriter()
@@ -53,7 +55,7 @@ class CliProjectResolverTest {
  fun `missing PklProject when explicit dir is provided`(@TempDir tempDir: Path) {
    val packager =
      CliProjectResolver(
-        CliBaseOptions(),
+        CliBaseOptions(noCache = true),
        listOf(tempDir),
        consoleWriter = StringWriter(),
        errWriter = StringWriter()
@@ -80,7 +82,9 @@ class CliProjectResolverTest {
    CliProjectResolver(
        CliBaseOptions(
          workingDir = tempDir,
-          caCertificates = listOf(FileTestUtils.selfSignedCertificate)
+          caCertificates = listOf(FileTestUtils.selfSignedCertificate),
+          testPort = packageServer.port,
+          noCache = true
        ),
        listOf(tempDir),
        consoleWriter = StringWriter(),
@@ -133,7 +137,9 @@ class CliProjectResolverTest {
    CliProjectResolver(
        CliBaseOptions(
          workingDir = tempDir,
-          caCertificates = listOf(FileTestUtils.selfSignedCertificate)
+          caCertificates = listOf(FileTestUtils.selfSignedCertificate),
+          testPort = packageServer.port,
+          noCache = true
        ),
        emptyList(),
        consoleWriter = StringWriter(),
@@ -228,7 +234,11 @@ class CliProjectResolverTest {
        .trimIndent()
    )
    CliProjectResolver(
-        CliBaseOptions(caCertificates = listOf(FileTestUtils.selfSignedCertificate)),
+        CliBaseOptions(
+          caCertificates = listOf(FileTestUtils.selfSignedCertificate),
+          testPort = packageServer.port,
+          noCache = true
+        ),
        listOf(projectDir),
        consoleWriter = StringWriter(),
        errWriter = StringWriter()
@@ -306,7 +316,11 @@ class CliProjectResolverTest {
    val consoleOut = StringWriter()
    val errOut = StringWriter()
    CliProjectResolver(
-        CliBaseOptions(caCertificates = listOf(FileTestUtils.selfSignedCertificate)),
+        CliBaseOptions(
+          caCertificates = listOf(FileTestUtils.selfSignedCertificate),
+          testPort = packageServer.port,
+          noCache = true
+        ),
        listOf(projectDir),
        consoleWriter = consoleOut,
        errWriter = errOut
@@ -377,7 +391,11 @@ class CliProjectResolverTest {
    val consoleOut = StringWriter()
    val errOut = StringWriter()
    CliProjectResolver(
-        CliBaseOptions(caCertificates = listOf(FileTestUtils.selfSignedCertificate)),
+        CliBaseOptions(
+          caCertificates = listOf(FileTestUtils.selfSignedCertificate),
+          testPort = packageServer.port,
+          noCache = true
+        ),
        listOf(tempDir.resolve("project1"), tempDir.resolve("project2")),
        consoleWriter = consoleOut,
        errWriter = errOut
--- a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliBaseOptions.kt
+++ b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliBaseOptions.kt
@@ -113,6 +113,12 @@ data class CliBaseOptions(
  /** Tells whether to run the CLI in test mode. This is an internal option. */
  val testMode: Boolean = false,

+  /**
+   * Unless -1, rewrites HTTP requests that specify port 12110 to the given port. This is an
+   * internal test option.
+   */
+  val testPort: Int = -1,
+
  /**
   * The CA certificates to trust.
   *
@@ -179,6 +185,7 @@ data class CliBaseOptions(
   */
  val httpClient: HttpClient by lazy {
    with(HttpClient.builder()) {
+      setTestPort(testPort)
      if (normalizedCaCertificates.isEmpty()) {
        addDefaultCliCertificates()
      } else {
--- a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseOptions.kt
+++ b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseOptions.kt
@@ -17,6 +17,7 @@ package org.pkl.commons.cli.commands

 import com.github.ajalt.clikt.parameters.groups.OptionGroup
 import com.github.ajalt.clikt.parameters.options.*
+import com.github.ajalt.clikt.parameters.types.int
 import com.github.ajalt.clikt.parameters.types.long
 import com.github.ajalt.clikt.parameters.types.path
 import java.io.File
@@ -135,6 +136,13 @@ class BaseOptions : OptionGroup() {
      .path()
      .multiple()

+  // hidden option used by native tests
+  private val testPort: Int by
+    option(names = arrayOf("--test-port"), help = "Internal test option", hidden = true)
+      .single()
+      .int()
+      .default(-1)
+
  fun baseOptions(
    modules: List<URI>,
    projectOptions: ProjectOptions? = null,
@@ -155,6 +163,7 @@ class BaseOptions : OptionGroup() {
      moduleCacheDir = cacheDir ?: defaults.normalizedModuleCacheDir,
      noCache = noCache,
      testMode = testMode,
+      testPort = testPort,
      omitProjectSettings = projectOptions?.omitProjectSettings ?: false,
      noProject = projectOptions?.noProject ?: false,
      caCertificates = caCertificates
--- a/pkl-commons-test/src/main/kotlin/org/pkl/commons/test/PackageServer.kt
+++ b/pkl-commons-test/src/main/kotlin/org/pkl/commons/test/PackageServer.kt
@@ -15,11 +15,7 @@
 */
 package org.pkl.commons.test

-import com.sun.net.httpserver.HttpHandler
-import com.sun.net.httpserver.HttpsConfigurator
-import com.sun.net.httpserver.HttpsParameters
-import com.sun.net.httpserver.HttpsServer
-import java.net.BindException
+import com.sun.net.httpserver.*
 import java.net.InetSocketAddress
 import java.nio.file.*
 import java.security.KeyStore
@@ -30,61 +26,103 @@ import kotlin.io.path.isRegularFile
 import org.pkl.commons.createParentDirectories
 import org.pkl.commons.deleteRecursively

-object PackageServer {
-  private val keystore = javaClass.getResource("/localhost.p12")!!
-
-  // When tests are run via Gradle (i.e. from ./gradlew check), resources are packaged into a jar.
-  // When run directly in IntelliJ, resources are just directories.
-  private val packagesDir: Path = let {
-    val uri = javaClass.getResource("packages")!!.toURI()
-    try {
-      Path.of(uri)
-    } catch (e: FileSystemNotFoundException) {
-      FileSystems.newFileSystem(uri, mapOf<String, String>())
-      Path.of(uri)
+/**
+ * A test HTTP server that serves the Pkl packages defined under
+ * `pkl-commons-test/src/main/files/packages`.
+ *
+ * To use this server from a test,
+ * 1. Instantiate the server.
+ * 2. (optional) Store the server in a companion or instance field.
+ * 3. When setting up your test, pass the server [port] to one of the following:
+ *     * `HttpClient.Builder.setTestPort`
+ *     * `CliBaseOptions` constructor
+ *     * `ExecutorOptions` constructor
+ *     * `testPort` Gradle property
+ *
+ *   If the server isn't already running, it is automatically started.
+ * 4. Use port `12110` in your test. `HttpClient` will replace this port with the server port.
+ * 4. [Close][close] the server, for example in [AfterAll][org.junit.jupiter.api.AfterAll].
+ */
+class PackageServer : AutoCloseable {
+  companion object {
+    fun populateCacheDir(cacheDir: Path) {
+      val basePath = cacheDir.resolve("package-1/localhost:$PORT")
+      basePath.deleteRecursively()
+      Files.walk(packagesDir).use { stream ->
+        stream.forEach { source ->
+          if (!source.isRegularFile()) return@forEach
+          val relativized =
+            source.toString().replaceFirst(packagesDir.toString(), "").drop(1).ifEmpty {
+              return@forEach
+            }
+          val dest = basePath.resolve(relativized)
+          dest.createParentDirectories()
+          Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING)
+        }
+      }
    }
-  }

-  fun populateCacheDir(cacheDir: Path) {
-    val basePath = cacheDir.resolve("package-1/localhost:$PORT")
-    basePath.deleteRecursively()
-    Files.walk(packagesDir).use { stream ->
-      stream.forEach { source ->
-        if (!source.isRegularFile()) return@forEach
-        val relativized =
-          source.toString().replaceFirst(packagesDir.toString(), "").drop(1).ifEmpty {
-            return@forEach
-          }
-        val dest = basePath.resolve(relativized)
-        dest.createParentDirectories()
-        Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING)
+    // Port declared in tests.
+    // Modified by RequestRewritingClient if testPort is set.
+    private const val PORT = 12110
+
+    // When tests are run via Gradle (i.e. from ./gradlew check), resources are packaged into a jar.
+    // When run directly in IntelliJ, resources are just directories.
+    private val packagesDir: Path by lazy {
+      val uri = PackageServer::class.java.getResource("packages")!!.toURI()
+      try {
+        Path.of(uri)
+      } catch (e: FileSystemNotFoundException) {
+        FileSystems.newFileSystem(uri, mapOf<String, String>())
+        Path.of(uri)
+      }
+    }
+
+    private val simpleHttpsConfigurator by lazy {
+      val sslContext =
+        SSLContext.getInstance("SSL").apply {
+          val pass = "password".toCharArray()
+          val keystore = PackageServer::class.java.getResource("/localhost.p12")!!
+          val ks = KeyStore.getInstance("PKCS12").apply { load(keystore.openStream(), pass) }
+          val kmf = KeyManagerFactory.getInstance("SunX509").apply { init(ks, pass) }
+          init(kmf.keyManagers, null, null)
+        }
+      val engine = sslContext.createSSLEngine()
+      object : HttpsConfigurator(sslContext) {
+        override fun configure(params: HttpsParameters) {
+          params.needClientAuth = false
+          params.cipherSuites = engine.enabledCipherSuites
+          params.protocols = engine.enabledProtocols
+          params.setSSLParameters(sslContext.supportedSSLParameters)
+        }
      }
    }
  }

-  private const val PORT = 12110
-  private var started = false
-
-  private val sslContext by lazy {
-    SSLContext.getInstance("SSL").apply {
-      val pass = "password".toCharArray()
-      val ks = KeyStore.getInstance("PKCS12").apply { load(keystore.openStream(), pass) }
-      val kmf = KeyManagerFactory.getInstance("SunX509").apply { init(ks, pass) }
-      init(kmf.keyManagers, null, null)
+  /** The ephemeral listening port of this server. Automatically starts the server if necessary. */
+  val port: Int by lazy {
+    with(server.value) {
+      bind(InetSocketAddress(0), 0)
+      start()
+      address.port
    }
  }

-  private val engine by lazy { sslContext.createSSLEngine() }
-
-  private val simpleHttpsConfigurator =
-    object : HttpsConfigurator(sslContext) {
-      override fun configure(params: HttpsParameters) {
-        params.needClientAuth = false
-        params.cipherSuites = engine.enabledCipherSuites
-        params.protocols = engine.enabledProtocols
-        params.setSSLParameters(sslContext.supportedSSLParameters)
-      }
+  /** Closes this server. */
+  override fun close() {
+    // don't start server just to stop it
+    if (server.isInitialized()) {
+      server.value.stop(0)
    }
+  }
+
+  private val server: Lazy<HttpsServer> = lazy {
+    HttpsServer.create().apply {
+      httpsConfigurator = simpleHttpsConfigurator
+      createContext("/", handler)
+      executor = Executors.newFixedThreadPool(1)
+    }
+  }

  private val handler = HttpHandler { exchange ->
    if (exchange.requestMethod != "GET") {
@@ -105,41 +143,4 @@ object PackageServer {
    exchange.responseBody.use { outputStream -> Files.copy(localPath, outputStream) }
    exchange.close()
  }
-
-  private val myExecutor = Executors.newFixedThreadPool(1)
-
-  private val server by lazy {
-    HttpsServer.create().apply {
-      httpsConfigurator = simpleHttpsConfigurator
-      createContext("/", handler)
-      executor = myExecutor
-    }
-  }
-
-  fun ensureStarted() =
-    synchronized(this) {
-      if (!started) {
-        // Crude hack to make sure that parallel tests don't try and use each others mock server
-        // otherwise you get flaky tests when a server instance is shutdown by one set of tests
-        // while another set of tests is still relying on it.
-        // Side effect is that tests that spin up a mock package server are now serialised, rather
-        // than running in parallel. But that seems like a reasonable tradeoff to avoid flaky
-        // tests.
-        for (i in 1..20) {
-          try {
-            server.bind(InetSocketAddress(PORT), 0)
-            server.start()
-            started = true
-            println("Mock package server started after $i attempt(s)")
-            return@synchronized
-          } catch (_: BindException) {
-            println(
-              "Port $PORT in use after $i/20 attempt(s), probably another test running in parallel. Sleeping for 1 second and trying again"
-            )
-            Thread.sleep(1000)
-          }
-        }
-        println("Unable to start package server! This will probably result in a test failures")
-      }
-    }
 }
--- a/pkl-core/src/main/java/org/pkl/core/http/HttpClient.java
+++ b/pkl-core/src/main/java/org/pkl/core/http/HttpClient.java
@@ -108,6 +108,14 @@ public interface HttpClient extends AutoCloseable {
     */
    Builder addBuiltInCertificates();

+    /**
+     * Sets a test server's listening port.
+     *
+     * <p>If set, requests that specify port 12110 will be modified to use the given port. This is
+     * an internal test option.
+     */
+    Builder setTestPort(int port);
+
    /**
     * Creates a new {@code HttpClient} from the current state of this builder.
     *
--- a/pkl-core/src/main/java/org/pkl/core/http/HttpClientBuilder.java
+++ b/pkl-core/src/main/java/org/pkl/core/http/HttpClientBuilder.java
@@ -35,6 +35,7 @@ final class HttpClientBuilder implements HttpClient.Builder {
  private final Path caCertsDir;
  private final List<Path> certificateFiles = new ArrayList<>();
  private final List<URI> certificateUris = new ArrayList<>();
+  private int testPort = -1;

  HttpClientBuilder() {
    this(IoUtils.getPklHomeDir().resolve("cacerts"));
@@ -102,6 +103,12 @@ final class HttpClientBuilder implements HttpClient.Builder {
    return this;
  }

+  @Override
+  public HttpClient.Builder setTestPort(int port) {
+    testPort = port;
+    return this;
+  }
+
  @Override
  public HttpClient build() {
    return doBuild().get();
@@ -118,7 +125,7 @@ final class HttpClientBuilder implements HttpClient.Builder {
    var certificateUris = List.copyOf(this.certificateUris);
    return () -> {
      var jdkClient = new JdkHttpClient(certificateFiles, certificateUris, connectTimeout);
-      return new RequestRewritingClient(userAgent, requestTimeout, jdkClient);
+      return new RequestRewritingClient(userAgent, requestTimeout, testPort, jdkClient);
    };
  }

--- a/pkl-core/src/main/java/org/pkl/core/http/RequestRewritingClient.java
+++ b/pkl-core/src/main/java/org/pkl/core/http/RequestRewritingClient.java
@@ -16,12 +16,14 @@
 package org.pkl.core.http;

 import java.io.IOException;
+import java.net.URI;
 import java.net.http.HttpRequest;
 import java.net.http.HttpResponse;
 import java.net.http.HttpResponse.BodyHandler;
 import java.time.Duration;
 import java.util.concurrent.atomic.AtomicBoolean;
 import javax.annotation.concurrent.ThreadSafe;
+import org.pkl.core.util.HttpUtils;

 /**
 * An {@code HttpClient} decorator that
@@ -40,13 +42,16 @@ final class RequestRewritingClient implements HttpClient {
  // non-private for testing
  final String userAgent;
  final Duration requestTimeout;
+  final int testPort;
  final HttpClient delegate;

  private final AtomicBoolean closed = new AtomicBoolean();

-  RequestRewritingClient(String userAgent, Duration requestTimeout, HttpClient delegate) {
+  RequestRewritingClient(
+      String userAgent, Duration requestTimeout, int testPort, HttpClient delegate) {
    this.userAgent = userAgent;
    this.requestTimeout = requestTimeout;
+    this.testPort = testPort;
    this.delegate = delegate;
  }

@@ -67,7 +72,7 @@ final class RequestRewritingClient implements HttpClient {
    HttpRequest.Builder builder = HttpRequest.newBuilder();

    builder
-        .uri(original.uri())
+        .uri(rewriteUri(original.uri()))
        .expectContinue(original.expectContinue())
        .timeout(original.timeout().orElse(requestTimeout))
        .version(original.version().orElse(java.net.http.HttpClient.Version.HTTP_2));
@@ -99,6 +104,15 @@ final class RequestRewritingClient implements HttpClient {
    return builder.build();
  }

+  private URI rewriteUri(URI uri) {
+    // Would be nice to use port 0 instead of 12110,
+    // but this is best done in a separate commit.
+    if (testPort != -1 && uri.getPort() == 12110) {
+      return HttpUtils.setPort(uri, testPort);
+    }
+    return uri;
+  }
+
  private void checkNotClosed(HttpRequest request) {
    if (closed.get()) {
      throw new IllegalStateException(
--- a/pkl-core/src/main/java/org/pkl/core/service/ExecutorSpiImpl.java
+++ b/pkl-core/src/main/java/org/pkl/core/service/ExecutorSpiImpl.java
@@ -132,22 +132,26 @@ public class ExecutorSpiImpl implements ExecutorSpi {
  private HttpClient getOrCreateHttpClient(ExecutorSpiOptions options) {
    List<Path> certificateFiles;
    List<URI> certificateUris;
+    int testPort;
    try {
      if (options instanceof ExecutorSpiOptions2) {
        var options2 = (ExecutorSpiOptions2) options;
        certificateFiles = options2.getCertificateFiles();
        certificateUris = options2.getCertificateUris();
+        testPort = options2.getTestPort();
      } else {
        certificateFiles = List.of();
        certificateUris = List.of();
+        testPort = -1;
      }
      // host pkl-executor does not have class ExecutorOptions2 defined.
      // this will happen if the pkl-executor distribution is too old.
    } catch (NoClassDefFoundError e) {
      certificateFiles = List.of();
      certificateUris = List.of();
+      testPort = -1;
    }
-    var clientKey = new HttpClientKey(certificateFiles, certificateUris);
+    var clientKey = new HttpClientKey(certificateFiles, certificateUris, testPort);
    return httpClients.computeIfAbsent(
        clientKey,
        (key) -> {
@@ -158,6 +162,7 @@ public class ExecutorSpiImpl implements ExecutorSpi {
          for (var uri : key.certificateUris) {
            builder.addCertificates(uri);
          }
+          builder.setTestPort(key.testPort);
          // If the above didn't add any certificates,
          // builder will use the JVM's default SSL context.
          return builder.buildLazily();
@@ -167,11 +172,13 @@ public class ExecutorSpiImpl implements ExecutorSpi {
  private static final class HttpClientKey {
    final Set<Path> certificateFiles;
    final Set<URI> certificateUris;
+    final int testPort;

-    HttpClientKey(List<Path> certificateFiles, List<URI> certificateUris) {
+    HttpClientKey(List<Path> certificateFiles, List<URI> certificateUris, int testPort) {
      // also serve as defensive copies
      this.certificateFiles = Set.copyOf(certificateFiles);
      this.certificateUris = Set.copyOf(certificateUris);
+      this.testPort = testPort;
    }

    @Override
@@ -184,12 +191,13 @@ public class ExecutorSpiImpl implements ExecutorSpi {
      }
      HttpClientKey that = (HttpClientKey) obj;
      return certificateFiles.equals(that.certificateFiles)
-          && certificateUris.equals(that.certificateUris);
+          && certificateUris.equals(that.certificateUris)
+          && testPort == that.testPort;
    }

    @Override
    public int hashCode() {
-      return Objects.hash(certificateFiles, certificateUris);
+      return Objects.hash(certificateFiles, certificateUris, testPort);
    }
  }
 }
--- a/pkl-core/src/main/java/org/pkl/core/util/HttpUtils.java
+++ b/pkl-core/src/main/java/org/pkl/core/util/HttpUtils.java
@@ -17,8 +17,10 @@ package org.pkl.core.util;

 import java.io.IOException;
 import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.net.http.HttpResponse;
+import org.pkl.core.PklBugException;

 public final class HttpUtils {
  private HttpUtils() {}
@@ -47,4 +49,22 @@ public final class HttpUtils {
    throw new IOException(
        ErrorMessages.create("badHttpStatusCode", response.statusCode(), response.uri()));
  }
+
+  public static URI setPort(URI uri, int port) {
+    if (port < 0 || port > 65535) {
+      throw new IllegalArgumentException(String.valueOf(port));
+    }
+    try {
+      return new URI(
+          uri.getScheme(),
+          uri.getUserInfo(),
+          uri.getHost(),
+          port,
+          uri.getPath(),
+          uri.getQuery(),
+          uri.getFragment());
+    } catch (URISyntaxException e) {
+      throw PklBugException.unreachableCode(); // only port changed
+    }
+  }
 }
--- a/pkl-core/src/test/kotlin/org/pkl/core/StackFrameTransformersTest.kt
+++ b/pkl-core/src/test/kotlin/org/pkl/core/StackFrameTransformersTest.kt
@@ -4,25 +4,30 @@ import org.pkl.commons.test.PackageServer
 import org.assertj.core.api.Assertions.assertThat
 import org.junit.jupiter.api.Disabled
 import org.junit.jupiter.api.Test
+import org.pkl.core.http.HttpClient

 class StackFrameTransformersTest {
  // TODO figure out how to test this; right now this fails because there is no VM context.
  @Test
  @Disabled
  fun replacePackageUriWithSourceCodeUrl() {
-    PackageServer.ensureStarted()
-    EvaluatorBuilder.preconfigured().build().use {
-      val frame = StackFrame(
-        "package://localhost:12110/birds@0.5.0#/Bird.pkl",
-        null,
-        listOf(),
-        1,
-        1,
-        2,
-        2)
-      val transformed =
-        StackFrameTransformers.replacePackageUriWithSourceCodeUrl.apply(frame)
-      assertThat(transformed.moduleUri).isEqualTo("https://example.com/birds/v0.5.0/blob/Bird.pkl#L1-L2")
+    PackageServer().use { server ->
+      val httpClient = HttpClient.builder().setTestPort(server.port).build()
+      EvaluatorBuilder.preconfigured()
+        .setHttpClient(httpClient)
+        .build().use {
+          val frame = StackFrame(
+            "package://localhost:12110/birds@0.5.0#/Bird.pkl",
+            null,
+            listOf(),
+            1,
+            1,
+            2,
+            2)
+          val transformed =
+            StackFrameTransformers.replacePackageUriWithSourceCodeUrl.apply(frame)
+          assertThat(transformed.moduleUri).isEqualTo("https://example.com/birds/v0.5.0/blob/Bird.pkl#L1-L2")
+        }
    }
  }
 }
--- a/pkl-core/src/test/kotlin/org/pkl/core/http/RequestRewritingClientTest.kt
+++ b/pkl-core/src/test/kotlin/org/pkl/core/http/RequestRewritingClientTest.kt
@@ -6,14 +6,13 @@ import org.junit.jupiter.api.Test
 import java.net.URI
 import java.net.http.HttpRequest
 import java.net.http.HttpRequest.BodyPublishers
-import java.net.http.HttpResponse
 import java.net.http.HttpResponse.BodyHandlers
 import java.time.Duration
 import java.net.http.HttpClient as JdkHttpClient

 class RequestRewritingClientTest {
  private val captured = RequestCapturingClient()
-  private val client = RequestRewritingClient("Pkl", Duration.ofSeconds(42), captured)
+  private val client = RequestRewritingClient("Pkl", Duration.ofSeconds(42), -1, captured)
  private val exampleUri = URI("https://example.com/foo/bar.html")
  private val exampleRequest = HttpRequest.newBuilder(exampleUri).build()

@@ -103,6 +102,26 @@ class RequestRewritingClientTest {
    
    assertThat(captured.request.bodyPublisher().get()).isSameAs(publisher)
  }
+  
+  @Test
+  fun `rewrites port 12110 if test port is set`() {
+    val captured = RequestCapturingClient()
+    val client = RequestRewritingClient("Pkl", Duration.ofSeconds(42), 5000, captured)
+    val request = HttpRequest.newBuilder(URI("https://example.com:12110")).build()
+
+    client.send(request, BodyHandlers.discarding())
+
+    assertThat(captured.request.uri().port).isEqualTo(5000)
+  }
+  
+  @Test
+  fun `leaves port 12110 intact if no test port is set`() {
+    val request = HttpRequest.newBuilder(URI("https://example.com:12110")).build()
+
+    client.send(request, BodyHandlers.discarding())
+    
+    assertThat(captured.request.uri().port).isEqualTo(12110)
+  }
 }


--- a/pkl-core/src/test/kotlin/org/pkl/core/packages/PackageResolversTest.kt
+++ b/pkl-core/src/test/kotlin/org/pkl/core/packages/PackageResolversTest.kt
@@ -6,8 +6,6 @@ import org.junit.jupiter.api.AfterAll
 import org.junit.jupiter.api.BeforeAll
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertThrows
-import org.junit.jupiter.api.parallel.Execution
-import org.junit.jupiter.api.parallel.ExecutionMode
 import org.pkl.commons.deleteRecursively
 import org.pkl.commons.readString
 import org.pkl.commons.test.FileTestUtils
@@ -23,7 +21,6 @@ import kotlin.io.path.exists
 import kotlin.io.path.readBytes

 class PackageResolversTest {
-  @Execution(ExecutionMode.SAME_THREAD)
  abstract class AbstractPackageResolverTest {

    abstract val resolver: PackageResolver
@@ -31,15 +28,20 @@ class PackageResolversTest {
    private val packageRoot = FileTestUtils.rootProjectDir.resolve("pkl-commons-test/src/main/files/packages")

    companion object {
+      private val packageServer = PackageServer()
+      
      @JvmStatic
-      @BeforeAll
-      fun beforeAll() {
-        PackageServer.ensureStarted()
+      @AfterAll
+      fun afterAll() {
+        packageServer.close()
      }
      
-      val httpClient: HttpClient = HttpClient.builder()
-        .addCertificates(FileTestUtils.selfSignedCertificate)
-        .build()
+      val httpClient: HttpClient by lazy {
+        HttpClient.builder()
+          .addCertificates(FileTestUtils.selfSignedCertificate)
+          .setTestPort(packageServer.port)
+          .build()
+      }
    }

    @Test
@@ -199,7 +201,6 @@ class PackageResolversTest {
      @BeforeAll
      @JvmStatic
      fun beforeAll() {
-        PackageServer.ensureStarted()
        cacheDir.deleteRecursively()
      }
    }
--- a/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectDependenciesResolverTest.kt
+++ b/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectDependenciesResolverTest.kt
@@ -1,7 +1,7 @@
 package org.pkl.core.project

 import org.assertj.core.api.Assertions.assertThat
-import org.junit.jupiter.api.BeforeAll
+import org.junit.jupiter.api.AfterAll
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertThrows
 import org.pkl.commons.test.FileTestUtils
@@ -16,15 +16,20 @@ import java.nio.file.Path

 class ProjectDependenciesResolverTest {
  companion object {
+    private val packageServer = PackageServer()
+    
    @JvmStatic
-    @BeforeAll
-    fun beforeAll() {
-      PackageServer.ensureStarted()
+    @AfterAll
+    fun afterAll() {
+      packageServer.close()
    }
    
-    val httpClient: HttpClient = HttpClient.builder()
-      .addCertificates(FileTestUtils.selfSignedCertificate)
-      .build()
+    val httpClient: HttpClient by lazy {
+      HttpClient.builder()
+        .addCertificates(FileTestUtils.selfSignedCertificate)
+        .setTestPort(packageServer.port)
+        .build()
+    }
  }

  @Test
--- a/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectTest.kt
+++ b/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectTest.kt
@@ -136,19 +136,20 @@ class ProjectTest {

  @Test
  fun `evaluate project module -- invalid checksum`() {
-    PackageServer.ensureStarted()
-    val projectDir = Path.of(javaClass.getResource("badProjectChecksum2/")!!.path)
-    val project = Project.loadFromPath(projectDir.resolve("PklProject"))
-    val httpClient = HttpClient.builder()
-      .addCertificates(FileTestUtils.selfSignedCertificate)
-      .build()
-    val evaluator = EvaluatorBuilder.preconfigured()
-      .applyFromProject(project)
-      .setModuleCacheDir(null)
-      .setHttpClient(httpClient)
-      .build()
-    assertThatCode { evaluator.evaluate(ModuleSource.path(projectDir.resolve("bug.pkl"))) }
-      .hasMessageStartingWith("""
+    PackageServer().use { server ->
+      val projectDir = Path.of(javaClass.getResource("badProjectChecksum2/")!!.path)
+      val project = Project.loadFromPath(projectDir.resolve("PklProject"))
+      val httpClient = HttpClient.builder()
+        .addCertificates(FileTestUtils.selfSignedCertificate)
+        .setTestPort(server.port)
+        .build()
+      val evaluator = EvaluatorBuilder.preconfigured()
+        .applyFromProject(project)
+        .setModuleCacheDir(null)
+        .setHttpClient(httpClient)
+        .build()
+      assertThatCode { evaluator.evaluate(ModuleSource.path(projectDir.resolve("bug.pkl"))) }
+        .hasMessageStartingWith("""
        –– Pkl Error ––
        Cannot download package `package://localhost:12110/fruit@1.0.5` because the computed checksum for package metadata does not match the expected checksum.
        
@@ -159,5 +160,6 @@ class ProjectTest {
        1 | import "@fruit/Fruit.pkl"
            ^^^^^^^^^^^^^^^^^^^^^^^^^
      """.trimIndent())
+    }
  }
 }
--- a/pkl-core/src/test/kotlin/org/pkl/core/util/HttpUtilsTest.kt
+++ b/pkl-core/src/test/kotlin/org/pkl/core/util/HttpUtilsTest.kt
@@ -39,4 +39,22 @@ class HttpUtilsTest {
      HttpUtils.checkHasStatusCode200(response2)
    }
  }
+  
+  @Test
+  fun setPort() {
+    assertThrows<IllegalArgumentException> {
+      HttpUtils.setPort(URI("https://example.com"), -1)
+    }
+    assertThrows<IllegalArgumentException> {
+      HttpUtils.setPort(URI("https://example.com"), 65536)
+    }
+    assertThat(HttpUtils.setPort(URI("http://example.com"), 123))
+      .isEqualTo(URI("http://example.com:123"))
+    assertThat(HttpUtils.setPort(URI("http://example.com:456"), 123))
+      .isEqualTo(URI("http://example.com:123"))
+    assertThat(HttpUtils.setPort(URI("https://example.com/foo/bar.baz?query=1#fragment"), 123))
+      .isEqualTo(URI("https://example.com:123/foo/bar.baz?query=1#fragment"))
+    assertThat(HttpUtils.setPort(URI("https://example.com:456/foo/bar.baz?query=1#fragment"), 123))
+      .isEqualTo(URI("https://example.com:123/foo/bar.baz?query=1#fragment"))
+  }
 }
--- a/pkl-executor/src/main/java/org/pkl/executor/ExecutorOptions.java
+++ b/pkl-executor/src/main/java/org/pkl/executor/ExecutorOptions.java
@@ -15,47 +15,226 @@
 */
 package org.pkl.executor;

+import java.net.URI;
 import java.nio.file.Path;
 import java.time.Duration;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import org.pkl.executor.spi.v1.ExecutorSpiOptions;
+import org.pkl.executor.spi.v1.ExecutorSpiOptions2;

 /**
 * Options for {@link Executor#evaluatePath}.
 *
- * <p>Note that subclasses of {@code ExecutorOptions} offer additional options.
+ * <p>To create {@code ExecutorOptions}, use its {@linkplain #builder builder}.
 */
-public class ExecutorOptions {
-  protected final List<String> allowedModules;
+public final class ExecutorOptions {
+  private final List<String> allowedModules;

-  protected final List<String> allowedResources;
+  private final List<String> allowedResources;

-  protected final Map<String, String> environmentVariables;
+  private final Map<String, String> environmentVariables;

-  protected final Map<String, String> externalProperties;
+  private final Map<String, String> externalProperties;

-  protected final List<Path> modulePath;
+  private final List<Path> modulePath;

-  protected final /* @Nullable */ Path rootDir;
+  private final /* @Nullable */ Path rootDir;

-  protected final /* @Nullable */ Duration timeout;
+  private final /* @Nullable */ Duration timeout;

-  protected final /* @Nullable */ String outputFormat;
+  private final /* @Nullable */ String outputFormat;

-  protected final /* @Nullable */ Path moduleCacheDir;
+  private final /* @Nullable */ Path moduleCacheDir;

-  protected final /* @Nullable */ Path projectDir;
+  private final /* @Nullable */ Path projectDir;
+
+  private final List<Path> certificateFiles;
+
+  private final List<URI> certificateUris;
+
+  private final int testPort; // -1 means disabled
+
+  private final int spiOptionsVersion; // -1 means use latest

  /** Returns the module cache dir that the CLI uses by default. */
  public static Path defaultModuleCacheDir() {
    return Path.of(System.getProperty("user.home"), ".pkl", "cache");
  }

+  public static Builder builder() {
+    return new Builder();
+  }
+
+  /**
+   * A builder of {@link ExecutorOptions}.
+   *
+   * <p>It is safe to create multiple options objects with the same builder.
+   */
+  public static final class Builder {
+    private List<String> allowedModules = List.of();
+    private List<String> allowedResources = List.of();
+    private Map<String, String> environmentVariables = Map.of();
+    private Map<String, String> externalProperties = Map.of();
+    private List<Path> modulePath = List.of();
+    private /* @Nullable */ Path rootDir;
+    private /* @Nullable */ Duration timeout;
+    private /* @Nullable */ String outputFormat;
+    private /* @Nullable */ Path moduleCacheDir;
+    private /* @Nullable */ Path projectDir;
+    private List<Path> certificateFiles = List.of();
+    private List<URI> certificateUris = List.of();
+    private int testPort = -1; // -1 means disabled
+    private int spiOptionsVersion = -1; // -1 means use latest
+
+    private Builder() {}
+
+    /** API equivalent of the {@code --allowed-modules} CLI option. */
+    public Builder allowedModules(List<String> allowedModules) {
+      this.allowedModules = allowedModules;
+      return this;
+    }
+
+    /** API equivalent of the {@code --allowed-modules} CLI option. */
+    public Builder allowedModules(String... allowedModules) {
+      this.allowedModules = List.of(allowedModules);
+      return this;
+    }
+
+    /** API equivalent of the {@code --allowed-resources} CLI option. */
+    public Builder allowedResources(List<String> allowedResources) {
+      this.allowedResources = allowedResources;
+      return this;
+    }
+
+    /** API equivalent of the {@code --allowed-resources} CLI option. */
+    public Builder allowedResources(String... allowedResources) {
+      this.allowedResources = List.of(allowedResources);
+      return this;
+    }
+
+    /** API equivalent of the repeatable {@code --env-var} CLI option. */
+    public Builder environmentVariables(Map<String, String> environmentVariables) {
+      this.environmentVariables = environmentVariables;
+      return this;
+    }
+
+    /** API equivalent of the repeatable {@code --property} CLI option. */
+    public Builder externalProperties(Map<String, String> externalProperties) {
+      this.externalProperties = externalProperties;
+      return this;
+    }
+
+    /** API equivalent of the {@code --module-path} CLI option. */
+    public Builder modulePath(List<Path> modulePath) {
+      this.modulePath = modulePath;
+      return this;
+    }
+
+    /** API equivalent of the {@code --module-path} CLI option. */
+    public Builder modulePath(Path... modulePath) {
+      this.modulePath = List.of(modulePath);
+      return this;
+    }
+
+    /** API equivalent of the {@code --root-dir} CLI option. */
+    public Builder rootDir(/*Nullable*/ Path rootDir) {
+      this.rootDir = rootDir;
+      return this;
+    }
+
+    /** API equivalent of the {@code --timeout} CLI option. */
+    public Builder timeout(/*Nullable*/ Duration timeout) {
+      this.timeout = timeout;
+      return this;
+    }
+
+    /** API equivalent of the {@code --format} CLI option. */
+    public Builder outputFormat(/*Nullable*/ String outputFormat) {
+      this.outputFormat = outputFormat;
+      return this;
+    }
+
+    /**
+     * API equivalent of the {@code --cache-dir} CLI option. Passing {@code null} is equivalent to
+     * {@code --no-cache}.
+     */
+    public Builder moduleCacheDir(/*Nullable*/ Path moduleCacheDir) {
+      this.moduleCacheDir = moduleCacheDir;
+      return this;
+    }
+
+    /**
+     * API equivalent of the {@code --project-dir} CLI option.
+     *
+     * <p>Unlike the CLI, this option only sets project dependencies. It does not set evaluator
+     * settings.
+     */
+    public Builder projectDir(/*Nullable*/ Path projectDir) {
+      this.projectDir = projectDir;
+      return this;
+    }
+
+    /** API equivalent of the {@code --ca-certificates} CLI option. */
+    public Builder certificateFiles(List<Path> certificateFiles) {
+      this.certificateFiles = certificateFiles;
+      return this;
+    }
+
+    /** API equivalent of the {@code --ca-certificates} CLI option. */
+    public Builder certificateFiles(Path... certificateFiles) {
+      this.certificateFiles = List.of(certificateFiles);
+      return this;
+    }
+
+    /** API equivalent of the {@code --ca-certificates} CLI option. */
+    public Builder certificateUris(List<URI> certificateUris) {
+      this.certificateUris = certificateUris;
+      return this;
+    }
+
+    /** API equivalent of the {@code --ca-certificates} CLI option. */
+    public Builder certificateUris(URI... certificateUris) {
+      this.certificateUris = List.of(certificateUris);
+      return this;
+    }
+
+    /** Internal test option. -1 means disabled. */
+    Builder testPort(int testPort) {
+      this.testPort = testPort;
+      return this;
+    }
+
+    /** Internal test option. -1 means use latest. */
+    Builder spiOptionsVersion(int version) {
+      this.spiOptionsVersion = version;
+      return this;
+    }
+
+    public ExecutorOptions build() {
+      return new ExecutorOptions(
+          allowedModules,
+          allowedResources,
+          environmentVariables,
+          externalProperties,
+          modulePath,
+          rootDir,
+          timeout,
+          outputFormat,
+          moduleCacheDir,
+          projectDir,
+          certificateFiles,
+          certificateUris,
+          testPort,
+          spiOptionsVersion);
+    }
+  }
+
  /**
   * Constructs an options object.
   *
+   * @deprecated use {@link #builder} instead
   * @param allowedModules API equivalent of the {@code --allowed-modules} CLI option
   * @param allowedResources API equivalent of the {@code --allowed-resources} CLI option
   * @param environmentVariables API equivalent of the repeatable {@code --env-var} CLI option
@@ -69,6 +248,7 @@ public class ExecutorOptions {
   *     null} is equivalent to {@code --no-cache}.
   * @param projectDir API equivalent of the {@code --project-dir} CLI option.
   */
+  @Deprecated(forRemoval = true)
  public ExecutorOptions(
      List<String> allowedModules,
      List<String> allowedResources,
@@ -81,16 +261,53 @@ public class ExecutorOptions {
      /* @Nullable */ Path moduleCacheDir,
      /* @Nullable */ Path projectDir) {

-    this.allowedModules = allowedModules;
-    this.allowedResources = allowedResources;
-    this.environmentVariables = environmentVariables;
-    this.externalProperties = externalProperties;
+    this(
+        allowedModules,
+        allowedResources,
+        environmentVariables,
+        externalProperties,
+        modulePath,
+        rootDir,
+        timeout,
+        outputFormat,
+        moduleCacheDir,
+        projectDir,
+        List.of(),
+        List.of(),
+        -1,
+        -1);
+  }
+
+  private ExecutorOptions(
+      List<String> allowedModules,
+      List<String> allowedResources,
+      Map<String, String> environmentVariables,
+      Map<String, String> externalProperties,
+      List<Path> modulePath,
+      /* @Nullable */ Path rootDir,
+      /* @Nullable */ Duration timeout,
+      /* @Nullable */ String outputFormat,
+      /* @Nullable */ Path moduleCacheDir,
+      /* @Nullable */ Path projectDir,
+      List<Path> certificateFiles,
+      List<URI> certificateUris,
+      int testPort,
+      int spiOptionsVersion) {
+
+    this.allowedModules = List.copyOf(allowedModules);
+    this.allowedResources = List.copyOf(allowedResources);
+    this.environmentVariables = Map.copyOf(environmentVariables);
+    this.externalProperties = Map.copyOf(externalProperties);
    this.modulePath = modulePath;
    this.rootDir = rootDir;
    this.timeout = timeout;
    this.outputFormat = outputFormat;
    this.moduleCacheDir = moduleCacheDir;
    this.projectDir = projectDir;
+    this.certificateFiles = List.copyOf(certificateFiles);
+    this.certificateUris = List.copyOf(certificateUris);
+    this.testPort = testPort;
+    this.spiOptionsVersion = spiOptionsVersion;
  }

  /** API equivalent of the {@code --allowed-modules} CLI option. */
@@ -151,6 +368,16 @@ public class ExecutorOptions {
    return projectDir;
  }

+  /** API equivalent of the {@code --ca-certificates} CLI option. */
+  public List<Path> getCertificateFiles() {
+    return certificateFiles;
+  }
+
+  /** API equivalent of the {@code --ca-certificates} CLI option. */
+  public List<URI> getCertificateUris() {
+    return certificateUris;
+  }
+
  @Override
  public boolean equals(/* @Nullable */ Object obj) {
    if (this == obj) return true;
@@ -166,7 +393,11 @@ public class ExecutorOptions {
        && Objects.equals(timeout, other.timeout)
        && Objects.equals(outputFormat, other.outputFormat)
        && Objects.equals(moduleCacheDir, other.moduleCacheDir)
-        && Objects.equals(projectDir, other.projectDir);
+        && Objects.equals(projectDir, other.projectDir)
+        && Objects.equals(certificateFiles, other.certificateFiles)
+        && Objects.equals(certificateUris, other.certificateUris)
+        && testPort == other.testPort
+        && spiOptionsVersion == other.spiOptionsVersion;
  }

  @Override
@@ -181,7 +412,11 @@ public class ExecutorOptions {
        timeout,
        outputFormat,
        moduleCacheDir,
-        projectDir);
+        projectDir,
+        certificateFiles,
+        certificateUris,
+        testPort,
+        spiOptionsVersion);
  }

  @Override
@@ -207,20 +442,49 @@ public class ExecutorOptions {
        + moduleCacheDir
        + ", projectDir="
        + projectDir
+        + ", certificateFiles="
+        + certificateFiles
+        + ", certificateUris="
+        + certificateUris
+        + ", testPort="
+        + testPort
+        + ", spiOptionsVersion="
+        + spiOptionsVersion
        + '}';
  }

  ExecutorSpiOptions toSpiOptions() {
-    return new ExecutorSpiOptions(
-        allowedModules,
-        allowedResources,
-        environmentVariables,
-        externalProperties,
-        modulePath,
-        rootDir,
-        timeout,
-        outputFormat,
-        moduleCacheDir,
-        projectDir);
+    switch (spiOptionsVersion) {
+      case -1:
+      case 2:
+        return new ExecutorSpiOptions2(
+            allowedModules,
+            allowedResources,
+            environmentVariables,
+            externalProperties,
+            modulePath,
+            rootDir,
+            timeout,
+            outputFormat,
+            moduleCacheDir,
+            projectDir,
+            certificateFiles,
+            certificateUris,
+            testPort);
+      case 1: // for testing only
+        return new ExecutorSpiOptions(
+            allowedModules,
+            allowedResources,
+            environmentVariables,
+            externalProperties,
+            modulePath,
+            rootDir,
+            timeout,
+            outputFormat,
+            moduleCacheDir,
+            projectDir);
+      default:
+        throw new AssertionError("Unknown ExecutorSpiOptions version: " + spiOptionsVersion);
+    }
  }
 }
--- a/pkl-executor/src/main/java/org/pkl/executor/ExecutorOptions2.java
+++ b/pkl-executor/src/main/java/org/pkl/executor/ExecutorOptions2.java
@@ -1,174 +0,0 @@
-/**
- * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.pkl.executor;
-
-import java.net.URI;
-import java.nio.file.Path;
-import java.time.Duration;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import org.pkl.executor.spi.v1.ExecutorSpiOptions2;
-
-/**
- * Options for {@link Executor#evaluatePath}.
- *
- * <p>This class offers additional options not available in {@code ExecutorOptions}.
- */
-public class ExecutorOptions2 extends ExecutorOptions {
-  protected final List<Path> certificateFiles;
-  protected final List<URI> certificateUris;
-
-  /**
-   * Constructs an options object.
-   *
-   * @param allowedModules API equivalent of the {@code --allowed-modules} CLI option
-   * @param allowedResources API equivalent of the {@code --allowed-resources} CLI option
-   * @param environmentVariables API equivalent of the repeatable {@code --env-var} CLI option
-   * @param externalProperties API equivalent of the repeatable {@code --property} CLI option
-   * @param modulePath API equivalent of the {@code --module-path} CLI option
-   * @param rootDir API equivalent of the {@code --root-dir} CLI option
-   * @param timeout API equivalent of the {@code --timeout} CLI option
-   * @param outputFormat API equivalent of the {@code --format} CLI option
-   * @param moduleCacheDir API equivalent of the {@code --cache-dir} CLI option. Passing {@link
-   *     #defaultModuleCacheDir()} is equivalent to omitting {@code --cache-dir}. Passing {@code
-   *     null} is equivalent to {@code --no-cache}.
-   * @param projectDir API equivalent of the {@code --project-dir} CLI option.
-   * @param certificateFiles API equivalent of the {@code --ca-certificates} CLI option
-   * @param certificateUris API equivalent of the {@code --ca-certificates} CLI option
-   */
-  public ExecutorOptions2(
-      List<String> allowedModules,
-      List<String> allowedResources,
-      Map<String, String> environmentVariables,
-      Map<String, String> externalProperties,
-      List<Path> modulePath,
-      /* @Nullable */ Path rootDir,
-      /* @Nullable */ Duration timeout,
-      /* @Nullable */ String outputFormat,
-      /* @Nullable */ Path moduleCacheDir,
-      /* @Nullable */ Path projectDir,
-      List<Path> certificateFiles,
-      List<URI> certificateUris) {
-
-    super(
-        allowedModules,
-        allowedResources,
-        environmentVariables,
-        externalProperties,
-        modulePath,
-        rootDir,
-        timeout,
-        outputFormat,
-        moduleCacheDir,
-        projectDir);
-    this.certificateFiles = certificateFiles;
-    this.certificateUris = certificateUris;
-  }
-
-  /** API equivalent of the {@code --ca-certificates} CLI option. */
-  public List<Path> getCertificateFiles() {
-    return certificateFiles;
-  }
-
-  /** API equivalent of the {@code --ca-certificates} CLI option. */
-  public List<URI> getCertificateUris() {
-    return certificateUris;
-  }
-
-  @Override
-  public boolean equals(/* @Nullable */ Object obj) {
-    if (this == obj) return true;
-    if (obj.getClass() != ExecutorOptions2.class) return false;
-
-    var other = (ExecutorOptions2) obj;
-    return allowedModules.equals(other.allowedModules)
-        && allowedResources.equals(other.allowedResources)
-        && environmentVariables.equals(other.environmentVariables)
-        && externalProperties.equals(other.externalProperties)
-        && modulePath.equals(other.modulePath)
-        && Objects.equals(rootDir, other.rootDir)
-        && Objects.equals(timeout, other.timeout)
-        && Objects.equals(outputFormat, other.outputFormat)
-        && Objects.equals(moduleCacheDir, other.moduleCacheDir)
-        && Objects.equals(projectDir, other.projectDir)
-        && Objects.equals(certificateFiles, other.certificateFiles)
-        && Objects.equals(certificateUris, other.certificateUris);
-  }
-
-  @Override
-  public int hashCode() {
-    return Objects.hash(
-        allowedModules,
-        allowedResources,
-        environmentVariables,
-        externalProperties,
-        modulePath,
-        rootDir,
-        timeout,
-        outputFormat,
-        moduleCacheDir,
-        projectDir,
-        certificateFiles,
-        certificateUris);
-  }
-
-  @Override
-  public String toString() {
-    return "ExecutorOptions2{"
-        + "allowedModules="
-        + allowedModules
-        + ", allowedResources="
-        + allowedResources
-        + ", environmentVariables="
-        + environmentVariables
-        + ", externalProperties="
-        + externalProperties
-        + ", modulePath="
-        + modulePath
-        + ", rootDir="
-        + rootDir
-        + ", timeout="
-        + timeout
-        + ", outputFormat="
-        + outputFormat
-        + ", cacheDir="
-        + moduleCacheDir
-        + ", projectDir="
-        + projectDir
-        + ", certificateFiles="
-        + certificateFiles
-        + ", certificateUris="
-        + certificateUris
-        + '}';
-  }
-
-  ExecutorSpiOptions2 toSpiOptions() {
-    return new ExecutorSpiOptions2(
-        allowedModules,
-        allowedResources,
-        environmentVariables,
-        externalProperties,
-        modulePath,
-        rootDir,
-        timeout,
-        outputFormat,
-        moduleCacheDir,
-        projectDir,
-        certificateFiles,
-        certificateUris);
-  }
-}
--- a/pkl-executor/src/main/java/org/pkl/executor/spi/v1/ExecutorSpiOptions2.java
+++ b/pkl-executor/src/main/java/org/pkl/executor/spi/v1/ExecutorSpiOptions2.java
@@ -26,6 +26,8 @@ public class ExecutorSpiOptions2 extends ExecutorSpiOptions {

  private final List<URI> certificateUris;

+  private final int testPort;
+
  public ExecutorSpiOptions2(
      List<String> allowedModules,
      List<String> allowedResources,
@@ -38,7 +40,8 @@ public class ExecutorSpiOptions2 extends ExecutorSpiOptions {
      Path moduleCacheDir,
      Path projectDir,
      List<Path> certificateFiles,
-      List<URI> certificateUris) {
+      List<URI> certificateUris,
+      int testPort) {
    super(
        allowedModules,
        allowedResources,
@@ -52,6 +55,7 @@ public class ExecutorSpiOptions2 extends ExecutorSpiOptions {
        projectDir);
    this.certificateFiles = certificateFiles;
    this.certificateUris = certificateUris;
+    this.testPort = testPort;
  }

  public List<Path> getCertificateFiles() {
@@ -61,4 +65,8 @@ public class ExecutorSpiOptions2 extends ExecutorSpiOptions {
  public List<URI> getCertificateUris() {
    return certificateUris;
  }
+
+  public int getTestPort() {
+    return testPort;
+  }
 }
--- a/pkl-executor/src/test/kotlin/org/pkl/executor/EmbeddedExecutorTest.kt
+++ b/pkl-executor/src/test/kotlin/org/pkl/executor/EmbeddedExecutorTest.kt
@@ -20,36 +20,44 @@ import kotlin.io.path.exists

 class EmbeddedExecutorTest {
  /**
-   * A combination of ExecutorOptions version, pkl-executor version, 
-   * and Pkl distribution version that parameterized tests should be run against.
+   * An executor that uses a particular combination of ExecutorSpiOptions version, 
+   * pkl-executor version, and Pkl distribution version.
   */
-  data class ExecutionContext(
-    val executor: Executor,
-    val options: (ExecutorOptions) -> ExecutorOptions,
-    val name: String
+  class TestExecutor(
+    private val executor: Executor,
+    private val spiOptionsVersion: Int,
+    private val name: String
  ) {
+    fun evaluatePath(modulePath: Path, optionSpec: ExecutorOptions.Builder.() -> Unit): String {
+      val options = ExecutorOptions.builder()
+        .apply(optionSpec)
+        .spiOptionsVersion(spiOptionsVersion)
+        .build()
+      return executor.evaluatePath(modulePath, options)
+    }
    override fun toString(): String = name
  }
  
  companion object {
    @JvmStatic
-    private val allExecutionContexts: List<ExecutionContext> by lazy {
+    private val allTestExecutors: List<TestExecutor> by lazy {
      listOf(
-        ExecutionContext(executor1_1.value, ::convertToOptions1, "Options1, Executor1, Distribution1"),
+        TestExecutor(executor1_1.value, 1, "SpiOptions1, Executor1, Distribution1"),

        // This context has a pkl-executor version that is lower than the distribution version.
-        // It can be enabled once there is a distribution that includes pkl-executor.
-        ExecutionContext(executor1_2.value, ::convertToOptions1, "Options1, Executor1, Distribution2"),
+        TestExecutor(executor1_2.value, 1, "SpiOptions1, Executor1, Distribution2"),

-        ExecutionContext(executor2_1.value, ::convertToOptions1, "Options1, Executor2, Distribution1"),
-        ExecutionContext(executor2_1.value, ::convertToOptions2, "Options2, Executor2, Distribution1"),
+        TestExecutor(executor2_1.value, 1, "SpiOptions1, Executor2, Distribution1"),
+        TestExecutor(executor2_1.value, 2, "SpiOptions2, Executor2, Distribution1"),

-        ExecutionContext(executor2_2.value, ::convertToOptions1, "Options1, Executor2, Distribution2"),
-        ExecutionContext(executor2_2.value, ::convertToOptions2, "Options2, Executor2, Distribution2")
+        TestExecutor(executor2_2.value, 1, "SpiOptions1, Executor2, Distribution2"),
+        TestExecutor(executor2_2.value, 2, "SpiOptions2, Executor2, Distribution2")
      )
    }

-    private val currentExecutor: Executor by lazy { executor2_2.value }
+    private val currentExecutor: TestExecutor by lazy { 
+      TestExecutor(executor2_2.value, -1, "currentExecutor") 
+    }
    
    // A pkl-executor library that supports ExecutorSpiOptions up to v1
    // and a Pkl distribution that supports ExecutorSpiOptions up to v1.
@@ -101,11 +109,9 @@ class EmbeddedExecutorTest {

    // a Pkl distribution that supports ExecutorSpiOptions up to v1
    private val pklDistribution1: Path by lazy {
-      FileTestUtils.rootProjectDir.resolve("pkl-executor/build/pklHistoricalDistributions/pkl-config-java-all-0.25.0.jar").apply {
-        if (!exists()) {
-          throw AssertionError("Missing test fixture. " +
-            "To fix this problem, run `./gradlew :pkl-executor:prepareTest`.")
-        }
+      FileTestUtils.rootProjectDir
+        .resolve("pkl-executor/build/pklHistoricalDistributions/pkl-config-java-all-0.25.0.jar").apply {
+        if (!exists()) missingTestFixture()
      }
    }

@@ -114,40 +120,13 @@ class EmbeddedExecutorTest {
      FileTestUtils.rootProjectDir
        .resolve("pkl-config-java/build/libs/pkl-config-java-all-" +
          "${Release.current().version().withBuild(null).toString().replaceFirst("dev", "SNAPSHOT")}.jar").apply {
-        if (!exists()) throw AssertionError("Missing test fixture. " +
-          "To fix this problem, run `./gradlew :pkl-executor:prepareTest`.")
+        if (!exists()) missingTestFixture()
      }
    }
-
-    private fun convertToOptions2(options: ExecutorOptions): ExecutorOptions2 =
-      if (options is ExecutorOptions2) options else ExecutorOptions2(
-        options.allowedModules,
-        options.allowedResources,
-        options.environmentVariables,
-        options.externalProperties,
-        options.modulePath,
-        options.rootDir,
-        options.timeout,
-        options.outputFormat,
-        options.moduleCacheDir,
-        options.projectDir,
-        listOf(),
-        listOf()
-      )
-
-    private fun convertToOptions1(options: ExecutorOptions): ExecutorOptions =
-      if (options.javaClass == ExecutorOptions::class.java) options else ExecutorOptions(
-        options.allowedModules,
-        options.allowedResources,
-        options.environmentVariables,
-        options.externalProperties,
-        options.modulePath,
-        options.rootDir,
-        options.timeout,
-        options.outputFormat,
-        options.moduleCacheDir,
-        options.projectDir
-      )
+    
+    private fun missingTestFixture(): Nothing =
+      throw AssertionError("Missing test fixture. " +
+        "To fix this problem, run `./gradlew :pkl-executor:prepareTest`.")
  }

  @Test
@@ -233,8 +212,8 @@ class EmbeddedExecutorTest {
  }

  @ParameterizedTest
-  @MethodSource("getAllExecutionContexts")
-  fun `evaluate a module that is missing a ModuleInfo annotation`(context: ExecutionContext, @TempDir tempDir: Path) {
+  @MethodSource("getAllTestExecutors")
+  fun `evaluate a module that is missing a ModuleInfo annotation`(executor: TestExecutor, @TempDir tempDir: Path) {
    val pklFile = tempDir.resolve("test.pkl")
    pklFile.toFile().writeText(
      """
@@ -245,23 +224,11 @@ class EmbeddedExecutorTest {
    )

    val e = assertThrows<ExecutorException> {
-      context.executor.evaluatePath(
-          pklFile,
-          context.options(ExecutorOptions2(
-            listOf("file:"),
-            listOf("prop:"),
-            mapOf(),
-            mapOf(),
-            listOf(),
-            tempDir,
-            null,
-            null,
-            null,
-            null,
-            listOf(),
-            listOf()
-          )
-        ))
+      executor.evaluatePath(pklFile) {
+        allowedModules("file:")
+        allowedResources("prop:")
+        rootDir(tempDir)
+      }
    }

    assertThat(e.message)
@@ -269,8 +236,8 @@ class EmbeddedExecutorTest {
  }

  @ParameterizedTest
-  @MethodSource("getAllExecutionContexts")
-  fun `evaluate a module that requests an incompatible Pkl version`(context: ExecutionContext, @TempDir tempDir: Path) {
+  @MethodSource("getAllTestExecutors")
+  fun `evaluate a module that requests an incompatible Pkl version`(executor: TestExecutor, @TempDir tempDir: Path) {
    val pklFile = tempDir.resolve("test.pkl")
    pklFile.toFile().writeText(
      """
@@ -282,23 +249,11 @@ class EmbeddedExecutorTest {
    )

    val e = assertThrows<ExecutorException> {
-      context.executor.evaluatePath(
-          pklFile,
-          context.options(ExecutorOptions2(
-            listOf("file:"),
-            listOf("prop:"),
-            mapOf(),
-            mapOf(),
-            listOf(),
-            tempDir,
-            null,
-            null,
-            null,
-            null,
-            listOf(),
-            listOf()
-          ))
-        )
+      executor.evaluatePath(pklFile) {
+        allowedModules("file:")
+        allowedResources("prop:")
+        rootDir(tempDir)
+      }
    }

    assertThat(e.message)
@@ -306,8 +261,8 @@ class EmbeddedExecutorTest {
  }

  @ParameterizedTest
-  @MethodSource("getAllExecutionContexts")
-  fun `evaluate a module that reads environment variables and external properties`(context: ExecutionContext, @TempDir tempDir: Path) {
+  @MethodSource("getAllTestExecutors")
+  fun `evaluate a module that reads environment variables and external properties`(executor: TestExecutor, @TempDir tempDir: Path) {
    val pklFile = tempDir.resolve("test.pkl")
    pklFile.toFile().writeText(
      """
@@ -319,24 +274,13 @@ class EmbeddedExecutorTest {
    """.trimIndent()
    )

-    val result = context.executor.evaluatePath(
-        pklFile,
-        context.options(ExecutorOptions2(
-          listOf("file:"),
-          // should `prop:pkl.outputFormat` be allowed automatically?
-          listOf("prop:", "env:"),
-          mapOf("ENV_VAR" to "ENV_VAR"),
-          mapOf("property" to "property"),
-          listOf(),
-          null,
-          null,
-          null,
-          null,
-          null,
-          listOf(),
-          listOf()
-        ))
-      )
+    val result = executor.evaluatePath(pklFile) {
+      allowedModules("file:")
+      // should `prop:pkl.outputFormat` be allowed automatically?
+      allowedResources("prop:", "env:")
+      environmentVariables(mapOf("ENV_VAR" to "ENV_VAR"))
+      externalProperties(mapOf("property" to "property"))
+    }

    assertThat(result.trim()).isEqualTo(
      """
@@ -347,8 +291,8 @@ class EmbeddedExecutorTest {
  }

  @ParameterizedTest
-  @MethodSource("getAllExecutionContexts")
-  fun `evaluate a module that depends on another module`(context: ExecutionContext, @TempDir tempDir: Path) {
+  @MethodSource("getAllTestExecutors")
+  fun `evaluate a module that depends on another module`(executor: TestExecutor, @TempDir tempDir: Path) {
    val pklFile = tempDir.resolve("test.pkl")
    pklFile.toFile().writeText(
      """
@@ -372,24 +316,10 @@ class EmbeddedExecutorTest {
    """.trimIndent()
    )

-    val result = context.executor.evaluatePath(
-        pklFile,
-        context.options(ExecutorOptions2(
-          listOf("file:"),
-          listOf("prop:"),
-          mapOf(),
-          mapOf(),
-          listOf(),
-          null,
-          null,
-          null,
-          null,
-          null,
-          listOf(),
-          listOf()
-        )
-      )
-    )
+    val result = executor.evaluatePath(pklFile) {
+      allowedModules("file:")
+      allowedResources("prop:")
+    }

    assertThat(result.trim()).isEqualTo(
      """
@@ -401,8 +331,8 @@ class EmbeddedExecutorTest {
  }

  @ParameterizedTest
-  @MethodSource("getAllExecutionContexts")
-  fun `evaluate a module whose evaluation fails`(context: ExecutionContext, @TempDir tempDir: Path) {
+  @MethodSource("getAllTestExecutors")
+  fun `evaluate a module whose evaluation fails`(executor: TestExecutor, @TempDir tempDir: Path) {
    val pklFile = tempDir.resolve("test.pkl")
    pklFile.toFile().writeText(
      """
@@ -414,23 +344,11 @@ class EmbeddedExecutorTest {
    )

    val e = assertThrows<ExecutorException> {
-      context.executor.evaluatePath(
-          pklFile,
-          context.options(ExecutorOptions2(
-            listOf("file:"),
-            listOf("prop:"),
-            mapOf(),
-            mapOf(),
-            listOf(),
-            tempDir,
-            null,
-            null,
-            null,
-            null,
-            listOf(),
-            listOf()
-          )
-        ))
+      executor.evaluatePath(pklFile) {
+        allowedModules("file:")
+        allowedResources("prop:")
+        rootDir(tempDir)
+      }
    }

    assertThat(e.message)
@@ -441,8 +359,8 @@ class EmbeddedExecutorTest {
  }

  @ParameterizedTest
-  @MethodSource("getAllExecutionContexts")
-  fun `time out a module`(context: ExecutionContext, @TempDir tempDir: Path) {
+  @MethodSource("getAllTestExecutors")
+  fun `time out a module`(executor: TestExecutor, @TempDir tempDir: Path) {
    val pklFile = tempDir.resolve("test.pkl")
    pklFile.toFile().writeText(
      """
@@ -456,23 +374,12 @@ class EmbeddedExecutorTest {
    )

    val e = assertThrows<ExecutorException> {
-      context.executor.evaluatePath(
-          pklFile,
-          context.options(ExecutorOptions2(
-            listOf("file:"),
-            listOf("prop:"),
-            mapOf(),
-            mapOf(),
-            listOf(),
-            tempDir,
-            Duration.ofSeconds(1),
-            null,
-            null,
-            null,
-            listOf(),
-            listOf()
-          ))
-        )
+      executor.evaluatePath(pklFile) {
+        allowedModules("file:")
+        allowedResources("prop:")
+        rootDir(tempDir)
+        timeout(Duration.ofSeconds(1))
+      }
    }

    assertThat(e.message)
@@ -493,22 +400,15 @@ class EmbeddedExecutorTest {
      chirpy = new Bird { name = "Chirpy"; favoriteFruit { name = "Orange" } }
    """.trimIndent()
    )
-    PackageServer.ensureStarted()
-    val result = currentExecutor.evaluatePath(pklFile,
-        ExecutorOptions2(
-          listOf("file:", "package:", "https:"),
-          listOf("prop:", "package:", "https:"),
-          mapOf(),
-          mapOf(),
-          listOf(),
-          null,
-          null,
-          null,
-          cacheDir,
-          null,
-          listOf(FileTestUtils.selfSignedCertificate),
-          listOf())
-      )
+    val result = PackageServer().use { server ->
+      currentExecutor.evaluatePath(pklFile) {
+        allowedModules("file:", "package:", "https:")
+        allowedResources("prop:", "package:", "https:")
+        moduleCacheDir(cacheDir)
+        certificateFiles(FileTestUtils.selfSignedCertificate)
+        testPort(server.port)
+      }
+    }
    assertThat(result.trim()).isEqualTo("""
      chirpy {
        name = "Chirpy"
@@ -523,8 +423,8 @@ class EmbeddedExecutorTest {
  }

  @ParameterizedTest
-  @MethodSource("getAllExecutionContexts")
-  fun `evaluate a project dependency`(context: ExecutionContext, @TempDir tempDir: Path) {
+  @MethodSource("getAllTestExecutors")
+  fun `evaluate a project dependency`(executor: TestExecutor, @TempDir tempDir: Path) {
    val cacheDir = tempDir.resolve("packages")
    PackageServer.populateCacheDir(cacheDir)
    val projectDir = tempDir.resolve("project/")
@@ -569,21 +469,12 @@ class EmbeddedExecutorTest {
        result = Swallow
      """.trimIndent()
    )
-    val result = context.executor.evaluatePath(pklFile,
-        context.options(ExecutorOptions2(
-          listOf("file:", "package:", "projectpackage:", "https:"),
-          listOf("prop:", "package:", "projectpackage:", "https:"),
-          mapOf(),
-          mapOf(),
-          listOf(),
-          null,
-          null,
-          null,
-          cacheDir,
-          projectDir,
-          listOf(),
-          listOf())
-      ))
+    val result = executor.evaluatePath(pklFile) {
+      allowedModules("file:", "package:", "projectpackage:", "https:")
+      allowedResources("prop:", "package:", "projectpackage:", "https:")
+      moduleCacheDir(cacheDir)
+      projectDir(projectDir)
+    }
    assertThat(result).isEqualTo("""
      result {
        name = "Swallow"
--- a/pkl-gradle/src/main/java/org/pkl/gradle/PklPlugin.java
+++ b/pkl-gradle/src/main/java/org/pkl/gradle/PklPlugin.java
@@ -279,6 +279,8 @@ public class PklPlugin implements Plugin<Project> {
    spec.getModuleCacheDir().set(IoUtils.getDefaultModuleCacheDir().toFile());

    spec.getNoCache().convention(false);
+
+    spec.getTestPort().convention(-1);
  }

  private void configureCodeGenSpec(CodeGenSpec spec) {
@@ -421,6 +423,7 @@ public class PklPlugin implements Plugin<Project> {
    task.getNoCache().set(spec.getNoCache());
    task.getModuleCacheDir().set(spec.getModuleCacheDir());
    task.getEvalTimeout().set(spec.getEvalTimeout());
+    task.getTestPort().set(spec.getTestPort());
  }

  private <T extends ModulesTask, S extends ModulesSpec> void configureModulesTask(T task, S spec) {
--- a/pkl-gradle/src/main/java/org/pkl/gradle/spec/BasePklSpec.java
+++ b/pkl-gradle/src/main/java/org/pkl/gradle/spec/BasePklSpec.java
@@ -48,4 +48,6 @@ public interface BasePklSpec {

  // use same type (Duration) as Gradle's `Task.timeout`
  Property<Duration> getEvalTimeout();
+
+  Property<Integer> getTestPort();
 }
--- a/pkl-gradle/src/main/java/org/pkl/gradle/task/BasePklTask.java
+++ b/pkl-gradle/src/main/java/org/pkl/gradle/task/BasePklTask.java
@@ -123,6 +123,10 @@ public abstract class BasePklTask extends DefaultTask {
  @Optional
  public abstract Property<Duration> getEvalTimeout();

+  @Input
+  @Optional
+  public abstract Property<Integer> getTestPort();
+
  @TaskAction
  public void runTask() {
    doRunTask();
@@ -153,6 +157,7 @@ public abstract class BasePklTask extends DefaultTask {
              false,
              false,
              false,
+              getTestPort().getOrElse(-1),
              Collections.emptyList());
    }
    return cachedOptions;
--- a/pkl-gradle/src/main/java/org/pkl/gradle/task/ModulesTask.java
+++ b/pkl-gradle/src/main/java/org/pkl/gradle/task/ModulesTask.java
@@ -178,6 +178,7 @@ public abstract class ModulesTask extends BasePklTask {
              getOmitProjectSettings().getOrElse(false),
              getNoProject().getOrElse(false),
              false,
+              getTestPort().getOrElse(-1),
              Collections.emptyList());
    }
    return cachedOptions;