Bind PackageServer to ephemeral port to avoid port conflicts (#227)

This is a comprehensive solution to the "flaky PackageServer tests" problem. It rules out port conflicts and imposes no limits on test parallelism. The same solution can be used for other test servers in the future. Major changes: - Turn `PackageServer` from a singleton into a class that is instantiated per test class or test method. - Start the server the first time its `port` property is read. Bind the server to an ephemeral port instead of port 12110. - For every test that uses `PackageServer`, pass the server port to `--test-port`, `HttpClient.Builder.setTestPort`, the `CliBaseOptions` or `ExecutorOptions` constructor, or the Gradle plugin's `testPort` property. Wire all of these to `RequestRewritingClient`'s `testPort` constructor parameter. - Enhance `RequestRewritingClient` to replace port 12110 with `testPort` in request URIs unless `testPort` is -1 (its default). - Introduce `ExecutorOptions.Builder`. This makes executor options more comfortable to create and allows to hide options such as `testPort`. - Deprecate the `ExecutorOptions` constructor to steer users towards the builder. - Get rid of `ExecutorOptions2`, which is no longer needed. - Clean up `EmbeddedExecutorTest` with the help of the builder.
2026-04-01 14:43:12 +02:00 · 2024-03-13 10:40:55 -07:00
parent 1e608b2aae
commit 014b3a8816
26 changed files with 756 additions and 581 deletions
--- a/pkl-cli/src/test/kotlin/org/pkl/cli/CliEvaluatorTest.kt
+++ b/pkl-cli/src/test/kotlin/org/pkl/cli/CliEvaluatorTest.kt
@@ -24,6 +24,7 @@ import java.time.Duration
 import kotlin.io.path.*
 import org.assertj.core.api.Assertions.assertThat
 import org.assertj.core.api.Assertions.assertThatCode
+import org.junit.jupiter.api.AfterAll
 import org.junit.jupiter.api.AfterEach
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertThrows
@@ -40,12 +41,22 @@ import org.pkl.core.util.IoUtils

 class CliEvaluatorTest {
  companion object {
-    const val defaultContents = """
-person {
-  name = "pigeon"
-  age = 20 + 10
-}
+    private val defaultContents =
+      """
+      person {
+        name = "pigeon"
+        age = 20 + 10
+      }
    """
+        .trimIndent()
+
+    private val packageServer = PackageServer()
+
+    @AfterAll
+    @JvmStatic
+    fun afterAll() {
+      packageServer.close()
+    }
  }

  // use manually constructed temp dir instead of @TempDir to work around
@@ -1117,7 +1128,6 @@ result = someLib.x

  @Test
  fun `setting noCache will skip writing to the cache dir`() {
-    PackageServer.ensureStarted()
    val moduleUri =
      writePklFile(
        "test.pkl",
@@ -1136,7 +1146,8 @@ result = someLib.x
          workingDir = tempDir,
          moduleCacheDir = tempDir,
          noCache = true,
-          caCertificates = listOf(FileTestUtils.selfSignedCertificate)
+          caCertificates = listOf(FileTestUtils.selfSignedCertificate),
+          testPort = packageServer.port
        ),
      )
    CliEvaluator(options, consoleWriter = buffer).run()
@@ -1185,10 +1196,10 @@ result = someLib.x

  @Test
  fun `gives decent error message if CLI doesn't have the required CA certificate`() {
-    PackageServer.ensureStarted()
    // provide SOME certs to prevent CliEvaluator from falling back to ~/.pkl/cacerts
    val builtInCerts = FileTestUtils.writePklBuiltInCertificates(tempDir)
-    val err = assertThrows<CliException> { evalModuleThatImportsPackage(builtInCerts) }
+    val err =
+      assertThrows<CliException> { evalModuleThatImportsPackage(builtInCerts, packageServer.port) }
    assertThat(err)
      // on some JDK11's this doesn't cause SSLHandshakeException but some other SSLException
      // .hasMessageContaining("Error during SSL handshake with host `localhost`:")
@@ -1196,7 +1207,7 @@ result = someLib.x
      .hasMessageNotContainingAny("java.", "sun.") // class names have been filtered out
  }

-  private fun evalModuleThatImportsPackage(certsFile: Path) {
+  private fun evalModuleThatImportsPackage(certsFile: Path, testPort: Int = -1) {
    val moduleUri =
      writePklFile(
        "test.pkl",
@@ -1212,7 +1223,9 @@ result = someLib.x
        CliBaseOptions(
          sourceModules = listOf(moduleUri),
          caCertificates = listOf(certsFile),
-          noCache = true
+          workingDir = tempDir,
+          noCache = true,
+          testPort = testPort
        ),
      )
    CliEvaluator(options).run()
--- a/pkl-cli/src/test/kotlin/org/pkl/cli/CliPackageDownloaderTest.kt
+++ b/pkl-cli/src/test/kotlin/org/pkl/cli/CliPackageDownloaderTest.kt
@@ -18,7 +18,7 @@ package org.pkl.cli
 import java.nio.file.Path
 import org.assertj.core.api.Assertions.assertThat
 import org.assertj.core.api.Assertions.assertThatCode
-import org.junit.jupiter.api.BeforeAll
+import org.junit.jupiter.api.AfterAll
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.io.TempDir
 import org.pkl.commons.cli.CliBaseOptions
@@ -28,10 +28,12 @@ import org.pkl.core.packages.PackageUri

 class CliPackageDownloaderTest {
  companion object {
-    @BeforeAll
+    val server = PackageServer()
+
+    @AfterAll
    @JvmStatic
-    fun beforeAll() {
-      PackageServer.ensureStarted()
+    fun afterAll() {
+      server.close()
    }
  }

@@ -42,7 +44,8 @@ class CliPackageDownloaderTest {
        baseOptions =
          CliBaseOptions(
            moduleCacheDir = tempDir,
-            caCertificates = listOf(FileTestUtils.selfSignedCertificate)
+            caCertificates = listOf(FileTestUtils.selfSignedCertificate),
+            testPort = server.port
          ),
        packageUris =
          listOf(
@@ -80,7 +83,8 @@ class CliPackageDownloaderTest {
        baseOptions =
          CliBaseOptions(
            workingDir = tempDir,
-            caCertificates = listOf(FileTestUtils.selfSignedCertificate)
+            caCertificates = listOf(FileTestUtils.selfSignedCertificate),
+            testPort = server.port
          ),
        packageUris = listOf(PackageUri("package://localhost:12110/birds@0.5.0")),
        noTransitive = true
@@ -99,7 +103,8 @@ class CliPackageDownloaderTest {
        baseOptions =
          CliBaseOptions(
            moduleCacheDir = tempDir,
-            caCertificates = listOf(FileTestUtils.selfSignedCertificate)
+            caCertificates = listOf(FileTestUtils.selfSignedCertificate),
+            testPort = server.port
          ),
        packageUris =
          listOf(
@@ -121,7 +126,8 @@ class CliPackageDownloaderTest {
        baseOptions =
          CliBaseOptions(
            moduleCacheDir = tempDir,
-            caCertificates = listOf(FileTestUtils.selfSignedCertificate)
+            caCertificates = listOf(FileTestUtils.selfSignedCertificate),
+            testPort = server.port
          ),
        packageUris =
          listOf(
@@ -161,7 +167,8 @@ class CliPackageDownloaderTest {
        baseOptions =
          CliBaseOptions(
            moduleCacheDir = tempDir,
-            caCertificates = listOf(FileTestUtils.selfSignedCertificate)
+            caCertificates = listOf(FileTestUtils.selfSignedCertificate),
+            testPort = server.port
          ),
        packageUris = listOf(PackageUri("package://localhost:12110/badChecksum@1.0.0")),
        noTransitive = true
@@ -179,7 +186,8 @@ class CliPackageDownloaderTest {
        baseOptions =
          CliBaseOptions(
            moduleCacheDir = tempDir,
-            caCertificates = listOf(FileTestUtils.selfSignedCertificate)
+            caCertificates = listOf(FileTestUtils.selfSignedCertificate),
+            testPort = server.port
          ),
        packageUris =
          listOf(
@@ -215,7 +223,8 @@ class CliPackageDownloaderTest {
        baseOptions =
          CliBaseOptions(
            moduleCacheDir = tempDir,
-            caCertificates = listOf(FileTestUtils.selfSignedCertificate)
+            caCertificates = listOf(FileTestUtils.selfSignedCertificate),
+            testPort = server.port
          ),
        packageUris = listOf(PackageUri("package://localhost:12110/birds@0.5.0")),
        noTransitive = false
--- a/pkl-cli/src/test/kotlin/org/pkl/cli/CliProjectPackagerTest.kt
+++ b/pkl-cli/src/test/kotlin/org/pkl/cli/CliProjectPackagerTest.kt
@@ -24,6 +24,7 @@ import java.util.stream.Collectors
 import kotlin.io.path.createDirectories
 import org.assertj.core.api.Assertions.assertThat
 import org.assertj.core.api.Assertions.assertThatCode
+import org.junit.jupiter.api.AfterAll
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertThrows
 import org.junit.jupiter.api.io.TempDir
@@ -36,6 +37,16 @@ import org.pkl.commons.test.PackageServer
 import org.pkl.commons.writeString

 class CliProjectPackagerTest {
+  companion object {
+    private val packageServer = PackageServer()
+
+    @AfterAll
+    @JvmStatic
+    fun afterAll() {
+      packageServer.close()
+    }
+  }
+
  @Test
  fun `missing PklProject when inferring a project dir`(@TempDir tempDir: Path) {
    val packager =
@@ -866,7 +877,6 @@ class CliProjectPackagerTest {

  @Test
  fun `publish checks`(@TempDir tempDir: Path) {
-    PackageServer.ensureStarted()
    tempDir.writeFile("project/main.pkl", "res = 1")
    tempDir.writeFile(
      "project/PklProject",
@@ -888,7 +898,8 @@ class CliProjectPackagerTest {
        CliProjectPackager(
            CliBaseOptions(
              workingDir = tempDir,
-              caCertificates = listOf(FileTestUtils.selfSignedCertificate)
+              caCertificates = listOf(FileTestUtils.selfSignedCertificate),
+              testPort = packageServer.port
            ),
            listOf(tempDir.resolve("project")),
            CliTestOptions(),
@@ -912,7 +923,6 @@ class CliProjectPackagerTest {

  @Test
  fun `publish check when package is not yet published`(@TempDir tempDir: Path) {
-    PackageServer.ensureStarted()
    tempDir.writeFile("project/main.pkl", "res = 1")
    tempDir.writeFile(
      "project/PklProject",
@@ -932,7 +942,8 @@ class CliProjectPackagerTest {
    CliProjectPackager(
        CliBaseOptions(
          workingDir = tempDir,
-          caCertificates = listOf(FileTestUtils.selfSignedCertificate)
+          caCertificates = listOf(FileTestUtils.selfSignedCertificate),
+          testPort = packageServer.port
        ),
        listOf(tempDir.resolve("project")),
        CliTestOptions(),
--- a/pkl-cli/src/test/kotlin/org/pkl/cli/CliProjectResolverTest.kt
+++ b/pkl-cli/src/test/kotlin/org/pkl/cli/CliProjectResolverTest.kt
@@ -18,7 +18,7 @@ package org.pkl.cli
 import java.io.StringWriter
 import java.nio.file.Path
 import org.assertj.core.api.Assertions.assertThat
-import org.junit.jupiter.api.BeforeAll
+import org.junit.jupiter.api.AfterAll
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertThrows
 import org.junit.jupiter.api.io.TempDir
@@ -29,10 +29,12 @@ import org.pkl.commons.test.PackageServer

 class CliProjectResolverTest {
  companion object {
-    @BeforeAll
+    private val packageServer = PackageServer()
+
+    @AfterAll
    @JvmStatic
-    fun beforeAll() {
-      PackageServer.ensureStarted()
+    fun afterAll() {
+      packageServer.close()
    }
  }

@@ -40,7 +42,7 @@ class CliProjectResolverTest {
  fun `missing PklProject when inferring a project dir`(@TempDir tempDir: Path) {
    val packager =
      CliProjectResolver(
-        CliBaseOptions(workingDir = tempDir),
+        CliBaseOptions(workingDir = tempDir, noCache = true),
        emptyList(),
        consoleWriter = StringWriter(),
        errWriter = StringWriter()
@@ -53,7 +55,7 @@ class CliProjectResolverTest {
  fun `missing PklProject when explicit dir is provided`(@TempDir tempDir: Path) {
    val packager =
      CliProjectResolver(
-        CliBaseOptions(),
+        CliBaseOptions(noCache = true),
        listOf(tempDir),
        consoleWriter = StringWriter(),
        errWriter = StringWriter()
@@ -80,7 +82,9 @@ class CliProjectResolverTest {
    CliProjectResolver(
        CliBaseOptions(
          workingDir = tempDir,
-          caCertificates = listOf(FileTestUtils.selfSignedCertificate)
+          caCertificates = listOf(FileTestUtils.selfSignedCertificate),
+          testPort = packageServer.port,
+          noCache = true
        ),
        listOf(tempDir),
        consoleWriter = StringWriter(),
@@ -133,7 +137,9 @@ class CliProjectResolverTest {
    CliProjectResolver(
        CliBaseOptions(
          workingDir = tempDir,
-          caCertificates = listOf(FileTestUtils.selfSignedCertificate)
+          caCertificates = listOf(FileTestUtils.selfSignedCertificate),
+          testPort = packageServer.port,
+          noCache = true
        ),
        emptyList(),
        consoleWriter = StringWriter(),
@@ -228,7 +234,11 @@ class CliProjectResolverTest {
        .trimIndent()
    )
    CliProjectResolver(
-        CliBaseOptions(caCertificates = listOf(FileTestUtils.selfSignedCertificate)),
+        CliBaseOptions(
+          caCertificates = listOf(FileTestUtils.selfSignedCertificate),
+          testPort = packageServer.port,
+          noCache = true
+        ),
        listOf(projectDir),
        consoleWriter = StringWriter(),
        errWriter = StringWriter()
@@ -306,7 +316,11 @@ class CliProjectResolverTest {
    val consoleOut = StringWriter()
    val errOut = StringWriter()
    CliProjectResolver(
-        CliBaseOptions(caCertificates = listOf(FileTestUtils.selfSignedCertificate)),
+        CliBaseOptions(
+          caCertificates = listOf(FileTestUtils.selfSignedCertificate),
+          testPort = packageServer.port,
+          noCache = true
+        ),
        listOf(projectDir),
        consoleWriter = consoleOut,
        errWriter = errOut
@@ -377,7 +391,11 @@ class CliProjectResolverTest {
    val consoleOut = StringWriter()
    val errOut = StringWriter()
    CliProjectResolver(
-        CliBaseOptions(caCertificates = listOf(FileTestUtils.selfSignedCertificate)),
+        CliBaseOptions(
+          caCertificates = listOf(FileTestUtils.selfSignedCertificate),
+          testPort = packageServer.port,
+          noCache = true
+        ),
        listOf(tempDir.resolve("project1"), tempDir.resolve("project2")),
        consoleWriter = consoleOut,
        errWriter = errOut