Initial commit

pkl-commons-test/src/main/files/packages/badChecksum@1.0.0/badChecksum@1.0.0.json

@@ -0,0 +1,11 @@
+{
+  "schemaVersion": 1,
+  "packageUri": "package://localhost:12110/badChecksum@1.0.0",
+  "name": "bugs",
+  "packageZipUrl": "https://localhost:12110/badChecksum@1.0.0/badChecksum@1.0.0.zip",
+  "dependencies": {},
+  "version": "1.0.0",
+  "packageZipChecksums": {
+    "sha256": "intentionally bogus checksum"
+  }
+}

pkl-commons-test/src/main/files/packages/badChecksum@1.0.0/package/Bug.pkl

@@ -0,0 +1 @@
+module bugs.Bug

pkl-commons-test/src/main/files/packages/badImportsWithinPackage@1.0.0/badImportsWithinPackage@1.0.0.json

@@ -0,0 +1,11 @@
+{
+  "schemaVersion": 1,
+  "packageUri": "package://localhost:12110/badImportsWithinPackage@1.1.0",
+  "name": "badImportsWithinPackage",
+  "packageZipUrl": "https://localhost:12110/badImportsWithinPackage@1.0.0/badImportsWithinPackage@1.0.0.zip",
+  "dependencies": {},
+  "version": "1.0.0",
+  "packageZipChecksums": {
+    "sha256": "$computedChecksum"
+  }
+}

pkl-commons-test/src/main/files/packages/badImportsWithinPackage@1.0.0/package/invalidPath.pkl

@@ -0,0 +1 @@
+res = import("not/a/valid/path.pkl")

pkl-commons-test/src/main/files/packages/badImportsWithinPackage@1.0.0/package/invalidPathRead.pkl

@@ -0,0 +1 @@
+res = read("not/a/valid/path.txt")

pkl-commons-test/src/main/files/packages/badImportsWithinPackage@1.0.0/package/unknownDependency.pkl

@@ -0,0 +1 @@
+res = import("@fruits/Foo.pkl")

pkl-commons-test/src/main/files/packages/badImportsWithinPackage@1.0.0/package/unknownDependencyRead.pkl

@@ -0,0 +1 @@
+res = read("@notapackage/Foo.txt")

pkl-commons-test/src/main/files/packages/badMetadataJson@1.0.0/badMetadataJson@1.0.0.json

@@ -0,0 +1 @@
+this is not json

pkl-commons-test/src/main/files/packages/badMetadataJson@1.0.0/package/Bug.pkl

@@ -0,0 +1 @@
+module bugs.Bug

pkl-commons-test/src/main/files/packages/badPackageZipUrl@1.0.0/badPackageZipUrl@1.0.0.json

@@ -0,0 +1,11 @@
+{
+  "schemaVersion": 1,
+  "packageUri": "package://localhost:12110/badPackagezipUrl@1.1.0",
+  "name": "bugs",
+  "packageZipUrl": "ftp://wait/a/minute",
+  "dependencies": {},
+  "version": "1.0.0",
+  "packageZipChecksums": {
+    "sha256": "$computedChecksum"
+  }
+}

pkl-commons-test/src/main/files/packages/badPackageZipUrl@1.0.0/package/Bug.pkl

@@ -0,0 +1 @@
+module bugs.Bug

pkl-commons-test/src/main/files/packages/birds@0.5.0/birds@0.5.0.json

@@ -0,0 +1,27 @@
+{
+  "schemaVersion": 1,
+  "name": "birds",
+  "packageUri": "package://localhost:12110/birds@0.5.0",
+  "packageZipUrl": "https://localhost:12110/birds@0.5.0/birds@0.5.0.zip",
+  "dependencies": {
+    "fruities": {
+      "uri": "package://localhost:12110/fruit@1.0.5",
+      "checksums": {
+        "sha256": "b4ea243de781feeab7921227591e6584db5d0673340f30fab2ffe8ad5c9f75f5"
+      }
+    }
+  },
+  "version": "0.5.0",
+  "packageZipChecksums": {
+    "sha256": "$computedChecksum"
+  },
+  "sourceCodeUrlScheme": "https://example.com/birds/v0.5.0/blob%{path}#L%{line}-L%{endLine}",
+  "sourceCode": "https://example.com/birds",
+  "documentation": "https://example.com/bird-docs",
+  "license": "UNLICENSED",
+  "authors": [
+    "petey-bird@example.com",
+    "polly-bird@example.com"
+  ],
+  "issueTracker": "https://example.com/birds/issues"
+}

pkl-commons-test/src/main/files/packages/birds@0.5.0/package/Bird.pkl

@@ -0,0 +1,7 @@
+open module birds.Bird
+
+import "@fruities/Fruit.pkl"
+
+name: String
+
+favoriteFruit: Fruit

pkl-commons-test/src/main/files/packages/birds@0.5.0/package/allFruit.pkl

@@ -0,0 +1,4 @@
+module birds.allFruit
+
+fruit = import*("@fruities/catalog/*.pkl")
+fruitFiles = read*("@fruities/catalog/*.pkl")

pkl-commons-test/src/main/files/packages/birds@0.5.0/package/catalog.pkl

@@ -0,0 +1,4 @@
+module birds.catalog
+
+catalog = import*("catalog/*.pkl")
+catalogFiles = read*("catalog/*.pkl")

pkl-commons-test/src/main/files/packages/birds@0.5.0/package/catalog/Ostritch.pkl

@@ -0,0 +1,7 @@
+amends "../Bird.pkl"
+
+name = "Ostritch"
+
+favoriteFruit {
+  name = "Orange"
+}

pkl-commons-test/src/main/files/packages/birds@0.5.0/package/catalog/Swallow.pkl

@@ -0,0 +1,7 @@
+amends "../Bird.pkl"
+
+import "@fruities/catalog/apple.pkl"
+
+name = "Swallow"
+
+favoriteFruit = apple

pkl-commons-test/src/main/files/packages/birds@0.5.0/package/some/dir/Bird.pkl

@@ -0,0 +1,7 @@
+amends "..."
+
+name = "Bird"
+
+favoriteFruit {
+  name = "Fruit"
+}

pkl-commons-test/src/main/files/packages/fruit@1.0.5/fruit@1.0.5.json

@@ -0,0 +1,19 @@
+{
+  "schemaVersion": 1,
+  "packageUri": "package://localhost:12110/fruit@1.0.5",
+  "name": "fruit",
+  "version": "1.0.5",
+  "packageZipUrl": "https://localhost:12110/fruit@1.0.5/fruit@1.0.5.zip",
+  "dependencies": {},
+  "packageZipChecksums": {
+    "sha256": "$computedChecksum"
+  },
+  "sourceCode": "https://example.com/fruit",
+  "documentation": "https://example.com/fruit-docs",
+  "license": "UNLICENSED",
+  "authors": [
+    "apple-1@example.com",
+    "banana-2@example.com"
+  ],
+  "issueTracker": "https://example.com/fruit/issues"
+}

pkl-commons-test/src/main/files/packages/fruit@1.0.5/package/Fruit.pkl

@@ -0,0 +1,3 @@
+module fruit.Fruit
+
+name: String

pkl-commons-test/src/main/files/packages/fruit@1.0.5/package/catalog/apple.pkl

@@ -0,0 +1,3 @@
+amends "../Fruit.pkl"
+
+name = "Apple"

pkl-commons-test/src/main/files/packages/fruit@1.1.0/fruit@1.1.0.json

@@ -0,0 +1,19 @@
+{
+  "schemaVersion": 1,
+  "packageUri": "package://localhost:12110/fruit@1.1.0",
+  "name": "fruit",
+  "version": "1.1.0",
+  "packageZipUrl": "https://localhost:12110/fruit@1.1.0/fruit@1.1.0.zip",
+  "dependencies": {},
+  "packageZipChecksums": {
+    "sha256": "$computedChecksum"
+  },
+  "sourceCode": "https://example.com/fruit",
+  "documentation": "https://example.com/fruit-docs",
+  "license": "UNLICENSED",
+  "authors": [
+    "apple-1@example.com",
+    "banana-2@example.com"
+  ],
+  "issueTracker": "https://example.com/fruit/issues"
+}

pkl-commons-test/src/main/files/packages/fruit@1.1.0/package/Fruit.pkl

@@ -0,0 +1,3 @@
+module fruit.Fruit
+
+name: String

pkl-commons-test/src/main/files/packages/fruit@1.1.0/package/catalog/apple.pkl

@@ -0,0 +1,3 @@
+amends "../Fruit.pkl"
+
+name = "Apple 🍎"

pkl-commons-test/src/main/files/packages/fruit@1.1.0/package/catalog/pineapple.pkl

@@ -0,0 +1,3 @@
+amends "../Fruit.pkl"
+
+name = "Pineapple 🍍"

pkl-commons-test/src/main/files/packages/packageContainingWildcardCharacters@1.0.0/packageContainingWildcardCharacters@1.0.0.json

@@ -0,0 +1,11 @@
+{
+  "schemaVersion": 1,
+  "packageUri": "package://localhost:12110/packageContainingWildcardCharacters@1.0.0",
+  "name": "packageContainingWildcardCharacters",
+  "version": "1.0.0",
+  "packageZipUrl": "https://localhost:12110/packageContainingWildcardCharacters@1.0.0/packageContainingWildcardCharacters@1.0.0.zip",
+  "dependencies": {},
+  "packageZipChecksums": {
+    "sha256": "$computedChecksum"
+  }
+}

pkl-commons-test/src/main/kotlin/org/pkl/commons/test/FileTestUtils.kt

@@ -0,0 +1,106 @@
+/**
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.commons.test
+
+import java.nio.file.Path
+import kotlin.io.path.*
+import kotlin.streams.toList
+import org.assertj.core.api.Assertions.fail
+import org.pkl.commons.*
+
+object FileTestUtils {
+  val rootProjectDir: Path by lazy {
+    val workingDir = currentWorkingDir
+    workingDir.takeIf { it.resolve("settings.gradle.kts").exists() }
+      ?: workingDir.parent.takeIf { it.resolve("settings.gradle.kts").exists() }
+        ?: workingDir.parent.parent.takeIf { it.resolve("settings.gradle.kts").exists() }
+        ?: throw AssertionError("Failed to locate root project directory.")
+  }
+  val selfSignedCertificate: Path by lazy {
+    rootProjectDir.resolve("pkl-commons-test/build/keystore/localhost.pem")
+  }
+}
+
+fun Path.listFilesRecursively(): List<Path> =
+  walk(99).use { paths -> paths.filter { it.isRegularFile() || it.isSymbolicLink() }.toList() }
+
+data class SnippetOutcome(val expectedOutFile: Path, val actual: String, val success: Boolean) {
+  private val expectedErrFile =
+    expectedOutFile.resolveSibling(expectedOutFile.toString().replaceAfterLast('.', "err"))
+
+  private val expectedOutExists = expectedOutFile.exists()
+  private val expectedErrExists = expectedErrFile.exists()
+  private val overwrite
+    get() = System.getenv().containsKey("OVERWRITE_SNIPPETS")
+
+  private val expected by lazy {
+    when {
+      expectedOutExists && expectedErrExists ->
+        fail("Test has both expected out and .err files: $displayName")
+      expectedOutExists -> expectedOutFile.readString()
+      expectedErrExists -> expectedErrFile.readString()
+      else -> ""
+    }
+  }
+
+  private val displayName by lazy {
+    val path = expectedOutFile.toString()
+    val baseDir = "src/test/files"
+    val index = path.indexOf(baseDir)
+    val endIndex = path.lastIndexOf('.')
+    if (index == -1 || endIndex == -1) path else path.substring(index + baseDir.length, endIndex)
+  }
+
+  fun check() {
+    when {
+      success && !expectedOutExists && !expectedErrExists && actual.isBlank() -> return
+      !success && expectedOutExists && !overwrite ->
+        failWithDiff("Test was expected to succeed, but failed: $displayName")
+      !success && expectedOutExists -> {
+        expectedOutFile.deleteExisting()
+        expectedErrFile.writeString(actual)
+        fail("Wrote file $expectedErrFile for $displayName and deleted $expectedOutFile")
+      }
+      success && expectedErrExists && !overwrite ->
+        failWithDiff("Test was expected to fail, but succeeded: $displayName")
+      success && expectedErrExists -> {
+        expectedErrFile.deleteExisting()
+        expectedOutFile.writeString(actual)
+        fail("Wrote file $expectedOutFile for $displayName and deleted $expectedErrFile")
+      }
+      !expectedOutExists && !expectedErrExists && actual.isNotBlank() -> {
+        val file = if (success) expectedOutFile else expectedErrFile
+        file.createParentDirectories().writeString(actual)
+        failWithDiff("Created missing file $file for $displayName")
+      }
+      else -> {
+        assert(success && expectedOutExists || !success && expectedErrExists)
+        if (actual != expected) {
+          if (overwrite) {
+            val file = if (success) expectedOutFile else expectedErrFile
+            file.writeString(actual)
+            fail("Overwrote file $file for $displayName")
+          } else {
+            failWithDiff("Output was different from expected: $displayName")
+          }
+        }
+      }
+    }
+  }
+
+  private fun failWithDiff(message: String): Nothing =
+    throw PklAssertionFailedError(message, expected, actual)
+}

pkl-commons-test/src/main/kotlin/org/pkl/commons/test/InputOutputTestEngine.kt

@@ -0,0 +1,155 @@
+/**
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.commons.test
+
+import java.nio.file.Path
+import java.util.Locale
+import kotlin.io.path.isRegularFile
+import kotlin.io.path.useDirectoryEntries
+import kotlin.reflect.KClass
+import org.junit.platform.engine.*
+import org.junit.platform.engine.TestDescriptor.Type
+import org.junit.platform.engine.discovery.ClassSelector
+import org.junit.platform.engine.discovery.MethodSelector
+import org.junit.platform.engine.discovery.PackageSelector
+import org.junit.platform.engine.discovery.UniqueIdSelector
+import org.junit.platform.engine.support.descriptor.*
+import org.junit.platform.engine.support.hierarchical.EngineExecutionContext
+import org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine
+import org.junit.platform.engine.support.hierarchical.Node
+import org.junit.platform.engine.support.hierarchical.Node.DynamicTestExecutor
+
+abstract class InputOutputTestEngine :
+  HierarchicalTestEngine<InputOutputTestEngine.ExecutionContext>() {
+  protected val rootProjectDir = FileTestUtils.rootProjectDir
+
+  protected abstract val testClass: KClass<*>
+
+  protected open val includedTests: List<Regex> = listOf(Regex(".*"))
+
+  @Suppress("RegExpUnexpectedAnchor")
+  protected open val excludedTests: List<Regex> = listOf(Regex("$^"))
+
+  protected abstract val inputDir: Path
+
+  protected abstract val isInputFile: (Path) -> Boolean
+
+  protected abstract fun expectedOutputFileFor(inputFile: Path): Path
+
+  protected abstract fun generateOutputFor(inputFile: Path): Pair<Boolean, String>
+
+  class ExecutionContext : EngineExecutionContext
+
+  override fun getId(): String = this::class.java.simpleName
+
+  init {
+    // Enforce consistent locale for tests to avoid inconsistent formatting.
+    Locale.setDefault(Locale.ROOT)
+  }
+
+  override fun discover(
+    discoveryRequest: EngineDiscoveryRequest,
+    uniqueId: UniqueId
+  ): TestDescriptor {
+    val packageSelectors = discoveryRequest.getSelectorsByType(PackageSelector::class.java)
+    val classSelectors = discoveryRequest.getSelectorsByType(ClassSelector::class.java)
+    val methodSelectors = discoveryRequest.getSelectorsByType(MethodSelector::class.java)
+    val uniqueIdSelectors = discoveryRequest.getSelectorsByType(UniqueIdSelector::class.java)
+
+    val packageName = testClass.java.`package`.name
+    val className = testClass.java.name
+
+    if (
+      methodSelectors.isEmpty() &&
+        (packageSelectors.isEmpty() || packageSelectors.any { it.packageName == packageName }) &&
+        (classSelectors.isEmpty() || classSelectors.any { it.className == className })
+    ) {
+
+      val rootNode = InputDirNode(uniqueId, inputDir, ClassSource.from(testClass.java))
+      return doDiscover(rootNode, uniqueIdSelectors)
+    }
+
+    // return empty descriptor w/o children
+    return EngineDescriptor(uniqueId, javaClass.simpleName)
+  }
+
+  private fun doDiscover(
+    dirNode: InputDirNode,
+    uniqueIdSelectors: List<UniqueIdSelector>
+  ): TestDescriptor {
+    dirNode.inputDir.useDirectoryEntries { children ->
+      for (child in children) {
+        val testPath = child.toString()
+        val testName = child.fileName.toString()
+        if (child.isRegularFile()) {
+          if (
+            isInputFile(child) &&
+              includedTests.any { it.matches(testPath) } &&
+              !excludedTests.any { it.matches(testPath) }
+          ) {
+            val childId = dirNode.uniqueId.append("inputFileNode", testName)
+            if (
+              uniqueIdSelectors.isEmpty() ||
+                uniqueIdSelectors.any { childId.hasPrefix(it.uniqueId) }
+            ) {
+              dirNode.addChild(InputFileNode(childId, child))
+            } // else skip
+          }
+        } else {
+          val childId = dirNode.uniqueId.append("inputDirNode", testName)
+          dirNode.addChild(
+            doDiscover(
+              InputDirNode(childId, child, DirectorySource.from(child.toFile())),
+              uniqueIdSelectors
+            )
+          )
+        }
+      }
+    }
+    return dirNode
+  }
+
+  override fun createExecutionContext(request: ExecutionRequest) = ExecutionContext()
+
+  private inner class InputDirNode(uniqueId: UniqueId, val inputDir: Path, source: TestSource) :
+    AbstractTestDescriptor(uniqueId, inputDir.fileName.toString(), source), Node<ExecutionContext> {
+    override fun getType() = Type.CONTAINER
+  }
+
+  private inner class InputFileNode(uniqueId: UniqueId, private val inputFile: Path) :
+    AbstractTestDescriptor(
+      uniqueId,
+      inputFile.fileName.toString(),
+      FileSource.from(inputFile.toFile())
+    ),
+    Node<ExecutionContext> {
+
+    override fun getType() = Type.TEST
+
+    override fun execute(
+      context: ExecutionContext,
+      dynamicTestExecutor: DynamicTestExecutor
+    ): ExecutionContext {
+
+      val (success, actualOutput) = generateOutputFor(inputFile)
+      val expectedOutputFile = expectedOutputFileFor(inputFile)
+
+      SnippetOutcome(expectedOutputFile, actualOutput, success).check()
+
+      return context
+    }
+  }
+}

pkl-commons-test/src/main/kotlin/org/pkl/commons/test/PackageServer.kt

@@ -0,0 +1,145 @@
+/**
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.commons.test
+
+import com.sun.net.httpserver.HttpHandler
+import com.sun.net.httpserver.HttpsConfigurator
+import com.sun.net.httpserver.HttpsParameters
+import com.sun.net.httpserver.HttpsServer
+import java.net.BindException
+import java.net.InetSocketAddress
+import java.nio.file.*
+import java.security.KeyStore
+import java.util.concurrent.Executors
+import javax.net.ssl.KeyManagerFactory
+import javax.net.ssl.SSLContext
+import kotlin.io.path.isRegularFile
+import org.pkl.commons.createParentDirectories
+import org.pkl.commons.deleteRecursively
+
+object PackageServer {
+  private val keystore = javaClass.getResource("/localhost.p12")!!
+
+  // When tests are run via Gradle (i.e. from ./gradlew check), resources are packaged into a jar.
+  // When run directly in IntelliJ, resources are just directories.
+  private val packagesDir: Path = let {
+    val uri = javaClass.getResource("packages")!!.toURI()
+    try {
+      Path.of(uri)
+    } catch (e: FileSystemNotFoundException) {
+      FileSystems.newFileSystem(uri, mapOf<String, String>())
+      Path.of(uri)
+    }
+  }
+
+  fun populateCacheDir(cacheDir: Path) {
+    val basePath = cacheDir.resolve("package-1/localhost:$PORT")
+    basePath.deleteRecursively()
+    Files.walk(packagesDir).use { stream ->
+      stream.forEach { source ->
+        if (!source.isRegularFile()) return@forEach
+        val relativized =
+          source.toString().replaceFirst(packagesDir.toString(), "").drop(1).ifEmpty {
+            return@forEach
+          }
+        val dest = basePath.resolve(relativized)
+        dest.createParentDirectories()
+        Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING)
+      }
+    }
+  }
+
+  private const val PORT = 12110
+  private var started = false
+
+  private val sslContext by lazy {
+    SSLContext.getInstance("SSL").apply {
+      val pass = "password".toCharArray()
+      val ks = KeyStore.getInstance("PKCS12").apply { load(keystore.openStream(), pass) }
+      val kmf = KeyManagerFactory.getInstance("SunX509").apply { init(ks, pass) }
+      init(kmf.keyManagers, null, null)
+    }
+  }
+
+  private val engine by lazy { sslContext.createSSLEngine() }
+
+  private val simpleHttpsConfigurator =
+    object : HttpsConfigurator(sslContext) {
+      override fun configure(params: HttpsParameters) {
+        params.needClientAuth = false
+        params.cipherSuites = engine.enabledCipherSuites
+        params.protocols = engine.enabledProtocols
+        params.setSSLParameters(sslContext.supportedSSLParameters)
+      }
+    }
+
+  private val handler = HttpHandler { exchange ->
+    if (exchange.requestMethod != "GET") {
+      exchange.sendResponseHeaders(405, 0)
+      exchange.close()
+      return@HttpHandler
+    }
+    val path = exchange.requestURI.path
+    val localPath =
+      if (path.endsWith(".zip")) packagesDir.resolve(path.drop(1))
+      else packagesDir.resolve("${path.drop(1)}${path}.json")
+    if (!Files.exists(localPath)) {
+      exchange.sendResponseHeaders(404, 0)
+      exchange.close()
+      return@HttpHandler
+    }
+    exchange.sendResponseHeaders(200, 0)
+    exchange.responseBody.use { outputStream -> Files.copy(localPath, outputStream) }
+    exchange.close()
+  }
+
+  private val myExecutor = Executors.newFixedThreadPool(1)
+
+  private val server by lazy {
+    HttpsServer.create().apply {
+      httpsConfigurator = simpleHttpsConfigurator
+      createContext("/", handler)
+      executor = myExecutor
+    }
+  }
+
+  fun ensureStarted() =
+    synchronized(this) {
+      if (!started) {
+        // Crude hack to make sure that parrallel tests don't try and use each others mock server
+        // otherwise you get flaky tests when a server instance is shutdown by one set of tests
+        // while another set of tests is still relying on it.
+        // Side effect is that tests that spin up a mock package server are now serialised, rather
+        // than running in parrallel. But that seems like a reasonable tradeoff to avoid flaky
+        // tests.
+        for (i in 1..20) {
+          try {
+            server.bind(InetSocketAddress(PORT), 0)
+            server.start()
+            started = true
+            println("Mock package server started after $i attempt(s)")
+            return@synchronized
+          } catch (_: BindException) {
+            println(
+              "Port $PORT in use after $i/20 attempt(s), probably another test running in parrallel. Sleeping for 1 second and trying again"
+            )
+            Thread.sleep(1000)
+          }
+        }
+        println("Unable to start package server! This will probably result in a test failures")
+      }
+    }
+}

pkl-commons-test/src/main/kotlin/org/pkl/commons/test/PklAssertionFailedError.kt

@@ -0,0 +1,33 @@
+/**
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.commons.test
+
+import org.assertj.core.util.diff.DiffUtils
+import org.opentest4j.AssertionFailedError
+
+/**
+ * Makes up for the fact that [AssertionFailedError] doesn't print a diff, resulting in
+ * unintelligible errors outside IDEs (which show a diff dialog).
+ * https://github.com/ota4j-team/opentest4j/issues/59
+ */
+class PklAssertionFailedError(message: String, expected: Any?, actual: Any?) :
+  AssertionFailedError(message, expected, actual) {
+  override fun toString(): String {
+    val patch =
+      DiffUtils.diff(expected.stringRepresentation.lines(), actual.stringRepresentation.lines())
+    return patch.deltas.joinToString("\n\n")
+  }
+}