Improve HTTP headers logic (#1584)

* Relax forbidden headers constraints
  - remove restriction on browser-related headers
- allow any glob pattern (no need to end with `/` or `*`, because glob
patterns already require users to explicitly declare prefix matches if
that's the intention)
* Replace `List<Pair<, ...>>`; use `Map<String, ...>` instead
* Use glob pattern strings as an API throughout, instead of `Pattern`
(e.g. in `HttpClientBuilder`)
* Add HTTP headers to message passing API
* Add HTTP headers to executor API (introduces `ExecutorSpiOptions4`)
* Add tests for Gradle, CLI, and pkl-executor invocations
* Improve documentation
* Add `isGlobPattern` API to class `String` for in-language validation
of http headers
* Behavior change: make sure explicitly configured `User-Agent` in
`HttpClientBuilder` can be shadowed by headers (allows users to set
`--http-header "**=User-Agent: My User Agent"` and for this to be the
only user agent).

CC @kyokuping

pkl-server/src/test/kotlin/org/pkl/server/AbstractServerTest.kt

@@ -15,6 +15,15 @@
  */
 package org.pkl.server
 
+import com.github.tomakehurst.wiremock.client.WireMock.equalTo
+import com.github.tomakehurst.wiremock.client.WireMock.get
+import com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor
+import com.github.tomakehurst.wiremock.client.WireMock.ok
+import com.github.tomakehurst.wiremock.client.WireMock.stubFor
+import com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo
+import com.github.tomakehurst.wiremock.client.WireMock.verify
+import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo
+import com.github.tomakehurst.wiremock.junit5.WireMockTest
 import java.io.PipedInputStream
 import java.io.PipedOutputStream
 import java.net.URI
@@ -30,6 +39,7 @@ import org.assertj.core.api.Assertions.assertThat
 import org.junit.jupiter.api.AfterAll
 import org.junit.jupiter.api.BeforeAll
 import org.junit.jupiter.api.Disabled
+import org.junit.jupiter.api.Nested
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.io.TempDir
 import org.msgpack.core.MessagePack
@@ -1065,6 +1075,7 @@ abstract class AbstractServerTest {
             caCertificates = null,
             proxy = null,
             rewrites = mapOf(URI("https://example.com/") to URI("https://example.example/")),
+            headers = null,
           )
       )
     client.send(
@@ -1092,6 +1103,7 @@ abstract class AbstractServerTest {
             caCertificates = null,
             proxy = null,
             rewrites = mapOf(URI("https://example.com") to URI("https://example.example/")),
+            headers = null,
           )
       )
     )
@@ -1100,6 +1112,37 @@ abstract class AbstractServerTest {
       .contains("Rewrite rule must end with '/', but was 'https://example.com'")
   }
 
+  @Nested
+  @WireMockTest
+  inner class HttpTests {
+    @Test
+    fun `http headers`(wwRuntimeInfo: WireMockRuntimeInfo) {
+      stubFor(get(urlEqualTo("/foo.pkl")).willReturn(ok("foo = 1")))
+      val evaluatorId =
+        client.sendCreateEvaluatorRequest(
+          http =
+            Http(
+              caCertificates = null,
+              proxy = null,
+              rewrites = null,
+              headers = mapOf("**" to mapOf("X-Foo" to listOf("Foo"))),
+            )
+        )
+      client.send(
+        EvaluateRequest(
+          1,
+          evaluatorId,
+          URI("repl:text"),
+          "res = import(\"${wwRuntimeInfo.httpBaseUrl}/foo.pkl\")",
+          "output.text",
+        )
+      )
+      val response = client.receive<EvaluateResponse>()
+      assertThat(response.error).isNull()
+      verify(getRequestedFor(urlEqualTo("/foo.pkl")).withHeader("X-Foo", equalTo("Foo")))
+    }
+  }
+
   private fun TestTransport.sendCreateEvaluatorRequest(
     requestId: Long = 123,
     resourceReaders: List<ResourceReaderSpec> = listOf(),

pkl-server/src/test/kotlin/org/pkl/server/ServerMessagePackCodecTest.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -45,7 +45,7 @@ class ServerMessagePackCodecTest {
   private fun roundtrip(message: Message) {
     encoder.encode(message)
     val decoded = decoder.decode()
-    assertThat(decoded).isEqualTo(message)
+    assertThat(decoded).usingRecursiveComparison().isEqualTo(message)
   }
 
   @Test
@@ -98,6 +98,7 @@ class ServerMessagePackCodecTest {
             proxy = Proxy(URI("http://foo.com:1234"), listOf("bar", "baz")),
             caCertificates = byteArrayOf(1, 2, 3, 4),
             rewrites = mapOf(URI("https://foo.com/") to URI("https://bar.com/")),
+            headers = mapOf("**" to mapOf("X-Foo" to listOf("Foo", "Bar"))),
           ),
         externalModuleReaders = mapOf("external" to externalReader, "external2" to externalReader),
         externalResourceReaders = mapOf("external" to externalReader),