Improve HTTP headers logic (#1584)

* Relax forbidden headers constraints
  - remove restriction on browser-related headers
- allow any glob pattern (no need to end with `/` or `*`, because glob
patterns already require users to explicitly declare prefix matches if
that's the intention)
* Replace `List<Pair<, ...>>`; use `Map<String, ...>` instead
* Use glob pattern strings as an API throughout, instead of `Pattern`
(e.g. in `HttpClientBuilder`)
* Add HTTP headers to message passing API
* Add HTTP headers to executor API (introduces `ExecutorSpiOptions4`)
* Add tests for Gradle, CLI, and pkl-executor invocations
* Improve documentation
* Add `isGlobPattern` API to class `String` for in-language validation
of http headers
* Behavior change: make sure explicitly configured `User-Agent` in
`HttpClientBuilder` can be shadowed by headers (allows users to set
`--http-header "**=User-Agent: My User Agent"` and for this to be the
only user agent).

CC @kyokuping

pkl-server/src/main/kotlin/org/pkl/server/Server.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -200,6 +200,7 @@ class Server(private val transport: MessageTransport) : AutoCloseable {
             }
             message.http?.caCertificates?.let(::addCertificates)
             message.http?.rewrites?.let(::setRewrites)
+            message.http?.headers?.let(::setHeaders)
             buildLazily()
           }
         securityManager =

pkl-server/src/main/kotlin/org/pkl/server/ServerMessagePackDecoder.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -103,7 +103,21 @@ class ServerMessagePackDecoder(unpacker: MessageUnpacker) : BaseMessagePackDecod
         ?.map()
         ?.mapKeys { URI(it.key.asStringValue().asString()) }
         ?.mapValues { URI(it.value.asStringValue().asString()) }
-    return Http(caCertificates, proxy, rewrites)
+    val headers =
+      getNullable(httpMap, "headers")
+        ?.asMapValue()
+        ?.map()
+        ?.mapKeys { it.key.asStringValue().asString() }
+        ?.mapValues { (_, value) ->
+          value
+            .asMapValue()
+            .map()
+            .mapKeys { it.key.asStringValue().asString() }
+            .mapValues { value ->
+              value.value.asArrayValue().list().map { it.asStringValue().asString() }
+            }
+        }
+    return Http(caCertificates, proxy, rewrites, headers)
   }
 
   private fun Map<Value, Value>.unpackProxy(): Proxy? {

pkl-server/src/main/kotlin/org/pkl/server/ServerMessagePackEncoder.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -36,7 +36,7 @@ class ServerMessagePackEncoder(packer: MessagePacker) : BaseMessagePackEncoder(p
   }
 
   private fun MessagePacker.packHttp(http: Http) {
-    packMapHeader(0, http.caCertificates, http.proxy, http.rewrites)
+    packMapHeader(0, http.caCertificates, http.proxy, http.rewrites, http.headers)
     http.caCertificates?.let { packKeyValue("caCertificates", it) }
     http.proxy?.let { proxy ->
       packString("proxy")
@@ -52,6 +52,21 @@ class ServerMessagePackEncoder(packer: MessagePacker) : BaseMessagePackEncoder(p
         packString(value.toString())
       }
     }
+    http.headers?.let { headers ->
+      packString("headers")
+      packMapHeader(headers.size)
+      for ((pattern, headerMap) in headers) {
+        packString(pattern)
+        packMapHeader(headerMap.size)
+        for ((headerName, headerValue) in headerMap) {
+          packString(headerName)
+          packArrayHeader(headerValue.size)
+          for (elem in headerValue) {
+            packString(elem)
+          }
+        }
+      }
+    }
   }
 
   private fun MessagePacker.packDependencies(dependencies: Map<String, Dependency>) {

pkl-server/src/main/kotlin/org/pkl/server/ServerMessages.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -60,6 +60,8 @@ data class Http(
   val proxy: Proxy?,
   /** HTTP rewrites */
   val rewrites: Map<URI, URI>?,
+  /** HTTP headers */
+  val headers: Map<String, Map<String, List<String>>>?,
 ) {
   override fun equals(other: Any?): Boolean {
     if (this === other) return true
@@ -69,13 +71,16 @@ data class Http(
       if (other.caCertificates == null) return false
       if (!caCertificates.contentEquals(other.caCertificates)) return false
     } else if (other.caCertificates != null) return false
-    return Objects.equals(rewrites, other.rewrites) && Objects.equals(proxy, other.proxy)
+    return Objects.equals(rewrites, other.rewrites) &&
+      Objects.equals(proxy, other.proxy) &&
+      Objects.equals(headers, other.headers)
   }
 
   override fun hashCode(): Int {
     var result = caCertificates?.contentHashCode() ?: 0
     result = 31 * result + (proxy?.hashCode() ?: 0)
     result = 31 * result + (rewrites?.hashCode() ?: 0)
+    result = 31 * result + (headers?.hashCode() ?: 0)
     return result
   }
 }

pkl-server/src/test/kotlin/org/pkl/server/AbstractServerTest.kt

@@ -15,6 +15,15 @@
  */
 package org.pkl.server
 
+import com.github.tomakehurst.wiremock.client.WireMock.equalTo
+import com.github.tomakehurst.wiremock.client.WireMock.get
+import com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor
+import com.github.tomakehurst.wiremock.client.WireMock.ok
+import com.github.tomakehurst.wiremock.client.WireMock.stubFor
+import com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo
+import com.github.tomakehurst.wiremock.client.WireMock.verify
+import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo
+import com.github.tomakehurst.wiremock.junit5.WireMockTest
 import java.io.PipedInputStream
 import java.io.PipedOutputStream
 import java.net.URI
@@ -30,6 +39,7 @@ import org.assertj.core.api.Assertions.assertThat
 import org.junit.jupiter.api.AfterAll
 import org.junit.jupiter.api.BeforeAll
 import org.junit.jupiter.api.Disabled
+import org.junit.jupiter.api.Nested
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.io.TempDir
 import org.msgpack.core.MessagePack
@@ -1065,6 +1075,7 @@ abstract class AbstractServerTest {
             caCertificates = null,
             proxy = null,
             rewrites = mapOf(URI("https://example.com/") to URI("https://example.example/")),
+            headers = null,
           )
       )
     client.send(
@@ -1092,6 +1103,7 @@ abstract class AbstractServerTest {
             caCertificates = null,
             proxy = null,
             rewrites = mapOf(URI("https://example.com") to URI("https://example.example/")),
+            headers = null,
           )
       )
     )
@@ -1100,6 +1112,37 @@ abstract class AbstractServerTest {
       .contains("Rewrite rule must end with '/', but was 'https://example.com'")
   }
 
+  @Nested
+  @WireMockTest
+  inner class HttpTests {
+    @Test
+    fun `http headers`(wwRuntimeInfo: WireMockRuntimeInfo) {
+      stubFor(get(urlEqualTo("/foo.pkl")).willReturn(ok("foo = 1")))
+      val evaluatorId =
+        client.sendCreateEvaluatorRequest(
+          http =
+            Http(
+              caCertificates = null,
+              proxy = null,
+              rewrites = null,
+              headers = mapOf("**" to mapOf("X-Foo" to listOf("Foo"))),
+            )
+        )
+      client.send(
+        EvaluateRequest(
+          1,
+          evaluatorId,
+          URI("repl:text"),
+          "res = import(\"${wwRuntimeInfo.httpBaseUrl}/foo.pkl\")",
+          "output.text",
+        )
+      )
+      val response = client.receive<EvaluateResponse>()
+      assertThat(response.error).isNull()
+      verify(getRequestedFor(urlEqualTo("/foo.pkl")).withHeader("X-Foo", equalTo("Foo")))
+    }
+  }
+
   private fun TestTransport.sendCreateEvaluatorRequest(
     requestId: Long = 123,
     resourceReaders: List<ResourceReaderSpec> = listOf(),

pkl-server/src/test/kotlin/org/pkl/server/ServerMessagePackCodecTest.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -45,7 +45,7 @@ class ServerMessagePackCodecTest {
   private fun roundtrip(message: Message) {
     encoder.encode(message)
     val decoded = decoder.decode()
-    assertThat(decoded).isEqualTo(message)
+    assertThat(decoded).usingRecursiveComparison().isEqualTo(message)
   }
 
   @Test
@@ -98,6 +98,7 @@ class ServerMessagePackCodecTest {
             proxy = Proxy(URI("http://foo.com:1234"), listOf("bar", "baz")),
             caCertificates = byteArrayOf(1, 2, 3, 4),
             rewrites = mapOf(URI("https://foo.com/") to URI("https://bar.com/")),
+            headers = mapOf("**" to mapOf("X-Foo" to listOf("Foo", "Bar"))),
           ),
         externalModuleReaders = mapOf("external" to externalReader, "external2" to externalReader),
         externalResourceReaders = mapOf("external" to externalReader),