Improve HTTP headers logic (#1584)

* Relax forbidden headers constraints
  - remove restriction on browser-related headers
- allow any glob pattern (no need to end with `/` or `*`, because glob
patterns already require users to explicitly declare prefix matches if
that's the intention)
* Replace `List<Pair<, ...>>`; use `Map<String, ...>` instead
* Use glob pattern strings as an API throughout, instead of `Pattern`
(e.g. in `HttpClientBuilder`)
* Add HTTP headers to message passing API
* Add HTTP headers to executor API (introduces `ExecutorSpiOptions4`)
* Add tests for Gradle, CLI, and pkl-executor invocations
* Improve documentation
* Add `isGlobPattern` API to class `String` for in-language validation
of http headers
* Behavior change: make sure explicitly configured `User-Agent` in
`HttpClientBuilder` can be shadowed by headers (allows users to set
`--http-header "**=User-Agent: My User Agent"` and for this to be the
only user agent).

CC @kyokuping

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliBaseOptions.kt

@@ -20,7 +20,6 @@ import java.nio.file.Files
 import java.nio.file.Path
 import java.time.Duration
 import java.util.regex.Pattern
-import org.pkl.core.Pair
 import org.pkl.core.evaluatorSettings.Color
 import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader
 import org.pkl.core.evaluatorSettings.TraceMode
@@ -146,7 +145,7 @@ data class CliBaseOptions(
   val httpRewrites: Map<URI, URI>? = null,
 
   /** HTTP headers to add to the request. */
-  val httpHeaders: List<Pair<Pattern, List<Pair<String, String>>>>? = null,
+  val httpHeaders: Map<String, Map<String, List<String>>>? = null,
 
   /** External module reader process specs */
   val externalModuleReaders: Map<String, ExternalReader> = mapOf(),

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliCommand.kt

@@ -185,11 +185,11 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
   }
 
   protected val httpRewrites: Map<URI, URI>? by lazy {
-    cliOptions.httpRewrites ?: evaluatorSettings?.http?.rewrites ?: settings.http?.rewrites()
+    cliOptions.httpRewrites ?: evaluatorSettings?.http?.rewrites ?: settings.http?.rewrites
   }
 
-  private val httpHeaders: List<Pair<Pattern, List<Pair<String, String>>>>? by lazy {
-    cliOptions.httpHeaders ?: project?.evaluatorSettings?.http?.headers ?: settings.http?.headers
+  private val httpHeaders: Map<String, Map<String, List<String>>>? by lazy {
+    cliOptions.httpHeaders ?: evaluatorSettings?.http?.headers ?: settings.http?.headers
   }
 
   protected val externalModuleReaders: Map<String, PklEvaluatorSettings.ExternalReader> by lazy {

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseOptions.kt

@@ -31,7 +31,6 @@ import java.util.regex.Pattern
 import org.pkl.commons.cli.CliBaseOptions
 import org.pkl.commons.cli.CliException
 import org.pkl.commons.shlex
-import org.pkl.core.Pair as PPair
 import org.pkl.core.evaluatorSettings.Color
 import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader
 import org.pkl.core.evaluatorSettings.TraceMode
@@ -95,6 +94,8 @@ class BaseOptions : OptionGroup() {
         Pair(it.first, ExternalReader(cmd.first(), cmd.drop(1)))
       }
     }
+
+    private val headerRegex = Regex("""^(.+?):[ \t]*(.*)$""")
   }
 
   private val defaults = CliBaseOptions()
@@ -287,34 +288,34 @@ class BaseOptions : OptionGroup() {
       .multiple()
       .toMap()
 
-  val httpHeaders: List<PPair<Pattern, List<PPair<String, String>>>> by
+  val httpHeaders: Map<String, Map<String, List<String>>> by
     option(
-        names = arrayOf("--http-headers"),
+        names = arrayOf("--http-header"),
         metavar = "<url-pattern>=<header name>:<header value>",
         help = "HTTP header to add to the request.",
       )
       .splitPair()
-      .transformAll { it ->
-        val headersMap = mutableMapOf<String, MutableList<PPair<String, String>>>()
-
-        try {
-          val headerRegex = Regex("""^(.+?):[ \t]*(.+)$""")
-          for ((stringPattern, header) in it) {
+      .transformAll { pairs ->
+        buildMap<String, MutableMap<String, MutableList<String>>> {
+          for ((pattern, headerNameAndValue) in pairs) {
+            try {
+              GlobResolver.toRegexPattern(pattern)
+            } catch (e: GlobResolver.InvalidGlobPatternException) {
+              fail(e.message!!)
+            }
             val (headerName, headerValue) =
-              headerRegex.find(header)?.destructured
-                ?: fail("Header '$header' is not in 'name:value' format.")
-            IoUtils.validateHeaderName(headerName)
-            IoUtils.validateHeaderValue(headerValue)
-            headersMap
-              .computeIfAbsent(stringPattern) { mutableListOf() }
-              .add(PPair(headerName, headerValue))
+              headerRegex.find(headerNameAndValue)?.destructured
+                ?: fail("Header '$headerNameAndValue' is not in 'name:value' format.")
+            try {
+              IoUtils.validateHeaderName(headerName)
+              IoUtils.validateHeaderValue(headerValue)
+            } catch (e: IllegalArgumentException) {
+              fail(e.message!!)
+            }
+            getOrPut(pattern) { mutableMapOf() }
+              .getOrPut(headerName) { mutableListOf() }
+              .add(headerValue)
           }
-
-          headersMap.entries.map { PPair(GlobResolver.toRegexPattern(it.key), it.value) }
-        } catch (e: IllegalArgumentException) {
-          fail(e.message!!)
-        } catch (e: GlobResolver.InvalidGlobPatternException) {
-          fail(e.message!!)
         }
       }