From 8e2e5e4ba84cad25f4b8b89e9ec97529eabda6a3 Mon Sep 17 00:00:00 2001 From: Daniel Chao Date: Thu, 21 May 2026 20:07:06 -0700 Subject: [PATCH] Improve HTTP headers logic (#1584) * Relax forbidden headers constraints - remove restriction on browser-related headers - allow any glob pattern (no need to end with `/` or `*`, because glob patterns already require users to explicitly declare prefix matches if that's the intention) * Replace `List>`; use `Map` instead * Use glob pattern strings as an API throughout, instead of `Pattern` (e.g. in `HttpClientBuilder`) * Add HTTP headers to message passing API * Add HTTP headers to executor API (introduces `ExecutorSpiOptions4`) * Add tests for Gradle, CLI, and pkl-executor invocations * Improve documentation * Add `isGlobPattern` API to class `String` for in-language validation of http headers * Behavior change: make sure explicitly configured `User-Agent` in `HttpClientBuilder` can be shadowed by headers (allows users to set `--http-header "**=User-Agent: My User Agent"` and for this to be the only user agent). CC @kyokuping --- .../pages/message-passing-api.adoc | 5 + .../language-reference/pages/index.adoc | 2 - .../pkl-cli/partials/cli-common-options.adoc | 17 +++ .../partials/gradle-common-properties.adoc | 13 ++ .../kotlin/org/pkl/cli/CliEvaluatorTest.kt | 18 +++ .../test/kotlin/org/pkl/cli/CliMainTest.kt | 56 +++++++++ .../org/pkl/commons/cli/CliBaseOptions.kt | 3 +- .../kotlin/org/pkl/commons/cli/CliCommand.kt | 6 +- .../pkl/commons/cli/commands/BaseOptions.kt | 45 +++---- .../java/org/pkl/core/EvaluatorBuilder.java | 3 + .../PklEvaluatorSettings.java | 96 +++++++------- .../java/org/pkl/core/http/HttpClient.java | 35 +++++- .../org/pkl/core/http/HttpClientBuilder.java | 57 ++++++++- .../org/pkl/core/http/LazyHttpClient.java | 6 +- .../pkl/core/http/RequestRewritingClient.java | 62 +++++++-- .../messaging/AbstractMessagePackEncoder.java | 12 +- .../org/pkl/core/service/ExecutorSpiImpl.java | 33 ++--- .../org/pkl/core/stdlib/base/StringNodes.java | 14 +++ .../main/java/org/pkl/core/util/IoUtils.java | 68 ++++++---- .../org/pkl/core/errorMessages.properties | 16 ++- .../input/api/evaluatorSettings.pkl | 68 ++++++++++ .../LanguageSnippetTests/input/api/string.pkl | 6 + .../output/api/evaluatorSettings.pcf | 28 +++++ .../output/api/reflectedDeclaration.pcf | 42 +++++++ .../output/api/string.pcf | 5 + .../org/pkl/core/EvaluatorBuilderTest.kt | 11 +- .../pkl/core/http/HttpClientBuilderTest.kt | 46 +++++++ .../core/http/RequestRewritingClientTest.kt | 74 ++++++++--- .../org/pkl/core/settings/PklSettingsTest.kt | 15 ++- .../org/pkl/core/project/project1/PklProject | 16 +++ pkl-executor/pkl-executor.gradle.kts | 1 + .../org/pkl/executor/ExecutorOptions.java | 43 ++++++- .../executor/spi/v1/ExecutorSpiOptions4.java | 66 ++++++++++ .../org/pkl/executor/EmbeddedExecutorTest.kt | 36 +++++- pkl-gradle/pkl-gradle.gradle.kts | 1 + .../main/java/org/pkl/gradle/PklPlugin.java | 1 + .../java/org/pkl/gradle/spec/BasePklSpec.java | 4 + .../java/org/pkl/gradle/task/BasePklTask.java | 4 +- .../kotlin/org/pkl/gradle/EvaluatorsTest.kt | 32 +++++ pkl-server/pkl-server.gradle.kts | 1 + .../src/main/kotlin/org/pkl/server/Server.kt | 3 +- .../pkl/server/ServerMessagePackDecoder.kt | 18 ++- .../pkl/server/ServerMessagePackEncoder.kt | 19 ++- .../kotlin/org/pkl/server/ServerMessages.kt | 9 +- .../org/pkl/server/AbstractServerTest.kt | 43 +++++++ .../pkl/server/ServerMessagePackCodecTest.kt | 5 +- stdlib/EvaluatorSettings.pkl | 118 +++++++++++++----- stdlib/base.pkl | 7 ++ 48 files changed, 1067 insertions(+), 222 deletions(-) create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/input/api/evaluatorSettings.pkl create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/output/api/evaluatorSettings.pcf create mode 100644 pkl-core/src/test/kotlin/org/pkl/core/http/HttpClientBuilderTest.kt create mode 100644 pkl-executor/src/main/java/org/pkl/executor/spi/v1/ExecutorSpiOptions4.java diff --git a/docs/modules/bindings-specification/pages/message-passing-api.adoc b/docs/modules/bindings-specification/pages/message-passing-api.adoc index 6b6674df..c05149b9 100644 --- a/docs/modules/bindings-specification/pages/message-passing-api.adoc +++ b/docs/modules/bindings-specification/pages/message-passing-api.adoc @@ -210,6 +210,11 @@ class Http { /// /// Each rewrite must start with `http://` or `https://`, and must end with `/`. rewrites: Mapping? + + /// HTTP headers. + /// Each entry key is a glob pattern that is matched against outbound request URLs. + /// Each value is a map of headers that is added to the matching request. + headers: Mapping>>? } /// Settings that control how Pkl talks to HTTP proxies. diff --git a/docs/modules/language-reference/pages/index.adoc b/docs/modules/language-reference/pages/index.adoc index 2f2ff5b3..8b4f2551 100644 --- a/docs/modules/language-reference/pages/index.adoc +++ b/docs/modules/language-reference/pages/index.adoc @@ -5059,8 +5059,6 @@ in the context of that module. [[glob-patterns]] === Glob Patterns -Resources and modules may be imported at the same time by globbing with the <> and <> features. - Pkl's glob patterns mostly follow the rules described by link:{uri-glob-7}[glob(7)], with the following differences: * `*` includes names that start with a dot (`.`). diff --git a/docs/modules/pkl-cli/partials/cli-common-options.adoc b/docs/modules/pkl-cli/partials/cli-common-options.adoc index baf3da6d..13d94bcd 100644 --- a/docs/modules/pkl-cli/partials/cli-common-options.adoc +++ b/docs/modules/pkl-cli/partials/cli-common-options.adoc @@ -158,6 +158,23 @@ This option is commonly used to enable package mirroring. The above example will rewrite URL `\https://pkg.pkl-lang.org/pkl-k8s/k8s@1.0.0` to `\https://my.internal.mirror/pkl-k8s/k8s@1.0.0`. ==== +.--http-header +[%collapsible] +==== +Default: (none) + +Example: `**=User-Agent: My User Agent` + + +Additional headers to add to outbound HTTP requests. + +The syntax is `=
:
`, and any whitespace after the leading colon is trimmed. + +The glob pattern is used to match against the URLs of outbound HTTP calls, and the value is the header name and value to add. +In the case of multiple matches, every header is added. + +To describe a prefix match, the `\\**` wildcard should be added to the pattern. +For example, `https?://example.com/**` matches against every request to host `example.com`. +==== + .--trace-mode [%collapsible] ==== diff --git a/docs/modules/pkl-gradle/partials/gradle-common-properties.adoc b/docs/modules/pkl-gradle/partials/gradle-common-properties.adoc index cf53c996..10fad87c 100644 --- a/docs/modules/pkl-gradle/partials/gradle-common-properties.adoc +++ b/docs/modules/pkl-gradle/partials/gradle-common-properties.adoc @@ -119,6 +119,19 @@ This option is commonly used to enable package mirroring. The above example will rewrite URL `\https://pkg.pkl-lang.org/pkl-k8s/k8s@1.0.0` to `\https://my.internal.mirror/pkl-k8s/k8s@1.0.0`. ==== +.httpHeaders: MapProperty>> +[%collapsible] +==== +Default: `null` + +Example: `httpHeaders = ["**": ["User-Agent": ["My User Agent"]]]` + +Additional headers to add to outbound HTTP requests. +The key is a glob pattern that is used to match against the URLs of outbound HTTP calls, and the value is the header name and values to add. +Each header value becomes its own HTTP header. + +To describe a prefix match, the `\\**` wildcard should be added to the pattern. +For example, `https?://example.com/**` matches against every request to host `example.com`. +==== + .powerAssertions: Property [%collapsible] ==== diff --git a/pkl-cli/src/test/kotlin/org/pkl/cli/CliEvaluatorTest.kt b/pkl-cli/src/test/kotlin/org/pkl/cli/CliEvaluatorTest.kt index 8bc67340..a0d4fa26 100644 --- a/pkl-cli/src/test/kotlin/org/pkl/cli/CliEvaluatorTest.kt +++ b/pkl-cli/src/test/kotlin/org/pkl/cli/CliEvaluatorTest.kt @@ -1614,6 +1614,24 @@ result = someLib.x assertThat(output).isEqualTo("result = 1\n") } + @Test + fun `eval configured http headers`(wwRuntimeInfo: WireMockRuntimeInfo) { + stubFor(get(anyUrl()).willReturn(ok("result = 1"))) + val file = URI("${wwRuntimeInfo.httpBaseUrl}/foo.pkl") + val output = + evalToConsole( + CliEvaluatorOptions( + CliBaseOptions( + sourceModules = listOf(file), + httpHeaders = mapOf("**" to mapOf("X-Foo" to listOf("Foo"))), + allowedModules = + listOf(Pattern.compile("http:"), Pattern.compile("file:"), Pattern.compile("pkl:")), + ) + ) + ) + verify(getRequestedFor(urlEqualTo("/foo.pkl")).withHeader("X-Foo", equalTo("Foo"))) + } + @Test fun `eval file with non-ASCII name`() { val tempDirUri = tempDir.toUri() diff --git a/pkl-cli/src/test/kotlin/org/pkl/cli/CliMainTest.kt b/pkl-cli/src/test/kotlin/org/pkl/cli/CliMainTest.kt index 29c1575e..092ada73 100644 --- a/pkl-cli/src/test/kotlin/org/pkl/cli/CliMainTest.kt +++ b/pkl-cli/src/test/kotlin/org/pkl/cli/CliMainTest.kt @@ -169,4 +169,60 @@ class CliMainTest { link.createSymbolicLinkPointingTo(dir) return link } + + @Test + fun `invalid http header glob pattern`() { + val ex = + assertThrows { + rootCmd.parse(arrayOf("eval", "--http-header", "foo{{}}=bar:baz", "myModule.pkl")) + } + assertThat(ex.message) + .contains("Sub-patterns cannot be nested. To fix, remove or escape the inner `{` character.") + } + + @Test + fun `forbidden http header name`() { + val ex = + assertThrows { + rootCmd.parse(arrayOf("eval", "--http-header", "**=Connection: baz", "myModule.pkl")) + } + assertThat(ex.message).contains("HTTP header `Connection` is a reserved header") + } + + @Test + fun `bad http header value`() { + val ex = + assertThrows { + rootCmd.parse(arrayOf("eval", "--http-header", "**=X-Foo:🙃", "myModule.pkl")) + } + assertThat(ex.message).contains("HTTP header value `🙃` has invalid syntax") + } + + @Test + fun `multiple headers`() { + val cmd = RootCommand() + cmd.parse( + arrayOf( + "eval", + "--http-header", + "**=X-Foo:Foo", + "--http-header", + "**=X-Foo:Foo2", + "--http-header", + "**=X-Bar:Bar", + "--http-header", + "https://example.com/**=X-Qux:Qux", + "pkl:base", + ) + ) + + val evalCmd = cmd.registeredSubcommands().filterIsInstance().first() + assertThat(evalCmd.baseOptions.httpHeaders) + .isEqualTo( + mapOf( + "**" to mapOf("X-Foo" to listOf("Foo", "Foo2"), "X-Bar" to listOf("Bar")), + "https://example.com/**" to mapOf("X-Qux" to listOf("Qux")), + ) + ) + } } diff --git a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliBaseOptions.kt b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliBaseOptions.kt index ec07e1b9..af58ad48 100644 --- a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliBaseOptions.kt +++ b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliBaseOptions.kt @@ -20,7 +20,6 @@ import java.nio.file.Files import java.nio.file.Path import java.time.Duration import java.util.regex.Pattern -import org.pkl.core.Pair import org.pkl.core.evaluatorSettings.Color import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader import org.pkl.core.evaluatorSettings.TraceMode @@ -146,7 +145,7 @@ data class CliBaseOptions( val httpRewrites: Map? = null, /** HTTP headers to add to the request. */ - val httpHeaders: List>>>? = null, + val httpHeaders: Map>>? = null, /** External module reader process specs */ val externalModuleReaders: Map = mapOf(), diff --git a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliCommand.kt b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliCommand.kt index 954112e4..c944a0ee 100644 --- a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliCommand.kt +++ b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliCommand.kt @@ -185,11 +185,11 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) { } protected val httpRewrites: Map? by lazy { - cliOptions.httpRewrites ?: evaluatorSettings?.http?.rewrites ?: settings.http?.rewrites() + cliOptions.httpRewrites ?: evaluatorSettings?.http?.rewrites ?: settings.http?.rewrites } - private val httpHeaders: List>>>? by lazy { - cliOptions.httpHeaders ?: project?.evaluatorSettings?.http?.headers ?: settings.http?.headers + private val httpHeaders: Map>>? by lazy { + cliOptions.httpHeaders ?: evaluatorSettings?.http?.headers ?: settings.http?.headers } protected val externalModuleReaders: Map by lazy { diff --git a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseOptions.kt b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseOptions.kt index a4cae35f..62ba7653 100644 --- a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseOptions.kt +++ b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseOptions.kt @@ -31,7 +31,6 @@ import java.util.regex.Pattern import org.pkl.commons.cli.CliBaseOptions import org.pkl.commons.cli.CliException import org.pkl.commons.shlex -import org.pkl.core.Pair as PPair import org.pkl.core.evaluatorSettings.Color import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader import org.pkl.core.evaluatorSettings.TraceMode @@ -95,6 +94,8 @@ class BaseOptions : OptionGroup() { Pair(it.first, ExternalReader(cmd.first(), cmd.drop(1))) } } + + private val headerRegex = Regex("""^(.+?):[ \t]*(.*)$""") } private val defaults = CliBaseOptions() @@ -287,34 +288,34 @@ class BaseOptions : OptionGroup() { .multiple() .toMap() - val httpHeaders: List>>> by + val httpHeaders: Map>> by option( - names = arrayOf("--http-headers"), + names = arrayOf("--http-header"), metavar = "=
:
", help = "HTTP header to add to the request.", ) .splitPair() - .transformAll { it -> - val headersMap = mutableMapOf>>() - - try { - val headerRegex = Regex("""^(.+?):[ \t]*(.+)$""") - for ((stringPattern, header) in it) { + .transformAll { pairs -> + buildMap>> { + for ((pattern, headerNameAndValue) in pairs) { + try { + GlobResolver.toRegexPattern(pattern) + } catch (e: GlobResolver.InvalidGlobPatternException) { + fail(e.message!!) + } val (headerName, headerValue) = - headerRegex.find(header)?.destructured - ?: fail("Header '$header' is not in 'name:value' format.") - IoUtils.validateHeaderName(headerName) - IoUtils.validateHeaderValue(headerValue) - headersMap - .computeIfAbsent(stringPattern) { mutableListOf() } - .add(PPair(headerName, headerValue)) + headerRegex.find(headerNameAndValue)?.destructured + ?: fail("Header '$headerNameAndValue' is not in 'name:value' format.") + try { + IoUtils.validateHeaderName(headerName) + IoUtils.validateHeaderValue(headerValue) + } catch (e: IllegalArgumentException) { + fail(e.message!!) + } + getOrPut(pattern) { mutableMapOf() } + .getOrPut(headerName) { mutableListOf() } + .add(headerValue) } - - headersMap.entries.map { PPair(GlobResolver.toRegexPattern(it.key), it.value) } - } catch (e: IllegalArgumentException) { - fail(e.message!!) - } catch (e: GlobResolver.InvalidGlobPatternException) { - fail(e.message!!) } } diff --git a/pkl-core/src/main/java/org/pkl/core/EvaluatorBuilder.java b/pkl-core/src/main/java/org/pkl/core/EvaluatorBuilder.java index b46c444f..8e1d9c4f 100644 --- a/pkl-core/src/main/java/org/pkl/core/EvaluatorBuilder.java +++ b/pkl-core/src/main/java/org/pkl/core/EvaluatorBuilder.java @@ -557,6 +557,9 @@ public final class EvaluatorBuilder { if (settings.http().rewrites() != null) { httpClientBuilder.setRewrites(settings.http().rewrites()); } + if (settings.http().headers() != null) { + httpClientBuilder.setHeaders(settings.http().headers()); + } setHttpClient(httpClientBuilder.buildLazily()); } diff --git a/pkl-core/src/main/java/org/pkl/core/evaluatorSettings/PklEvaluatorSettings.java b/pkl-core/src/main/java/org/pkl/core/evaluatorSettings/PklEvaluatorSettings.java index f24a9224..8d94344d 100644 --- a/pkl-core/src/main/java/org/pkl/core/evaluatorSettings/PklEvaluatorSettings.java +++ b/pkl-core/src/main/java/org/pkl/core/evaluatorSettings/PklEvaluatorSettings.java @@ -18,9 +18,9 @@ package org.pkl.core.evaluatorSettings; import java.net.URI; import java.net.URISyntaxException; import java.nio.file.Path; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -28,18 +28,14 @@ import java.util.Objects; import java.util.function.BiFunction; import java.util.regex.Pattern; import java.util.stream.Collectors; -import java.util.stream.Stream; import org.jspecify.annotations.Nullable; import org.pkl.core.Duration; import org.pkl.core.PNull; import org.pkl.core.PObject; -import org.pkl.core.Pair; import org.pkl.core.PklBugException; import org.pkl.core.PklException; import org.pkl.core.Value; import org.pkl.core.util.ErrorMessages; -import org.pkl.core.util.GlobResolver; -import org.pkl.core.util.GlobResolver.InvalidGlobPatternException; /** Java version of {@code pkl.EvaluatorSettings}. */ public record PklEvaluatorSettings( @@ -134,63 +130,61 @@ public record PklEvaluatorSettings( public record Http( @Nullable Proxy proxy, @Nullable Map rewrites, - @Nullable List>>> headers) { + @Nullable Map>> headers) { public static final Http DEFAULT = new Http(null, Collections.emptyMap(), null); - @SuppressWarnings("unchecked") public static @Nullable Http parse(@Nullable Value input) { if (input == null || input instanceof PNull) { return null; } else if (input instanceof PObject http) { var proxy = Proxy.parse((Value) http.getProperty("proxy")); - var rewrites = http.getProperty("rewrites"); - HashMap parsedRewrites = null; - if (!(rewrites instanceof PNull)) { - parsedRewrites = new HashMap<>(); - for (var entry : ((Map) rewrites).entrySet()) { - var key = entry.getKey(); - var value = entry.getValue(); - try { - parsedRewrites.put(new URI(key), new URI(value)); - } catch (URISyntaxException e) { - throw new PklException(ErrorMessages.create("invalidUri", e.getInput())); - } - } - } - var headerDefs = http.getProperty("headers"); - List>>> parsedHeaderDefs = null; - if (!(headerDefs instanceof PNull)) { - parsedHeaderDefs = new ArrayList<>(); - var headerDefsMap = (Map>) headerDefs; - for (var entry : headerDefsMap.entrySet()) { - var stringPattern = entry.getKey(); - var headersMap = entry.getValue(); - try { - var urlPattern = GlobResolver.toRegexPattern(stringPattern); - var pairs = - headersMap.entrySet().stream() - .flatMap( - header -> { - var value = header.getValue(); - if (value instanceof List) { - return ((List) value) - .stream().map(v -> new Pair(header.getKey(), v)); - } else { - return Stream.of(new Pair(header.getKey(), value)); - } - }) - .toList(); - parsedHeaderDefs.add(new Pair(urlPattern, pairs)); - } catch (InvalidGlobPatternException e) { - throw new PklException(ErrorMessages.create("invalidUri", stringPattern)); - } - } - } - return new Http(proxy, parsedRewrites, parsedHeaderDefs); + var parsedRewrites = parseHttpRewrites(http.getProperty("rewrites")); + var parsedHeaders = parseHttpHeaders(http.getProperty("headers")); + return new Http(proxy, parsedRewrites, parsedHeaders); } else { throw PklBugException.unreachableCode(); } } + + @SuppressWarnings("unchecked") + private static @Nullable Map parseHttpRewrites(Object rewrites) { + if (rewrites instanceof PNull) { + return null; + } + var parsedRewrites = new HashMap(); + for (var entry : ((Map) rewrites).entrySet()) { + var key = entry.getKey(); + var value = entry.getValue(); + try { + parsedRewrites.put(new URI(key), new URI(value)); + } catch (URISyntaxException e) { + throw new PklException(ErrorMessages.create("invalidUri", e.getInput())); + } + } + return parsedRewrites; + } + + @SuppressWarnings("unchecked") + private static @Nullable Map>> parseHttpHeaders( + Object headerDefs) { + if (headerDefs instanceof PNull) { + return null; + } + var defs = (Map>) headerDefs; + var ret = new LinkedHashMap>>(defs.size()); + for (var entry : defs.entrySet()) { + var headers = entry.getValue(); + var map = new LinkedHashMap>(headers.size()); + for (var header : headers.entrySet()) { + var value = header.getValue(); + var headerValues = + value instanceof List ? (List) value : List.of((String) value); + map.put(header.getKey(), headerValues); + } + ret.put(entry.getKey(), map); + } + return ret; + } } public record Proxy(@Nullable URI address, @Nullable List noProxy) { diff --git a/pkl-core/src/main/java/org/pkl/core/http/HttpClient.java b/pkl-core/src/main/java/org/pkl/core/http/HttpClient.java index ca564f84..c1e33605 100644 --- a/pkl-core/src/main/java/org/pkl/core/http/HttpClient.java +++ b/pkl-core/src/main/java/org/pkl/core/http/HttpClient.java @@ -23,10 +23,8 @@ import java.net.http.HttpTimeoutException; import java.nio.file.Path; import java.util.List; import java.util.Map; -import java.util.regex.Pattern; import javax.net.ssl.SSLContext; import org.jspecify.annotations.Nullable; -import org.pkl.core.Pair; /** * An HTTP client. @@ -47,6 +45,9 @@ public interface HttpClient extends AutoCloseable { * Sets the {@code User-Agent} header. * *

Defaults to {@code "Pkl/$version ($os; $flavor)"}. + * + *

An existing "User-Agent" from {@link Builder#setHeaders} and {@link Builder#addHeaders} + * takes precedence over this field. */ Builder setUserAgent(String userAgent); @@ -146,7 +147,6 @@ public interface HttpClient extends AutoCloseable { /** * Adds a rewrite rule. * - * @see Builder#setRewrites(Map) * @throws IllegalArgumentException if {@code sourcePrefix} or {@code targetPrefix} is invalid. * @since 0.29.0 */ @@ -157,8 +157,35 @@ public interface HttpClient extends AutoCloseable { * *

This method clears all existing headers and replaces them with the contents of the * provided map. + * + *

{@code headerRules} is a map whose keys are glob + * patterns, and values is a map of header names and values. Multiple header values turn + * into multiple individual headers in the HTTP request. + * + *

To add headers to all requests, use {@code **} as the glob pattern. + * + *

To describe a prefix match, add {@code **} to the glob pattern (e.g. {@code + * https://example.com/**}). + * + *

Before an HTTP request is made, each key is matched against the request URL. If any + * matches are found, each of their headers are added to the request. + * + * @throws IllegalArgumentException if any of the keys are invalid glob patterns, or if any of + * the header names or values are invalid. + * @since 0.32.0 */ - Builder setHeaders(List>>> headers); + Builder setHeaders(Map>> headerRules); + + /** + * Adds HTTP headers for URL requests that match {@code globPattern}. + * + * @throws IllegalArgumentException if {@code globPattern} is an invalid glob pattern, or if any + * of the header names or values are invalid. + * @since 0.32.0 + * @see Builder#setHeaders + */ + Builder addHeaders(String globPattern, Map> headers); /** * Creates a new {@code HttpClient} from the current state of this builder. diff --git a/pkl-core/src/main/java/org/pkl/core/http/HttpClientBuilder.java b/pkl-core/src/main/java/org/pkl/core/http/HttpClientBuilder.java index 5b3fc30f..20e39e35 100644 --- a/pkl-core/src/main/java/org/pkl/core/http/HttpClientBuilder.java +++ b/pkl-core/src/main/java/org/pkl/core/http/HttpClientBuilder.java @@ -24,14 +24,17 @@ import java.nio.file.Path; import java.time.Duration; import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.function.Supplier; import java.util.regex.Pattern; import org.jspecify.annotations.Nullable; -import org.pkl.core.Pair; import org.pkl.core.Release; import org.pkl.core.http.HttpClient.Builder; +import org.pkl.core.util.GlobResolver; +import org.pkl.core.util.GlobResolver.InvalidGlobPatternException; +import org.pkl.core.util.IoUtils; final class HttpClientBuilder implements HttpClient.Builder { private String userAgent; @@ -42,7 +45,10 @@ final class HttpClientBuilder implements HttpClient.Builder { private int testPort = -1; private @Nullable ProxySelector proxySelector; private Map rewrites = new HashMap<>(); - private List>>> headers = new ArrayList<>(); + // okay to use Pattern as a map key here because `GlobResolver.toRegexPattern()` caches and + // gives the same `Pattern` instance for an existing glob pattern. + // use LinkedHashMap to preserve insertion order. + private Map>> headers = new LinkedHashMap<>(); HttpClientBuilder() { var release = Release.current(); @@ -115,11 +121,54 @@ final class HttpClientBuilder implements HttpClient.Builder { } @Override - public Builder setHeaders(List>>> headers) { - this.headers = headers; + public Builder setHeaders(Map>> headers) { + var newHeaders = new LinkedHashMap>>(headers.size()); + for (var rule : headers.entrySet()) { + Pattern pattern; + try { + pattern = GlobResolver.toRegexPattern(rule.getKey()); + } catch (InvalidGlobPatternException e) { + throw new IllegalArgumentException(e.getMessage(), e); + } + var map = new LinkedHashMap>(); + for (var entry : rule.getValue().entrySet()) { + IoUtils.validateHeaderName(entry.getKey()); + for (var value : entry.getValue()) { + IoUtils.validateHeaderValue(value); + } + map.put(entry.getKey(), new ArrayList<>(entry.getValue())); + } + newHeaders.put(pattern, map); + } + this.headers = newHeaders; return this; } + @Override + public Builder addHeaders(String globPattern, Map> headers) { + try { + var pattern = GlobResolver.toRegexPattern(globPattern); + var existingHeaders = this.headers.computeIfAbsent(pattern, k -> new HashMap<>()); + for (var entry : headers.entrySet()) { + var headerName = entry.getKey(); + var headerValues = entry.getValue(); + + IoUtils.validateHeaderName(headerName); + for (var value : headerValues) { + IoUtils.validateHeaderValue(value); + } + + var existingList = existingHeaders.putIfAbsent(headerName, new ArrayList<>(headerValues)); + if (existingList != null) { + existingList.addAll(headerValues); + } + } + return this; + } catch (InvalidGlobPatternException e) { + throw new IllegalArgumentException(e.getMessage(), e); + } + } + @Override public HttpClient build() { return doBuild().get(); diff --git a/pkl-core/src/main/java/org/pkl/core/http/LazyHttpClient.java b/pkl-core/src/main/java/org/pkl/core/http/LazyHttpClient.java index 00387034..9a44b58d 100644 --- a/pkl-core/src/main/java/org/pkl/core/http/LazyHttpClient.java +++ b/pkl-core/src/main/java/org/pkl/core/http/LazyHttpClient.java @@ -29,8 +29,9 @@ import org.jspecify.annotations.Nullable; * An {@code HttpClient} decorator that defers creating the underlying HTTP client until the first * send. */ +// visible for testing @ThreadSafe -final class LazyHttpClient implements HttpClient { +public final class LazyHttpClient implements HttpClient { private final Supplier supplier; private final Object lock = new Object(); @@ -55,7 +56,8 @@ final class LazyHttpClient implements HttpClient { getClient().ifPresent(HttpClient::close); } - private HttpClient getOrCreateClient() { + // visible for testing + public HttpClient getOrCreateClient() { synchronized (lock) { // only try to create client once if (exception != null) { diff --git a/pkl-core/src/main/java/org/pkl/core/http/RequestRewritingClient.java b/pkl-core/src/main/java/org/pkl/core/http/RequestRewritingClient.java index 2af696a3..1e8f47f6 100644 --- a/pkl-core/src/main/java/org/pkl/core/http/RequestRewritingClient.java +++ b/pkl-core/src/main/java/org/pkl/core/http/RequestRewritingClient.java @@ -22,6 +22,7 @@ import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.net.http.HttpResponse.BodyHandler; import java.time.Duration; +import java.util.ArrayList; import java.util.Comparator; import java.util.List; import java.util.Map; @@ -29,12 +30,12 @@ import java.util.Map.Entry; import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Pattern; -import java.util.stream.Stream; import javax.annotation.concurrent.ThreadSafe; import org.jspecify.annotations.Nullable; -import org.pkl.core.Pair; import org.pkl.core.PklBugException; import org.pkl.core.util.HttpUtils; +import org.pkl.core.util.IoUtils; +import org.pkl.core.util.Pair; /** * An {@code HttpClient} decorator that @@ -49,15 +50,17 @@ import org.pkl.core.util.HttpUtils; *

Both {@code User-Agent} header and default request timeout are configurable through {@link * HttpClient.Builder}. */ +// visible for testing @ThreadSafe -final class RequestRewritingClient implements HttpClient { +public final class RequestRewritingClient implements HttpClient { // non-private for testing final String userAgent; final Duration requestTimeout; final int testPort; final HttpClient delegate; + private final Map rewritesMap; private final List> rewrites; - private final List>>> headers; + private final Map>> headers; private final AtomicBoolean closed = new AtomicBoolean(); @@ -67,11 +70,12 @@ final class RequestRewritingClient implements HttpClient { int testPort, HttpClient delegate, Map rewrites, - List>>> headers) { + Map>> headers) { this.userAgent = userAgent; this.requestTimeout = requestTimeout; this.testPort = testPort; this.delegate = delegate; + this.rewritesMap = rewrites; this.rewrites = rewrites.entrySet().stream() .map((it) -> Map.entry(normalizeRewrite(it.getKey()), normalizeRewrite(it.getValue()))) @@ -117,11 +121,17 @@ final class RequestRewritingClient implements HttpClient { .headers() .map() .forEach((name, values) -> values.forEach(value -> builder.header(name, value))); - builder.setHeader("User-Agent", userAgent); + var isUserAgentSet = false; for (var header : this.getHeaders(original.uri())) { + var headerName = header.getFirst(); + isUserAgentSet = isUserAgentSet || headerName.equalsIgnoreCase("user-agent"); builder.header(header.getFirst(), header.getSecond()); } + if (!isUserAgentSet) { + builder.setHeader("User-Agent", userAgent); + } + var method = original.method(); original .bodyPublisher() @@ -225,14 +235,30 @@ final class RequestRewritingClient implements HttpClient { return ret; } + private boolean matches(Pattern pattern, URI uri) { + // optimization: "**" always matches, no need to execute regex + // okay to use `==` here because `GlobResolver.toRegexPattern()` caches and + // gives the same `Pattern` instance for an existing glob pattern. + if (pattern == IoUtils.doubleStarGlob) { + return true; + } + return pattern.matcher(uri.toString()).matches(); + } + private List> getHeaders(URI uri) { - return headers.stream() - .flatMap( - rule -> - rule.getFirst().asPredicate().test(uri.toString()) - ? rule.getSecond().stream() - : Stream.empty()) - .toList(); + var result = new ArrayList>(); + for (var rule : headers.entrySet()) { + var pattern = rule.getKey(); + if (!matches(pattern, uri)) { + continue; + } + for (var header : rule.getValue().entrySet()) { + for (var value : header.getValue()) { + result.add(Pair.of(header.getKey(), value)); + } + } + } + return result; } private void checkNotClosed(HttpRequest request) { @@ -241,4 +267,14 @@ final class RequestRewritingClient implements HttpClient { "Cannot send request " + request + " because this client has already been closed."); } } + + // visible for testing + public Map getRewritesMap() { + return rewritesMap; + } + + // visible for testing + public Map>> getHeaders() { + return headers; + } } diff --git a/pkl-core/src/main/java/org/pkl/core/messaging/AbstractMessagePackEncoder.java b/pkl-core/src/main/java/org/pkl/core/messaging/AbstractMessagePackEncoder.java index 66a55fca..29633324 100644 --- a/pkl-core/src/main/java/org/pkl/core/messaging/AbstractMessagePackEncoder.java +++ b/pkl-core/src/main/java/org/pkl/core/messaging/AbstractMessagePackEncoder.java @@ -63,10 +63,18 @@ public abstract class AbstractMessagePackEncoder implements MessageEncoder { } protected void packMapHeader( - int size, @Nullable Object value1, @Nullable Object value2, @Nullable Object value3) + int size, + @Nullable Object value1, + @Nullable Object value2, + @Nullable Object value3, + @Nullable Object value4) throws IOException { packer.packMapHeader( - size + (value1 != null ? 1 : 0) + (value2 != null ? 1 : 0) + (value3 != null ? 1 : 0)); + size + + (value1 != null ? 1 : 0) + + (value2 != null ? 1 : 0) + + (value3 != null ? 1 : 0) + + (value4 != null ? 1 : 0)); } protected void packMapHeader( diff --git a/pkl-core/src/main/java/org/pkl/core/service/ExecutorSpiImpl.java b/pkl-core/src/main/java/org/pkl/core/service/ExecutorSpiImpl.java index 2f865a7d..84d5029c 100644 --- a/pkl-core/src/main/java/org/pkl/core/service/ExecutorSpiImpl.java +++ b/pkl-core/src/main/java/org/pkl/core/service/ExecutorSpiImpl.java @@ -1,5 +1,5 @@ /* - * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. + * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -37,6 +37,7 @@ import org.pkl.executor.spi.v1.ExecutorSpiException; import org.pkl.executor.spi.v1.ExecutorSpiOptions; import org.pkl.executor.spi.v1.ExecutorSpiOptions2; import org.pkl.executor.spi.v1.ExecutorSpiOptions3; +import org.pkl.executor.spi.v1.ExecutorSpiOptions4; public final class ExecutorSpiImpl implements ExecutorSpi { private static final int MAX_HTTP_CLIENTS = 3; @@ -140,15 +141,17 @@ public final class ExecutorSpiImpl implements ExecutorSpi { } private HttpClient getOrCreateHttpClient(ExecutorSpiOptions options) { - List certificateFiles; - List certificateBytes; - Map rewrites; - int testPort; + List certificateFiles = List.of(); + List certificateBytes = List.of(); + Map rewrites = Map.of(); + Map>> headers = Map.of(); + int testPort = -1; try { + if (options instanceof ExecutorSpiOptions4 options4) { + headers = options4.getHttpHeaders(); + } if (options instanceof ExecutorSpiOptions3 options3) { rewrites = options3.getHttpRewrites(); - } else { - rewrites = Map.of(); } if (options instanceof ExecutorSpiOptions2 options2) { certificateFiles = options2.getCertificateFiles(); @@ -157,17 +160,13 @@ public final class ExecutorSpiImpl implements ExecutorSpi { } else { certificateFiles = List.of(); certificateBytes = List.of(); - testPort = -1; } - // host pkl-executor does not have class ExecutorOptions2/ExecutorOptions3 defined. + // host pkl-executor does not have class ExecutorOptions2+ defined. // this will happen if the pkl-executor distribution is too old. - } catch (NoClassDefFoundError e) { - certificateFiles = List.of(); - certificateBytes = List.of(); - rewrites = Map.of(); - testPort = -1; + } catch (NoClassDefFoundError ignored) { } - var clientKey = new HttpClientKey(certificateFiles, certificateBytes, testPort, rewrites); + var clientKey = + new HttpClientKey(certificateFiles, certificateBytes, testPort, rewrites, headers); return httpClients.computeIfAbsent( clientKey, (key) -> { @@ -180,6 +179,7 @@ public final class ExecutorSpiImpl implements ExecutorSpi { } builder.setRewrites(key.rewrites); builder.setTestPort(key.testPort); + builder.setHeaders(key.headers); // If the above didn't add any certificates, // builder will use the JVM's default SSL context. return builder.buildLazily(); @@ -190,5 +190,6 @@ public final class ExecutorSpiImpl implements ExecutorSpi { List certificateFiles, List certificateBytes, int testPort, - Map rewrites) {} + Map rewrites, + Map>> headers) {} } diff --git a/pkl-core/src/main/java/org/pkl/core/stdlib/base/StringNodes.java b/pkl-core/src/main/java/org/pkl/core/stdlib/base/StringNodes.java index ead693f7..b34e6bdf 100644 --- a/pkl-core/src/main/java/org/pkl/core/stdlib/base/StringNodes.java +++ b/pkl-core/src/main/java/org/pkl/core/stdlib/base/StringNodes.java @@ -29,6 +29,8 @@ import org.pkl.core.ast.lambda.ApplyVmFunction1NodeGen; import org.pkl.core.runtime.*; import org.pkl.core.stdlib.*; import org.pkl.core.util.ByteArrayUtils; +import org.pkl.core.util.GlobResolver; +import org.pkl.core.util.GlobResolver.InvalidGlobPatternException; import org.pkl.core.util.Pair; import org.pkl.core.util.StringUtils; @@ -165,6 +167,18 @@ public final class StringNodes { } } + public abstract static class isGlobPattern extends ExternalPropertyNode { + @Specialization + protected boolean eval(String self) { + try { + GlobResolver.toRegexString(self); + return true; + } catch (InvalidGlobPatternException e) { + return false; + } + } + } + public abstract static class isBase64 extends ExternalPropertyNode { @Specialization @TruffleBoundary diff --git a/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java b/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java index 952abf5b..63811ccf 100644 --- a/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java +++ b/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java @@ -45,6 +45,7 @@ import org.pkl.core.packages.PackageLoadError; import org.pkl.core.runtime.ReaderBase; import org.pkl.core.runtime.VmContext; import org.pkl.core.runtime.VmExceptionBuilder; +import org.pkl.core.util.GlobResolver.InvalidGlobPatternException; public final class IoUtils { @@ -53,33 +54,36 @@ public final class IoUtils { private static final Pattern windowsPathLike = Pattern.compile("\\w:\\\\.*"); - private static final Pattern headerNameLike = Pattern.compile("^[a-zA-Z0-9!#$%&'*+-.^_`|~]+$"); + private static final Pattern headerNameLike = Pattern.compile("[a-zA-Z0-9!#$%&'*+-.^_`|~]+"); private static final Pattern headerValueLike = - Pattern.compile("^[\\t\\u0020-\\u007E\\u0080-\\u00FF]*$"); + Pattern.compile("[\\t\\u0020-\\u007E\\u0080-\\u00FF]*"); + public static final Pattern doubleStarGlob; + + static { + try { + doubleStarGlob = GlobResolver.toRegexPattern("**"); + } catch (InvalidGlobPatternException e) { + throw PklBugException.unreachableCode(); + } + } + + // keep in sync with stdlib `EvaluatorSettings.ReservedHttpHeaderName` private static final String[] reservedHeaderNames = { - "accept-charset", - "accept-encoding", "connection", + "keep-alive", "content-length", - "cookie", - "date", - "dnt", "expect", "host", - "keep-alive", - "origin", - "permissions-policy", - "referer", - "te", - "trailer", - "transfer-encoding", "upgrade", - "via" + "te", + "transfer-encoding", + "trailer" }; - private static final String[] reservedHeaderPrefixes = {"proxy-", "sec-", "access-control-"}; + // keep in sync with stdlib `EvaluatorSettings.reservedHttpHeaderPrefix` + private static final String[] reservedHeaderPrefixes = {"proxy-", "sec-"}; private IoUtils() {} @@ -885,36 +889,50 @@ public final class IoUtils { } private static boolean isReservedHeaderName(String headerName) { - return Arrays.stream(reservedHeaderNames).anyMatch((reserved) -> headerName.equals(reserved)); + var normalizedHeader = headerName.toLowerCase(); + for (var reservedHeader : reservedHeaderNames) { + if (normalizedHeader.equals(reservedHeader)) { + return true; + } + } + return false; } private static boolean hasReservedHeaderPrefix(String headerName) { - return Arrays.stream(reservedHeaderPrefixes) - .anyMatch((prefix) -> headerName.startsWith(prefix)); + var normalizedHeader = headerName.toLowerCase(); + for (var prefix : reservedHeaderPrefixes) { + if (normalizedHeader.startsWith(prefix)) { + return true; + } + } + return false; } + // keep in sync with stdlib EvaluatorSettings.HttpHeaderName public static void validateHeaderName(String headerName) { - if (isReservedHeaderName(headerName)) { throw new IllegalArgumentException( - "HTTP header '%s' is a reserved header".formatted(headerName)); + ErrorMessages.create("invalidHttpHeaderReserved", headerName)); } if (hasReservedHeaderPrefix(headerName)) { throw new IllegalArgumentException( - "HTTP header '%s' starts with a reserved header prefix".formatted(headerName)); + ErrorMessages.create("invalidHttpHeaderReservedPrefix", headerName)); } if (!headerNameLike.matcher(headerName).matches()) { - throw new IllegalArgumentException( - "HTTP header name '%s' has an invalid syntax".formatted(headerName)); + throw new IllegalArgumentException(ErrorMessages.create("invalidHttpHeaderName", headerName)); } } public static void validateHeaderValue(String headerValue) { if (!headerValueLike.matcher(headerValue).matches()) { throw new IllegalArgumentException( - "HTTP header value '%s' has an invalid syntax".formatted(headerValue)); + ErrorMessages.create("invalidHttpHeaderValue", headerValue)); + } + if (headerValue.length() > 4096) { + throw new IllegalArgumentException( + ErrorMessages.create("invalidHttpHeaderValueTooLong", headerValue)); } } } diff --git a/pkl-core/src/main/resources/org/pkl/core/errorMessages.properties b/pkl-core/src/main/resources/org/pkl/core/errorMessages.properties index 588fc61c..98fcd8ba 100644 --- a/pkl-core/src/main/resources/org/pkl/core/errorMessages.properties +++ b/pkl-core/src/main/resources/org/pkl/core/errorMessages.properties @@ -1133,8 +1133,18 @@ commandFlagInvalidType=\ Option `{0}` with annotation `@{1}` has invalid type `{2}`.\n\ Expected type: `{3}` -invalidHeaderName=\ -HTTP header name `{0}` has invalid syntax. +invalidHttpHeaderReserved=\ +HTTP header `{0}` is a reserved header. -invalidHeaderValue=\ +invalidHttpHeaderReservedPrefix=\ +HTTP header `{0}` starts with a reserved header prefix. + +invalidHttpHeaderName=\ +HTTP header `{0}` has invalid syntax. + +invalidHttpHeaderValue=\ HTTP header value `{0}` has invalid syntax. + +invalidHttpHeaderValueTooLong=\ +HTTP Header value is invalid because it is longer than 4096 characters. \ +Value: `{0}` diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/api/evaluatorSettings.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/api/evaluatorSettings.pkl new file mode 100644 index 00000000..f9ec1c2b --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/api/evaluatorSettings.pkl @@ -0,0 +1,68 @@ +amends "../snippetTest.pkl" + +import "pkl:EvaluatorSettings" + +local function force(http: EvaluatorSettings.Http) = new PcfRenderer {}.renderValue(http) + +examples { + ["http headers"] { + new EvaluatorSettings.Http { + headers { + ["**"] { + ["X-Foo"] = "Foo" + } + ["https://example.com/**"] { + ["X-Bar"] { + "bar" + "baz" + } + } + } + } + } + + ["http headers -- invalid glob pattern"] { + module.catch(() -> force(new EvaluatorSettings.Http { + headers { + ["{{}}"] { + ["X-Foo"] = "Foo" + } + } + })) + } + + ["http headers -- invalid header name"] { + module.catch(() -> force(new EvaluatorSettings.Http { + headers { + ["**"] { + ["Connection"] = "Foo" + } + } + })) + module.catch(() -> force(new EvaluatorSettings.Http { + headers { + ["**"] { + ["Sec-Foo-Bar"] = "Foo" + } + } + })) + + module.catch(() -> force(new EvaluatorSettings.Http { + headers { + ["**"] { + ["Foo:Bar"] = "Foo" + } + } + })) + } + + ["http headers -- invalid header value"] { + module.catch(() -> force(new EvaluatorSettings.Http { + headers { + ["**"] { + ["My-Header"] = "🙃" + } + } + })) + } +} diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/api/string.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/api/string.pkl index c0ec950c..90100a81 100644 --- a/pkl-core/src/test/files/LanguageSnippetTests/input/api/string.pkl +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/api/string.pkl @@ -85,6 +85,12 @@ facts { !"(abc".isRegex } + ["isGlobPattern"] { + "**".isGlobPattern + "hello".isGlobPattern + !"{{}}".isGlobPattern + } + ["toBoolean()"] { "true".toBoolean() "tRuE".toBoolean() diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/api/evaluatorSettings.pcf b/pkl-core/src/test/files/LanguageSnippetTests/output/api/evaluatorSettings.pcf new file mode 100644 index 00000000..6b13ef40 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/api/evaluatorSettings.pcf @@ -0,0 +1,28 @@ +examples { + ["http headers"] { + new { + headers { + ["**"] { + ["X-Foo"] = "Foo" + } + ["https://example.com/**"] { + ["X-Bar"] { + "bar" + "baz" + } + } + } + } + } + ["http headers -- invalid glob pattern"] { + "Type constraint `isGlobPattern` violated. Value: \"{{}}\"" + } + ["http headers -- invalid header name"] { + "Type constraint `isNotReservedHeaderName` violated. Value: \"Connection\"" + "Type constraint `doesNotStartWithReservedPrefix` violated. Value: \"Sec-Foo-Bar\"" + "Type constraint `hasValidHeaderNameSyntax` violated. Value: \"Foo:Bar\"" + } + ["http headers -- invalid header value"] { + "Expected value of type `*Listing | HttpHeaderValue`, but got a different `String`. Value: \"🙃\"" + } +} diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/api/reflectedDeclaration.pcf b/pkl-core/src/test/files/LanguageSnippetTests/output/api/reflectedDeclaration.pcf index 39c32903..86d54173 100644 --- a/pkl-core/src/test/files/LanguageSnippetTests/output/api/reflectedDeclaration.pcf +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/api/reflectedDeclaration.pcf @@ -914,6 +914,27 @@ alias { name = "isRegex" allModifiers = Set() allAnnotations = List() + }, "isGlobPattern", new { + location { + line = XXXX + column = 3 + displayUri = "https://github.com/apple/pkl/blob/$commitId/stdlib/base.pkl#LXXXX" + } + docComment = """ + Tells if this string is a valid glob pattern. + + For reference, see + . + """ + annotations = List(new { + version = "0.32.0" + }) + modifiers = Set() + name = "isGlobPattern" + allModifiers = Set() + allAnnotations = List(new { + version = "0.32.0" + }) }, "isBase64", new { location { line = XXXX @@ -1258,6 +1279,27 @@ alias { name = "isRegex" allModifiers = Set() allAnnotations = List() + }, "isGlobPattern", new { + location { + line = XXXX + column = 3 + displayUri = "https://github.com/apple/pkl/blob/$commitId/stdlib/base.pkl#LXXXX" + } + docComment = """ + Tells if this string is a valid glob pattern. + + For reference, see + . + """ + annotations = List(new { + version = "0.32.0" + }) + modifiers = Set() + name = "isGlobPattern" + allModifiers = Set() + allAnnotations = List(new { + version = "0.32.0" + }) }, "isBase64", new { location { line = XXXX diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/api/string.pcf b/pkl-core/src/test/files/LanguageSnippetTests/output/api/string.pcf index f5b79bf0..c14daed1 100644 --- a/pkl-core/src/test/files/LanguageSnippetTests/output/api/string.pcf +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/api/string.pcf @@ -65,6 +65,11 @@ facts { true true } + ["isGlobPattern"] { + true + true + true + } ["toBoolean()"] { true true diff --git a/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorBuilderTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorBuilderTest.kt index 8baa0059..67a49fd1 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorBuilderTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorBuilderTest.kt @@ -1,5 +1,5 @@ /* - * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. + * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,13 +15,17 @@ */ package org.pkl.core +import java.net.URI import java.nio.file.Path import java.time.Duration import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows +import org.pkl.core.http.LazyHttpClient +import org.pkl.core.http.RequestRewritingClient import org.pkl.core.project.Project import org.pkl.core.resource.TestResourceReader +import org.pkl.core.util.IoUtils class EvaluatorBuilderTest { @Test @@ -93,5 +97,10 @@ class EvaluatorBuilderTest { .isEqualTo(3) // two external readers, one module path assertThat(builder.resourceReaders.find { it.uriScheme == "scheme3" }).isNotNull assertThat(builder.resourceReaders.find { it.uriScheme == "scheme4" }).isNotNull + val client = (builder.httpClient as LazyHttpClient).orCreateClient as RequestRewritingClient + assertThat(client.headers) + .isEqualTo(mapOf(IoUtils.doubleStarGlob to mapOf("X-Foo" to listOf("Foo")))) + assertThat(client.rewritesMap) + .isEqualTo(mapOf(URI("https://foo.com/") to URI("https://bar.com/"))) } } diff --git a/pkl-core/src/test/kotlin/org/pkl/core/http/HttpClientBuilderTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/http/HttpClientBuilderTest.kt new file mode 100644 index 00000000..ec3c2e80 --- /dev/null +++ b/pkl-core/src/test/kotlin/org/pkl/core/http/HttpClientBuilderTest.kt @@ -0,0 +1,46 @@ +/* + * Copyright © 2026 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.http + +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test + +class HttpClientBuilderTest { + @Test + fun `addHeader merges values for duplicate header names`() { + val client = + HttpClient.builder() + .setHeaders(mapOf("**" to mapOf("X-Foo" to listOf("v1")))) + .addHeaders("**", mapOf("X-Foo" to listOf("v2"))) + .build() as RequestRewritingClient + + val headerValues = client.headers.values.single()["X-Foo"] + assertThat(headerValues).containsExactly("v1", "v2") + } + + @Test + fun `addHeader preserves non-overlapping header names`() { + val client = + HttpClient.builder() + .addHeaders("**", mapOf("X-Foo" to listOf("v1"))) + .addHeaders("**", mapOf("X-Bar" to listOf("v2"))) + .build() as RequestRewritingClient + + val headerMap = client.headers.values.single() + assertThat(headerMap["X-Foo"]).containsExactly("v1") + assertThat(headerMap["X-Bar"]).containsExactly("v2") + } +} diff --git a/pkl-core/src/test/kotlin/org/pkl/core/http/RequestRewritingClientTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/http/RequestRewritingClientTest.kt index a1176a83..cf94a6d7 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/http/RequestRewritingClientTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/http/RequestRewritingClientTest.kt @@ -21,12 +21,13 @@ import java.net.http.HttpRequest import java.net.http.HttpRequest.BodyPublishers import java.net.http.HttpResponse.BodyHandlers import java.time.Duration -import java.util.regex.Pattern import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.assertThatList import org.junit.jupiter.api.Test -import org.pkl.core.Pair as PPair +import org.pkl.core.util.GlobResolver +import org.pkl.core.util.IoUtils +@Suppress("UastIncorrectHttpHeaderInspection") class RequestRewritingClientTest { private val captured = RequestCapturingClient() private val client = @@ -36,7 +37,7 @@ class RequestRewritingClientTest { -1, captured, mapOf(URI("https://foo/") to URI("https://bar/")), - listOf(), + mapOf(), ) private val exampleUri = URI("https://example.com/foo/bar.html") private val exampleRequest = HttpRequest.newBuilder(exampleUri).build() @@ -48,6 +49,21 @@ class RequestRewritingClientTest { assertThatList(captured.request.headers().allValues("User-Agent")).containsOnly("Pkl") } + @Test + fun `User-Agent from configured headers takes precedence`() { + val client = + RequestRewritingClient( + "Pkl", + Duration.ofSeconds(42), + -1, + captured, + mapOf(URI("https://foo/") to URI("https://bar/")), + mapOf(IoUtils.doubleStarGlob to mapOf("User-Agent" to listOf("My-User-Agent"))), + ) + client.send(exampleRequest, BodyHandlers.discarding()) + assertThatList(captured.request.headers().allValues("User-Agent")).containsOnly("My-User-Agent") + } + @Test fun `overrides existing User-Agent headers`() { val request = @@ -125,7 +141,7 @@ class RequestRewritingClientTest { fun `rewrites port 0 if test port is set`() { val captured = RequestCapturingClient() val client = - RequestRewritingClient("Pkl", Duration.ofSeconds(42), 5000, captured, mapOf(), listOf()) + RequestRewritingClient("Pkl", Duration.ofSeconds(42), 5000, captured, mapOf(), mapOf()) val request = HttpRequest.newBuilder(URI("https://example.com:0")).build() client.send(request, BodyHandlers.discarding()) @@ -307,8 +323,7 @@ class RequestRewritingClientTest { private fun rewrittenRequest(uri: String, rules: Map): String { val captured = RequestCapturingClient() - val client = - RequestRewritingClient("Pkl", Duration.ofSeconds(42), -1, captured, rules, listOf()) + val client = RequestRewritingClient("Pkl", Duration.ofSeconds(42), -1, captured, rules, mapOf()) val request = HttpRequest.newBuilder(URI(uri)).build() client.send(request, BodyHandlers.discarding()) return captured.request.uri().toString() @@ -324,12 +339,10 @@ class RequestRewritingClientTest { -1, captured, mapOf(), - listOf( - PPair(Pattern.compile("^https://example\\.com/.*"), listOf(PPair("x-one", "one"))), - PPair( - Pattern.compile("^https://example\\.com/foo/.*"), - listOf(PPair("x-two", "two-a"), PPair("x-two", "two-b")), - ), + mapOf( + GlobResolver.toRegexPattern("https://example.com/**") to mapOf("x-one" to listOf("one")), + GlobResolver.toRegexPattern("https://example.com/foo/**") to + mapOf("x-two" to listOf("two-a", "two-b")), ), ) val request = HttpRequest.newBuilder(URI("https://example.com/foo/bar")).build() @@ -350,9 +363,9 @@ class RequestRewritingClientTest { -1, captured, mapOf(), - listOf( - PPair(Pattern.compile("^https://foo\\.com/.*"), listOf(PPair("x-foo", "foo"))), - PPair(Pattern.compile("^https://bar\\.com/.*"), listOf(PPair("x-bar", "bar"))), + mapOf( + GlobResolver.toRegexPattern("https://foo.com/**") to mapOf("x-foo" to listOf("foo")), + GlobResolver.toRegexPattern("https://bar.com/**") to mapOf("x-bar" to listOf("bar")), ), ) val request = HttpRequest.newBuilder(URI("https://example.com/foo/bar")).build() @@ -373,11 +386,9 @@ class RequestRewritingClientTest { -1, captured, mapOf(), - listOf( - PPair( - Pattern.compile("^https://example\\.com/.*"), - listOf(PPair("x-foo", "rule-a"), PPair("x-foo", "rule-b")), - ) + mapOf( + GlobResolver.toRegexPattern("https://example.com/**") to + mapOf("x-foo" to listOf("rule-a", "rule-b")) ), ) val request = @@ -388,4 +399,27 @@ class RequestRewritingClientTest { assertThatList(captured.request.headers().allValues("x-foo")) .containsExactly("request", "rule-a", "rule-b") } + + @Test + fun `configured headers wins over configured user-agent header`() { + val captured = RequestCapturingClient() + val client = + RequestRewritingClient( + "Pkl", + Duration.ofSeconds(42), + -1, + captured, + mapOf(), + mapOf( + GlobResolver.toRegexPattern("https://example.com/**") to + mapOf("user-agent" to listOf("My User Agent")) + ), + ) + val request = HttpRequest.newBuilder(URI("https://example.com/foo/bar")).build() + + client.send(request, BodyHandlers.discarding()) + + assertThatList(captured.request.headers().allValues("user-agent")) + .containsExactly("My User Agent") + } } diff --git a/pkl-core/src/test/kotlin/org/pkl/core/settings/PklSettingsTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/settings/PklSettingsTest.kt index fd52e3ea..a4094ac4 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/settings/PklSettingsTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/settings/PklSettingsTest.kt @@ -26,10 +26,8 @@ import org.pkl.commons.writeString import org.pkl.core.Evaluator import org.pkl.core.ModuleSource import org.pkl.core.PObject -import org.pkl.core.Pair as PPair import org.pkl.core.evaluatorSettings.PklEvaluatorSettings import org.pkl.core.settings.PklSettings.Editor -import org.pkl.core.util.GlobResolver class PklSettingsTest { @Test @@ -67,9 +65,15 @@ class PklSettingsTest { ["https://foo.com/"] = "https://bar.com/" } headers { - ["https://foo.com/"] { + ["https://foo.com/**"] { ["x-foo"] = "bar" } + ["https://bar.com/**"] { + ["x-bar"] { + "bar" + "baz" + } + } } } """ @@ -84,8 +88,9 @@ class PklSettingsTest { listOf("example.com", "pkg.pkl-lang.org"), ), mapOf(URI("https://foo.com/") to URI("https://bar.com/")), - listOf( - PPair(GlobResolver.toRegexPattern("https://foo.com/"), listOf(PPair("x-foo", "bar"))) + mapOf( + "https://foo.com/**" to mapOf("x-foo" to listOf("bar")), + "https://bar.com/**" to mapOf("x-bar" to listOf("bar", "baz")), ), ) diff --git a/pkl-core/src/test/resources/org/pkl/core/project/project1/PklProject b/pkl-core/src/test/resources/org/pkl/core/project/project1/PklProject index af9b32df..bfb5dc97 100644 --- a/pkl-core/src/test/resources/org/pkl/core/project/project1/PklProject +++ b/pkl-core/src/test/resources/org/pkl/core/project/project1/PklProject @@ -41,4 +41,20 @@ evaluatorSettings { arguments { "with"; "args" } } } + http { + headers { + ["**"] { + ["X-Foo"] = "Foo" + } + } + rewrites { + ["https://foo.com/"] = "https://bar.com/" + } + proxy { + noProxy { + "my-no-proxy" + } + address = "http://localhost:5619" + } + } } diff --git a/pkl-executor/pkl-executor.gradle.kts b/pkl-executor/pkl-executor.gradle.kts index 232b58ae..57311269 100644 --- a/pkl-executor/pkl-executor.gradle.kts +++ b/pkl-executor/pkl-executor.gradle.kts @@ -39,6 +39,7 @@ dependencies { testImplementation(projects.pklCommonsTest) testImplementation(projects.pklCore) testImplementation(libs.slf4jSimple) + testImplementation(libs.wiremock) } // TODO why is this needed? Without this, we get error: diff --git a/pkl-executor/src/main/java/org/pkl/executor/ExecutorOptions.java b/pkl-executor/src/main/java/org/pkl/executor/ExecutorOptions.java index e81960aa..0f534dd0 100644 --- a/pkl-executor/src/main/java/org/pkl/executor/ExecutorOptions.java +++ b/pkl-executor/src/main/java/org/pkl/executor/ExecutorOptions.java @@ -25,6 +25,7 @@ import org.jspecify.annotations.Nullable; import org.pkl.executor.spi.v1.ExecutorSpiOptions; import org.pkl.executor.spi.v1.ExecutorSpiOptions2; import org.pkl.executor.spi.v1.ExecutorSpiOptions3; +import org.pkl.executor.spi.v1.ExecutorSpiOptions4; /** * Options for {@link Executor#evaluatePath}. @@ -58,6 +59,8 @@ public final class ExecutorOptions { private final Map httpRewrites; + private final Map>> httpHeaders; + private final int testPort; // -1 means disabled private final int spiOptionsVersion; // -1 means use latest @@ -92,6 +95,7 @@ public final class ExecutorOptions { private Map httpRewrites = Map.of(); private int testPort = -1; // -1 means disabled private int spiOptionsVersion = -1; // -1 means use latest + private Map>> httpHeaders = Map.of(); private Builder() {} @@ -215,6 +219,18 @@ public final class ExecutorOptions { return this; } + /** + * API equivalent of the {@code --http-header} CLI option. + * + *

This option is ignored on Pkl 0.31 and older. + * + * @since 0.32.0 + */ + public Builder httpHeaders(Map>> httpHeaders) { + this.httpHeaders = httpHeaders; + return this; + } + /** Internal test option. -1 means disabled. */ Builder testPort(int testPort) { this.testPort = testPort; @@ -242,6 +258,7 @@ public final class ExecutorOptions { certificateFiles, certificateBytes, httpRewrites, + httpHeaders, testPort, spiOptionsVersion); } @@ -291,6 +308,7 @@ public final class ExecutorOptions { List.of(), List.of(), Map.of(), + Map.of(), -1, -1); } @@ -309,6 +327,7 @@ public final class ExecutorOptions { List certificateFiles, List certificateBytes, Map httpRewrites, + Map>> httpHeaders, int testPort, int spiOptionsVersion) { @@ -325,6 +344,7 @@ public final class ExecutorOptions { this.certificateFiles = List.copyOf(certificateFiles); this.certificateBytes = List.copyOf(certificateBytes); this.httpRewrites = Map.copyOf(httpRewrites); + this.httpHeaders = Map.copyOf(httpHeaders); this.testPort = testPort; this.spiOptionsVersion = spiOptionsVersion; } @@ -425,6 +445,7 @@ public final class ExecutorOptions { && Objects.equals(certificateFiles, other.certificateFiles) && Objects.equals(certificateBytes, other.certificateBytes) && Objects.equals(httpRewrites, other.httpRewrites) + && Objects.equals(httpHeaders, other.httpHeaders) && testPort == other.testPort && spiOptionsVersion == other.spiOptionsVersion; } @@ -445,6 +466,7 @@ public final class ExecutorOptions { certificateFiles, certificateBytes, httpRewrites, + httpHeaders, testPort, spiOptionsVersion); } @@ -478,6 +500,8 @@ public final class ExecutorOptions { + certificateBytes + ", httpRewrites=" + httpRewrites + + ", httpHeaders=" + + httpHeaders + ", testPort=" + testPort + ", spiOptionsVersion=" @@ -487,7 +511,24 @@ public final class ExecutorOptions { ExecutorSpiOptions toSpiOptions() { return switch (spiOptionsVersion) { - case -1, 3 -> + case -1, 4 -> + new ExecutorSpiOptions4( + allowedModules, + allowedResources, + environmentVariables, + externalProperties, + modulePath, + rootDir, + timeout, + outputFormat, + moduleCacheDir, + projectDir, + certificateFiles, + certificateBytes, + testPort, + httpRewrites, + httpHeaders); + case 3 -> new ExecutorSpiOptions3( allowedModules, allowedResources, diff --git a/pkl-executor/src/main/java/org/pkl/executor/spi/v1/ExecutorSpiOptions4.java b/pkl-executor/src/main/java/org/pkl/executor/spi/v1/ExecutorSpiOptions4.java new file mode 100644 index 00000000..f30e677a --- /dev/null +++ b/pkl-executor/src/main/java/org/pkl/executor/spi/v1/ExecutorSpiOptions4.java @@ -0,0 +1,66 @@ +/* + * Copyright © 2026 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.executor.spi.v1; + +import java.net.URI; +import java.nio.file.Path; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import org.jspecify.annotations.Nullable; + +public class ExecutorSpiOptions4 extends ExecutorSpiOptions3 { + + private final Map>> httpHeaders; + + public ExecutorSpiOptions4( + List allowedModules, + List allowedResources, + Map environmentVariables, + Map externalProperties, + List modulePath, + @Nullable Path rootDir, + @Nullable Duration timeout, + @Nullable String outputFormat, + @Nullable Path moduleCacheDir, + @Nullable Path projectDir, + List certificateFiles, + List certificateBytes, + int testPort, + Map httpRewrites, + Map>> httpHeaders) { + super( + allowedModules, + allowedResources, + environmentVariables, + externalProperties, + modulePath, + rootDir, + timeout, + outputFormat, + moduleCacheDir, + projectDir, + certificateFiles, + certificateBytes, + testPort, + httpRewrites); + this.httpHeaders = httpHeaders; + } + + public Map>> getHttpHeaders() { + return httpHeaders; + } +} diff --git a/pkl-executor/src/test/kotlin/org/pkl/executor/EmbeddedExecutorTest.kt b/pkl-executor/src/test/kotlin/org/pkl/executor/EmbeddedExecutorTest.kt index 6b7ade01..ac664e4a 100644 --- a/pkl-executor/src/test/kotlin/org/pkl/executor/EmbeddedExecutorTest.kt +++ b/pkl-executor/src/test/kotlin/org/pkl/executor/EmbeddedExecutorTest.kt @@ -15,6 +15,15 @@ */ package org.pkl.executor +import com.github.tomakehurst.wiremock.client.WireMock.equalTo +import com.github.tomakehurst.wiremock.client.WireMock.get +import com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor +import com.github.tomakehurst.wiremock.client.WireMock.ok +import com.github.tomakehurst.wiremock.client.WireMock.stubFor +import com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo +import com.github.tomakehurst.wiremock.client.WireMock.verify +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo +import com.github.tomakehurst.wiremock.junit5.WireMockTest import java.io.File import java.net.URI import java.nio.file.Files @@ -39,6 +48,7 @@ import org.pkl.commons.test.PackageServer import org.pkl.commons.toPath import org.pkl.core.Release +@WireMockTest class EmbeddedExecutorTest { /** * An executor that uses a particular combination of ExecutorSpiOptions version, pkl-executor @@ -139,7 +149,7 @@ class EmbeddedExecutorTest { .apply { if (!exists()) missingTestFixture() } } - // a Pkl distribution that supports ExecutorSpiOptions up to v3 + // a Pkl distribution that supports ExecutorSpiOptions up to v4 private val pklDistribution2: Path by lazy { FileTestUtils.rootProjectDir .resolve( @@ -590,6 +600,30 @@ class EmbeddedExecutorTest { ) } + @Test + fun `http headers option`(@TempDir tempDir: Path, wwRuntimeInfo: WireMockRuntimeInfo) { + stubFor(get(urlEqualTo("/foo.pkl")).willReturn(ok("foo = 1"))) + val pklFile = + tempDir.resolve("test.pkl").also { + it.writeText( + """ + @ModuleInfo { minPklVersion = "0.32.0" } + module test + + res = import("${wwRuntimeInfo.httpBaseUrl}/foo.pkl") + """ + .trimIndent() + ) + } + + currentExecutor.evaluatePath(pklFile) { + allowedModules("file:", "https:", "http:") + allowedResources("prop:") + httpHeaders(mapOf("**" to mapOf("X-Foo" to listOf("Foo")))) + } + verify(getRequestedFor(urlEqualTo("/foo.pkl")).withHeader("X-Foo", equalTo("Foo"))) + } + @ParameterizedTest @MethodSource("getCompatibleTestExecutors") @DisabledOnOs(OS.WINDOWS, disabledReason = "Can't populate legacy cache dir on Windows") diff --git a/pkl-gradle/pkl-gradle.gradle.kts b/pkl-gradle/pkl-gradle.gradle.kts index ae9b2925..449d85a7 100644 --- a/pkl-gradle/pkl-gradle.gradle.kts +++ b/pkl-gradle/pkl-gradle.gradle.kts @@ -47,6 +47,7 @@ dependencies { } testImplementation(projects.pklCommonsTest) + testImplementation(libs.wiremock) } sourceSets { diff --git a/pkl-gradle/src/main/java/org/pkl/gradle/PklPlugin.java b/pkl-gradle/src/main/java/org/pkl/gradle/PklPlugin.java index ef9b347f..fcc76faf 100644 --- a/pkl-gradle/src/main/java/org/pkl/gradle/PklPlugin.java +++ b/pkl-gradle/src/main/java/org/pkl/gradle/PklPlugin.java @@ -487,6 +487,7 @@ public class PklPlugin implements Plugin { task.getTestPort().set(spec.getTestPort()); task.getHttpProxy().set(spec.getHttpProxy()); task.getHttpNoProxy().set(spec.getHttpNoProxy()); + task.getHttpHeaders().set(spec.getHttpHeaders()); task.getHttpRewrites().set(spec.getHttpRewrites()); task.getExternalModuleReaders() .set(providers.provider(() -> spec.getExternalModuleReaders().getAsMap())); diff --git a/pkl-gradle/src/main/java/org/pkl/gradle/spec/BasePklSpec.java b/pkl-gradle/src/main/java/org/pkl/gradle/spec/BasePklSpec.java index 32ca81b5..609c0392 100644 --- a/pkl-gradle/src/main/java/org/pkl/gradle/spec/BasePklSpec.java +++ b/pkl-gradle/src/main/java/org/pkl/gradle/spec/BasePklSpec.java @@ -17,6 +17,8 @@ package org.pkl.gradle.spec; import java.net.URI; import java.time.Duration; +import java.util.List; +import java.util.Map; import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.file.ConfigurableFileCollection; import org.gradle.api.file.DirectoryProperty; @@ -61,6 +63,8 @@ public interface BasePklSpec { MapProperty getHttpRewrites(); + MapProperty>> getHttpHeaders(); + NamedDomainObjectContainer getExternalModuleReaders(); NamedDomainObjectContainer getExternalResourceReaders(); diff --git a/pkl-gradle/src/main/java/org/pkl/gradle/task/BasePklTask.java b/pkl-gradle/src/main/java/org/pkl/gradle/task/BasePklTask.java index 44dac3d3..f842e6ed 100644 --- a/pkl-gradle/src/main/java/org/pkl/gradle/task/BasePklTask.java +++ b/pkl-gradle/src/main/java/org/pkl/gradle/task/BasePklTask.java @@ -25,6 +25,7 @@ import java.nio.file.Paths; import java.time.Duration; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.regex.Pattern; import java.util.stream.Collectors; import javax.inject.Inject; @@ -50,7 +51,6 @@ import org.gradle.api.tasks.PathSensitivity; import org.gradle.api.tasks.TaskAction; import org.jspecify.annotations.Nullable; import org.pkl.commons.cli.CliBaseOptions; -import org.pkl.core.Pair; import org.pkl.core.evaluatorSettings.Color; import org.pkl.gradle.spec.ExternalReaderSpec; import org.pkl.gradle.utils.PluginUtils; @@ -167,7 +167,7 @@ public abstract class BasePklTask extends DefaultTask { @Input @Optional - public abstract ListProperty>>> getHttpHeaders(); + public abstract MapProperty>> getHttpHeaders(); @Input @Optional diff --git a/pkl-gradle/src/test/kotlin/org/pkl/gradle/EvaluatorsTest.kt b/pkl-gradle/src/test/kotlin/org/pkl/gradle/EvaluatorsTest.kt index 6db78751..1f9cf90e 100644 --- a/pkl-gradle/src/test/kotlin/org/pkl/gradle/EvaluatorsTest.kt +++ b/pkl-gradle/src/test/kotlin/org/pkl/gradle/EvaluatorsTest.kt @@ -15,6 +15,15 @@ */ package org.pkl.gradle +import com.github.tomakehurst.wiremock.client.WireMock.equalTo +import com.github.tomakehurst.wiremock.client.WireMock.get +import com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor +import com.github.tomakehurst.wiremock.client.WireMock.ok +import com.github.tomakehurst.wiremock.client.WireMock.stubFor +import com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo +import com.github.tomakehurst.wiremock.client.WireMock.verify +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo +import com.github.tomakehurst.wiremock.junit5.WireMockTest import java.nio.file.Path import kotlin.io.path.readText import org.assertj.core.api.Assertions.assertThat @@ -24,6 +33,7 @@ import org.junit.jupiter.api.io.TempDir import org.pkl.commons.readString import org.pkl.commons.test.PackageServer +@WireMockTest class EvaluatorsTest : AbstractTest() { @Test fun `render Pcf`() { @@ -907,6 +917,28 @@ class EvaluatorsTest : AbstractTest() { assertThat(secondRun.output).contains(CONFIG_CACHE_REUSED) } + @Test + fun `http headers`(wwRuntimeInfo: WireMockRuntimeInfo) { + stubFor(get(urlEqualTo("/foo.pkl")).willReturn(ok("foo = 1"))) + writeBuildFile( + "pcf", + """ + httpHeaders = [ + "**": ["X-Foo": ["Foo"]] + ] + allowedModules = ["http:", "pkl:", "repl:", "file:"] + """ + .trimIndent(), + ) + writePklFile( + """ + res = import("${wwRuntimeInfo.httpBaseUrl}/foo.pkl") + """ + ) + runTask("evalTest") + verify(getRequestedFor(urlEqualTo("/foo.pkl")).withHeader("X-Foo", equalTo("Foo"))) + } + private fun writeBuildFile( // don't use `org.pkl.core.OutputFormat` // because test compile class path doesn't contain pkl-core diff --git a/pkl-server/pkl-server.gradle.kts b/pkl-server/pkl-server.gradle.kts index 99e6c095..0cbf1319 100644 --- a/pkl-server/pkl-server.gradle.kts +++ b/pkl-server/pkl-server.gradle.kts @@ -25,6 +25,7 @@ dependencies { implementation(libs.truffleApi) testImplementation(projects.pklCommonsTest) + testImplementation(libs.wiremock) } tasks.test { exclude("**/NativeServerTest.*") } diff --git a/pkl-server/src/main/kotlin/org/pkl/server/Server.kt b/pkl-server/src/main/kotlin/org/pkl/server/Server.kt index 3c75f252..8ec95b4b 100644 --- a/pkl-server/src/main/kotlin/org/pkl/server/Server.kt +++ b/pkl-server/src/main/kotlin/org/pkl/server/Server.kt @@ -1,5 +1,5 @@ /* - * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. + * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -200,6 +200,7 @@ class Server(private val transport: MessageTransport) : AutoCloseable { } message.http?.caCertificates?.let(::addCertificates) message.http?.rewrites?.let(::setRewrites) + message.http?.headers?.let(::setHeaders) buildLazily() } securityManager = diff --git a/pkl-server/src/main/kotlin/org/pkl/server/ServerMessagePackDecoder.kt b/pkl-server/src/main/kotlin/org/pkl/server/ServerMessagePackDecoder.kt index 53dbd9d7..0039cf04 100644 --- a/pkl-server/src/main/kotlin/org/pkl/server/ServerMessagePackDecoder.kt +++ b/pkl-server/src/main/kotlin/org/pkl/server/ServerMessagePackDecoder.kt @@ -1,5 +1,5 @@ /* - * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. + * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -103,7 +103,21 @@ class ServerMessagePackDecoder(unpacker: MessageUnpacker) : BaseMessagePackDecod ?.map() ?.mapKeys { URI(it.key.asStringValue().asString()) } ?.mapValues { URI(it.value.asStringValue().asString()) } - return Http(caCertificates, proxy, rewrites) + val headers = + getNullable(httpMap, "headers") + ?.asMapValue() + ?.map() + ?.mapKeys { it.key.asStringValue().asString() } + ?.mapValues { (_, value) -> + value + .asMapValue() + .map() + .mapKeys { it.key.asStringValue().asString() } + .mapValues { value -> + value.value.asArrayValue().list().map { it.asStringValue().asString() } + } + } + return Http(caCertificates, proxy, rewrites, headers) } private fun Map.unpackProxy(): Proxy? { diff --git a/pkl-server/src/main/kotlin/org/pkl/server/ServerMessagePackEncoder.kt b/pkl-server/src/main/kotlin/org/pkl/server/ServerMessagePackEncoder.kt index b5186e39..e19f6220 100644 --- a/pkl-server/src/main/kotlin/org/pkl/server/ServerMessagePackEncoder.kt +++ b/pkl-server/src/main/kotlin/org/pkl/server/ServerMessagePackEncoder.kt @@ -1,5 +1,5 @@ /* - * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. + * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -36,7 +36,7 @@ class ServerMessagePackEncoder(packer: MessagePacker) : BaseMessagePackEncoder(p } private fun MessagePacker.packHttp(http: Http) { - packMapHeader(0, http.caCertificates, http.proxy, http.rewrites) + packMapHeader(0, http.caCertificates, http.proxy, http.rewrites, http.headers) http.caCertificates?.let { packKeyValue("caCertificates", it) } http.proxy?.let { proxy -> packString("proxy") @@ -52,6 +52,21 @@ class ServerMessagePackEncoder(packer: MessagePacker) : BaseMessagePackEncoder(p packString(value.toString()) } } + http.headers?.let { headers -> + packString("headers") + packMapHeader(headers.size) + for ((pattern, headerMap) in headers) { + packString(pattern) + packMapHeader(headerMap.size) + for ((headerName, headerValue) in headerMap) { + packString(headerName) + packArrayHeader(headerValue.size) + for (elem in headerValue) { + packString(elem) + } + } + } + } } private fun MessagePacker.packDependencies(dependencies: Map) { diff --git a/pkl-server/src/main/kotlin/org/pkl/server/ServerMessages.kt b/pkl-server/src/main/kotlin/org/pkl/server/ServerMessages.kt index b53ef274..02cbf959 100644 --- a/pkl-server/src/main/kotlin/org/pkl/server/ServerMessages.kt +++ b/pkl-server/src/main/kotlin/org/pkl/server/ServerMessages.kt @@ -1,5 +1,5 @@ /* - * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. + * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -60,6 +60,8 @@ data class Http( val proxy: Proxy?, /** HTTP rewrites */ val rewrites: Map?, + /** HTTP headers */ + val headers: Map>>?, ) { override fun equals(other: Any?): Boolean { if (this === other) return true @@ -69,13 +71,16 @@ data class Http( if (other.caCertificates == null) return false if (!caCertificates.contentEquals(other.caCertificates)) return false } else if (other.caCertificates != null) return false - return Objects.equals(rewrites, other.rewrites) && Objects.equals(proxy, other.proxy) + return Objects.equals(rewrites, other.rewrites) && + Objects.equals(proxy, other.proxy) && + Objects.equals(headers, other.headers) } override fun hashCode(): Int { var result = caCertificates?.contentHashCode() ?: 0 result = 31 * result + (proxy?.hashCode() ?: 0) result = 31 * result + (rewrites?.hashCode() ?: 0) + result = 31 * result + (headers?.hashCode() ?: 0) return result } } diff --git a/pkl-server/src/test/kotlin/org/pkl/server/AbstractServerTest.kt b/pkl-server/src/test/kotlin/org/pkl/server/AbstractServerTest.kt index da3c1e77..bfc8b3b4 100644 --- a/pkl-server/src/test/kotlin/org/pkl/server/AbstractServerTest.kt +++ b/pkl-server/src/test/kotlin/org/pkl/server/AbstractServerTest.kt @@ -15,6 +15,15 @@ */ package org.pkl.server +import com.github.tomakehurst.wiremock.client.WireMock.equalTo +import com.github.tomakehurst.wiremock.client.WireMock.get +import com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor +import com.github.tomakehurst.wiremock.client.WireMock.ok +import com.github.tomakehurst.wiremock.client.WireMock.stubFor +import com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo +import com.github.tomakehurst.wiremock.client.WireMock.verify +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo +import com.github.tomakehurst.wiremock.junit5.WireMockTest import java.io.PipedInputStream import java.io.PipedOutputStream import java.net.URI @@ -30,6 +39,7 @@ import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.AfterAll import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Disabled +import org.junit.jupiter.api.Nested import org.junit.jupiter.api.Test import org.junit.jupiter.api.io.TempDir import org.msgpack.core.MessagePack @@ -1065,6 +1075,7 @@ abstract class AbstractServerTest { caCertificates = null, proxy = null, rewrites = mapOf(URI("https://example.com/") to URI("https://example.example/")), + headers = null, ) ) client.send( @@ -1092,6 +1103,7 @@ abstract class AbstractServerTest { caCertificates = null, proxy = null, rewrites = mapOf(URI("https://example.com") to URI("https://example.example/")), + headers = null, ) ) ) @@ -1100,6 +1112,37 @@ abstract class AbstractServerTest { .contains("Rewrite rule must end with '/', but was 'https://example.com'") } + @Nested + @WireMockTest + inner class HttpTests { + @Test + fun `http headers`(wwRuntimeInfo: WireMockRuntimeInfo) { + stubFor(get(urlEqualTo("/foo.pkl")).willReturn(ok("foo = 1"))) + val evaluatorId = + client.sendCreateEvaluatorRequest( + http = + Http( + caCertificates = null, + proxy = null, + rewrites = null, + headers = mapOf("**" to mapOf("X-Foo" to listOf("Foo"))), + ) + ) + client.send( + EvaluateRequest( + 1, + evaluatorId, + URI("repl:text"), + "res = import(\"${wwRuntimeInfo.httpBaseUrl}/foo.pkl\")", + "output.text", + ) + ) + val response = client.receive() + assertThat(response.error).isNull() + verify(getRequestedFor(urlEqualTo("/foo.pkl")).withHeader("X-Foo", equalTo("Foo"))) + } + } + private fun TestTransport.sendCreateEvaluatorRequest( requestId: Long = 123, resourceReaders: List = listOf(), diff --git a/pkl-server/src/test/kotlin/org/pkl/server/ServerMessagePackCodecTest.kt b/pkl-server/src/test/kotlin/org/pkl/server/ServerMessagePackCodecTest.kt index 849e40fe..c3c18092 100644 --- a/pkl-server/src/test/kotlin/org/pkl/server/ServerMessagePackCodecTest.kt +++ b/pkl-server/src/test/kotlin/org/pkl/server/ServerMessagePackCodecTest.kt @@ -1,5 +1,5 @@ /* - * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. + * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -45,7 +45,7 @@ class ServerMessagePackCodecTest { private fun roundtrip(message: Message) { encoder.encode(message) val decoded = decoder.decode() - assertThat(decoded).isEqualTo(message) + assertThat(decoded).usingRecursiveComparison().isEqualTo(message) } @Test @@ -98,6 +98,7 @@ class ServerMessagePackCodecTest { proxy = Proxy(URI("http://foo.com:1234"), listOf("bar", "baz")), caCertificates = byteArrayOf(1, 2, 3, 4), rewrites = mapOf(URI("https://foo.com/") to URI("https://bar.com/")), + headers = mapOf("**" to mapOf("X-Foo" to listOf("Foo", "Bar"))), ), externalModuleReaders = mapOf("external" to externalReader, "external2" to externalReader), externalResourceReaders = mapOf("external" to externalReader), diff --git a/stdlib/EvaluatorSettings.pkl b/stdlib/EvaluatorSettings.pkl index ab9472d4..33c608e2 100644 --- a/stdlib/EvaluatorSettings.pkl +++ b/stdlib/EvaluatorSettings.pkl @@ -128,9 +128,6 @@ local const hasNonEmptyHostname = (it: String) -> @Since { version = "0.29.0" } typealias HttpRewrite = String(startsWith(Regex("https?://")), endsWith("/"), hasNonEmptyHostname) -@Since { version = "0.32.0" } -typealias UrlPattern = String(endsWith(Regex("[/*]"))) - /// Settings that control how Pkl talks to HTTP(S) servers. class Http { /// Configuration of the HTTP proxy to use. @@ -173,9 +170,49 @@ class Http { @Since { version = "0.29.0" } rewrites: Mapping? - /// HTTP headers to add to outbound requests targeting specified URLs. + /// HTTP headers to add to outbound requests. + /// + /// Each key is a glob pattern, and each value is a mapping of header name to header value(s). + /// + /// Before an HTTP request is made, each key is matched against the request URL. + /// If any matches are found, each of their described headers are added to the request. + /// + /// To add headers to all HTTP requests, use `**` as the glob pattern. + /// + /// Example: + /// + /// ``` + /// headers { + /// // Add this user agent to all out-bound requests. + /// ["**"] { + /// ["User-Agent"] = "MyApp-Pkl" + /// } + /// + /// // Add an authorization header to all requests made to the + /// // `https://my.internal.service` host. + /// ["https://my.internal.service/**"] { + /// ["Authorization"] = "Bearer abc123" + /// } + /// } + /// ``` + /// + /// To repeat a header multiple times, set the value to a listing: + /// + /// ``` + /// headers { + /// // Adds: + /// // My-Custom-Header: foo + /// // My-Custom-Header: bar + /// ["**"] { + /// ["My-Custom-Header"] { "foo"; "bar" } + /// } + /// } + /// ``` + /// + /// For details on glob patterns, see + /// . @Since { version = "0.32.0" } - headers: Mapping | HttpHeaderValue>>? + headers: Mapping? } /// Settings that control how Pkl talks to HTTP proxies. @@ -243,50 +280,63 @@ class ExternalReader { arguments: Listing? } -@Since { version = "0.32.0" } -typealias ReservedHttpHeaderName = - "accept-charset" - | "accept-encoding" - | "connection" +local typealias ReservedHttpHeaderName = + "connection" | "content-length" - | "cookie" - | "date" - | "dnt" | "expect" | "host" | "keep-alive" - | "origin" - | "permissions-policy" - | "referer" | "te" | "trailer" | "transfer-encoding" | "upgrade" - | "via" -local const ReservedHttpHeaderPrefix = new Listing { - "proxy-" - "sec-" - "access-control-" -} +local const reservedHttpHeaderPrefix = + Set( + "proxy-", + "sec-", + ) -local const hasReservedHttpHeaderPrefix = (header: String) -> - ReservedHttpHeaderPrefix.any((it) -> header.startsWith(it)) +local const isNotReservedHeaderName = (header: String) -> + !(header.toLowerCase() is ReservedHttpHeaderName) -local const httpHeaderNameRegex = Regex("^[a-zA-Z0-9!#\\$%&'*+-.^_`|~]+$") -local const hasValidHttpHeaderName = (header: String) -> - !httpHeaderNameRegex.findMatchesIn(header).isEmpty +local const doesNotStartWithReservedPrefix = (header: String) -> + !reservedHttpHeaderPrefix.any((it) -> header.toLowerCase().startsWith(it)) +local const hasValidHeaderNameSyntax = (header: String) -> + header.matches(Regex(#"[a-zA-Z0-9!#$%&'*+-.^_`|~]+"#)) + +/// A mapping of header names to header value(s). +/// +/// The mapping must have distinct header keys (case insensitive). +@Since { version = "0.32.0" } +typealias HttpHeaders = + Mapping | HttpHeaderValue>( + keys.toList().isDistinctBy((it) -> it.toLowerCase()), + ) + +/// An HTTP header name. +/// +/// It conforms to the following rules: +/// +/// * It is not one of "connection", "keep-alive", "content-length", "expect", "host", "upgrade", +/// "te", "transfer-encoding", "trailer" (case insensitive). +/// * It does not start with "proxy-" nor "sec-". +/// * It consists of a-z, A-Z, or a character in "!#$%&'*+-.^_`|~". +/// * It is not a blank string. @Since { version = "0.32.0" } typealias HttpHeaderName = String( - this == toLowerCase(), - !(this is ReservedHttpHeaderName), - !hasReservedHttpHeaderPrefix.apply(this), - hasValidHttpHeaderName, + isNotReservedHeaderName, + doesNotStartWithReservedPrefix, + hasValidHeaderNameSyntax, + isNotBlank, ) -local const httpHeaderValueRegex = Regex("^[\\t\\u0020-\\u007E\\u0080-\\u00FF]*$") - +/// An HTTP header value. +/// +/// It consists of ASCII characters between the range of 0x20 - 0x7E, and 0x80 - 0xFF (all visible +/// ASCII characters), and also the tab character. @Since { version = "0.32.0" } -typealias HttpHeaderValue = String(!httpHeaderValueRegex.findMatchesIn(this).isEmpty) +typealias HttpHeaderValue = + String(matches(Regex(#"[\t\u0020-\u007E\u0080-\u00FF]*"#)), length < 4096) diff --git a/stdlib/base.pkl b/stdlib/base.pkl index 3c9b22f3..a684ad6a 100644 --- a/stdlib/base.pkl +++ b/stdlib/base.pkl @@ -1323,6 +1323,13 @@ external class String extends Any { /// Tells if this string is a valid regular expression according to [Regex]. external isRegex: Boolean + /// Tells if this string is a valid glob pattern. + /// + /// For reference, see + /// . + @Since { version = "0.32.0" } + external isGlobPattern: Boolean + /// Tells if this is a valid base64-encoded string. /// /// Facts: