Improve HTTP headers logic (#1584)

* Relax forbidden headers constraints
  - remove restriction on browser-related headers
- allow any glob pattern (no need to end with `/` or `*`, because glob
patterns already require users to explicitly declare prefix matches if
that's the intention)
* Replace `List<Pair<, ...>>`; use `Map<String, ...>` instead
* Use glob pattern strings as an API throughout, instead of `Pattern`
(e.g. in `HttpClientBuilder`)
* Add HTTP headers to message passing API
* Add HTTP headers to executor API (introduces `ExecutorSpiOptions4`)
* Add tests for Gradle, CLI, and pkl-executor invocations
* Improve documentation
* Add `isGlobPattern` API to class `String` for in-language validation
of http headers
* Behavior change: make sure explicitly configured `User-Agent` in
`HttpClientBuilder` can be shadowed by headers (allows users to set
`--http-header "**=User-Agent: My User Agent"` and for this to be the
only user agent).

CC @kyokuping
This commit is contained in:
Daniel Chao
2026-05-21 20:07:06 -07:00
committed by GitHub
parent 87ea28260b
commit 8e2e5e4ba8
48 changed files with 1067 additions and 222 deletions
@@ -210,6 +210,11 @@ class Http {
/// ///
/// Each rewrite must start with `http://` or `https://`, and must end with `/`. /// Each rewrite must start with `http://` or `https://`, and must end with `/`.
rewrites: Mapping<String, String>? rewrites: Mapping<String, String>?
/// HTTP headers.
/// Each entry key is a glob pattern that is matched against outbound request URLs.
/// Each value is a map of headers that is added to the matching request.
headers: Mapping<String, Mapping<String, Listing<String>>>?
} }
/// Settings that control how Pkl talks to HTTP proxies. /// Settings that control how Pkl talks to HTTP proxies.
@@ -5059,8 +5059,6 @@ in the context of that module.
[[glob-patterns]] [[glob-patterns]]
=== Glob Patterns === Glob Patterns
Resources and modules may be imported at the same time by globbing with the <<globbed-imports>> and <<globbed-reads>> features.
Pkl's glob patterns mostly follow the rules described by link:{uri-glob-7}[glob(7)], with the following differences: Pkl's glob patterns mostly follow the rules described by link:{uri-glob-7}[glob(7)], with the following differences:
* `*` includes names that start with a dot (`.`). * `*` includes names that start with a dot (`.`).
@@ -158,6 +158,23 @@ This option is commonly used to enable package mirroring.
The above example will rewrite URL `\https://pkg.pkl-lang.org/pkl-k8s/k8s@1.0.0` to `\https://my.internal.mirror/pkl-k8s/k8s@1.0.0`. The above example will rewrite URL `\https://pkg.pkl-lang.org/pkl-k8s/k8s@1.0.0` to `\https://my.internal.mirror/pkl-k8s/k8s@1.0.0`.
==== ====
.--http-header
[%collapsible]
====
Default: (none) +
Example: `**=User-Agent: My User Agent` +
Additional headers to add to outbound HTTP requests.
The syntax is `<glob pattern>=<header name>:<header value>`, and any whitespace after the leading colon is trimmed.
The glob pattern is used to match against the URLs of outbound HTTP calls, and the value is the header name and value to add.
In the case of multiple matches, every header is added.
To describe a prefix match, the `\\**` wildcard should be added to the pattern.
For example, `https?://example.com/**` matches against every request to host `example.com`.
====
.--trace-mode .--trace-mode
[%collapsible] [%collapsible]
==== ====
@@ -119,6 +119,19 @@ This option is commonly used to enable package mirroring.
The above example will rewrite URL `\https://pkg.pkl-lang.org/pkl-k8s/k8s@1.0.0` to `\https://my.internal.mirror/pkl-k8s/k8s@1.0.0`. The above example will rewrite URL `\https://pkg.pkl-lang.org/pkl-k8s/k8s@1.0.0` to `\https://my.internal.mirror/pkl-k8s/k8s@1.0.0`.
==== ====
.httpHeaders: MapProperty<String, Map<String, List<String>>>
[%collapsible]
====
Default: `null` +
Example: `httpHeaders = ["**": ["User-Agent": ["My User Agent"]]]` +
Additional headers to add to outbound HTTP requests.
The key is a glob pattern that is used to match against the URLs of outbound HTTP calls, and the value is the header name and values to add.
Each header value becomes its own HTTP header.
To describe a prefix match, the `\\**` wildcard should be added to the pattern.
For example, `https?://example.com/**` matches against every request to host `example.com`.
====
.powerAssertions: Property<Boolean> .powerAssertions: Property<Boolean>
[%collapsible] [%collapsible]
==== ====
@@ -1614,6 +1614,24 @@ result = someLib.x
assertThat(output).isEqualTo("result = 1\n") assertThat(output).isEqualTo("result = 1\n")
} }
@Test
fun `eval configured http headers`(wwRuntimeInfo: WireMockRuntimeInfo) {
stubFor(get(anyUrl()).willReturn(ok("result = 1")))
val file = URI("${wwRuntimeInfo.httpBaseUrl}/foo.pkl")
val output =
evalToConsole(
CliEvaluatorOptions(
CliBaseOptions(
sourceModules = listOf(file),
httpHeaders = mapOf("**" to mapOf("X-Foo" to listOf("Foo"))),
allowedModules =
listOf(Pattern.compile("http:"), Pattern.compile("file:"), Pattern.compile("pkl:")),
)
)
)
verify(getRequestedFor(urlEqualTo("/foo.pkl")).withHeader("X-Foo", equalTo("Foo")))
}
@Test @Test
fun `eval file with non-ASCII name`() { fun `eval file with non-ASCII name`() {
val tempDirUri = tempDir.toUri() val tempDirUri = tempDir.toUri()
@@ -169,4 +169,60 @@ class CliMainTest {
link.createSymbolicLinkPointingTo(dir) link.createSymbolicLinkPointingTo(dir)
return link return link
} }
@Test
fun `invalid http header glob pattern`() {
val ex =
assertThrows<BadParameterValue> {
rootCmd.parse(arrayOf("eval", "--http-header", "foo{{}}=bar:baz", "myModule.pkl"))
}
assertThat(ex.message)
.contains("Sub-patterns cannot be nested. To fix, remove or escape the inner `{` character.")
}
@Test
fun `forbidden http header name`() {
val ex =
assertThrows<BadParameterValue> {
rootCmd.parse(arrayOf("eval", "--http-header", "**=Connection: baz", "myModule.pkl"))
}
assertThat(ex.message).contains("HTTP header `Connection` is a reserved header")
}
@Test
fun `bad http header value`() {
val ex =
assertThrows<BadParameterValue> {
rootCmd.parse(arrayOf("eval", "--http-header", "**=X-Foo:🙃", "myModule.pkl"))
}
assertThat(ex.message).contains("HTTP header value `🙃` has invalid syntax")
}
@Test
fun `multiple headers`() {
val cmd = RootCommand()
cmd.parse(
arrayOf(
"eval",
"--http-header",
"**=X-Foo:Foo",
"--http-header",
"**=X-Foo:Foo2",
"--http-header",
"**=X-Bar:Bar",
"--http-header",
"https://example.com/**=X-Qux:Qux",
"pkl:base",
)
)
val evalCmd = cmd.registeredSubcommands().filterIsInstance<EvalCommand>().first()
assertThat(evalCmd.baseOptions.httpHeaders)
.isEqualTo(
mapOf(
"**" to mapOf("X-Foo" to listOf("Foo", "Foo2"), "X-Bar" to listOf("Bar")),
"https://example.com/**" to mapOf("X-Qux" to listOf("Qux")),
)
)
}
} }
@@ -20,7 +20,6 @@ import java.nio.file.Files
import java.nio.file.Path import java.nio.file.Path
import java.time.Duration import java.time.Duration
import java.util.regex.Pattern import java.util.regex.Pattern
import org.pkl.core.Pair
import org.pkl.core.evaluatorSettings.Color import org.pkl.core.evaluatorSettings.Color
import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader
import org.pkl.core.evaluatorSettings.TraceMode import org.pkl.core.evaluatorSettings.TraceMode
@@ -146,7 +145,7 @@ data class CliBaseOptions(
val httpRewrites: Map<URI, URI>? = null, val httpRewrites: Map<URI, URI>? = null,
/** HTTP headers to add to the request. */ /** HTTP headers to add to the request. */
val httpHeaders: List<Pair<Pattern, List<Pair<String, String>>>>? = null, val httpHeaders: Map<String, Map<String, List<String>>>? = null,
/** External module reader process specs */ /** External module reader process specs */
val externalModuleReaders: Map<String, ExternalReader> = mapOf(), val externalModuleReaders: Map<String, ExternalReader> = mapOf(),
@@ -185,11 +185,11 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
} }
protected val httpRewrites: Map<URI, URI>? by lazy { protected val httpRewrites: Map<URI, URI>? by lazy {
cliOptions.httpRewrites ?: evaluatorSettings?.http?.rewrites ?: settings.http?.rewrites() cliOptions.httpRewrites ?: evaluatorSettings?.http?.rewrites ?: settings.http?.rewrites
} }
private val httpHeaders: List<Pair<Pattern, List<Pair<String, String>>>>? by lazy { private val httpHeaders: Map<String, Map<String, List<String>>>? by lazy {
cliOptions.httpHeaders ?: project?.evaluatorSettings?.http?.headers ?: settings.http?.headers cliOptions.httpHeaders ?: evaluatorSettings?.http?.headers ?: settings.http?.headers
} }
protected val externalModuleReaders: Map<String, PklEvaluatorSettings.ExternalReader> by lazy { protected val externalModuleReaders: Map<String, PklEvaluatorSettings.ExternalReader> by lazy {
@@ -31,7 +31,6 @@ import java.util.regex.Pattern
import org.pkl.commons.cli.CliBaseOptions import org.pkl.commons.cli.CliBaseOptions
import org.pkl.commons.cli.CliException import org.pkl.commons.cli.CliException
import org.pkl.commons.shlex import org.pkl.commons.shlex
import org.pkl.core.Pair as PPair
import org.pkl.core.evaluatorSettings.Color import org.pkl.core.evaluatorSettings.Color
import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader
import org.pkl.core.evaluatorSettings.TraceMode import org.pkl.core.evaluatorSettings.TraceMode
@@ -95,6 +94,8 @@ class BaseOptions : OptionGroup() {
Pair(it.first, ExternalReader(cmd.first(), cmd.drop(1))) Pair(it.first, ExternalReader(cmd.first(), cmd.drop(1)))
} }
} }
private val headerRegex = Regex("""^(.+?):[ \t]*(.*)$""")
} }
private val defaults = CliBaseOptions() private val defaults = CliBaseOptions()
@@ -287,35 +288,35 @@ class BaseOptions : OptionGroup() {
.multiple() .multiple()
.toMap() .toMap()
val httpHeaders: List<PPair<Pattern, List<PPair<String, String>>>> by val httpHeaders: Map<String, Map<String, List<String>>> by
option( option(
names = arrayOf("--http-headers"), names = arrayOf("--http-header"),
metavar = "<url-pattern>=<header name>:<header value>", metavar = "<url-pattern>=<header name>:<header value>",
help = "HTTP header to add to the request.", help = "HTTP header to add to the request.",
) )
.splitPair() .splitPair()
.transformAll { it -> .transformAll { pairs ->
val headersMap = mutableMapOf<String, MutableList<PPair<String, String>>>() buildMap<String, MutableMap<String, MutableList<String>>> {
for ((pattern, headerNameAndValue) in pairs) {
try { try {
val headerRegex = Regex("""^(.+?):[ \t]*(.+)$""") GlobResolver.toRegexPattern(pattern)
for ((stringPattern, header) in it) {
val (headerName, headerValue) =
headerRegex.find(header)?.destructured
?: fail("Header '$header' is not in 'name:value' format.")
IoUtils.validateHeaderName(headerName)
IoUtils.validateHeaderValue(headerValue)
headersMap
.computeIfAbsent(stringPattern) { mutableListOf() }
.add(PPair(headerName, headerValue))
}
headersMap.entries.map { PPair(GlobResolver.toRegexPattern(it.key), it.value) }
} catch (e: IllegalArgumentException) {
fail(e.message!!)
} catch (e: GlobResolver.InvalidGlobPatternException) { } catch (e: GlobResolver.InvalidGlobPatternException) {
fail(e.message!!) fail(e.message!!)
} }
val (headerName, headerValue) =
headerRegex.find(headerNameAndValue)?.destructured
?: fail("Header '$headerNameAndValue' is not in 'name:value' format.")
try {
IoUtils.validateHeaderName(headerName)
IoUtils.validateHeaderValue(headerValue)
} catch (e: IllegalArgumentException) {
fail(e.message!!)
}
getOrPut(pattern) { mutableMapOf() }
.getOrPut(headerName) { mutableListOf() }
.add(headerValue)
}
}
} }
val externalModuleReaders: Map<String, ExternalReader> by val externalModuleReaders: Map<String, ExternalReader> by
@@ -557,6 +557,9 @@ public final class EvaluatorBuilder {
if (settings.http().rewrites() != null) { if (settings.http().rewrites() != null) {
httpClientBuilder.setRewrites(settings.http().rewrites()); httpClientBuilder.setRewrites(settings.http().rewrites());
} }
if (settings.http().headers() != null) {
httpClientBuilder.setHeaders(settings.http().headers());
}
setHttpClient(httpClientBuilder.buildLazily()); setHttpClient(httpClientBuilder.buildLazily());
} }
@@ -18,9 +18,9 @@ package org.pkl.core.evaluatorSettings;
import java.net.URI; import java.net.URI;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
@@ -28,18 +28,14 @@ import java.util.Objects;
import java.util.function.BiFunction; import java.util.function.BiFunction;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.jspecify.annotations.Nullable; import org.jspecify.annotations.Nullable;
import org.pkl.core.Duration; import org.pkl.core.Duration;
import org.pkl.core.PNull; import org.pkl.core.PNull;
import org.pkl.core.PObject; import org.pkl.core.PObject;
import org.pkl.core.Pair;
import org.pkl.core.PklBugException; import org.pkl.core.PklBugException;
import org.pkl.core.PklException; import org.pkl.core.PklException;
import org.pkl.core.Value; import org.pkl.core.Value;
import org.pkl.core.util.ErrorMessages; import org.pkl.core.util.ErrorMessages;
import org.pkl.core.util.GlobResolver;
import org.pkl.core.util.GlobResolver.InvalidGlobPatternException;
/** Java version of {@code pkl.EvaluatorSettings}. */ /** Java version of {@code pkl.EvaluatorSettings}. */
public record PklEvaluatorSettings( public record PklEvaluatorSettings(
@@ -134,19 +130,28 @@ public record PklEvaluatorSettings(
public record Http( public record Http(
@Nullable Proxy proxy, @Nullable Proxy proxy,
@Nullable Map<URI, URI> rewrites, @Nullable Map<URI, URI> rewrites,
@Nullable List<Pair<Pattern, List<Pair<String, String>>>> headers) { @Nullable Map<String, Map<String, List<String>>> headers) {
public static final Http DEFAULT = new Http(null, Collections.emptyMap(), null); public static final Http DEFAULT = new Http(null, Collections.emptyMap(), null);
@SuppressWarnings("unchecked")
public static @Nullable Http parse(@Nullable Value input) { public static @Nullable Http parse(@Nullable Value input) {
if (input == null || input instanceof PNull) { if (input == null || input instanceof PNull) {
return null; return null;
} else if (input instanceof PObject http) { } else if (input instanceof PObject http) {
var proxy = Proxy.parse((Value) http.getProperty("proxy")); var proxy = Proxy.parse((Value) http.getProperty("proxy"));
var rewrites = http.getProperty("rewrites"); var parsedRewrites = parseHttpRewrites(http.getProperty("rewrites"));
HashMap<URI, URI> parsedRewrites = null; var parsedHeaders = parseHttpHeaders(http.getProperty("headers"));
if (!(rewrites instanceof PNull)) { return new Http(proxy, parsedRewrites, parsedHeaders);
parsedRewrites = new HashMap<>(); } else {
throw PklBugException.unreachableCode();
}
}
@SuppressWarnings("unchecked")
private static @Nullable Map<URI, URI> parseHttpRewrites(Object rewrites) {
if (rewrites instanceof PNull) {
return null;
}
var parsedRewrites = new HashMap<URI, URI>();
for (var entry : ((Map<String, String>) rewrites).entrySet()) { for (var entry : ((Map<String, String>) rewrites).entrySet()) {
var key = entry.getKey(); var key = entry.getKey();
var value = entry.getValue(); var value = entry.getValue();
@@ -156,40 +161,29 @@ public record PklEvaluatorSettings(
throw new PklException(ErrorMessages.create("invalidUri", e.getInput())); throw new PklException(ErrorMessages.create("invalidUri", e.getInput()));
} }
} }
return parsedRewrites;
} }
var headerDefs = http.getProperty("headers");
List<Pair<Pattern, List<Pair<String, String>>>> parsedHeaderDefs = null; @SuppressWarnings("unchecked")
if (!(headerDefs instanceof PNull)) { private static @Nullable Map<String, Map<String, List<String>>> parseHttpHeaders(
parsedHeaderDefs = new ArrayList<>(); Object headerDefs) {
var headerDefsMap = (Map<String, Map<String, Object>>) headerDefs; if (headerDefs instanceof PNull) {
for (var entry : headerDefsMap.entrySet()) { return null;
var stringPattern = entry.getKey(); }
var headersMap = entry.getValue(); var defs = (Map<String, Map<String, Object>>) headerDefs;
try { var ret = new LinkedHashMap<String, Map<String, List<String>>>(defs.size());
var urlPattern = GlobResolver.toRegexPattern(stringPattern); for (var entry : defs.entrySet()) {
var pairs = var headers = entry.getValue();
headersMap.entrySet().stream() var map = new LinkedHashMap<String, List<String>>(headers.size());
.flatMap( for (var header : headers.entrySet()) {
header -> {
var value = header.getValue(); var value = header.getValue();
if (value instanceof List) { var headerValues =
return ((List<String>) value) value instanceof List<?> ? (List<String>) value : List.of((String) value);
.stream().map(v -> new Pair(header.getKey(), v)); map.put(header.getKey(), headerValues);
} else {
return Stream.of(new Pair(header.getKey(), value));
} }
}) ret.put(entry.getKey(), map);
.toList();
parsedHeaderDefs.add(new Pair(urlPattern, pairs));
} catch (InvalidGlobPatternException e) {
throw new PklException(ErrorMessages.create("invalidUri", stringPattern));
}
}
}
return new Http(proxy, parsedRewrites, parsedHeaderDefs);
} else {
throw PklBugException.unreachableCode();
} }
return ret;
} }
} }
@@ -23,10 +23,8 @@ import java.net.http.HttpTimeoutException;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.regex.Pattern;
import javax.net.ssl.SSLContext; import javax.net.ssl.SSLContext;
import org.jspecify.annotations.Nullable; import org.jspecify.annotations.Nullable;
import org.pkl.core.Pair;
/** /**
* An HTTP client. * An HTTP client.
@@ -47,6 +45,9 @@ public interface HttpClient extends AutoCloseable {
* Sets the {@code User-Agent} header. * Sets the {@code User-Agent} header.
* *
* <p>Defaults to {@code "Pkl/$version ($os; $flavor)"}. * <p>Defaults to {@code "Pkl/$version ($os; $flavor)"}.
*
* <p>An existing "User-Agent" from {@link Builder#setHeaders} and {@link Builder#addHeaders}
* takes precedence over this field.
*/ */
Builder setUserAgent(String userAgent); Builder setUserAgent(String userAgent);
@@ -146,7 +147,6 @@ public interface HttpClient extends AutoCloseable {
/** /**
* Adds a rewrite rule. * Adds a rewrite rule.
* *
* @see Builder#setRewrites(Map)
* @throws IllegalArgumentException if {@code sourcePrefix} or {@code targetPrefix} is invalid. * @throws IllegalArgumentException if {@code sourcePrefix} or {@code targetPrefix} is invalid.
* @since 0.29.0 * @since 0.29.0
*/ */
@@ -157,8 +157,35 @@ public interface HttpClient extends AutoCloseable {
* *
* <p>This method clears all existing headers and replaces them with the contents of the * <p>This method clears all existing headers and replaces them with the contents of the
* provided map. * provided map.
*
* <p>{@code headerRules} is a map whose keys are <a
* href="https://pkl-lang.org/main/current/language-reference/index.html#glob-patterns">glob
* patterns</a>, and values is a map of header names and values. Multiple header values turn
* into multiple individual headers in the HTTP request.
*
* <p>To add headers to all requests, use {@code **} as the glob pattern.
*
* <p>To describe a prefix match, add {@code **} to the glob pattern (e.g. {@code
* https://example.com/**}).
*
* <p>Before an HTTP request is made, each key is matched against the request URL. If any
* matches are found, each of their headers are added to the request.
*
* @throws IllegalArgumentException if any of the keys are invalid glob patterns, or if any of
* the header names or values are invalid.
* @since 0.32.0
*/ */
Builder setHeaders(List<Pair<Pattern, List<Pair<String, String>>>> headers); Builder setHeaders(Map<String, Map<String, List<String>>> headerRules);
/**
* Adds HTTP headers for URL requests that match {@code globPattern}.
*
* @throws IllegalArgumentException if {@code globPattern} is an invalid glob pattern, or if any
* of the header names or values are invalid.
* @since 0.32.0
* @see Builder#setHeaders
*/
Builder addHeaders(String globPattern, Map<String, List<String>> headers);
/** /**
* Creates a new {@code HttpClient} from the current state of this builder. * Creates a new {@code HttpClient} from the current state of this builder.
@@ -24,14 +24,17 @@ import java.nio.file.Path;
import java.time.Duration; import java.time.Duration;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.function.Supplier; import java.util.function.Supplier;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.jspecify.annotations.Nullable; import org.jspecify.annotations.Nullable;
import org.pkl.core.Pair;
import org.pkl.core.Release; import org.pkl.core.Release;
import org.pkl.core.http.HttpClient.Builder; import org.pkl.core.http.HttpClient.Builder;
import org.pkl.core.util.GlobResolver;
import org.pkl.core.util.GlobResolver.InvalidGlobPatternException;
import org.pkl.core.util.IoUtils;
final class HttpClientBuilder implements HttpClient.Builder { final class HttpClientBuilder implements HttpClient.Builder {
private String userAgent; private String userAgent;
@@ -42,7 +45,10 @@ final class HttpClientBuilder implements HttpClient.Builder {
private int testPort = -1; private int testPort = -1;
private @Nullable ProxySelector proxySelector; private @Nullable ProxySelector proxySelector;
private Map<URI, URI> rewrites = new HashMap<>(); private Map<URI, URI> rewrites = new HashMap<>();
private List<Pair<Pattern, List<Pair<String, String>>>> headers = new ArrayList<>(); // okay to use Pattern as a map key here because `GlobResolver.toRegexPattern()` caches and
// gives the same `Pattern` instance for an existing glob pattern.
// use LinkedHashMap to preserve insertion order.
private Map<Pattern, Map<String, List<String>>> headers = new LinkedHashMap<>();
HttpClientBuilder() { HttpClientBuilder() {
var release = Release.current(); var release = Release.current();
@@ -115,11 +121,54 @@ final class HttpClientBuilder implements HttpClient.Builder {
} }
@Override @Override
public Builder setHeaders(List<Pair<Pattern, List<Pair<String, String>>>> headers) { public Builder setHeaders(Map<String, Map<String, List<String>>> headers) {
this.headers = headers; var newHeaders = new LinkedHashMap<Pattern, Map<String, List<String>>>(headers.size());
for (var rule : headers.entrySet()) {
Pattern pattern;
try {
pattern = GlobResolver.toRegexPattern(rule.getKey());
} catch (InvalidGlobPatternException e) {
throw new IllegalArgumentException(e.getMessage(), e);
}
var map = new LinkedHashMap<String, List<String>>();
for (var entry : rule.getValue().entrySet()) {
IoUtils.validateHeaderName(entry.getKey());
for (var value : entry.getValue()) {
IoUtils.validateHeaderValue(value);
}
map.put(entry.getKey(), new ArrayList<>(entry.getValue()));
}
newHeaders.put(pattern, map);
}
this.headers = newHeaders;
return this; return this;
} }
@Override
public Builder addHeaders(String globPattern, Map<String, List<String>> headers) {
try {
var pattern = GlobResolver.toRegexPattern(globPattern);
var existingHeaders = this.headers.computeIfAbsent(pattern, k -> new HashMap<>());
for (var entry : headers.entrySet()) {
var headerName = entry.getKey();
var headerValues = entry.getValue();
IoUtils.validateHeaderName(headerName);
for (var value : headerValues) {
IoUtils.validateHeaderValue(value);
}
var existingList = existingHeaders.putIfAbsent(headerName, new ArrayList<>(headerValues));
if (existingList != null) {
existingList.addAll(headerValues);
}
}
return this;
} catch (InvalidGlobPatternException e) {
throw new IllegalArgumentException(e.getMessage(), e);
}
}
@Override @Override
public HttpClient build() { public HttpClient build() {
return doBuild().get(); return doBuild().get();
@@ -29,8 +29,9 @@ import org.jspecify.annotations.Nullable;
* An {@code HttpClient} decorator that defers creating the underlying HTTP client until the first * An {@code HttpClient} decorator that defers creating the underlying HTTP client until the first
* send. * send.
*/ */
// visible for testing
@ThreadSafe @ThreadSafe
final class LazyHttpClient implements HttpClient { public final class LazyHttpClient implements HttpClient {
private final Supplier<HttpClient> supplier; private final Supplier<HttpClient> supplier;
private final Object lock = new Object(); private final Object lock = new Object();
@@ -55,7 +56,8 @@ final class LazyHttpClient implements HttpClient {
getClient().ifPresent(HttpClient::close); getClient().ifPresent(HttpClient::close);
} }
private HttpClient getOrCreateClient() { // visible for testing
public HttpClient getOrCreateClient() {
synchronized (lock) { synchronized (lock) {
// only try to create client once // only try to create client once
if (exception != null) { if (exception != null) {
@@ -22,6 +22,7 @@ import java.net.http.HttpRequest;
import java.net.http.HttpResponse; import java.net.http.HttpResponse;
import java.net.http.HttpResponse.BodyHandler; import java.net.http.HttpResponse.BodyHandler;
import java.time.Duration; import java.time.Duration;
import java.util.ArrayList;
import java.util.Comparator; import java.util.Comparator;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@@ -29,12 +30,12 @@ import java.util.Map.Entry;
import java.util.Objects; import java.util.Objects;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import java.util.stream.Stream;
import javax.annotation.concurrent.ThreadSafe; import javax.annotation.concurrent.ThreadSafe;
import org.jspecify.annotations.Nullable; import org.jspecify.annotations.Nullable;
import org.pkl.core.Pair;
import org.pkl.core.PklBugException; import org.pkl.core.PklBugException;
import org.pkl.core.util.HttpUtils; import org.pkl.core.util.HttpUtils;
import org.pkl.core.util.IoUtils;
import org.pkl.core.util.Pair;
/** /**
* An {@code HttpClient} decorator that * An {@code HttpClient} decorator that
@@ -49,15 +50,17 @@ import org.pkl.core.util.HttpUtils;
* <p>Both {@code User-Agent} header and default request timeout are configurable through {@link * <p>Both {@code User-Agent} header and default request timeout are configurable through {@link
* HttpClient.Builder}. * HttpClient.Builder}.
*/ */
// visible for testing
@ThreadSafe @ThreadSafe
final class RequestRewritingClient implements HttpClient { public final class RequestRewritingClient implements HttpClient {
// non-private for testing // non-private for testing
final String userAgent; final String userAgent;
final Duration requestTimeout; final Duration requestTimeout;
final int testPort; final int testPort;
final HttpClient delegate; final HttpClient delegate;
private final Map<URI, URI> rewritesMap;
private final List<Entry<URI, URI>> rewrites; private final List<Entry<URI, URI>> rewrites;
private final List<Pair<Pattern, List<Pair<String, String>>>> headers; private final Map<Pattern, Map<String, List<String>>> headers;
private final AtomicBoolean closed = new AtomicBoolean(); private final AtomicBoolean closed = new AtomicBoolean();
@@ -67,11 +70,12 @@ final class RequestRewritingClient implements HttpClient {
int testPort, int testPort,
HttpClient delegate, HttpClient delegate,
Map<URI, URI> rewrites, Map<URI, URI> rewrites,
List<Pair<Pattern, List<Pair<String, String>>>> headers) { Map<Pattern, Map<String, List<String>>> headers) {
this.userAgent = userAgent; this.userAgent = userAgent;
this.requestTimeout = requestTimeout; this.requestTimeout = requestTimeout;
this.testPort = testPort; this.testPort = testPort;
this.delegate = delegate; this.delegate = delegate;
this.rewritesMap = rewrites;
this.rewrites = this.rewrites =
rewrites.entrySet().stream() rewrites.entrySet().stream()
.map((it) -> Map.entry(normalizeRewrite(it.getKey()), normalizeRewrite(it.getValue()))) .map((it) -> Map.entry(normalizeRewrite(it.getKey()), normalizeRewrite(it.getValue())))
@@ -117,11 +121,17 @@ final class RequestRewritingClient implements HttpClient {
.headers() .headers()
.map() .map()
.forEach((name, values) -> values.forEach(value -> builder.header(name, value))); .forEach((name, values) -> values.forEach(value -> builder.header(name, value)));
builder.setHeader("User-Agent", userAgent); var isUserAgentSet = false;
for (var header : this.getHeaders(original.uri())) { for (var header : this.getHeaders(original.uri())) {
var headerName = header.getFirst();
isUserAgentSet = isUserAgentSet || headerName.equalsIgnoreCase("user-agent");
builder.header(header.getFirst(), header.getSecond()); builder.header(header.getFirst(), header.getSecond());
} }
if (!isUserAgentSet) {
builder.setHeader("User-Agent", userAgent);
}
var method = original.method(); var method = original.method();
original original
.bodyPublisher() .bodyPublisher()
@@ -225,14 +235,30 @@ final class RequestRewritingClient implements HttpClient {
return ret; return ret;
} }
private boolean matches(Pattern pattern, URI uri) {
// optimization: "**" always matches, no need to execute regex
// okay to use `==` here because `GlobResolver.toRegexPattern()` caches and
// gives the same `Pattern` instance for an existing glob pattern.
if (pattern == IoUtils.doubleStarGlob) {
return true;
}
return pattern.matcher(uri.toString()).matches();
}
private List<Pair<String, String>> getHeaders(URI uri) { private List<Pair<String, String>> getHeaders(URI uri) {
return headers.stream() var result = new ArrayList<Pair<String, String>>();
.flatMap( for (var rule : headers.entrySet()) {
rule -> var pattern = rule.getKey();
rule.getFirst().asPredicate().test(uri.toString()) if (!matches(pattern, uri)) {
? rule.getSecond().stream() continue;
: Stream.empty()) }
.toList(); for (var header : rule.getValue().entrySet()) {
for (var value : header.getValue()) {
result.add(Pair.of(header.getKey(), value));
}
}
}
return result;
} }
private void checkNotClosed(HttpRequest request) { private void checkNotClosed(HttpRequest request) {
@@ -241,4 +267,14 @@ final class RequestRewritingClient implements HttpClient {
"Cannot send request " + request + " because this client has already been closed."); "Cannot send request " + request + " because this client has already been closed.");
} }
} }
// visible for testing
public Map<URI, URI> getRewritesMap() {
return rewritesMap;
}
// visible for testing
public Map<Pattern, Map<String, List<String>>> getHeaders() {
return headers;
}
} }
@@ -63,10 +63,18 @@ public abstract class AbstractMessagePackEncoder implements MessageEncoder {
} }
protected void packMapHeader( protected void packMapHeader(
int size, @Nullable Object value1, @Nullable Object value2, @Nullable Object value3) int size,
@Nullable Object value1,
@Nullable Object value2,
@Nullable Object value3,
@Nullable Object value4)
throws IOException { throws IOException {
packer.packMapHeader( packer.packMapHeader(
size + (value1 != null ? 1 : 0) + (value2 != null ? 1 : 0) + (value3 != null ? 1 : 0)); size
+ (value1 != null ? 1 : 0)
+ (value2 != null ? 1 : 0)
+ (value3 != null ? 1 : 0)
+ (value4 != null ? 1 : 0));
} }
protected void packMapHeader( protected void packMapHeader(
@@ -1,5 +1,5 @@
/* /*
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -37,6 +37,7 @@ import org.pkl.executor.spi.v1.ExecutorSpiException;
import org.pkl.executor.spi.v1.ExecutorSpiOptions; import org.pkl.executor.spi.v1.ExecutorSpiOptions;
import org.pkl.executor.spi.v1.ExecutorSpiOptions2; import org.pkl.executor.spi.v1.ExecutorSpiOptions2;
import org.pkl.executor.spi.v1.ExecutorSpiOptions3; import org.pkl.executor.spi.v1.ExecutorSpiOptions3;
import org.pkl.executor.spi.v1.ExecutorSpiOptions4;
public final class ExecutorSpiImpl implements ExecutorSpi { public final class ExecutorSpiImpl implements ExecutorSpi {
private static final int MAX_HTTP_CLIENTS = 3; private static final int MAX_HTTP_CLIENTS = 3;
@@ -140,15 +141,17 @@ public final class ExecutorSpiImpl implements ExecutorSpi {
} }
private HttpClient getOrCreateHttpClient(ExecutorSpiOptions options) { private HttpClient getOrCreateHttpClient(ExecutorSpiOptions options) {
List<Path> certificateFiles; List<Path> certificateFiles = List.of();
List<byte[]> certificateBytes; List<byte[]> certificateBytes = List.of();
Map<URI, URI> rewrites; Map<URI, URI> rewrites = Map.of();
int testPort; Map<String, Map<String, List<String>>> headers = Map.of();
int testPort = -1;
try { try {
if (options instanceof ExecutorSpiOptions4 options4) {
headers = options4.getHttpHeaders();
}
if (options instanceof ExecutorSpiOptions3 options3) { if (options instanceof ExecutorSpiOptions3 options3) {
rewrites = options3.getHttpRewrites(); rewrites = options3.getHttpRewrites();
} else {
rewrites = Map.of();
} }
if (options instanceof ExecutorSpiOptions2 options2) { if (options instanceof ExecutorSpiOptions2 options2) {
certificateFiles = options2.getCertificateFiles(); certificateFiles = options2.getCertificateFiles();
@@ -157,17 +160,13 @@ public final class ExecutorSpiImpl implements ExecutorSpi {
} else { } else {
certificateFiles = List.of(); certificateFiles = List.of();
certificateBytes = List.of(); certificateBytes = List.of();
testPort = -1;
} }
// host pkl-executor does not have class ExecutorOptions2/ExecutorOptions3 defined. // host pkl-executor does not have class ExecutorOptions2+ defined.
// this will happen if the pkl-executor distribution is too old. // this will happen if the pkl-executor distribution is too old.
} catch (NoClassDefFoundError e) { } catch (NoClassDefFoundError ignored) {
certificateFiles = List.of();
certificateBytes = List.of();
rewrites = Map.of();
testPort = -1;
} }
var clientKey = new HttpClientKey(certificateFiles, certificateBytes, testPort, rewrites); var clientKey =
new HttpClientKey(certificateFiles, certificateBytes, testPort, rewrites, headers);
return httpClients.computeIfAbsent( return httpClients.computeIfAbsent(
clientKey, clientKey,
(key) -> { (key) -> {
@@ -180,6 +179,7 @@ public final class ExecutorSpiImpl implements ExecutorSpi {
} }
builder.setRewrites(key.rewrites); builder.setRewrites(key.rewrites);
builder.setTestPort(key.testPort); builder.setTestPort(key.testPort);
builder.setHeaders(key.headers);
// If the above didn't add any certificates, // If the above didn't add any certificates,
// builder will use the JVM's default SSL context. // builder will use the JVM's default SSL context.
return builder.buildLazily(); return builder.buildLazily();
@@ -190,5 +190,6 @@ public final class ExecutorSpiImpl implements ExecutorSpi {
List<Path> certificateFiles, List<Path> certificateFiles,
List<byte[]> certificateBytes, List<byte[]> certificateBytes,
int testPort, int testPort,
Map<URI, URI> rewrites) {} Map<URI, URI> rewrites,
Map<String, Map<String, List<String>>> headers) {}
} }
@@ -29,6 +29,8 @@ import org.pkl.core.ast.lambda.ApplyVmFunction1NodeGen;
import org.pkl.core.runtime.*; import org.pkl.core.runtime.*;
import org.pkl.core.stdlib.*; import org.pkl.core.stdlib.*;
import org.pkl.core.util.ByteArrayUtils; import org.pkl.core.util.ByteArrayUtils;
import org.pkl.core.util.GlobResolver;
import org.pkl.core.util.GlobResolver.InvalidGlobPatternException;
import org.pkl.core.util.Pair; import org.pkl.core.util.Pair;
import org.pkl.core.util.StringUtils; import org.pkl.core.util.StringUtils;
@@ -165,6 +167,18 @@ public final class StringNodes {
} }
} }
public abstract static class isGlobPattern extends ExternalPropertyNode {
@Specialization
protected boolean eval(String self) {
try {
GlobResolver.toRegexString(self);
return true;
} catch (InvalidGlobPatternException e) {
return false;
}
}
}
public abstract static class isBase64 extends ExternalPropertyNode { public abstract static class isBase64 extends ExternalPropertyNode {
@Specialization @Specialization
@TruffleBoundary @TruffleBoundary
@@ -45,6 +45,7 @@ import org.pkl.core.packages.PackageLoadError;
import org.pkl.core.runtime.ReaderBase; import org.pkl.core.runtime.ReaderBase;
import org.pkl.core.runtime.VmContext; import org.pkl.core.runtime.VmContext;
import org.pkl.core.runtime.VmExceptionBuilder; import org.pkl.core.runtime.VmExceptionBuilder;
import org.pkl.core.util.GlobResolver.InvalidGlobPatternException;
public final class IoUtils { public final class IoUtils {
@@ -53,33 +54,36 @@ public final class IoUtils {
private static final Pattern windowsPathLike = Pattern.compile("\\w:\\\\.*"); private static final Pattern windowsPathLike = Pattern.compile("\\w:\\\\.*");
private static final Pattern headerNameLike = Pattern.compile("^[a-zA-Z0-9!#$%&'*+-.^_`|~]+$"); private static final Pattern headerNameLike = Pattern.compile("[a-zA-Z0-9!#$%&'*+-.^_`|~]+");
private static final Pattern headerValueLike = private static final Pattern headerValueLike =
Pattern.compile("^[\\t\\u0020-\\u007E\\u0080-\\u00FF]*$"); Pattern.compile("[\\t\\u0020-\\u007E\\u0080-\\u00FF]*");
public static final Pattern doubleStarGlob;
static {
try {
doubleStarGlob = GlobResolver.toRegexPattern("**");
} catch (InvalidGlobPatternException e) {
throw PklBugException.unreachableCode();
}
}
// keep in sync with stdlib `EvaluatorSettings.ReservedHttpHeaderName`
private static final String[] reservedHeaderNames = { private static final String[] reservedHeaderNames = {
"accept-charset",
"accept-encoding",
"connection", "connection",
"keep-alive",
"content-length", "content-length",
"cookie",
"date",
"dnt",
"expect", "expect",
"host", "host",
"keep-alive",
"origin",
"permissions-policy",
"referer",
"te",
"trailer",
"transfer-encoding",
"upgrade", "upgrade",
"via" "te",
"transfer-encoding",
"trailer"
}; };
private static final String[] reservedHeaderPrefixes = {"proxy-", "sec-", "access-control-"}; // keep in sync with stdlib `EvaluatorSettings.reservedHttpHeaderPrefix`
private static final String[] reservedHeaderPrefixes = {"proxy-", "sec-"};
private IoUtils() {} private IoUtils() {}
@@ -885,36 +889,50 @@ public final class IoUtils {
} }
private static boolean isReservedHeaderName(String headerName) { private static boolean isReservedHeaderName(String headerName) {
return Arrays.stream(reservedHeaderNames).anyMatch((reserved) -> headerName.equals(reserved)); var normalizedHeader = headerName.toLowerCase();
for (var reservedHeader : reservedHeaderNames) {
if (normalizedHeader.equals(reservedHeader)) {
return true;
}
}
return false;
} }
private static boolean hasReservedHeaderPrefix(String headerName) { private static boolean hasReservedHeaderPrefix(String headerName) {
return Arrays.stream(reservedHeaderPrefixes) var normalizedHeader = headerName.toLowerCase();
.anyMatch((prefix) -> headerName.startsWith(prefix)); for (var prefix : reservedHeaderPrefixes) {
if (normalizedHeader.startsWith(prefix)) {
return true;
}
}
return false;
} }
// keep in sync with stdlib EvaluatorSettings.HttpHeaderName
public static void validateHeaderName(String headerName) { public static void validateHeaderName(String headerName) {
if (isReservedHeaderName(headerName)) { if (isReservedHeaderName(headerName)) {
throw new IllegalArgumentException( throw new IllegalArgumentException(
"HTTP header '%s' is a reserved header".formatted(headerName)); ErrorMessages.create("invalidHttpHeaderReserved", headerName));
} }
if (hasReservedHeaderPrefix(headerName)) { if (hasReservedHeaderPrefix(headerName)) {
throw new IllegalArgumentException( throw new IllegalArgumentException(
"HTTP header '%s' starts with a reserved header prefix".formatted(headerName)); ErrorMessages.create("invalidHttpHeaderReservedPrefix", headerName));
} }
if (!headerNameLike.matcher(headerName).matches()) { if (!headerNameLike.matcher(headerName).matches()) {
throw new IllegalArgumentException( throw new IllegalArgumentException(ErrorMessages.create("invalidHttpHeaderName", headerName));
"HTTP header name '%s' has an invalid syntax".formatted(headerName));
} }
} }
public static void validateHeaderValue(String headerValue) { public static void validateHeaderValue(String headerValue) {
if (!headerValueLike.matcher(headerValue).matches()) { if (!headerValueLike.matcher(headerValue).matches()) {
throw new IllegalArgumentException( throw new IllegalArgumentException(
"HTTP header value '%s' has an invalid syntax".formatted(headerValue)); ErrorMessages.create("invalidHttpHeaderValue", headerValue));
}
if (headerValue.length() > 4096) {
throw new IllegalArgumentException(
ErrorMessages.create("invalidHttpHeaderValueTooLong", headerValue));
} }
} }
} }
@@ -1133,8 +1133,18 @@ commandFlagInvalidType=\
Option `{0}` with annotation `@{1}` has invalid type `{2}`.\n\ Option `{0}` with annotation `@{1}` has invalid type `{2}`.\n\
Expected type: `{3}` Expected type: `{3}`
invalidHeaderName=\ invalidHttpHeaderReserved=\
HTTP header name `{0}` has invalid syntax. HTTP header `{0}` is a reserved header.
invalidHeaderValue=\ invalidHttpHeaderReservedPrefix=\
HTTP header `{0}` starts with a reserved header prefix.
invalidHttpHeaderName=\
HTTP header `{0}` has invalid syntax.
invalidHttpHeaderValue=\
HTTP header value `{0}` has invalid syntax. HTTP header value `{0}` has invalid syntax.
invalidHttpHeaderValueTooLong=\
HTTP Header value is invalid because it is longer than 4096 characters. \
Value: `{0}`
@@ -0,0 +1,68 @@
amends "../snippetTest.pkl"
import "pkl:EvaluatorSettings"
local function force(http: EvaluatorSettings.Http) = new PcfRenderer {}.renderValue(http)
examples {
["http headers"] {
new EvaluatorSettings.Http {
headers {
["**"] {
["X-Foo"] = "Foo"
}
["https://example.com/**"] {
["X-Bar"] {
"bar"
"baz"
}
}
}
}
}
["http headers -- invalid glob pattern"] {
module.catch(() -> force(new EvaluatorSettings.Http {
headers {
["{{}}"] {
["X-Foo"] = "Foo"
}
}
}))
}
["http headers -- invalid header name"] {
module.catch(() -> force(new EvaluatorSettings.Http {
headers {
["**"] {
["Connection"] = "Foo"
}
}
}))
module.catch(() -> force(new EvaluatorSettings.Http {
headers {
["**"] {
["Sec-Foo-Bar"] = "Foo"
}
}
}))
module.catch(() -> force(new EvaluatorSettings.Http {
headers {
["**"] {
["Foo:Bar"] = "Foo"
}
}
}))
}
["http headers -- invalid header value"] {
module.catch(() -> force(new EvaluatorSettings.Http {
headers {
["**"] {
["My-Header"] = "🙃"
}
}
}))
}
}
@@ -85,6 +85,12 @@ facts {
!"(abc".isRegex !"(abc".isRegex
} }
["isGlobPattern"] {
"**".isGlobPattern
"hello".isGlobPattern
!"{{}}".isGlobPattern
}
["toBoolean()"] { ["toBoolean()"] {
"true".toBoolean() "true".toBoolean()
"tRuE".toBoolean() "tRuE".toBoolean()
@@ -0,0 +1,28 @@
examples {
["http headers"] {
new {
headers {
["**"] {
["X-Foo"] = "Foo"
}
["https://example.com/**"] {
["X-Bar"] {
"bar"
"baz"
}
}
}
}
}
["http headers -- invalid glob pattern"] {
"Type constraint `isGlobPattern` violated. Value: \"{{}}\""
}
["http headers -- invalid header name"] {
"Type constraint `isNotReservedHeaderName` violated. Value: \"Connection\""
"Type constraint `doesNotStartWithReservedPrefix` violated. Value: \"Sec-Foo-Bar\""
"Type constraint `hasValidHeaderNameSyntax` violated. Value: \"Foo:Bar\""
}
["http headers -- invalid header value"] {
"Expected value of type `*Listing<HttpHeaderValue> | HttpHeaderValue`, but got a different `String`. Value: \"🙃\""
}
}
@@ -914,6 +914,27 @@ alias {
name = "isRegex" name = "isRegex"
allModifiers = Set() allModifiers = Set()
allAnnotations = List() allAnnotations = List()
}, "isGlobPattern", new {
location {
line = XXXX
column = 3
displayUri = "https://github.com/apple/pkl/blob/$commitId/stdlib/base.pkl#LXXXX"
}
docComment = """
Tells if this string is a valid glob pattern.
For reference, see
<https://pkl-lang.org/main/current/language-reference/index.html#glob-patterns>.
"""
annotations = List(new {
version = "0.32.0"
})
modifiers = Set()
name = "isGlobPattern"
allModifiers = Set()
allAnnotations = List(new {
version = "0.32.0"
})
}, "isBase64", new { }, "isBase64", new {
location { location {
line = XXXX line = XXXX
@@ -1258,6 +1279,27 @@ alias {
name = "isRegex" name = "isRegex"
allModifiers = Set() allModifiers = Set()
allAnnotations = List() allAnnotations = List()
}, "isGlobPattern", new {
location {
line = XXXX
column = 3
displayUri = "https://github.com/apple/pkl/blob/$commitId/stdlib/base.pkl#LXXXX"
}
docComment = """
Tells if this string is a valid glob pattern.
For reference, see
<https://pkl-lang.org/main/current/language-reference/index.html#glob-patterns>.
"""
annotations = List(new {
version = "0.32.0"
})
modifiers = Set()
name = "isGlobPattern"
allModifiers = Set()
allAnnotations = List(new {
version = "0.32.0"
})
}, "isBase64", new { }, "isBase64", new {
location { location {
line = XXXX line = XXXX
@@ -65,6 +65,11 @@ facts {
true true
true true
} }
["isGlobPattern"] {
true
true
true
}
["toBoolean()"] { ["toBoolean()"] {
true true
true true
@@ -1,5 +1,5 @@
/* /*
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -15,13 +15,17 @@
*/ */
package org.pkl.core package org.pkl.core
import java.net.URI
import java.nio.file.Path import java.nio.file.Path
import java.time.Duration import java.time.Duration
import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test import org.junit.jupiter.api.Test
import org.junit.jupiter.api.assertThrows import org.junit.jupiter.api.assertThrows
import org.pkl.core.http.LazyHttpClient
import org.pkl.core.http.RequestRewritingClient
import org.pkl.core.project.Project import org.pkl.core.project.Project
import org.pkl.core.resource.TestResourceReader import org.pkl.core.resource.TestResourceReader
import org.pkl.core.util.IoUtils
class EvaluatorBuilderTest { class EvaluatorBuilderTest {
@Test @Test
@@ -93,5 +97,10 @@ class EvaluatorBuilderTest {
.isEqualTo(3) // two external readers, one module path .isEqualTo(3) // two external readers, one module path
assertThat(builder.resourceReaders.find { it.uriScheme == "scheme3" }).isNotNull assertThat(builder.resourceReaders.find { it.uriScheme == "scheme3" }).isNotNull
assertThat(builder.resourceReaders.find { it.uriScheme == "scheme4" }).isNotNull assertThat(builder.resourceReaders.find { it.uriScheme == "scheme4" }).isNotNull
val client = (builder.httpClient as LazyHttpClient).orCreateClient as RequestRewritingClient
assertThat(client.headers)
.isEqualTo(mapOf(IoUtils.doubleStarGlob to mapOf("X-Foo" to listOf("Foo"))))
assertThat(client.rewritesMap)
.isEqualTo(mapOf(URI("https://foo.com/") to URI("https://bar.com/")))
} }
} }
@@ -0,0 +1,46 @@
/*
* Copyright © 2026 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.http
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test
class HttpClientBuilderTest {
@Test
fun `addHeader merges values for duplicate header names`() {
val client =
HttpClient.builder()
.setHeaders(mapOf("**" to mapOf("X-Foo" to listOf("v1"))))
.addHeaders("**", mapOf("X-Foo" to listOf("v2")))
.build() as RequestRewritingClient
val headerValues = client.headers.values.single()["X-Foo"]
assertThat(headerValues).containsExactly("v1", "v2")
}
@Test
fun `addHeader preserves non-overlapping header names`() {
val client =
HttpClient.builder()
.addHeaders("**", mapOf("X-Foo" to listOf("v1")))
.addHeaders("**", mapOf("X-Bar" to listOf("v2")))
.build() as RequestRewritingClient
val headerMap = client.headers.values.single()
assertThat(headerMap["X-Foo"]).containsExactly("v1")
assertThat(headerMap["X-Bar"]).containsExactly("v2")
}
}
@@ -21,12 +21,13 @@ import java.net.http.HttpRequest
import java.net.http.HttpRequest.BodyPublishers import java.net.http.HttpRequest.BodyPublishers
import java.net.http.HttpResponse.BodyHandlers import java.net.http.HttpResponse.BodyHandlers
import java.time.Duration import java.time.Duration
import java.util.regex.Pattern
import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.assertThat
import org.assertj.core.api.Assertions.assertThatList import org.assertj.core.api.Assertions.assertThatList
import org.junit.jupiter.api.Test import org.junit.jupiter.api.Test
import org.pkl.core.Pair as PPair import org.pkl.core.util.GlobResolver
import org.pkl.core.util.IoUtils
@Suppress("UastIncorrectHttpHeaderInspection")
class RequestRewritingClientTest { class RequestRewritingClientTest {
private val captured = RequestCapturingClient() private val captured = RequestCapturingClient()
private val client = private val client =
@@ -36,7 +37,7 @@ class RequestRewritingClientTest {
-1, -1,
captured, captured,
mapOf(URI("https://foo/") to URI("https://bar/")), mapOf(URI("https://foo/") to URI("https://bar/")),
listOf(), mapOf(),
) )
private val exampleUri = URI("https://example.com/foo/bar.html") private val exampleUri = URI("https://example.com/foo/bar.html")
private val exampleRequest = HttpRequest.newBuilder(exampleUri).build() private val exampleRequest = HttpRequest.newBuilder(exampleUri).build()
@@ -48,6 +49,21 @@ class RequestRewritingClientTest {
assertThatList(captured.request.headers().allValues("User-Agent")).containsOnly("Pkl") assertThatList(captured.request.headers().allValues("User-Agent")).containsOnly("Pkl")
} }
@Test
fun `User-Agent from configured headers takes precedence`() {
val client =
RequestRewritingClient(
"Pkl",
Duration.ofSeconds(42),
-1,
captured,
mapOf(URI("https://foo/") to URI("https://bar/")),
mapOf(IoUtils.doubleStarGlob to mapOf("User-Agent" to listOf("My-User-Agent"))),
)
client.send(exampleRequest, BodyHandlers.discarding())
assertThatList(captured.request.headers().allValues("User-Agent")).containsOnly("My-User-Agent")
}
@Test @Test
fun `overrides existing User-Agent headers`() { fun `overrides existing User-Agent headers`() {
val request = val request =
@@ -125,7 +141,7 @@ class RequestRewritingClientTest {
fun `rewrites port 0 if test port is set`() { fun `rewrites port 0 if test port is set`() {
val captured = RequestCapturingClient() val captured = RequestCapturingClient()
val client = val client =
RequestRewritingClient("Pkl", Duration.ofSeconds(42), 5000, captured, mapOf(), listOf()) RequestRewritingClient("Pkl", Duration.ofSeconds(42), 5000, captured, mapOf(), mapOf())
val request = HttpRequest.newBuilder(URI("https://example.com:0")).build() val request = HttpRequest.newBuilder(URI("https://example.com:0")).build()
client.send(request, BodyHandlers.discarding()) client.send(request, BodyHandlers.discarding())
@@ -307,8 +323,7 @@ class RequestRewritingClientTest {
private fun rewrittenRequest(uri: String, rules: Map<URI, URI>): String { private fun rewrittenRequest(uri: String, rules: Map<URI, URI>): String {
val captured = RequestCapturingClient() val captured = RequestCapturingClient()
val client = val client = RequestRewritingClient("Pkl", Duration.ofSeconds(42), -1, captured, rules, mapOf())
RequestRewritingClient("Pkl", Duration.ofSeconds(42), -1, captured, rules, listOf())
val request = HttpRequest.newBuilder(URI(uri)).build() val request = HttpRequest.newBuilder(URI(uri)).build()
client.send(request, BodyHandlers.discarding()) client.send(request, BodyHandlers.discarding())
return captured.request.uri().toString() return captured.request.uri().toString()
@@ -324,12 +339,10 @@ class RequestRewritingClientTest {
-1, -1,
captured, captured,
mapOf(), mapOf(),
listOf( mapOf(
PPair(Pattern.compile("^https://example\\.com/.*"), listOf(PPair("x-one", "one"))), GlobResolver.toRegexPattern("https://example.com/**") to mapOf("x-one" to listOf("one")),
PPair( GlobResolver.toRegexPattern("https://example.com/foo/**") to
Pattern.compile("^https://example\\.com/foo/.*"), mapOf("x-two" to listOf("two-a", "two-b")),
listOf(PPair("x-two", "two-a"), PPair("x-two", "two-b")),
),
), ),
) )
val request = HttpRequest.newBuilder(URI("https://example.com/foo/bar")).build() val request = HttpRequest.newBuilder(URI("https://example.com/foo/bar")).build()
@@ -350,9 +363,9 @@ class RequestRewritingClientTest {
-1, -1,
captured, captured,
mapOf(), mapOf(),
listOf( mapOf(
PPair(Pattern.compile("^https://foo\\.com/.*"), listOf(PPair("x-foo", "foo"))), GlobResolver.toRegexPattern("https://foo.com/**") to mapOf("x-foo" to listOf("foo")),
PPair(Pattern.compile("^https://bar\\.com/.*"), listOf(PPair("x-bar", "bar"))), GlobResolver.toRegexPattern("https://bar.com/**") to mapOf("x-bar" to listOf("bar")),
), ),
) )
val request = HttpRequest.newBuilder(URI("https://example.com/foo/bar")).build() val request = HttpRequest.newBuilder(URI("https://example.com/foo/bar")).build()
@@ -373,11 +386,9 @@ class RequestRewritingClientTest {
-1, -1,
captured, captured,
mapOf(), mapOf(),
listOf( mapOf(
PPair( GlobResolver.toRegexPattern("https://example.com/**") to
Pattern.compile("^https://example\\.com/.*"), mapOf("x-foo" to listOf("rule-a", "rule-b"))
listOf(PPair("x-foo", "rule-a"), PPair("x-foo", "rule-b")),
)
), ),
) )
val request = val request =
@@ -388,4 +399,27 @@ class RequestRewritingClientTest {
assertThatList(captured.request.headers().allValues("x-foo")) assertThatList(captured.request.headers().allValues("x-foo"))
.containsExactly("request", "rule-a", "rule-b") .containsExactly("request", "rule-a", "rule-b")
} }
@Test
fun `configured headers wins over configured user-agent header`() {
val captured = RequestCapturingClient()
val client =
RequestRewritingClient(
"Pkl",
Duration.ofSeconds(42),
-1,
captured,
mapOf(),
mapOf(
GlobResolver.toRegexPattern("https://example.com/**") to
mapOf("user-agent" to listOf("My User Agent"))
),
)
val request = HttpRequest.newBuilder(URI("https://example.com/foo/bar")).build()
client.send(request, BodyHandlers.discarding())
assertThatList(captured.request.headers().allValues("user-agent"))
.containsExactly("My User Agent")
}
} }
@@ -26,10 +26,8 @@ import org.pkl.commons.writeString
import org.pkl.core.Evaluator import org.pkl.core.Evaluator
import org.pkl.core.ModuleSource import org.pkl.core.ModuleSource
import org.pkl.core.PObject import org.pkl.core.PObject
import org.pkl.core.Pair as PPair
import org.pkl.core.evaluatorSettings.PklEvaluatorSettings import org.pkl.core.evaluatorSettings.PklEvaluatorSettings
import org.pkl.core.settings.PklSettings.Editor import org.pkl.core.settings.PklSettings.Editor
import org.pkl.core.util.GlobResolver
class PklSettingsTest { class PklSettingsTest {
@Test @Test
@@ -67,9 +65,15 @@ class PklSettingsTest {
["https://foo.com/"] = "https://bar.com/" ["https://foo.com/"] = "https://bar.com/"
} }
headers { headers {
["https://foo.com/"] { ["https://foo.com/**"] {
["x-foo"] = "bar" ["x-foo"] = "bar"
} }
["https://bar.com/**"] {
["x-bar"] {
"bar"
"baz"
}
}
} }
} }
""" """
@@ -84,8 +88,9 @@ class PklSettingsTest {
listOf("example.com", "pkg.pkl-lang.org"), listOf("example.com", "pkg.pkl-lang.org"),
), ),
mapOf(URI("https://foo.com/") to URI("https://bar.com/")), mapOf(URI("https://foo.com/") to URI("https://bar.com/")),
listOf( mapOf(
PPair(GlobResolver.toRegexPattern("https://foo.com/"), listOf(PPair("x-foo", "bar"))) "https://foo.com/**" to mapOf("x-foo" to listOf("bar")),
"https://bar.com/**" to mapOf("x-bar" to listOf("bar", "baz")),
), ),
) )
@@ -41,4 +41,20 @@ evaluatorSettings {
arguments { "with"; "args" } arguments { "with"; "args" }
} }
} }
http {
headers {
["**"] {
["X-Foo"] = "Foo"
}
}
rewrites {
["https://foo.com/"] = "https://bar.com/"
}
proxy {
noProxy {
"my-no-proxy"
}
address = "http://localhost:5619"
}
}
} }
+1
View File
@@ -39,6 +39,7 @@ dependencies {
testImplementation(projects.pklCommonsTest) testImplementation(projects.pklCommonsTest)
testImplementation(projects.pklCore) testImplementation(projects.pklCore)
testImplementation(libs.slf4jSimple) testImplementation(libs.slf4jSimple)
testImplementation(libs.wiremock)
} }
// TODO why is this needed? Without this, we get error: // TODO why is this needed? Without this, we get error:
@@ -25,6 +25,7 @@ import org.jspecify.annotations.Nullable;
import org.pkl.executor.spi.v1.ExecutorSpiOptions; import org.pkl.executor.spi.v1.ExecutorSpiOptions;
import org.pkl.executor.spi.v1.ExecutorSpiOptions2; import org.pkl.executor.spi.v1.ExecutorSpiOptions2;
import org.pkl.executor.spi.v1.ExecutorSpiOptions3; import org.pkl.executor.spi.v1.ExecutorSpiOptions3;
import org.pkl.executor.spi.v1.ExecutorSpiOptions4;
/** /**
* Options for {@link Executor#evaluatePath}. * Options for {@link Executor#evaluatePath}.
@@ -58,6 +59,8 @@ public final class ExecutorOptions {
private final Map<URI, URI> httpRewrites; private final Map<URI, URI> httpRewrites;
private final Map<String, Map<String, List<String>>> httpHeaders;
private final int testPort; // -1 means disabled private final int testPort; // -1 means disabled
private final int spiOptionsVersion; // -1 means use latest private final int spiOptionsVersion; // -1 means use latest
@@ -92,6 +95,7 @@ public final class ExecutorOptions {
private Map<URI, URI> httpRewrites = Map.of(); private Map<URI, URI> httpRewrites = Map.of();
private int testPort = -1; // -1 means disabled private int testPort = -1; // -1 means disabled
private int spiOptionsVersion = -1; // -1 means use latest private int spiOptionsVersion = -1; // -1 means use latest
private Map<String, Map<String, List<String>>> httpHeaders = Map.of();
private Builder() {} private Builder() {}
@@ -215,6 +219,18 @@ public final class ExecutorOptions {
return this; return this;
} }
/**
* API equivalent of the {@code --http-header} CLI option.
*
* <p>This option is ignored on Pkl 0.31 and older.
*
* @since 0.32.0
*/
public Builder httpHeaders(Map<String, Map<String, List<String>>> httpHeaders) {
this.httpHeaders = httpHeaders;
return this;
}
/** Internal test option. -1 means disabled. */ /** Internal test option. -1 means disabled. */
Builder testPort(int testPort) { Builder testPort(int testPort) {
this.testPort = testPort; this.testPort = testPort;
@@ -242,6 +258,7 @@ public final class ExecutorOptions {
certificateFiles, certificateFiles,
certificateBytes, certificateBytes,
httpRewrites, httpRewrites,
httpHeaders,
testPort, testPort,
spiOptionsVersion); spiOptionsVersion);
} }
@@ -291,6 +308,7 @@ public final class ExecutorOptions {
List.of(), List.of(),
List.of(), List.of(),
Map.of(), Map.of(),
Map.of(),
-1, -1,
-1); -1);
} }
@@ -309,6 +327,7 @@ public final class ExecutorOptions {
List<Path> certificateFiles, List<Path> certificateFiles,
List<byte[]> certificateBytes, List<byte[]> certificateBytes,
Map<URI, URI> httpRewrites, Map<URI, URI> httpRewrites,
Map<String, Map<String, List<String>>> httpHeaders,
int testPort, int testPort,
int spiOptionsVersion) { int spiOptionsVersion) {
@@ -325,6 +344,7 @@ public final class ExecutorOptions {
this.certificateFiles = List.copyOf(certificateFiles); this.certificateFiles = List.copyOf(certificateFiles);
this.certificateBytes = List.copyOf(certificateBytes); this.certificateBytes = List.copyOf(certificateBytes);
this.httpRewrites = Map.copyOf(httpRewrites); this.httpRewrites = Map.copyOf(httpRewrites);
this.httpHeaders = Map.copyOf(httpHeaders);
this.testPort = testPort; this.testPort = testPort;
this.spiOptionsVersion = spiOptionsVersion; this.spiOptionsVersion = spiOptionsVersion;
} }
@@ -425,6 +445,7 @@ public final class ExecutorOptions {
&& Objects.equals(certificateFiles, other.certificateFiles) && Objects.equals(certificateFiles, other.certificateFiles)
&& Objects.equals(certificateBytes, other.certificateBytes) && Objects.equals(certificateBytes, other.certificateBytes)
&& Objects.equals(httpRewrites, other.httpRewrites) && Objects.equals(httpRewrites, other.httpRewrites)
&& Objects.equals(httpHeaders, other.httpHeaders)
&& testPort == other.testPort && testPort == other.testPort
&& spiOptionsVersion == other.spiOptionsVersion; && spiOptionsVersion == other.spiOptionsVersion;
} }
@@ -445,6 +466,7 @@ public final class ExecutorOptions {
certificateFiles, certificateFiles,
certificateBytes, certificateBytes,
httpRewrites, httpRewrites,
httpHeaders,
testPort, testPort,
spiOptionsVersion); spiOptionsVersion);
} }
@@ -478,6 +500,8 @@ public final class ExecutorOptions {
+ certificateBytes + certificateBytes
+ ", httpRewrites=" + ", httpRewrites="
+ httpRewrites + httpRewrites
+ ", httpHeaders="
+ httpHeaders
+ ", testPort=" + ", testPort="
+ testPort + testPort
+ ", spiOptionsVersion=" + ", spiOptionsVersion="
@@ -487,7 +511,24 @@ public final class ExecutorOptions {
ExecutorSpiOptions toSpiOptions() { ExecutorSpiOptions toSpiOptions() {
return switch (spiOptionsVersion) { return switch (spiOptionsVersion) {
case -1, 3 -> case -1, 4 ->
new ExecutorSpiOptions4(
allowedModules,
allowedResources,
environmentVariables,
externalProperties,
modulePath,
rootDir,
timeout,
outputFormat,
moduleCacheDir,
projectDir,
certificateFiles,
certificateBytes,
testPort,
httpRewrites,
httpHeaders);
case 3 ->
new ExecutorSpiOptions3( new ExecutorSpiOptions3(
allowedModules, allowedModules,
allowedResources, allowedResources,
@@ -0,0 +1,66 @@
/*
* Copyright © 2026 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.executor.spi.v1;
import java.net.URI;
import java.nio.file.Path;
import java.time.Duration;
import java.util.List;
import java.util.Map;
import org.jspecify.annotations.Nullable;
public class ExecutorSpiOptions4 extends ExecutorSpiOptions3 {
private final Map<String, Map<String, List<String>>> httpHeaders;
public ExecutorSpiOptions4(
List<String> allowedModules,
List<String> allowedResources,
Map<String, String> environmentVariables,
Map<String, String> externalProperties,
List<Path> modulePath,
@Nullable Path rootDir,
@Nullable Duration timeout,
@Nullable String outputFormat,
@Nullable Path moduleCacheDir,
@Nullable Path projectDir,
List<Path> certificateFiles,
List<byte[]> certificateBytes,
int testPort,
Map<URI, URI> httpRewrites,
Map<String, Map<String, List<String>>> httpHeaders) {
super(
allowedModules,
allowedResources,
environmentVariables,
externalProperties,
modulePath,
rootDir,
timeout,
outputFormat,
moduleCacheDir,
projectDir,
certificateFiles,
certificateBytes,
testPort,
httpRewrites);
this.httpHeaders = httpHeaders;
}
public Map<String, Map<String, List<String>>> getHttpHeaders() {
return httpHeaders;
}
}
@@ -15,6 +15,15 @@
*/ */
package org.pkl.executor package org.pkl.executor
import com.github.tomakehurst.wiremock.client.WireMock.equalTo
import com.github.tomakehurst.wiremock.client.WireMock.get
import com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor
import com.github.tomakehurst.wiremock.client.WireMock.ok
import com.github.tomakehurst.wiremock.client.WireMock.stubFor
import com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo
import com.github.tomakehurst.wiremock.client.WireMock.verify
import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo
import com.github.tomakehurst.wiremock.junit5.WireMockTest
import java.io.File import java.io.File
import java.net.URI import java.net.URI
import java.nio.file.Files import java.nio.file.Files
@@ -39,6 +48,7 @@ import org.pkl.commons.test.PackageServer
import org.pkl.commons.toPath import org.pkl.commons.toPath
import org.pkl.core.Release import org.pkl.core.Release
@WireMockTest
class EmbeddedExecutorTest { class EmbeddedExecutorTest {
/** /**
* An executor that uses a particular combination of ExecutorSpiOptions version, pkl-executor * An executor that uses a particular combination of ExecutorSpiOptions version, pkl-executor
@@ -139,7 +149,7 @@ class EmbeddedExecutorTest {
.apply { if (!exists()) missingTestFixture() } .apply { if (!exists()) missingTestFixture() }
} }
// a Pkl distribution that supports ExecutorSpiOptions up to v3 // a Pkl distribution that supports ExecutorSpiOptions up to v4
private val pklDistribution2: Path by lazy { private val pklDistribution2: Path by lazy {
FileTestUtils.rootProjectDir FileTestUtils.rootProjectDir
.resolve( .resolve(
@@ -590,6 +600,30 @@ class EmbeddedExecutorTest {
) )
} }
@Test
fun `http headers option`(@TempDir tempDir: Path, wwRuntimeInfo: WireMockRuntimeInfo) {
stubFor(get(urlEqualTo("/foo.pkl")).willReturn(ok("foo = 1")))
val pklFile =
tempDir.resolve("test.pkl").also {
it.writeText(
"""
@ModuleInfo { minPklVersion = "0.32.0" }
module test
res = import("${wwRuntimeInfo.httpBaseUrl}/foo.pkl")
"""
.trimIndent()
)
}
currentExecutor.evaluatePath(pklFile) {
allowedModules("file:", "https:", "http:")
allowedResources("prop:")
httpHeaders(mapOf("**" to mapOf("X-Foo" to listOf("Foo"))))
}
verify(getRequestedFor(urlEqualTo("/foo.pkl")).withHeader("X-Foo", equalTo("Foo")))
}
@ParameterizedTest @ParameterizedTest
@MethodSource("getCompatibleTestExecutors") @MethodSource("getCompatibleTestExecutors")
@DisabledOnOs(OS.WINDOWS, disabledReason = "Can't populate legacy cache dir on Windows") @DisabledOnOs(OS.WINDOWS, disabledReason = "Can't populate legacy cache dir on Windows")
+1
View File
@@ -47,6 +47,7 @@ dependencies {
} }
testImplementation(projects.pklCommonsTest) testImplementation(projects.pklCommonsTest)
testImplementation(libs.wiremock)
} }
sourceSets { sourceSets {
@@ -487,6 +487,7 @@ public class PklPlugin implements Plugin<Project> {
task.getTestPort().set(spec.getTestPort()); task.getTestPort().set(spec.getTestPort());
task.getHttpProxy().set(spec.getHttpProxy()); task.getHttpProxy().set(spec.getHttpProxy());
task.getHttpNoProxy().set(spec.getHttpNoProxy()); task.getHttpNoProxy().set(spec.getHttpNoProxy());
task.getHttpHeaders().set(spec.getHttpHeaders());
task.getHttpRewrites().set(spec.getHttpRewrites()); task.getHttpRewrites().set(spec.getHttpRewrites());
task.getExternalModuleReaders() task.getExternalModuleReaders()
.set(providers.provider(() -> spec.getExternalModuleReaders().getAsMap())); .set(providers.provider(() -> spec.getExternalModuleReaders().getAsMap()));
@@ -17,6 +17,8 @@ package org.pkl.gradle.spec;
import java.net.URI; import java.net.URI;
import java.time.Duration; import java.time.Duration;
import java.util.List;
import java.util.Map;
import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.NamedDomainObjectContainer;
import org.gradle.api.file.ConfigurableFileCollection; import org.gradle.api.file.ConfigurableFileCollection;
import org.gradle.api.file.DirectoryProperty; import org.gradle.api.file.DirectoryProperty;
@@ -61,6 +63,8 @@ public interface BasePklSpec {
MapProperty<URI, URI> getHttpRewrites(); MapProperty<URI, URI> getHttpRewrites();
MapProperty<String, Map<String, List<String>>> getHttpHeaders();
NamedDomainObjectContainer<ExternalReaderSpec> getExternalModuleReaders(); NamedDomainObjectContainer<ExternalReaderSpec> getExternalModuleReaders();
NamedDomainObjectContainer<ExternalReaderSpec> getExternalResourceReaders(); NamedDomainObjectContainer<ExternalReaderSpec> getExternalResourceReaders();
@@ -25,6 +25,7 @@ import java.nio.file.Paths;
import java.time.Duration; import java.time.Duration;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import javax.inject.Inject; import javax.inject.Inject;
@@ -50,7 +51,6 @@ import org.gradle.api.tasks.PathSensitivity;
import org.gradle.api.tasks.TaskAction; import org.gradle.api.tasks.TaskAction;
import org.jspecify.annotations.Nullable; import org.jspecify.annotations.Nullable;
import org.pkl.commons.cli.CliBaseOptions; import org.pkl.commons.cli.CliBaseOptions;
import org.pkl.core.Pair;
import org.pkl.core.evaluatorSettings.Color; import org.pkl.core.evaluatorSettings.Color;
import org.pkl.gradle.spec.ExternalReaderSpec; import org.pkl.gradle.spec.ExternalReaderSpec;
import org.pkl.gradle.utils.PluginUtils; import org.pkl.gradle.utils.PluginUtils;
@@ -167,7 +167,7 @@ public abstract class BasePklTask extends DefaultTask {
@Input @Input
@Optional @Optional
public abstract ListProperty<Pair<Pattern, List<Pair<String, String>>>> getHttpHeaders(); public abstract MapProperty<String, Map<String, List<String>>> getHttpHeaders();
@Input @Input
@Optional @Optional
@@ -15,6 +15,15 @@
*/ */
package org.pkl.gradle package org.pkl.gradle
import com.github.tomakehurst.wiremock.client.WireMock.equalTo
import com.github.tomakehurst.wiremock.client.WireMock.get
import com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor
import com.github.tomakehurst.wiremock.client.WireMock.ok
import com.github.tomakehurst.wiremock.client.WireMock.stubFor
import com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo
import com.github.tomakehurst.wiremock.client.WireMock.verify
import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo
import com.github.tomakehurst.wiremock.junit5.WireMockTest
import java.nio.file.Path import java.nio.file.Path
import kotlin.io.path.readText import kotlin.io.path.readText
import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.assertThat
@@ -24,6 +33,7 @@ import org.junit.jupiter.api.io.TempDir
import org.pkl.commons.readString import org.pkl.commons.readString
import org.pkl.commons.test.PackageServer import org.pkl.commons.test.PackageServer
@WireMockTest
class EvaluatorsTest : AbstractTest() { class EvaluatorsTest : AbstractTest() {
@Test @Test
fun `render Pcf`() { fun `render Pcf`() {
@@ -907,6 +917,28 @@ class EvaluatorsTest : AbstractTest() {
assertThat(secondRun.output).contains(CONFIG_CACHE_REUSED) assertThat(secondRun.output).contains(CONFIG_CACHE_REUSED)
} }
@Test
fun `http headers`(wwRuntimeInfo: WireMockRuntimeInfo) {
stubFor(get(urlEqualTo("/foo.pkl")).willReturn(ok("foo = 1")))
writeBuildFile(
"pcf",
"""
httpHeaders = [
"**": ["X-Foo": ["Foo"]]
]
allowedModules = ["http:", "pkl:", "repl:", "file:"]
"""
.trimIndent(),
)
writePklFile(
"""
res = import("${wwRuntimeInfo.httpBaseUrl}/foo.pkl")
"""
)
runTask("evalTest")
verify(getRequestedFor(urlEqualTo("/foo.pkl")).withHeader("X-Foo", equalTo("Foo")))
}
private fun writeBuildFile( private fun writeBuildFile(
// don't use `org.pkl.core.OutputFormat` // don't use `org.pkl.core.OutputFormat`
// because test compile class path doesn't contain pkl-core // because test compile class path doesn't contain pkl-core
+1
View File
@@ -25,6 +25,7 @@ dependencies {
implementation(libs.truffleApi) implementation(libs.truffleApi)
testImplementation(projects.pklCommonsTest) testImplementation(projects.pklCommonsTest)
testImplementation(libs.wiremock)
} }
tasks.test { exclude("**/NativeServerTest.*") } tasks.test { exclude("**/NativeServerTest.*") }
@@ -1,5 +1,5 @@
/* /*
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -200,6 +200,7 @@ class Server(private val transport: MessageTransport) : AutoCloseable {
} }
message.http?.caCertificates?.let(::addCertificates) message.http?.caCertificates?.let(::addCertificates)
message.http?.rewrites?.let(::setRewrites) message.http?.rewrites?.let(::setRewrites)
message.http?.headers?.let(::setHeaders)
buildLazily() buildLazily()
} }
securityManager = securityManager =
@@ -1,5 +1,5 @@
/* /*
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -103,7 +103,21 @@ class ServerMessagePackDecoder(unpacker: MessageUnpacker) : BaseMessagePackDecod
?.map() ?.map()
?.mapKeys { URI(it.key.asStringValue().asString()) } ?.mapKeys { URI(it.key.asStringValue().asString()) }
?.mapValues { URI(it.value.asStringValue().asString()) } ?.mapValues { URI(it.value.asStringValue().asString()) }
return Http(caCertificates, proxy, rewrites) val headers =
getNullable(httpMap, "headers")
?.asMapValue()
?.map()
?.mapKeys { it.key.asStringValue().asString() }
?.mapValues { (_, value) ->
value
.asMapValue()
.map()
.mapKeys { it.key.asStringValue().asString() }
.mapValues { value ->
value.value.asArrayValue().list().map { it.asStringValue().asString() }
}
}
return Http(caCertificates, proxy, rewrites, headers)
} }
private fun Map<Value, Value>.unpackProxy(): Proxy? { private fun Map<Value, Value>.unpackProxy(): Proxy? {
@@ -1,5 +1,5 @@
/* /*
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -36,7 +36,7 @@ class ServerMessagePackEncoder(packer: MessagePacker) : BaseMessagePackEncoder(p
} }
private fun MessagePacker.packHttp(http: Http) { private fun MessagePacker.packHttp(http: Http) {
packMapHeader(0, http.caCertificates, http.proxy, http.rewrites) packMapHeader(0, http.caCertificates, http.proxy, http.rewrites, http.headers)
http.caCertificates?.let { packKeyValue("caCertificates", it) } http.caCertificates?.let { packKeyValue("caCertificates", it) }
http.proxy?.let { proxy -> http.proxy?.let { proxy ->
packString("proxy") packString("proxy")
@@ -52,6 +52,21 @@ class ServerMessagePackEncoder(packer: MessagePacker) : BaseMessagePackEncoder(p
packString(value.toString()) packString(value.toString())
} }
} }
http.headers?.let { headers ->
packString("headers")
packMapHeader(headers.size)
for ((pattern, headerMap) in headers) {
packString(pattern)
packMapHeader(headerMap.size)
for ((headerName, headerValue) in headerMap) {
packString(headerName)
packArrayHeader(headerValue.size)
for (elem in headerValue) {
packString(elem)
}
}
}
}
} }
private fun MessagePacker.packDependencies(dependencies: Map<String, Dependency>) { private fun MessagePacker.packDependencies(dependencies: Map<String, Dependency>) {
@@ -1,5 +1,5 @@
/* /*
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -60,6 +60,8 @@ data class Http(
val proxy: Proxy?, val proxy: Proxy?,
/** HTTP rewrites */ /** HTTP rewrites */
val rewrites: Map<URI, URI>?, val rewrites: Map<URI, URI>?,
/** HTTP headers */
val headers: Map<String, Map<String, List<String>>>?,
) { ) {
override fun equals(other: Any?): Boolean { override fun equals(other: Any?): Boolean {
if (this === other) return true if (this === other) return true
@@ -69,13 +71,16 @@ data class Http(
if (other.caCertificates == null) return false if (other.caCertificates == null) return false
if (!caCertificates.contentEquals(other.caCertificates)) return false if (!caCertificates.contentEquals(other.caCertificates)) return false
} else if (other.caCertificates != null) return false } else if (other.caCertificates != null) return false
return Objects.equals(rewrites, other.rewrites) && Objects.equals(proxy, other.proxy) return Objects.equals(rewrites, other.rewrites) &&
Objects.equals(proxy, other.proxy) &&
Objects.equals(headers, other.headers)
} }
override fun hashCode(): Int { override fun hashCode(): Int {
var result = caCertificates?.contentHashCode() ?: 0 var result = caCertificates?.contentHashCode() ?: 0
result = 31 * result + (proxy?.hashCode() ?: 0) result = 31 * result + (proxy?.hashCode() ?: 0)
result = 31 * result + (rewrites?.hashCode() ?: 0) result = 31 * result + (rewrites?.hashCode() ?: 0)
result = 31 * result + (headers?.hashCode() ?: 0)
return result return result
} }
} }
@@ -15,6 +15,15 @@
*/ */
package org.pkl.server package org.pkl.server
import com.github.tomakehurst.wiremock.client.WireMock.equalTo
import com.github.tomakehurst.wiremock.client.WireMock.get
import com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor
import com.github.tomakehurst.wiremock.client.WireMock.ok
import com.github.tomakehurst.wiremock.client.WireMock.stubFor
import com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo
import com.github.tomakehurst.wiremock.client.WireMock.verify
import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo
import com.github.tomakehurst.wiremock.junit5.WireMockTest
import java.io.PipedInputStream import java.io.PipedInputStream
import java.io.PipedOutputStream import java.io.PipedOutputStream
import java.net.URI import java.net.URI
@@ -30,6 +39,7 @@ import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.AfterAll import org.junit.jupiter.api.AfterAll
import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.BeforeAll
import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Disabled
import org.junit.jupiter.api.Nested
import org.junit.jupiter.api.Test import org.junit.jupiter.api.Test
import org.junit.jupiter.api.io.TempDir import org.junit.jupiter.api.io.TempDir
import org.msgpack.core.MessagePack import org.msgpack.core.MessagePack
@@ -1065,6 +1075,7 @@ abstract class AbstractServerTest {
caCertificates = null, caCertificates = null,
proxy = null, proxy = null,
rewrites = mapOf(URI("https://example.com/") to URI("https://example.example/")), rewrites = mapOf(URI("https://example.com/") to URI("https://example.example/")),
headers = null,
) )
) )
client.send( client.send(
@@ -1092,6 +1103,7 @@ abstract class AbstractServerTest {
caCertificates = null, caCertificates = null,
proxy = null, proxy = null,
rewrites = mapOf(URI("https://example.com") to URI("https://example.example/")), rewrites = mapOf(URI("https://example.com") to URI("https://example.example/")),
headers = null,
) )
) )
) )
@@ -1100,6 +1112,37 @@ abstract class AbstractServerTest {
.contains("Rewrite rule must end with '/', but was 'https://example.com'") .contains("Rewrite rule must end with '/', but was 'https://example.com'")
} }
@Nested
@WireMockTest
inner class HttpTests {
@Test
fun `http headers`(wwRuntimeInfo: WireMockRuntimeInfo) {
stubFor(get(urlEqualTo("/foo.pkl")).willReturn(ok("foo = 1")))
val evaluatorId =
client.sendCreateEvaluatorRequest(
http =
Http(
caCertificates = null,
proxy = null,
rewrites = null,
headers = mapOf("**" to mapOf("X-Foo" to listOf("Foo"))),
)
)
client.send(
EvaluateRequest(
1,
evaluatorId,
URI("repl:text"),
"res = import(\"${wwRuntimeInfo.httpBaseUrl}/foo.pkl\")",
"output.text",
)
)
val response = client.receive<EvaluateResponse>()
assertThat(response.error).isNull()
verify(getRequestedFor(urlEqualTo("/foo.pkl")).withHeader("X-Foo", equalTo("Foo")))
}
}
private fun TestTransport.sendCreateEvaluatorRequest( private fun TestTransport.sendCreateEvaluatorRequest(
requestId: Long = 123, requestId: Long = 123,
resourceReaders: List<ResourceReaderSpec> = listOf(), resourceReaders: List<ResourceReaderSpec> = listOf(),
@@ -1,5 +1,5 @@
/* /*
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@@ -45,7 +45,7 @@ class ServerMessagePackCodecTest {
private fun roundtrip(message: Message) { private fun roundtrip(message: Message) {
encoder.encode(message) encoder.encode(message)
val decoded = decoder.decode() val decoded = decoder.decode()
assertThat(decoded).isEqualTo(message) assertThat(decoded).usingRecursiveComparison().isEqualTo(message)
} }
@Test @Test
@@ -98,6 +98,7 @@ class ServerMessagePackCodecTest {
proxy = Proxy(URI("http://foo.com:1234"), listOf("bar", "baz")), proxy = Proxy(URI("http://foo.com:1234"), listOf("bar", "baz")),
caCertificates = byteArrayOf(1, 2, 3, 4), caCertificates = byteArrayOf(1, 2, 3, 4),
rewrites = mapOf(URI("https://foo.com/") to URI("https://bar.com/")), rewrites = mapOf(URI("https://foo.com/") to URI("https://bar.com/")),
headers = mapOf("**" to mapOf("X-Foo" to listOf("Foo", "Bar"))),
), ),
externalModuleReaders = mapOf("external" to externalReader, "external2" to externalReader), externalModuleReaders = mapOf("external" to externalReader, "external2" to externalReader),
externalResourceReaders = mapOf("external" to externalReader), externalResourceReaders = mapOf("external" to externalReader),
+84 -34
View File
@@ -128,9 +128,6 @@ local const hasNonEmptyHostname = (it: String) ->
@Since { version = "0.29.0" } @Since { version = "0.29.0" }
typealias HttpRewrite = String(startsWith(Regex("https?://")), endsWith("/"), hasNonEmptyHostname) typealias HttpRewrite = String(startsWith(Regex("https?://")), endsWith("/"), hasNonEmptyHostname)
@Since { version = "0.32.0" }
typealias UrlPattern = String(endsWith(Regex("[/*]")))
/// Settings that control how Pkl talks to HTTP(S) servers. /// Settings that control how Pkl talks to HTTP(S) servers.
class Http { class Http {
/// Configuration of the HTTP proxy to use. /// Configuration of the HTTP proxy to use.
@@ -173,9 +170,49 @@ class Http {
@Since { version = "0.29.0" } @Since { version = "0.29.0" }
rewrites: Mapping<HttpRewrite, HttpRewrite>? rewrites: Mapping<HttpRewrite, HttpRewrite>?
/// HTTP headers to add to outbound requests targeting specified URLs. /// HTTP headers to add to outbound requests.
///
/// Each key is a glob pattern, and each value is a mapping of header name to header value(s).
///
/// Before an HTTP request is made, each key is matched against the request URL.
/// If any matches are found, each of their described headers are added to the request.
///
/// To add headers to all HTTP requests, use `**` as the glob pattern.
///
/// Example:
///
/// ```
/// headers {
/// // Add this user agent to all out-bound requests.
/// ["**"] {
/// ["User-Agent"] = "MyApp-Pkl"
/// }
///
/// // Add an authorization header to all requests made to the
/// // `https://my.internal.service` host.
/// ["https://my.internal.service/**"] {
/// ["Authorization"] = "Bearer abc123"
/// }
/// }
/// ```
///
/// To repeat a header multiple times, set the value to a listing:
///
/// ```
/// headers {
/// // Adds:
/// // My-Custom-Header: foo
/// // My-Custom-Header: bar
/// ["**"] {
/// ["My-Custom-Header"] { "foo"; "bar" }
/// }
/// }
/// ```
///
/// For details on glob patterns, see
/// <https://pkl-lang.org/main/current/language-reference/index.html#glob-patterns>.
@Since { version = "0.32.0" } @Since { version = "0.32.0" }
headers: Mapping<UrlPattern, Mapping<HttpHeaderName, *Listing<HttpHeaderValue> | HttpHeaderValue>>? headers: Mapping<String(isGlobPattern), HttpHeaders>?
} }
/// Settings that control how Pkl talks to HTTP proxies. /// Settings that control how Pkl talks to HTTP proxies.
@@ -243,50 +280,63 @@ class ExternalReader {
arguments: Listing<String>? arguments: Listing<String>?
} }
@Since { version = "0.32.0" } local typealias ReservedHttpHeaderName =
typealias ReservedHttpHeaderName = "connection"
"accept-charset"
| "accept-encoding"
| "connection"
| "content-length" | "content-length"
| "cookie"
| "date"
| "dnt"
| "expect" | "expect"
| "host" | "host"
| "keep-alive" | "keep-alive"
| "origin"
| "permissions-policy"
| "referer"
| "te" | "te"
| "trailer" | "trailer"
| "transfer-encoding" | "transfer-encoding"
| "upgrade" | "upgrade"
| "via"
local const ReservedHttpHeaderPrefix = new Listing { local const reservedHttpHeaderPrefix =
"proxy-" Set(
"sec-" "proxy-",
"access-control-" "sec-",
} )
local const hasReservedHttpHeaderPrefix = (header: String) -> local const isNotReservedHeaderName = (header: String) ->
ReservedHttpHeaderPrefix.any((it) -> header.startsWith(it)) !(header.toLowerCase() is ReservedHttpHeaderName)
local const httpHeaderNameRegex = Regex("^[a-zA-Z0-9!#\\$%&'*+-.^_`|~]+$") local const doesNotStartWithReservedPrefix = (header: String) ->
local const hasValidHttpHeaderName = (header: String) -> !reservedHttpHeaderPrefix.any((it) -> header.toLowerCase().startsWith(it))
!httpHeaderNameRegex.findMatchesIn(header).isEmpty
local const hasValidHeaderNameSyntax = (header: String) ->
header.matches(Regex(#"[a-zA-Z0-9!#$%&'*+-.^_`|~]+"#))
/// A mapping of header names to header value(s).
///
/// The mapping must have distinct header keys (case insensitive).
@Since { version = "0.32.0" }
typealias HttpHeaders =
Mapping<HttpHeaderName, *Listing<HttpHeaderValue> | HttpHeaderValue>(
keys.toList().isDistinctBy((it) -> it.toLowerCase()),
)
/// An HTTP header name.
///
/// It conforms to the following rules:
///
/// * It is not one of "connection", "keep-alive", "content-length", "expect", "host", "upgrade",
/// "te", "transfer-encoding", "trailer" (case insensitive).
/// * It does not start with "proxy-" nor "sec-".
/// * It consists of a-z, A-Z, or a character in "!#$%&'*+-.^_`|~".
/// * It is not a blank string.
@Since { version = "0.32.0" } @Since { version = "0.32.0" }
typealias HttpHeaderName = typealias HttpHeaderName =
String( String(
this == toLowerCase(), isNotReservedHeaderName,
!(this is ReservedHttpHeaderName), doesNotStartWithReservedPrefix,
!hasReservedHttpHeaderPrefix.apply(this), hasValidHeaderNameSyntax,
hasValidHttpHeaderName, isNotBlank,
) )
local const httpHeaderValueRegex = Regex("^[\\t\\u0020-\\u007E\\u0080-\\u00FF]*$") /// An HTTP header value.
///
/// It consists of ASCII characters between the range of 0x20 - 0x7E, and 0x80 - 0xFF (all visible
/// ASCII characters), and also the tab character.
@Since { version = "0.32.0" } @Since { version = "0.32.0" }
typealias HttpHeaderValue = String(!httpHeaderValueRegex.findMatchesIn(this).isEmpty) typealias HttpHeaderValue =
String(matches(Regex(#"[\t\u0020-\u007E\u0080-\u00FF]*"#)), length < 4096)
+7
View File
@@ -1323,6 +1323,13 @@ external class String extends Any {
/// Tells if this string is a valid regular expression according to [Regex]. /// Tells if this string is a valid regular expression according to [Regex].
external isRegex: Boolean external isRegex: Boolean
/// Tells if this string is a valid glob pattern.
///
/// For reference, see
/// <https://pkl-lang.org/main/current/language-reference/index.html#glob-patterns>.
@Since { version = "0.32.0" }
external isGlobPattern: Boolean
/// Tells if this is a valid base64-encoded string. /// Tells if this is a valid base64-encoded string.
/// ///
/// Facts: /// Facts: