starred/pkl
1
0
Fork 0
mirror of https://github.com/apple/pkl.git synced 2026-05-25 16:19:20 +02:00
Code Issues 198 Packages Projects Releases 10 Wiki Activity

Improve HTTP headers logic (#1584)

Browse Source 
* Relax forbidden headers constraints
  - remove restriction on browser-related headers
- allow any glob pattern (no need to end with `/` or `*`, because glob
patterns already require users to explicitly declare prefix matches if
that's the intention)
* Replace `List<Pair<, ...>>`; use `Map<String, ...>` instead
* Use glob pattern strings as an API throughout, instead of `Pattern`
(e.g. in `HttpClientBuilder`)
* Add HTTP headers to message passing API
* Add HTTP headers to executor API (introduces `ExecutorSpiOptions4`)
* Add tests for Gradle, CLI, and pkl-executor invocations
* Improve documentation
* Add `isGlobPattern` API to class `String` for in-language validation
of http headers
* Behavior change: make sure explicitly configured `User-Agent` in
`HttpClientBuilder` can be shadowed by headers (allows users to set
`--http-header "**=User-Agent: My User Agent"` and for this to be the
only user agent).

CC @kyokuping
This commit is contained in:
Daniel Chao
2026-05-21 20:07:06 -07:00
committed by GitHub
parent 87ea28260b
commit 8e2e5e4ba8
48 changed files with 1067 additions and 222 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkl-core/src/main/java/org/pkl/core/EvaluatorBuilder.java
+3
View File
@@ -557,6 +557,9 @@ public final class EvaluatorBuilder {
if (settings.http().rewrites() != null) {
httpClientBuilder.setRewrites(settings.http().rewrites());
}
if (settings.http().headers() != null) {
httpClientBuilder.setHeaders(settings.http().headers());
}
setHttpClient(httpClientBuilder.buildLazily());
}
pkl-core/src/main/java/org/pkl/core/evaluatorSettings/PklEvaluatorSettings.java
+45 -51
View File
@@ -18,9 +18,9 @@ package org.pkl.core.evaluatorSettings;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -28,18 +28,14 @@ import java.util.Objects;
import java.util.function.BiFunction;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.jspecify.annotations.Nullable;
import org.pkl.core.Duration;
import org.pkl.core.PNull;
import org.pkl.core.PObject;
import org.pkl.core.Pair;
import org.pkl.core.PklBugException;
import org.pkl.core.PklException;
import org.pkl.core.Value;
import org.pkl.core.util.ErrorMessages;
import org.pkl.core.util.GlobResolver;
import org.pkl.core.util.GlobResolver.InvalidGlobPatternException;
/** Java version of {@code pkl.EvaluatorSettings}. */
public record PklEvaluatorSettings(
@@ -134,63 +130,61 @@ public record PklEvaluatorSettings(
public record Http(
@Nullable Proxy proxy,
@Nullable Map<URI, URI> rewrites,
@Nullable List<Pair<Pattern, List<Pair<String, String>>>> headers) {
@Nullable Map<String, Map<String, List<String>>> headers) {
public static final Http DEFAULT = new Http(null, Collections.emptyMap(), null);
@SuppressWarnings("unchecked")
public static @Nullable Http parse(@Nullable Value input) {
if (input == null || input instanceof PNull) {
return null;
} else if (input instanceof PObject http) {
var proxy = Proxy.parse((Value) http.getProperty("proxy"));
var rewrites = http.getProperty("rewrites");
HashMap<URI, URI> parsedRewrites = null;
if (!(rewrites instanceof PNull)) {
parsedRewrites = new HashMap<>();
for (var entry : ((Map<String, String>) rewrites).entrySet()) {
var key = entry.getKey();
var value = entry.getValue();
try {
parsedRewrites.put(new URI(key), new URI(value));
} catch (URISyntaxException e) {
throw new PklException(ErrorMessages.create("invalidUri", e.getInput()));
}
}
}
var headerDefs = http.getProperty("headers");
List<Pair<Pattern, List<Pair<String, String>>>> parsedHeaderDefs = null;
if (!(headerDefs instanceof PNull)) {
parsedHeaderDefs = new ArrayList<>();
var headerDefsMap = (Map<String, Map<String, Object>>) headerDefs;
for (var entry : headerDefsMap.entrySet()) {
var stringPattern = entry.getKey();
var headersMap = entry.getValue();
try {
var urlPattern = GlobResolver.toRegexPattern(stringPattern);
var pairs =
headersMap.entrySet().stream()
.flatMap(
header -> {
var value = header.getValue();
if (value instanceof List) {
return ((List<String>) value)
.stream().map(v -> new Pair(header.getKey(), v));
} else {
return Stream.of(new Pair(header.getKey(), value));
}
})
.toList();
parsedHeaderDefs.add(new Pair(urlPattern, pairs));
} catch (InvalidGlobPatternException e) {
throw new PklException(ErrorMessages.create("invalidUri", stringPattern));
}
}
}
return new Http(proxy, parsedRewrites, parsedHeaderDefs);
var parsedRewrites = parseHttpRewrites(http.getProperty("rewrites"));
var parsedHeaders = parseHttpHeaders(http.getProperty("headers"));
return new Http(proxy, parsedRewrites, parsedHeaders);
} else {
throw PklBugException.unreachableCode();
}
}
@SuppressWarnings("unchecked")
private static @Nullable Map<URI, URI> parseHttpRewrites(Object rewrites) {
if (rewrites instanceof PNull) {
return null;
}
var parsedRewrites = new HashMap<URI, URI>();
for (var entry : ((Map<String, String>) rewrites).entrySet()) {
var key = entry.getKey();
var value = entry.getValue();
try {
parsedRewrites.put(new URI(key), new URI(value));
} catch (URISyntaxException e) {
throw new PklException(ErrorMessages.create("invalidUri", e.getInput()));
}
}
return parsedRewrites;
}
@SuppressWarnings("unchecked")
private static @Nullable Map<String, Map<String, List<String>>> parseHttpHeaders(
Object headerDefs) {
if (headerDefs instanceof PNull) {
return null;
}
var defs = (Map<String, Map<String, Object>>) headerDefs;
var ret = new LinkedHashMap<String, Map<String, List<String>>>(defs.size());
for (var entry : defs.entrySet()) {
var headers = entry.getValue();
var map = new LinkedHashMap<String, List<String>>(headers.size());
for (var header : headers.entrySet()) {
var value = header.getValue();
var headerValues =
value instanceof List<?> ? (List<String>) value : List.of((String) value);
map.put(header.getKey(), headerValues);
}
ret.put(entry.getKey(), map);
}
return ret;
}
}
public record Proxy(@Nullable URI address, @Nullable List<String> noProxy) {
pkl-core/src/main/java/org/pkl/core/http/HttpClient.java
+31 -4
View File
@@ -23,10 +23,8 @@ import java.net.http.HttpTimeoutException;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
import javax.net.ssl.SSLContext;
import org.jspecify.annotations.Nullable;
import org.pkl.core.Pair;
/**
* An HTTP client.
@@ -47,6 +45,9 @@ public interface HttpClient extends AutoCloseable {
* Sets the {@code User-Agent} header.
*
* <p>Defaults to {@code "Pkl/$version ($os; $flavor)"}.
*
* <p>An existing "User-Agent" from {@link Builder#setHeaders} and {@link Builder#addHeaders}
* takes precedence over this field.
*/
Builder setUserAgent(String userAgent);
@@ -146,7 +147,6 @@ public interface HttpClient extends AutoCloseable {
/**
* Adds a rewrite rule.
*
* @see Builder#setRewrites(Map)
* @throws IllegalArgumentException if {@code sourcePrefix} or {@code targetPrefix} is invalid.
* @since 0.29.0
*/
@@ -157,8 +157,35 @@ public interface HttpClient extends AutoCloseable {
*
* <p>This method clears all existing headers and replaces them with the contents of the
* provided map.
*
* <p>{@code headerRules} is a map whose keys are <a
* href="https://pkl-lang.org/main/current/language-reference/index.html#glob-patterns">glob
* patterns</a>, and values is a map of header names and values. Multiple header values turn
* into multiple individual headers in the HTTP request.
*
* <p>To add headers to all requests, use {@code **} as the glob pattern.
*
* <p>To describe a prefix match, add {@code **} to the glob pattern (e.g. {@code
* https://example.com/**}).
*
* <p>Before an HTTP request is made, each key is matched against the request URL. If any
* matches are found, each of their headers are added to the request.
*
* @throws IllegalArgumentException if any of the keys are invalid glob patterns, or if any of
* the header names or values are invalid.
* @since 0.32.0
*/
Builder setHeaders(List<Pair<Pattern, List<Pair<String, String>>>> headers);
Builder setHeaders(Map<String, Map<String, List<String>>> headerRules);
/**
* Adds HTTP headers for URL requests that match {@code globPattern}.
*
* @throws IllegalArgumentException if {@code globPattern} is an invalid glob pattern, or if any
* of the header names or values are invalid.
* @since 0.32.0
* @see Builder#setHeaders
*/
Builder addHeaders(String globPattern, Map<String, List<String>> headers);
/**
* Creates a new {@code HttpClient} from the current state of this builder.
pkl-core/src/main/java/org/pkl/core/http/HttpClientBuilder.java
+53 -4
View File
@@ -24,14 +24,17 @@ import java.nio.file.Path;
import java.time.Duration;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;
import java.util.regex.Pattern;
import org.jspecify.annotations.Nullable;
import org.pkl.core.Pair;
import org.pkl.core.Release;
import org.pkl.core.http.HttpClient.Builder;
import org.pkl.core.util.GlobResolver;
import org.pkl.core.util.GlobResolver.InvalidGlobPatternException;
import org.pkl.core.util.IoUtils;
final class HttpClientBuilder implements HttpClient.Builder {
private String userAgent;
@@ -42,7 +45,10 @@ final class HttpClientBuilder implements HttpClient.Builder {
private int testPort = -1;
private @Nullable ProxySelector proxySelector;
private Map<URI, URI> rewrites = new HashMap<>();
private List<Pair<Pattern, List<Pair<String, String>>>> headers = new ArrayList<>();
// okay to use Pattern as a map key here because `GlobResolver.toRegexPattern()` caches and
// gives the same `Pattern` instance for an existing glob pattern.
// use LinkedHashMap to preserve insertion order.
private Map<Pattern, Map<String, List<String>>> headers = new LinkedHashMap<>();
HttpClientBuilder() {
var release = Release.current();
@@ -115,11 +121,54 @@ final class HttpClientBuilder implements HttpClient.Builder {
}
@Override
public Builder setHeaders(List<Pair<Pattern, List<Pair<String, String>>>> headers) {
this.headers = headers;
public Builder setHeaders(Map<String, Map<String, List<String>>> headers) {
var newHeaders = new LinkedHashMap<Pattern, Map<String, List<String>>>(headers.size());
for (var rule : headers.entrySet()) {
Pattern pattern;
try {
pattern = GlobResolver.toRegexPattern(rule.getKey());
} catch (InvalidGlobPatternException e) {
throw new IllegalArgumentException(e.getMessage(), e);
}
var map = new LinkedHashMap<String, List<String>>();
for (var entry : rule.getValue().entrySet()) {
IoUtils.validateHeaderName(entry.getKey());
for (var value : entry.getValue()) {
IoUtils.validateHeaderValue(value);
}
map.put(entry.getKey(), new ArrayList<>(entry.getValue()));
}
newHeaders.put(pattern, map);
}
this.headers = newHeaders;
return this;
}
@Override
public Builder addHeaders(String globPattern, Map<String, List<String>> headers) {
try {
var pattern = GlobResolver.toRegexPattern(globPattern);
var existingHeaders = this.headers.computeIfAbsent(pattern, k -> new HashMap<>());
for (var entry : headers.entrySet()) {
var headerName = entry.getKey();
var headerValues = entry.getValue();
IoUtils.validateHeaderName(headerName);
for (var value : headerValues) {
IoUtils.validateHeaderValue(value);
}
var existingList = existingHeaders.putIfAbsent(headerName, new ArrayList<>(headerValues));
if (existingList != null) {
existingList.addAll(headerValues);
}
}
return this;
} catch (InvalidGlobPatternException e) {
throw new IllegalArgumentException(e.getMessage(), e);
}
}
@Override
public HttpClient build() {
return doBuild().get();
pkl-core/src/main/java/org/pkl/core/http/LazyHttpClient.java
+4 -2
View File
@@ -29,8 +29,9 @@ import org.jspecify.annotations.Nullable;
* An {@code HttpClient} decorator that defers creating the underlying HTTP client until the first
* send.
*/
// visible for testing
@ThreadSafe
final class LazyHttpClient implements HttpClient {
public final class LazyHttpClient implements HttpClient {
private final Supplier<HttpClient> supplier;
private final Object lock = new Object();
@@ -55,7 +56,8 @@ final class LazyHttpClient implements HttpClient {
getClient().ifPresent(HttpClient::close);
}
private HttpClient getOrCreateClient() {
// visible for testing
public HttpClient getOrCreateClient() {
synchronized (lock) {
// only try to create client once
if (exception != null) {
pkl-core/src/main/java/org/pkl/core/http/RequestRewritingClient.java
+49 -13
View File
@@ -22,6 +22,7 @@ import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.net.http.HttpResponse.BodyHandler;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
@@ -29,12 +30,12 @@ import java.util.Map.Entry;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.regex.Pattern;
import java.util.stream.Stream;
import javax.annotation.concurrent.ThreadSafe;
import org.jspecify.annotations.Nullable;
import org.pkl.core.Pair;
import org.pkl.core.PklBugException;
import org.pkl.core.util.HttpUtils;
import org.pkl.core.util.IoUtils;
import org.pkl.core.util.Pair;
/**
* An {@code HttpClient} decorator that
@@ -49,15 +50,17 @@ import org.pkl.core.util.HttpUtils;
* <p>Both {@code User-Agent} header and default request timeout are configurable through {@link
* HttpClient.Builder}.
*/
// visible for testing
@ThreadSafe
final class RequestRewritingClient implements HttpClient {
public final class RequestRewritingClient implements HttpClient {
// non-private for testing
final String userAgent;
final Duration requestTimeout;
final int testPort;
final HttpClient delegate;
private final Map<URI, URI> rewritesMap;
private final List<Entry<URI, URI>> rewrites;
private final List<Pair<Pattern, List<Pair<String, String>>>> headers;
private final Map<Pattern, Map<String, List<String>>> headers;
private final AtomicBoolean closed = new AtomicBoolean();
@@ -67,11 +70,12 @@ final class RequestRewritingClient implements HttpClient {
int testPort,
HttpClient delegate,
Map<URI, URI> rewrites,
List<Pair<Pattern, List<Pair<String, String>>>> headers) {
Map<Pattern, Map<String, List<String>>> headers) {
this.userAgent = userAgent;
this.requestTimeout = requestTimeout;
this.testPort = testPort;
this.delegate = delegate;
this.rewritesMap = rewrites;
this.rewrites =
rewrites.entrySet().stream()
.map((it) -> Map.entry(normalizeRewrite(it.getKey()), normalizeRewrite(it.getValue())))
@@ -117,11 +121,17 @@ final class RequestRewritingClient implements HttpClient {
.headers()
.map()
.forEach((name, values) -> values.forEach(value -> builder.header(name, value)));
builder.setHeader("User-Agent", userAgent);
var isUserAgentSet = false;
for (var header : this.getHeaders(original.uri())) {
var headerName = header.getFirst();
isUserAgentSet = isUserAgentSet || headerName.equalsIgnoreCase("user-agent");
builder.header(header.getFirst(), header.getSecond());
}
if (!isUserAgentSet) {
builder.setHeader("User-Agent", userAgent);
}
var method = original.method();
original
.bodyPublisher()
@@ -225,14 +235,30 @@ final class RequestRewritingClient implements HttpClient {
return ret;
}
private boolean matches(Pattern pattern, URI uri) {
// optimization: "**" always matches, no need to execute regex
// okay to use `==` here because `GlobResolver.toRegexPattern()` caches and
// gives the same `Pattern` instance for an existing glob pattern.
if (pattern == IoUtils.doubleStarGlob) {
return true;
}
return pattern.matcher(uri.toString()).matches();
}
private List<Pair<String, String>> getHeaders(URI uri) {
return headers.stream()
.flatMap(
rule ->
rule.getFirst().asPredicate().test(uri.toString())
? rule.getSecond().stream()
: Stream.empty())
.toList();
var result = new ArrayList<Pair<String, String>>();
for (var rule : headers.entrySet()) {
var pattern = rule.getKey();
if (!matches(pattern, uri)) {
continue;
}
for (var header : rule.getValue().entrySet()) {
for (var value : header.getValue()) {
result.add(Pair.of(header.getKey(), value));
}
}
}
return result;
}
private void checkNotClosed(HttpRequest request) {
@@ -241,4 +267,14 @@ final class RequestRewritingClient implements HttpClient {
"Cannot send request " + request + " because this client has already been closed.");
}
}
// visible for testing
public Map<URI, URI> getRewritesMap() {
return rewritesMap;
}
// visible for testing
public Map<Pattern, Map<String, List<String>>> getHeaders() {
return headers;
}
}
pkl-core/src/main/java/org/pkl/core/messaging/AbstractMessagePackEncoder.java
+10 -2
View File
@@ -63,10 +63,18 @@ public abstract class AbstractMessagePackEncoder implements MessageEncoder {
}
protected void packMapHeader(
int size, @Nullable Object value1, @Nullable Object value2, @Nullable Object value3)
int size,
@Nullable Object value1,
@Nullable Object value2,
@Nullable Object value3,
@Nullable Object value4)
throws IOException {
packer.packMapHeader(
size + (value1 != null ? 1 : 0) + (value2 != null ? 1 : 0) + (value3 != null ? 1 : 0));
size
+ (value1 != null ? 1 : 0)
+ (value2 != null ? 1 : 0)
+ (value3 != null ? 1 : 0)
+ (value4 != null ? 1 : 0));
}
protected void packMapHeader(
pkl-core/src/main/java/org/pkl/core/service/ExecutorSpiImpl.java
+17 -16
View File
@@ -1,5 +1,5 @@
/*
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -37,6 +37,7 @@ import org.pkl.executor.spi.v1.ExecutorSpiException;
import org.pkl.executor.spi.v1.ExecutorSpiOptions;
import org.pkl.executor.spi.v1.ExecutorSpiOptions2;
import org.pkl.executor.spi.v1.ExecutorSpiOptions3;
import org.pkl.executor.spi.v1.ExecutorSpiOptions4;
public final class ExecutorSpiImpl implements ExecutorSpi {
private static final int MAX_HTTP_CLIENTS = 3;
@@ -140,15 +141,17 @@ public final class ExecutorSpiImpl implements ExecutorSpi {
}
private HttpClient getOrCreateHttpClient(ExecutorSpiOptions options) {
List<Path> certificateFiles;
List<byte[]> certificateBytes;
Map<URI, URI> rewrites;
int testPort;
List<Path> certificateFiles = List.of();
List<byte[]> certificateBytes = List.of();
Map<URI, URI> rewrites = Map.of();
Map<String, Map<String, List<String>>> headers = Map.of();
int testPort = -1;
try {
if (options instanceof ExecutorSpiOptions4 options4) {
headers = options4.getHttpHeaders();
}
if (options instanceof ExecutorSpiOptions3 options3) {
rewrites = options3.getHttpRewrites();
} else {
rewrites = Map.of();
}
if (options instanceof ExecutorSpiOptions2 options2) {
certificateFiles = options2.getCertificateFiles();
@@ -157,17 +160,13 @@ public final class ExecutorSpiImpl implements ExecutorSpi {
} else {
certificateFiles = List.of();
certificateBytes = List.of();
testPort = -1;
}
// host pkl-executor does not have class ExecutorOptions2/ExecutorOptions3 defined.
// host pkl-executor does not have class ExecutorOptions2+ defined.
// this will happen if the pkl-executor distribution is too old.
} catch (NoClassDefFoundError e) {
certificateFiles = List.of();
certificateBytes = List.of();
rewrites = Map.of();
testPort = -1;
} catch (NoClassDefFoundError ignored) {
}
var clientKey = new HttpClientKey(certificateFiles, certificateBytes, testPort, rewrites);
var clientKey =
new HttpClientKey(certificateFiles, certificateBytes, testPort, rewrites, headers);
return httpClients.computeIfAbsent(
clientKey,
(key) -> {
@@ -180,6 +179,7 @@ public final class ExecutorSpiImpl implements ExecutorSpi {
}
builder.setRewrites(key.rewrites);
builder.setTestPort(key.testPort);
builder.setHeaders(key.headers);
// If the above didn't add any certificates,
// builder will use the JVM's default SSL context.
return builder.buildLazily();
@@ -190,5 +190,6 @@ public final class ExecutorSpiImpl implements ExecutorSpi {
List<Path> certificateFiles,
List<byte[]> certificateBytes,
int testPort,
Map<URI, URI> rewrites) {}
Map<URI, URI> rewrites,
Map<String, Map<String, List<String>>> headers) {}
}
pkl-core/src/main/java/org/pkl/core/stdlib/base/StringNodes.java
+14
View File
@@ -29,6 +29,8 @@ import org.pkl.core.ast.lambda.ApplyVmFunction1NodeGen;
import org.pkl.core.runtime.*;
import org.pkl.core.stdlib.*;
import org.pkl.core.util.ByteArrayUtils;
import org.pkl.core.util.GlobResolver;
import org.pkl.core.util.GlobResolver.InvalidGlobPatternException;
import org.pkl.core.util.Pair;
import org.pkl.core.util.StringUtils;
@@ -165,6 +167,18 @@ public final class StringNodes {
}
}
public abstract static class isGlobPattern extends ExternalPropertyNode {
@Specialization
protected boolean eval(String self) {
try {
GlobResolver.toRegexString(self);
return true;
} catch (InvalidGlobPatternException e) {
return false;
}
}
}
public abstract static class isBase64 extends ExternalPropertyNode {
@Specialization
@TruffleBoundary
pkl-core/src/main/java/org/pkl/core/util/IoUtils.java
+43 -25
View File
@@ -45,6 +45,7 @@ import org.pkl.core.packages.PackageLoadError;
import org.pkl.core.runtime.ReaderBase;
import org.pkl.core.runtime.VmContext;
import org.pkl.core.runtime.VmExceptionBuilder;
import org.pkl.core.util.GlobResolver.InvalidGlobPatternException;
public final class IoUtils {
@@ -53,33 +54,36 @@ public final class IoUtils {
private static final Pattern windowsPathLike = Pattern.compile("\\w:\\\\.*");
private static final Pattern headerNameLike = Pattern.compile("^[a-zA-Z0-9!#$%&'*+-.^_`|~]+$");
private static final Pattern headerNameLike = Pattern.compile("[a-zA-Z0-9!#$%&'*+-.^_`|~]+");
private static final Pattern headerValueLike =
Pattern.compile("^[\\t\\u0020-\\u007E\\u0080-\\u00FF]*$");
Pattern.compile("[\\t\\u0020-\\u007E\\u0080-\\u00FF]*");
public static final Pattern doubleStarGlob;
static {
try {
doubleStarGlob = GlobResolver.toRegexPattern("**");
} catch (InvalidGlobPatternException e) {
throw PklBugException.unreachableCode();
}
}
// keep in sync with stdlib `EvaluatorSettings.ReservedHttpHeaderName`
private static final String[] reservedHeaderNames = {
"accept-charset",
"accept-encoding",
"connection",
"keep-alive",
"content-length",
"cookie",
"date",
"dnt",
"expect",
"host",
"keep-alive",
"origin",
"permissions-policy",
"referer",
"te",
"trailer",
"transfer-encoding",
"upgrade",
"via"
"te",
"transfer-encoding",
"trailer"
};
private static final String[] reservedHeaderPrefixes = {"proxy-", "sec-", "access-control-"};
// keep in sync with stdlib `EvaluatorSettings.reservedHttpHeaderPrefix`
private static final String[] reservedHeaderPrefixes = {"proxy-", "sec-"};
private IoUtils() {}
@@ -885,36 +889,50 @@ public final class IoUtils {
}
private static boolean isReservedHeaderName(String headerName) {
return Arrays.stream(reservedHeaderNames).anyMatch((reserved) -> headerName.equals(reserved));
var normalizedHeader = headerName.toLowerCase();
for (var reservedHeader : reservedHeaderNames) {
if (normalizedHeader.equals(reservedHeader)) {
return true;
}
}
return false;
}
private static boolean hasReservedHeaderPrefix(String headerName) {
return Arrays.stream(reservedHeaderPrefixes)
.anyMatch((prefix) -> headerName.startsWith(prefix));
var normalizedHeader = headerName.toLowerCase();
for (var prefix : reservedHeaderPrefixes) {
if (normalizedHeader.startsWith(prefix)) {
return true;
}
}
return false;
}
// keep in sync with stdlib EvaluatorSettings.HttpHeaderName
public static void validateHeaderName(String headerName) {
if (isReservedHeaderName(headerName)) {
throw new IllegalArgumentException(
"HTTP header '%s' is a reserved header".formatted(headerName));
ErrorMessages.create("invalidHttpHeaderReserved", headerName));
}
if (hasReservedHeaderPrefix(headerName)) {
throw new IllegalArgumentException(
"HTTP header '%s' starts with a reserved header prefix".formatted(headerName));
ErrorMessages.create("invalidHttpHeaderReservedPrefix", headerName));
}
if (!headerNameLike.matcher(headerName).matches()) {
throw new IllegalArgumentException(
"HTTP header name '%s' has an invalid syntax".formatted(headerName));
throw new IllegalArgumentException(ErrorMessages.create("invalidHttpHeaderName", headerName));
}
}
public static void validateHeaderValue(String headerValue) {
if (!headerValueLike.matcher(headerValue).matches()) {
throw new IllegalArgumentException(
"HTTP header value '%s' has an invalid syntax".formatted(headerValue));
ErrorMessages.create("invalidHttpHeaderValue", headerValue));
}
if (headerValue.length() > 4096) {
throw new IllegalArgumentException(
ErrorMessages.create("invalidHttpHeaderValueTooLong", headerValue));
}
}
}
pkl-core/src/main/resources/org/pkl/core/errorMessages.properties
+13 -3
View File
@@ -1133,8 +1133,18 @@ commandFlagInvalidType=\
Option `{0}` with annotation `@{1}` has invalid type `{2}`.\n\
Expected type: `{3}`
invalidHeaderName=\
HTTP header name `{0}` has invalid syntax.
invalidHttpHeaderReserved=\
HTTP header `{0}` is a reserved header.
invalidHeaderValue=\
invalidHttpHeaderReservedPrefix=\
HTTP header `{0}` starts with a reserved header prefix.
invalidHttpHeaderName=\
HTTP header `{0}` has invalid syntax.
invalidHttpHeaderValue=\
HTTP header value `{0}` has invalid syntax.
invalidHttpHeaderValueTooLong=\
HTTP Header value is invalid because it is longer than 4096 characters. \
Value: `{0}`
pkl-core/src/test/files/LanguageSnippetTests/input/api/evaluatorSettings.pkl
Vendored
+68
View File
@@ -0,0 +1,68 @@
amends "../snippetTest.pkl"
import "pkl:EvaluatorSettings"
local function force(http: EvaluatorSettings.Http) = new PcfRenderer {}.renderValue(http)
examples {
["http headers"] {
new EvaluatorSettings.Http {
headers {
["**"] {
["X-Foo"] = "Foo"
}
["https://example.com/**"] {
["X-Bar"] {
"bar"
"baz"
}
}
}
}
}
["http headers -- invalid glob pattern"] {
module.catch(() -> force(new EvaluatorSettings.Http {
headers {
["{{}}"] {
["X-Foo"] = "Foo"
}
}
}))
}
["http headers -- invalid header name"] {
module.catch(() -> force(new EvaluatorSettings.Http {
headers {
["**"] {
["Connection"] = "Foo"
}
}
}))
module.catch(() -> force(new EvaluatorSettings.Http {
headers {
["**"] {
["Sec-Foo-Bar"] = "Foo"
}
}
}))
module.catch(() -> force(new EvaluatorSettings.Http {
headers {
["**"] {
["Foo:Bar"] = "Foo"
}
}
}))
}
["http headers -- invalid header value"] {
module.catch(() -> force(new EvaluatorSettings.Http {
headers {
["**"] {
["My-Header"] = "🙃"
}
}
}))
}
}
pkl-core/src/test/files/LanguageSnippetTests/input/api/string.pkl
Vendored
+6
View File
@@ -85,6 +85,12 @@ facts {
!"(abc".isRegex
}
["isGlobPattern"] {
"**".isGlobPattern
"hello".isGlobPattern
!"{{}}".isGlobPattern
}
["toBoolean()"] {
"true".toBoolean()
"tRuE".toBoolean()
pkl-core/src/test/files/LanguageSnippetTests/output/api/evaluatorSettings.pcf
Vendored
+28
View File
@@ -0,0 +1,28 @@
examples {
["http headers"] {
new {
headers {
["**"] {
["X-Foo"] = "Foo"
}
["https://example.com/**"] {
["X-Bar"] {
"bar"
"baz"
}
}
}
}
}
["http headers -- invalid glob pattern"] {
"Type constraint `isGlobPattern` violated. Value: \"{{}}\""
}
["http headers -- invalid header name"] {
"Type constraint `isNotReservedHeaderName` violated. Value: \"Connection\""
"Type constraint `doesNotStartWithReservedPrefix` violated. Value: \"Sec-Foo-Bar\""
"Type constraint `hasValidHeaderNameSyntax` violated. Value: \"Foo:Bar\""
}
["http headers -- invalid header value"] {
"Expected value of type `*Listing<HttpHeaderValue> | HttpHeaderValue`, but got a different `String`. Value: \"🙃\""
}
}
pkl-core/src/test/files/LanguageSnippetTests/output/api/reflectedDeclaration.pcf
Vendored
+42
View File
@@ -914,6 +914,27 @@ alias {
name = "isRegex"
allModifiers = Set()
allAnnotations = List()
}, "isGlobPattern", new {
location {
line = XXXX
column = 3
displayUri = "https://github.com/apple/pkl/blob/$commitId/stdlib/base.pkl#LXXXX"
}
docComment = """
Tells if this string is a valid glob pattern.
For reference, see
<https://pkl-lang.org/main/current/language-reference/index.html#glob-patterns>.
"""
annotations = List(new {
version = "0.32.0"
})
modifiers = Set()
name = "isGlobPattern"
allModifiers = Set()
allAnnotations = List(new {
version = "0.32.0"
})
}, "isBase64", new {
location {
line = XXXX
@@ -1258,6 +1279,27 @@ alias {
name = "isRegex"
allModifiers = Set()
allAnnotations = List()
}, "isGlobPattern", new {
location {
line = XXXX
column = 3
displayUri = "https://github.com/apple/pkl/blob/$commitId/stdlib/base.pkl#LXXXX"
}
docComment = """
Tells if this string is a valid glob pattern.
For reference, see
<https://pkl-lang.org/main/current/language-reference/index.html#glob-patterns>.
"""
annotations = List(new {
version = "0.32.0"
})
modifiers = Set()
name = "isGlobPattern"
allModifiers = Set()
allAnnotations = List(new {
version = "0.32.0"
})
}, "isBase64", new {
location {
line = XXXX
pkl-core/src/test/files/LanguageSnippetTests/output/api/string.pcf
Vendored
+5
View File
@@ -65,6 +65,11 @@ facts {
true
true
}
["isGlobPattern"] {
true
true
true
}
["toBoolean()"] {
true
true
pkl-core/src/test/kotlin/org/pkl/core/EvaluatorBuilderTest.kt
+10 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -15,13 +15,17 @@
*/
package org.pkl.core
import java.net.URI
import java.nio.file.Path
import java.time.Duration
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.assertThrows
import org.pkl.core.http.LazyHttpClient
import org.pkl.core.http.RequestRewritingClient
import org.pkl.core.project.Project
import org.pkl.core.resource.TestResourceReader
import org.pkl.core.util.IoUtils
class EvaluatorBuilderTest {
@Test
@@ -93,5 +97,10 @@ class EvaluatorBuilderTest {
.isEqualTo(3) // two external readers, one module path
assertThat(builder.resourceReaders.find { it.uriScheme == "scheme3" }).isNotNull
assertThat(builder.resourceReaders.find { it.uriScheme == "scheme4" }).isNotNull
val client = (builder.httpClient as LazyHttpClient).orCreateClient as RequestRewritingClient
assertThat(client.headers)
.isEqualTo(mapOf(IoUtils.doubleStarGlob to mapOf("X-Foo" to listOf("Foo"))))
assertThat(client.rewritesMap)
.isEqualTo(mapOf(URI("https://foo.com/") to URI("https://bar.com/")))
}
}
pkl-core/src/test/kotlin/org/pkl/core/http/HttpClientBuilderTest.kt
+46
View File
@@ -0,0 +1,46 @@
/*
* Copyright © 2026 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.http
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test
class HttpClientBuilderTest {
@Test
fun `addHeader merges values for duplicate header names`() {
val client =
HttpClient.builder()
.setHeaders(mapOf("**" to mapOf("X-Foo" to listOf("v1"))))
.addHeaders("**", mapOf("X-Foo" to listOf("v2")))
.build() as RequestRewritingClient
val headerValues = client.headers.values.single()["X-Foo"]
assertThat(headerValues).containsExactly("v1", "v2")
}
@Test
fun `addHeader preserves non-overlapping header names`() {
val client =
HttpClient.builder()
.addHeaders("**", mapOf("X-Foo" to listOf("v1")))
.addHeaders("**", mapOf("X-Bar" to listOf("v2")))
.build() as RequestRewritingClient
val headerMap = client.headers.values.single()
assertThat(headerMap["X-Foo"]).containsExactly("v1")
assertThat(headerMap["X-Bar"]).containsExactly("v2")
}
}
pkl-core/src/test/kotlin/org/pkl/core/http/RequestRewritingClientTest.kt
+54 -20
View File
@@ -21,12 +21,13 @@ import java.net.http.HttpRequest
import java.net.http.HttpRequest.BodyPublishers
import java.net.http.HttpResponse.BodyHandlers
import java.time.Duration
import java.util.regex.Pattern
import org.assertj.core.api.Assertions.assertThat
import org.assertj.core.api.Assertions.assertThatList
import org.junit.jupiter.api.Test
import org.pkl.core.Pair as PPair
import org.pkl.core.util.GlobResolver
import org.pkl.core.util.IoUtils
@Suppress("UastIncorrectHttpHeaderInspection")
class RequestRewritingClientTest {
private val captured = RequestCapturingClient()
private val client =
@@ -36,7 +37,7 @@ class RequestRewritingClientTest {
-1,
captured,
mapOf(URI("https://foo/") to URI("https://bar/")),
listOf(),
mapOf(),
)
private val exampleUri = URI("https://example.com/foo/bar.html")
private val exampleRequest = HttpRequest.newBuilder(exampleUri).build()
@@ -48,6 +49,21 @@ class RequestRewritingClientTest {
assertThatList(captured.request.headers().allValues("User-Agent")).containsOnly("Pkl")
}
@Test
fun `User-Agent from configured headers takes precedence`() {
val client =
RequestRewritingClient(
"Pkl",
Duration.ofSeconds(42),
-1,
captured,
mapOf(URI("https://foo/") to URI("https://bar/")),
mapOf(IoUtils.doubleStarGlob to mapOf("User-Agent" to listOf("My-User-Agent"))),
)
client.send(exampleRequest, BodyHandlers.discarding())
assertThatList(captured.request.headers().allValues("User-Agent")).containsOnly("My-User-Agent")
}
@Test
fun `overrides existing User-Agent headers`() {
val request =
@@ -125,7 +141,7 @@ class RequestRewritingClientTest {
fun `rewrites port 0 if test port is set`() {
val captured = RequestCapturingClient()
val client =
RequestRewritingClient("Pkl", Duration.ofSeconds(42), 5000, captured, mapOf(), listOf())
RequestRewritingClient("Pkl", Duration.ofSeconds(42), 5000, captured, mapOf(), mapOf())
val request = HttpRequest.newBuilder(URI("https://example.com:0")).build()
client.send(request, BodyHandlers.discarding())
@@ -307,8 +323,7 @@ class RequestRewritingClientTest {
private fun rewrittenRequest(uri: String, rules: Map<URI, URI>): String {
val captured = RequestCapturingClient()
val client =
RequestRewritingClient("Pkl", Duration.ofSeconds(42), -1, captured, rules, listOf())
val client = RequestRewritingClient("Pkl", Duration.ofSeconds(42), -1, captured, rules, mapOf())
val request = HttpRequest.newBuilder(URI(uri)).build()
client.send(request, BodyHandlers.discarding())
return captured.request.uri().toString()
@@ -324,12 +339,10 @@ class RequestRewritingClientTest {
-1,
captured,
mapOf(),
listOf(
PPair(Pattern.compile("^https://example\\.com/.*"), listOf(PPair("x-one", "one"))),
PPair(
Pattern.compile("^https://example\\.com/foo/.*"),
listOf(PPair("x-two", "two-a"), PPair("x-two", "two-b")),
),
mapOf(
GlobResolver.toRegexPattern("https://example.com/**") to mapOf("x-one" to listOf("one")),
GlobResolver.toRegexPattern("https://example.com/foo/**") to
mapOf("x-two" to listOf("two-a", "two-b")),
),
)
val request = HttpRequest.newBuilder(URI("https://example.com/foo/bar")).build()
@@ -350,9 +363,9 @@ class RequestRewritingClientTest {
-1,
captured,
mapOf(),
listOf(
PPair(Pattern.compile("^https://foo\\.com/.*"), listOf(PPair("x-foo", "foo"))),
PPair(Pattern.compile("^https://bar\\.com/.*"), listOf(PPair("x-bar", "bar"))),
mapOf(
GlobResolver.toRegexPattern("https://foo.com/**") to mapOf("x-foo" to listOf("foo")),
GlobResolver.toRegexPattern("https://bar.com/**") to mapOf("x-bar" to listOf("bar")),
),
)
val request = HttpRequest.newBuilder(URI("https://example.com/foo/bar")).build()
@@ -373,11 +386,9 @@ class RequestRewritingClientTest {
-1,
captured,
mapOf(),
listOf(
PPair(
Pattern.compile("^https://example\\.com/.*"),
listOf(PPair("x-foo", "rule-a"), PPair("x-foo", "rule-b")),
)
mapOf(
GlobResolver.toRegexPattern("https://example.com/**") to
mapOf("x-foo" to listOf("rule-a", "rule-b"))
),
)
val request =
@@ -388,4 +399,27 @@ class RequestRewritingClientTest {
assertThatList(captured.request.headers().allValues("x-foo"))
.containsExactly("request", "rule-a", "rule-b")
}
@Test
fun `configured headers wins over configured user-agent header`() {
val captured = RequestCapturingClient()
val client =
RequestRewritingClient(
"Pkl",
Duration.ofSeconds(42),
-1,
captured,
mapOf(),
mapOf(
GlobResolver.toRegexPattern("https://example.com/**") to
mapOf("user-agent" to listOf("My User Agent"))
),
)
val request = HttpRequest.newBuilder(URI("https://example.com/foo/bar")).build()
client.send(request, BodyHandlers.discarding())
assertThatList(captured.request.headers().allValues("user-agent"))
.containsExactly("My User Agent")
}
}
pkl-core/src/test/kotlin/org/pkl/core/settings/PklSettingsTest.kt
+10 -5
View File
@@ -26,10 +26,8 @@ import org.pkl.commons.writeString
import org.pkl.core.Evaluator
import org.pkl.core.ModuleSource
import org.pkl.core.PObject
import org.pkl.core.Pair as PPair
import org.pkl.core.evaluatorSettings.PklEvaluatorSettings
import org.pkl.core.settings.PklSettings.Editor
import org.pkl.core.util.GlobResolver
class PklSettingsTest {
@Test
@@ -67,9 +65,15 @@ class PklSettingsTest {
["https://foo.com/"] = "https://bar.com/"
}
headers {
["https://foo.com/"] {
["https://foo.com/**"] {
["x-foo"] = "bar"
}
["https://bar.com/**"] {
["x-bar"] {
"bar"
"baz"
}
}
}
}
"""
@@ -84,8 +88,9 @@ class PklSettingsTest {
listOf("example.com", "pkg.pkl-lang.org"),
),
mapOf(URI("https://foo.com/") to URI("https://bar.com/")),
listOf(
PPair(GlobResolver.toRegexPattern("https://foo.com/"), listOf(PPair("x-foo", "bar")))
mapOf(
"https://foo.com/**" to mapOf("x-foo" to listOf("bar")),
"https://bar.com/**" to mapOf("x-bar" to listOf("bar", "baz")),
),
)
pkl-core/src/test/resources/org/pkl/core/project/project1/PklProject
+16
View File
@@ -41,4 +41,20 @@ evaluatorSettings {
arguments { "with"; "args" }
}
}
http {
headers {
["**"] {
["X-Foo"] = "Foo"
}
}
rewrites {
["https://foo.com/"] = "https://bar.com/"
}
proxy {
noProxy {
"my-no-proxy"
}
address = "http://localhost:5619"
}
}
}