Add support for HTTP rewrites (#1062)

This adds a new configuration option for the HTTP client to replace URI prefixes when making outbound calls.

Follows the design of https://github.com/apple/pkl-evolution/pull/17
This commit is contained in:
Daniel Chao
2025-07-16 15:53:31 -07:00
committed by GitHub
parent fea031a138
commit 99020bb79d
27 changed files with 607 additions and 47 deletions

View File

@@ -517,6 +517,20 @@ public final class EvaluatorBuilder {
procs.computeIfAbsent(entry.getValue(), ExternalReaderProcess::of)));
}
}
if (settings.http() != null) {
var httpClientBuilder = HttpClient.builder();
if (settings.http().proxy() != null) {
var noProxy = settings.http().proxy().noProxy();
if (noProxy == null) {
noProxy = Collections.emptyList();
}
httpClientBuilder.setProxy(settings.http().proxy().address(), noProxy);
}
if (settings.http().rewrites() != null) {
httpClientBuilder.setRewrites(settings.http().rewrites());
}
setHttpClient(httpClientBuilder.buildLazily());
}
return this;
}

View File

@@ -18,6 +18,8 @@ package org.pkl.core.evaluatorSettings;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.Path;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -121,15 +123,31 @@ public record PklEvaluatorSettings(
externalResourceReaders);
}
public record Http(@Nullable Proxy proxy) {
public static final Http DEFAULT = new Http(null);
public record Http(@Nullable Proxy proxy, @Nullable Map<URI, URI> rewrites) {
public static final Http DEFAULT = new Http(null, Collections.emptyMap());
@SuppressWarnings("unchecked")
public static @Nullable Http parse(@Nullable Value input) {
if (input == null || input instanceof PNull) {
return null;
} else if (input instanceof PObject http) {
var proxy = Proxy.parse((Value) http.getProperty("proxy"));
return proxy == null ? DEFAULT : new Http(proxy);
var rewrites = http.getProperty("rewrites");
if (rewrites instanceof PNull) {
return new Http(proxy, null);
} else {
var parsedRewrites = new HashMap<URI, URI>();
for (var entry : ((Map<String, String>) rewrites).entrySet()) {
var key = entry.getKey();
var value = entry.getValue();
try {
parsedRewrites.put(new URI(key), new URI(value));
} catch (URISyntaxException e) {
throw new PklException(ErrorMessages.create("invalidUri", e.getInput()));
}
}
return new Http(proxy, parsedRewrites);
}
} else {
throw PklBugException.unreachableCode();
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -22,6 +22,7 @@ import java.net.http.HttpResponse;
import java.net.http.HttpTimeoutException;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import javax.net.ssl.SSLContext;
import org.pkl.core.util.Nullable;
@@ -118,6 +119,10 @@ public interface HttpClient extends AutoCloseable {
*/
Builder setProxy(@Nullable URI proxyAddress, List<String> noProxy);
Builder setRewrites(Map<URI, URI> rewrites);
Builder addRewrite(URI sourcePrefix, URI targetPrefix);
/**
* Creates a new {@code HttpClient} from the current state of this builder.
*

View File

@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -15,13 +15,17 @@
*/
package org.pkl.core.http;
import static org.pkl.core.util.IoUtils.validateRewriteRule;
import java.net.ProxySelector;
import java.net.URI;
import java.nio.ByteBuffer;
import java.nio.file.Path;
import java.time.Duration;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;
import org.pkl.core.Release;
import org.pkl.core.http.HttpClient.Builder;
@@ -34,6 +38,7 @@ final class HttpClientBuilder implements HttpClient.Builder {
private final List<ByteBuffer> certificateBytes = new ArrayList<>();
private int testPort = -1;
private ProxySelector proxySelector;
private Map<URI, URI> rewrites = new HashMap<>();
HttpClientBuilder() {
var release = Release.current();
@@ -87,6 +92,24 @@ final class HttpClientBuilder implements HttpClient.Builder {
return this;
}
@Override
public Builder setRewrites(Map<URI, URI> rewrites) {
for (var entry : rewrites.entrySet()) {
validateRewriteRule(entry.getKey());
validateRewriteRule(entry.getValue());
}
this.rewrites = new HashMap<>(rewrites);
return this;
}
@Override
public Builder addRewrite(URI sourcePrefix, URI targetPrefix) {
validateRewriteRule(sourcePrefix);
validateRewriteRule(targetPrefix);
this.rewrites.put(sourcePrefix, targetPrefix);
return this;
}
@Override
public HttpClient build() {
return doBuild().get();
@@ -105,7 +128,7 @@ final class HttpClientBuilder implements HttpClient.Builder {
return () -> {
var jdkClient =
new JdkHttpClient(certificateFiles, certificateBytes, connectTimeout, proxySelector);
return new RequestRewritingClient(userAgent, requestTimeout, testPort, jdkClient);
return new RequestRewritingClient(userAgent, requestTimeout, testPort, jdkClient, rewrites);
};
}
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -17,13 +17,21 @@ package org.pkl.core.http;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.net.http.HttpResponse.BodyHandler;
import java.time.Duration;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicBoolean;
import javax.annotation.concurrent.ThreadSafe;
import org.pkl.core.PklBugException;
import org.pkl.core.util.HttpUtils;
import org.pkl.core.util.Nullable;
/**
* An {@code HttpClient} decorator that
@@ -32,6 +40,7 @@ import org.pkl.core.util.HttpUtils;
* <li>overrides the {@code User-Agent} header of {@code HttpRequest}s
* <li>sets a request timeout if none is present
* <li>ensures that {@link #close()} is idempotent.
* <li>rewrites outbound URI prefixes with another prefix.
* </ul>
*
* <p>Both {@code User-Agent} header and default request timeout are configurable through {@link
@@ -44,22 +53,43 @@ final class RequestRewritingClient implements HttpClient {
final Duration requestTimeout;
final int testPort;
final HttpClient delegate;
private final List<Entry<URI, URI>> rewrites;
private final AtomicBoolean closed = new AtomicBoolean();
RequestRewritingClient(
String userAgent, Duration requestTimeout, int testPort, HttpClient delegate) {
String userAgent,
Duration requestTimeout,
int testPort,
HttpClient delegate,
Map<URI, URI> rewrites) {
this.userAgent = userAgent;
this.requestTimeout = requestTimeout;
this.testPort = testPort;
this.delegate = delegate;
this.rewrites =
rewrites.entrySet().stream()
.map((it) -> Map.entry(normalizeRewrite(it.getKey()), normalizeRewrite(it.getValue())))
.sorted(Comparator.comparingInt((it) -> -it.getKey().toString().length()))
.toList();
}
@Override
public <T> HttpResponse<T> send(HttpRequest request, BodyHandler<T> responseBodyHandler)
throws IOException {
checkNotClosed(request);
return delegate.send(rewriteRequest(request), responseBodyHandler);
try {
return delegate.send(rewriteRequest(request), responseBodyHandler);
} catch (IOException e) {
var rewrittenUri = rewriteUri(request.uri());
if (rewrittenUri != request.uri()) {
throw new IOException(
e.getMessage()
+ " (request was rewritten: %s -> %s)".formatted(request.uri(), rewrittenUri),
e);
}
throw e;
}
}
@Override
@@ -99,11 +129,91 @@ final class RequestRewritingClient implements HttpClient {
return builder.build();
}
private URI rewriteUri(URI uri) {
if (testPort != -1 && uri.getPort() == 0) {
return HttpUtils.setPort(uri, testPort);
private static boolean notEqualCaseInsensitive(@Nullable String a, @Nullable String b) {
if (a == null || b == null) {
return !Objects.equals(a, b);
}
return uri;
return !a.equalsIgnoreCase(b);
}
// Our docs say to not include query string or fragment in a rewrite rule, but technically they
// are supported.
public static boolean matchesRewriteRule(URI uri, URI rule) {
if (notEqualCaseInsensitive(uri.getScheme(), rule.getScheme())) {
return false;
}
if (!Objects.equals(uri.getUserInfo(), rule.getUserInfo())) {
return false;
}
if (notEqualCaseInsensitive(uri.getHost(), rule.getHost())) {
return false;
}
if (!Objects.equals(uri.getPath(), rule.getPath())) {
if (uri.getPath() != null
&& rule.getPath() != null
&& rule.getQuery() == null
&& rule.getFragment() == null) {
return uri.getPath().startsWith(rule.getPath());
}
return false;
}
if (!Objects.equals(uri.getQuery(), rule.getQuery())) {
if (uri.getQuery() != null && rule.getQuery() != null && rule.getFragment() == null) {
return uri.getQuery().startsWith(rule.getQuery());
}
return false;
}
if (uri.getFragment() != null && rule.getFragment() != null) {
return uri.getFragment().startsWith(rule.getFragment());
}
return Objects.equals(uri.getFragment(), rule.getFragment());
}
private @Nullable Entry<URI, URI> findRewrite(URI uri) {
for (var entry : rewrites) {
if (matchesRewriteRule(uri, entry.getKey())) {
return entry;
}
}
return null;
}
private URI normalizeRewrite(URI uri) {
try {
return new URI(
uri.getScheme().toLowerCase(),
uri.getUserInfo(),
uri.getHost().toLowerCase(),
uri.getPort(),
uri.getPath(),
uri.getQuery(),
uri.getFragment());
} catch (URISyntaxException e) {
// impossible condition, we started from a valid URI to begin with
throw PklBugException.unreachableCode();
}
}
private URI rewriteUri(URI uri) {
var rewrite = findRewrite(uri);
var ret = uri;
if (rewrite != null) {
var normalized = normalizeRewrite(uri);
var fromUri = rewrite.getKey();
var toUri = rewrite.getValue();
var relativePath = fromUri.relativize(normalized);
ret = toUri.resolve(relativePath);
}
if (testPort != -1 && ret.getPort() == 0) {
ret = HttpUtils.setPort(ret, testPort);
}
return ret;
}
private void checkNotClosed(HttpRequest request) {

View File

@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -56,6 +56,13 @@ public abstract class AbstractMessagePackEncoder implements MessageEncoder {
packer.packMapHeader(size + (value1 != null ? 1 : 0) + (value2 != null ? 1 : 0));
}
protected void packMapHeader(
int size, @Nullable Object value1, @Nullable Object value2, @Nullable Object value3)
throws IOException {
packer.packMapHeader(
size + (value1 != null ? 1 : 0) + (value2 != null ? 1 : 0) + (value3 != null ? 1 : 0));
}
protected void packMapHeader(
int size,
@Nullable Object value1,

View File

@@ -841,4 +841,17 @@ public final class IoUtils {
throw new URISyntaxException(uri.toString(), ErrorMessages.create("invalidOpaqueFileUri"));
}
}
public static void validateRewriteRule(URI rewrite) {
if (!Objects.equals(rewrite.getScheme(), "http")
&& !Objects.equals(rewrite.getScheme(), "https")) {
throw new IllegalArgumentException(
"Rewrite rule must start with 'http://' or 'https://', but was '%s'".formatted(rewrite));
}
if (!rewrite.toString().endsWith("/")) {
throw new IllegalArgumentException(
"Rewrite rule must end with '/', but was '%s'".formatted(rewrite));
}
}
}

View File

@@ -1,3 +0,0 @@
import "pkl:analyze"
result = analyze.importGraph(Set("http://localhost:0/foo.pkl"))

View File

@@ -1,14 +0,0 @@
Pkl Error
HTTP/1.1 header parser received no bytes
x | result = analyze.importGraph(Set("http://localhost:0/foo.pkl"))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
at analyzeInvalidHttpModule#result (file:///$snippetsDir/input/errors/analyzeInvalidHttpModule.pkl)
xxx | text = renderer.renderDocument(value)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
at pkl.base#Module.output.text (pkl:base)
xxx | bytes = text.encodeToBytes("UTF-8")
^^^^
at pkl.base#Module.output.bytes (pkl:base)

View File

@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -27,7 +27,14 @@ import org.junit.jupiter.api.Test
class RequestRewritingClientTest {
private val captured = RequestCapturingClient()
private val client = RequestRewritingClient("Pkl", Duration.ofSeconds(42), -1, captured)
private val client =
RequestRewritingClient(
"Pkl",
Duration.ofSeconds(42),
-1,
captured,
mapOf(URI("https://foo/") to URI("https://bar/")),
)
private val exampleUri = URI("https://example.com/foo/bar.html")
private val exampleRequest = HttpRequest.newBuilder(exampleUri).build()
@@ -114,7 +121,7 @@ class RequestRewritingClientTest {
@Test
fun `rewrites port 0 if test port is set`() {
val captured = RequestCapturingClient()
val client = RequestRewritingClient("Pkl", Duration.ofSeconds(42), 5000, captured)
val client = RequestRewritingClient("Pkl", Duration.ofSeconds(42), 5000, captured, mapOf())
val request = HttpRequest.newBuilder(URI("https://example.com:0")).build()
client.send(request, BodyHandlers.discarding())
@@ -130,4 +137,175 @@ class RequestRewritingClientTest {
assertThat(captured.request.uri().port).isEqualTo(0)
}
@Test
fun `matches rewrite rule`() {
fun assertThatRewriteMatches(uri: String, rule: String) =
assertThat(RequestRewritingClient.matchesRewriteRule(URI(uri), URI(rule)))
.`as`("$uri matches $rule")
assertThatRewriteMatches("https://www.foo.com/path/to/qux.html", "https://www.foo.com/").isTrue
assertThatRewriteMatches("HTTPS://www.foo.com/path/to/qux.html", "https://www.foo.com/").isTrue
assertThatRewriteMatches("HTTPS://WWW.FOO.COM/path/to/qux.html", "https://www.foo.com/").isTrue
assertThatRewriteMatches("https://www.foo.com/path/to/qux.html", "https://www.foo.com/path/")
.isTrue
assertThatRewriteMatches("https://www.foo.com/path/to/qux.html", "https://www.foo.com/PATH/")
.isFalse
assertThatRewriteMatches("https://www.foo.com", "https://www.foo.com/").isFalse
assertThatRewriteMatches(
"https://www.foo.com/path/to/qux.html?foo&bar",
"https://www.foo.com/path/to/qux.html?foo&bar",
)
.isTrue
assertThatRewriteMatches(
"https://www.foo.com/path/to/qux.html?foo&baz",
"https://www.foo.com/path/to/qux.html?foo&bar",
)
.isFalse
assertThatRewriteMatches(
"https://www.foo.com/path/to/qux.html?foo&bar#qux",
"https://www.foo.com/path/to/qux.html?foo&bar#q",
)
.isTrue
assertThatRewriteMatches(
"https://www.foo.com/path/to/qux.html?foo&bar#qux",
"https://www.foo.com/path/to/qux.html?foo&bar#w",
)
.isFalse
assertThatRewriteMatches(
"https://www.foo.com/path/to/qux.html?foo&bar",
"https://www.foo.com/path/to/qux.html?foo&bar#w",
)
.isFalse
assertThatRewriteMatches("https:///", "https:///").isTrue
assertThatRewriteMatches("https:///", "http:///").isFalse
// userinfo
assertThatRewriteMatches("https://foo@foo.com/", "http://foo.com/").isFalse
assertThatRewriteMatches("https://foo@foo.com/", "http://foo@foo.com/").isFalse
assertThatRewriteMatches("https://foo@foo.com/", "http://FOO@foo.com/").isFalse
}
@Test
fun `rewrites URIs`() {
assertThat(
rewrittenRequest(
"https://foo.com/bar/baz",
mapOf(URI("https://foo.com/") to URI("https://bar.com/")),
)
)
.isEqualTo("https://bar.com/bar/baz")
assertThat(
rewrittenRequest(
"https://FOO.COM/bar/baz",
mapOf(URI("https://foo.com/") to URI("https://bar.com/")),
)
)
.isEqualTo("https://bar.com/bar/baz")
assertThat(
rewrittenRequest(
"https://foo.com/bar/baz",
mapOf(URI("https://FOO.COM/") to URI("https://bar.com/")),
)
)
.isEqualTo("https://bar.com/bar/baz")
assertThat(
rewrittenRequest(
"https://foo.com/bar/baz",
mapOf(URI("https://foo.com/") to URI("https://bar.com/qux/baz/")),
)
)
.isEqualTo("https://bar.com/qux/baz/bar/baz")
assertThat(
rewrittenRequest(
"https://foo.com/bar/baz",
mapOf(URI("https://foo.com/") to URI("https://bar.com/qux/baz/")),
)
)
.isEqualTo("https://bar.com/qux/baz/bar/baz")
assertThat(
rewrittenRequest(
"https://foo.com/bar/baz?qux=foo#corge",
mapOf(URI("https://foo.com/") to URI("https://bar.com/")),
)
)
.isEqualTo("https://bar.com/bar/baz?qux=foo#corge")
assertThat(
rewrittenRequest(
"https://fooey@foo.com/bar/baz",
mapOf(URI("https://fooey@foo.com/") to URI("https://bar.com/")),
)
)
.isEqualTo("https://bar.com/bar/baz")
}
@Test
fun `rewrites URIs - longest rewrite wins`() {
assertThat(
rewrittenRequest(
"https://foo.com/qux/bar/baz",
mapOf(
URI("https://foo.com/") to URI("https://bar.com/"),
URI("https://foo.com/qux") to URI("https://corge.com/"),
),
)
)
.isEqualTo("https://corge.com/bar/baz")
}
@Test
fun `rewrites URIs - hostname is always lowercased`() {
assertThat(
rewrittenRequest(
"https://foo.com/bar/baz",
mapOf(URI("https://FOO.com/") to URI("https://bar.com/")),
)
)
.isEqualTo("https://bar.com/bar/baz")
assertThat(
rewrittenRequest(
"https://FOO.com/bar/baz",
mapOf(URI("https://foo.com/") to URI("https://bar.com/")),
)
)
.isEqualTo("https://bar.com/bar/baz")
}
@Test
fun `rewrites URIs - scheme is always lowercased`() {
assertThat(
rewrittenRequest(
"HTTPS://foo.com/bar/baz",
mapOf(URI("https://foo.com/") to URI("https://bar.com/")),
)
)
.isEqualTo("https://bar.com/bar/baz")
assertThat(
rewrittenRequest(
"https://FOO.com/bar/baz",
mapOf(URI("HTTPS://foo.com/") to URI("HTTPS://bar.com/")),
)
)
.isEqualTo("https://bar.com/bar/baz")
}
private fun rewrittenRequest(uri: String, rules: Map<URI, URI>): String {
val captured = RequestCapturingClient()
val client = RequestRewritingClient("Pkl", Duration.ofSeconds(42), -1, captured, rules)
val request = HttpRequest.newBuilder(URI(uri)).build()
client.send(request, BodyHandlers.discarding())
return captured.request.uri().toString()
}
}

View File

@@ -61,6 +61,9 @@ class PklSettingsTest {
"pkg.pkl-lang.org"
}
}
rewrites {
["https://foo.com/"] = "https://bar.com/"
}
}
"""
.trimIndent()
@@ -72,7 +75,8 @@ class PklSettingsTest {
PklEvaluatorSettings.Proxy(
URI("http://localhost:8080"),
listOf("example.com", "pkg.pkl-lang.org"),
)
),
mapOf(URI("https://foo.com/") to URI("https://bar.com/")),
)
assertThat(settings).isEqualTo(PklSettings(Editor.SYSTEM, expectedHttp))
}
@@ -95,7 +99,10 @@ class PklSettingsTest {
val settings = PklSettings.loadFromPklHomeDir(tempDir)
val expectedHttp =
PklEvaluatorSettings.Http(PklEvaluatorSettings.Proxy(URI("http://localhost:8080"), listOf()))
PklEvaluatorSettings.Http(
PklEvaluatorSettings.Proxy(URI("http://localhost:8080"), listOf()),
null,
)
assertThat(settings).isEqualTo(PklSettings(Editor.SYSTEM, expectedHttp))
}