Add support for HTTP proxying (#506)

* Add `--proxy` and `--no-proxy` CLI flags
* Add property `http` to `pkl:settings`
* Move `EvaluatorSettings` from `pkl:Project` to its own module and add property `http`
* Add support for proxying in server mode, and through Gradle
* Add `setProxy()` to `HttpClient`
* Add documentation

pkl-server/gradle.lockfile

@@ -2,10 +2,10 @@
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
 com.tunnelvisionlabs:antlr4-runtime:4.9.0=compileClasspath,default,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
-net.bytebuddy:byte-buddy:1.14.11=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+net.bytebuddy:byte-buddy:1.14.16=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 net.java.dev.jna:jna:5.6.0=kotlinCompilerClasspath,kotlinKlibCommonizerClasspath
 org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeOnlyDependenciesMetadata
-org.assertj:assertj-core:3.25.3=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+org.assertj:assertj-core:3.26.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.graalvm.sdk:graal-sdk:23.0.2=compileClasspath,default,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.graalvm.truffle:truffle-api:23.0.2=compileClasspath,default,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.jetbrains.intellij.deps:trove4j:1.0.20200330=kotlinCompilerClasspath,kotlinKlibCommonizerClasspath

pkl-server/src/main/kotlin/org/pkl/server/Message.kt

@@ -20,6 +20,7 @@ import java.nio.file.Path
 import java.time.Duration
 import java.util.*
 import java.util.regex.Pattern
+import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.*
 import org.pkl.core.module.PathElement
 import org.pkl.core.packages.Checksums
 
@@ -123,6 +124,7 @@ data class CreateEvaluatorRequest(
   val cacheDir: Path?,
   val outputFormat: String?,
   val project: Project?,
+  val http: Http?,
 ) : ClientRequestMessage() {
   override val type = MessageType.CREATE_EVALUATOR_REQUEST
 

pkl-server/src/main/kotlin/org/pkl/server/MessagePackDecoder.kt

@@ -23,6 +23,7 @@ import org.msgpack.core.MessageTypeException
 import org.msgpack.core.MessageUnpacker
 import org.msgpack.value.Value
 import org.msgpack.value.impl.ImmutableStringValueImpl
+import org.pkl.core.evaluatorSettings.PklEvaluatorSettings
 import org.pkl.core.module.PathElement
 import org.pkl.core.packages.Checksums
 
@@ -50,8 +51,8 @@ internal class MessagePackDecoder(private val unpacker: MessageUnpacker) : Messa
             allowedModules = map.unpackStringListOrNull("allowedModules")?.map(Pattern::compile),
             allowedResources =
               map.unpackStringListOrNull("allowedResources")?.map(Pattern::compile),
-            clientModuleReaders = map.unpackModuleReaderSpec("clientModuleReaders"),
-            clientResourceReaders = map.unpackResourceReaderSpec("clientResourceReaders"),
+            clientModuleReaders = map.unpackModuleReaderSpec(),
+            clientResourceReaders = map.unpackResourceReaderSpec(),
             modulePaths = map.unpackStringListOrNull("modulePaths")?.map(Path::of),
             env = map.unpackStringMapOrNull("env"),
             properties = map.unpackStringMapOrNull("properties"),
@@ -59,7 +60,8 @@ internal class MessagePackDecoder(private val unpacker: MessageUnpacker) : Messa
             rootDir = map.unpackStringOrNull("rootDir")?.let(Path::of),
             cacheDir = map.unpackStringOrNull("cacheDir")?.let(Path::of),
             outputFormat = map.unpackStringOrNull("outputFormat"),
-            project = map.unpackProject("project")
+            project = map.unpackProject(),
+            http = map.unpackHttp(),
           )
         }
         MessageType.CREATE_EVALUATOR_RESPONSE.code -> {
@@ -221,8 +223,8 @@ internal class MessagePackDecoder(private val unpacker: MessageUnpacker) : Messa
       PathElement(map.unpackString("name"), map.unpackBoolean("isDirectory"))
     }
 
-  private fun Map<Value, Value>.unpackModuleReaderSpec(name: String): List<ModuleReaderSpec>? {
-    val keys = getNullable(name) ?: return null
+  private fun Map<Value, Value>.unpackModuleReaderSpec(): List<ModuleReaderSpec>? {
+    val keys = getNullable("clientModuleReaders") ?: return null
     return keys.asArrayValue().toList().map { value ->
       val readerMap = value.asMapValue().map()
       ModuleReaderSpec(
@@ -234,8 +236,8 @@ internal class MessagePackDecoder(private val unpacker: MessageUnpacker) : Messa
     }
   }
 
-  private fun Map<Value, Value>.unpackResourceReaderSpec(name: String): List<ResourceReaderSpec> {
-    val keys = getNullable(name) ?: return emptyList()
+  private fun Map<Value, Value>.unpackResourceReaderSpec(): List<ResourceReaderSpec> {
+    val keys = getNullable("clientResourceReaders") ?: return emptyList()
     return keys.asArrayValue().toList().map { value ->
       val readerMap = value.asMapValue().map()
       ResourceReaderSpec(
@@ -246,13 +248,26 @@ internal class MessagePackDecoder(private val unpacker: MessageUnpacker) : Messa
     }
   }
 
-  private fun Map<Value, Value>.unpackProject(name: String): Project? {
-    val projMap = getNullable(name)?.asMapValue()?.map() ?: return null
+  private fun Map<Value, Value>.unpackProject(): Project? {
+    val projMap = getNullable("project")?.asMapValue()?.map() ?: return null
     val projectFileUri = URI(projMap.unpackString("projectFileUri"))
     val dependencies = projMap.unpackDependencies("dependencies")
     return Project(projectFileUri, null, dependencies)
   }
 
+  private fun Map<Value, Value>.unpackHttp(): PklEvaluatorSettings.Http? {
+    val httpMap = getNullable("http")?.asMapValue()?.map() ?: return null
+    val proxy = httpMap.unpackProxy()
+    return PklEvaluatorSettings.Http(proxy)
+  }
+
+  private fun Map<Value, Value>.unpackProxy(): PklEvaluatorSettings.Proxy? {
+    val proxyMap = getNullable("proxy")?.asMapValue()?.map() ?: return null
+    val address = proxyMap.unpackString("address")
+    val noProxy = proxyMap.unpackStringListOrNull("noProxy")
+    return PklEvaluatorSettings.Proxy.create(address, noProxy)
+  }
+
   private fun Map<Value, Value>.unpackDependencies(name: String): Map<String, Dependency> {
     val mapValue = get(name).asMapValue().map()
     return mapValue.entries.associate { (key, value) ->

pkl-server/src/main/kotlin/org/pkl/server/Server.kt

@@ -29,9 +29,9 @@ import org.pkl.core.packages.PackageUri
 import org.pkl.core.project.DeclaredDependencies
 import org.pkl.core.resource.ResourceReader
 import org.pkl.core.resource.ResourceReaders
+import org.pkl.core.util.IoUtils
 
-class Server(private val transport: MessageTransport, private val httpClient: HttpClient) :
-  AutoCloseable {
+class Server(private val transport: MessageTransport) : AutoCloseable {
   private val evaluators: MutableMap<Long, BinaryEvaluator> = ConcurrentHashMap()
 
   // https://github.com/jano7/executor would be the perfect executor here
@@ -162,6 +162,14 @@ class Server(private val transport: MessageTransport, private val httpClient: Ht
     val properties = message.properties ?: emptyMap()
     val timeout = message.timeout
     val cacheDir = message.cacheDir
+    val http =
+      with(HttpClient.builder()) {
+        message.http?.proxy?.let { proxy ->
+          setProxy(proxy.address, message.http.proxy?.noProxy ?: listOf())
+          proxy.address?.let(IoUtils::setSystemProxy)
+        }
+        buildLazily()
+      }
     val dependencies =
       message.project?.let { proj ->
         buildDeclaredDependencies(proj.projectFileUri, proj.dependencies, null)
@@ -175,7 +183,7 @@ class Server(private val transport: MessageTransport, private val httpClient: Ht
         SecurityManagers.defaultTrustLevels,
         rootDir
       ),
-      httpClient,
+      http,
       ClientLogger(evaluatorId, transport),
       createModuleKeyFactories(message, evaluatorId, resolver),
       createResourceReaders(message, evaluatorId, resolver),

pkl-server/src/test/kotlin/org/pkl/server/AbstractServerTest.kt

@@ -1029,7 +1029,8 @@ abstract class AbstractServerTest {
         rootDir = null,
         cacheDir = cacheDir,
         outputFormat = null,
-        project = project
+        project = project,
+        http = null,
       )
 
     send(message)

pkl-server/src/test/kotlin/org/pkl/server/JvmServerTest.kt

@@ -19,7 +19,6 @@ import java.io.PipedInputStream
 import java.io.PipedOutputStream
 import org.junit.jupiter.api.AfterEach
 import org.junit.jupiter.api.BeforeEach
-import org.pkl.core.http.HttpClient
 
 class JvmServerTest : AbstractServerTest() {
   private val transports: Pair<MessageTransport, MessageTransport> = run {
@@ -35,7 +34,7 @@ class JvmServerTest : AbstractServerTest() {
   }
 
   override val client: TestTransport = TestTransport(transports.first)
-  private val server: Server = Server(transports.second, HttpClient.dummyClient())
+  private val server: Server = Server(transports.second)
 
   @BeforeEach
   fun beforeEach() {

pkl-server/src/test/kotlin/org/pkl/server/MessagePackCodecTest.kt

@@ -112,7 +112,8 @@ class MessagePackCodecTest {
                 "baz" to
                   RemoteDependency(URI("package://localhost:0/baz@1.1.0"), Checksums("abc123"))
               )
-          )
+          ),
+        http = null,
       )
     )
   }