Add support for HTTP proxying (#506)

* Add `--proxy` and `--no-proxy` CLI flags
* Add property `http` to `pkl:settings`
* Move `EvaluatorSettings` from `pkl:Project` to its own module and add property `http`
* Add support for proxying in server mode, and through Gradle
* Add `setProxy()` to `HttpClient`
* Add documentation

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliBaseOptions.kt

@@ -20,7 +20,6 @@ import java.nio.file.Files
 import java.nio.file.Path
 import java.time.Duration
 import java.util.regex.Pattern
-import org.pkl.core.http.HttpClient
 import org.pkl.core.module.ProjectDependenciesManager
 import org.pkl.core.util.IoUtils
 
@@ -129,6 +128,12 @@ data class CliBaseOptions(
    * `~/.pkl/cacerts/` does not exist or is empty, Pkl's built-in CA certificates are used.
    */
   val caCertificates: List<Path> = listOf(),
+
+  /** The proxy to connect to. */
+  val proxyAddress: URI? = null,
+
+  /** Hostnames, IP addresses, or CIDR blocks to not proxy. */
+  val noProxy: List<String>? = null,
 ) {
 
   companion object {
@@ -177,24 +182,4 @@ data class CliBaseOptions(
 
   /** [caCertificates] after normalization. */
   val normalizedCaCertificates: List<Path> = caCertificates.map(normalizedWorkingDir::resolve)
-
-  /**
-   * The HTTP client shared between CLI commands created with this [CliBaseOptions] instance.
-   *
-   * To release the resources held by the HTTP client in a timely manner, call its `close()` method.
-   */
-  val httpClient: HttpClient by lazy {
-    with(HttpClient.builder()) {
-      setTestPort(testPort)
-      if (normalizedCaCertificates.isEmpty()) {
-        addDefaultCliCertificates()
-      } else {
-        for (file in normalizedCaCertificates) addCertificates(file)
-      }
-      // Lazy building significantly reduces execution time of commands that do minimal work.
-      // However, it means that HTTP client initialization errors won't surface until an HTTP
-      // request is made.
-      buildLazily()
-    }
-  }
 }

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliCommand.kt

@@ -18,6 +18,7 @@ package org.pkl.commons.cli
 import java.nio.file.Path
 import java.util.regex.Pattern
 import org.pkl.core.*
+import org.pkl.core.evaluatorSettings.PklEvaluatorSettings
 import org.pkl.core.http.HttpClient
 import org.pkl.core.module.ModuleKeyFactories
 import org.pkl.core.module.ModuleKeyFactory
@@ -35,6 +36,9 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
     if (cliOptions.testMode) {
       IoUtils.setTestMode()
     }
+
+    proxyAddress?.let(IoUtils::setSystemProxy)
+
     try {
       doRun()
     } catch (e: PklException) {
@@ -97,42 +101,44 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
     )
   }
 
-  private val projectSettings: Project.EvaluatorSettings? by lazy {
-    if (cliOptions.omitProjectSettings) null else project?.settings
+  private val evaluatorSettings: PklEvaluatorSettings? by lazy {
+    if (cliOptions.omitProjectSettings) null else project?.evaluatorSettings
   }
 
   protected val allowedModules: List<Pattern> by lazy {
     cliOptions.allowedModules
-      ?: projectSettings?.allowedModules ?: SecurityManagers.defaultAllowedModules
+      ?: evaluatorSettings?.allowedModules ?: SecurityManagers.defaultAllowedModules
   }
 
   protected val allowedResources: List<Pattern> by lazy {
     cliOptions.allowedResources
-      ?: projectSettings?.allowedResources ?: SecurityManagers.defaultAllowedResources
+      ?: evaluatorSettings?.allowedResources ?: SecurityManagers.defaultAllowedResources
   }
 
-  protected val rootDir: Path? by lazy { cliOptions.normalizedRootDir ?: projectSettings?.rootDir }
+  protected val rootDir: Path? by lazy {
+    cliOptions.normalizedRootDir ?: evaluatorSettings?.rootDir
+  }
 
   protected val environmentVariables: Map<String, String> by lazy {
-    cliOptions.environmentVariables ?: projectSettings?.env ?: System.getenv()
+    cliOptions.environmentVariables ?: evaluatorSettings?.env ?: System.getenv()
   }
 
   protected val externalProperties: Map<String, String> by lazy {
-    cliOptions.externalProperties ?: projectSettings?.externalProperties ?: emptyMap()
+    cliOptions.externalProperties ?: evaluatorSettings?.externalProperties ?: emptyMap()
   }
 
   protected val moduleCacheDir: Path? by lazy {
     if (cliOptions.noCache) null
     else
       cliOptions.normalizedModuleCacheDir
-        ?: projectSettings?.let { settings ->
-          if (settings.isNoCache == true) null else settings.moduleCacheDir
+        ?: evaluatorSettings?.let { settings ->
+          if (settings.noCache == true) null else settings.moduleCacheDir
         }
           ?: IoUtils.getDefaultModuleCacheDir()
   }
 
   protected val modulePath: List<Path> by lazy {
-    cliOptions.normalizedModulePath ?: projectSettings?.modulePath ?: emptyList()
+    cliOptions.normalizedModulePath ?: evaluatorSettings?.modulePath ?: emptyList()
   }
 
   protected val stackFrameTransformer: StackFrameTransformer by lazy {
@@ -152,9 +158,36 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
     )
   }
 
-  // share HTTP client with other commands with the same cliOptions
-  protected val httpClient: HttpClient
-    get() = cliOptions.httpClient
+  private val proxyAddress =
+    cliOptions.proxyAddress
+      ?: project?.evaluatorSettings?.http?.proxy?.address ?: settings.http?.proxy?.address
+
+  private val noProxy =
+    cliOptions.noProxy
+      ?: project?.evaluatorSettings?.http?.proxy?.noProxy ?: settings.http?.proxy?.noProxy
+
+  /**
+   * The HTTP client used for this command.
+   *
+   * To release resources held by the HTTP client in a timely manner, call [HttpClient.close].
+   */
+  val httpClient: HttpClient by lazy {
+    with(HttpClient.builder()) {
+      setTestPort(cliOptions.testPort)
+      if (cliOptions.normalizedCaCertificates.isEmpty()) {
+        addDefaultCliCertificates()
+      } else {
+        for (file in cliOptions.normalizedCaCertificates) addCertificates(file)
+      }
+      if ((proxyAddress ?: noProxy) != null) {
+        setProxy(proxyAddress, noProxy ?: listOf())
+      }
+      // Lazy building significantly reduces execution time of commands that do minimal work.
+      // However, it means that HTTP client initialization errors won't surface until an HTTP
+      // request is made.
+      buildLazily()
+    }
+  }
 
   protected fun moduleKeyFactories(modulePathResolver: ModulePathResolver): List<ModuleKeyFactory> {
     return buildList {

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliMain.kt

@@ -27,6 +27,9 @@ fun cliMain(block: () -> Unit) {
     if (!message.endsWith('\n')) stream.println()
   }
 
+  // Force `native-image` to use system proxies (which does not happen with `-D`).
+  System.setProperty("java.net.useSystemProxies", "true")
+
   try {
     block()
   } catch (e: CliTestException) {

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseOptions.kt

@@ -172,6 +172,32 @@ class BaseOptions : OptionGroup() {
       .path()
       .multiple()
 
+  @Suppress("HttpUrlsUsage")
+  val proxy: URI? by
+    option(
+        names = arrayOf("--proxy"),
+        metavar = "<address>",
+        help = "Proxy to use for HTTP(S) connections."
+      )
+      .single()
+      .convert { URI(it) }
+      .validate { uri ->
+        require(
+          uri.scheme == "http" && uri.host != null && uri.path.isEmpty() && uri.userInfo == null
+        ) {
+          "Malformed proxy URI (expecting `http://<host>[:<port>]`)"
+        }
+      }
+
+  val noProxy: List<String>? by
+    option(
+        names = arrayOf("--no-proxy"),
+        metavar = "<pattern1,pattern2>",
+        help = "Hostnames that should not be connected to via a proxy."
+      )
+      .single()
+      .split(",")
+
   // hidden option used by native tests
   private val testPort: Int by
     option(names = arrayOf("--test-port"), help = "Internal test option", hidden = true)
@@ -202,7 +228,9 @@ class BaseOptions : OptionGroup() {
       testPort = testPort,
       omitProjectSettings = projectOptions?.omitProjectSettings ?: false,
       noProject = projectOptions?.noProject ?: false,
-      caCertificates = caCertificates
+      caCertificates = caCertificates,
+      proxyAddress = proxy,
+      noProxy = noProxy ?: emptyList()
     )
   }
 }