Use java.net.http.HttpClient instead of java.net.Http(s)URLConnection (#217)

Moving to java.net.http.HttpClient brings many benefits, including
HTTP/2 support and the ability to make asynchronous requests.

Major additions and changes:
- Introduce a lightweight org.pkl.core.http.HttpClient API.
  This keeps some flexibility and allows to enforce behavior
  such as setting the User-Agent header.
- Provide an implementation that delegates to java.net.http.HttpClient.
- Use HttpClient for all HTTP(s) requests across the codebase.
  This required adding an HttpClient parameter to constructors and
  factory methods of multiple classes, some of which are public APIs.
- Manage CA certificates per HTTP client instead of per JVM.
  This makes it unnecessary to set JVM-wide system/security properties
  and default SSLSocketFactory's.
- Add executor v2 options to the executor SPI
- Add pkl-certs as a new artifact, and remove certs from pkl-commons-cli artifact

Each HTTP client maintains its own connection pool and SSLContext.
For efficiency reasons, It's best to reuse clients whenever feasible.
To avoid memory leaks, clients are not stored in static fields.

HTTP clients are expensive to create. For this reason,
EvaluatorBuilder defaults to a "lazy" client that creates the underlying
java.net.http.HttpClient on the first send (which may never happen).

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliBaseOptions.kt

@@ -20,6 +20,7 @@ import java.nio.file.Files
 import java.nio.file.Path
 import java.time.Duration
 import java.util.regex.Pattern
+import org.pkl.core.http.HttpClient
 import org.pkl.core.module.ProjectDependenciesManager
 import org.pkl.core.util.IoUtils
 
@@ -113,15 +114,15 @@ data class CliBaseOptions(
   val testMode: Boolean = false,
 
   /**
-   * [X.509 certificates](https://en.wikipedia.org/wiki/X.509) in PEM format.
+   * The CA certificates to trust.
    *
-   * Elements can either be a [Path] or a [java.io.InputStream]. Input streams are closed when
-   * [CliCommand] is initialized.
+   * The given files must contain [X.509](https://en.wikipedia.org/wiki/X.509) certificates in PEM
+   * format.
    *
-   * If not empty, this determines the CA root certs used for all HTTPS requests. Warning: this
-   * affects the whole Java runtime, not just the Pkl API!
+   * If [caCertificates] is the empty list, the certificate files in `~/.pkl/cacerts/` are used. If
+   * `~/.pkl/cacerts/` does not exist or is empty, Pkl's built-in CA certificates are used.
    */
-  val caCertificates: List<Any> = emptyList(),
+  val caCertificates: List<Path> = listOf(),
 ) {
 
   companion object {
@@ -167,4 +168,26 @@ data class CliBaseOptions(
     projectDir?.resolve(ProjectDependenciesManager.PKL_PROJECT_FILENAME)
       ?: normalizedWorkingDir.getProjectFile(rootDir)
   }
+
+  /** [caCertificates] after normalization. */
+  val normalizedCaCertificates: List<Path> = caCertificates.map(normalizedWorkingDir::resolve)
+
+  /**
+   * The HTTP client shared between CLI commands created with this [CliBaseOptions] instance.
+   *
+   * To release the resources held by the HTTP client in a timely manner, call its `close()` method.
+   */
+  val httpClient: HttpClient by lazy {
+    with(HttpClient.builder()) {
+      if (normalizedCaCertificates.isEmpty()) {
+        addDefaultCliCertificates()
+      } else {
+        for (file in normalizedCaCertificates) addCertificates(file)
+      }
+      // Lazy building significantly reduces execution time of commands that do minimal work.
+      // However, it means that HTTP client initialization errors won't surface until an HTTP
+      // request is made.
+      buildLazily()
+    }
+  }
 }

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliCommand.kt

@@ -18,24 +18,18 @@ package org.pkl.commons.cli
 import java.nio.file.Path
 import java.util.regex.Pattern
 import org.pkl.core.*
+import org.pkl.core.http.HttpClient
 import org.pkl.core.module.ModuleKeyFactories
 import org.pkl.core.module.ModuleKeyFactory
 import org.pkl.core.module.ModulePathResolver
 import org.pkl.core.project.Project
 import org.pkl.core.resource.ResourceReader
 import org.pkl.core.resource.ResourceReaders
-import org.pkl.core.runtime.CertificateUtils
 import org.pkl.core.settings.PklSettings
 import org.pkl.core.util.IoUtils
 
 /** Building block for CLI commands. Configured programmatically to allow for embedding. */
 abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
-  init {
-    if (cliOptions.caCertificates.isNotEmpty()) {
-      CertificateUtils.setupAllX509CertificatesGlobally(cliOptions.caCertificates)
-    }
-  }
-
   /** Runs this command. */
   fun run() {
     if (cliOptions.testMode) {
@@ -158,6 +152,10 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
     )
   }
 
+  // share HTTP client with other commands with the same cliOptions
+  protected val httpClient: HttpClient
+    get() = cliOptions.httpClient
+
   protected fun moduleKeyFactories(modulePathResolver: ModulePathResolver): List<ModuleKeyFactory> {
     return buildList {
       add(ModuleKeyFactories.standardLibrary)
@@ -195,6 +193,7 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
       .setStackFrameTransformer(stackFrameTransformer)
       .apply { project?.let { setProjectDependencies(it.dependencies) } }
       .setSecurityManager(securityManager)
+      .setHttpClient(httpClient)
       .setExternalProperties(externalProperties)
       .setEnvironmentVariables(environmentVariables)
       .addModuleKeyFactories(moduleKeyFactories(modulePathResolver))

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseOptions.kt

@@ -20,27 +20,15 @@ import com.github.ajalt.clikt.parameters.options.*
 import com.github.ajalt.clikt.parameters.types.long
 import com.github.ajalt.clikt.parameters.types.path
 import java.io.File
-import java.io.InputStream
 import java.net.URI
-import java.nio.file.Files
 import java.nio.file.Path
 import java.time.Duration
 import java.util.regex.Pattern
-import java.util.stream.Collectors
-import kotlin.io.path.exists
-import kotlin.io.path.isDirectory
-import kotlin.io.path.isRegularFile
 import org.pkl.commons.cli.CliBaseOptions
 import org.pkl.core.util.IoUtils
 
 @Suppress("MemberVisibilityCanBePrivate")
 class BaseOptions : OptionGroup() {
-  companion object {
-    fun includedCARootCerts(): InputStream {
-      return BaseOptions::class.java.getResourceAsStream("IncludedCARoots.pem")!!
-    }
-  }
-
   private val defaults = CliBaseOptions()
 
   private val output =
@@ -142,28 +130,11 @@ class BaseOptions : OptionGroup() {
     option(
         names = arrayOf("--ca-certificates"),
         metavar = "<path>",
-        help = "Replaces the built-in CA certificates with the provided certificate file."
+        help = "Only trust CA certificates from the provided file(s)."
       )
       .path()
       .multiple()
 
-  /**
-   * 1. If `--ca-certificates` option is not empty, use that.
-   * 2. If directory `~/.pkl/cacerts` is not empty, use that.
-   * 3. Use the bundled CA certificates.
-   */
-  private fun getEffectiveCaCertificates(): List<Any> {
-    return caCertificates
-      .ifEmpty {
-        val home = System.getProperty("user.home")
-        val cacerts = Path.of(home, ".pkl", "cacerts")
-        if (cacerts.exists() && cacerts.isDirectory())
-          Files.list(cacerts).filter(Path::isRegularFile).collect(Collectors.toList())
-        else emptyList()
-      }
-      .ifEmpty { listOf(includedCARootCerts()) }
-  }
-
   fun baseOptions(
     modules: List<URI>,
     projectOptions: ProjectOptions? = null,
@@ -186,7 +157,7 @@ class BaseOptions : OptionGroup() {
       testMode = testMode,
       omitProjectSettings = projectOptions?.omitProjectSettings ?: false,
       noProject = projectOptions?.noProject ?: false,
-      caCertificates = getEffectiveCaCertificates()
+      caCertificates = caCertificates
     )
   }
 }