Use java.net.http.HttpClient instead of java.net.Http(s)URLConnection (#217)

Moving to java.net.http.HttpClient brings many benefits, including
HTTP/2 support and the ability to make asynchronous requests.

Major additions and changes:
- Introduce a lightweight org.pkl.core.http.HttpClient API.
  This keeps some flexibility and allows to enforce behavior
  such as setting the User-Agent header.
- Provide an implementation that delegates to java.net.http.HttpClient.
- Use HttpClient for all HTTP(s) requests across the codebase.
  This required adding an HttpClient parameter to constructors and
  factory methods of multiple classes, some of which are public APIs.
- Manage CA certificates per HTTP client instead of per JVM.
  This makes it unnecessary to set JVM-wide system/security properties
  and default SSLSocketFactory's.
- Add executor v2 options to the executor SPI
- Add pkl-certs as a new artifact, and remove certs from pkl-commons-cli artifact

Each HTTP client maintains its own connection pool and SSLContext.
For efficiency reasons, It's best to reuse clients whenever feasible.
To avoid memory leaks, clients are not stored in static fields.

HTTP clients are expensive to create. For this reason,
EvaluatorBuilder defaults to a "lazy" client that creates the underlying
java.net.http.HttpClient on the first send (which may never happen).

pkl-cli/pkl-cli.gradle.kts

@@ -156,7 +156,7 @@ fun Exec.configureExecutable(isEnabled: Boolean, outputFile: File, extraArgs: Li
         ,"--no-fallback"
         ,"-H:IncludeResources=org/pkl/core/stdlib/.*\\.pkl"
         ,"-H:IncludeResources=org/jline/utils/.*"
-        ,"-H:IncludeResources=org/pkl/commons/cli/commands/IncludedCARoots.pem"
+        ,"-H:IncludeResources=org/pkl/certs/PklCARoots.pem"
         //,"-H:IncludeResources=org/pkl/core/Release.properties"
         ,"-H:IncludeResourceBundles=org.pkl.core.errorMessages"
         ,"--macro:truffle"

pkl-cli/src/main/kotlin/org/pkl/cli/CliPackageDownloader.kt

@@ -31,7 +31,7 @@ class CliPackageDownloader(
     if (moduleCacheDir == null) {
       throw CliException("Cannot download packages because no cache directory is specified.")
     }
-    val packageResolver = PackageResolver.getInstance(securityManager, moduleCacheDir)
+    val packageResolver = PackageResolver.getInstance(securityManager, httpClient, moduleCacheDir)
     val errors = mutableMapOf<PackageUri, Throwable>()
     for (pkg in packageUris) {
       try {

pkl-cli/src/main/kotlin/org/pkl/cli/CliProjectPackager.kt

@@ -82,6 +82,7 @@ class CliProjectPackager(
         outputPath,
         stackFrameTransformer,
         securityManager,
+        httpClient,
         skipPublishCheck,
         consoleWriter
       )

pkl-cli/src/main/kotlin/org/pkl/cli/CliProjectResolver.kt

@@ -40,6 +40,7 @@ class CliProjectResolver(
             SecurityManagers.defaultTrustLevels,
             rootDir
           ),
+          httpClient,
           moduleCacheDir
         )
       val dependencies = ProjectDependenciesResolver(project, packageResolver, errWriter).resolve()

pkl-cli/src/main/kotlin/org/pkl/cli/CliRepl.kt

@@ -36,6 +36,7 @@ internal class CliRepl(private val options: CliEvaluatorOptions) : CliCommand(op
             SecurityManagers.defaultTrustLevels,
             rootDir
           ),
+          httpClient,
           Loggers.stdErr(),
           listOf(
             ModuleKeyFactories.standardLibrary,

pkl-cli/src/main/kotlin/org/pkl/cli/CliServer.kt

@@ -25,7 +25,7 @@ import org.pkl.server.Server
 class CliServer(options: CliBaseOptions) : CliCommand(options) {
   override fun doRun() =
     try {
-      val server = Server(MessageTransports.stream(System.`in`, System.out))
+      val server = Server(MessageTransports.stream(System.`in`, System.out), httpClient)
       server.use { it.start() }
     } catch (e: ProtocolException) {
       throw CliException(e.message!!)

pkl-cli/src/test/kotlin/org/pkl/cli/CliEvaluatorTest.kt

@@ -21,19 +21,18 @@ import java.net.URI
 import java.nio.file.Files
 import java.nio.file.Path
 import java.time.Duration
-import kotlin.io.path.createDirectories
-import kotlin.io.path.listDirectoryEntries
+import kotlin.io.path.*
 import org.assertj.core.api.Assertions.assertThat
 import org.assertj.core.api.Assertions.assertThatCode
 import org.junit.jupiter.api.AfterEach
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertThrows
+import org.junit.jupiter.api.io.TempDir
 import org.junit.jupiter.params.ParameterizedTest
 import org.junit.jupiter.params.provider.EnumSource
 import org.pkl.commons.*
 import org.pkl.commons.cli.CliBaseOptions
 import org.pkl.commons.cli.CliException
-import org.pkl.commons.cli.commands.BaseOptions
 import org.pkl.commons.test.FileTestUtils
 import org.pkl.commons.test.PackageServer
 import org.pkl.core.OutputFormat
@@ -1158,8 +1157,46 @@ result = someLib.x
   }
 
   @Test
-  fun `not including the self signed certificate will result in a error`() {
+  fun `gives decent error message if certificate file contains random text`() {
+    val certsFile = tempDir.writeFile("random.pem", "RANDOM")
+    val err = assertThrows<CliException> { evalModuleThatImportsPackage(certsFile) }
+    assertThat(err)
+      .hasMessageContaining("Error parsing CA certificate file `${certsFile.pathString}`:")
+      .hasMessageContaining("No certificate data found")
+      .hasMessageNotContainingAny("java.", "sun.") // class names have been filtered out
+  }
+
+  @Test
+  fun `gives decent error message if certificate file is emtpy`(@TempDir tempDir: Path) {
+    val emptyCerts = tempDir.writeEmptyFile("empty.pem")
+    val err = assertThrows<CliException> { evalModuleThatImportsPackage(emptyCerts) }
+    assertThat(err).hasMessageContaining("CA certificate file `${emptyCerts.pathString}` is empty.")
+  }
+
+  @Test
+  fun `gives decent error message if certificate cannot be parsed`(@TempDir tempDir: Path) {
+    val invalidCerts = FileTestUtils.writeCertificateWithMissingLines(tempDir)
+    val err = assertThrows<CliException> { evalModuleThatImportsPackage(invalidCerts) }
+    assertThat(err)
+      // no assert for detail message because it differs between JDK implementations
+      .hasMessageContaining("Error parsing CA certificate file `${invalidCerts.pathString}`:")
+      .hasMessageNotContainingAny("java.", "sun.") // class names have been filtered out
+  }
+
+  @Test
+  fun `gives decent error message if CLI doesn't have the required CA certificate`() {
     PackageServer.ensureStarted()
+    // provide SOME certs to prevent CliEvaluator from falling back to ~/.pkl/cacerts
+    val builtInCerts = FileTestUtils.writePklBuiltInCertificates(tempDir)
+    val err = assertThrows<CliException> { evalModuleThatImportsPackage(builtInCerts) }
+    assertThat(err)
+      // on some JDK11's this doesn't cause SSLHandshakeException but some other SSLException
+      // .hasMessageContaining("Error during SSL handshake with host `localhost`:")
+      .hasMessageContaining("unable to find valid certification path to requested target")
+      .hasMessageNotContainingAny("java.", "sun.") // class names have been filtered out
+  }
+
+  private fun evalModuleThatImportsPackage(certsFile: Path) {
     val moduleUri =
       writePklFile(
         "test.pkl",
@@ -1168,22 +1205,17 @@ result = someLib.x
       
       res = Swallow
     """
-          .trimIndent()
       )
-    val buffer = StringWriter()
+
     val options =
       CliEvaluatorOptions(
         CliBaseOptions(
           sourceModules = listOf(moduleUri),
-          workingDir = tempDir,
-          moduleCacheDir = tempDir,
-          noCache = true,
-          // ensure we override any previously set root cert to the default buundle.
-          caCertificates = listOf(BaseOptions.Companion.includedCARootCerts())
+          caCertificates = listOf(certsFile),
+          noCache = true
         ),
       )
-    val err = assertThrows<CliException> { CliEvaluator(options, consoleWriter = buffer).run() }
-    assertThat(err.message).contains("unable to find valid certification path to requested target")
+    CliEvaluator(options).run()
   }
 
   private fun writePklFile(fileName: String, contents: String = defaultContents): URI {

pkl-cli/src/test/kotlin/org/pkl/cli/CliPackageDownloaderTest.kt

@@ -202,7 +202,7 @@ class CliPackageDownloaderTest {
 
         Failed to download package://bogus.domain/notAPackage@1.0.0 because:
         Exception when making request `GET https://bogus.domain/notAPackage@1.0.0`:
-        bogus.domain
+        Error connecting to host `bogus.domain`.
 
       """
           .trimIndent()

pkl-cli/src/test/kotlin/org/pkl/cli/CliProjectPackagerTest.kt

@@ -34,7 +34,6 @@ import org.pkl.commons.readString
 import org.pkl.commons.test.FileTestUtils
 import org.pkl.commons.test.PackageServer
 import org.pkl.commons.writeString
-import org.pkl.core.runtime.CertificateUtils
 
 class CliProjectPackagerTest {
   @Test
@@ -868,7 +867,6 @@ class CliProjectPackagerTest {
   @Test
   fun `publish checks`(@TempDir tempDir: Path) {
     PackageServer.ensureStarted()
-    CertificateUtils.setupAllX509CertificatesGlobally(listOf(FileTestUtils.selfSignedCertificate))
     tempDir.writeFile("project/main.pkl", "res = 1")
     tempDir.writeFile(
       "project/PklProject",
@@ -888,7 +886,10 @@ class CliProjectPackagerTest {
     val e =
       assertThrows<CliException> {
         CliProjectPackager(
-            CliBaseOptions(workingDir = tempDir),
+            CliBaseOptions(
+              workingDir = tempDir,
+              caCertificates = listOf(FileTestUtils.selfSignedCertificate)
+            ),
             listOf(tempDir.resolve("project")),
             CliTestOptions(),
             ".out/%{name}@%{version}",
@@ -912,7 +913,6 @@ class CliProjectPackagerTest {
   @Test
   fun `publish check when package is not yet published`(@TempDir tempDir: Path) {
     PackageServer.ensureStarted()
-    CertificateUtils.setupAllX509CertificatesGlobally(listOf(FileTestUtils.selfSignedCertificate))
     tempDir.writeFile("project/main.pkl", "res = 1")
     tempDir.writeFile(
       "project/PklProject",
@@ -930,7 +930,10 @@ class CliProjectPackagerTest {
     )
     val out = StringWriter()
     CliProjectPackager(
-        CliBaseOptions(workingDir = tempDir),
+        CliBaseOptions(
+          workingDir = tempDir,
+          caCertificates = listOf(FileTestUtils.selfSignedCertificate)
+        ),
         listOf(tempDir.resolve("project")),
         CliTestOptions(),
         ".out/%{name}@%{version}",

pkl-cli/src/test/kotlin/org/pkl/cli/repl/ReplMessagesTest.kt

@@ -21,6 +21,7 @@ import org.pkl.commons.toPath
 import org.pkl.core.Loggers
 import org.pkl.core.SecurityManagers
 import org.pkl.core.StackFrameTransformers
+import org.pkl.core.http.HttpClient
 import org.pkl.core.module.ModuleKeyFactories
 import org.pkl.core.repl.ReplRequest
 import org.pkl.core.repl.ReplResponse
@@ -30,6 +31,7 @@ class ReplMessagesTest {
   private val server =
     ReplServer(
       SecurityManagers.defaultManager,
+      HttpClient.dummyClient(),
       Loggers.stdErr(),
       listOf(ModuleKeyFactories.standardLibrary),
       listOf(),