Improve handling of CA certificates (#518)

Instead of bundling Pkl's built-in CA certificates as a class path resource and loading them at runtime,
pass them to the native image compiler as the default SSL context's trust store.
This results in faster SSL initialization and is more consistent with how default certificates
are handled when running on the JVM.

Further related improvements:
- Remove HttpClientBuilder methods `addDefaultCliCertificates` and `addBuiltInCertificates`.
- Remove pkl-certs subproject and the optional dependencies on it.
- Move `PklCARoots.pem` to `pkl-cli/src/certs`.
- Fix certificate related error messages that were missing an argument.
- Prevent PklBugException if initialization of `CliBaseOptions.httpClient` fails.
- Add ability to set CA certificates as a byte array
- Add CA certificates option to message passing API

pkl-server/src/main/kotlin/org/pkl/server/Message.kt

@@ -20,7 +20,7 @@ import java.nio.file.Path
 import java.time.Duration
 import java.util.*
 import java.util.regex.Pattern
-import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.*
+import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.Proxy
 import org.pkl.core.module.PathElement
 import org.pkl.core.packages.Checksums
 
@@ -124,7 +124,7 @@ data class CreateEvaluatorRequest(
   val cacheDir: Path?,
   val outputFormat: String?,
   val project: Project?,
-  val http: Http?,
+  val http: Http?
 ) : ClientRequestMessage() {
   override val type = MessageType.CREATE_EVALUATOR_REQUEST
 
@@ -151,7 +151,8 @@ data class CreateEvaluatorRequest(
       rootDir.equalsNullable(other.rootDir) &&
       cacheDir.equalsNullable(other.cacheDir) &&
       outputFormat.equalsNullable(other.outputFormat) &&
-      project.equalsNullable(other.project)
+      project.equalsNullable(other.project) &&
+      http.equalsNullable(other.http)
   }
 
   @Suppress("DuplicatedCode") // false duplicate within method
@@ -170,6 +171,31 @@ data class CreateEvaluatorRequest(
     result = 31 * result + outputFormat.hashCode()
     result = 31 * result + project.hashCode()
     result = 31 * result + type.hashCode()
+    result = 31 * result + http.hashCode()
+    return result
+  }
+}
+
+data class Http(
+  /** PEM-format CA certificates as raw bytes. */
+  val caCertificates: ByteArray?,
+  /** Proxy settings */
+  val proxy: Proxy?
+) {
+  override fun equals(other: Any?): Boolean {
+    if (this === other) return true
+    if (other !is Http) return false
+
+    if (caCertificates != null) {
+      if (other.caCertificates == null) return false
+      if (!caCertificates.contentEquals(other.caCertificates)) return false
+    } else if (other.caCertificates != null) return false
+    return Objects.equals(proxy, other.proxy)
+  }
+
+  override fun hashCode(): Int {
+    var result = caCertificates?.contentHashCode() ?: 0
+    result = 31 * result + (proxy?.hashCode() ?: 0)
     return result
   }
 }

pkl-server/src/main/kotlin/org/pkl/server/MessagePackDecoder.kt

@@ -255,10 +255,11 @@ internal class MessagePackDecoder(private val unpacker: MessageUnpacker) : Messa
     return Project(projectFileUri, null, dependencies)
   }
 
-  private fun Map<Value, Value>.unpackHttp(): PklEvaluatorSettings.Http? {
+  private fun Map<Value, Value>.unpackHttp(): Http? {
     val httpMap = getNullable("http")?.asMapValue()?.map() ?: return null
     val proxy = httpMap.unpackProxy()
-    return PklEvaluatorSettings.Http(proxy)
+    val caCertificates = httpMap.getNullable("caCertificates")?.asBinaryValue()?.asByteArray()
+    return Http(caCertificates, proxy)
   }
 
   private fun Map<Value, Value>.unpackProxy(): PklEvaluatorSettings.Proxy? {

pkl-server/src/main/kotlin/org/pkl/server/MessagePackEncoder.kt

@@ -49,6 +49,21 @@ internal class MessagePackEncoder(private val packer: MessagePacker) : MessageEn
     packDependencies(project.dependencies)
   }
 
+  private fun MessagePacker.packHttp(http: Http) {
+    if ((http.caCertificates ?: http.proxy) == null) {
+      packMapHeader(0)
+      return
+    }
+    packMapHeader(0, http.caCertificates, http.proxy)
+    packKeyValue("caCertificates", http.caCertificates)
+    http.proxy?.let { proxy ->
+      packString("proxy")
+      packMapHeader(0, proxy.address, proxy.noProxy)
+      packKeyValue("address", proxy.address?.toString())
+      packKeyValue("noProxy", proxy.noProxy)
+    }
+  }
+
   private fun MessagePacker.packDependencies(dependencies: Map<String, Dependency>) {
     packMapHeader(dependencies.size)
     for ((name, dep) in dependencies) {
@@ -87,7 +102,15 @@ internal class MessagePackEncoder(private val packer: MessagePacker) : MessageEn
       when (msg.type.code) {
         MessageType.CREATE_EVALUATOR_REQUEST.code -> {
           msg as CreateEvaluatorRequest
-          packMapHeader(8, msg.timeout, msg.rootDir, msg.cacheDir, msg.outputFormat, msg.project)
+          packMapHeader(
+            8,
+            msg.timeout,
+            msg.rootDir,
+            msg.cacheDir,
+            msg.outputFormat,
+            msg.project,
+            msg.http
+          )
           packKeyValue("requestId", msg.requestId)
           packKeyValue("allowedModules", msg.allowedModules?.map { it.toString() })
           packKeyValue("allowedResources", msg.allowedResources?.map { it.toString() })
@@ -116,6 +139,10 @@ internal class MessagePackEncoder(private val packer: MessagePacker) : MessageEn
             packString("project")
             packProject(msg.project)
           }
+          if (msg.http != null) {
+            packString("http")
+            packHttp(msg.http)
+          }
         }
         MessageType.CREATE_EVALUATOR_RESPONSE.code -> {
           msg as CreateEvaluatorResponse
@@ -243,7 +270,8 @@ internal class MessagePackEncoder(private val packer: MessagePacker) : MessageEn
     value2: Any?,
     value3: Any?,
     value4: Any?,
-    value5: Any?
+    value5: Any?,
+    value6: Any?
   ) =
     packMapHeader(
       size +
@@ -251,7 +279,8 @@ internal class MessagePackEncoder(private val packer: MessagePacker) : MessageEn
         (if (value2 != null) 1 else 0) +
         (if (value3 != null) 1 else 0) +
         (if (value4 != null) 1 else 0) +
-        (if (value5 != null) 1 else 0)
+        (if (value5 != null) 1 else 0) +
+        (if (value6 != null) 1 else 0)
     )
 
   private fun MessagePacker.packKeyValue(name: String, value: Int?) {

pkl-server/src/main/kotlin/org/pkl/server/Server.kt

@@ -162,12 +162,13 @@ class Server(private val transport: MessageTransport) : AutoCloseable {
     val properties = message.properties ?: emptyMap()
     val timeout = message.timeout
     val cacheDir = message.cacheDir
-    val http =
+    val httpClient =
       with(HttpClient.builder()) {
         message.http?.proxy?.let { proxy ->
-          setProxy(proxy.address, message.http.proxy?.noProxy ?: listOf())
+          setProxy(proxy.address, proxy.noProxy ?: listOf())
           proxy.address?.let(IoUtils::setSystemProxy)
         }
+        message.http?.caCertificates?.let { caCertificates -> addCertificates(caCertificates) }
         buildLazily()
       }
     val dependencies =
@@ -183,7 +184,7 @@ class Server(private val transport: MessageTransport) : AutoCloseable {
         SecurityManagers.defaultTrustLevels,
         rootDir
       ),
-      http,
+      httpClient,
       ClientLogger(evaluatorId, transport),
       createModuleKeyFactories(message, evaluatorId, resolver),
       createResourceReaders(message, evaluatorId, resolver),

pkl-server/src/test/kotlin/org/pkl/server/AbstractServerTest.kt

@@ -1013,7 +1013,8 @@ abstract class AbstractServerTest {
     moduleReaders: List<ModuleReaderSpec> = listOf(),
     modulePaths: List<Path> = listOf(),
     project: Project? = null,
-    cacheDir: Path? = null
+    cacheDir: Path? = null,
+    http: Http? = null,
   ): Long {
     val message =
       CreateEvaluatorRequest(
@@ -1030,7 +1031,7 @@ abstract class AbstractServerTest {
         cacheDir = cacheDir,
         outputFormat = null,
         project = project,
-        http = null,
+        http = http
       )
 
     send(message)

pkl-server/src/test/kotlin/org/pkl/server/MessagePackCodecTest.kt

@@ -25,6 +25,7 @@ import org.assertj.core.api.Assertions.assertThat
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertThrows
 import org.msgpack.core.MessagePack
+import org.pkl.core.evaluatorSettings.PklEvaluatorSettings
 import org.pkl.core.module.PathElement
 import org.pkl.core.packages.Checksums
 
@@ -73,6 +74,7 @@ class MessagePackCodecTest {
         isGlobbable = false,
         isLocal = false
       )
+    @Suppress("HttpUrlsUsage")
     roundtrip(
       CreateEvaluatorRequest(
         requestId = 123,
@@ -113,7 +115,11 @@ class MessagePackCodecTest {
                   RemoteDependency(URI("package://localhost:0/baz@1.1.0"), Checksums("abc123"))
               )
           ),
-        http = null,
+        http =
+          Http(
+            proxy = PklEvaluatorSettings.Proxy(URI("http://foo.com:1234"), listOf("bar", "baz")),
+            caCertificates = byteArrayOf(1, 2, 3, 4)
+          )
       )
     )
   }