diff --git a/docs/modules/bindings-specification/pages/message-passing-api.adoc b/docs/modules/bindings-specification/pages/message-passing-api.adoc index b4b54aeb..d75a6eb0 100644 --- a/docs/modules/bindings-specification/pages/message-passing-api.adoc +++ b/docs/modules/bindings-specification/pages/message-passing-api.adoc @@ -121,7 +121,7 @@ outputFormat: String? /// The project dependency settings. project: Project? -/// Configuration of outgoing HTTP requests. +/// Configuration of outgoing HTTP(s) requests. http: Http? class ClientResourceReader { @@ -178,8 +178,27 @@ class Project { dependencies: Mapping } -/// Settings that control how Pkl talks to HTTP(S) servers. +class RemoteDependency { + type: "remote" + + /// The canonical URI of this dependency + packageUri: String? + + /// The checksums of this remote dependency + checksums: Checksums? +} + +class Checksums { + /// The sha-256 checksum of this dependency's metadata. + sha256: String +} + class Http { + /// PEM format certificates to trust when making HTTP requests. + /// + /// If [null], Pkl will trust its own built-in certificates. + caCertificates: Binary? + /// Configuration of the HTTP proxy to use. /// /// If [null], uses the operating system's proxy configuration. @@ -226,21 +245,9 @@ class Proxy { noProxy: Listing(isDistinct) } -class RemoteDependency { - type: "remote" - - /// The canonical URI of this dependency - packageUri: String? - - /// The checksums of this remote dependency - checksums: Checksums? -} - -class Checksums { - /// The sha-256 checksum of this dependency's metadata. - sha256: String -} +typealias Binary = Any // <1> ---- +<1> link:{uri-messagepack-bin}[bin format] (not expressable in Pkl) Example: [source,json5] diff --git a/pkl-certs/pkl-certs.gradle.kts b/pkl-certs/pkl-certs.gradle.kts deleted file mode 100644 index 15322c78..00000000 --- a/pkl-certs/pkl-certs.gradle.kts +++ /dev/null @@ -1,19 +0,0 @@ -plugins { - pklAllProjects - pklJavaLibrary - pklPublishLibrary -} - -publishing { - publications { - named("library") { - pom { - url.set("https://github.com/apple/pkl/tree/main/pkl-certs") - description.set(""" - Pkl's built-in CA certificates. - Used by Pkl CLIs and optionally supported by pkl-core.") - """.trimIndent()) - } - } - } -} diff --git a/pkl-cli/pkl-cli.gradle.kts b/pkl-cli/pkl-cli.gradle.kts index cc80cfd6..71d880b2 100644 --- a/pkl-cli/pkl-cli.gradle.kts +++ b/pkl-cli/pkl-cli.gradle.kts @@ -1,3 +1,6 @@ +import java.security.KeyStore +import java.security.cert.CertificateFactory + plugins { pklAllProjects pklKotlinLibrary @@ -35,6 +38,8 @@ val stagedLinuxAarch64Executable: Configuration by configurations.creating val stagedAlpineLinuxAmd64Executable: Configuration by configurations.creating val stagedWindowsAmd64Executable: Configuration by configurations.creating +val certs: SourceSet by sourceSets.creating + dependencies { compileOnly(libs.svm) @@ -142,11 +147,38 @@ tasks.check { dependsOn(testStartJavaExecutable) } +val trustStore = layout.buildDirectory.dir("generateTrustStore/PklCARoots.p12") +val trustStorePassword = "password" // no sensitive data to protect + +// generate a trust store for Pkl's built-in CA certificates +val generateTrustStore by tasks.registering { + inputs.file(certs.resources.singleFile) + outputs.file(trustStore) + doLast { + val certificates = certs.resources.singleFile.inputStream().use { stream -> + CertificateFactory.getInstance("X.509").generateCertificates(stream) + } + KeyStore.getInstance("PKCS12").apply { + load(null, trustStorePassword.toCharArray()) // initialize empty trust store + for ((index, certificate) in certificates.withIndex()) { + setCertificateEntry("cert-$index", certificate) + } + val trustStoreFile = trustStore.get().asFile + trustStoreFile.parentFile.mkdirs() + trustStoreFile.outputStream().use { stream -> + store(stream, trustStorePassword.toCharArray()) + } + } + } +} + fun Exec.configureExecutable( graalVm: BuildInfo.GraalVm, outputFile: Provider, extraArgs: List = listOf() ) { + dependsOn(generateTrustStore) + inputs.files(sourceSets.main.map { it.output }) .withPropertyName("mainSourceSets") .withPathSensitivity(PathSensitivity.RELATIVE) @@ -175,9 +207,13 @@ fun Exec.configureExecutable( // needed for messagepack-java (see https://github.com/msgpack/msgpack-java/issues/600) ,"--initialize-at-run-time=org.msgpack.core.buffer.DirectBufferAccess" ,"--no-fallback" + ,"-Djavax.net.ssl.trustStore=${trustStore.get().asFile}" + ,"-Djavax.net.ssl.trustStorePassword=$trustStorePassword" + ,"-Djavax.net.ssl.trustStoreType=PKCS12" + // security property "ocsp.enable=true" is set in Main.kt + ,"-Dcom.sun.net.ssl.checkRevocation=true" ,"-H:IncludeResources=org/pkl/core/stdlib/.*\\.pkl" ,"-H:IncludeResources=org/jline/utils/.*" - ,"-H:IncludeResources=org/pkl/certs/PklCARoots.pem" ,"-H:IncludeResourceBundles=org.pkl.core.errorMessages" ,"--macro:truffle" ,"-H:Class=org.pkl.cli.Main" diff --git a/pkl-certs/src/main/resources/org/pkl/certs/PklCARoots.pem b/pkl-cli/src/certs/resources/PklCARoots.pem similarity index 100% rename from pkl-certs/src/main/resources/org/pkl/certs/PklCARoots.pem rename to pkl-cli/src/certs/resources/PklCARoots.pem diff --git a/pkl-cli/src/test/kotlin/org/pkl/cli/CliEvaluatorTest.kt b/pkl-cli/src/test/kotlin/org/pkl/cli/CliEvaluatorTest.kt index 34b273d0..5ce38755 100644 --- a/pkl-cli/src/test/kotlin/org/pkl/cli/CliEvaluatorTest.kt +++ b/pkl-cli/src/test/kotlin/org/pkl/cli/CliEvaluatorTest.kt @@ -1260,10 +1260,7 @@ result = someLib.x @Test fun `gives decent error message if CLI doesn't have the required CA certificate`() { - // provide SOME certs to prevent CliEvaluator from falling back to ~/.pkl/cacerts - val builtInCerts = FileTestUtils.writePklBuiltInCertificates(tempDir) - val err = - assertThrows { evalModuleThatImportsPackage(builtInCerts, packageServer.port) } + val err = assertThrows { evalModuleThatImportsPackage(null, packageServer.port) } assertThat(err) .hasMessageContaining("Error during SSL handshake with host `localhost`:") .hasMessageContaining("unable to find valid certification path to requested target") @@ -1460,7 +1457,7 @@ result = someLib.x assertThat(output).isEqualTo("result = 1\n") } - private fun evalModuleThatImportsPackage(certsFile: Path, testPort: Int = -1) { + private fun evalModuleThatImportsPackage(certsFile: Path?, testPort: Int = -1) { val moduleUri = writePklFile( "test.pkl", @@ -1475,7 +1472,7 @@ result = someLib.x CliEvaluatorOptions( CliBaseOptions( sourceModules = listOf(moduleUri), - caCertificates = listOf(certsFile), + caCertificates = buildList { if (certsFile != null) add(certsFile) }, workingDir = tempDir, noCache = true, testPort = testPort diff --git a/pkl-commons-cli/pkl-commons-cli.gradle.kts b/pkl-commons-cli/pkl-commons-cli.gradle.kts index a49d393c..d836b138 100644 --- a/pkl-commons-cli/pkl-commons-cli.gradle.kts +++ b/pkl-commons-cli/pkl-commons-cli.gradle.kts @@ -14,7 +14,6 @@ dependencies { implementation(projects.pklCommons) testImplementation(projects.pklCommonsTest) - runtimeOnly(projects.pklCerts) } publishing { diff --git a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliCommand.kt b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliCommand.kt index 054c6722..ca283072 100644 --- a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliCommand.kt +++ b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliCommand.kt @@ -15,8 +15,10 @@ */ package org.pkl.commons.cli +import java.nio.file.Files import java.nio.file.Path import java.util.regex.Pattern +import kotlin.io.path.isRegularFile import org.pkl.core.* import org.pkl.core.evaluatorSettings.PklEvaluatorSettings import org.pkl.core.http.HttpClient @@ -166,6 +168,13 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) { cliOptions.noProxy ?: project?.evaluatorSettings?.http?.proxy?.noProxy ?: settings.http?.proxy?.noProxy + private fun HttpClient.Builder.addDefaultCliCertificates() { + val caCertsDir = IoUtils.getPklHomeDir().resolve("cacerts") + if (Files.isDirectory(caCertsDir)) { + Files.list(caCertsDir).filter { it.isRegularFile() }.forEach { addCertificates(it) } + } + } + /** * The HTTP client used for this command. * diff --git a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliMain.kt b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliMain.kt index c586a6cb..200b7414 100644 --- a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliMain.kt +++ b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliMain.kt @@ -16,6 +16,7 @@ package org.pkl.commons.cli import java.io.PrintStream +import java.security.Security import kotlin.system.exitProcess /** Building block for CLIs. Intended to be called from a `main` method. */ @@ -29,6 +30,8 @@ fun cliMain(block: () -> Unit) { // Force `native-image` to use system proxies (which does not happen with `-D`). System.setProperty("java.net.useSystemProxies", "true") + // enable OCSP for default SSL context + Security.setProperty("ocsp.enable", "true") try { block() diff --git a/pkl-commons-test/pkl-commons-test.gradle.kts b/pkl-commons-test/pkl-commons-test.gradle.kts index fc3e40b0..a9927f86 100644 --- a/pkl-commons-test/pkl-commons-test.gradle.kts +++ b/pkl-commons-test/pkl-commons-test.gradle.kts @@ -13,7 +13,6 @@ dependencies { api(libs.junitParams) api(projects.pklCommons) // for convenience implementation(libs.assertj) - runtimeOnly(projects.pklCerts) } /** diff --git a/pkl-commons-test/src/main/kotlin/org/pkl/commons/test/FileTestUtils.kt b/pkl-commons-test/src/main/kotlin/org/pkl/commons/test/FileTestUtils.kt index 43d88146..2bc36723 100644 --- a/pkl-commons-test/src/main/kotlin/org/pkl/commons/test/FileTestUtils.kt +++ b/pkl-commons-test/src/main/kotlin/org/pkl/commons/test/FileTestUtils.kt @@ -38,11 +38,6 @@ object FileTestUtils { // drop some lines in the middle return dir.resolve("invalidCerts.pem").writeLines(lines.take(5) + lines.takeLast(5)) } - - fun writePklBuiltInCertificates(dir: Path): Path { - val text = javaClass.getResource("/org/pkl/certs/PklCARoots.pem")!!.readText() - return dir.resolve("PklCARoots.pem").apply { writeText(text) } - } } fun Path.listFilesRecursively(): List = diff --git a/pkl-core/src/main/java/org/pkl/core/http/HttpClient.java b/pkl-core/src/main/java/org/pkl/core/http/HttpClient.java index 1d26106b..d0f3c581 100644 --- a/pkl-core/src/main/java/org/pkl/core/http/HttpClient.java +++ b/pkl-core/src/main/java/org/pkl/core/http/HttpClient.java @@ -67,49 +67,22 @@ public interface HttpClient extends AutoCloseable { * *

The given file must contain X.509 * certificates in PEM format. + * + *

If no CA certificates are added via this method nor {@link #addCertificates(byte[])}, the + * built-in CA certificates of the Pkl native executable or JVM are used. */ - Builder addCertificates(Path file); + Builder addCertificates(Path path); /** - * Adds a CA certificate file to the client's trust store. + * Adds CA certificate bytes to the client's trust store. * - *

The given file must contain X.509 - * certificates in PEM format. + *

The given cert must be an X.509 + * certificate in PEM format. * - *

This method is intended to be used for adding certificate files located on the class path. - * To add certificate files located on the file system, use {@link #addCertificates(Path)}. - * - * @throws HttpClientInitException if the given URI has a scheme other than {@code jar:} or - * {@code file:} + *

If no CA certificates are added via this method nor {@link #addCertificates(Path)}, the + * built-in CA certificates of the Pkl native executable or JVM are used. */ - Builder addCertificates(URI file); - - /** - * Adds the CA certificate files in {@code ~/.pkl/cacerts/} to the client's trust store. - * - *

Each file must contain X.509 - * certificates in PEM format. If {@code ~/.pkl/cacerts/} does not exist or is empty, Pkl's - * {@link #addBuiltInCertificates() built-in certificates} are added instead. - * - *

This method implements the default behavior of Pkl CLIs. - * - *

NOTE: This method requires the optional {@code pkl-certs} JAR to be present on the class - * path. - * - * @throws HttpClientInitException if an I/O error occurs while scanning {@code ~/.pkl/cacerts/} - * or the {@code pkl-certs} JAR is not found on the class path - */ - Builder addDefaultCliCertificates(); - - /** - * Adds Pkl's built-in CA certificates to the client's trust store. - * - *

NOTE: This method requires the optional {@code pkl-certs} JAR to be present on the class - * path. - * - * @throws HttpClientInitException if the {@code pkl-certs} JAR is not found on the class path - */ - Builder addBuiltInCertificates(); + Builder addCertificates(byte[] certificateBytes); /** * Sets a test server's listening port. diff --git a/pkl-core/src/main/java/org/pkl/core/http/HttpClientBuilder.java b/pkl-core/src/main/java/org/pkl/core/http/HttpClientBuilder.java index 2a99b17d..7cc96bf1 100644 --- a/pkl-core/src/main/java/org/pkl/core/http/HttpClientBuilder.java +++ b/pkl-core/src/main/java/org/pkl/core/http/HttpClientBuilder.java @@ -15,11 +15,9 @@ */ package org.pkl.core.http; -import java.io.IOException; import java.net.ProxySelector; import java.net.URI; -import java.net.URISyntaxException; -import java.nio.file.Files; +import java.nio.ByteBuffer; import java.nio.file.Path; import java.time.Duration; import java.util.ArrayList; @@ -27,27 +25,18 @@ import java.util.List; import java.util.function.Supplier; import org.pkl.core.Release; import org.pkl.core.http.HttpClient.Builder; -import org.pkl.core.util.ErrorMessages; -import org.pkl.core.util.IoUtils; final class HttpClientBuilder implements HttpClient.Builder { private String userAgent; private Duration connectTimeout = Duration.ofSeconds(60); private Duration requestTimeout = Duration.ofSeconds(60); - private final Path caCertsDir; private final List certificateFiles = new ArrayList<>(); - private final List certificateUris = new ArrayList<>(); + private final List certificateBytes = new ArrayList<>(); private int testPort = -1; private ProxySelector proxySelector; HttpClientBuilder() { - this(IoUtils.getPklHomeDir().resolve("cacerts")); - } - - // only exists for testing - HttpClientBuilder(Path caCertsDir) { var release = Release.current(); - this.caCertsDir = caCertsDir; this.userAgent = "Pkl/" + release.version() + " (" + release.os() + "; " + release.flavor() + ")"; } @@ -70,39 +59,14 @@ final class HttpClientBuilder implements HttpClient.Builder { } @Override - public HttpClient.Builder addCertificates(Path file) { - certificateFiles.add(file); + public HttpClient.Builder addCertificates(Path path) { + certificateFiles.add(path); return this; } @Override - public HttpClient.Builder addCertificates(URI url) { - var scheme = url.getScheme(); - if (!"jar".equalsIgnoreCase(scheme) && !"file".equalsIgnoreCase(scheme)) { - throw new HttpClientInitException(ErrorMessages.create("expectedJarOrFileUrl", url)); - } - certificateUris.add(url); - return this; - } - - public HttpClient.Builder addDefaultCliCertificates() { - var fileCount = certificateFiles.size(); - if (Files.isDirectory(caCertsDir)) { - try (var files = Files.list(caCertsDir)) { - files.filter(Files::isRegularFile).forEach(certificateFiles::add); - } catch (IOException e) { - throw new HttpClientInitException(e); - } - } - if (certificateFiles.size() == fileCount) { - addBuiltInCertificates(); - } - return this; - } - - @Override - public HttpClient.Builder addBuiltInCertificates() { - certificateUris.add(getBuiltInCertificates()); + public Builder addCertificates(byte[] certificateBytes) { + this.certificateBytes.add(ByteBuffer.wrap(certificateBytes)); return this; } @@ -134,27 +98,14 @@ final class HttpClientBuilder implements HttpClient.Builder { } private Supplier doBuild() { - // make defensive copies because Supplier may get called after builder was mutated + // make defensive copy because Supplier may get called after builder was mutated var certificateFiles = List.copyOf(this.certificateFiles); - var certificateUris = List.copyOf(this.certificateUris); var proxySelector = this.proxySelector != null ? this.proxySelector : java.net.ProxySelector.getDefault(); return () -> { var jdkClient = - new JdkHttpClient(certificateFiles, certificateUris, connectTimeout, proxySelector); + new JdkHttpClient(certificateFiles, certificateBytes, connectTimeout, proxySelector); return new RequestRewritingClient(userAgent, requestTimeout, testPort, jdkClient); }; } - - private static URI getBuiltInCertificates() { - var resource = HttpClientBuilder.class.getResource("/org/pkl/certs/PklCARoots.pem"); - if (resource == null) { - throw new HttpClientInitException(ErrorMessages.create("cannotFindBuiltInCertificates")); - } - try { - return resource.toURI(); - } catch (URISyntaxException e) { - throw new AssertionError("unreachable"); - } - } } diff --git a/pkl-core/src/main/java/org/pkl/core/http/JdkHttpClient.java b/pkl-core/src/main/java/org/pkl/core/http/JdkHttpClient.java index 64406096..87bcda4f 100644 --- a/pkl-core/src/main/java/org/pkl/core/http/JdkHttpClient.java +++ b/pkl-core/src/main/java/org/pkl/core/http/JdkHttpClient.java @@ -15,17 +15,18 @@ */ package org.pkl.core.http; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodType; import java.net.ConnectException; -import java.net.URI; import java.net.http.HttpClient.Redirect; import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.net.http.HttpResponse.BodyHandler; +import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; @@ -79,12 +80,12 @@ final class JdkHttpClient implements HttpClient { JdkHttpClient( List certificateFiles, - List certificateUris, + List certificateBytes, Duration connectTimeout, java.net.ProxySelector proxySelector) { underlying = java.net.http.HttpClient.newBuilder() - .sslContext(createSslContext(certificateFiles, certificateUris)) + .sslContext(createSslContext(certificateFiles, certificateBytes)) .connectTimeout(connectTimeout) .proxy(proxySelector) .followRedirects(Redirect.NORMAL) @@ -126,10 +127,10 @@ final class JdkHttpClient implements HttpClient { // https://docs.oracle.com/en/java/javase/11/docs/specs/security/standard-names.html#security-algorithm-implementation-requirements private static SSLContext createSslContext( - List certificateFiles, List certificateUris) { + List certificateFiles, List certificateBytes) { try { - if (certificateFiles.isEmpty() && certificateUris.isEmpty()) { - // fall back to JVM defaults (not Pkl built-in certs) + if (certificateFiles.isEmpty() && certificateBytes.isEmpty()) { + // use Pkl native executable's or JVM's built-in CA certificates return SSLContext.getDefault(); } @@ -141,7 +142,7 @@ final class JdkHttpClient implements HttpClient { var certFactory = CertificateFactory.getInstance("X.509"); Set trustAnchors = - createTrustAnchors(certFactory, certificateFiles, certificateUris); + createTrustAnchors(certFactory, certificateFiles, certificateBytes); var pkixParameters = new PKIXBuilderParameters(trustAnchors, new X509CertSelector()); // equivalent of "com.sun.net.ssl.checkRevocation=true" pkixParameters.setRevocationEnabled(true); @@ -161,9 +162,8 @@ final class JdkHttpClient implements HttpClient { } private static Set createTrustAnchors( - CertificateFactory factory, List certificateFiles, List certificateUris) { + CertificateFactory factory, List certificateFiles, List certificateBytes) { var anchors = new HashSet(); - for (var file : certificateFiles) { try (var stream = Files.newInputStream(file)) { collectTrustAnchors(anchors, factory, stream, file); @@ -174,16 +174,10 @@ final class JdkHttpClient implements HttpClient { ErrorMessages.create("cannotReadCertFile", Exceptions.getRootReason(e))); } } - - for (var uri : certificateUris) { - try (var stream = uri.toURL().openStream()) { - collectTrustAnchors(anchors, factory, stream, uri); - } catch (IOException e) { - throw new HttpClientInitException( - ErrorMessages.create("cannotReadCertFile", Exceptions.getRootReason(e))); - } + for (var byteBuffer : certificateBytes) { + var stream = new ByteArrayInputStream(byteBuffer.array()); + collectTrustAnchors(anchors, factory, stream, ""); } - return anchors; } diff --git a/pkl-core/src/main/java/org/pkl/core/service/ExecutorSpiImpl.java b/pkl-core/src/main/java/org/pkl/core/service/ExecutorSpiImpl.java index 077854a0..45dc508c 100644 --- a/pkl-core/src/main/java/org/pkl/core/service/ExecutorSpiImpl.java +++ b/pkl-core/src/main/java/org/pkl/core/service/ExecutorSpiImpl.java @@ -17,7 +17,6 @@ package org.pkl.core.service; import static org.pkl.core.module.ProjectDependenciesManager.PKL_PROJECT_FILENAME; -import java.net.URI; import java.nio.file.Path; import java.util.Collections; import java.util.LinkedHashMap; @@ -132,35 +131,35 @@ public final class ExecutorSpiImpl implements ExecutorSpi { private HttpClient getOrCreateHttpClient(ExecutorSpiOptions options) { List certificateFiles; - List certificateUris; + List certificateBytes; int testPort; try { if (options instanceof ExecutorSpiOptions2 options2) { certificateFiles = options2.getCertificateFiles(); - certificateUris = options2.getCertificateUris(); + certificateBytes = options2.getCertificateBytes(); testPort = options2.getTestPort(); } else { certificateFiles = List.of(); - certificateUris = List.of(); + certificateBytes = List.of(); testPort = -1; } // host pkl-executor does not have class ExecutorOptions2 defined. // this will happen if the pkl-executor distribution is too old. } catch (NoClassDefFoundError e) { certificateFiles = List.of(); - certificateUris = List.of(); + certificateBytes = List.of(); testPort = -1; } - var clientKey = new HttpClientKey(certificateFiles, certificateUris, testPort); + var clientKey = new HttpClientKey(certificateFiles, certificateBytes, testPort); return httpClients.computeIfAbsent( clientKey, (key) -> { var builder = HttpClient.builder(); - for (var file : key.certificateFiles) { - builder.addCertificates(file); + for (var path : key.certificateFiles) { + builder.addCertificates(path); } - for (var uri : key.certificateUris) { - builder.addCertificates(uri); + for (var bytes : key.certificateBytes) { + builder.addCertificates(bytes); } builder.setTestPort(key.testPort); // If the above didn't add any certificates, @@ -171,13 +170,13 @@ public final class ExecutorSpiImpl implements ExecutorSpi { private static final class HttpClientKey { final Set certificateFiles; - final Set certificateUris; + final Set certificateBytes; final int testPort; - HttpClientKey(List certificateFiles, List certificateUris, int testPort) { - // also serve as defensive copies + HttpClientKey(List certificateFiles, List certificateBytes, int testPort) { + // also serves as defensive copy this.certificateFiles = Set.copyOf(certificateFiles); - this.certificateUris = Set.copyOf(certificateUris); + this.certificateBytes = Set.copyOf(certificateBytes); this.testPort = testPort; } @@ -191,13 +190,13 @@ public final class ExecutorSpiImpl implements ExecutorSpi { } HttpClientKey that = (HttpClientKey) obj; return certificateFiles.equals(that.certificateFiles) - && certificateUris.equals(that.certificateUris) + && certificateBytes.equals(that.certificateBytes) && testPort == that.testPort; } @Override public int hashCode() { - return Objects.hash(certificateFiles, certificateUris, testPort); + return Objects.hash(certificateFiles, certificateBytes, testPort); } } } diff --git a/pkl-core/src/test/kotlin/org/pkl/core/http/HttpClientTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/http/HttpClientTest.kt index d0e1ac17..d0399906 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/http/HttpClientTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/http/HttpClientTest.kt @@ -12,9 +12,8 @@ import java.net.http.HttpRequest import java.net.http.HttpResponse import java.nio.file.Path import java.time.Duration -import kotlin.io.path.copyTo -import kotlin.io.path.createDirectories import kotlin.io.path.createFile +import kotlin.io.path.readBytes class HttpClientTest { @Test @@ -52,14 +51,21 @@ class HttpClientTest { } @Test - fun `can load certificates from file system`() { + fun `can load certificates from regular file`() { assertDoesNotThrow { HttpClient.builder().addCertificates(FileTestUtils.selfSignedCertificate).build() } } @Test - fun `certificate file located on file system cannot be empty`(@TempDir tempDir: Path) { + fun `can load certificates from a byte array`() { + assertDoesNotThrow { + HttpClient.builder().addCertificates(FileTestUtils.selfSignedCertificate.readBytes()).build() + } + } + + @Test + fun `certificate file cannot be empty`(@TempDir tempDir: Path) { val file = tempDir.resolve("certs.pem").createFile() val e = assertThrows { @@ -69,56 +75,10 @@ class HttpClientTest { assertThat(e).hasMessageContaining("empty") } - @Test - fun `can load certificates from class path`() { - assertDoesNotThrow { - HttpClient.builder().addCertificates(javaClass.getResource("/org/pkl/certs/PklCARoots.pem")!!.toURI()).build() - } - } - - @Test - fun `only allows loading jar and file certificate URIs`() { - assertThrows { - HttpClient.builder().addCertificates(URI("https://example.com")) - } - } - - @Test - fun `certificate file located on class path cannot be empty`() { - val uri = javaClass.getResource("emptyCerts.pem")!!.toURI() - - val e = assertThrows { - HttpClient.builder().addCertificates(uri).build() - } - - assertThat(e).hasMessageContaining("empty") - } - @Test fun `can load built-in certificates`() { assertDoesNotThrow { - HttpClient.builder().addBuiltInCertificates().build() - } - } - - @Test - fun `can load certificates from Pkl user home cacerts directory`(@TempDir tempDir: Path) { - val certsDir = tempDir.resolve(".pkl") - .resolve("cacerts") - .createDirectories() - .also { dir -> - FileTestUtils.selfSignedCertificate.copyTo(dir.resolve("certs.pem")) - } - - assertDoesNotThrow { - HttpClientBuilder(certsDir).addDefaultCliCertificates().build() - } - } - - @Test - fun `loading certificates from cacerts directory falls back to built-in certificates`(@TempDir certsDir: Path) { - assertDoesNotThrow { - HttpClientBuilder(certsDir).addDefaultCliCertificates().build() + HttpClient.builder().build() } } diff --git a/pkl-core/src/test/kotlin/org/pkl/core/http/LazyHttpClientTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/http/LazyHttpClientTest.kt index e6a77c0e..a6204024 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/http/LazyHttpClientTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/http/LazyHttpClientTest.kt @@ -3,15 +3,20 @@ package org.pkl.core.http import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertDoesNotThrow import org.junit.jupiter.api.assertThrows +import org.junit.jupiter.api.io.TempDir +import org.pkl.commons.createTempFile +import org.pkl.commons.writeString import java.net.URI import java.net.http.HttpRequest import java.net.http.HttpResponse.BodyHandlers +import java.nio.file.Path class LazyHttpClientTest { @Test - fun `builds underlying client on first send`() { + fun `builds underlying client on first send`(@TempDir tempDir: Path) { + val certFile = tempDir.resolve("cert.pem").apply { writeString("broken") } val client = HttpClient.builder() - .addCertificates(javaClass.getResource("brokenCerts.pem")!!.toURI()) + .addCertificates(certFile) .buildLazily() val request = HttpRequest.newBuilder(URI("https://example.com")).build() @@ -21,9 +26,10 @@ class LazyHttpClientTest { } @Test - fun `does not build underlying client unnecessarily`() { + fun `does not build underlying client unnecessarily`(@TempDir tempDir: Path) { + val certFile = tempDir.createTempFile().apply { writeString("broken") } val client = HttpClient.builder() - .addCertificates(javaClass.getResource("brokenCerts.pem")!!.toURI()) + .addCertificates(certFile) .buildLazily() assertDoesNotThrow { diff --git a/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectDependenciesResolverTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectDependenciesResolverTest.kt index a5fb22b7..db1b682e 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectDependenciesResolverTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectDependenciesResolverTest.kt @@ -13,7 +13,6 @@ import org.pkl.core.SecurityManagers import org.pkl.core.packages.PackageResolver import java.io.ByteArrayOutputStream import java.nio.charset.StandardCharsets -import java.nio.file.Path class ProjectDependenciesResolverTest { companion object { diff --git a/pkl-core/src/test/resources/org/pkl/core/http/brokenCerts.pem b/pkl-core/src/test/resources/org/pkl/core/http/brokenCerts.pem deleted file mode 100644 index aef41d77..00000000 --- a/pkl-core/src/test/resources/org/pkl/core/http/brokenCerts.pem +++ /dev/null @@ -1 +0,0 @@ -broken diff --git a/pkl-core/src/test/resources/org/pkl/core/http/emptyCerts.pem b/pkl-core/src/test/resources/org/pkl/core/http/emptyCerts.pem deleted file mode 100644 index e69de29b..00000000 diff --git a/pkl-executor/src/main/java/org/pkl/executor/ExecutorOptions.java b/pkl-executor/src/main/java/org/pkl/executor/ExecutorOptions.java index 5bb6f072..e04dcf39 100644 --- a/pkl-executor/src/main/java/org/pkl/executor/ExecutorOptions.java +++ b/pkl-executor/src/main/java/org/pkl/executor/ExecutorOptions.java @@ -15,7 +15,6 @@ */ package org.pkl.executor; -import java.net.URI; import java.nio.file.Path; import java.time.Duration; import java.util.List; @@ -52,7 +51,7 @@ public final class ExecutorOptions { private final List certificateFiles; - private final List certificateUris; + private final List certificateBytes; private final int testPort; // -1 means disabled @@ -84,7 +83,7 @@ public final class ExecutorOptions { private /* @Nullable */ Path moduleCacheDir; private /* @Nullable */ Path projectDir; private List certificateFiles = List.of(); - private List certificateUris = List.of(); + private List certificateBytes = List.of(); private int testPort = -1; // -1 means disabled private int spiOptionsVersion = -1; // -1 means use latest @@ -188,15 +187,13 @@ public final class ExecutorOptions { return this; } - /** API equivalent of the {@code --ca-certificates} CLI option. */ - public Builder certificateUris(List certificateUris) { - this.certificateUris = certificateUris; + public Builder certificateBytes(List certificateBytes) { + this.certificateBytes = certificateBytes; return this; } - /** API equivalent of the {@code --ca-certificates} CLI option. */ - public Builder certificateUris(URI... certificateUris) { - this.certificateUris = List.of(certificateUris); + public Builder certificateBytes(byte[]... certificateBytes) { + this.certificateBytes = List.of(certificateBytes); return this; } @@ -225,7 +222,7 @@ public final class ExecutorOptions { moduleCacheDir, projectDir, certificateFiles, - certificateUris, + certificateBytes, testPort, spiOptionsVersion); } @@ -290,7 +287,7 @@ public final class ExecutorOptions { /* @Nullable */ Path moduleCacheDir, /* @Nullable */ Path projectDir, List certificateFiles, - List certificateUris, + List certificateBytes, int testPort, int spiOptionsVersion) { @@ -305,7 +302,7 @@ public final class ExecutorOptions { this.moduleCacheDir = moduleCacheDir; this.projectDir = projectDir; this.certificateFiles = List.copyOf(certificateFiles); - this.certificateUris = List.copyOf(certificateUris); + this.certificateBytes = List.copyOf(certificateBytes); this.testPort = testPort; this.spiOptionsVersion = spiOptionsVersion; } @@ -373,9 +370,8 @@ public final class ExecutorOptions { return certificateFiles; } - /** API equivalent of the {@code --ca-certificates} CLI option. */ - public List getCertificateUris() { - return certificateUris; + public List getCertificateBytes() { + return certificateBytes; } @Override @@ -395,7 +391,7 @@ public final class ExecutorOptions { && Objects.equals(moduleCacheDir, other.moduleCacheDir) && Objects.equals(projectDir, other.projectDir) && Objects.equals(certificateFiles, other.certificateFiles) - && Objects.equals(certificateUris, other.certificateUris) + && Objects.equals(certificateBytes, other.certificateBytes) && testPort == other.testPort && spiOptionsVersion == other.spiOptionsVersion; } @@ -414,7 +410,7 @@ public final class ExecutorOptions { moduleCacheDir, projectDir, certificateFiles, - certificateUris, + certificateBytes, testPort, spiOptionsVersion); } @@ -444,8 +440,8 @@ public final class ExecutorOptions { + projectDir + ", certificateFiles=" + certificateFiles - + ", certificateUris=" - + certificateUris + + ", certificateBytes=" + + certificateBytes + ", testPort=" + testPort + ", spiOptionsVersion=" @@ -468,7 +464,7 @@ public final class ExecutorOptions { moduleCacheDir, projectDir, certificateFiles, - certificateUris, + certificateBytes, testPort); case 1 -> // for testing only new ExecutorSpiOptions( diff --git a/pkl-executor/src/main/java/org/pkl/executor/spi/v1/ExecutorSpiOptions2.java b/pkl-executor/src/main/java/org/pkl/executor/spi/v1/ExecutorSpiOptions2.java index f12bd373..c3fb53d8 100644 --- a/pkl-executor/src/main/java/org/pkl/executor/spi/v1/ExecutorSpiOptions2.java +++ b/pkl-executor/src/main/java/org/pkl/executor/spi/v1/ExecutorSpiOptions2.java @@ -15,7 +15,6 @@ */ package org.pkl.executor.spi.v1; -import java.net.URI; import java.nio.file.Path; import java.time.Duration; import java.util.List; @@ -24,7 +23,7 @@ import java.util.Map; public class ExecutorSpiOptions2 extends ExecutorSpiOptions { private final List certificateFiles; - private final List certificateUris; + private final List certificateBytes; private final int testPort; @@ -40,7 +39,7 @@ public class ExecutorSpiOptions2 extends ExecutorSpiOptions { Path moduleCacheDir, Path projectDir, List certificateFiles, - List certificateUris, + List certificateBytes, int testPort) { super( allowedModules, @@ -54,7 +53,7 @@ public class ExecutorSpiOptions2 extends ExecutorSpiOptions { moduleCacheDir, projectDir); this.certificateFiles = certificateFiles; - this.certificateUris = certificateUris; + this.certificateBytes = certificateBytes; this.testPort = testPort; } @@ -62,8 +61,8 @@ public class ExecutorSpiOptions2 extends ExecutorSpiOptions { return certificateFiles; } - public List getCertificateUris() { - return certificateUris; + public List getCertificateBytes() { + return certificateBytes; } public int getTestPort() { diff --git a/pkl-server/src/main/kotlin/org/pkl/server/Message.kt b/pkl-server/src/main/kotlin/org/pkl/server/Message.kt index 530000bb..3e2ce1e7 100644 --- a/pkl-server/src/main/kotlin/org/pkl/server/Message.kt +++ b/pkl-server/src/main/kotlin/org/pkl/server/Message.kt @@ -20,7 +20,7 @@ import java.nio.file.Path import java.time.Duration import java.util.* import java.util.regex.Pattern -import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.* +import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.Proxy import org.pkl.core.module.PathElement import org.pkl.core.packages.Checksums @@ -124,7 +124,7 @@ data class CreateEvaluatorRequest( val cacheDir: Path?, val outputFormat: String?, val project: Project?, - val http: Http?, + val http: Http? ) : ClientRequestMessage() { override val type = MessageType.CREATE_EVALUATOR_REQUEST @@ -151,7 +151,8 @@ data class CreateEvaluatorRequest( rootDir.equalsNullable(other.rootDir) && cacheDir.equalsNullable(other.cacheDir) && outputFormat.equalsNullable(other.outputFormat) && - project.equalsNullable(other.project) + project.equalsNullable(other.project) && + http.equalsNullable(other.http) } @Suppress("DuplicatedCode") // false duplicate within method @@ -170,6 +171,31 @@ data class CreateEvaluatorRequest( result = 31 * result + outputFormat.hashCode() result = 31 * result + project.hashCode() result = 31 * result + type.hashCode() + result = 31 * result + http.hashCode() + return result + } +} + +data class Http( + /** PEM-format CA certificates as raw bytes. */ + val caCertificates: ByteArray?, + /** Proxy settings */ + val proxy: Proxy? +) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is Http) return false + + if (caCertificates != null) { + if (other.caCertificates == null) return false + if (!caCertificates.contentEquals(other.caCertificates)) return false + } else if (other.caCertificates != null) return false + return Objects.equals(proxy, other.proxy) + } + + override fun hashCode(): Int { + var result = caCertificates?.contentHashCode() ?: 0 + result = 31 * result + (proxy?.hashCode() ?: 0) return result } } diff --git a/pkl-server/src/main/kotlin/org/pkl/server/MessagePackDecoder.kt b/pkl-server/src/main/kotlin/org/pkl/server/MessagePackDecoder.kt index 3cee911c..e87c3506 100644 --- a/pkl-server/src/main/kotlin/org/pkl/server/MessagePackDecoder.kt +++ b/pkl-server/src/main/kotlin/org/pkl/server/MessagePackDecoder.kt @@ -255,10 +255,11 @@ internal class MessagePackDecoder(private val unpacker: MessageUnpacker) : Messa return Project(projectFileUri, null, dependencies) } - private fun Map.unpackHttp(): PklEvaluatorSettings.Http? { + private fun Map.unpackHttp(): Http? { val httpMap = getNullable("http")?.asMapValue()?.map() ?: return null val proxy = httpMap.unpackProxy() - return PklEvaluatorSettings.Http(proxy) + val caCertificates = httpMap.getNullable("caCertificates")?.asBinaryValue()?.asByteArray() + return Http(caCertificates, proxy) } private fun Map.unpackProxy(): PklEvaluatorSettings.Proxy? { diff --git a/pkl-server/src/main/kotlin/org/pkl/server/MessagePackEncoder.kt b/pkl-server/src/main/kotlin/org/pkl/server/MessagePackEncoder.kt index 6952bd0c..2bbd4ccc 100644 --- a/pkl-server/src/main/kotlin/org/pkl/server/MessagePackEncoder.kt +++ b/pkl-server/src/main/kotlin/org/pkl/server/MessagePackEncoder.kt @@ -49,6 +49,21 @@ internal class MessagePackEncoder(private val packer: MessagePacker) : MessageEn packDependencies(project.dependencies) } + private fun MessagePacker.packHttp(http: Http) { + if ((http.caCertificates ?: http.proxy) == null) { + packMapHeader(0) + return + } + packMapHeader(0, http.caCertificates, http.proxy) + packKeyValue("caCertificates", http.caCertificates) + http.proxy?.let { proxy -> + packString("proxy") + packMapHeader(0, proxy.address, proxy.noProxy) + packKeyValue("address", proxy.address?.toString()) + packKeyValue("noProxy", proxy.noProxy) + } + } + private fun MessagePacker.packDependencies(dependencies: Map) { packMapHeader(dependencies.size) for ((name, dep) in dependencies) { @@ -87,7 +102,15 @@ internal class MessagePackEncoder(private val packer: MessagePacker) : MessageEn when (msg.type.code) { MessageType.CREATE_EVALUATOR_REQUEST.code -> { msg as CreateEvaluatorRequest - packMapHeader(8, msg.timeout, msg.rootDir, msg.cacheDir, msg.outputFormat, msg.project) + packMapHeader( + 8, + msg.timeout, + msg.rootDir, + msg.cacheDir, + msg.outputFormat, + msg.project, + msg.http + ) packKeyValue("requestId", msg.requestId) packKeyValue("allowedModules", msg.allowedModules?.map { it.toString() }) packKeyValue("allowedResources", msg.allowedResources?.map { it.toString() }) @@ -116,6 +139,10 @@ internal class MessagePackEncoder(private val packer: MessagePacker) : MessageEn packString("project") packProject(msg.project) } + if (msg.http != null) { + packString("http") + packHttp(msg.http) + } } MessageType.CREATE_EVALUATOR_RESPONSE.code -> { msg as CreateEvaluatorResponse @@ -243,7 +270,8 @@ internal class MessagePackEncoder(private val packer: MessagePacker) : MessageEn value2: Any?, value3: Any?, value4: Any?, - value5: Any? + value5: Any?, + value6: Any? ) = packMapHeader( size + @@ -251,7 +279,8 @@ internal class MessagePackEncoder(private val packer: MessagePacker) : MessageEn (if (value2 != null) 1 else 0) + (if (value3 != null) 1 else 0) + (if (value4 != null) 1 else 0) + - (if (value5 != null) 1 else 0) + (if (value5 != null) 1 else 0) + + (if (value6 != null) 1 else 0) ) private fun MessagePacker.packKeyValue(name: String, value: Int?) { diff --git a/pkl-server/src/main/kotlin/org/pkl/server/Server.kt b/pkl-server/src/main/kotlin/org/pkl/server/Server.kt index 3b078130..a0a71a8b 100644 --- a/pkl-server/src/main/kotlin/org/pkl/server/Server.kt +++ b/pkl-server/src/main/kotlin/org/pkl/server/Server.kt @@ -162,12 +162,13 @@ class Server(private val transport: MessageTransport) : AutoCloseable { val properties = message.properties ?: emptyMap() val timeout = message.timeout val cacheDir = message.cacheDir - val http = + val httpClient = with(HttpClient.builder()) { message.http?.proxy?.let { proxy -> - setProxy(proxy.address, message.http.proxy?.noProxy ?: listOf()) + setProxy(proxy.address, proxy.noProxy ?: listOf()) proxy.address?.let(IoUtils::setSystemProxy) } + message.http?.caCertificates?.let { caCertificates -> addCertificates(caCertificates) } buildLazily() } val dependencies = @@ -183,7 +184,7 @@ class Server(private val transport: MessageTransport) : AutoCloseable { SecurityManagers.defaultTrustLevels, rootDir ), - http, + httpClient, ClientLogger(evaluatorId, transport), createModuleKeyFactories(message, evaluatorId, resolver), createResourceReaders(message, evaluatorId, resolver), diff --git a/pkl-server/src/test/kotlin/org/pkl/server/AbstractServerTest.kt b/pkl-server/src/test/kotlin/org/pkl/server/AbstractServerTest.kt index f1246199..f7a78ce3 100644 --- a/pkl-server/src/test/kotlin/org/pkl/server/AbstractServerTest.kt +++ b/pkl-server/src/test/kotlin/org/pkl/server/AbstractServerTest.kt @@ -1013,7 +1013,8 @@ abstract class AbstractServerTest { moduleReaders: List = listOf(), modulePaths: List = listOf(), project: Project? = null, - cacheDir: Path? = null + cacheDir: Path? = null, + http: Http? = null, ): Long { val message = CreateEvaluatorRequest( @@ -1030,7 +1031,7 @@ abstract class AbstractServerTest { cacheDir = cacheDir, outputFormat = null, project = project, - http = null, + http = http ) send(message) diff --git a/pkl-server/src/test/kotlin/org/pkl/server/MessagePackCodecTest.kt b/pkl-server/src/test/kotlin/org/pkl/server/MessagePackCodecTest.kt index 1e511bd9..e8c553a3 100644 --- a/pkl-server/src/test/kotlin/org/pkl/server/MessagePackCodecTest.kt +++ b/pkl-server/src/test/kotlin/org/pkl/server/MessagePackCodecTest.kt @@ -25,6 +25,7 @@ import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows import org.msgpack.core.MessagePack +import org.pkl.core.evaluatorSettings.PklEvaluatorSettings import org.pkl.core.module.PathElement import org.pkl.core.packages.Checksums @@ -73,6 +74,7 @@ class MessagePackCodecTest { isGlobbable = false, isLocal = false ) + @Suppress("HttpUrlsUsage") roundtrip( CreateEvaluatorRequest( requestId = 123, @@ -113,7 +115,11 @@ class MessagePackCodecTest { RemoteDependency(URI("package://localhost:0/baz@1.1.0"), Checksums("abc123")) ) ), - http = null, + http = + Http( + proxy = PklEvaluatorSettings.Proxy(URI("http://foo.com:1234"), listOf("bar", "baz")), + caCertificates = byteArrayOf(1, 2, 3, 4) + ) ) ) } diff --git a/settings.gradle.kts b/settings.gradle.kts index 9f9bd5ec..521a2785 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -4,7 +4,6 @@ include("bench") include("docs") include("stdlib") -include("pkl-certs") include("pkl-cli") include("pkl-codegen-java") include("pkl-codegen-kotlin")