Follow HTTP redirects (#328)

- Change HttpClient to follow all redirects except HTTPS to HTTP.
- Run language snippet tests with --no-cache and real PackageServer
  instead of pre-seeded cache.
  This increases HTTP test coverage and enables testing of package redirects.
- Change PackageServer to return 301 for request paths starting with /HTTP301/
  and 307 for request paths starting with /HTTP307/.
- Update some outdated test package checksums that apparently weren't verified.

pkl-cli/src/test/kotlin/org/pkl/cli/CliProjectPackagerTest.kt

@@ -588,7 +588,7 @@ class CliProjectPackagerTest {
               "type": "remote",
               "uri": "projectpackage://localhost:0/birds@0.5.0",
               "checksums": {
-                "sha256": "3f19ab9fcee2f44f93a75a09e531db278c6d2cd25206836c8c2c4071cd7d3118"
+                "sha256": "0a5ad2dc13f06f73f96ba94e8d01d48252bc934e2de71a837620ca0fef8a7453"
               }
             },
             "package://localhost:0/project2@5": {

pkl-commons-test/src/main/kotlin/org/pkl/commons/test/InputOutputTestEngine.kt

@@ -51,6 +51,10 @@ abstract class InputOutputTestEngine :
 
   protected abstract fun generateOutputFor(inputFile: Path): Pair<Boolean, String>
 
+  protected open fun beforeAll() {}
+
+  protected open fun afterAll() {}
+
   class ExecutionContext : EngineExecutionContext
 
   override fun getId(): String = this::class.java.simpleName
@@ -78,7 +82,17 @@ abstract class InputOutputTestEngine :
         (classSelectors.isEmpty() || classSelectors.any { it.className == className })
     ) {
 
-      val rootNode = InputDirNode(uniqueId, inputDir, ClassSource.from(testClass.java))
+      val rootNode =
+        object : InputDirNode(uniqueId, inputDir, ClassSource.from(testClass.java)) {
+          override fun before(context: ExecutionContext): ExecutionContext {
+            beforeAll()
+            return context
+          }
+
+          override fun after(context: ExecutionContext) {
+            afterAll()
+          }
+        }
       return doDiscover(rootNode, uniqueIdSelectors)
     }
 
@@ -124,7 +138,11 @@ abstract class InputOutputTestEngine :
 
   override fun createExecutionContext(request: ExecutionRequest) = ExecutionContext()
 
-  private inner class InputDirNode(uniqueId: UniqueId, val inputDir: Path, source: TestSource) :
+  private open inner class InputDirNode(
+    uniqueId: UniqueId,
+    val inputDir: Path,
+    source: TestSource
+  ) :
     AbstractTestDescriptor(uniqueId, inputDir.fileName.toString(), source), Node<ExecutionContext> {
     override fun getType() = Type.CONTAINER
   }

pkl-commons-test/src/main/kotlin/org/pkl/commons/test/PackageServer.kt

@@ -130,11 +130,23 @@ class PackageServer : AutoCloseable {
       return@HttpHandler
     }
     val path = exchange.requestURI.path
+    if (path.startsWith("/HTTP301/")) {
+      exchange.responseHeaders.add("Location", path.removePrefix("/HTTP301"))
+      exchange.sendResponseHeaders(301, -1)
+      exchange.close()
+      return@HttpHandler
+    }
+    if (path.startsWith("/HTTP307/")) {
+      exchange.responseHeaders.add("Location", path.removePrefix("/HTTP307"))
+      exchange.sendResponseHeaders(307, -1)
+      exchange.close()
+      return@HttpHandler
+    }
     val localPath =
       if (path.endsWith(".zip")) packagesDir.resolve(path.drop(1))
       else packagesDir.resolve("${path.drop(1)}${path}.json")
     if (!Files.exists(localPath)) {
-      exchange.sendResponseHeaders(404, 0)
+      exchange.sendResponseHeaders(404, -1)
       exchange.close()
       return@HttpHandler
     }

pkl-core/src/main/java/org/pkl/core/http/JdkHttpClient.java

@@ -22,6 +22,7 @@ import java.lang.invoke.MethodHandles;
 import java.lang.invoke.MethodType;
 import java.net.ConnectException;
 import java.net.URI;
+import java.net.http.HttpClient.Redirect;
 import java.net.http.HttpRequest;
 import java.net.http.HttpResponse;
 import java.net.http.HttpResponse.BodyHandler;
@@ -81,6 +82,7 @@ final class JdkHttpClient implements HttpClient {
         java.net.http.HttpClient.newBuilder()
             .sslContext(createSslContext(certificateFiles, certificateUris))
             .connectTimeout(connectTimeout)
+            .followRedirects(Redirect.NORMAL)
             .build();
   }
 

pkl-core/src/test/files/LanguageSnippetTests/input/packages/packages1.pkl

@@ -16,7 +16,7 @@ examples {
     import("package://localhost:0/birds@0.5.0#/catalog/Ostritch.pkl")
   }
   ["importing while specifying checksum"] {
-    import("package://localhost:0/birds@0.5.0::sha256:3f19ab9fcee2f44f93a75a09e531db278c6d2cd25206836c8c2c4071cd7d3118#/catalog/Swallow.pkl")
+    import("package://localhost:0/birds@0.5.0::sha256:bfaf5281613d170a740505cc87561041f4e0cad1f0e6938bf94f7609f9a4673d#/catalog/Swallow.pkl")
   }
   ["reads"] {
     read("package://localhost:0/birds@0.5.0#/Bird.pkl")

pkl-core/src/test/files/LanguageSnippetTests/input/packages/packages2.pkl

@@ -20,6 +20,6 @@ examples {
     import("package://localhost:0/birds@0.5.0#/allFruit.pkl").fruitFiles
   }
   ["glob import while specifying checksum"] {
-    import*("package://localhost:0/birds@0.5.0::sha256:3f19ab9fcee2f44f93a75a09e531db278c6d2cd25206836c8c2c4071cd7d3118#/catalog/*.pkl")
+    import*("package://localhost:0/birds@0.5.0::sha256:bfaf5281613d170a740505cc87561041f4e0cad1f0e6938bf94f7609f9a4673d#/catalog/*.pkl")
   }
 }

pkl-core/src/test/files/LanguageSnippetTests/input/packages/redirects.pkl

@@ -0,0 +1,15 @@
+amends ".../snippetTest.pkl"
+
+examples {
+  ["permanent redirect is followed"] {
+    import("package://localhost:0/HTTP301/birds@0.5.0#/catalog/Swallow.pkl")
+  }
+
+  ["temporary redirect is followed"] {
+    import("package://localhost:0/HTTP307/birds@0.5.0#/catalog/Swallow.pkl")
+  }
+  
+  ["double redirect is followed"] {
+    import("package://localhost:0/HTTP301/HTTP307/birds@0.5.0#/catalog/Swallow.pkl")
+  }
+}

pkl-core/src/test/files/LanguageSnippetTests/output/packages/packages2.pcf

@@ -118,13 +118,13 @@ examples {
   }
   ["glob import while specifying checksum"] {
     new {
-      ["package://localhost:0/birds@0.5.0::sha256:3f19ab9fcee2f44f93a75a09e531db278c6d2cd25206836c8c2c4071cd7d3118#/catalog/Ostritch.pkl"] {
+      ["package://localhost:0/birds@0.5.0::sha256:bfaf5281613d170a740505cc87561041f4e0cad1f0e6938bf94f7609f9a4673d#/catalog/Ostritch.pkl"] {
         name = "Ostritch"
         favoriteFruit {
           name = "Orange"
         }
       }
-      ["package://localhost:0/birds@0.5.0::sha256:3f19ab9fcee2f44f93a75a09e531db278c6d2cd25206836c8c2c4071cd7d3118#/catalog/Swallow.pkl"] {
+      ["package://localhost:0/birds@0.5.0::sha256:bfaf5281613d170a740505cc87561041f4e0cad1f0e6938bf94f7609f9a4673d#/catalog/Swallow.pkl"] {
         name = "Swallow"
         favoriteFruit {
           name = "Apple"

pkl-core/src/test/files/LanguageSnippetTests/output/packages/redirects.pcf

@@ -0,0 +1,26 @@
+examples {
+  ["permanent redirect is followed"] {
+    new {
+      name = "Swallow"
+      favoriteFruit {
+        name = "Apple"
+      }
+    }
+  }
+  ["temporary redirect is followed"] {
+    new {
+      name = "Swallow"
+      favoriteFruit {
+        name = "Apple"
+      }
+    }
+  }
+  ["double redirect is followed"] {
+    new {
+      name = "Swallow"
+      favoriteFruit {
+        name = "Apple"
+      }
+    }
+  }
+}

pkl-core/src/test/kotlin/org/pkl/core/LanguageSnippetTestsEngine.kt

@@ -4,9 +4,12 @@ import org.junit.platform.engine.EngineDiscoveryRequest
 import org.junit.platform.engine.TestDescriptor
 import org.junit.platform.engine.UniqueId
 import org.junit.platform.engine.support.descriptor.EngineDescriptor
+import org.pkl.commons.test.FileTestUtils
 import org.pkl.commons.test.InputOutputTestEngine
 import org.pkl.commons.test.PackageServer
+import org.pkl.core.http.HttpClient
 import org.pkl.core.project.Project
+import org.pkl.core.util.IoUtils
 import java.io.PrintWriter
 import java.io.StringWriter
 import java.nio.file.Files
@@ -33,6 +36,8 @@ abstract class AbstractLanguageSnippetTestsEngine : InputOutputTestEngine() {
   //language=regexp
   internal val selection: String = ""
 
+  protected val packageServer: PackageServer = PackageServer()
+  
   override val includedTests: List<Regex> = listOf(Regex(".*$selection\\.pkl"))
 
   override val excludedTests: List<Regex> = listOf(Regex(".*/native/.*"))
@@ -41,11 +46,6 @@ abstract class AbstractLanguageSnippetTestsEngine : InputOutputTestEngine() {
 
   override val isInputFile: (Path) -> Boolean = { it.isRegularFile() }
 
-  protected val cacheDir: Path by lazy {
-    rootProjectDir.resolve("pkl-core/build/packages-cache")
-      .also { PackageServer.populateCacheDir(it) }
-  }
-
   protected tailrec fun Path.getProjectDir(): Path? =
     if (Files.exists(this.resolve("PklProject"))) this
     else parent?.getProjectDir()
@@ -58,6 +58,15 @@ abstract class AbstractLanguageSnippetTestsEngine : InputOutputTestEngine() {
     return expectedOutputDir.resolve(stdoutPath)
   }
 
+  override fun beforeAll() {
+    // disable SHA verification for packages
+    IoUtils.setTestMode()
+  }
+  
+  override fun afterAll() {
+    packageServer.close()
+  }
+  
   protected fun String.stripFilePaths() = replace(snippetsDir.toString(), "/\$snippetsDir")
 
   protected fun String.stripLineNumbers() = replace(lineNumberRegex) { result ->
@@ -92,7 +101,11 @@ class LanguageSnippetTestsEngine : AbstractLanguageSnippetTestsEngine() {
         "name2" to "value2",
         "/foo/bar" to "foobar"
       ))
-      .setModuleCacheDir(cacheDir)
+      .setModuleCacheDir(null)
+      .setHttpClient(HttpClient.builder()
+        .setTestPort(packageServer.port)
+        .addCertificates(FileTestUtils.selfSignedCertificate)
+        .buildLazily())
   }
 
   override val testClass: KClass<*> = LanguageSnippetTests::class
@@ -158,8 +171,7 @@ abstract class AbstractNativeLanguageSnippetTestsEngine : AbstractLanguageSnippe
     val args = buildList {
       add(pklExecutablePath.toString())
       add("eval")
-      add("--cache-dir")
+      add("--no-cache")
-      add(cacheDir.toString())
       if (inputFile.startsWith(projectsDir)) {
         val projectDir = inputFile.getProjectDir()
         if (projectDir != null) {
@@ -187,7 +199,11 @@ abstract class AbstractNativeLanguageSnippetTestsEngine : AbstractLanguageSnippe
       }
       add("--settings")
       add("pkl:settings")
+      add("--ca-certificates")
+      add(FileTestUtils.selfSignedCertificate.toString())
       add("--test-mode")
+      add("--test-port")
+      add(packageServer.port.toString())
       add(inputFile.toString())
     }