diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/ModuleResolver.java b/pkl-core/src/main/java/org/pkl/core/runtime/ModuleResolver.java index fa925081a..855503c1b 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/ModuleResolver.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/ModuleResolver.java @@ -27,6 +27,7 @@ import org.pkl.core.externalreader.ExternalReaderProcessException; import org.pkl.core.module.ModuleKey; import org.pkl.core.module.ModuleKeyFactory; import org.pkl.core.module.ModuleKeys; +import org.pkl.core.util.IoUtils; public final class ModuleResolver { private final Collection factories; @@ -78,6 +79,9 @@ public final class ModuleResolver { Optional key; try { key = factory.create(normalized); + if (key.isPresent()) { + key = Optional.of(renormalize(factory, normalized, key.get())); + } } catch (URISyntaxException e) { throw new VmExceptionBuilder() .withOptionalLocation(importNode) @@ -105,4 +109,20 @@ public final class ModuleResolver { .withOptionalLocation(importNode) .build(); } + + /** + * Ensures that the URI of the module key is normalized according to the key's own URI semantics + * (which {@link URI#normalize()} knows nothing about), so that security checks and loading + * observe the same URI. + * + *

Module keys are immutable, so a new one is created if {@code uri} changes. + */ + private ModuleKey renormalize(ModuleKeyFactory factory, URI uri, ModuleKey key) + throws URISyntaxException, IOException, ExternalReaderProcessException { + var renormalized = IoUtils.normalizeUri(uri, key.hasHierarchicalUris()); + if (renormalized.equals(uri)) return key; + return factory + .create(renormalized) + .orElseThrow(() -> new IllegalStateException("failed to renormalize a module URI")); + } } diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/ResourceManager.java b/pkl-core/src/main/java/org/pkl/core/runtime/ResourceManager.java index aa0ac34aa..c5cc8ad9a 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/ResourceManager.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/ResourceManager.java @@ -34,6 +34,7 @@ import org.pkl.core.resource.Resource; import org.pkl.core.resource.ResourceReader; import org.pkl.core.resource.ResourceReaders; import org.pkl.core.stdlib.VmObjectFactory; +import org.pkl.core.util.IoUtils; public final class ResourceManager { private final Map resourceReaders = new HashMap<>(); @@ -96,10 +97,17 @@ public final class ResourceManager { @TruffleBoundary public Optional read(URI resourceUri, @Nullable Node readNode) { + var reader = getResourceReader(resourceUri); + URI normalizedUri; + try { + normalizedUri = + IoUtils.normalizeUri(resourceUri, reader != null && reader.hasHierarchicalUris()); + } catch (ExternalReaderProcessException | IOException e) { + throw new VmExceptionBuilder().withCause(e).withOptionalLocation(readNode).build(); + } return resources.computeIfAbsent( - resourceUri.normalize(), + normalizedUri, (uri) -> { - var reader = getResourceReader(uri); // hack: we don't want to call `checkReadResource` here for these resources because those // readers defer to HttpClient to do the actual checks. if (!(reader instanceof ResourceReaders.HttpResource) diff --git a/pkl-core/src/main/java/org/pkl/core/util/GlobResolver.java b/pkl-core/src/main/java/org/pkl/core/util/GlobResolver.java index 121d9d07c..190408f18 100644 --- a/pkl-core/src/main/java/org/pkl/core/util/GlobResolver.java +++ b/pkl-core/src/main/java/org/pkl/core/util/GlobResolver.java @@ -339,16 +339,17 @@ public final class GlobResolver { if (listElementCallCount.getAndIncrement() > maxListElements()) { throw new InvalidGlobPatternException(ErrorMessages.create("invalidGlobTooComplex")); } - var elements = reader.listElements(securityManager, baseUri); + var normalizedBaseUri = normalizeUri(reader, baseUri); + var elements = reader.listElements(securityManager, normalizedBaseUri); for (var element : sorted(elements)) { var elementPath = resolvePath(expandedGlobSoFar, element.getName(), hasAbsoluteGlob); if (globPartPattern.matcher(element.getName()).matches()) { var name = element.isDirectory() ? element.getName() + "/" : element.getName(); - var elementUri = IoUtils.resolve(reader, baseUri, name); + var elementUri = IoUtils.resolve(reader, normalizedBaseUri, name); result.add(new ResolvedGlobElement(elementPath, elementUri, element.isDirectory())); } if (element.isDirectory() && isGlobStar) { - var elementUri = IoUtils.resolve(reader, baseUri, element.getName() + "/"); + var elementUri = IoUtils.resolve(reader, normalizedBaseUri, element.getName() + "/"); var newExpandedGlobPattern = resolvePath(expandedGlobSoFar, element.getName(), hasAbsoluteGlob); doExpandHierarchicalGlobPart( @@ -365,6 +366,15 @@ public final class GlobResolver { } } + /** + * Normalizes URIs before they are handed to the reader, so that security checks and I/O both see + * the same URI. + */ + private static URI normalizeUri(ReaderBase reader, URI uri) + throws IOException, ExternalReaderProcessException { + return IoUtils.normalizeUri(uri, reader.hasHierarchicalUris()); + } + private static void resolveHierarchicalGlob( SecurityManager securityManager, ReaderBase reader, @@ -384,7 +394,7 @@ public final class GlobResolver { if (isRegularPathPart(patternPart)) { var newPath = resolvePath(expandedGlobPatternSoFar, patternPart, hasAbsoluteGlob); if (isLeaf) { - var newBaseUri = IoUtils.resolve(reader, baseUri, patternPart); + var newBaseUri = normalizeUri(reader, IoUtils.resolve(reader, baseUri, patternPart)); if (reader.hasElement(securityManager, newBaseUri)) { // Note: isDirectory is not meaningful here. Setting it to false is a way to skip setting // it. @@ -501,9 +511,11 @@ public final class GlobResolver { // short-circuit for glob pattern with no wildcards (can only match 0 or 1 element) if (globParts.length == 0) { var resolvedUri = - enclosingUri == null - ? URI.create(globPattern) - : IoUtils.resolve(reader, enclosingUri, globPattern); + normalizeUri( + reader, + enclosingUri == null + ? URI.create(globPattern) + : IoUtils.resolve(reader, enclosingUri, globPattern)); if (reader.hasElement(securityManager, resolvedUri)) { result.put(globPattern, new ResolvedGlobElement(globPattern, resolvedUri, true)); } diff --git a/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java b/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java index ba67261b5..d23a68408 100644 --- a/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java +++ b/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java @@ -85,6 +85,11 @@ public final class IoUtils { // keep in sync with stdlib `EvaluatorSettings.reservedHttpHeaderPrefix` private static final String[] reservedHeaderPrefixes = {"proxy-", "sec-"}; + private static final Pattern DOT_SEGMENT = Pattern.compile("\\.|%2[eE]"); + private static final Pattern DOT_DOT_SEGMENT = Pattern.compile("(?:\\.|%2[eE]){2}"); + private static final Pattern PATH_SEPARATOR = Pattern.compile("/|%2[fF]"); + private static final Pattern WINDOWS_PATH_SEPARATOR = Pattern.compile("/|%2[fF]|%5[cC]"); + private IoUtils() {} public static URL toUrl(URI uri) throws IOException { @@ -438,6 +443,143 @@ public final class IoUtils { throw new PackageLoadError("cannotFindDependencyInProject", name); } + /** + * Normalizes {@code uri} for the purpose of security checks. + * + *

In hierarchical URIs of any scheme, removes {@code .} and {@code ..} path segments, and + * treats {@code %2E} as equivalent to {@code .}. {@code .} is an unreserved character, so a + * percent-encoded {@code .} is equivalent to a literal one (RFC 3986, section 6.2.2.2). All other + * parts of the URI are left as is. In particular, {@code %2F} is not treated as a path separator, + * because it might be distinct from {@code /} (for example, in HTTP). + * + *

For {@code file:} URIs, also treats {@code %2F} as equivalent to {@code /}, because file + * systems do. + * + *

For {@code package:} and {@code projectpackage:} URIs, also normalizes the path of the asset + * in the fragment, treating {@code %2E} and {@code %2F} the same way as for {@code file:} URIs. + * + *

For {@code jar:} URIs, only the URI of the archive is normalized. The path within the + * archive is left as is, because zip file systems treat {@code ..} as a verbatim name. + */ + public static URI normalizeUri(URI uri, boolean hasHierarchicalUris) { + var scheme = uri.getScheme(); + if ("jar".equalsIgnoreCase(scheme)) { + return normalizeJarUri(uri); + } + if ("file".equalsIgnoreCase(scheme)) { + return normalizeUriPath(uri, true); + } + if ("package".equalsIgnoreCase(scheme) || "projectpackage".equalsIgnoreCase(scheme)) { + return normalizePackageUri(uri); + } + if (hasHierarchicalUris) { + return normalizeUriPath(uri, false); + } + return uri; + } + + private static URI normalizePackageUri(URI uri) { + var assetPath = uri.getRawFragment(); + if (assetPath == null || !assetPath.startsWith("/")) { + return uri; + } + var normalizedPath = normalizeRawPath(assetPath, true, isWindows()); + if (normalizedPath.equals(uri.getRawFragment())) { + return uri; + } + var uriString = uri.toString(); + var hashIndex = uriString.indexOf('#'); + return URI.create(uriString.substring(0, hashIndex) + "#" + normalizedPath); + } + + /** + * Normalizes the path of {@code uri}. + * + * @param decodeSlash whether to treat {@code %2F} as a path separator + */ + private static URI normalizeUriPath(URI uri, boolean decodeSlash) { + if (uri.isOpaque() || uri.getScheme() == null) { + return uri; + } + // Use string builder instead of component-wise accessors/constructors because component-wise + // will re-encode (e.g. turn %2F into `/`), and can fail for unusual authorities. + var uriString = uri.toString(); + var rawPath = uri.getRawPath(); + var pathStart = uriString.indexOf(':') + 1; + var hasAuthority = uriString.startsWith("//", pathStart); + if (hasAuthority) { + var rawAuthority = uri.getRawAuthority(); + pathStart += 2 + (rawAuthority == null ? 0 : rawAuthority.length()); + } + assert uriString.startsWith(rawPath, pathStart); + var normalizedPath = normalizeRawPath(rawPath, decodeSlash, isWindows()); + if (normalizedPath.equals(rawPath)) { + return uri; + } + var sb = new StringBuilder(uriString.substring(0, pathStart)); + if (!hasAuthority && normalizedPath.startsWith("//")) { + // avoid the path being misinterpreted as an authority + sb.append("/."); + } + sb.append(normalizedPath); + var pathEnd = pathStart + rawPath.length(); + sb.append(uriString, pathEnd, uriString.length()); + return URI.create(sb.toString()); + } + + /** Normalizes the URI of the archive of {@code jar:!/}. */ + private static URI normalizeJarUri(URI uri) { + var uriString = uri.toString(); + var index = uriString.indexOf('!'); + if (index == -1) { + return uri; + } + var archiveUri = URI.create(uriString.substring(uri.getScheme().length() + 1, index)); + var normalizedArchiveUri = normalizeUri(archiveUri, true); + if (normalizedArchiveUri.equals(archiveUri)) { + return uri; + } + return URI.create(uri.getScheme() + ":" + normalizedArchiveUri + uriString.substring(index)); + } + + /** + * Removes {@code .} and {@code ..} segments from a raw (percent-encoded) path. + * + *

Segments written as {@code %2e} are treated as dots, because readers decode the path before + * using it. All other segments are kept verbatim. A {@code ..} that cannot be resolved is kept, + * so the result does not match patterns it shouldn't. + * + *

If {@code decodeSlash} is true, treats either {@code /} or {@code %2F} as the path + * separator, and replaces {@code %2F} with {@code /}. Otherwise, {@code %2F} is kept as is. + * + *

If {@code isWindows}, then {@code %5c} is also treated as a slash. + */ + static String normalizeRawPath(String rawPath, boolean decodeSlash, boolean isWindows) { + var isAbsolute = rawPath.startsWith("/"); + var rawSegments = (isAbsolute ? rawPath.substring(1) : rawPath); + var separator = isWindows ? WINDOWS_PATH_SEPARATOR : PATH_SEPARATOR; + var segments = decodeSlash ? separator.split(rawSegments, -1) : rawSegments.split("/", -1); + var result = new ArrayList(segments.length); + for (var i = 0; i < segments.length; i++) { + var segment = segments[i]; + var isLast = i == segments.length - 1; + if (DOT_SEGMENT.matcher(segment).matches()) { + if (isLast) result.add(""); + } else if (DOT_DOT_SEGMENT.matcher(segment).matches()) { + if (!result.isEmpty() + && !DOT_DOT_SEGMENT.matcher(result.get(result.size() - 1)).matches()) { + result.remove(result.size() - 1); + } else { + result.add(segment); + } + if (isLast) result.add(""); + } else { + result.add(segment); + } + } + return (isAbsolute ? "/" : "") + String.join("/", result); + } + /** * Resolves {@code importUri} against the module key. * diff --git a/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorTest.kt index a1b2123d3..9d7015733 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorTest.kt @@ -362,6 +362,142 @@ class EvaluatorTest { ) } + private fun evaluatorAllowingOnly(allowedDir: Path): Evaluator { + val allowedUri = allowedDir.toUri().toString() + val allowed = Pattern.compile(Pattern.quote(allowedUri)) + val allowedJar = Pattern.compile(Pattern.quote("jar:$allowedUri")) + return with(EvaluatorBuilder.preconfigured()) { + setAllowedModules( + listOf(Pattern.compile("pkl:"), Pattern.compile("repl:"), allowed, allowedJar) + ) + setAllowedResources(listOf(allowed, allowedJar)) + build() + } + } + + /** Creates `allowed/` and `secret/` directories inside [tempDir], and returns `allowed/`. */ + private fun setUpAllowedAndSecretDirs(tempDir: Path): Path { + // resolve symlinks (e.g. `/var` -> `/private/var` on macOS) so that module URIs match + val allowedDir = tempDir.toRealPath().resolve("allowed") + allowedDir.resolve("ok.pkl").createParentDirectories().writeText("x = 1") + allowedDir.resolve("ok.txt").writeText("ok") + val secretDir = allowedDir.resolveSibling("secret") + secretDir.resolve("secret.pkl").createParentDirectories().writeText("x = 2") + secretDir.resolve("secret.txt").writeText("secret") + return allowedDir + } + + @Test + fun `allowed modules - baseline - can import module inside allowed dir`(@TempDir tempDir: Path) { + val allowedDir = setUpAllowedAndSecretDirs(tempDir) + val allowedUri = allowedDir.toUri() + val module = allowedDir.resolve("test.pkl").writeString("res = import(\"${allowedUri}ok.pkl\")") + val result = evaluatorAllowingOnly(allowedDir).evaluateExpression(path(module), "res.x") + assertThat(result).isEqualTo(1L) + } + + @Test + fun `allowed modules - cannot import module using path traversal`(@TempDir tempDir: Path) { + val allowedDir = setUpAllowedAndSecretDirs(tempDir) + val allowedUri = allowedDir.toUri() + val evaluator = evaluatorAllowingOnly(allowedDir) + for (traversal in listOf("../", "%2e%2e/", "%2e./", ".%2e/", "..%2f", "%2e%2e%2f", ".%2E%2F")) { + val module = + allowedDir + .resolve("test.pkl") + .writeString("res = import(\"${allowedUri}${traversal}secret/secret.pkl\")") + val e = assertThrows { evaluator.evaluate(path(module)) } + assertThat(e) + .hasMessageContaining("/secret/") + .hasMessageContaining("module allowlist (`--allowed-modules`)") + } + } + + @Test + fun `allowed modules - cannot import module glob using path traversal`(@TempDir tempDir: Path) { + val allowedDir = setUpAllowedAndSecretDirs(tempDir) + val allowedUri = allowedDir.toUri() + val evaluator = evaluatorAllowingOnly(allowedDir) + for (traversal in listOf("../", "%2e%2e/", "..%2f", "%2e%2e%2f")) { + val module = + allowedDir + .resolve("test.pkl") + .writeString("res = import*(\"${allowedUri}${traversal}secret/*.pkl\")") + val e = assertThrows { evaluator.evaluate(path(module)) } + assertThat(e) + .hasMessageContaining("/secret/") + .hasMessageContaining("module allowlist (`--allowed-modules`)") + } + } + + @Test + fun `allowed modules - cannot import module from jar using path traversal`( + @TempDir tempDir: Path + ) { + val allowedDir = setUpAllowedAndSecretDirs(tempDir) + val allowedUri = allowedDir.toUri() + createModulesZip(allowedDir.resolveSibling("secret")) + val evaluator = evaluatorAllowingOnly(allowedDir) + for (traversal in listOf("../", "%2e%2e/", "..%2f", "%2e%2e%2f")) { + val module = + allowedDir + .resolve("test.pkl") + .writeString( + "res = import(\"jar:${allowedUri}${traversal}secret/modules.zip!/foo/baz/module2.pkl\")" + ) + val e = assertThrows { evaluator.evaluate(path(module)) } + assertThat(e) + .hasMessageContaining("/secret/modules.zip") + .hasMessageContaining("module allowlist (`--allowed-modules`)") + } + } + + @Test + fun `allowed resources - baseline - can read resource inside allowed dir`( + @TempDir tempDir: Path + ) { + val allowedDir = setUpAllowedAndSecretDirs(tempDir) + val allowedUri = allowedDir.toUri() + val module = + allowedDir.resolve("test.pkl").writeString("res = read(\"${allowedUri}ok.txt\").text") + val result = evaluatorAllowingOnly(allowedDir).evaluateExpression(path(module), "res") + assertThat(result).isEqualTo("ok") + } + + @Test + fun `allowed resources - cannot read resource using path traversal`(@TempDir tempDir: Path) { + val allowedDir = setUpAllowedAndSecretDirs(tempDir) + val allowedUri = allowedDir.toUri() + val evaluator = evaluatorAllowingOnly(allowedDir) + for (traversal in listOf("../", "%2e%2e/", "%2e./", ".%2e/", "..%2f", "%2e%2e%2f", ".%2E%2F")) { + val module = + allowedDir + .resolve("test.pkl") + .writeString("res = read(\"${allowedUri}${traversal}secret/secret.txt\").text") + val e = assertThrows { evaluator.evaluate(path(module)) } + assertThat(e) + .hasMessageContaining("/secret/") + .hasMessageContaining("resource allowlist (`--allowed-resources`)") + } + } + + @Test + fun `allowed resources - cannot read resource glob using path traversal`(@TempDir tempDir: Path) { + val allowedDir = setUpAllowedAndSecretDirs(tempDir) + val allowedUri = allowedDir.toUri() + val evaluator = evaluatorAllowingOnly(allowedDir) + for (traversal in listOf("../", "%2e%2e/", "..%2f", "%2e%2e%2f")) { + val module = + allowedDir + .resolve("test.pkl") + .writeString("res = read*(\"${allowedUri}${traversal}secret/*.txt\")") + val e = assertThrows { evaluator.evaluate(path(module)) } + assertThat(e) + .hasMessageContaining("/secret/") + .hasMessageContaining("resource allowlist (`--allowed-resources`)") + } + } + @Test fun `multiple-file output`() { val evaluator = Evaluator.preconfigured() @@ -560,17 +696,16 @@ class EvaluatorTest { build() } - val exc = - assertThrows { - evaluator.evaluate( - text( - """ - foo: String(chars.first == "a") = "boo" - """ - .trimIndent() - ) + assertThrows { + evaluator.evaluate( + text( + """ + foo: String(chars.first == "a") = "boo" + """ + .trimIndent() ) - } + ) + } assertThat((evaluator as EvaluatorImpl).isInstrumentationEverUsed()).isTrue } diff --git a/pkl-core/src/test/kotlin/org/pkl/core/util/IoUtilsTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/util/IoUtilsTest.kt index a4c83a023..10299f222 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/util/IoUtilsTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/util/IoUtilsTest.kt @@ -416,4 +416,227 @@ class IoUtilsTest { assertThat(IoUtils.encodePath("3a)")).isEqualTo("3a)") assertThat(IoUtils.encodePath("foo/bar/baz")).isEqualTo("foo/bar/baz") } + + @Test + fun `normalizeUri removes dot segments from hierarchical URIs`() { + assertThat(IoUtils.normalizeUri(URI("file:///foo/../baz.pkl"), true)) + .isEqualTo(URI("file:///baz.pkl")) + assertThat(IoUtils.normalizeUri(URI("file:///foo/%2e./baz.pkl"), true)) + .isEqualTo(URI("file:///baz.pkl")) + assertThat(IoUtils.normalizeUri(URI("file:///foo/%2e%2e/baz.pkl"), true)) + .isEqualTo(URI("file:///baz.pkl")) + assertThat(IoUtils.normalizeUri(URI("https://example.com/a/../b?q=1"), true)) + .isEqualTo(URI("https://example.com/b?q=1")) + } + + @Test + fun `normalizeUri decodes 2e but not 2f for schemes other than file, jar and package`() { + assertThat(IoUtils.normalizeUri(URI("https://example.com/a/%2e%2e/b"), true)) + .isEqualTo(URI("https://example.com/b")) + assertThat(IoUtils.normalizeUri(URI("https://example.com/a/%2E./b"), true)) + .isEqualTo(URI("https://example.com/b")) + assertThat(IoUtils.normalizeUri(URI("https://example.com/a/../b"), true)) + .isEqualTo(URI("https://example.com/b")) + // `%2F` is distinct from `/`, and so is preserved + assertThat(IoUtils.normalizeUri(URI("https://example.com/foo%2fbar/./baz"), true)) + .isEqualTo(URI("https://example.com/foo%2fbar/baz")) + val uri = URI("https://example.com/a%2f..%2fb") + assertThat(IoUtils.normalizeUri(uri, true)).isSameAs(uri) + } + + @Test + fun `normalizeUri decodes both 2e and 2f for file URIs`() { + assertThat(IoUtils.normalizeUri(URI("file:///allowed/..%2fsecret/x"), true)) + .isEqualTo(URI("file:///secret/x")) + assertThat(IoUtils.normalizeUri(URI("file:///allowed/%2E%2e%2Fsecret/x"), true)) + .isEqualTo(URI("file:///secret/x")) + } + + @Test + fun `normalizeUri decodes both 2e and 2f in package asset paths`() { + assertThat(IoUtils.normalizeUri(URI("package://example.com/foo@1.0.0#/a%2f..%2fb.pkl"), true)) + .isEqualTo(URI("package://example.com/foo@1.0.0#/b.pkl")) + assertThat( + IoUtils.normalizeUri(URI("projectpackage://example.com/foo@1.0.0#/a/%2e%2e/b.pkl"), true) + ) + .isEqualTo(URI("projectpackage://example.com/foo@1.0.0#/b.pkl")) + } + + @Test + fun `normalizeUri decodes 2f in the archive URI of jar file URIs`() { + assertThat(IoUtils.normalizeUri(URI("jar:file:///allowed/..%2fsecret/x.zip!/a%2f..%2fb"), true)) + .isEqualTo(URI("jar:file:///secret/x.zip!/a%2f..%2fb")) + } + + @Test + fun `normalizeUri leaves non-hierarchical URIs unchanged`() { + val uri = URI("foo:/bar/../baz") + assertThat(IoUtils.normalizeUri(uri, false)).isSameAs(uri) + } + + @Test + fun `normalizeUri only normalizes fragment if URI is a package`() { + assertThat(IoUtils.normalizeUri(URI("package://example.com/foo@1.0.0#/bar/../baz.pkl"), true)) + .isEqualTo(URI("package://example.com/foo@1.0.0#/baz.pkl")) + assertThat(IoUtils.normalizeUri(URI("foo://example.com/bar#/bar/../baz.pkl"), true)) + .isEqualTo(URI("foo://example.com/bar#/bar/../baz.pkl")) + } + + @Test + fun `normalizeUri does not add a fragment`() { + val uri = URI("https://example.com/foo.pkl") + assertThat(IoUtils.normalizeUri(uri, true)).isEqualTo(uri) + assertThat(IoUtils.normalizeUri(uri, true).toString()).doesNotContain("#") + } + + @Test + fun `normalizeUri returns the same URI if already normalized`() { + val uri = URI("file:///foo/b%C3%A9/baz%20qux.pkl") + assertThat(IoUtils.normalizeUri(uri, true)).isSameAs(uri) + } + + @Test + fun `normalizeUri only normalizes the archive URI of jar URIs`() { + assertThat(IoUtils.normalizeUri(URI("jar:file:///foo/../bar.zip!/a/../b.pkl"), true)) + .isEqualTo(URI("jar:file:///bar.zip!/a/../b.pkl")) + assertThat(IoUtils.normalizeUri(URI("jar:file:///foo/%2e%2e/bar.zip!/a/b.pkl"), true)) + .isEqualTo(URI("jar:file:///bar.zip!/a/b.pkl")) + assertThat(IoUtils.normalizeUri(URI("JAR:file:///foo/../bar.zip!/b.pkl"), true)) + .isEqualTo(URI("JAR:file:///bar.zip!/b.pkl")) + val uri = URI("jar:file:///foo/bar.zip!/a/../b.pkl") + assertThat(IoUtils.normalizeUri(uri, true)).isSameAs(uri) + } + + @Test + fun `normalizeUri preserves encoding of everything except dot and slash segments`() { + assertThat(IoUtils.normalizeUri(URI("file://example.com/a%2Fb/./c"), true)) + .isEqualTo(URI("file://example.com/a/b/c")) + assertThat(IoUtils.normalizeUri(URI("file:///b%C3%A9/./x%20y"), true)) + .isEqualTo(URI("file:///b%C3%A9/x%20y")) + assertThat(IoUtils.normalizeUri(URI("file://h/a/../b?x=%2e%2e#frag"), true)) + .isEqualTo(URI("file://h/b?x=%2e%2e#frag")) + } + + @Test + fun `normalizeUri handles unusual authorities`() { + assertThat(IoUtils.normalizeUri(URI("file://user:p%40ss@example.com:8080/a/../b"), true)) + .isEqualTo(URI("file://user:p%40ss@example.com:8080/b")) + assertThat(IoUtils.normalizeUri(URI("file://[::1]:8080/a/../b"), true)) + .isEqualTo(URI("file://[::1]:8080/b")) + } + + @Test + fun `normalizeUri does not turn a path into an authority`() { + assertThat(IoUtils.normalizeUri(URI("file:////etc/../x"), true)).isEqualTo(URI("file:////x")) + assertThat(IoUtils.normalizeUri(URI("file:/a/..//b"), true)).isEqualTo(URI("file:/.//b")) + } + + @Test + fun `normalizeUri handles trailing dot segments and unresolvable dot-dot segments`() { + assertThat(IoUtils.normalizeUri(URI("file:///a/b/.."), true)).isEqualTo(URI("file:///a/")) + assertThat(IoUtils.normalizeUri(URI("file:///a/."), true)).isEqualTo(URI("file:///a/")) + assertThat(IoUtils.normalizeUri(URI("file:///a/../.."), true)).isEqualTo(URI("file:///../")) + val uri = URI("file:///../x") + assertThat(IoUtils.normalizeUri(uri, true)).isSameAs(uri) + } + + @Test + fun `normalizeUri is idempotent`() { + for (str in + listOf( + "file:///a/../b/./c/%2e%2e/d", + "file:/a/..//b", + "file:////etc/../x", + "https://example.com/a%2Fb/../c?q#f", + "package://example.com/foo@1.0.0#/a/%2e%2e/../b.pkl", + "jar:file:///a/../b.zip!/c/../d", + )) { + val once = IoUtils.normalizeUri(URI(str), true) + assertThat(IoUtils.normalizeUri(once, true)).isEqualTo(once) + } + } + + @Test + fun `normalizeRawPath removes dot segments`() { + for (isWindows in listOf(false, true)) { + for (decodeSlash in listOf(false, true)) { + fun normalize(path: String) = IoUtils.normalizeRawPath(path, decodeSlash, isWindows) + assertThat(normalize("/a/b/../c")).isEqualTo("/a/c") + assertThat(normalize("/a/%2e%2E/b")).isEqualTo("/b") + assertThat(normalize("/a/./b/.")).isEqualTo("/a/b/") + assertThat(normalize("/a/b/..")).isEqualTo("/a/") + assertThat(normalize("/../x")).isEqualTo("/../x") + assertThat(normalize("/a/../..")).isEqualTo("/../") + assertThat(normalize("//etc/../x")).isEqualTo("//x") + assertThat(normalize("")).isEqualTo("") + assertThat(normalize("/")).isEqualTo("/") + } + } + } + + @Test + fun `normalizeRawPath treats 2f as a path separator only if decodeSlash`() { + for (isWindows in listOf(false, true)) { + assertThat(IoUtils.normalizeRawPath("/a%2f..%2fb", true, isWindows)).isEqualTo("/b") + assertThat(IoUtils.normalizeRawPath("/a%2F..%2Fb", true, isWindows)).isEqualTo("/b") + assertThat(IoUtils.normalizeRawPath("/a%2fb", true, isWindows)).isEqualTo("/a/b") + assertThat(IoUtils.normalizeRawPath("/a/.%2f./b", true, isWindows)).isEqualTo("/a/b") + assertThat(IoUtils.normalizeRawPath("/%2f..%2fx", true, isWindows)).isEqualTo("/x") + // `%2F` is distinct from `/` if not decodeSlash + assertThat(IoUtils.normalizeRawPath("/a%2f..%2fb", false, isWindows)).isEqualTo("/a%2f..%2fb") + assertThat(IoUtils.normalizeRawPath("/a%2fb/./c", false, isWindows)).isEqualTo("/a%2fb/c") + } + } + + @Test + fun `normalizeRawPath does not decode 2f twice`() { + assertThat(IoUtils.normalizeRawPath("/a%252f..%2fb", true, false)).isEqualTo("/a%252f../b") + assertThat(IoUtils.normalizeRawPath("/a%255c..%5cb", true, true)).isEqualTo("/a%255c../b") + } + + @Test + fun `normalizeRawPath treats 5c as a path separator only on Windows`() { + // on Windows, `\` is a path separator + assertThat(IoUtils.normalizeRawPath("/a%5c..%5cb", true, true)).isEqualTo("/b") + assertThat(IoUtils.normalizeRawPath("/a%5C..%5Cb", true, true)).isEqualTo("/b") + assertThat(IoUtils.normalizeRawPath("/a%5cb", true, true)).isEqualTo("/a/b") + assertThat(IoUtils.normalizeRawPath("/a/..%5c..%5cb", true, true)).isEqualTo("/../b") + assertThat(IoUtils.normalizeRawPath("/a%5c.%2f..%2fb", true, true)).isEqualTo("/b") + // on other platforms, `\` is a regular file name character + assertThat(IoUtils.normalizeRawPath("/a%5c..%5cb", true, false)).isEqualTo("/a%5c..%5cb") + assertThat(IoUtils.normalizeRawPath("/a%5C..%5Cb/./c", true, false)).isEqualTo("/a%5C..%5Cb/c") + } + + @Test + fun `normalizeRawPath does not treat 5c as a path separator if not decodeSlash`() { + for (isWindows in listOf(false, true)) { + assertThat(IoUtils.normalizeRawPath("/a%5c..%5cb", false, isWindows)).isEqualTo("/a%5c..%5cb") + } + } + + @Test + fun `normalizeRawPath is idempotent`() { + for (isWindows in listOf(false, true)) { + for (decodeSlash in listOf(false, true)) { + for (path in + listOf("/a/../b", "/a%2f..%2fb", "/a%5c..%5cb", "/a/.%5c./b", "//x/../y", "/../../a")) { + val once = IoUtils.normalizeRawPath(path, decodeSlash, isWindows) + assertThat(IoUtils.normalizeRawPath(once, decodeSlash, isWindows)).isEqualTo(once) + } + } + } + } + + @Test + fun `normalizeUri does not treat 5c as a path separator in other schemes`() { + val uri = URI("https://example.com/allowed/..%5csecret/x") + assertThat(IoUtils.normalizeUri(uri, true)).isSameAs(uri) + } + + @Test + fun `normalizeUri replaces the whole asset path of a package URI if it changed`() { + // the normalized asset path must not be compared to the path of the package + assertThat(IoUtils.normalizeUri(URI("package://example.com/x@1.0.0#/a/../x@1.0.0"), true)) + .isEqualTo(URI("package://example.com/x@1.0.0#/x@1.0.0")) + } }