Improve normalization of URIs (#1937)
This adds changes such that URIs like `file:///path/to/%2E%2E/bar.txt` are considered path traversal sequences. These URIs are treated as dot segments, and normalized prior to calling into security checks. For package URIs, the same treatment is applied to the URI fragment. jar-file URIs are special-cased and normalized as well. Additionally, `%2f` is treated as `/` for file, jar-file and asset path segments. However, these are left alone for other URIs (RFC-3986 treats `%2e` as identical to `.`, but `%2f` as distinct from `/`)..
This commit is contained in:
1 parent
5cb5d41a44
commit
89cb6c98a1
6 files changed
+559
-19
No files matched your search
@@ -27,6 +27,7 @@ import org.pkl.core.externalreader.ExternalReaderProcessException;
|
||||
import org.pkl.core.module.ModuleKey;
|
||||
import org.pkl.core.module.ModuleKeyFactory;
|
||||
import org.pkl.core.module.ModuleKeys;
|
||||
import org.pkl.core.util.IoUtils;
|
||||
|
||||
public final class ModuleResolver {
|
||||
private final Collection<ModuleKeyFactory> factories;
|
||||
@@ -78,6 +79,9 @@ public final class ModuleResolver {
|
||||
Optional<ModuleKey> key;
|
||||
try {
|
||||
key = factory.create(normalized);
|
||||
if (key.isPresent()) {
|
||||
key = Optional.of(renormalize(factory, normalized, key.get()));
|
||||
}
|
||||
} catch (URISyntaxException e) {
|
||||
throw new VmExceptionBuilder()
|
||||
.withOptionalLocation(importNode)
|
||||
@@ -105,4 +109,20 @@ public final class ModuleResolver {
|
||||
.withOptionalLocation(importNode)
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensures that the URI of the module key is normalized according to the key's own URI semantics
|
||||
* (which {@link URI#normalize()} knows nothing about), so that security checks and loading
|
||||
* observe the same URI.
|
||||
*
|
||||
* <p>Module keys are immutable, so a new one is created if {@code uri} changes.
|
||||
*/
|
||||
private ModuleKey renormalize(ModuleKeyFactory factory, URI uri, ModuleKey key)
|
||||
throws URISyntaxException, IOException, ExternalReaderProcessException {
|
||||
var renormalized = IoUtils.normalizeUri(uri, key.hasHierarchicalUris());
|
||||
if (renormalized.equals(uri)) return key;
|
||||
return factory
|
||||
.create(renormalized)
|
||||
.orElseThrow(() -> new IllegalStateException("failed to renormalize a module URI"));
|
||||
}
|
||||
}
|
||||
@@ -34,6 +34,7 @@ import org.pkl.core.resource.Resource;
|
||||
import org.pkl.core.resource.ResourceReader;
|
||||
import org.pkl.core.resource.ResourceReaders;
|
||||
import org.pkl.core.stdlib.VmObjectFactory;
|
||||
import org.pkl.core.util.IoUtils;
|
||||
|
||||
public final class ResourceManager {
|
||||
private final Map<String, ResourceReader> resourceReaders = new HashMap<>();
|
||||
@@ -96,10 +97,17 @@ public final class ResourceManager {
|
||||
|
||||
@TruffleBoundary
|
||||
public Optional<Object> read(URI resourceUri, @Nullable Node readNode) {
|
||||
var reader = getResourceReader(resourceUri);
|
||||
URI normalizedUri;
|
||||
try {
|
||||
normalizedUri =
|
||||
IoUtils.normalizeUri(resourceUri, reader != null && reader.hasHierarchicalUris());
|
||||
} catch (ExternalReaderProcessException | IOException e) {
|
||||
throw new VmExceptionBuilder().withCause(e).withOptionalLocation(readNode).build();
|
||||
}
|
||||
return resources.computeIfAbsent(
|
||||
resourceUri.normalize(),
|
||||
normalizedUri,
|
||||
(uri) -> {
|
||||
var reader = getResourceReader(uri);
|
||||
// hack: we don't want to call `checkReadResource` here for these resources because those
|
||||
// readers defer to HttpClient to do the actual checks.
|
||||
if (!(reader instanceof ResourceReaders.HttpResource)
|
||||
|
||||
@@ -339,16 +339,17 @@ public final class GlobResolver {
|
||||
if (listElementCallCount.getAndIncrement() > maxListElements()) {
|
||||
throw new InvalidGlobPatternException(ErrorMessages.create("invalidGlobTooComplex"));
|
||||
}
|
||||
var elements = reader.listElements(securityManager, baseUri);
|
||||
var normalizedBaseUri = normalizeUri(reader, baseUri);
|
||||
var elements = reader.listElements(securityManager, normalizedBaseUri);
|
||||
for (var element : sorted(elements)) {
|
||||
var elementPath = resolvePath(expandedGlobSoFar, element.getName(), hasAbsoluteGlob);
|
||||
if (globPartPattern.matcher(element.getName()).matches()) {
|
||||
var name = element.isDirectory() ? element.getName() + "/" : element.getName();
|
||||
var elementUri = IoUtils.resolve(reader, baseUri, name);
|
||||
var elementUri = IoUtils.resolve(reader, normalizedBaseUri, name);
|
||||
result.add(new ResolvedGlobElement(elementPath, elementUri, element.isDirectory()));
|
||||
}
|
||||
if (element.isDirectory() && isGlobStar) {
|
||||
var elementUri = IoUtils.resolve(reader, baseUri, element.getName() + "/");
|
||||
var elementUri = IoUtils.resolve(reader, normalizedBaseUri, element.getName() + "/");
|
||||
var newExpandedGlobPattern =
|
||||
resolvePath(expandedGlobSoFar, element.getName(), hasAbsoluteGlob);
|
||||
doExpandHierarchicalGlobPart(
|
||||
@@ -365,6 +366,15 @@ public final class GlobResolver {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalizes URIs before they are handed to the reader, so that security checks and I/O both see
|
||||
* the same URI.
|
||||
*/
|
||||
private static URI normalizeUri(ReaderBase reader, URI uri)
|
||||
throws IOException, ExternalReaderProcessException {
|
||||
return IoUtils.normalizeUri(uri, reader.hasHierarchicalUris());
|
||||
}
|
||||
|
||||
private static void resolveHierarchicalGlob(
|
||||
SecurityManager securityManager,
|
||||
ReaderBase reader,
|
||||
@@ -384,7 +394,7 @@ public final class GlobResolver {
|
||||
if (isRegularPathPart(patternPart)) {
|
||||
var newPath = resolvePath(expandedGlobPatternSoFar, patternPart, hasAbsoluteGlob);
|
||||
if (isLeaf) {
|
||||
var newBaseUri = IoUtils.resolve(reader, baseUri, patternPart);
|
||||
var newBaseUri = normalizeUri(reader, IoUtils.resolve(reader, baseUri, patternPart));
|
||||
if (reader.hasElement(securityManager, newBaseUri)) {
|
||||
// Note: isDirectory is not meaningful here. Setting it to false is a way to skip setting
|
||||
// it.
|
||||
@@ -501,9 +511,11 @@ public final class GlobResolver {
|
||||
// short-circuit for glob pattern with no wildcards (can only match 0 or 1 element)
|
||||
if (globParts.length == 0) {
|
||||
var resolvedUri =
|
||||
enclosingUri == null
|
||||
? URI.create(globPattern)
|
||||
: IoUtils.resolve(reader, enclosingUri, globPattern);
|
||||
normalizeUri(
|
||||
reader,
|
||||
enclosingUri == null
|
||||
? URI.create(globPattern)
|
||||
: IoUtils.resolve(reader, enclosingUri, globPattern));
|
||||
if (reader.hasElement(securityManager, resolvedUri)) {
|
||||
result.put(globPattern, new ResolvedGlobElement(globPattern, resolvedUri, true));
|
||||
}
|
||||
|
||||
@@ -85,6 +85,11 @@ public final class IoUtils {
|
||||
// keep in sync with stdlib `EvaluatorSettings.reservedHttpHeaderPrefix`
|
||||
private static final String[] reservedHeaderPrefixes = {"proxy-", "sec-"};
|
||||
|
||||
private static final Pattern DOT_SEGMENT = Pattern.compile("\\.|%2[eE]");
|
||||
private static final Pattern DOT_DOT_SEGMENT = Pattern.compile("(?:\\.|%2[eE]){2}");
|
||||
private static final Pattern PATH_SEPARATOR = Pattern.compile("/|%2[fF]");
|
||||
private static final Pattern WINDOWS_PATH_SEPARATOR = Pattern.compile("/|%2[fF]|%5[cC]");
|
||||
|
||||
private IoUtils() {}
|
||||
|
||||
public static URL toUrl(URI uri) throws IOException {
|
||||
@@ -438,6 +443,143 @@ public final class IoUtils {
|
||||
throw new PackageLoadError("cannotFindDependencyInProject", name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalizes {@code uri} for the purpose of security checks.
|
||||
*
|
||||
* <p>In hierarchical URIs of any scheme, removes {@code .} and {@code ..} path segments, and
|
||||
* treats {@code %2E} as equivalent to {@code .}. {@code .} is an unreserved character, so a
|
||||
* percent-encoded {@code .} is equivalent to a literal one (RFC 3986, section 6.2.2.2). All other
|
||||
* parts of the URI are left as is. In particular, {@code %2F} is not treated as a path separator,
|
||||
* because it might be distinct from {@code /} (for example, in HTTP).
|
||||
*
|
||||
* <p>For {@code file:} URIs, also treats {@code %2F} as equivalent to {@code /}, because file
|
||||
* systems do.
|
||||
*
|
||||
* <p>For {@code package:} and {@code projectpackage:} URIs, also normalizes the path of the asset
|
||||
* in the fragment, treating {@code %2E} and {@code %2F} the same way as for {@code file:} URIs.
|
||||
*
|
||||
* <p>For {@code jar:} URIs, only the URI of the archive is normalized. The path within the
|
||||
* archive is left as is, because zip file systems treat {@code ..} as a verbatim name.
|
||||
*/
|
||||
public static URI normalizeUri(URI uri, boolean hasHierarchicalUris) {
|
||||
var scheme = uri.getScheme();
|
||||
if ("jar".equalsIgnoreCase(scheme)) {
|
||||
return normalizeJarUri(uri);
|
||||
}
|
||||
if ("file".equalsIgnoreCase(scheme)) {
|
||||
return normalizeUriPath(uri, true);
|
||||
}
|
||||
if ("package".equalsIgnoreCase(scheme) || "projectpackage".equalsIgnoreCase(scheme)) {
|
||||
return normalizePackageUri(uri);
|
||||
}
|
||||
if (hasHierarchicalUris) {
|
||||
return normalizeUriPath(uri, false);
|
||||
}
|
||||
return uri;
|
||||
}
|
||||
|
||||
private static URI normalizePackageUri(URI uri) {
|
||||
var assetPath = uri.getRawFragment();
|
||||
if (assetPath == null || !assetPath.startsWith("/")) {
|
||||
return uri;
|
||||
}
|
||||
var normalizedPath = normalizeRawPath(assetPath, true, isWindows());
|
||||
if (normalizedPath.equals(uri.getRawFragment())) {
|
||||
return uri;
|
||||
}
|
||||
var uriString = uri.toString();
|
||||
var hashIndex = uriString.indexOf('#');
|
||||
return URI.create(uriString.substring(0, hashIndex) + "#" + normalizedPath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalizes the path of {@code uri}.
|
||||
*
|
||||
* @param decodeSlash whether to treat {@code %2F} as a path separator
|
||||
*/
|
||||
private static URI normalizeUriPath(URI uri, boolean decodeSlash) {
|
||||
if (uri.isOpaque() || uri.getScheme() == null) {
|
||||
return uri;
|
||||
}
|
||||
// Use string builder instead of component-wise accessors/constructors because component-wise
|
||||
// will re-encode (e.g. turn %2F into `/`), and can fail for unusual authorities.
|
||||
var uriString = uri.toString();
|
||||
var rawPath = uri.getRawPath();
|
||||
var pathStart = uriString.indexOf(':') + 1;
|
||||
var hasAuthority = uriString.startsWith("//", pathStart);
|
||||
if (hasAuthority) {
|
||||
var rawAuthority = uri.getRawAuthority();
|
||||
pathStart += 2 + (rawAuthority == null ? 0 : rawAuthority.length());
|
||||
}
|
||||
assert uriString.startsWith(rawPath, pathStart);
|
||||
var normalizedPath = normalizeRawPath(rawPath, decodeSlash, isWindows());
|
||||
if (normalizedPath.equals(rawPath)) {
|
||||
return uri;
|
||||
}
|
||||
var sb = new StringBuilder(uriString.substring(0, pathStart));
|
||||
if (!hasAuthority && normalizedPath.startsWith("//")) {
|
||||
// avoid the path being misinterpreted as an authority
|
||||
sb.append("/.");
|
||||
}
|
||||
sb.append(normalizedPath);
|
||||
var pathEnd = pathStart + rawPath.length();
|
||||
sb.append(uriString, pathEnd, uriString.length());
|
||||
return URI.create(sb.toString());
|
||||
}
|
||||
|
||||
/** Normalizes the URI of the archive of {@code jar:<archive-uri>!/<path-in-archive>}. */
|
||||
private static URI normalizeJarUri(URI uri) {
|
||||
var uriString = uri.toString();
|
||||
var index = uriString.indexOf('!');
|
||||
if (index == -1) {
|
||||
return uri;
|
||||
}
|
||||
var archiveUri = URI.create(uriString.substring(uri.getScheme().length() + 1, index));
|
||||
var normalizedArchiveUri = normalizeUri(archiveUri, true);
|
||||
if (normalizedArchiveUri.equals(archiveUri)) {
|
||||
return uri;
|
||||
}
|
||||
return URI.create(uri.getScheme() + ":" + normalizedArchiveUri + uriString.substring(index));
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes {@code .} and {@code ..} segments from a raw (percent-encoded) path.
|
||||
*
|
||||
* <p>Segments written as {@code %2e} are treated as dots, because readers decode the path before
|
||||
* using it. All other segments are kept verbatim. A {@code ..} that cannot be resolved is kept,
|
||||
* so the result does not match patterns it shouldn't.
|
||||
*
|
||||
* <p>If {@code decodeSlash} is true, treats either {@code /} or {@code %2F} as the path
|
||||
* separator, and replaces {@code %2F} with {@code /}. Otherwise, {@code %2F} is kept as is.
|
||||
*
|
||||
* <p>If {@code isWindows}, then {@code %5c} is also treated as a slash.
|
||||
*/
|
||||
static String normalizeRawPath(String rawPath, boolean decodeSlash, boolean isWindows) {
|
||||
var isAbsolute = rawPath.startsWith("/");
|
||||
var rawSegments = (isAbsolute ? rawPath.substring(1) : rawPath);
|
||||
var separator = isWindows ? WINDOWS_PATH_SEPARATOR : PATH_SEPARATOR;
|
||||
var segments = decodeSlash ? separator.split(rawSegments, -1) : rawSegments.split("/", -1);
|
||||
var result = new ArrayList<String>(segments.length);
|
||||
for (var i = 0; i < segments.length; i++) {
|
||||
var segment = segments[i];
|
||||
var isLast = i == segments.length - 1;
|
||||
if (DOT_SEGMENT.matcher(segment).matches()) {
|
||||
if (isLast) result.add("");
|
||||
} else if (DOT_DOT_SEGMENT.matcher(segment).matches()) {
|
||||
if (!result.isEmpty()
|
||||
&& !DOT_DOT_SEGMENT.matcher(result.get(result.size() - 1)).matches()) {
|
||||
result.remove(result.size() - 1);
|
||||
} else {
|
||||
result.add(segment);
|
||||
}
|
||||
if (isLast) result.add("");
|
||||
} else {
|
||||
result.add(segment);
|
||||
}
|
||||
}
|
||||
return (isAbsolute ? "/" : "") + String.join("/", result);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves {@code importUri} against the module key.
|
||||
*
|
||||
|
||||
@@ -362,6 +362,142 @@ class EvaluatorTest {
|
||||
)
|
||||
}
|
||||
|
||||
private fun evaluatorAllowingOnly(allowedDir: Path): Evaluator {
|
||||
val allowedUri = allowedDir.toUri().toString()
|
||||
val allowed = Pattern.compile(Pattern.quote(allowedUri))
|
||||
val allowedJar = Pattern.compile(Pattern.quote("jar:$allowedUri"))
|
||||
return with(EvaluatorBuilder.preconfigured()) {
|
||||
setAllowedModules(
|
||||
listOf(Pattern.compile("pkl:"), Pattern.compile("repl:"), allowed, allowedJar)
|
||||
)
|
||||
setAllowedResources(listOf(allowed, allowedJar))
|
||||
build()
|
||||
}
|
||||
}
|
||||
|
||||
/** Creates `allowed/` and `secret/` directories inside [tempDir], and returns `allowed/`. */
|
||||
private fun setUpAllowedAndSecretDirs(tempDir: Path): Path {
|
||||
// resolve symlinks (e.g. `/var` -> `/private/var` on macOS) so that module URIs match
|
||||
val allowedDir = tempDir.toRealPath().resolve("allowed")
|
||||
allowedDir.resolve("ok.pkl").createParentDirectories().writeText("x = 1")
|
||||
allowedDir.resolve("ok.txt").writeText("ok")
|
||||
val secretDir = allowedDir.resolveSibling("secret")
|
||||
secretDir.resolve("secret.pkl").createParentDirectories().writeText("x = 2")
|
||||
secretDir.resolve("secret.txt").writeText("secret")
|
||||
return allowedDir
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `allowed modules - baseline - can import module inside allowed dir`(@TempDir tempDir: Path) {
|
||||
val allowedDir = setUpAllowedAndSecretDirs(tempDir)
|
||||
val allowedUri = allowedDir.toUri()
|
||||
val module = allowedDir.resolve("test.pkl").writeString("res = import(\"${allowedUri}ok.pkl\")")
|
||||
val result = evaluatorAllowingOnly(allowedDir).evaluateExpression(path(module), "res.x")
|
||||
assertThat(result).isEqualTo(1L)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `allowed modules - cannot import module using path traversal`(@TempDir tempDir: Path) {
|
||||
val allowedDir = setUpAllowedAndSecretDirs(tempDir)
|
||||
val allowedUri = allowedDir.toUri()
|
||||
val evaluator = evaluatorAllowingOnly(allowedDir)
|
||||
for (traversal in listOf("../", "%2e%2e/", "%2e./", ".%2e/", "..%2f", "%2e%2e%2f", ".%2E%2F")) {
|
||||
val module =
|
||||
allowedDir
|
||||
.resolve("test.pkl")
|
||||
.writeString("res = import(\"${allowedUri}${traversal}secret/secret.pkl\")")
|
||||
val e = assertThrows<PklException> { evaluator.evaluate(path(module)) }
|
||||
assertThat(e)
|
||||
.hasMessageContaining("/secret/")
|
||||
.hasMessageContaining("module allowlist (`--allowed-modules`)")
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `allowed modules - cannot import module glob using path traversal`(@TempDir tempDir: Path) {
|
||||
val allowedDir = setUpAllowedAndSecretDirs(tempDir)
|
||||
val allowedUri = allowedDir.toUri()
|
||||
val evaluator = evaluatorAllowingOnly(allowedDir)
|
||||
for (traversal in listOf("../", "%2e%2e/", "..%2f", "%2e%2e%2f")) {
|
||||
val module =
|
||||
allowedDir
|
||||
.resolve("test.pkl")
|
||||
.writeString("res = import*(\"${allowedUri}${traversal}secret/*.pkl\")")
|
||||
val e = assertThrows<PklException> { evaluator.evaluate(path(module)) }
|
||||
assertThat(e)
|
||||
.hasMessageContaining("/secret/")
|
||||
.hasMessageContaining("module allowlist (`--allowed-modules`)")
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `allowed modules - cannot import module from jar using path traversal`(
|
||||
@TempDir tempDir: Path
|
||||
) {
|
||||
val allowedDir = setUpAllowedAndSecretDirs(tempDir)
|
||||
val allowedUri = allowedDir.toUri()
|
||||
createModulesZip(allowedDir.resolveSibling("secret"))
|
||||
val evaluator = evaluatorAllowingOnly(allowedDir)
|
||||
for (traversal in listOf("../", "%2e%2e/", "..%2f", "%2e%2e%2f")) {
|
||||
val module =
|
||||
allowedDir
|
||||
.resolve("test.pkl")
|
||||
.writeString(
|
||||
"res = import(\"jar:${allowedUri}${traversal}secret/modules.zip!/foo/baz/module2.pkl\")"
|
||||
)
|
||||
val e = assertThrows<PklException> { evaluator.evaluate(path(module)) }
|
||||
assertThat(e)
|
||||
.hasMessageContaining("/secret/modules.zip")
|
||||
.hasMessageContaining("module allowlist (`--allowed-modules`)")
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `allowed resources - baseline - can read resource inside allowed dir`(
|
||||
@TempDir tempDir: Path
|
||||
) {
|
||||
val allowedDir = setUpAllowedAndSecretDirs(tempDir)
|
||||
val allowedUri = allowedDir.toUri()
|
||||
val module =
|
||||
allowedDir.resolve("test.pkl").writeString("res = read(\"${allowedUri}ok.txt\").text")
|
||||
val result = evaluatorAllowingOnly(allowedDir).evaluateExpression(path(module), "res")
|
||||
assertThat(result).isEqualTo("ok")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `allowed resources - cannot read resource using path traversal`(@TempDir tempDir: Path) {
|
||||
val allowedDir = setUpAllowedAndSecretDirs(tempDir)
|
||||
val allowedUri = allowedDir.toUri()
|
||||
val evaluator = evaluatorAllowingOnly(allowedDir)
|
||||
for (traversal in listOf("../", "%2e%2e/", "%2e./", ".%2e/", "..%2f", "%2e%2e%2f", ".%2E%2F")) {
|
||||
val module =
|
||||
allowedDir
|
||||
.resolve("test.pkl")
|
||||
.writeString("res = read(\"${allowedUri}${traversal}secret/secret.txt\").text")
|
||||
val e = assertThrows<PklException> { evaluator.evaluate(path(module)) }
|
||||
assertThat(e)
|
||||
.hasMessageContaining("/secret/")
|
||||
.hasMessageContaining("resource allowlist (`--allowed-resources`)")
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `allowed resources - cannot read resource glob using path traversal`(@TempDir tempDir: Path) {
|
||||
val allowedDir = setUpAllowedAndSecretDirs(tempDir)
|
||||
val allowedUri = allowedDir.toUri()
|
||||
val evaluator = evaluatorAllowingOnly(allowedDir)
|
||||
for (traversal in listOf("../", "%2e%2e/", "..%2f", "%2e%2e%2f")) {
|
||||
val module =
|
||||
allowedDir
|
||||
.resolve("test.pkl")
|
||||
.writeString("res = read*(\"${allowedUri}${traversal}secret/*.txt\")")
|
||||
val e = assertThrows<PklException> { evaluator.evaluate(path(module)) }
|
||||
assertThat(e)
|
||||
.hasMessageContaining("/secret/")
|
||||
.hasMessageContaining("resource allowlist (`--allowed-resources`)")
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `multiple-file output`() {
|
||||
val evaluator = Evaluator.preconfigured()
|
||||
@@ -560,17 +696,16 @@ class EvaluatorTest {
|
||||
build()
|
||||
}
|
||||
|
||||
val exc =
|
||||
assertThrows<PklException> {
|
||||
evaluator.evaluate(
|
||||
text(
|
||||
"""
|
||||
foo: String(chars.first == "a") = "boo"
|
||||
"""
|
||||
.trimIndent()
|
||||
)
|
||||
assertThrows<PklException> {
|
||||
evaluator.evaluate(
|
||||
text(
|
||||
"""
|
||||
foo: String(chars.first == "a") = "boo"
|
||||
"""
|
||||
.trimIndent()
|
||||
)
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
assertThat((evaluator as EvaluatorImpl).isInstrumentationEverUsed()).isTrue
|
||||
}
|
||||
|
||||
@@ -416,4 +416,227 @@ class IoUtilsTest {
|
||||
assertThat(IoUtils.encodePath("3a)")).isEqualTo("3a)")
|
||||
assertThat(IoUtils.encodePath("foo/bar/baz")).isEqualTo("foo/bar/baz")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `normalizeUri removes dot segments from hierarchical URIs`() {
|
||||
assertThat(IoUtils.normalizeUri(URI("file:///foo/../baz.pkl"), true))
|
||||
.isEqualTo(URI("file:///baz.pkl"))
|
||||
assertThat(IoUtils.normalizeUri(URI("file:///foo/%2e./baz.pkl"), true))
|
||||
.isEqualTo(URI("file:///baz.pkl"))
|
||||
assertThat(IoUtils.normalizeUri(URI("file:///foo/%2e%2e/baz.pkl"), true))
|
||||
.isEqualTo(URI("file:///baz.pkl"))
|
||||
assertThat(IoUtils.normalizeUri(URI("https://example.com/a/../b?q=1"), true))
|
||||
.isEqualTo(URI("https://example.com/b?q=1"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `normalizeUri decodes 2e but not 2f for schemes other than file, jar and package`() {
|
||||
assertThat(IoUtils.normalizeUri(URI("https://example.com/a/%2e%2e/b"), true))
|
||||
.isEqualTo(URI("https://example.com/b"))
|
||||
assertThat(IoUtils.normalizeUri(URI("https://example.com/a/%2E./b"), true))
|
||||
.isEqualTo(URI("https://example.com/b"))
|
||||
assertThat(IoUtils.normalizeUri(URI("https://example.com/a/../b"), true))
|
||||
.isEqualTo(URI("https://example.com/b"))
|
||||
// `%2F` is distinct from `/`, and so is preserved
|
||||
assertThat(IoUtils.normalizeUri(URI("https://example.com/foo%2fbar/./baz"), true))
|
||||
.isEqualTo(URI("https://example.com/foo%2fbar/baz"))
|
||||
val uri = URI("https://example.com/a%2f..%2fb")
|
||||
assertThat(IoUtils.normalizeUri(uri, true)).isSameAs(uri)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `normalizeUri decodes both 2e and 2f for file URIs`() {
|
||||
assertThat(IoUtils.normalizeUri(URI("file:///allowed/..%2fsecret/x"), true))
|
||||
.isEqualTo(URI("file:///secret/x"))
|
||||
assertThat(IoUtils.normalizeUri(URI("file:///allowed/%2E%2e%2Fsecret/x"), true))
|
||||
.isEqualTo(URI("file:///secret/x"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `normalizeUri decodes both 2e and 2f in package asset paths`() {
|
||||
assertThat(IoUtils.normalizeUri(URI("package://example.com/foo@1.0.0#/a%2f..%2fb.pkl"), true))
|
||||
.isEqualTo(URI("package://example.com/foo@1.0.0#/b.pkl"))
|
||||
assertThat(
|
||||
IoUtils.normalizeUri(URI("projectpackage://example.com/foo@1.0.0#/a/%2e%2e/b.pkl"), true)
|
||||
)
|
||||
.isEqualTo(URI("projectpackage://example.com/foo@1.0.0#/b.pkl"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `normalizeUri decodes 2f in the archive URI of jar file URIs`() {
|
||||
assertThat(IoUtils.normalizeUri(URI("jar:file:///allowed/..%2fsecret/x.zip!/a%2f..%2fb"), true))
|
||||
.isEqualTo(URI("jar:file:///secret/x.zip!/a%2f..%2fb"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `normalizeUri leaves non-hierarchical URIs unchanged`() {
|
||||
val uri = URI("foo:/bar/../baz")
|
||||
assertThat(IoUtils.normalizeUri(uri, false)).isSameAs(uri)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `normalizeUri only normalizes fragment if URI is a package`() {
|
||||
assertThat(IoUtils.normalizeUri(URI("package://example.com/foo@1.0.0#/bar/../baz.pkl"), true))
|
||||
.isEqualTo(URI("package://example.com/foo@1.0.0#/baz.pkl"))
|
||||
assertThat(IoUtils.normalizeUri(URI("foo://example.com/bar#/bar/../baz.pkl"), true))
|
||||
.isEqualTo(URI("foo://example.com/bar#/bar/../baz.pkl"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `normalizeUri does not add a fragment`() {
|
||||
val uri = URI("https://example.com/foo.pkl")
|
||||
assertThat(IoUtils.normalizeUri(uri, true)).isEqualTo(uri)
|
||||
assertThat(IoUtils.normalizeUri(uri, true).toString()).doesNotContain("#")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `normalizeUri returns the same URI if already normalized`() {
|
||||
val uri = URI("file:///foo/b%C3%A9/baz%20qux.pkl")
|
||||
assertThat(IoUtils.normalizeUri(uri, true)).isSameAs(uri)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `normalizeUri only normalizes the archive URI of jar URIs`() {
|
||||
assertThat(IoUtils.normalizeUri(URI("jar:file:///foo/../bar.zip!/a/../b.pkl"), true))
|
||||
.isEqualTo(URI("jar:file:///bar.zip!/a/../b.pkl"))
|
||||
assertThat(IoUtils.normalizeUri(URI("jar:file:///foo/%2e%2e/bar.zip!/a/b.pkl"), true))
|
||||
.isEqualTo(URI("jar:file:///bar.zip!/a/b.pkl"))
|
||||
assertThat(IoUtils.normalizeUri(URI("JAR:file:///foo/../bar.zip!/b.pkl"), true))
|
||||
.isEqualTo(URI("JAR:file:///bar.zip!/b.pkl"))
|
||||
val uri = URI("jar:file:///foo/bar.zip!/a/../b.pkl")
|
||||
assertThat(IoUtils.normalizeUri(uri, true)).isSameAs(uri)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `normalizeUri preserves encoding of everything except dot and slash segments`() {
|
||||
assertThat(IoUtils.normalizeUri(URI("file://example.com/a%2Fb/./c"), true))
|
||||
.isEqualTo(URI("file://example.com/a/b/c"))
|
||||
assertThat(IoUtils.normalizeUri(URI("file:///b%C3%A9/./x%20y"), true))
|
||||
.isEqualTo(URI("file:///b%C3%A9/x%20y"))
|
||||
assertThat(IoUtils.normalizeUri(URI("file://h/a/../b?x=%2e%2e#frag"), true))
|
||||
.isEqualTo(URI("file://h/b?x=%2e%2e#frag"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `normalizeUri handles unusual authorities`() {
|
||||
assertThat(IoUtils.normalizeUri(URI("file://user:p%40ss@example.com:8080/a/../b"), true))
|
||||
.isEqualTo(URI("file://user:p%40ss@example.com:8080/b"))
|
||||
assertThat(IoUtils.normalizeUri(URI("file://[::1]:8080/a/../b"), true))
|
||||
.isEqualTo(URI("file://[::1]:8080/b"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `normalizeUri does not turn a path into an authority`() {
|
||||
assertThat(IoUtils.normalizeUri(URI("file:////etc/../x"), true)).isEqualTo(URI("file:////x"))
|
||||
assertThat(IoUtils.normalizeUri(URI("file:/a/..//b"), true)).isEqualTo(URI("file:/.//b"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `normalizeUri handles trailing dot segments and unresolvable dot-dot segments`() {
|
||||
assertThat(IoUtils.normalizeUri(URI("file:///a/b/.."), true)).isEqualTo(URI("file:///a/"))
|
||||
assertThat(IoUtils.normalizeUri(URI("file:///a/."), true)).isEqualTo(URI("file:///a/"))
|
||||
assertThat(IoUtils.normalizeUri(URI("file:///a/../.."), true)).isEqualTo(URI("file:///../"))
|
||||
val uri = URI("file:///../x")
|
||||
assertThat(IoUtils.normalizeUri(uri, true)).isSameAs(uri)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `normalizeUri is idempotent`() {
|
||||
for (str in
|
||||
listOf(
|
||||
"file:///a/../b/./c/%2e%2e/d",
|
||||
"file:/a/..//b",
|
||||
"file:////etc/../x",
|
||||
"https://example.com/a%2Fb/../c?q#f",
|
||||
"package://example.com/foo@1.0.0#/a/%2e%2e/../b.pkl",
|
||||
"jar:file:///a/../b.zip!/c/../d",
|
||||
)) {
|
||||
val once = IoUtils.normalizeUri(URI(str), true)
|
||||
assertThat(IoUtils.normalizeUri(once, true)).isEqualTo(once)
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `normalizeRawPath removes dot segments`() {
|
||||
for (isWindows in listOf(false, true)) {
|
||||
for (decodeSlash in listOf(false, true)) {
|
||||
fun normalize(path: String) = IoUtils.normalizeRawPath(path, decodeSlash, isWindows)
|
||||
assertThat(normalize("/a/b/../c")).isEqualTo("/a/c")
|
||||
assertThat(normalize("/a/%2e%2E/b")).isEqualTo("/b")
|
||||
assertThat(normalize("/a/./b/.")).isEqualTo("/a/b/")
|
||||
assertThat(normalize("/a/b/..")).isEqualTo("/a/")
|
||||
assertThat(normalize("/../x")).isEqualTo("/../x")
|
||||
assertThat(normalize("/a/../..")).isEqualTo("/../")
|
||||
assertThat(normalize("//etc/../x")).isEqualTo("//x")
|
||||
assertThat(normalize("")).isEqualTo("")
|
||||
assertThat(normalize("/")).isEqualTo("/")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `normalizeRawPath treats 2f as a path separator only if decodeSlash`() {
|
||||
for (isWindows in listOf(false, true)) {
|
||||
assertThat(IoUtils.normalizeRawPath("/a%2f..%2fb", true, isWindows)).isEqualTo("/b")
|
||||
assertThat(IoUtils.normalizeRawPath("/a%2F..%2Fb", true, isWindows)).isEqualTo("/b")
|
||||
assertThat(IoUtils.normalizeRawPath("/a%2fb", true, isWindows)).isEqualTo("/a/b")
|
||||
assertThat(IoUtils.normalizeRawPath("/a/.%2f./b", true, isWindows)).isEqualTo("/a/b")
|
||||
assertThat(IoUtils.normalizeRawPath("/%2f..%2fx", true, isWindows)).isEqualTo("/x")
|
||||
// `%2F` is distinct from `/` if not decodeSlash
|
||||
assertThat(IoUtils.normalizeRawPath("/a%2f..%2fb", false, isWindows)).isEqualTo("/a%2f..%2fb")
|
||||
assertThat(IoUtils.normalizeRawPath("/a%2fb/./c", false, isWindows)).isEqualTo("/a%2fb/c")
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `normalizeRawPath does not decode 2f twice`() {
|
||||
assertThat(IoUtils.normalizeRawPath("/a%252f..%2fb", true, false)).isEqualTo("/a%252f../b")
|
||||
assertThat(IoUtils.normalizeRawPath("/a%255c..%5cb", true, true)).isEqualTo("/a%255c../b")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `normalizeRawPath treats 5c as a path separator only on Windows`() {
|
||||
// on Windows, `\` is a path separator
|
||||
assertThat(IoUtils.normalizeRawPath("/a%5c..%5cb", true, true)).isEqualTo("/b")
|
||||
assertThat(IoUtils.normalizeRawPath("/a%5C..%5Cb", true, true)).isEqualTo("/b")
|
||||
assertThat(IoUtils.normalizeRawPath("/a%5cb", true, true)).isEqualTo("/a/b")
|
||||
assertThat(IoUtils.normalizeRawPath("/a/..%5c..%5cb", true, true)).isEqualTo("/../b")
|
||||
assertThat(IoUtils.normalizeRawPath("/a%5c.%2f..%2fb", true, true)).isEqualTo("/b")
|
||||
// on other platforms, `\` is a regular file name character
|
||||
assertThat(IoUtils.normalizeRawPath("/a%5c..%5cb", true, false)).isEqualTo("/a%5c..%5cb")
|
||||
assertThat(IoUtils.normalizeRawPath("/a%5C..%5Cb/./c", true, false)).isEqualTo("/a%5C..%5Cb/c")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `normalizeRawPath does not treat 5c as a path separator if not decodeSlash`() {
|
||||
for (isWindows in listOf(false, true)) {
|
||||
assertThat(IoUtils.normalizeRawPath("/a%5c..%5cb", false, isWindows)).isEqualTo("/a%5c..%5cb")
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `normalizeRawPath is idempotent`() {
|
||||
for (isWindows in listOf(false, true)) {
|
||||
for (decodeSlash in listOf(false, true)) {
|
||||
for (path in
|
||||
listOf("/a/../b", "/a%2f..%2fb", "/a%5c..%5cb", "/a/.%5c./b", "//x/../y", "/../../a")) {
|
||||
val once = IoUtils.normalizeRawPath(path, decodeSlash, isWindows)
|
||||
assertThat(IoUtils.normalizeRawPath(once, decodeSlash, isWindows)).isEqualTo(once)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `normalizeUri does not treat 5c as a path separator in other schemes`() {
|
||||
val uri = URI("https://example.com/allowed/..%5csecret/x")
|
||||
assertThat(IoUtils.normalizeUri(uri, true)).isSameAs(uri)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `normalizeUri replaces the whole asset path of a package URI if it changed`() {
|
||||
// the normalized asset path must not be compared to the path of the package
|
||||
assertThat(IoUtils.normalizeUri(URI("package://example.com/x@1.0.0#/a/../x@1.0.0"), true))
|
||||
.isEqualTo(URI("package://example.com/x@1.0.0#/x@1.0.0"))
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user