mirror of
https://github.com/mountain-loop/yaak.git
synced 2026-11-20 07:21:35 +01:00
Add Bruno imports and a file-tree importer API (#706)
This commit is contained in:
1 parent
46464cfacf
commit
aa10c44a45
109 files changed
+4684
-879
No files matched your search
@@ -51,6 +51,7 @@ import type {
|
||||
} from "@yaakapp-internal/plugins";
|
||||
import { applyDynamicFormInput } from "./common";
|
||||
import { EventChannel } from "./EventChannel";
|
||||
import { runImporter } from "./importFiles";
|
||||
import { migrateTemplateFunctionSelectOptions } from "./migrations";
|
||||
import { createResponseBody, decodeBase64Chunk } from "./responseBody";
|
||||
|
||||
@@ -157,19 +158,14 @@ export class PluginInstance {
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
payload.type === "import_request" &&
|
||||
typeof this.#mod?.importer?.onImport === "function"
|
||||
) {
|
||||
const reply = await this.#mod.importer.onImport(ctx, {
|
||||
text: payload.content,
|
||||
});
|
||||
if (payload.type === "import_request" && this.#mod?.importer != null) {
|
||||
const reply = await runImporter(this.#mod.importer, ctx, payload);
|
||||
if (reply != null) {
|
||||
const replyPayload: InternalEventPayload = {
|
||||
type: "import_response",
|
||||
importer: this.#mod.importer.name,
|
||||
resources: reply.resources as ImportResources,
|
||||
sourceKeys: reply.sourceKeys ?? null,
|
||||
sourceKeys: reply.sourceKeys,
|
||||
};
|
||||
this.#sendPayload(context, replyPayload, replyId);
|
||||
return;
|
||||
@@ -649,10 +645,12 @@ export class PluginInstance {
|
||||
complete: info.complete,
|
||||
},
|
||||
async (offset, length) => {
|
||||
const chunk = await this.#sendForReply<ReadHttpResponseBodyChunkResponse>(
|
||||
context,
|
||||
{ type: "read_http_response_body_chunk_request", responseId, offset, length },
|
||||
);
|
||||
const chunk = await this.#sendForReply<ReadHttpResponseBodyChunkResponse>(context, {
|
||||
type: "read_http_response_body_chunk_request",
|
||||
responseId,
|
||||
offset,
|
||||
length,
|
||||
});
|
||||
return decodeBase64Chunk(chunk.data);
|
||||
},
|
||||
{ refresh: bodyInfo },
|
||||
@@ -853,7 +851,10 @@ export class PluginInstance {
|
||||
// carries the only copy of its body. A saved one is read back from
|
||||
// the host like any other. Callers get the same thing either way.
|
||||
if (body == null) {
|
||||
return { httpResponse: forPlugin(httpResponse), body: await storedBody(httpResponse.id) };
|
||||
return {
|
||||
httpResponse: forPlugin(httpResponse),
|
||||
body: await storedBody(httpResponse.id),
|
||||
};
|
||||
}
|
||||
|
||||
const bytes = decodeBase64Chunk(body);
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,224 @@
|
||||
import { mkdtemp, mkdir, readFile, rm, symlink, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import type { Context, ImportRequest, ImportSource } from "@yaakapp/api";
|
||||
import { afterEach, beforeEach, describe, expect, test, vi } from "vite-plus/test";
|
||||
import { createImportFiles, runImporter } from "./importFiles";
|
||||
|
||||
const dirs: string[] = [];
|
||||
beforeEach(() => {
|
||||
vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
});
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks();
|
||||
await Promise.all(dirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true })));
|
||||
});
|
||||
const input = (name: string, data: Uint8Array): ImportRequest => ({
|
||||
content: "",
|
||||
source: { type: "file", name, base64: Buffer.from(data).toString("base64") },
|
||||
});
|
||||
async function zip(name = "import-files.zip") {
|
||||
return input(name, await readFile(new URL(`./fixtures/${name}`, import.meta.url)));
|
||||
}
|
||||
const ctx = {} as Context;
|
||||
|
||||
describe("import file tree", () => {
|
||||
test("exposes legacy text and single binary files through the same read API", async () => {
|
||||
const session = await createImportFiles({ content: "café" });
|
||||
expect(session.text).toBe("café");
|
||||
expect(await session.files.readDir()).toEqual([{ name: "input", path: "input", type: "file" }]);
|
||||
expect(await session.files.readTextFile("input")).toBe("café");
|
||||
session.close();
|
||||
await expect(session.files.readFile("input")).rejects.toThrow("closed");
|
||||
const binary = await createImportFiles(input("body.bin", Uint8Array.from([255, 0, 128])));
|
||||
expect(binary.isText).toBe(false);
|
||||
expect([...(await binary.files.readFile("body.bin"))]).toEqual([255, 0, 128]);
|
||||
await expect(binary.files.readTextFile("body.bin")).rejects.toThrow();
|
||||
binary.close();
|
||||
});
|
||||
|
||||
test("lists immediate ZIP children, including implicit/empty directories, and reads binary/UTF-8", async () => {
|
||||
const session = await createImportFiles(await zip());
|
||||
expect(session.files.kind).toBe("zip");
|
||||
expect(session.text).toBe("");
|
||||
expect(await session.files.readDir()).toEqual([
|
||||
{ name: "binary.bin", path: "binary.bin", type: "file" },
|
||||
{ name: "collection", path: "collection", type: "directory" },
|
||||
]);
|
||||
expect((await session.files.readDir("collection")).map((e) => e.name)).toEqual([
|
||||
"café.yml",
|
||||
"empty",
|
||||
"nested",
|
||||
]);
|
||||
expect(await session.files.readDir("collection/empty")).toEqual([]);
|
||||
expect(await session.files.readTextFile("collection/café.yml")).toBe("name: café\n");
|
||||
expect([...(await session.files.readFile("binary.bin"))]).toEqual([0, 255, 128]);
|
||||
await expect(session.files.readFile("collection")).rejects.toThrow("not found");
|
||||
await expect(session.files.readDir("missing")).rejects.toThrow("not found");
|
||||
session.close();
|
||||
});
|
||||
|
||||
test("a misleading ZIP suffix still dispatches as text", async () => {
|
||||
const content = '{"openapi":"3.0.0","info":{"title":"Example","version":"1"},"paths":{}}';
|
||||
const source = input("api.zip", Buffer.from(content));
|
||||
const onImport = vi.fn().mockResolvedValue(null);
|
||||
await runImporter({ name: "Text", onImport }, ctx, source);
|
||||
expect(onImport).toHaveBeenCalledExactlyOnceWith(ctx, { text: content });
|
||||
const onImportSource = vi.fn().mockResolvedValue(null);
|
||||
await runImporter({ name: "Source", onImportSource }, ctx, source);
|
||||
expect(onImportSource).toHaveBeenCalledExactlyOnceWith(ctx, {
|
||||
source: { type: "text", name: "api.zip", text: content },
|
||||
});
|
||||
});
|
||||
|
||||
test("ZIP signatures take precedence over names, including empty archives", async () => {
|
||||
const bytes = await readFile(new URL("./fixtures/import-files.zip", import.meta.url));
|
||||
const empty = Buffer.alloc(22);
|
||||
empty.writeUInt32LE(0x06054b50);
|
||||
for (const data of [bytes, empty]) {
|
||||
const source = input("collection.yaml", data);
|
||||
const session = await createImportFiles(source);
|
||||
try {
|
||||
expect(session.files.kind).toBe("zip");
|
||||
if (data === empty) expect(await session.files.readDir()).toEqual([]);
|
||||
else expect(await session.files.readTextFile("collection/café.yml")).toBe("name: café\n");
|
||||
} finally {
|
||||
session.close();
|
||||
}
|
||||
const onImport = vi.fn().mockResolvedValue(null);
|
||||
expect(await runImporter({ name: "Text", onImport }, ctx, source)).toBeNull();
|
||||
expect(onImport).not.toHaveBeenCalled();
|
||||
}
|
||||
});
|
||||
|
||||
test("a ZIP made by compressing a folder is rooted at that folder", async () => {
|
||||
const session = await createImportFiles(await zip("wrapped.zip"));
|
||||
try {
|
||||
expect((await session.files.readDir()).map((e) => e.name)).toEqual(["nested", "root.txt"]);
|
||||
expect(await session.files.readTextFile("nested/file.txt")).toBe("inner\n");
|
||||
await expect(session.files.readDir("Demo")).rejects.toThrow("not found");
|
||||
} finally {
|
||||
session.close();
|
||||
}
|
||||
});
|
||||
|
||||
test("malformed input with a ZIP signature still fails archive validation", async () => {
|
||||
const source = input("broken.zip", Buffer.from([0x50, 0x4b, 0x03, 0x04]));
|
||||
await expect(createImportFiles(source)).rejects.toThrow();
|
||||
});
|
||||
|
||||
test("directory view reads lazily and blocks links and escaping paths", async () => {
|
||||
const root = await mkdtemp(path.join(tmpdir(), "yaak-import-"));
|
||||
dirs.push(root);
|
||||
await mkdir(path.join(root, "nested"));
|
||||
await writeFile(path.join(root, "nested", "file"), "first");
|
||||
await symlink(tmpdir(), path.join(root, "escape"));
|
||||
const session = await createImportFiles({
|
||||
content: "",
|
||||
source: { type: "directory", path: root },
|
||||
});
|
||||
expect(await session.files.readDir()).toEqual([
|
||||
{ name: "nested", path: "nested", type: "directory" },
|
||||
]);
|
||||
await writeFile(path.join(root, "nested", "file"), "updated");
|
||||
expect(await session.files.readTextFile("nested/file")).toBe("updated");
|
||||
await expect(session.files.readDir("escape")).rejects.toThrow("symbolic");
|
||||
for (const p of [
|
||||
"../outside",
|
||||
"/absolute",
|
||||
"C:/outside",
|
||||
"nested/../../outside",
|
||||
"nested\\file",
|
||||
]) {
|
||||
await expect(session.files.readFile(p)).rejects.toThrow();
|
||||
}
|
||||
session.close();
|
||||
});
|
||||
|
||||
test.each(["import-traversal.zip", "import-duplicate.zip", "import-symlink.zip"])(
|
||||
"rejects unsafe ZIP %s",
|
||||
async (name) => {
|
||||
await expect(createImportFiles(await zip(name))).rejects.toThrow();
|
||||
},
|
||||
);
|
||||
|
||||
test("ZIP entries are decompressed only on read, with a read-size limit", async () => {
|
||||
const session = await createImportFiles(await zip("import-large.zip"));
|
||||
expect((await session.files.readDir())[0]?.name).toBe("large.txt");
|
||||
await expect(session.files.readFile("large.txt")).rejects.toThrow("32 MiB");
|
||||
session.close();
|
||||
});
|
||||
|
||||
test("the deprecated text hook preserves its arguments and skips other input", async () => {
|
||||
const onImport = vi.fn().mockResolvedValue(null);
|
||||
const importer = { name: "Legacy", onImport };
|
||||
await runImporter(importer, ctx, { content: "original" });
|
||||
expect(onImport).toHaveBeenCalledExactlyOnceWith(ctx, { text: "original" });
|
||||
onImport.mockClear();
|
||||
await runImporter(importer, ctx, input("empty.txt", new Uint8Array()));
|
||||
expect(onImport).toHaveBeenCalledExactlyOnceWith(ctx, { text: "" });
|
||||
onImport.mockClear();
|
||||
for (const source of [
|
||||
await zip(),
|
||||
input("binary", Uint8Array.from([255])),
|
||||
{
|
||||
content: "",
|
||||
source: { type: "directory", path: "/does-not-exist" },
|
||||
} satisfies ImportRequest,
|
||||
]) {
|
||||
expect(await runImporter(importer, ctx, source)).toBeNull();
|
||||
}
|
||||
expect(onImport).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("the source hook gets text for documents and a tree for directories and ZIPs", async () => {
|
||||
const onImport = vi.fn().mockResolvedValue(null);
|
||||
const onImportSource = vi.fn().mockResolvedValue(null);
|
||||
const importer = { name: "Both", onImport, onImportSource };
|
||||
const root = await mkdtemp(path.join(tmpdir(), "yaak-import-hooks-"));
|
||||
dirs.push(root);
|
||||
const cases: [ImportRequest, ImportSource["type"] | null, string?][] = [
|
||||
[{ content: "pasted" }, "text", "pasted"],
|
||||
[input("text.txt", Buffer.from("text")), "text", "text"],
|
||||
[input("empty.txt", new Uint8Array()), "text", ""],
|
||||
[input("binary", Uint8Array.from([255])), null],
|
||||
[await zip(), "directory"],
|
||||
[{ content: "", source: { type: "directory", path: root } }, "directory"],
|
||||
];
|
||||
for (const [request, type, text] of cases) {
|
||||
onImportSource.mockClear();
|
||||
expect(await runImporter(importer, ctx, request)).toBeNull();
|
||||
if (type == null) {
|
||||
expect(onImportSource).not.toHaveBeenCalled();
|
||||
continue;
|
||||
}
|
||||
expect(onImportSource).toHaveBeenCalledOnce();
|
||||
const { source } = onImportSource.mock.calls[0]![1] as { source: ImportSource };
|
||||
expect(source.type).toBe(type);
|
||||
if (source.type === "text") expect(source.text).toBe(text);
|
||||
else await expect(source.files.readDir()).rejects.toThrow("closed");
|
||||
}
|
||||
onImportSource.mockResolvedValue(undefined);
|
||||
await runImporter(importer, ctx, { content: "pasted" });
|
||||
expect(onImport).not.toHaveBeenCalled();
|
||||
expect(console.warn).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("closes file access even if the importer throws", async () => {
|
||||
let saved: import("@yaakapp/api").ImportFiles | undefined;
|
||||
await expect(
|
||||
runImporter(
|
||||
{
|
||||
name: "Fails",
|
||||
onImportSource(_ctx, { source }) {
|
||||
if (source.type === "directory") saved = source.files;
|
||||
throw new Error("failure");
|
||||
},
|
||||
},
|
||||
ctx,
|
||||
await zip(),
|
||||
),
|
||||
).rejects.toThrow("failure");
|
||||
await expect(saved!.readDir()).rejects.toThrow("closed");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,326 @@
|
||||
import { constants } from "node:fs";
|
||||
import { lstat, open, opendir, realpath } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { Context, ImportFileEntry, ImportFiles, ImportRequest } from "@yaakapp/api";
|
||||
import type { ImporterPlugin, ImportSource } from "@yaakapp/api/lib/plugins/ImporterPlugin";
|
||||
import { type Entry, fromBuffer, type ZipFile } from "yauzl";
|
||||
|
||||
const MAX_FILE_BYTES = 32 * 1024 * 1024;
|
||||
const MAX_SOURCE_BYTES = 64 * 1024 * 1024;
|
||||
const MAX_ENTRIES = 10_000;
|
||||
const MAX_EXPANDED_BYTES = 256 * 1024 * 1024;
|
||||
|
||||
function relativePath(value: string): string {
|
||||
if (
|
||||
value.includes("\\") ||
|
||||
value.includes("\0") ||
|
||||
value.startsWith("/") ||
|
||||
/^[a-z]:/i.test(value)
|
||||
) {
|
||||
throw new Error(`Invalid import path: ${value}`);
|
||||
}
|
||||
const parts = value.split("/");
|
||||
if (parts.includes("..")) throw new Error("Import paths cannot leave the selected source");
|
||||
return parts.filter((p) => p !== "" && p !== ".").join("/");
|
||||
}
|
||||
|
||||
function isZip(bytes: Buffer): boolean {
|
||||
// Names are hints only: a text specification can legitimately be named *.zip.
|
||||
return (
|
||||
bytes[0] === 0x50 &&
|
||||
bytes[1] === 0x4b &&
|
||||
((bytes[2] === 3 && bytes[3] === 4) ||
|
||||
(bytes[2] === 5 && bytes[3] === 6) ||
|
||||
(bytes[2] === 7 && bytes[3] === 8))
|
||||
);
|
||||
}
|
||||
|
||||
export interface ImportFileSession {
|
||||
files: ImportFiles;
|
||||
kind: "file" | ImportFiles["kind"];
|
||||
name: string;
|
||||
text: string;
|
||||
/** Whether text-only plugins can handle this input, including an empty text file. */
|
||||
isText: boolean;
|
||||
close(): void;
|
||||
}
|
||||
|
||||
/** Construct one scoped view for one invocation. No ZIP entries are extracted to disk. */
|
||||
export async function createImportFiles(input: ImportRequest): Promise<ImportFileSession> {
|
||||
let closed = false;
|
||||
let zip: ZipFile | undefined;
|
||||
let zipError: Error | undefined;
|
||||
const assertOpen = () => {
|
||||
if (closed) throw new Error("Import source is closed");
|
||||
if (zipError) throw zipError;
|
||||
};
|
||||
let kind: ImportFileSession["kind"] = "file";
|
||||
let name = "input";
|
||||
let text = input.content;
|
||||
let isText = true;
|
||||
let readDir: ImportFiles["readDir"];
|
||||
let readFile: ImportFiles["readFile"];
|
||||
|
||||
if (input.source?.type === "directory") {
|
||||
kind = "directory";
|
||||
isText = false;
|
||||
text = "";
|
||||
const root = await realpath(input.source.path);
|
||||
name = path.basename(root);
|
||||
const resolve = async (value: string) => {
|
||||
assertOpen();
|
||||
const rel = relativePath(value);
|
||||
let candidate = root;
|
||||
// Reject links visible during validation, including intermediate components.
|
||||
// These path checks are not atomic with open/opendir: concurrent filesystem
|
||||
// changes can redirect a later operation. This is not a confinement boundary.
|
||||
for (const component of rel.split("/").filter(Boolean)) {
|
||||
candidate = path.join(candidate, component);
|
||||
if ((await lstat(candidate)).isSymbolicLink())
|
||||
throw new Error("Import sources do not follow symbolic links");
|
||||
}
|
||||
const canonical = await realpath(candidate);
|
||||
const within = path.relative(root, canonical);
|
||||
if (within === ".." || within.startsWith(`..${path.sep}`) || path.isAbsolute(within))
|
||||
throw new Error("Import path is outside the selected source");
|
||||
return canonical;
|
||||
};
|
||||
readDir = async (value = "") => {
|
||||
const rel = relativePath(value);
|
||||
const entries: ImportFileEntry[] = [];
|
||||
for await (const entry of await opendir(await resolve(rel))) {
|
||||
if (!entry.isFile() && !entry.isDirectory()) continue;
|
||||
entries.push({
|
||||
name: entry.name,
|
||||
path: rel ? `${rel}/${entry.name}` : entry.name,
|
||||
type: entry.isDirectory() ? "directory" : "file",
|
||||
});
|
||||
if (entries.length > MAX_ENTRIES)
|
||||
throw new Error("Import directory contains too many entries");
|
||||
}
|
||||
assertOpen();
|
||||
return entries.sort((a, b) => a.path.localeCompare(b.path));
|
||||
};
|
||||
readFile = async (value) => {
|
||||
const handle = await open(
|
||||
await resolve(value),
|
||||
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
|
||||
);
|
||||
try {
|
||||
const stat = await handle.stat();
|
||||
if (!stat.isFile()) throw new Error("Import path is not a regular file");
|
||||
if (stat.size > MAX_FILE_BYTES)
|
||||
throw new Error("Import file exceeds the 32 MiB read limit");
|
||||
const chunks: Buffer[] = [];
|
||||
let total = 0;
|
||||
for await (const chunk of handle.createReadStream({ autoClose: false })) {
|
||||
const bytes = Buffer.from(chunk as Uint8Array);
|
||||
total += bytes.length;
|
||||
if (total > MAX_FILE_BYTES) throw new Error("Import file exceeds the 32 MiB read limit");
|
||||
chunks.push(bytes);
|
||||
}
|
||||
assertOpen();
|
||||
return Buffer.concat(chunks);
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
};
|
||||
} else {
|
||||
const source = input.source;
|
||||
if (source && source.base64.length > Math.ceil(MAX_SOURCE_BYTES / 3) * 4)
|
||||
throw new Error("Import source exceeds the 64 MiB limit");
|
||||
const bytes = source ? Buffer.from(source.base64, "base64") : Buffer.from(input.content);
|
||||
name = source?.name || "input";
|
||||
if (bytes.length > MAX_SOURCE_BYTES) throw new Error("Import source exceeds the 64 MiB limit");
|
||||
const entries = new Map<string, { type: "file" | "directory"; entry?: Entry; data?: Buffer }>([
|
||||
["", { type: "directory" }],
|
||||
]);
|
||||
if (source && isZip(bytes)) {
|
||||
kind = "zip";
|
||||
text = "";
|
||||
isText = false;
|
||||
zip = await new Promise<ZipFile>((resolve, reject) => {
|
||||
fromBuffer(
|
||||
bytes,
|
||||
{ lazyEntries: true, autoClose: false, strictFileNames: false, validateEntrySizes: true },
|
||||
(err, result) => (err ? reject(err) : resolve(result)),
|
||||
);
|
||||
});
|
||||
zip.on("error", (err: Error) => {
|
||||
zipError = err;
|
||||
});
|
||||
const archive = zip;
|
||||
try {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
let count = 0;
|
||||
let expandedBytes = 0;
|
||||
archive.once("error", reject);
|
||||
archive.once("end", () => {
|
||||
archive.removeListener("error", reject);
|
||||
resolve();
|
||||
});
|
||||
archive.on("entry", (entry: Entry) => {
|
||||
try {
|
||||
if (++count > MAX_ENTRIES) throw new Error("ZIP contains too many entries");
|
||||
expandedBytes += entry.uncompressedSize;
|
||||
if (expandedBytes > MAX_EXPANDED_BYTES)
|
||||
throw new Error("ZIP expanded size exceeds the 256 MiB limit");
|
||||
if (entry.isEncrypted()) throw new Error("Encrypted ZIP entries are not supported");
|
||||
const mode = (entry.externalFileAttributes >>> 16) & 0xf000;
|
||||
if (mode === 0xa000) throw new Error("ZIP symbolic links are not supported");
|
||||
const entryPath = relativePath(entry.fileName);
|
||||
if (!entryPath) throw new Error("ZIP contains an empty entry path");
|
||||
const type = entry.fileName.endsWith("/") ? "directory" : "file";
|
||||
const previous = entries.get(entryPath);
|
||||
if (previous && (type !== "directory" || previous.type !== "directory"))
|
||||
throw new Error(`Duplicate ZIP path: ${entryPath}`);
|
||||
entries.set(entryPath, { type, entry });
|
||||
let parent = path.posix.dirname(entryPath);
|
||||
while (parent !== ".") {
|
||||
if (entries.get(parent)?.type === "file")
|
||||
throw new Error(`ZIP file/directory conflict: ${parent}`);
|
||||
if (!entries.has(parent)) entries.set(parent, { type: "directory" });
|
||||
parent = path.posix.dirname(parent);
|
||||
}
|
||||
if (entries.size > MAX_ENTRIES) throw new Error("ZIP contains too many paths");
|
||||
archive.readEntry();
|
||||
} catch (err) {
|
||||
reject(err);
|
||||
}
|
||||
});
|
||||
archive.readEntry();
|
||||
});
|
||||
} catch (err) {
|
||||
archive.close();
|
||||
throw err;
|
||||
}
|
||||
unwrapSingleRoot(entries);
|
||||
} else {
|
||||
const entryPath = relativePath(name);
|
||||
if (!entryPath || entryPath.includes("/"))
|
||||
throw new Error("Single import files must have a basename");
|
||||
entries.set(entryPath, { type: "file", data: bytes });
|
||||
try {
|
||||
text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
||||
} catch {
|
||||
text = "";
|
||||
isText = false;
|
||||
}
|
||||
}
|
||||
readDir = async (value = "") => {
|
||||
assertOpen();
|
||||
const rel = relativePath(value);
|
||||
if (entries.get(rel)?.type !== "directory")
|
||||
throw new Error(`Import directory not found: ${rel}`);
|
||||
return [...entries]
|
||||
.filter(
|
||||
([p]) => p !== "" && (path.posix.dirname(p) === "." ? "" : path.posix.dirname(p)) === rel,
|
||||
)
|
||||
.map(([p, entry]) => ({ name: path.posix.basename(p), path: p, type: entry.type }))
|
||||
.sort((a, b) => a.path.localeCompare(b.path));
|
||||
};
|
||||
readFile = async (value) => {
|
||||
assertOpen();
|
||||
const found = entries.get(relativePath(value));
|
||||
if (!found || found.type !== "file") throw new Error(`Import file not found: ${value}`);
|
||||
if (found.data) {
|
||||
if (found.data.length > MAX_FILE_BYTES)
|
||||
throw new Error("Import file exceeds the 32 MiB read limit");
|
||||
return Buffer.from(found.data);
|
||||
}
|
||||
const entry = found.entry!;
|
||||
if (entry.uncompressedSize > MAX_FILE_BYTES)
|
||||
throw new Error("Import file exceeds the 32 MiB read limit");
|
||||
const stream = await new Promise<import("node:stream").Readable>((resolve, reject) => {
|
||||
zip!.openReadStream(entry, (err, result) => (err ? reject(err) : resolve(result)));
|
||||
});
|
||||
const chunks: Buffer[] = [];
|
||||
let total = 0;
|
||||
for await (const chunk of stream) {
|
||||
const bytes = Buffer.from(chunk as Uint8Array);
|
||||
total += bytes.length;
|
||||
if (total > MAX_FILE_BYTES) {
|
||||
stream.destroy();
|
||||
throw new Error("Import file exceeds the 32 MiB read limit");
|
||||
}
|
||||
chunks.push(bytes);
|
||||
}
|
||||
assertOpen();
|
||||
return Buffer.concat(chunks);
|
||||
};
|
||||
}
|
||||
return {
|
||||
text,
|
||||
isText,
|
||||
kind,
|
||||
name,
|
||||
files: {
|
||||
// A lone file is only ever handed to plugins as text, never as a tree
|
||||
kind: kind === "file" ? "directory" : kind,
|
||||
name,
|
||||
readDir,
|
||||
readFile,
|
||||
async readTextFile(value) {
|
||||
return new TextDecoder("utf-8", { fatal: true }).decode(await readFile(value));
|
||||
},
|
||||
},
|
||||
close() {
|
||||
closed = true;
|
||||
zip?.close();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Compressing a folder puts everything under that folder inside the archive. Present the
|
||||
* folder's contents as the root so plugins see the same tree they would for the directory.
|
||||
*/
|
||||
function unwrapSingleRoot<T extends { type: "file" | "directory" }>(entries: Map<string, T>) {
|
||||
const tops = new Set<string>();
|
||||
for (const entryPath of entries.keys()) {
|
||||
if (entryPath === "") continue;
|
||||
const top = entryPath.split("/")[0]!;
|
||||
if (top === "__MACOSX" || top.startsWith(".")) continue;
|
||||
tops.add(top);
|
||||
}
|
||||
if (tops.size !== 1) return;
|
||||
const [root] = tops;
|
||||
if (entries.get(root!)?.type !== "directory") return;
|
||||
const prefix = `${root}/`;
|
||||
const snapshot = Array.from(entries);
|
||||
for (const [entryPath, value] of snapshot) {
|
||||
if (entryPath === "") continue;
|
||||
entries.delete(entryPath);
|
||||
if (entryPath === root) continue;
|
||||
if (entryPath.startsWith(prefix)) entries.set(entryPath.slice(prefix.length), value);
|
||||
}
|
||||
}
|
||||
|
||||
/** Keep legacy text plugins out of directories, archives, and arbitrary binary inputs. */
|
||||
export async function runImporter(importer: ImporterPlugin, ctx: Context, input: ImportRequest) {
|
||||
if (!importer.onImportSource && input.source?.type === "directory") return null;
|
||||
// Avoid opening archives once for every legacy importer.
|
||||
if (!importer.onImportSource && input.source?.type === "file") {
|
||||
const bytes = Buffer.from(input.source.base64, "base64");
|
||||
if (isZip(bytes)) return null;
|
||||
}
|
||||
const session = await createImportFiles(input);
|
||||
try {
|
||||
if (importer.onImportSource) {
|
||||
const source: ImportSource | null =
|
||||
session.kind === "file"
|
||||
? session.isText
|
||||
? { type: "text", name: session.name, text: session.text }
|
||||
: null
|
||||
: { type: "directory", files: session.files };
|
||||
if (source == null) return null;
|
||||
return await importer.onImportSource(ctx, { source });
|
||||
}
|
||||
if (session.isText && importer.onImport) {
|
||||
return await importer.onImport(ctx, { text: session.text });
|
||||
}
|
||||
return null;
|
||||
} finally {
|
||||
session.close();
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user