mirror of
https://github.com/mountain-loop/yaak.git
synced 2026-11-13 12:13:42 +01:00
Add Bruno imports and a file-tree importer API (#706)
This commit is contained in:
1 parent
46464cfacf
commit
aa10c44a45
109 files changed
+4684
-879
No files matched your search
@@ -288,10 +288,10 @@ const DECLINED: Partial<Record<AppCmd, [reason: string, capability: CapabilityNa
|
||||
"Importing from a file needs a filesystem, which a browser tab has no",
|
||||
"localFiles",
|
||||
],
|
||||
cmd_import_url: ["Importing from a URL needs the Yaak server, which isn't available yet", null],
|
||||
cmd_commit_import: ["Importing needs a plugin, which this host doesn't run", null],
|
||||
cmd_list_import_sources: ["Importing isn't available in the browser yet", null],
|
||||
cmd_import_sources_for_origin: ["Importing isn't available in the browser yet", null],
|
||||
cmd_detect_import_source: ["Importing isn't available in the browser yet", null],
|
||||
cmd_save_response: ["Saving a response to disk isn't available in the browser", "localFiles"],
|
||||
cmd_save_base64_to_binary: ["Saving to disk isn't available in the browser", "localFiles"],
|
||||
cmd_format_graphql: ["Formatting GraphQL needs a plugin, which this host doesn't run", null],
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
# Importers
|
||||
|
||||
Implement **`onImportSource(ctx, { source })`**. The source is one of two things:
|
||||
|
||||
- `{ type: "text", name, text }`: a single UTF-8 document, or pasted text (named `input`).
|
||||
- `{ type: "directory", files }`: a directory or a ZIP archive, read as one file tree.
|
||||
|
||||
Return `null` or `undefined` when the source isn't your format. Only one importer's result is used per source.
|
||||
|
||||
```ts
|
||||
import type { PluginDefinition } from "@yaakapp/api";
|
||||
|
||||
export const plugin: PluginDefinition = {
|
||||
importer: {
|
||||
name: "Example",
|
||||
async onImportSource(ctx, { source }) {
|
||||
if (source.type === "text") return convertDocument(source.text);
|
||||
for (const entry of await source.files.readDir()) {
|
||||
if (entry.name === "example.json") {
|
||||
return convertDocument(await source.files.readTextFile(entry.path));
|
||||
}
|
||||
}
|
||||
return null;
|
||||
},
|
||||
},
|
||||
};
|
||||
```
|
||||
|
||||
The older `onImport(ctx, { text })` hook still works but is deprecated. It only receives single text documents and is ignored when `onImportSource` is defined.
|
||||
|
||||
## The file tree
|
||||
|
||||
- `files.kind` is `"directory"` or `"zip"`; `files.name` is the selected basename.
|
||||
- `readDir(path = "")` lists immediate children. `readFile(path)` returns bytes. `readTextFile(path)` decodes UTF-8 strictly.
|
||||
- Paths use `/` and are relative to the root. Absolute paths, `..`, and backslashes are rejected. Symbolic links are skipped in listings and rejected on read. Nothing is extracted to disk.
|
||||
- ZIPs are detected by signature, not extension. A ZIP made by compressing a folder is rooted at that folder, so plugins see the same tree as for the directory. Directory reads are not a snapshot: a symlink swapped in under a validated path can redirect a read outside the root, so treat directory imports as trusted input.
|
||||
- Limits: 64 MiB per source file or download, 32 MiB per file read, 10,000 entries, 256 MiB declared ZIP expansion. Encrypted ZIPs are unsupported.
|
||||
- The tree is read-only and only valid until the hook settles.
|
||||
|
||||
Plugins run with the normal Node.js permissions of the plugin runtime; the tree scopes convenience, not security.
|
||||
+10
-1
@@ -470,7 +470,16 @@ export type HttpRequestAction = { label: string, icon?: Icon, };
|
||||
|
||||
export type Icon = "alert_triangle" | "check" | "check_circle" | "chevron_down" | "copy" | "info" | "pin" | "search" | "trash" | "_unknown";
|
||||
|
||||
export type ImportRequest = { content: string, };
|
||||
/**
|
||||
* Transport descriptor; plugins see a scoped ImportFiles API, not this representation.
|
||||
*/
|
||||
export type ImportFileSource = { "type": "file", name: string, base64: string, } | { "type": "directory", path: string, };
|
||||
|
||||
export type ImportRequest = {
|
||||
/**
|
||||
* Kept for text-only importers and older runtimes.
|
||||
*/
|
||||
content: string, source?: ImportFileSource, };
|
||||
|
||||
export type ImportResources = { workspaces: Array<Workspace>, environments: Array<Environment>, folders: Array<Folder>, httpRequests: Array<HttpRequest>, grpcRequests: Array<GrpcRequest>, websocketRequests: Array<WebsocketRequest>, };
|
||||
|
||||
|
||||
@@ -6,4 +6,10 @@ export * from "./bindings/gen_events";
|
||||
|
||||
// Some extras for utility
|
||||
|
||||
export type { PartialImportResources } from "./plugins/ImporterPlugin";
|
||||
export type {
|
||||
PartialImportResources,
|
||||
ImportFiles,
|
||||
ImportFileEntry,
|
||||
ImportSource,
|
||||
ImportSourceHandler,
|
||||
} from "./plugins/ImporterPlugin";
|
||||
@@ -21,11 +21,45 @@ export type ImportPluginResponse =
|
||||
resources: PartialImportResources;
|
||||
});
|
||||
|
||||
/** A read-only view of a selected directory or ZIP archive. Paths are relative and use
|
||||
* forward slashes. Directory path checks are best-effort under concurrent filesystem
|
||||
* changes, not a security boundary. Valid only while onImportSource runs. */
|
||||
export interface ImportFiles {
|
||||
readonly name: string;
|
||||
readonly kind: "zip" | "directory";
|
||||
/** List immediate children. The root directory is "" (the default). */
|
||||
readDir(path?: string): Promise<ImportFileEntry[]>;
|
||||
readFile(path: string): Promise<Uint8Array>;
|
||||
/** Decode UTF-8 strictly; binary or invalid UTF-8 content throws. */
|
||||
readTextFile(path: string): Promise<string>;
|
||||
}
|
||||
|
||||
export interface ImportFileEntry {
|
||||
name: string;
|
||||
path: string;
|
||||
type: "file" | "directory";
|
||||
}
|
||||
|
||||
/** What the user chose to import: one text document, or a directory or ZIP read as a tree. */
|
||||
export type ImportSource =
|
||||
| { type: "text"; name: string; text: string }
|
||||
| { type: "directory"; files: ImportFiles };
|
||||
|
||||
type ImportTextHandler = (
|
||||
ctx: Context,
|
||||
args: { text: string },
|
||||
) => MaybePromise<ImportPluginResponse | undefined>;
|
||||
|
||||
export type ImportSourceHandler = (
|
||||
ctx: Context,
|
||||
args: { source: ImportSource },
|
||||
) => MaybePromise<ImportPluginResponse | undefined>;
|
||||
|
||||
export type ImporterPlugin = {
|
||||
name: string;
|
||||
description?: string;
|
||||
onImport(
|
||||
ctx: Context,
|
||||
args: { text: string },
|
||||
): MaybePromise<ImportPluginResponse | null | undefined>;
|
||||
/** Return null when the source isn't this importer's format. Takes precedence over onImport. */
|
||||
onImportSource?: ImportSourceHandler;
|
||||
/** @deprecated Use onImportSource. Only receives single text documents. */
|
||||
onImport?: ImportTextHandler;
|
||||
};
|
||||
@@ -6,11 +6,13 @@
|
||||
"build:main": "esbuild src/index.ts --bundle --platform=node --outfile=../../crates-tauri/yaak-app-client/vendored/plugin-runtime/index.cjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"ws": "^8.21.0"
|
||||
"ws": "^8.21.0",
|
||||
"yauzl": "^3.4.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^24.0.13",
|
||||
"@types/ws": "^8.5.13",
|
||||
"@types/yauzl": "^3.4.0",
|
||||
"esbuild": "^0.28.0"
|
||||
}
|
||||
}
|
||||
@@ -51,6 +51,7 @@ import type {
|
||||
} from "@yaakapp-internal/plugins";
|
||||
import { applyDynamicFormInput } from "./common";
|
||||
import { EventChannel } from "./EventChannel";
|
||||
import { runImporter } from "./importFiles";
|
||||
import { migrateTemplateFunctionSelectOptions } from "./migrations";
|
||||
import { createResponseBody, decodeBase64Chunk } from "./responseBody";
|
||||
|
||||
@@ -157,19 +158,14 @@ export class PluginInstance {
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
payload.type === "import_request" &&
|
||||
typeof this.#mod?.importer?.onImport === "function"
|
||||
) {
|
||||
const reply = await this.#mod.importer.onImport(ctx, {
|
||||
text: payload.content,
|
||||
});
|
||||
if (payload.type === "import_request" && this.#mod?.importer != null) {
|
||||
const reply = await runImporter(this.#mod.importer, ctx, payload);
|
||||
if (reply != null) {
|
||||
const replyPayload: InternalEventPayload = {
|
||||
type: "import_response",
|
||||
importer: this.#mod.importer.name,
|
||||
resources: reply.resources as ImportResources,
|
||||
sourceKeys: reply.sourceKeys ?? null,
|
||||
sourceKeys: reply.sourceKeys,
|
||||
};
|
||||
this.#sendPayload(context, replyPayload, replyId);
|
||||
return;
|
||||
@@ -649,10 +645,12 @@ export class PluginInstance {
|
||||
complete: info.complete,
|
||||
},
|
||||
async (offset, length) => {
|
||||
const chunk = await this.#sendForReply<ReadHttpResponseBodyChunkResponse>(
|
||||
context,
|
||||
{ type: "read_http_response_body_chunk_request", responseId, offset, length },
|
||||
);
|
||||
const chunk = await this.#sendForReply<ReadHttpResponseBodyChunkResponse>(context, {
|
||||
type: "read_http_response_body_chunk_request",
|
||||
responseId,
|
||||
offset,
|
||||
length,
|
||||
});
|
||||
return decodeBase64Chunk(chunk.data);
|
||||
},
|
||||
{ refresh: bodyInfo },
|
||||
@@ -853,7 +851,10 @@ export class PluginInstance {
|
||||
// carries the only copy of its body. A saved one is read back from
|
||||
// the host like any other. Callers get the same thing either way.
|
||||
if (body == null) {
|
||||
return { httpResponse: forPlugin(httpResponse), body: await storedBody(httpResponse.id) };
|
||||
return {
|
||||
httpResponse: forPlugin(httpResponse),
|
||||
body: await storedBody(httpResponse.id),
|
||||
};
|
||||
}
|
||||
|
||||
const bytes = decodeBase64Chunk(body);
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,224 @@
|
||||
import { mkdtemp, mkdir, readFile, rm, symlink, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import type { Context, ImportRequest, ImportSource } from "@yaakapp/api";
|
||||
import { afterEach, beforeEach, describe, expect, test, vi } from "vite-plus/test";
|
||||
import { createImportFiles, runImporter } from "./importFiles";
|
||||
|
||||
const dirs: string[] = [];
|
||||
beforeEach(() => {
|
||||
vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
});
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks();
|
||||
await Promise.all(dirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true })));
|
||||
});
|
||||
const input = (name: string, data: Uint8Array): ImportRequest => ({
|
||||
content: "",
|
||||
source: { type: "file", name, base64: Buffer.from(data).toString("base64") },
|
||||
});
|
||||
async function zip(name = "import-files.zip") {
|
||||
return input(name, await readFile(new URL(`./fixtures/${name}`, import.meta.url)));
|
||||
}
|
||||
const ctx = {} as Context;
|
||||
|
||||
describe("import file tree", () => {
|
||||
test("exposes legacy text and single binary files through the same read API", async () => {
|
||||
const session = await createImportFiles({ content: "café" });
|
||||
expect(session.text).toBe("café");
|
||||
expect(await session.files.readDir()).toEqual([{ name: "input", path: "input", type: "file" }]);
|
||||
expect(await session.files.readTextFile("input")).toBe("café");
|
||||
session.close();
|
||||
await expect(session.files.readFile("input")).rejects.toThrow("closed");
|
||||
const binary = await createImportFiles(input("body.bin", Uint8Array.from([255, 0, 128])));
|
||||
expect(binary.isText).toBe(false);
|
||||
expect([...(await binary.files.readFile("body.bin"))]).toEqual([255, 0, 128]);
|
||||
await expect(binary.files.readTextFile("body.bin")).rejects.toThrow();
|
||||
binary.close();
|
||||
});
|
||||
|
||||
test("lists immediate ZIP children, including implicit/empty directories, and reads binary/UTF-8", async () => {
|
||||
const session = await createImportFiles(await zip());
|
||||
expect(session.files.kind).toBe("zip");
|
||||
expect(session.text).toBe("");
|
||||
expect(await session.files.readDir()).toEqual([
|
||||
{ name: "binary.bin", path: "binary.bin", type: "file" },
|
||||
{ name: "collection", path: "collection", type: "directory" },
|
||||
]);
|
||||
expect((await session.files.readDir("collection")).map((e) => e.name)).toEqual([
|
||||
"café.yml",
|
||||
"empty",
|
||||
"nested",
|
||||
]);
|
||||
expect(await session.files.readDir("collection/empty")).toEqual([]);
|
||||
expect(await session.files.readTextFile("collection/café.yml")).toBe("name: café\n");
|
||||
expect([...(await session.files.readFile("binary.bin"))]).toEqual([0, 255, 128]);
|
||||
await expect(session.files.readFile("collection")).rejects.toThrow("not found");
|
||||
await expect(session.files.readDir("missing")).rejects.toThrow("not found");
|
||||
session.close();
|
||||
});
|
||||
|
||||
test("a misleading ZIP suffix still dispatches as text", async () => {
|
||||
const content = '{"openapi":"3.0.0","info":{"title":"Example","version":"1"},"paths":{}}';
|
||||
const source = input("api.zip", Buffer.from(content));
|
||||
const onImport = vi.fn().mockResolvedValue(null);
|
||||
await runImporter({ name: "Text", onImport }, ctx, source);
|
||||
expect(onImport).toHaveBeenCalledExactlyOnceWith(ctx, { text: content });
|
||||
const onImportSource = vi.fn().mockResolvedValue(null);
|
||||
await runImporter({ name: "Source", onImportSource }, ctx, source);
|
||||
expect(onImportSource).toHaveBeenCalledExactlyOnceWith(ctx, {
|
||||
source: { type: "text", name: "api.zip", text: content },
|
||||
});
|
||||
});
|
||||
|
||||
test("ZIP signatures take precedence over names, including empty archives", async () => {
|
||||
const bytes = await readFile(new URL("./fixtures/import-files.zip", import.meta.url));
|
||||
const empty = Buffer.alloc(22);
|
||||
empty.writeUInt32LE(0x06054b50);
|
||||
for (const data of [bytes, empty]) {
|
||||
const source = input("collection.yaml", data);
|
||||
const session = await createImportFiles(source);
|
||||
try {
|
||||
expect(session.files.kind).toBe("zip");
|
||||
if (data === empty) expect(await session.files.readDir()).toEqual([]);
|
||||
else expect(await session.files.readTextFile("collection/café.yml")).toBe("name: café\n");
|
||||
} finally {
|
||||
session.close();
|
||||
}
|
||||
const onImport = vi.fn().mockResolvedValue(null);
|
||||
expect(await runImporter({ name: "Text", onImport }, ctx, source)).toBeNull();
|
||||
expect(onImport).not.toHaveBeenCalled();
|
||||
}
|
||||
});
|
||||
|
||||
test("a ZIP made by compressing a folder is rooted at that folder", async () => {
|
||||
const session = await createImportFiles(await zip("wrapped.zip"));
|
||||
try {
|
||||
expect((await session.files.readDir()).map((e) => e.name)).toEqual(["nested", "root.txt"]);
|
||||
expect(await session.files.readTextFile("nested/file.txt")).toBe("inner\n");
|
||||
await expect(session.files.readDir("Demo")).rejects.toThrow("not found");
|
||||
} finally {
|
||||
session.close();
|
||||
}
|
||||
});
|
||||
|
||||
test("malformed input with a ZIP signature still fails archive validation", async () => {
|
||||
const source = input("broken.zip", Buffer.from([0x50, 0x4b, 0x03, 0x04]));
|
||||
await expect(createImportFiles(source)).rejects.toThrow();
|
||||
});
|
||||
|
||||
test("directory view reads lazily and blocks links and escaping paths", async () => {
|
||||
const root = await mkdtemp(path.join(tmpdir(), "yaak-import-"));
|
||||
dirs.push(root);
|
||||
await mkdir(path.join(root, "nested"));
|
||||
await writeFile(path.join(root, "nested", "file"), "first");
|
||||
await symlink(tmpdir(), path.join(root, "escape"));
|
||||
const session = await createImportFiles({
|
||||
content: "",
|
||||
source: { type: "directory", path: root },
|
||||
});
|
||||
expect(await session.files.readDir()).toEqual([
|
||||
{ name: "nested", path: "nested", type: "directory" },
|
||||
]);
|
||||
await writeFile(path.join(root, "nested", "file"), "updated");
|
||||
expect(await session.files.readTextFile("nested/file")).toBe("updated");
|
||||
await expect(session.files.readDir("escape")).rejects.toThrow("symbolic");
|
||||
for (const p of [
|
||||
"../outside",
|
||||
"/absolute",
|
||||
"C:/outside",
|
||||
"nested/../../outside",
|
||||
"nested\\file",
|
||||
]) {
|
||||
await expect(session.files.readFile(p)).rejects.toThrow();
|
||||
}
|
||||
session.close();
|
||||
});
|
||||
|
||||
test.each(["import-traversal.zip", "import-duplicate.zip", "import-symlink.zip"])(
|
||||
"rejects unsafe ZIP %s",
|
||||
async (name) => {
|
||||
await expect(createImportFiles(await zip(name))).rejects.toThrow();
|
||||
},
|
||||
);
|
||||
|
||||
test("ZIP entries are decompressed only on read, with a read-size limit", async () => {
|
||||
const session = await createImportFiles(await zip("import-large.zip"));
|
||||
expect((await session.files.readDir())[0]?.name).toBe("large.txt");
|
||||
await expect(session.files.readFile("large.txt")).rejects.toThrow("32 MiB");
|
||||
session.close();
|
||||
});
|
||||
|
||||
test("the deprecated text hook preserves its arguments and skips other input", async () => {
|
||||
const onImport = vi.fn().mockResolvedValue(null);
|
||||
const importer = { name: "Legacy", onImport };
|
||||
await runImporter(importer, ctx, { content: "original" });
|
||||
expect(onImport).toHaveBeenCalledExactlyOnceWith(ctx, { text: "original" });
|
||||
onImport.mockClear();
|
||||
await runImporter(importer, ctx, input("empty.txt", new Uint8Array()));
|
||||
expect(onImport).toHaveBeenCalledExactlyOnceWith(ctx, { text: "" });
|
||||
onImport.mockClear();
|
||||
for (const source of [
|
||||
await zip(),
|
||||
input("binary", Uint8Array.from([255])),
|
||||
{
|
||||
content: "",
|
||||
source: { type: "directory", path: "/does-not-exist" },
|
||||
} satisfies ImportRequest,
|
||||
]) {
|
||||
expect(await runImporter(importer, ctx, source)).toBeNull();
|
||||
}
|
||||
expect(onImport).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("the source hook gets text for documents and a tree for directories and ZIPs", async () => {
|
||||
const onImport = vi.fn().mockResolvedValue(null);
|
||||
const onImportSource = vi.fn().mockResolvedValue(null);
|
||||
const importer = { name: "Both", onImport, onImportSource };
|
||||
const root = await mkdtemp(path.join(tmpdir(), "yaak-import-hooks-"));
|
||||
dirs.push(root);
|
||||
const cases: [ImportRequest, ImportSource["type"] | null, string?][] = [
|
||||
[{ content: "pasted" }, "text", "pasted"],
|
||||
[input("text.txt", Buffer.from("text")), "text", "text"],
|
||||
[input("empty.txt", new Uint8Array()), "text", ""],
|
||||
[input("binary", Uint8Array.from([255])), null],
|
||||
[await zip(), "directory"],
|
||||
[{ content: "", source: { type: "directory", path: root } }, "directory"],
|
||||
];
|
||||
for (const [request, type, text] of cases) {
|
||||
onImportSource.mockClear();
|
||||
expect(await runImporter(importer, ctx, request)).toBeNull();
|
||||
if (type == null) {
|
||||
expect(onImportSource).not.toHaveBeenCalled();
|
||||
continue;
|
||||
}
|
||||
expect(onImportSource).toHaveBeenCalledOnce();
|
||||
const { source } = onImportSource.mock.calls[0]![1] as { source: ImportSource };
|
||||
expect(source.type).toBe(type);
|
||||
if (source.type === "text") expect(source.text).toBe(text);
|
||||
else await expect(source.files.readDir()).rejects.toThrow("closed");
|
||||
}
|
||||
onImportSource.mockResolvedValue(undefined);
|
||||
await runImporter(importer, ctx, { content: "pasted" });
|
||||
expect(onImport).not.toHaveBeenCalled();
|
||||
expect(console.warn).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("closes file access even if the importer throws", async () => {
|
||||
let saved: import("@yaakapp/api").ImportFiles | undefined;
|
||||
await expect(
|
||||
runImporter(
|
||||
{
|
||||
name: "Fails",
|
||||
onImportSource(_ctx, { source }) {
|
||||
if (source.type === "directory") saved = source.files;
|
||||
throw new Error("failure");
|
||||
},
|
||||
},
|
||||
ctx,
|
||||
await zip(),
|
||||
),
|
||||
).rejects.toThrow("failure");
|
||||
await expect(saved!.readDir()).rejects.toThrow("closed");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,326 @@
|
||||
import { constants } from "node:fs";
|
||||
import { lstat, open, opendir, realpath } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { Context, ImportFileEntry, ImportFiles, ImportRequest } from "@yaakapp/api";
|
||||
import type { ImporterPlugin, ImportSource } from "@yaakapp/api/lib/plugins/ImporterPlugin";
|
||||
import { type Entry, fromBuffer, type ZipFile } from "yauzl";
|
||||
|
||||
const MAX_FILE_BYTES = 32 * 1024 * 1024;
|
||||
const MAX_SOURCE_BYTES = 64 * 1024 * 1024;
|
||||
const MAX_ENTRIES = 10_000;
|
||||
const MAX_EXPANDED_BYTES = 256 * 1024 * 1024;
|
||||
|
||||
function relativePath(value: string): string {
|
||||
if (
|
||||
value.includes("\\") ||
|
||||
value.includes("\0") ||
|
||||
value.startsWith("/") ||
|
||||
/^[a-z]:/i.test(value)
|
||||
) {
|
||||
throw new Error(`Invalid import path: ${value}`);
|
||||
}
|
||||
const parts = value.split("/");
|
||||
if (parts.includes("..")) throw new Error("Import paths cannot leave the selected source");
|
||||
return parts.filter((p) => p !== "" && p !== ".").join("/");
|
||||
}
|
||||
|
||||
function isZip(bytes: Buffer): boolean {
|
||||
// Names are hints only: a text specification can legitimately be named *.zip.
|
||||
return (
|
||||
bytes[0] === 0x50 &&
|
||||
bytes[1] === 0x4b &&
|
||||
((bytes[2] === 3 && bytes[3] === 4) ||
|
||||
(bytes[2] === 5 && bytes[3] === 6) ||
|
||||
(bytes[2] === 7 && bytes[3] === 8))
|
||||
);
|
||||
}
|
||||
|
||||
export interface ImportFileSession {
|
||||
files: ImportFiles;
|
||||
kind: "file" | ImportFiles["kind"];
|
||||
name: string;
|
||||
text: string;
|
||||
/** Whether text-only plugins can handle this input, including an empty text file. */
|
||||
isText: boolean;
|
||||
close(): void;
|
||||
}
|
||||
|
||||
/** Construct one scoped view for one invocation. No ZIP entries are extracted to disk. */
|
||||
export async function createImportFiles(input: ImportRequest): Promise<ImportFileSession> {
|
||||
let closed = false;
|
||||
let zip: ZipFile | undefined;
|
||||
let zipError: Error | undefined;
|
||||
const assertOpen = () => {
|
||||
if (closed) throw new Error("Import source is closed");
|
||||
if (zipError) throw zipError;
|
||||
};
|
||||
let kind: ImportFileSession["kind"] = "file";
|
||||
let name = "input";
|
||||
let text = input.content;
|
||||
let isText = true;
|
||||
let readDir: ImportFiles["readDir"];
|
||||
let readFile: ImportFiles["readFile"];
|
||||
|
||||
if (input.source?.type === "directory") {
|
||||
kind = "directory";
|
||||
isText = false;
|
||||
text = "";
|
||||
const root = await realpath(input.source.path);
|
||||
name = path.basename(root);
|
||||
const resolve = async (value: string) => {
|
||||
assertOpen();
|
||||
const rel = relativePath(value);
|
||||
let candidate = root;
|
||||
// Reject links visible during validation, including intermediate components.
|
||||
// These path checks are not atomic with open/opendir: concurrent filesystem
|
||||
// changes can redirect a later operation. This is not a confinement boundary.
|
||||
for (const component of rel.split("/").filter(Boolean)) {
|
||||
candidate = path.join(candidate, component);
|
||||
if ((await lstat(candidate)).isSymbolicLink())
|
||||
throw new Error("Import sources do not follow symbolic links");
|
||||
}
|
||||
const canonical = await realpath(candidate);
|
||||
const within = path.relative(root, canonical);
|
||||
if (within === ".." || within.startsWith(`..${path.sep}`) || path.isAbsolute(within))
|
||||
throw new Error("Import path is outside the selected source");
|
||||
return canonical;
|
||||
};
|
||||
readDir = async (value = "") => {
|
||||
const rel = relativePath(value);
|
||||
const entries: ImportFileEntry[] = [];
|
||||
for await (const entry of await opendir(await resolve(rel))) {
|
||||
if (!entry.isFile() && !entry.isDirectory()) continue;
|
||||
entries.push({
|
||||
name: entry.name,
|
||||
path: rel ? `${rel}/${entry.name}` : entry.name,
|
||||
type: entry.isDirectory() ? "directory" : "file",
|
||||
});
|
||||
if (entries.length > MAX_ENTRIES)
|
||||
throw new Error("Import directory contains too many entries");
|
||||
}
|
||||
assertOpen();
|
||||
return entries.sort((a, b) => a.path.localeCompare(b.path));
|
||||
};
|
||||
readFile = async (value) => {
|
||||
const handle = await open(
|
||||
await resolve(value),
|
||||
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
|
||||
);
|
||||
try {
|
||||
const stat = await handle.stat();
|
||||
if (!stat.isFile()) throw new Error("Import path is not a regular file");
|
||||
if (stat.size > MAX_FILE_BYTES)
|
||||
throw new Error("Import file exceeds the 32 MiB read limit");
|
||||
const chunks: Buffer[] = [];
|
||||
let total = 0;
|
||||
for await (const chunk of handle.createReadStream({ autoClose: false })) {
|
||||
const bytes = Buffer.from(chunk as Uint8Array);
|
||||
total += bytes.length;
|
||||
if (total > MAX_FILE_BYTES) throw new Error("Import file exceeds the 32 MiB read limit");
|
||||
chunks.push(bytes);
|
||||
}
|
||||
assertOpen();
|
||||
return Buffer.concat(chunks);
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
};
|
||||
} else {
|
||||
const source = input.source;
|
||||
if (source && source.base64.length > Math.ceil(MAX_SOURCE_BYTES / 3) * 4)
|
||||
throw new Error("Import source exceeds the 64 MiB limit");
|
||||
const bytes = source ? Buffer.from(source.base64, "base64") : Buffer.from(input.content);
|
||||
name = source?.name || "input";
|
||||
if (bytes.length > MAX_SOURCE_BYTES) throw new Error("Import source exceeds the 64 MiB limit");
|
||||
const entries = new Map<string, { type: "file" | "directory"; entry?: Entry; data?: Buffer }>([
|
||||
["", { type: "directory" }],
|
||||
]);
|
||||
if (source && isZip(bytes)) {
|
||||
kind = "zip";
|
||||
text = "";
|
||||
isText = false;
|
||||
zip = await new Promise<ZipFile>((resolve, reject) => {
|
||||
fromBuffer(
|
||||
bytes,
|
||||
{ lazyEntries: true, autoClose: false, strictFileNames: false, validateEntrySizes: true },
|
||||
(err, result) => (err ? reject(err) : resolve(result)),
|
||||
);
|
||||
});
|
||||
zip.on("error", (err: Error) => {
|
||||
zipError = err;
|
||||
});
|
||||
const archive = zip;
|
||||
try {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
let count = 0;
|
||||
let expandedBytes = 0;
|
||||
archive.once("error", reject);
|
||||
archive.once("end", () => {
|
||||
archive.removeListener("error", reject);
|
||||
resolve();
|
||||
});
|
||||
archive.on("entry", (entry: Entry) => {
|
||||
try {
|
||||
if (++count > MAX_ENTRIES) throw new Error("ZIP contains too many entries");
|
||||
expandedBytes += entry.uncompressedSize;
|
||||
if (expandedBytes > MAX_EXPANDED_BYTES)
|
||||
throw new Error("ZIP expanded size exceeds the 256 MiB limit");
|
||||
if (entry.isEncrypted()) throw new Error("Encrypted ZIP entries are not supported");
|
||||
const mode = (entry.externalFileAttributes >>> 16) & 0xf000;
|
||||
if (mode === 0xa000) throw new Error("ZIP symbolic links are not supported");
|
||||
const entryPath = relativePath(entry.fileName);
|
||||
if (!entryPath) throw new Error("ZIP contains an empty entry path");
|
||||
const type = entry.fileName.endsWith("/") ? "directory" : "file";
|
||||
const previous = entries.get(entryPath);
|
||||
if (previous && (type !== "directory" || previous.type !== "directory"))
|
||||
throw new Error(`Duplicate ZIP path: ${entryPath}`);
|
||||
entries.set(entryPath, { type, entry });
|
||||
let parent = path.posix.dirname(entryPath);
|
||||
while (parent !== ".") {
|
||||
if (entries.get(parent)?.type === "file")
|
||||
throw new Error(`ZIP file/directory conflict: ${parent}`);
|
||||
if (!entries.has(parent)) entries.set(parent, { type: "directory" });
|
||||
parent = path.posix.dirname(parent);
|
||||
}
|
||||
if (entries.size > MAX_ENTRIES) throw new Error("ZIP contains too many paths");
|
||||
archive.readEntry();
|
||||
} catch (err) {
|
||||
reject(err);
|
||||
}
|
||||
});
|
||||
archive.readEntry();
|
||||
});
|
||||
} catch (err) {
|
||||
archive.close();
|
||||
throw err;
|
||||
}
|
||||
unwrapSingleRoot(entries);
|
||||
} else {
|
||||
const entryPath = relativePath(name);
|
||||
if (!entryPath || entryPath.includes("/"))
|
||||
throw new Error("Single import files must have a basename");
|
||||
entries.set(entryPath, { type: "file", data: bytes });
|
||||
try {
|
||||
text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
||||
} catch {
|
||||
text = "";
|
||||
isText = false;
|
||||
}
|
||||
}
|
||||
readDir = async (value = "") => {
|
||||
assertOpen();
|
||||
const rel = relativePath(value);
|
||||
if (entries.get(rel)?.type !== "directory")
|
||||
throw new Error(`Import directory not found: ${rel}`);
|
||||
return [...entries]
|
||||
.filter(
|
||||
([p]) => p !== "" && (path.posix.dirname(p) === "." ? "" : path.posix.dirname(p)) === rel,
|
||||
)
|
||||
.map(([p, entry]) => ({ name: path.posix.basename(p), path: p, type: entry.type }))
|
||||
.sort((a, b) => a.path.localeCompare(b.path));
|
||||
};
|
||||
readFile = async (value) => {
|
||||
assertOpen();
|
||||
const found = entries.get(relativePath(value));
|
||||
if (!found || found.type !== "file") throw new Error(`Import file not found: ${value}`);
|
||||
if (found.data) {
|
||||
if (found.data.length > MAX_FILE_BYTES)
|
||||
throw new Error("Import file exceeds the 32 MiB read limit");
|
||||
return Buffer.from(found.data);
|
||||
}
|
||||
const entry = found.entry!;
|
||||
if (entry.uncompressedSize > MAX_FILE_BYTES)
|
||||
throw new Error("Import file exceeds the 32 MiB read limit");
|
||||
const stream = await new Promise<import("node:stream").Readable>((resolve, reject) => {
|
||||
zip!.openReadStream(entry, (err, result) => (err ? reject(err) : resolve(result)));
|
||||
});
|
||||
const chunks: Buffer[] = [];
|
||||
let total = 0;
|
||||
for await (const chunk of stream) {
|
||||
const bytes = Buffer.from(chunk as Uint8Array);
|
||||
total += bytes.length;
|
||||
if (total > MAX_FILE_BYTES) {
|
||||
stream.destroy();
|
||||
throw new Error("Import file exceeds the 32 MiB read limit");
|
||||
}
|
||||
chunks.push(bytes);
|
||||
}
|
||||
assertOpen();
|
||||
return Buffer.concat(chunks);
|
||||
};
|
||||
}
|
||||
return {
|
||||
text,
|
||||
isText,
|
||||
kind,
|
||||
name,
|
||||
files: {
|
||||
// A lone file is only ever handed to plugins as text, never as a tree
|
||||
kind: kind === "file" ? "directory" : kind,
|
||||
name,
|
||||
readDir,
|
||||
readFile,
|
||||
async readTextFile(value) {
|
||||
return new TextDecoder("utf-8", { fatal: true }).decode(await readFile(value));
|
||||
},
|
||||
},
|
||||
close() {
|
||||
closed = true;
|
||||
zip?.close();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Compressing a folder puts everything under that folder inside the archive. Present the
|
||||
* folder's contents as the root so plugins see the same tree they would for the directory.
|
||||
*/
|
||||
function unwrapSingleRoot<T extends { type: "file" | "directory" }>(entries: Map<string, T>) {
|
||||
const tops = new Set<string>();
|
||||
for (const entryPath of entries.keys()) {
|
||||
if (entryPath === "") continue;
|
||||
const top = entryPath.split("/")[0]!;
|
||||
if (top === "__MACOSX" || top.startsWith(".")) continue;
|
||||
tops.add(top);
|
||||
}
|
||||
if (tops.size !== 1) return;
|
||||
const [root] = tops;
|
||||
if (entries.get(root!)?.type !== "directory") return;
|
||||
const prefix = `${root}/`;
|
||||
const snapshot = Array.from(entries);
|
||||
for (const [entryPath, value] of snapshot) {
|
||||
if (entryPath === "") continue;
|
||||
entries.delete(entryPath);
|
||||
if (entryPath === root) continue;
|
||||
if (entryPath.startsWith(prefix)) entries.set(entryPath.slice(prefix.length), value);
|
||||
}
|
||||
}
|
||||
|
||||
/** Keep legacy text plugins out of directories, archives, and arbitrary binary inputs. */
|
||||
export async function runImporter(importer: ImporterPlugin, ctx: Context, input: ImportRequest) {
|
||||
if (!importer.onImportSource && input.source?.type === "directory") return null;
|
||||
// Avoid opening archives once for every legacy importer.
|
||||
if (!importer.onImportSource && input.source?.type === "file") {
|
||||
const bytes = Buffer.from(input.source.base64, "base64");
|
||||
if (isZip(bytes)) return null;
|
||||
}
|
||||
const session = await createImportFiles(input);
|
||||
try {
|
||||
if (importer.onImportSource) {
|
||||
const source: ImportSource | null =
|
||||
session.kind === "file"
|
||||
? session.isText
|
||||
? { type: "text", name: session.name, text: session.text }
|
||||
: null
|
||||
: { type: "directory", files: session.files };
|
||||
if (source == null) return null;
|
||||
return await importer.onImportSource(ctx, { source });
|
||||
}
|
||||
if (session.isText && importer.onImport) {
|
||||
return await importer.onImport(ctx, { text: session.text });
|
||||
}
|
||||
return null;
|
||||
} finally {
|
||||
session.close();
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,8 @@ export type ButtonProps = Omit<HTMLAttributes<HTMLButtonElement>, "color" | "onC
|
||||
size?: ButtonSize;
|
||||
justify?: "start" | "center";
|
||||
type?: "button" | "submit";
|
||||
/** Submit a form the button is not inside, by that form's id */
|
||||
form?: string;
|
||||
forDropdown?: boolean;
|
||||
disabled?: boolean;
|
||||
title?: string;
|
||||
|
||||
Reference in new issue
Block a user