Compare commits

...
Author SHA1 Message Date
Gregory SchierandClaude Opus 5.5 479a2af6c3 Patch the Brotli decoder CVE in bundled Node and yaak-http
CVE-2026-111497 is an out-of-bounds write in the Brotli decoder. It
reaches Yaak through the bundled Node (`yaaknode`, used by plugins) and
through the pure-Rust `brotli-decompressor` that yaak-http uses to decode
`br` responses.

- Bump the plugin runtime Node from 24.11.1 to 24.22.2 and update the
  vendoring script's SHA256 pins from the signed SHASUMS256.txt.
- Move yaak-http from `brotli = "8"` to `brotli = "9"`, locking brotli
  9.1.1 and brotli-decompressor 6.2.0.
- Bump compression-codecs 0.4.43 -> 0.4.45 (as on main) so
  async-compression's streaming decoder also uses brotli 9.1.1.

Tauri's pinned git rev still pulls brotli 8.0.1 and brotli-decompressor
5.0.0 for its embedded-asset compression; that needs a Tauri bump.

(cherry picked from commit 39ff1351)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-30 23:56:11 -07:00
Zhizhen He 919e58fc9d Prevent sync data loss from incomplete tracked files (#725)
Co-authored-by: Zhizhen He <hezhizhen.yi@gmail.com>
2026-10-06 21:45:26 -07:00
Gregory Schier b4417add33 Fix PDF viewer crash on older macOS WebKit (#730) 2026-10-06 20:50:07 -07:00
M kashif 7890281db3 Trim leading/trailing whitespace from request URLs (#724) 2026-10-06 20:50:03 -07:00
Ngo Quoc Viet d44d75dcef fix(template-function-xml): return attribute and text() values from XPath (#723) 2026-10-06 20:31:36 -07:00
Gregory Schier 7f30253616 Select new sub-environments and keep enabled state through bulk edit (#714) 2026-09-26 09:40:33 -07:00
Gregory Schier 6c2a18d506 Attach CodeMirror editors after building them to batch layout work (#713) 2026-09-26 08:54:30 -07:00
Gregory Schier 0a57d8eb61 Convert the remaining Insomnia tags to native Yaak functions (#712) 2026-09-25 10:50:22 -07:00
Gregory Schier 411aa262c7 Test the plugin's copy of the template tag scanner (#711) 2026-09-25 10:06:11 -07:00
Gregory Schier 34815f327a Fix Insomnia template and GraphQL import fidelity (#693) 2026-09-25 09:58:31 -07:00
Gregory Schier 6fc43a60e2 Make template tag scanners quote-aware (#710) 2026-09-25 09:32:59 -07:00
Gregory Schier 23d369d84e Print template strings in plain quotes unless they can't round-trip (#708) 2026-09-25 09:07:28 -07:00
62 changed files with 3457 additions and 217 deletions

No files matched your search

Generated
+49 -12
View File
@@ -65,13 +65,28 @@ version = "2.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3"
[[package]]
name = "alloc-no-stdlib"
version = "3.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2fb6cfd47bf496ff64095c20eaba0c201404ee38714d4142fcfa1dc334fcc7a"
[[package]]
name = "alloc-stdlib"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece"
dependencies = [
"alloc-no-stdlib",
"alloc-no-stdlib 2.0.4",
]
[[package]]
name = "alloc-stdlib"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b5c1865780388bfa186411ab5f247819487fc4864c6e9c3106611fa347586e1"
dependencies = [
"alloc-no-stdlib 3.0.0",
]
[[package]]
@@ -778,9 +793,20 @@ version = "8.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9991eea70ea4f293524138648e41ee89b0b2b12ddef3b255effa43c8056e0e0d"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
"brotli-decompressor",
"alloc-no-stdlib 2.0.4",
"alloc-stdlib 0.2.2",
"brotli-decompressor 5.0.0",
]
[[package]]
name = "brotli"
version = "9.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "627b7cfb0578f0eafce087a90fcd40add1231900b69e4d4de2d456260a4f5d81"
dependencies = [
"alloc-no-stdlib 3.0.0",
"alloc-stdlib 0.3.0",
"brotli-decompressor 6.2.0",
]
[[package]]
@@ -789,8 +815,18 @@ version = "5.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
"alloc-no-stdlib 2.0.4",
"alloc-stdlib 0.2.2",
]
[[package]]
name = "brotli-decompressor"
version = "6.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34c6783f42c21c952ad7112b354f0ba2f10760e9decea5b1b60523d6e270faaf"
dependencies = [
"alloc-no-stdlib 3.0.0",
"alloc-stdlib 0.3.0",
]
[[package]]
@@ -1241,11 +1277,11 @@ dependencies = [
[[package]]
name = "compression-codecs"
version = "0.4.43"
version = "0.4.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef16c47ba2797aa6a909cc37d39911f3a6743811fe7408ac0b0cc0276b656e9"
checksum = "98fc98460ba0ad5317075d3632b8dfc45d0be8c4a49347c2a38272019717614a"
dependencies = [
"brotli",
"brotli 9.1.1",
"compression-core",
"flate2",
"memchr",
@@ -8620,7 +8656,7 @@ version = "2.6.1"
source = "git+https://github.com/tauri-apps/tauri?rev=d9bc695c18d9a25baec21d8a5f36d72e3a14ee53#d9bc695c18d9a25baec21d8a5f36d72e3a14ee53"
dependencies = [
"base64 0.22.1",
"brotli",
"brotli 8.0.1",
"ico",
"json-patch",
"plist",
@@ -8961,7 +8997,7 @@ version = "2.9.1"
source = "git+https://github.com/tauri-apps/tauri?rev=d9bc695c18d9a25baec21d8a5f36d72e3a14ee53#d9bc695c18d9a25baec21d8a5f36d72e3a14ee53"
dependencies = [
"anyhow",
"brotli",
"brotli 8.0.1",
"cargo_metadata",
"ctor",
"dom_query",
@@ -11446,7 +11482,7 @@ version = "0.1.0"
dependencies = [
"async-compression",
"async-trait",
"brotli",
"brotli 9.1.1",
"bytes",
"cookie",
"flate2",
@@ -11667,6 +11703,7 @@ dependencies = [
"serde_path_to_error",
"serde_yaml",
"sha1 0.11.0",
"tempfile",
"thiserror 2.0.20",
"tokio",
"ts-rs",
@@ -6,11 +6,12 @@ import { showDialog } from "../lib/dialog";
import { jotaiStore } from "../lib/jotai";
import { setWorkspaceSearchParams } from "../lib/setWorkspaceSearchParams";
export const createSubEnvironmentAndActivate = createFastMutation<
string | null,
unknown,
Environment | null
>({
/**
* Prompt for and create a sub-environment of the given base environment, resolving with the new
* environment's ID (or null if the dialog was dismissed). This does not change the workspace's
* active environment.
*/
export const createSubEnvironment = createFastMutation<string | null, unknown, Environment | null>({
mutationKey: ["create_environment"],
mutationFn: async (baseEnvironment) => {
if (baseEnvironment == null) {
@@ -41,6 +42,17 @@ export const createSubEnvironmentAndActivate = createFastMutation<
});
});
},
});
/** Same as {@link createSubEnvironment}, but also makes the new environment the active one. */
export const createSubEnvironmentAndActivate = createFastMutation<
string | null,
unknown,
Environment | null
>({
mutationKey: ["create_environment_and_activate"],
mutationFn: (baseEnvironment) => createSubEnvironment.mutateAsync(baseEnvironment),
disableToastError: true, // Already shown by createSubEnvironment
onSuccess: async (environmentId) => {
if (environmentId == null) {
return; // Was not created
@@ -4,8 +4,16 @@ import type { TreeHandle, TreeNode, TreeProps } from "@yaakapp-internal/ui";
import { Banner, Icon, InlineCode, SplitLayout, Tree } from "@yaakapp-internal/ui";
import { atom, useAtomValue } from "jotai";
import { atomFamily } from "jotai-family";
import { useCallback, useLayoutEffect, useRef, useState } from "react";
import { createSubEnvironmentAndActivate } from "../commands/createEnvironment";
import {
createContext,
useCallback,
useContext,
useEffect,
useLayoutEffect,
useRef,
useState,
} from "react";
import { createSubEnvironment } from "../commands/createEnvironment";
import { activeWorkspaceAtom, activeWorkspaceIdAtom } from "../hooks/useActiveWorkspace";
import {
environmentsBreakdownAtom,
@@ -40,6 +48,9 @@ interface Props {
type TreeModel = Environment | Workspace;
/** Lets tree row slots create a sub-environment and select it in the editor. */
const CreateSubEnvironmentContext = createContext<() => Promise<void>>(async () => {});
export function EnvironmentEditDialog({ initialEnvironmentId, setRef }: Props) {
const { allEnvironments, baseEnvironment, baseEnvironments } = useEnvironmentsBreakdown();
const [selectedEnvironmentId, setSelectedEnvironmentId] = useState<string | null>(
@@ -115,7 +126,7 @@ function EnvironmentEditDialogSidebar({
const activeWorkspaceId = useAtomValue(activeWorkspaceIdAtom) ?? "";
const treeId = `environment.${activeWorkspaceId}.sidebar`;
const treeRef = useRef<TreeHandle>(null);
const { baseEnvironment, baseEnvironments } = useEnvironmentsBreakdown();
const { allEnvironments, baseEnvironment, baseEnvironments } = useEnvironmentsBreakdown();
// oxlint-disable-next-line react-hooks/exhaustive-deps -- none
useLayoutEffect(() => {
@@ -124,6 +135,31 @@ function EnvironmentEditDialogSidebar({
treeRef.current?.focus();
}, []);
// A just-created environment to select once it reaches the model store. Selecting it before
// then would show a "failed to find" error in the editor, and the tree would drop the
// selection of an item it doesn't have yet.
const [pendingSelectionId, setPendingSelectionId] = useState<string | null>(null);
const tree = useAtomValue(treeAtom);
useEffect(() => {
if (pendingSelectionId == null) return;
if (!allEnvironments.some((e) => e.id === pendingSelectionId)) return;
setPendingSelectionId(null);
setSelectedEnvironmentId(pendingSelectionId);
// The tree leaves out sub-environments while there are multiple base environments
const node = tree == null ? null : findTreeNode(tree, pendingSelectionId);
if (node == null) return;
if (node.parent != null) {
// Expand the parent so the new item is visible to be selected
const parentId = node.parent.item.id;
jotaiStore.set(collapsedFamily(treeId), (prev) => ({ ...prev, [parentId]: false }));
}
// Wait a frame so the tree's own pending selection fixups, which may have been scheduled
// before the new item arrived, don't reset this selection
requestAnimationFrame(() => treeRef.current?.selectItem(pendingSelectionId));
}, [allEnvironments, pendingSelectionId, setSelectedEnvironmentId, tree, treeId]);
const handleDeleteEnvironment = useCallback(
async (environment: Environment) => {
await deleteModelWithConfirm(environment);
@@ -134,6 +170,14 @@ function EnvironmentEditDialogSidebar({
[baseEnvironment?.id, selectedEnvironmentId, setSelectedEnvironmentId],
);
const handleCreateSubEnvironment = useCallback(async () => {
const { baseEnvironment } = jotaiStore.get(environmentsBreakdownAtom);
if (baseEnvironment == null) return;
const id = await createSubEnvironment.mutateAsync(baseEnvironment);
// Show the new environment in the editor without changing the workspace's active environment
if (id != null) setPendingSelectionId(id);
}, []);
const treeHasFocus = useCallback(() => treeRef.current?.hasFocus() ?? false, []);
const getSelectedTreeModels = useCallback(
@@ -195,13 +239,14 @@ function EnvironmentEditDialogSidebar({
const addEnvironmentItem: DropdownItem = {
label: "Create Sub Environment",
leftSlot: <Icon icon="plus" />,
onSelect: async () => {
await createSubEnvironment();
},
onSelect: handleCreateSubEnvironment,
};
// Sub-environments aren't shown while there are multiple base environments, same as the
// plus button on the base environment row
const canCreateSubEnvironment = baseEnvironments.length <= 1;
if (environment == null || environment.model !== "environment") {
return [addEnvironmentItem];
return canCreateSubEnvironment ? [addEnvironmentItem] : [];
}
const singleEnvironment = items.length === 1;
@@ -257,7 +302,7 @@ function EnvironmentEditDialogSidebar({
];
// Add sub environment to base environment
if (isBaseEnvironment(environment) && singleEnvironment) {
if (isBaseEnvironment(environment) && singleEnvironment && canCreateSubEnvironment) {
menuItems.push({ type: "separator" });
menuItems.push(addEnvironmentItem);
}
@@ -266,6 +311,7 @@ function EnvironmentEditDialogSidebar({
},
[
baseEnvironments.length,
handleCreateSubEnvironment,
handleDeleteEnvironment,
handleDuplicateSelected,
handleRenameSelected,
@@ -317,34 +363,37 @@ function EnvironmentEditDialogSidebar({
);
const renderContextMenuFn = useCallback<NonNullable<TreeProps<TreeModel>["renderContextMenu"]>>(
({ items, position, onClose }) => (
<ContextMenu items={items as DropdownItem[]} triggerPosition={position} onClose={onClose} />
),
({ items, position, onClose }) =>
// Right-clicking empty space has nothing to offer while there are multiple base environments
items.length === 0 ? null : (
<ContextMenu items={items as DropdownItem[]} triggerPosition={position} onClose={onClose} />
),
[],
);
const tree = useAtomValue(treeAtom);
return (
<aside className="x-theme-sidebar h-full w-full min-w-0 grid overflow-y-auto border-r border-border-subtle ">
{tree != null && (
<div className="pt-2">
<Tree
ref={treeRef}
treeId={treeId}
collapsedAtom={collapsedFamily(treeId)}
className="px-2 pb-10"
root={tree}
getContextMenu={getContextMenu}
renderContextMenu={renderContextMenuFn}
onDragEnd={handleDragEnd}
getItemKey={(i) => `${i.id}::${i.name}`}
ItemLeftSlotInner={ItemLeftSlotInner}
ItemRightSlot={ItemRightSlot}
ItemInner={ItemInner}
onActivate={handleActivate}
getEditOptions={getEditOptions}
/>
</div>
<CreateSubEnvironmentContext.Provider value={handleCreateSubEnvironment}>
<div className="pt-2">
<Tree
ref={treeRef}
treeId={treeId}
collapsedAtom={collapsedFamily(treeId)}
className="px-2 pb-10"
root={tree}
getContextMenu={getContextMenu}
renderContextMenu={renderContextMenuFn}
onDragEnd={handleDragEnd}
getItemKey={(i) => `${i.id}::${i.name}`}
ItemLeftSlotInner={ItemLeftSlotInner}
ItemRightSlot={ItemRightSlot}
ItemInner={ItemInner}
onActivate={handleActivate}
getEditOptions={getEditOptions}
/>
</div>
</CreateSubEnvironmentContext.Provider>
)}
</aside>
);
@@ -384,6 +433,15 @@ const treeAtom = atom<TreeNode<TreeModel> | null>((get) => {
return root;
});
function findTreeNode(node: TreeNode<TreeModel>, id: string): TreeNode<TreeModel> | null {
if (node.item.id === id) return node;
for (const child of node.children ?? []) {
const found = findTreeNode(child, id);
if (found != null) return found;
}
return null;
}
function ItemLeftSlotInner({ item }: { item: TreeModel }) {
const { baseEnvironments } = useEnvironmentsBreakdown();
return baseEnvironments.length > 1 ? (
@@ -395,6 +453,7 @@ function ItemLeftSlotInner({ item }: { item: TreeModel }) {
function ItemRightSlot({ item }: { item: TreeModel }) {
const { baseEnvironments } = useEnvironmentsBreakdown();
const handleCreateSubEnvironment = useContext(CreateSubEnvironmentContext);
return (
<>
{item.model === "environment" && baseEnvironments.length <= 1 && isBaseEnvironment(item) && (
@@ -405,7 +464,7 @@ function ItemRightSlot({ item }: { item: TreeModel }) {
icon="plus_circle"
className="opacity-50 hover:opacity-100"
title="Add Sub-Environment"
onClick={createSubEnvironment}
onClick={handleCreateSubEnvironment}
/>
)}
</>
@@ -425,13 +484,6 @@ function ItemInner({ item }: { item: TreeModel }) {
);
}
async function createSubEnvironment() {
const { baseEnvironment } = jotaiStore.get(environmentsBreakdownAtom);
if (baseEnvironment == null) return;
const id = await createSubEnvironmentAndActivate.mutateAsync(baseEnvironment);
return id;
}
function getEditOptions(item: TreeModel) {
const options: ReturnType<NonNullable<TreeProps<TreeModel>["getEditOptions"]>> = {
defaultValue: item.name,
+1 -10
View File
@@ -9,6 +9,7 @@ import type { HeaderValuePreset } from "../lib/data/headerValuePresets";
import { headerNames } from "../lib/data/headerNames";
import { mimeTypes } from "../lib/data/mimetypes";
import { userAgents } from "../lib/data/userAgents";
import { validateHttpHeader } from "../lib/validateHttpHeader";
import { CountBadge } from "./core/CountBadge";
import { DetailsBanner } from "./core/DetailsBanner";
import type { GenericCompletion, GenericCompletionConfig } from "./core/Editor/genericCompletion";
@@ -178,13 +179,3 @@ const nameAutocomplete: PairEditorProps["nameAutocomplete"] = {
},
),
};
const validateHttpHeader = (v: string) => {
if (v === "") {
return true;
}
// Template strings are not allowed so we replace them with a valid example string
const withoutTemplateStrings = v.replace(/\$\{\[\s*[^\]\s]+\s*]}/gi, "123");
return withoutTemplateStrings.match(/^[a-zA-Z0-9-_]+$/) !== null;
};
@@ -1,5 +1,10 @@
import { describe, expect, test } from "vite-plus/test";
import { parseBulkPairLine } from "./BulkPairEditor";
import {
formatBulkPairLine,
formatBulkPairs,
parseBulkPairLine,
parseBulkPairs,
} from "./BulkPairEditor";
describe("parseBulkPairLine", () => {
test("parses colon-space pairs as name and value", () => {
@@ -33,4 +38,130 @@ describe("parseBulkPairLine", () => {
value: "bar\nbaz",
});
});
test("parses commented-out pairs as disabled", () => {
expect(parseBulkPairLine("# foo: bar")).toMatchObject({
enabled: false,
name: "foo",
value: "bar",
});
});
test("allows any whitespace after the comment marker", () => {
expect(parseBulkPairLine("# foo: bar")).toMatchObject({
enabled: false,
name: "foo",
value: "bar",
});
expect(parseBulkPairLine("#\tfoo: bar")).toMatchObject({
enabled: false,
name: "foo",
value: "bar",
});
});
test("treats a # without whitespace after it as part of the name", () => {
expect(parseBulkPairLine("#foo: bar")).toMatchObject({
enabled: true,
name: "#foo",
value: "bar",
});
});
test("parses commented-out pairs with an empty value as disabled", () => {
expect(parseBulkPairLine("# token:")).toMatchObject({
enabled: false,
name: "token",
value: "",
});
expect(parseBulkPairLine("# token: ")).toMatchObject({
enabled: false,
name: "token",
value: "",
});
});
test("drops comments that are not pairs", () => {
expect(parseBulkPairLine("# just a comment")).toBeNull();
expect(parseBulkPairLine("# see http://example.com")).toBeNull();
expect(parseBulkPairLine("#")).toBeNull();
});
test("leaves values that start with # alone", () => {
expect(parseBulkPairLine("color: #fff")).toMatchObject({
enabled: true,
name: "color",
value: "#fff",
});
});
});
describe("formatBulkPairLine", () => {
test("formats enabled pairs as name and value", () => {
expect(formatBulkPairLine({ enabled: true, name: "foo", value: "bar" })).toBe("foo: bar");
});
test("treats pairs without an enabled flag as enabled", () => {
expect(formatBulkPairLine({ name: "foo", value: "bar" })).toBe("foo: bar");
});
test("comments out disabled pairs", () => {
expect(formatBulkPairLine({ enabled: false, name: "foo", value: "bar" })).toBe("# foo: bar");
});
test("escapes newlines in values", () => {
expect(formatBulkPairLine({ enabled: false, name: "foo", value: "a\nb" })).toBe("# foo: a\\nb");
});
});
describe("bulk pair round trip", () => {
test("preserves order and enabled state", () => {
const pairs = [
{ enabled: true, name: "host", value: "example.com" },
{ enabled: false, name: "token", value: "old" },
{ enabled: true, name: "color", value: "#fff" },
{ enabled: false, name: "multi", value: "a\nb" },
{ enabled: true, name: "token", value: "new" },
];
const text = formatBulkPairs(pairs);
expect(text).toBe(
["host: example.com", "# token: old", "color: #fff", "# multi: a\\nb", "token: new"].join(
"\n",
),
);
const parsed = parseBulkPairs(text);
expect(parsed.map(({ enabled, name, value }) => ({ enabled, name, value }))).toEqual(pairs);
});
test("preserves enabled pairs whose name starts with #", () => {
const pairs = [{ enabled: true, name: "#foo", value: "bar" }];
const text = formatBulkPairs(pairs);
expect(text).toBe("#foo: bar");
const parsed = parseBulkPairs(text);
expect(parsed.map(({ enabled, name, value }) => ({ enabled, name, value }))).toEqual(pairs);
});
test("preserves disabled pairs whose name starts with #", () => {
const pairs = [{ enabled: false, name: "#foo", value: "bar" }];
const text = formatBulkPairs(pairs);
expect(text).toBe("# #foo: bar");
const parsed = parseBulkPairs(text);
expect(parsed.map(({ enabled, name, value }) => ({ enabled, name, value }))).toEqual(pairs);
});
test("preserves disabled pairs with empty values", () => {
const pairs = [{ enabled: false, name: "token", value: "" }];
const parsed = parseBulkPairs(formatBulkPairs(pairs));
expect(parsed.map(({ enabled, name, value }) => ({ enabled, name, value }))).toEqual(pairs);
});
test("drops free-text comments and blank lines between pairs", () => {
const parsed = parseBulkPairs("# Auth\nfoo: bar\n\n# baz: qux\n# see the docs");
expect(parsed.map(({ enabled, name, value }) => ({ enabled, name, value }))).toEqual([
{ enabled: true, name: "foo", value: "bar" },
{ enabled: false, name: "baz", value: "qux" },
]);
});
});
@@ -14,20 +14,11 @@ export function BulkPairEditor({
forcedEnvironmentId,
stateKey,
}: Props) {
const pairsText = useMemo(() => {
return pairs
.filter((p) => !(p.name.trim() === "" && p.value.trim() === ""))
.map(formatBulkPairLine)
.join("\n");
}, [pairs]);
const pairsText = useMemo(() => formatBulkPairs(pairs), [pairs]);
const handleChange = useCallback(
(text: string) => {
const pairs = text
.split("\n")
.filter((l: string) => l.trim())
.map(parseBulkPairLine);
onChange(pairs);
onChange(parseBulkPairs(text));
},
[onChange],
);
@@ -47,13 +38,57 @@ export function BulkPairEditor({
);
}
export function formatBulkPairLine(pair: Pair) {
const value = pair.value.replaceAll("\n", "\\n");
return `${pair.name}: ${value}`;
export function formatBulkPairs(pairs: Pair[]): string {
return pairs
.filter((p) => !(p.name.trim() === "" && p.value.trim() === ""))
.map(formatBulkPairLine)
.join("\n");
}
export function parseBulkPairLine(line: string): PairWithId {
const [, name, value] = line.match(/^([^:]+):\s+(.*)$/) ?? [];
export function parseBulkPairs(text: string): PairWithId[] {
return text
.split("\n")
.filter((l: string) => l.trim())
.map(parseBulkPairLine)
.filter((p) => p != null);
}
/**
* Format a pair as a `name: value` line. Disabled pairs are commented out dotenv-style, as
* `# name: value`, so the enabled state survives a round trip through {@link parseBulkPairLine}.
*/
export function formatBulkPairLine(pair: Pair) {
const value = pair.value.replaceAll("\n", "\\n");
const line = `${pair.name}: ${value}`;
return pair.enabled === false ? `# ${line}` : line;
}
const PAIR_REGEX = /^([^:]+):\s+(.*)$/;
// Commented-out pairs may also end at the colon, so a hand-typed `# name:` isn't dropped
const COMMENTED_PAIR_REGEX = /^([^:]+):(?:\s+(.*))?$/;
// A # only marks a comment when followed by whitespace (or nothing), so `#foo: bar` stays an
// enabled pair named `#foo`
const COMMENT_PREFIX_REGEX = /^\s*#(?:\s+|$)/;
/**
* Parse a `name: value` line into an enabled pair. A line starting with `# ` is a disabled pair
* if the rest of it parses as `name: value`, otherwise it's a free-text comment and `null` is
* returned so it can be dropped.
*/
export function parseBulkPairLine(line: string): PairWithId | null {
const commentPrefix = line.match(COMMENT_PREFIX_REGEX);
if (commentPrefix != null) {
const uncommented = line.slice(commentPrefix[0].length);
const match = uncommented.match(COMMENTED_PAIR_REGEX);
if (match == null) return null;
return { ...pairFromMatch(uncommented, match), enabled: false };
}
return pairFromMatch(line, line.match(PAIR_REGEX));
}
function pairFromMatch(line: string, match: RegExpMatchArray | null): PairWithId {
const [, name, value] = match ?? [];
return {
enabled: true,
name: (name ?? line).trim(),
@@ -442,7 +442,21 @@ function EditorInner({
? EditorState.fromJSON(cachedJsonState, config, stateFields)
: EditorState.create(config);
const view = new EditorView({ state, parent: container });
// PERF: Built detached and attached in a microtask, rather than passing `parent` and
// letting the constructor attach it first.
//
// The constructor sets `contenteditable` on the content element and then reads
// `document.fonts.ready`. Both force the browser to flush style and layout when the
// element is already in the document, and whatever the previous editor attached has
// just dirtied it again — so mounting n editors in one commit costs n forced layouts.
// The environment editor mounts two per variable row (~60 of them), which is where its
// half-second open went.
//
// Building detached moves that work off the live document, and deferring the attach to
// a microtask means every editor in the commit is built before any of them is inserted:
// one forced layout for the batch instead of one each. The microtask still runs before
// paint, so nothing is visible in the meantime.
const view = new EditorView({ state });
// For large documents, the parser may parse the max number of lines and fail to add
// things like fold markers because of it.
@@ -450,12 +464,26 @@ function EditorInner({
forceParsing(view, 9e6, 100);
cm.current = { view, languageCompartment };
if (autoFocus) {
view.focus();
}
if (autoSelect) {
view.dispatch({ selection: { anchor: 0, head: view.state.doc.length } });
}
queueMicrotask(() => {
// The editor was torn down (or replaced) before we got here
if (cm.current?.view !== view) return;
// Runs after the surrounding try/catch has returned, so it needs its own
try {
container.appendChild(view.dom);
// Everything the constructor measured, it measured detached
view.requestMeasure();
if (autoFocus) {
view.focus();
}
if (autoSelect) {
view.dispatch({ selection: { anchor: 0, head: view.state.doc.length } });
}
} catch (e) {
console.log("Failed to attach Codemirror", e);
}
});
setRef?.(view);
} catch (e) {
console.log("Failed to initialize Codemirror", e);
@@ -0,0 +1,46 @@
import type { EditorView } from "@codemirror/view";
import { describe, expect, test } from "vite-plus/test";
import { jsonParseLinter } from "./json-lint";
// The linter only reads the document text off the view
function view(doc: string): EditorView {
return { state: { doc: { toString: () => doc } } } as EditorView;
}
const lint = jsonParseLinter();
describe("jsonParseLinter", () => {
test("reports nothing for an empty document", () => {
expect(lint(view(" "))).toEqual([]);
});
test("reports nothing for valid JSON", () => {
expect(lint(view('{"a": 1}'))).toEqual([]);
});
test("reports a message for invalid JSON", () => {
expect(lint(view('{"a": }'))).toHaveLength(1);
});
test("ignores a template tag standing in for a value", () => {
expect(lint(view('{"a": "${[ my_var ]}"}'))).toEqual([]);
});
test("ignores a tag whose quoted argument contains `]}`", () => {
expect(lint(view(`{"a": "\${[ fn(arg='x]}y') ]}"}`))).toEqual([]);
});
test("ignores a tag whose quoted argument contains an escaped quote", () => {
expect(lint(view(`{"a": "\${[ fn(arg='it\\'s ]}') ]}"}`))).toEqual([]);
});
test("still reports errors around a tag with a quoted `]}`", () => {
expect(lint(view(`{"a": "\${[ fn(arg='x]}y') ]}",}`))).toHaveLength(1);
});
test("keeps the tag's length so error offsets stay correct", () => {
const doc = `{"a" "\${[ fn(arg='x]}y') ]}"}`;
const [error] = lint(view(doc));
expect(error?.from).toEqual(doc.indexOf('"${['));
});
});
@@ -1,8 +1,7 @@
import type { Diagnostic } from "@codemirror/lint";
import type { EditorView } from "@codemirror/view";
import { type ParseError, parse, printParseErrorCode } from "jsonc-parser";
const TEMPLATE_SYNTAX_REGEX = /\$\{\[[\s\S]*?]}/g;
import { replaceTemplateTags } from "../../../lib/templateTags";
// jsonc-parser reports error codes, so these are the words the editor shows for them
const MESSAGES: Record<string, string> = {
@@ -36,7 +35,7 @@ export function jsonParseLinter(options?: JsonLintOptions) {
// We need lint to not break on stuff like {"foo:" ${[ ... ]}} so we'll replace all template
// syntax with repeating `1` characters, so it's valid JSON and the position is still correct.
const escapedDoc = doc.replace(TEMPLATE_SYNTAX_REGEX, (m) => "1".repeat(m.length));
const escapedDoc = replaceTemplateTags(doc, (m) => "1".repeat(m.end - m.start));
const errors: ParseError[] = [];
parse(escapedDoc, errors, {
@@ -4,4 +4,6 @@ export const highlight = styleTags({
TagOpen: t.bracket,
TagClose: t.bracket,
TagContent: t.keyword,
// Strings are part of the tag body, so they get the same color as the rest of it
TagString: t.keyword,
});
@@ -1,13 +1,17 @@
@top Template { (Tag | Text)* }
// `TagString` keeps a quoted argument from closing the tag, so `${[ fn(a='x]}y') ]}` stays a
// single Tag. Only a terminated string matches, so a half-typed `'` degrades to TagContent and
// the tag still closes at the next `]}`.
@local tokens {
TagClose { "]}" }
TagString { "'" (![\\'] | "\\" _)* "'" }
@else TagContent
}
@skip { } {
TagOpen { "${[" }
Tag { TagOpen (TagContent)+ TagClose }
Tag { TagOpen (TagContent | TagString)+ TagClose }
}
@tokens {
@@ -3,5 +3,6 @@ export const Template = 1,
Tag = 2,
TagOpen = 3,
TagContent = 4,
TagClose = 5,
Text = 6;
TagString = 5,
TagClose = 6,
Text = 7;
@@ -23,6 +23,10 @@ function hasError(input: string): boolean {
return getNodeNames(input).includes("⚠");
}
function countTags(input: string): number {
return getNodeNames(input).filter((n) => n === "Tag").length;
}
describe("twig grammar", () => {
describe("${[var]} format (valid template tags)", () => {
test("parses simple variable as Tag", () => {
@@ -83,6 +87,50 @@ describe("twig grammar", () => {
});
});
describe("quoted strings inside tags", () => {
test("keeps a quoted `]}` inside the tag", () => {
const input = "${[ fn(a='x]}y') ]}";
expect(countTags(input)).toBe(1);
expect(hasError(input)).toBe(false);
});
test("keeps an escaped quote inside the string", () => {
const input = "${[ fn(a='it\\'s ]}') ]}";
expect(countTags(input)).toBe(1);
expect(hasError(input)).toBe(false);
});
test("parses a b64 string as one Tag", () => {
const input = "${[ b64'Zm9v' ]}";
expect(countTags(input)).toBe(1);
expect(hasError(input)).toBe(false);
});
test("closes at the first `]}` when a quote is left unterminated", () => {
// Half-typed strings must not swallow the rest of the document
const input = "${[ fn(a='unterminated ]} after";
expect(countTags(input)).toBe(1);
expect(hasError(input)).toBe(false);
const tag = parser.parse(input).topNode.getChild("Tag");
expect(input.slice(tag?.from ?? 0, tag?.to ?? 0)).toBe("${[ fn(a='unterminated ]}");
});
test("handles two tags with quoted `]}` in the same string", () => {
const input = "${[ a(x='1]}') ]} and ${[ b(y='2]}') ]}";
expect(countTags(input)).toBe(2);
expect(hasError(input)).toBe(false);
});
test("keeps JSON around a tag with a quoted `]}` intact", () => {
const tag = "${[ fn(a='x]}y') ]}";
const json = '{"a": "' + tag + '"}';
expect(countTags(json)).toBe(1);
expect(hasError(json)).toBe(false);
const node = parser.parse(json).topNode.getChild("Tag");
expect(json.slice(node?.from ?? 0, node?.to ?? 0)).toBe(tag);
});
});
describe("edge cases", () => {
test("handles $ at end of string", () => {
expect(hasError("hello$")).toBe(false);
@@ -4,17 +4,24 @@ import { highlight } from "./highlight";
export const parser = LRParser.deserialize({
version: 14,
states:
"!^QQOPOOOOOO'#C_'#C_OYOQO'#C^OOOO'#Cc'#CcQQOPOOOOOO'#Cd'#CdO_OQO,58xOOOO-E6a-E6aOOOO-E6b-E6bOOOO1G.d1G.d",
stateData: "g~OUROYPO~OSTO~OSTOTXO~O",
goto: "nXPPY^PPPbhTROSTQOSQSORVSQUQRWU",
nodeNames: "⚠ Template Tag TagOpen TagContent TagClose Text",
maxTerm: 10,
"!^QQOPOOOOOO'#C_'#C_OYOQO'#C^OOOO'#Cd'#CdQQOPOOOOOO'#Ce'#CeObOQO,58xOOOO-E6b-E6bOOOO-E6c-E6cOOOO1G.d1G.d",
stateData: "m~OVROZPO~OSTOTTO~OSTOTTOUXO~O",
goto: "oYPPZ_PPPPciTROSTQOSQSORVSQUQRWU",
nodeNames: "⚠ Template Tag TagOpen TagContent TagString TagClose Text",
maxTerm: 11,
propSources: [highlight],
skippedNodes: [0],
repeatNodeCount: 2,
tokenData:
"#{~RTOtbtu!zu;'Sb;'S;=`!o<%lOb~gTU~Otbtuvu;'Sb;'S;=`!o<%lOb~yVO#ob#o#p!`#p;'Sb;'S;=`!o<%l~b~Ob~~!u~!cSO!}b#O;'Sb;'S;=`!o<%lOb~!rP;=`<%lb~!zOU~~!}VO#ob#o#p#d#p;'Sb;'S;=`!o<%l~b~Ob~~!u~#gTO!}b!}#O#v#O;'Sb;'S;=`!o<%lOb~#{OY~",
tokenizers: [1, new LocalTokenGroup("b~RP#P#QU~XP#q#r[~aOT~~", 17, 4)],
"#{~RTOtbtu!zu;'Sb;'S;=`!o<%lOb~gTV~Otbtuvu;'Sb;'S;=`!o<%lOb~yVO#ob#o#p!`#p;'Sb;'S;=`!o<%l~b~Ob~~!u~!cSO!}b#O;'Sb;'S;=`!o<%lOb~!rP;=`<%lb~!zOV~~!}VO#ob#o#p#d#p;'Sb;'S;=`!o<%l~b~Ob~~!u~#gTO!}b!}#O#v#O;'Sb;'S;=`!o<%lOb~#{OZ~",
tokenizers: [
1,
new LocalTokenGroup(
"#R~RQwxX#P#Q!u~[VOwXwxqx#OX#O#Pv#P;'SX;'S;=`!o<%lOX~vOT~~yRO;'SX;'S;=`!S;=`OX~!VWOwXwxqx#OX#O#Pv#P;'SX;'S;=`!o;=`<%lX<%lOX~!rP;=`<%lX~!xP#q#r!{~#QOU~~",
94,
4,
),
],
topRules: { Template: [0, 1] },
tokenPrec: 0,
});
@@ -5,13 +5,11 @@ import type { PDFDocumentProxy } from "pdfjs-dist";
import { useMemo, useRef, useState } from "react";
import { Document, Page, pdfjs } from "react-pdf";
import { useContainerSize } from "@yaakapp-internal/ui";
import pdfWorkerUrl from "./pdfWorker?worker&url";
// Document can start loading during render, so configure its worker synchronously.
// Vite bundles the installed PDF.js worker as a local asset, including offline builds.
pdfjs.GlobalWorkerOptions.workerSrc = new URL(
"pdfjs-dist/build/pdf.worker.min.mjs",
import.meta.url,
).toString();
pdfjs.GlobalWorkerOptions.workerSrc = new URL(pdfWorkerUrl, import.meta.url).toString();
interface Props {
/** A URL for the body the host already stored. */
@@ -0,0 +1,3 @@
// The worker has its own globals, so it needs the polyfills before PDF.js evaluates
import "../../polyfills";
import "pdfjs-dist/build/pdf.worker.min.mjs";
+1
View File
@@ -31,6 +31,7 @@
<div id="cm-portal" class="cm-portal"></div>
<div id="react-portal"></div>
<div id="radix-portal" class="cm-portal"></div>
<script type="module" src="/polyfills.ts"></script>
<script type="module" src="/theme.ts"></script>
<script type="module" src="/font-size.ts"></script>
<script type="module" src="/font.ts"></script>
@@ -0,0 +1,79 @@
/* oxlint-disable no-template-curly-in-string */
/*
* Shared expectations for `resolvedModelName`.
*
* The request template function plugin carries a hand-copied `resolvedModelName` and template-tag
* scanner, because importing this module into the plugin bundle fails to build on Windows in CI.
* Both `resolvedModelName.test.ts` here and `plugins/template-function-request/tests/
* resolvedModelName.test.ts` iterate these cases, so changing an expectation on this side
* automatically exercises the plugin's copy and a drifting copy fails its own suite.
*
* Keep this file dependency-free — the plugin test imports it by relative path, so anything it
* pulls in would have to resolve from both packages.
*/
export interface ResolvedModelNameCase {
/** Test name, used as the `test.each` title on both sides. */
name: string;
/** `url` of the `http_request` under test. */
url: string;
/** `name` of the `http_request`, when the case is about an explicit name. */
requestName?: string;
/** What `resolvedModelName` must return. */
expected: string;
}
export const resolvedModelNameCases: ResolvedModelNameCase[] = [
{
name: "returns the name when there is one",
url: "https://example.com",
requestName: "My Request",
expected: "My Request",
},
{
name: "replaces a simple tag with its contents",
url: "${[ base_url ]}/users",
expected: "base_url/users",
},
{
name: "replaces a tag containing a quoted argument with a space",
url: "${[ fn(arg='my key') ]}/users",
expected: "fn(arg='my key')/users",
},
{
name: "replaces each tag in a multi-tag string",
url: "${[ fn(arg='my key') ]}/a/${[ other(b='x y') ]}/b",
expected: "fn(arg='my key')/a/other(b='x y')/b",
},
{
name: "keeps a quoted `]}` inside the tag",
url: "${[ fn(arg='x]}y') ]}/users",
expected: "fn(arg='x]}y')/users",
},
{
name: "keeps an escaped quote inside the tag",
url: "${[ fn(arg='it\\'s ]}') ]}/users",
expected: "fn(arg='it\\'s ]}')/users",
},
{
name: "closes at the first `]}` when a quote is left unterminated",
url: "${[ fn(arg='oops ]}/users",
expected: "fn(arg='oops/users",
},
{
name: "stops each tag at its first closing bracket",
url: "${[ a ]}${[ b ]}",
expected: "ab",
},
{
name: "strips the protocol",
url: "https://example.com/${[ path ]}",
expected: "example.com/path",
},
{
name: "falls back to a generic name when the url is empty",
url: "",
expected: "HTTP Request",
},
];
@@ -0,0 +1,21 @@
import type { HttpRequest } from "@yaakapp-internal/models";
import { describe, expect, test, vi } from "vite-plus/test";
import { resolvedModelNameCases } from "./resolvedModelName.cases";
// resolvedModelName only reads plain model fields, but its module imports pull in the
// platform singleton, which needs a browser. Stub them out to keep this a pure unit test.
vi.mock("@yaakapp-internal/models", () => ({ foldersAtom: {} }));
vi.mock("./jotai", () => ({ jotaiStore: { get: () => [] } }));
const { resolvedModelName } = await import("./resolvedModelName");
function httpRequest(url: string, name = ""): HttpRequest {
return { id: "rq_test", model: "http_request", name, url } as HttpRequest;
}
describe("resolvedModelName", () => {
// Shared with the plugin's hand-copied scanner. See `resolvedModelName.cases.ts`.
test.each(resolvedModelNameCases)("$name", ({ url, requestName, expected }) => {
expect(resolvedModelName(httpRequest(url, requestName))).toEqual(expected);
});
});
+2 -1
View File
@@ -1,6 +1,7 @@
import type { AnyModel } from "@yaakapp-internal/models";
import { foldersAtom } from "@yaakapp-internal/models";
import { jotaiStore } from "./jotai";
import { replaceTemplateTags } from "./templateTags";
export function resolvedModelName(r: AnyModel | null): string {
if (r == null) return "";
@@ -15,7 +16,7 @@ export function resolvedModelName(r: AnyModel | null): string {
}
// Replace variable syntax with variable name
const withoutVariables = r.url.replace(/\$\{\[\s*([^\]\s]+)\s*]}/g, "$1");
const withoutVariables = replaceTemplateTags(r.url, (m) => m.inner.trim());
if (withoutVariables.trim() === "") {
return r.model === "http_request"
? r.bodyType && r.bodyType === "graphql"
+176
View File
@@ -0,0 +1,176 @@
/* oxlint-disable no-template-curly-in-string */
import { describe, expect, test } from "vite-plus/test";
import { findTemplateTags, replaceTemplateTags } from "./templateTags";
/** Seeded so a fuzz failure reproduces. */
function random(seed: number): () => number {
let state = seed;
return () => {
state = (state + 0x6d2b79f5) | 0;
let t = Math.imul(state ^ (state >>> 15), 1 | state);
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
};
}
function millis(run: () => void): number {
const started = performance.now();
run();
return performance.now() - started;
}
describe("findTemplateTags", () => {
test("finds a simple tag", () => {
expect(findTemplateTags("${[ my_var ]}")).toEqual([{ start: 0, end: 13, inner: " my_var " }]);
});
test("finds a tag surrounded by text", () => {
const text = 'a="${[ my_var ]}"';
expect(findTemplateTags(text)).toEqual([{ start: 3, end: 16, inner: " my_var " }]);
expect(text.slice(3, 16)).toEqual("${[ my_var ]}");
});
test("finds an empty tag", () => {
expect(findTemplateTags("${[]}")).toEqual([{ start: 0, end: 5, inner: "" }]);
});
test("keeps a quoted `]}` inside the tag", () => {
const text = "${[ fn(a='x]}y') ]}";
expect(findTemplateTags(text)).toEqual([
{ start: 0, end: text.length, inner: " fn(a='x]}y') " },
]);
});
test("keeps an escaped quote inside the string", () => {
const text = "${[ fn(a='it\\'s ]}') ]}";
expect(findTemplateTags(text)).toEqual([
{ start: 0, end: text.length, inner: " fn(a='it\\'s ]}') " },
]);
});
test("keeps a b64 string in one tag", () => {
const text = "${[ b64'Zm9v' ]}";
expect(findTemplateTags(text)).toEqual([{ start: 0, end: text.length, inner: " b64'Zm9v' " }]);
});
test("closes at the first `]}` when a quote is left unterminated", () => {
const text = "${[ fn(a='unterminated ]} rest";
expect(findTemplateTags(text)).toEqual([{ start: 0, end: 25, inner: " fn(a='unterminated " }]);
expect(text.slice(0, 25)).toEqual("${[ fn(a='unterminated ]}");
});
test("finds two tags with text between them", () => {
expect(findTemplateTags("one ${[ a ]} two ${[ b ]} three")).toEqual([
{ start: 4, end: 12, inner: " a " },
{ start: 17, end: 25, inner: " b " },
]);
});
test("finds two adjacent tags", () => {
expect(findTemplateTags("${[ a ]} ${[ b ]}")).toEqual([
{ start: 0, end: 8, inner: " a " },
{ start: 9, end: 17, inner: " b " },
]);
});
test("finds two tags that each quote a `]}`", () => {
const text = "${[ a(x='1]}') ]} and ${[ b(y='2]}') ]}";
expect(findTemplateTags(text)).toEqual([
{ start: 0, end: 17, inner: " a(x='1]}') " },
{ start: 22, end: 39, inner: " b(y='2]}') " },
]);
});
test("finds nothing in an unclosed tag", () => {
expect(findTemplateTags("${[ no close")).toEqual([]);
});
test("finds nothing in text without tags", () => {
expect(findTemplateTags('{"a": "${b}", "c": 1}')).toEqual([]);
});
test("treats a nested `${[` as ordinary content", () => {
expect(findTemplateTags("${[ a ${[ b ]}")).toEqual([{ start: 0, end: 14, inner: " a ${[ b " }]);
});
test("finishes fast on a long run of quotes", () => {
const text = `\${[${"'".repeat(20_000)} ]}`;
let matches: unknown;
const elapsed = millis(() => {
matches = findTemplateTags(text);
});
expect(matches).toEqual([{ start: 0, end: text.length, inner: `${"'".repeat(20_000)} ` }]);
expect(elapsed).toBeLessThan(200);
});
test("finishes fast on a long run of unpairable quotes", () => {
// Every quote is escaped, so none of them opens a string that closes
const text = `\${[${"\\'".repeat(20_000)} ]}`;
let matches: unknown;
const elapsed = millis(() => {
matches = findTemplateTags(text);
});
expect(matches).toHaveLength(1);
expect(elapsed).toBeLessThan(200);
});
test("finishes fast on a long run of openers that never close", () => {
const text = "${[".repeat(20_000);
let matches: unknown;
const elapsed = millis(() => {
matches = findTemplateTags(text);
});
expect(matches).toEqual([]);
expect(elapsed).toBeLessThan(200);
});
test("never throws, and always reports real bounds", () => {
const alphabet = ["$", "{", "[", "]", "}", "'", "\\", "a", "b"];
const next = random(0x5eed);
for (let n = 0; n < 2_000; n++) {
let text = "";
const length = Math.floor(next() * 31);
for (let i = 0; i < length; i++) {
text += alphabet[Math.floor(next() * alphabet.length)];
}
let matches: ReturnType<typeof findTemplateTags> = [];
expect(() => {
matches = findTemplateTags(text);
}, text).not.toThrow();
let previousEnd = 0;
for (const match of matches) {
expect(text.slice(match.start, match.start + 3), text).toEqual("${[");
expect(text.slice(match.end - 2, match.end), text).toEqual("]}");
expect(text.slice(match.start + 3, match.end - 2), text).toEqual(match.inner);
expect(match.start, text).toBeGreaterThanOrEqual(previousEnd);
previousEnd = match.end;
}
}
});
});
describe("replaceTemplateTags", () => {
test("leaves text without tags alone", () => {
expect(replaceTemplateTags('{"a": 1}', () => "x")).toEqual('{"a": 1}');
});
test("replaces each tag with its trimmed contents", () => {
expect(replaceTemplateTags("${[ a ]}/x/${[ b ]}", (m) => m.inner.trim())).toEqual("a/x/b");
});
test("keeps the replaced length when padding", () => {
const tag = "${[ fn(arg='x]}y') ]}";
const text = `{"a": "${tag}"}`;
const replaced = replaceTemplateTags(text, (m) => "1".repeat(m.end - m.start));
expect(replaced).toEqual(`{"a": "${"1".repeat(tag.length)}"}`);
expect(replaced).toHaveLength(text.length);
});
test("leaves an unclosed tag in place", () => {
expect(replaceTemplateTags("${[ oops", () => "x")).toEqual("${[ oops");
});
});
+137
View File
@@ -0,0 +1,137 @@
/*
* Finding template tags in text.
*
* A tag closes at `]}`, but a quoted argument is allowed to contain `]}` of its own — the
* printer in `crates/yaak-templates` emits `${[ fn(a='x]}y') ]}` verbatim — so the scan has to
* step over quoted strings. It mirrors the Lezer grammar in
* `components/core/Editor/twig/twig.grammar`: a terminated `'…'` (where `\` escapes whatever
* follows it) is consumed whole, and an unterminated quote is just another character, so a
* half-typed tag still closes at the next `]}` instead of swallowing the rest of the document.
*
* This was a regex until CodeQL flagged it (js/polynomial-redos): matching an unterminated
* quote needed a negative lookahead that rescanned to the end of the input at every unpaired
* quote, which is quadratic on text an attacker picks. The scan below is linear instead — see
* the notes on `closingQuoteTable` and on the dead-end set in `findTemplateTags` for why.
*/
const QUOTE = 0x27; // '
const BACKSLASH = 0x5c; // \
const CLOSE_BRACKET = 0x5d; // ]
const CLOSE_BRACE = 0x7d; // }
/** The `${[`…`]}` bounds of one tag, and what sits between them. */
export interface TemplateTagMatch {
/** Index of the `$` that opens the tag. */
start: number;
/** Index just past the closing `]}`. */
end: number;
/** Everything between `${[` and `]}`, not trimmed. */
inner: string;
}
/**
* For every index `i`, the index of the `'` that closes a string whose body starts at `i`, or
* `-1` when nothing does.
*
* Built right to left in a single pass, which is the trick that replaces the regex lookahead:
* whether a quote opens a terminated string depends only on the text after it, so the answer
* can be computed once for every position instead of re-derived at each quote. Skipping a
* string of any length then costs one lookup.
*/
function closingQuoteTable(text: string): Int32Array {
// Two extra slots so a `\` in the last position can read past the end and find `-1`
const table = new Int32Array(text.length + 2).fill(-1);
for (let i = text.length - 1; i >= 0; i--) {
const c = text.charCodeAt(i);
// A backslash escapes whatever follows it, including a quote
const next = c === BACKSLASH ? table[i + 2] : table[i + 1];
table[i] = c === QUOTE ? i : (next ?? -1);
}
return table;
}
/**
* Finds every template tag in `text`, in order and without overlaps.
*
* A `${[` with no `]}` after it isn't a tag, and neither is a `${[` nested inside a tag body —
* both are ordinary content, same as the regex this replaced treated them.
*/
export function findTemplateTags(text: string): TemplateTagMatch[] {
const matches: TemplateTagMatch[] = [];
// Both built only once a scan actually needs them, so short values allocate nothing
let closingQuotes: Int32Array | null = null;
/*
* Body positions a previous scan already ran off the end of the text from. Where a scan ends
* up depends only on where it starts, so a scan that walks into one of these would repeat
* that same walk and fail the same way, and can stop right there.
*
* That bound is what keeps the whole pass linear: every position enters this set at most
* once, and every step of every scan either visits a position for the first time or ends
* that scan.
*/
let deadEnds: Set<number> | null = null;
let searchFrom = 0;
while (searchFrom < text.length) {
const start = text.indexOf("${[", searchFrom);
if (start < 0) break;
const visited: number[] = [];
let end = -1;
let i = start + 3;
while (i < text.length) {
if (deadEnds?.has(i)) break;
visited.push(i);
const c = text.charCodeAt(i);
if (c === QUOTE) {
closingQuotes ??= closingQuoteTable(text);
const close = closingQuotes[i + 1] ?? -1;
// An unterminated quote is an ordinary character, so only step over a closed string
i = close < 0 ? i + 1 : close + 1;
continue;
}
if (c === CLOSE_BRACKET && text.charCodeAt(i + 1) === CLOSE_BRACE) {
end = i + 2;
break;
}
i += 1;
}
if (end < 0) {
// Not a tag. Resume past the opener: a real tag may start inside a string this scan
// stepped over.
deadEnds ??= new Set();
for (const position of visited) deadEnds.add(position);
searchFrom = start + 3;
continue;
}
matches.push({ start, end, inner: text.slice(start + 3, end - 2) });
searchFrom = end;
}
return matches;
}
/** Rewrites every template tag in `text` with whatever `replace` returns for it. */
export function replaceTemplateTags(
text: string,
replace: (match: TemplateTagMatch) => string,
): string {
const matches = findTemplateTags(text);
if (matches.length === 0) return text;
let result = "";
let copiedTo = 0;
for (const match of matches) {
result += text.slice(copiedTo, match.start) + replace(match);
copiedTo = match.end;
}
return result + text.slice(copiedTo);
}
@@ -0,0 +1,44 @@
import { describe, expect, test } from "vite-plus/test";
import { validateHttpHeader } from "./validateHttpHeader";
describe("validateHttpHeader", () => {
test("allows an empty value", () => {
expect(validateHttpHeader("")).toEqual(true);
});
test("allows a plain header name", () => {
expect(validateHttpHeader("X-Api-Key")).toEqual(true);
});
test("rejects a header name with a space", () => {
expect(validateHttpHeader("X Api Key")).toEqual(false);
});
test("allows a simple template tag", () => {
expect(validateHttpHeader("${[ my_var ]}")).toEqual(true);
});
test("allows a tag containing a quoted argument with a space", () => {
expect(validateHttpHeader("${[ fn(arg='my key') ]}")).toEqual(true);
});
test("allows a multi-tag value", () => {
expect(validateHttpHeader("${[ fn(arg='my key') ]}-${[ other(b='x y') ]}")).toEqual(true);
});
test("allows a tag whose quoted argument contains `]}`", () => {
expect(validateHttpHeader("${[ fn(arg='x]}y') ]}")).toEqual(true);
});
test("allows a tag whose quoted argument contains an escaped quote", () => {
expect(validateHttpHeader("${[ fn(arg='it\\'s ]}') ]}")).toEqual(true);
});
test("rejects a tag with a quoted `]}` followed by an invalid character", () => {
expect(validateHttpHeader("${[ fn(arg='x]}y') ]} oops")).toEqual(false);
});
test("rejects a tag followed by an invalid character", () => {
expect(validateHttpHeader("${[ my_var ]} oops")).toEqual(false);
});
});
@@ -0,0 +1,11 @@
import { replaceTemplateTags } from "./templateTags";
export function validateHttpHeader(v: string): boolean {
if (v === "") {
return true;
}
// Template strings are not allowed so we replace them with a valid example string
const withoutTemplateStrings = replaceTemplateTags(v, () => "123");
return withoutTemplateStrings.match(/^[a-zA-Z0-9-_]+$/) !== null;
}
+167
View File
@@ -0,0 +1,167 @@
import { describe, expect, test, vi } from "vite-plus/test";
import { getOrInsert, getOrInsertComputed, readableStreamValues, sumPrecise } from "./polyfills";
describe("getOrInsert", () => {
test("returns the existing value without replacing it", () => {
const map = new Map([["a", 1]]);
expect(getOrInsert.call(map, "a", 2)).toBe(1);
expect(map.get("a")).toBe(1);
});
test("inserts and returns the value for a missing key", () => {
const map = new Map<string, number>();
expect(getOrInsert.call(map, "a", 2)).toBe(2);
expect(map.get("a")).toBe(2);
});
test("works on a WeakMap", () => {
const key = {};
const map = new WeakMap<object, number>();
expect(getOrInsert.call(map, key, 1)).toBe(1);
expect(getOrInsert.call(map, key, 2)).toBe(1);
});
});
describe("getOrInsertComputed", () => {
test("returns the existing value without calling the callback", () => {
const map = new Map([["a", 1]]);
const callback = vi.fn(() => 2);
expect(getOrInsertComputed.call(map, "a", callback)).toBe(1);
expect(callback).not.toHaveBeenCalled();
});
test("calls the callback with the key and inserts its result", () => {
const map = new Map<string, string>();
expect(getOrInsertComputed.call(map, "a", (key) => `${String(key)}!`)).toBe("a!");
expect(map.get("a")).toBe("a!");
});
test("stores what the callback returns even if the callback set the key", () => {
const map = new Map<string, number>();
const result = getOrInsertComputed.call(map, "a", () => {
map.set("a", 1);
return 2;
});
expect(result).toBe(2);
expect(map.get("a")).toBe(2);
});
test("canonicalizes -0 to +0", () => {
const map = new Map<number, number>();
getOrInsertComputed.call(map, -0, (key) => {
expect(Object.is(key, 0)).toBe(true);
return 1;
});
expect(Object.is([...map.keys()][0], 0)).toBe(true);
});
test("throws when the callback is not a function, even for an existing key", () => {
const map = new Map([["a", 1]]);
// @ts-expect-error testing a non-callable callback
expect(() => getOrInsertComputed.call(map, "a", 1)).toThrow(TypeError);
});
test("is installed on Map and WeakMap", () => {
const map = new Map<string, number[]>();
map.getOrInsertComputed("a", () => []).push(1);
map.getOrInsertComputed("a", () => []).push(2);
expect(map.get("a")).toEqual([1, 2]);
const key = {};
const weakMap = new WeakMap<object, number>();
expect(weakMap.getOrInsertComputed(key, () => 1)).toBe(1);
expect(weakMap.getOrInsert(key, 2)).toBe(1);
});
});
describe("sumPrecise", () => {
test("sums without intermediate rounding", () => {
expect(sumPrecise([1e20, 0.1, -1e20])).toBe(0.1);
expect(sumPrecise([0.1, 0.2, 0.3])).toBe(0.6);
expect(sumPrecise(Array.from({ length: 10 }, () => 0.1))).toBe(1);
});
test("rounds half to even on the final step", () => {
expect(sumPrecise([1, 1e-16, 1e-16])).toBe(1.0000000000000002);
expect(sumPrecise([1, 2 ** -53])).toBe(1);
});
test("handles zeroes, infinities and NaN like the spec", () => {
expect(Object.is(sumPrecise([]), -0)).toBe(true);
expect(Object.is(sumPrecise([-0, -0]), -0)).toBe(true);
expect(Object.is(sumPrecise([-0, 0]), 0)).toBe(true);
expect(Object.is(sumPrecise([1, -1]), 0)).toBe(true);
expect(sumPrecise([1, Infinity])).toBe(Infinity);
expect(sumPrecise([1, -Infinity])).toBe(-Infinity);
expect(sumPrecise([Infinity, -Infinity])).toBeNaN();
expect(sumPrecise([1, NaN])).toBeNaN();
});
test("accepts any iterable and rejects non-numbers", () => {
expect(sumPrecise(new Set([1, 2, 3]))).toBe(6);
// @ts-expect-error testing a non-number item
expect(() => sumPrecise([1, "2"])).toThrow(TypeError);
});
test("is installed on Math", () => {
expect(Math.sumPrecise([1, 2, 3])).toBe(6);
});
});
describe("readableStreamValues", () => {
const values = readableStreamValues<number>;
function streamOf(chunks: number[], onCancel?: () => void) {
return new ReadableStream<number>({
start(controller) {
for (const chunk of chunks) controller.enqueue(chunk);
controller.close();
},
cancel: onCancel,
});
}
test("yields every chunk and releases the lock", async () => {
const stream = streamOf([1, 2, 3]);
const seen: number[] = [];
for await (const chunk of values.call(stream)) seen.push(chunk);
expect(seen).toEqual([1, 2, 3]);
expect(stream.locked).toBe(false);
});
test("cancels the stream when the loop exits early", async () => {
const onCancel = vi.fn();
const stream = new ReadableStream<number>({
pull(controller) {
controller.enqueue(1);
},
cancel: onCancel,
});
for await (const _ of values.call(stream)) break;
expect(onCancel).toHaveBeenCalled();
expect(stream.locked).toBe(false);
});
test("leaves the stream open with preventCancel", async () => {
const onCancel = vi.fn();
const stream = streamOf([1, 2], onCancel);
for await (const _ of values.call(stream, { preventCancel: true })) break;
expect(onCancel).not.toHaveBeenCalled();
expect(stream.locked).toBe(false);
});
test("releases the lock and rethrows when cancelling rejects", async () => {
const stream = new ReadableStream<number>({
pull(controller) {
controller.enqueue(1);
},
cancel() {
throw new Error("cancel failed");
},
});
await expect(async () => {
for await (const _ of values.call(stream)) break;
}).rejects.toThrow("cancel failed");
expect(stream.locked).toBe(false);
});
});
+172
View File
@@ -0,0 +1,172 @@
/**
* Built-ins that PDF.js's modern build calls unconditionally but the system WebKit on
* older macOS lacks: the upsert methods and `Math.sumPrecise` arrived in Safari 26.2,
* and async iteration of `ReadableStream` in Safari 27. Without them the PDF viewer
* throws as soon as it renders. Loaded first by `index.html`, and by `pdfWorker.ts`
* because the worker has its own globals.
*
* Each is defined only when missing, so newer engines keep their native versions.
*/
type KeyedCollection<K, V> = {
has(key: K): boolean;
get(key: K): V | undefined;
set(key: K, value: V): unknown;
};
/** https://tc39.es/proposal-upsert/#sec-map.prototype.getOrInsert */
export function getOrInsert<K, V>(this: KeyedCollection<K, V>, key: K, value: V): V {
if (this.has(key)) return this.get(key) as V;
this.set(key, value);
return value;
}
/** https://tc39.es/proposal-upsert/#sec-map.prototype.getOrInsertComputed */
export function getOrInsertComputed<K, V>(
this: KeyedCollection<K, V>,
key: K,
callbackfn: (key: K) => V,
): V {
if (typeof callbackfn !== "function") {
throw new TypeError("getOrInsertComputed callback is not a function");
}
if (this.has(key)) return this.get(key) as V;
// Map keys are canonicalized, so -0 is stored and passed on as +0
const k = (Object.is(key, -0) ? 0 : key) as K;
const value = callbackfn(k);
// The spec sets after the callback even if the callback inserted the key itself
this.set(k, value);
return value;
}
/**
* https://tc39.es/proposal-math-sum/
*
* Shewchuk's exact summation with round-half-even on the final step (the algorithm
* behind Python's `math.fsum`). A finite input set whose running sum overflows
* partway through is not handled exactly, which PDF.js's byte lengths and widths
* never come near.
*/
export function sumPrecise(items: Iterable<number>): number {
const partials: number[] = [];
let allNegativeZero = true;
let sawNaN = false;
let sawPositiveInfinity = false;
let sawNegativeInfinity = false;
for (const item of items) {
if (typeof item !== "number") {
throw new TypeError("Math.sumPrecise requires an iterable of numbers");
}
if (!Object.is(item, -0)) allNegativeZero = false;
if (Number.isNaN(item)) {
sawNaN = true;
} else if (item === Infinity) {
sawPositiveInfinity = true;
} else if (item === -Infinity) {
sawNegativeInfinity = true;
} else {
let x = item;
let i = 0;
for (let y of partials) {
if (Math.abs(x) < Math.abs(y)) [x, y] = [y, x];
const hi = x + y;
const lo = y - (hi - x);
if (lo !== 0) partials[i++] = lo;
x = hi;
}
partials.length = i;
partials.push(x);
}
}
if (sawNaN || (sawPositiveInfinity && sawNegativeInfinity)) return NaN;
if (sawPositiveInfinity) return Infinity;
if (sawNegativeInfinity) return -Infinity;
if (allNegativeZero) return -0;
let n = partials.length;
let hi = 0;
let lo = 0;
if (n > 0) {
hi = partials[--n] as number;
while (n > 0) {
const x = hi;
const y = partials[--n] as number;
hi = x + y;
lo = y - (hi - x);
if (lo !== 0) break;
}
const next = partials[n - 1];
if (n > 0 && next != null && ((lo < 0 && next < 0) || (lo > 0 && next > 0))) {
const y = lo * 2;
const x = hi + y;
if (y === x - hi) hi = x;
}
}
return hi === 0 ? 0 : hi;
}
/** https://streams.spec.whatwg.org/#rs-asynciterator */
export async function* readableStreamValues<R>(
this: ReadableStream<R>,
{ preventCancel = false }: { preventCancel?: boolean } = {},
): AsyncGenerator<R, undefined, undefined> {
const reader = this.getReader();
let finished = false;
try {
while (true) {
const { done, value } = await reader.read();
if (done) {
finished = true;
return;
}
yield value;
}
} catch (err) {
finished = true;
throw err;
} finally {
// Leaving the loop early (break, return, throw in the body) cancels the stream. The
// lock is released even if cancelling rejects, which then propagates as in the spec.
try {
if (!finished && !preventCancel) await reader.cancel();
} finally {
reader.releaseLock();
}
}
}
function define(target: object, key: PropertyKey, value: unknown) {
if (key in target) return;
Object.defineProperty(target, key, { value, writable: true, configurable: true });
}
define(Map.prototype, "getOrInsert", getOrInsert);
define(Map.prototype, "getOrInsertComputed", getOrInsertComputed);
define(WeakMap.prototype, "getOrInsert", getOrInsert);
define(WeakMap.prototype, "getOrInsertComputed", getOrInsertComputed);
define(Math, "sumPrecise", sumPrecise);
if (typeof ReadableStream !== "undefined") {
define(ReadableStream.prototype, "values", readableStreamValues);
define(
ReadableStream.prototype,
Symbol.asyncIterator,
Reflect.get(ReadableStream.prototype, "values"),
);
}
// TypeScript's lib does not describe these yet
declare global {
interface Map<K, V> {
getOrInsert(key: K, value: V): V;
getOrInsertComputed(key: K, callbackfn: (key: K) => V): V;
}
interface WeakMap<K extends WeakKey, V> {
getOrInsert(key: K, value: V): V;
getOrInsertComputed(key: K, callbackfn: (key: K) => V): V;
}
interface Math {
sumPrecise(items: Iterable<number>): number;
}
}
+2 -2
View File
@@ -29,7 +29,7 @@ pub(crate) async fn cmd_sync_calculate<R: Runtime>(
let db = app_handle.db();
let version = app_handle.package_info().version.to_string();
let db_candidates = get_db_candidates(&db, &version, workspace_id, sync_dir)?;
let fs_candidates = get_fs_candidates(sync_dir)?
let fs_candidates = get_fs_candidates(sync_dir, &db_candidates)?
.into_iter()
// Only keep items in the same workspace
.filter(|fs| fs.model.workspace_id() == workspace_id)
@@ -39,7 +39,7 @@ pub(crate) async fn cmd_sync_calculate<R: Runtime>(
pub(crate) async fn cmd_sync_calculate_fs(dir: &Path) -> Result<Vec<SyncOp>> {
let db_candidates = Vec::new();
let fs_candidates = get_fs_candidates(dir)?;
let fs_candidates = get_fs_candidates(dir, &db_candidates)?;
Ok(compute_sync_ops(db_candidates, fs_candidates))
}
+2 -2
View File
@@ -135,7 +135,7 @@ pub fn git_status(dir: &Path) -> crate::error::Result<GitStatusSummary> {
let obj = entry.to_object(&repo)?;
let content = obj.as_blob().unwrap().content();
let name = Path::new(entry.name().unwrap_or_default());
SyncModel::from_bytes(content.into(), name)?.map(|m| m.0)
SyncModel::from_bytes(content.into(), name).ok().flatten().map(|m| m.0)
}
Err(_) => None,
},
@@ -143,7 +143,7 @@ pub fn git_status(dir: &Path) -> crate::error::Result<GitStatusSummary> {
let next = {
let full_path = repo.workdir().unwrap().join(rela_path.clone());
SyncModel::from_file(full_path.as_path())?.map(|m| m.0)
SyncModel::from_file(full_path.as_path()).ok().flatten().map(|m| m.0)
};
entries.push(GitStatusEntry {
+1 -1
View File
@@ -7,7 +7,7 @@ publish = false
[dependencies]
async-compression = { version = "0.4", features = ["tokio", "gzip", "deflate", "brotli", "zstd"] }
async-trait = "0.1"
brotli = "8"
brotli = "9"
bytes = "1.11.1"
cookie = "0.18.1"
flate2 = "1"
+63 -1
View File
@@ -181,7 +181,11 @@ fn strip_query_params(url: &str, names: &[&str]) -> String {
}
fn build_url(r: &HttpRequest) -> String {
let (url_string, params) = apply_path_placeholders(&ensure_proto(&r.url), &r.url_parameters);
// Trim stray whitespace: a leading space breaks scheme
// detection in `ensure_proto` (IdnaError); a trailing space becomes `/path%20` -> 404.
let trimmed_url = r.url.trim();
let (url_string, params) =
apply_path_placeholders(&ensure_proto(trimmed_url), &r.url_parameters);
let mut url = append_query_params(
&url_string,
params
@@ -817,6 +821,64 @@ mod tests {
assert_eq!(result, "https://example.com/api?foo=bar#section");
}
#[test]
fn test_build_url_trims_leading_whitespace() {
// A leading space must not break scheme detection (it used to produce
// `http:// https://...`, which fails to parse with an IdnaError).
let r = HttpRequest {
url: " https://example.com/api".to_string(),
url_parameters: vec![],
..Default::default()
};
let result = build_url(&r);
assert_eq!(result, "https://example.com/api");
}
#[test]
fn test_build_url_trims_trailing_whitespace() {
let r = HttpRequest {
url: "https://example.com/api ".to_string(),
url_parameters: vec![],
..Default::default()
};
let result = build_url(&r);
assert_eq!(result, "https://example.com/api");
}
#[test]
fn test_build_url_trims_whitespace_with_params() {
// A trailing space must not end up percent-encoded in the path once query
// parameters are appended (e.g. `/api%20?foo=bar`, which 404s).
let r = HttpRequest {
url: " https://example.com/api ".to_string(),
url_parameters: vec![HttpUrlParameter {
enabled: true,
name: "foo".to_string(),
value: "bar".to_string(),
id: None,
}],
..Default::default()
};
let result = build_url(&r);
assert_eq!(result, "https://example.com/api?foo=bar");
}
#[test]
fn test_build_url_trims_whitespace_without_scheme() {
let r = HttpRequest {
url: " example.com/api\t".to_string(),
url_parameters: vec![],
..Default::default()
};
let result = build_url(&r);
// ensure_proto defaults to http:// for regular domains
assert_eq!(result, "http://example.com/api");
}
#[test]
fn test_build_url_with_existing_query_and_fragment() {
let r = HttpRequest {
+3
View File
@@ -18,3 +18,6 @@ tokio = { workspace = true, features = ["fs", "sync", "macros", "rt"] }
ts-rs = { workspace = true, features = ["chrono-impl", "serde-json-impl"] }
yaak-models = { workspace = true }
serde_path_to_error = "0.1.20"
[dev-dependencies]
tempfile = "3"
+12 -25
View File
@@ -1,7 +1,7 @@
use crate::error::Error::UnknownModel;
use crate::error::Result;
use chrono::NaiveDateTime;
use log::{debug, warn};
use log::debug;
use serde::{Deserialize, Deserializer, Serialize};
use serde_yaml::{Mapping, Value};
use sha1::{Digest, Sha1};
@@ -98,11 +98,13 @@ fn migrate_environment(obj: &mut Mapping) {
}
impl SyncModel {
/// Returns `Ok(None)` for files that don't look like Yaak models, and an error for ones that
/// do but can't be parsed.
pub fn from_bytes(content: Vec<u8>, file_path: &Path) -> Result<Option<(SyncModel, String)>> {
let mut hasher = Sha1::new();
hasher.update(&content);
let checksum = hex::encode(hasher.finalize());
let content_str = String::from_utf8(content.clone()).unwrap_or_default();
let content_str = String::from_utf8(content).unwrap_or_default();
// Check for some strings that will be in a model file for sure. If these strings
// don't exist, then it's probably not a Yaak file.
@@ -111,34 +113,19 @@ impl SyncModel {
}
let ext = file_path.extension().unwrap_or_default();
if ext == "yml" || ext == "yaml" {
Ok(match serde_yaml::from_str::<SyncModel>(&content_str) {
Ok(m) => Some((m, checksum)),
Err(e) => {
warn!("Error parsing {:?} {:?}", file_path.file_name(), e);
None
}
})
let model = if ext == "yml" || ext == "yaml" {
serde_yaml::from_str::<SyncModel>(&content_str)?
} else if ext == "json" {
Ok(match serde_json::from_str::<SyncModel>(&content_str) {
Ok(m) => Some((m, checksum)),
Err(e) => {
warn!("Error parsing {:?} {:?}", file_path.file_name(), e);
None
}
})
serde_json::from_str::<SyncModel>(&content_str)?
} else {
Ok(None)
}
return Ok(None);
};
Ok(Some((model, checksum)))
}
pub fn from_file(file_path: &Path) -> Result<Option<(SyncModel, String)>> {
let content = match fs::read(file_path) {
Ok(c) => c,
Err(_) => return Ok(None),
};
Self::from_bytes(content, file_path)
Self::from_bytes(fs::read(file_path)?, file_path)
}
pub fn to_file_contents(&self, rel_path: &Path) -> Result<(Vec<u8>, String)> {
+182 -19
View File
@@ -1,4 +1,4 @@
use crate::error::Result;
use crate::error::{Error, Result};
use crate::models::SyncModel;
use chrono::Utc;
use log::{info, warn};
@@ -7,8 +7,9 @@ use std::collections::HashMap;
use std::fmt::{Display, Formatter};
use std::fs;
use std::fs::File;
use std::io::Write;
use std::io::{self, Write};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicUsize, Ordering};
use ts_rs::TS;
use yaak_models::blob_manager::BlobManager;
use yaak_models::client_db::{ClientDb, WriteDb};
@@ -178,33 +179,77 @@ pub fn get_db_candidates(
Ok(candidates)
}
pub fn get_fs_candidates(dir: &Path) -> Result<Vec<FsCandidate>> {
/// Read sync files, validating paths tracked by existing database models.
///
/// Workspace sync callers must pass all candidates from [`get_db_candidates`] for
/// the same workspace and directory. An empty slice is for import discovery
/// without database sync state; it does not validate tracked model identities.
pub fn get_fs_candidates(dir: &Path, db_candidates: &[DbCandidate]) -> Result<Vec<FsCandidate>> {
// Ensure the root directory exists
fs::create_dir_all(dir)?;
let tracked_paths: HashMap<PathBuf, (&SyncModel, &SyncState)> = db_candidates
.iter()
.filter_map(|candidate| match candidate {
DbCandidate::Added(_) | DbCandidate::Deleted(_) => None,
DbCandidate::Modified(model, state) | DbCandidate::Unmodified(model, state) => {
Some((PathBuf::from(&state.rel_path), (model, state)))
}
})
.collect();
let mut candidates = Vec::new();
let entries = fs::read_dir(dir)?;
for dir_entry in entries {
let dir_entry = match dir_entry {
Ok(v) => v,
Err(_) => continue,
};
let dir_entry = dir_entry?;
let path = dir_entry.path();
let rel_path = PathBuf::from(dir_entry.file_name());
let tracked = tracked_paths.get(&rel_path);
let invalid =
|reason: String| Err(Error::InvalidSyncFile(format!("{}: {reason}", path.display())));
if !dir_entry.file_type()?.is_file() {
if tracked.is_some() {
return invalid("expected a regular sync file".into());
}
continue;
};
let path = dir_entry.path();
let (model, checksum) = match SyncModel::from_file(&path) {
Ok(Some(m)) => m,
Ok(None) => continue,
Err(e) => {
warn!("Failed to parse sync file {e}");
return Err(e);
// A present tracked file must not disappear from the candidates: that
// would turn a read/parse failure into a database deletion.
let (model, checksum) = match (SyncModel::from_file(&path), tracked) {
(Ok(Some(m)), _) => m,
(Ok(None), None) => continue,
(Err(e), None) => {
warn!("Skipping invalid sync file {}: {e}", path.display());
continue;
}
(Ok(None), Some(_)) => return invalid("file does not contain a Yaak model".into()),
(Err(e), Some(_)) => return invalid(e.to_string()),
};
let rel_path = Path::new(&dir_entry.file_name()).to_path_buf();
if let Some((previous, state)) = tracked {
if model.id() != state.model_id {
return invalid(format!(
"expected model ID {}, found {}",
state.model_id,
model.id()
));
}
// A wrong/defaulted workspace ID would be filtered out by the caller,
// making this tracked model look deleted even if its ID is intact.
if model.workspace_id() != state.workspace_id {
return invalid(format!(
"expected workspace ID {}, found {}",
state.workspace_id,
model.workspace_id()
));
}
if std::mem::discriminant(&model) != std::mem::discriminant(*previous) {
return invalid("model type does not match the tracked model".into());
}
}
candidates.push(FsCandidate { rel_path, model, checksum })
}
@@ -391,8 +436,7 @@ pub fn apply_fs_sync_ops(
let rel_path = derive_model_filename(&model);
let abs_path = sync_dir.join(rel_path.clone());
let (content, checksum) = model.to_file_contents(&rel_path)?;
let mut f = File::create(&abs_path)?;
f.write_all(&content)?;
write_sync_file(&abs_path, |file| file.write_all(&content))?;
SyncStateOp::Create { model_id: model.id(), checksum, rel_path }
}
SyncOp::FsUpdate { model, state } => {
@@ -400,8 +444,7 @@ pub fn apply_fs_sync_ops(
let rel_path = Path::new(&state.rel_path);
let abs_path = Path::new(&state.sync_dir).join(&rel_path);
let (content, checksum) = model.to_file_contents(&rel_path)?;
let mut f = File::create(&abs_path)?;
f.write_all(&content)?;
write_sync_file(&abs_path, |file| file.write_all(&content))?;
SyncStateOp::Update {
state: state.to_owned(),
checksum,
@@ -447,6 +490,39 @@ pub fn apply_fs_sync_ops(
Ok(pending)
}
/// Publish a complete file without exposing partial writes to sync readers.
///
/// The temporary file's extension keeps it out of sync candidates. A crash may
/// leave it behind, but it is never read as a model.
fn write_sync_file(path: &Path, write: impl FnOnce(&mut File) -> io::Result<()>) -> Result<()> {
static COUNTER: AtomicUsize = AtomicUsize::new(0);
// Opening the existing file for writing keeps read-only files from being replaced
let permissions = match fs::OpenOptions::new().write(true).open(path) {
Ok(file) => Some(file.metadata()?.permissions()),
Err(error) if error.kind() == io::ErrorKind::NotFound => None,
Err(error) => return Err(error.into()),
};
let tmp_path = path.with_file_name(format!(
".{}.{}-{}.tmp",
path.file_name().unwrap_or_default().to_string_lossy(),
std::process::id(),
COUNTER.fetch_add(1, Ordering::Relaxed),
));
let result = File::create_new(&tmp_path).and_then(|mut file| {
if let Some(permissions) = permissions {
file.set_permissions(permissions)?;
}
write(&mut file)?;
drop(file);
fs::rename(&tmp_path, path)
});
if result.is_err() {
let _ = fs::remove_file(&tmp_path);
}
Ok(result?)
}
impl PendingDbSyncOps {
/// Upserts are collected per model type and written in one batch so
/// foreign keys are satisfied.
@@ -603,3 +679,90 @@ fn delete_model(db: &WriteDb, blobs: &BlobManager, model: &SyncModel) -> Result<
};
Ok(())
}
#[cfg(test)]
mod write_tests {
use super::*;
#[test]
fn failed_partial_write_keeps_previous_file() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("request.yaml");
fs::write(&path, b"complete previous contents").unwrap();
let result = write_sync_file(&path, |file| {
file.write_all(b"partial new contents")?;
Err(io::Error::new(io::ErrorKind::WriteZero, "simulated write failure"))
});
assert!(
matches!(result, Err(Error::IoError(error)) if error.kind() == io::ErrorKind::WriteZero)
);
assert_eq!(fs::read(&path).unwrap(), b"complete previous contents");
assert_eq!(fs::read_dir(dir.path()).unwrap().count(), 1);
}
#[test]
fn readers_ignore_the_in_progress_file() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("request.yaml");
let request = HttpRequest {
id: "rq_write_test".into(),
workspace_id: "wk_write_test".into(),
name: "Previous request".into(),
..Default::default()
};
let model = SyncModel::HttpRequest(request.clone());
let (old_contents, checksum) = model.to_file_contents(&path).unwrap();
fs::write(&path, &old_contents).unwrap();
let db_candidates = vec![DbCandidate::Unmodified(
model,
SyncState {
model_id: request.id.clone(),
workspace_id: request.workspace_id.clone(),
rel_path: "request.yaml".into(),
checksum,
..Default::default()
},
)];
let model = SyncModel::HttpRequest(HttpRequest { name: "New request".into(), ..request });
let (new_contents, _) = model.to_file_contents(&path).unwrap();
let split = new_contents.len() / 2;
write_sync_file(&path, |file| {
file.write_all(&new_contents[..split])?;
assert_eq!(fs::read(&path).unwrap(), old_contents);
let fs_candidates = get_fs_candidates(dir.path(), &db_candidates).unwrap();
assert_eq!(fs_candidates.len(), 1);
assert!(compute_sync_ops(db_candidates.clone(), fs_candidates).is_empty());
file.write_all(&new_contents[split..])
})
.unwrap();
assert_eq!(fs::read(&path).unwrap(), new_contents);
assert_eq!(fs::read_dir(dir.path()).unwrap().count(), 1);
}
#[cfg(unix)]
#[test]
fn writes_preserve_existing_permissions_and_new_file_defaults() {
use std::os::unix::fs::PermissionsExt;
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("request.yaml");
fs::write(&path, b"old contents").unwrap();
fs::set_permissions(&path, fs::Permissions::from_mode(0o640)).unwrap();
write_sync_file(&path, |file| {
// Keep unpublished data at least as private as the original file.
assert_eq!(file.metadata()?.permissions().mode() & 0o777 & !0o640, 0);
file.write_all(b"new contents")
})
.unwrap();
assert_eq!(fs::metadata(&path).unwrap().permissions().mode() & 0o777, 0o640);
let reference = dir.path().join("reference.yaml");
File::create(&reference).unwrap();
let new_path = dir.path().join("new.yaml");
write_sync_file(&new_path, |file| file.write_all(b"new contents")).unwrap();
assert_eq!(
fs::metadata(&new_path).unwrap().permissions().mode() & 0o777,
fs::metadata(&reference).unwrap().permissions().mode() & 0o777
);
}
}
+217
View File
@@ -0,0 +1,217 @@
use std::fs;
use std::path::PathBuf;
use tempfile::TempDir;
use yaak_models::blob_manager::BlobManager;
use yaak_models::models::{HttpRequest, SyncState, Workspace};
use yaak_models::query_manager::QueryManager;
use yaak_sync::error::{Error, Result};
use yaak_sync::models::SyncModel;
use yaak_sync::sync::{
SyncOp, apply_db_sync_ops, apply_fs_sync_ops, apply_sync_state_ops, compute_sync_ops,
get_db_candidates, get_fs_candidates,
};
struct Fixture {
db: QueryManager,
blobs: BlobManager,
dir: TempDir,
workspace_id: String,
request: HttpRequest,
request_path: PathBuf,
}
impl Fixture {
fn new(request_filename: &str) -> Self {
let (db, blobs, _events) = yaak_models::init_in_memory().unwrap();
let dir = tempfile::tempdir().unwrap();
let workspace = Workspace { id: "wk_sync_test".into(), ..Default::default() };
let request = HttpRequest {
id: "rq_sync_test".into(),
workspace_id: workspace.id.clone(),
name: "Keep this request".into(),
..Default::default()
};
let request_path = dir.path().join(request_filename);
let fixture = Self {
db,
blobs,
dir,
workspace_id: workspace.id.clone(),
request: request.clone(),
request_path,
};
for (model, path) in [
(SyncModel::Workspace(workspace), fixture.dir.path().join("workspace.yaml")),
(SyncModel::HttpRequest(request), fixture.request_path.clone()),
] {
let (contents, _) = model.to_file_contents(&path).unwrap();
fs::write(path, contents).unwrap();
}
let fs_candidates = get_fs_candidates(fixture.dir.path(), &[]).unwrap();
fixture.apply(compute_sync_ops(Vec::new(), fs_candidates));
assert!(fixture.calculate().unwrap().is_empty());
fixture
}
fn calculate(&self) -> Result<Vec<SyncOp>> {
let db_candidates =
get_db_candidates(&self.db.connect(), "test", &self.workspace_id, self.dir.path())?;
let fs_candidates = get_fs_candidates(self.dir.path(), &db_candidates)?
.into_iter()
.filter(|candidate| candidate.model.workspace_id() == self.workspace_id)
.collect();
Ok(compute_sync_ops(db_candidates, fs_candidates))
}
fn apply(&self, ops: Vec<SyncOp>) {
let pending = apply_fs_sync_ops(&self.workspace_id, self.dir.path(), ops).unwrap();
let workspace = &self.workspace_id;
let dir = self.dir.path();
self.db
.with_tx::<_, Error>(|tx| {
let states = apply_db_sync_ops(tx, &self.blobs, workspace, dir, pending)?;
apply_sync_state_ops(tx, workspace, dir, states)
})
.unwrap();
}
fn states(&self) -> Vec<SyncState> {
let mut states = self
.db
.connect()
.list_sync_states_for_workspace(&self.workspace_id, self.dir.path())
.unwrap();
states.sort_by(|a, b| a.id.cmp(&b.id));
states
}
fn db_request(&self) -> HttpRequest {
self.db.connect().get_http_request(&self.request.id).unwrap()
}
fn edit(&self, edit: impl FnOnce(&mut serde_json::Value)) {
let mut value = serde_json::to_value(SyncModel::HttpRequest(self.request.clone())).unwrap();
edit(&mut value);
let content = if self.request_path.extension().unwrap() == "json" {
serde_json::to_string(&value).unwrap()
} else {
serde_yaml::to_string(&value).unwrap()
};
fs::write(&self.request_path, content).unwrap();
}
fn assert_invalid_file(&self) -> String {
let before_request = self.db_request();
let before_states = serde_json::to_value(self.states()).unwrap();
let error = self.calculate().unwrap_err();
assert!(matches!(&error, Error::InvalidSyncFile(_)), "{error:?}");
assert!(error.to_string().contains(self.request_path.to_str().unwrap()));
assert_eq!(self.db_request(), before_request);
assert_eq!(serde_json::to_value(self.states()).unwrap(), before_states);
error.to_string()
}
}
#[test]
fn tracked_model_type_changes_abort_sync() {
let fixture = Fixture::new("request.yaml");
fixture.edit(|value| {
value["model"] = "websocket_request".into();
});
fixture.assert_invalid_file();
}
#[test]
fn invalid_tracked_files_abort_without_changing_data() {
for (filename, contents) in [
("request.yaml", b"model: http_request\nid: [broken\n".as_slice()),
("request.json", b"{\"model\":\"http_request\",\"id\":".as_slice()),
("custom-request.yml", b"name: half-written\n".as_slice()),
("custom-request.yml", b"model: http_request\n".as_slice()),
("custom-request.yml", [0xff, 0xfe].as_slice()),
] {
let fixture = Fixture::new(filename);
fs::write(&fixture.request_path, contents).unwrap();
fixture.assert_invalid_file();
assert_eq!(fs::read(&fixture.request_path).unwrap(), contents);
}
}
#[test]
fn tracked_file_with_git_conflict_markers_aborts_sync() {
let fixture = Fixture::new("request.yaml");
let contents = fs::read_to_string(&fixture.request_path).unwrap().replace(
"name: Keep this request\n",
"<<<<<<< HEAD\nname: Mine\n=======\nname: Theirs\n>>>>>>> origin/main\n",
);
fs::write(&fixture.request_path, contents).unwrap();
let error = fixture.assert_invalid_file();
assert!(error.contains("line "), "{error}");
}
#[test]
fn empty_tracked_file_aborts_sync() {
let fixture = Fixture::new("request.yaml");
fs::write(&fixture.request_path, "").unwrap();
fixture.assert_invalid_file();
}
#[cfg(unix)]
#[test]
fn unreadable_tracked_file_aborts_sync() {
use std::os::unix::fs::PermissionsExt;
let fixture = Fixture::new("request.yaml");
fs::set_permissions(&fixture.request_path, fs::Permissions::from_mode(0o000)).unwrap();
// Root bypasses ordinary POSIX permission checks.
if fs::read(&fixture.request_path).is_ok() {
return;
}
let error = fixture.assert_invalid_file();
assert!(error.contains("Permission denied"), "{error}");
}
#[test]
fn mismatched_identities_abort_sync_for_yaml_and_json() {
for filename in ["request.yaml", "request.json"] {
for field in ["id", "workspaceId"] {
let fixture = Fixture::new(filename);
fixture.edit(|value| value[field] = "different".into());
fixture.assert_invalid_file();
}
}
}
#[test]
fn tracked_path_replaced_by_a_directory_aborts_sync() {
let fixture = Fixture::new("request.yaml");
fs::remove_file(&fixture.request_path).unwrap();
fs::create_dir(&fixture.request_path).unwrap();
fixture.assert_invalid_file();
}
#[test]
fn deleting_a_tracked_file_still_deletes_the_request() {
let fixture = Fixture::new("request.yaml");
fs::remove_file(&fixture.request_path).unwrap();
let ops = fixture.calculate().unwrap();
assert!(
matches!(ops.as_slice(), [SyncOp::DbDelete { model, .. }] if model.id() == fixture.request.id)
);
fixture.apply(ops);
assert!(fixture.db.connect().get_http_request(&fixture.request.id).is_err());
}
#[test]
fn unrelated_files_are_still_ignored() {
let fixture = Fixture::new("request.yaml");
for (name, contents) in [
("README.md", "model: http_request\nid: [broken\n"),
("config.yaml", "model: unrelated\nid: example\n"),
("unrelated.json", "{\"model\":\"config\",\"id\":"),
("empty.yaml", ""),
] {
fs::write(fixture.dir.path().join(name), contents).unwrap();
}
fs::create_dir(fixture.dir.path().join("other-files")).unwrap();
assert!(fixture.calculate().unwrap().is_empty());
}
+40 -3
View File
@@ -1,3 +1,5 @@
use crate::tag_scan::TagStrings;
enum FormatState {
TemplateTag,
String,
@@ -7,6 +9,7 @@ enum FormatState {
/// Formats JSON that might contain template tags (skipped entirely)
pub fn format_json(text: &str, tab: &str) -> String {
let mut chars = text.chars().peekable();
let mut tag_strings = TagStrings::default();
let mut new_json = "".to_string();
let mut depth = 0;
@@ -43,15 +46,22 @@ pub fn format_json(text: &str, tab: &str) -> String {
}
// Close Template tag states
if let FormatState::TemplateTag = state {
// A quoted argument is allowed to contain `]}`, so skip over strings whole
if current_char == '\'' {
new_json.push(current_char);
if let Some(rest) = tag_strings.take(&mut chars) {
new_json.push_str(&rest);
}
continue;
}
if rest_of_chars.take(2).collect::<String>() == "]}" {
state = FormatState::None;
new_json.push_str("]}");
chars.next(); // Skip the second closing bracket
continue;
} else {
new_json.push(current_char);
continue;
}
new_json.push(current_char);
continue;
}
if rest_of_chars.take(3).collect::<String>() == "${[" {
@@ -315,6 +325,33 @@ mod tests {
);
}
#[test]
fn test_skip_template_tags_with_quoted_brackets() {
// The `]}` inside the quotes must not end the tag, or the rest gets re-indented
assert_eq!(
format_json(r#"{"foo":${[ fn(a='x]}y') ]} }"#, " "),
r#"
{
"foo": ${[ fn(a='x]}y') ]}
}
"#
.trim()
);
}
#[test]
fn test_skip_template_tags_with_escaped_quote() {
assert_eq!(
format_json(r#"{"foo":${[ fn(a='it\'s ]}') ]} }"#, " "),
r#"
{
"foo": ${[ fn(a='it\'s ]}') ]}
}
"#
.trim()
);
}
#[test]
fn test_graphql_response() {
assert_eq!(
+1
View File
@@ -4,6 +4,7 @@ pub mod format_json;
pub mod parser;
pub mod renderer;
pub mod strip_json_comments;
mod tag_scan;
#[cfg(feature = "wasm")]
pub mod wasm;
+83 -8
View File
@@ -45,14 +45,35 @@ pub enum Val {
Null,
}
/// Strings are printed as plain single-quoted values whenever they can survive a round-trip
/// through both the parser and the client's editor. A quoted `]}` is fine — the editor's
/// grammar and regexes skip over quoted strings — but newlines and other control characters
/// don't belong in a single-line tag, so those fall back to base64.
fn requires_b64(text: &str) -> bool {
text.chars().any(char::is_control)
}
/// Escape the only two characters the parser treats specially inside a single-quoted string.
fn escape_str(text: &str) -> String {
let mut out = String::with_capacity(text.len());
for c in text.chars() {
match c {
'\\' => out.push_str("\\\\"),
'\'' => out.push_str("\\'"),
_ => out.push(c),
}
}
out
}
impl Display for Val {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let str = match self {
Val::Str { text } => {
if text.chars().all(|c| c.is_alphanumeric() || c == ' ' || c == '_' || c == '_') {
format!("'{}'", text)
} else {
if requires_b64(text) {
format!("b64'{}'", BASE64_URL_SAFE_NO_PAD.encode(text))
} else {
format!("'{}'", escape_str(text))
}
}
Val::Var { name } => name.to_string(),
@@ -835,10 +856,64 @@ mod tests {
#[test]
fn token_display_complex_str() -> Result<()> {
assert_eq!(
Val::Str { text: "Hello 'You'".to_string() }.to_string(),
"b64'SGVsbG8gJ1lvdSc'"
);
assert_eq!(Val::Str { text: "Hello 'You'".to_string() }.to_string(), r#"'Hello \'You\''"#);
Ok(())
}
#[test]
fn token_display_str_plain() -> Result<()> {
// Strings the parser can read back verbatim print in plain quotes
assert_eq!(Val::Str { text: "X-Api-Key".to_string() }.to_string(), "'X-Api-Key'");
assert_eq!(Val::Str { text: "$.token".to_string() }.to_string(), "'$.token'");
assert_eq!(Val::Str { text: "it's".to_string() }.to_string(), r#"'it\'s'"#);
assert_eq!(Val::Str { text: r"back\slash".to_string() }.to_string(), r"'back\\slash'");
assert_eq!(Val::Str { text: r#"{"a":1}"#.to_string() }.to_string(), r#"'{"a":1}'"#);
assert_eq!(Val::Str { text: "héllo".to_string() }.to_string(), "'héllo'");
// Quoting is enough to keep `]}` from closing the tag early in the client editor
assert_eq!(Val::Str { text: "a]}b".to_string() }.to_string(), "'a]}b'");
Ok(())
}
#[test]
fn token_display_str_b64_fallback() -> Result<()> {
// Control characters don't belong in a single-line tag
assert_eq!(Val::Str { text: "line\nbreak".to_string() }.to_string(), "b64'bGluZQpicmVhaw'");
assert_eq!(Val::Str { text: "line\rbreak".to_string() }.to_string(), "b64'bGluZQ1icmVhaw'");
assert_eq!(Val::Str { text: "tab\there".to_string() }.to_string(), "b64'dGFiCWhlcmU'");
Ok(())
}
#[test]
fn token_str_round_trip() -> Result<()> {
let cases = [
"X-Api-Key",
"$.token",
"hello world",
"it's",
r"back\slash",
r#"{"a":1}"#,
"a]}b",
"line\nbreak",
"héllo",
r"trailing\",
"'",
"",
];
for text in cases {
let val = Val::Str { text: text.to_string() };
let template = Token::Tag { val: val.clone() }.to_string();
let tokens = Parser::new(&template).parse()?;
assert_eq!(
tokens.tokens,
vec![Token::Tag { val: val.clone() }, Token::Eof],
"failed to round-trip {text:?} (printed as {template})"
);
}
Ok(())
}
@@ -879,7 +954,7 @@ mod tests {
}
}
.to_string(),
r#"${[ foo(arg=b64'diAneCc', arg2=my_var) ]}"#
r#"${[ foo(arg='v \'x\'', arg2=my_var) ]}"#
);
Ok(())
@@ -1,3 +1,5 @@
use crate::tag_scan::TagStrings;
/// Strips JSON comments only if the result is valid JSON. If stripping comments
/// produces invalid JSON, the original text is returned unchanged.
pub fn maybe_strip_json_comments(text: &str) -> String {
@@ -17,6 +19,7 @@ pub fn maybe_strip_json_comments(text: &str) -> String {
/// - Comments inside strings and template tags are left alone
pub fn strip_json_comments(text: &str) -> String {
let mut chars = text.chars().peekable();
let mut tag_strings = TagStrings::default();
let mut result = String::with_capacity(text.len());
let mut in_string = false;
let mut in_template_tag = false;
@@ -45,6 +48,13 @@ pub fn strip_json_comments(text: &str) -> String {
// Handle template tags
if in_template_tag {
result.push(current_char);
// A quoted argument is allowed to contain `]}`, so skip over strings whole
if current_char == '\'' {
if let Some(rest) = tag_strings.take(&mut chars) {
result.push_str(&rest);
}
continue;
}
if current_char == ']' && chars.peek() == Some(&'}') {
result.push(chars.next().unwrap());
in_template_tag = false;
@@ -280,6 +290,34 @@ mod tests {
assert_eq!(strip_json_comments(input), input);
}
#[test]
fn test_quoted_tag_close_inside_template_tag() {
// The `]}` is quoted, so the tag runs to the real close and the `//` stays put
let input = r#"{
"foo": ${[ fn(a='x]}y // not a comment') ]}
}"#;
assert_eq!(strip_json_comments(input), input);
}
#[test]
fn test_escaped_quote_inside_template_tag() {
let input = r#"{
"foo": ${[ fn(a='it\'s ]} // still fine') ]}
}"#;
assert_eq!(strip_json_comments(input), input);
}
#[test]
fn test_unterminated_quote_closes_tag_at_first_bracket() {
// A half-typed string must not swallow the rest of the document
assert_eq!(
strip_json_comments(r#"{"foo": ${[ fn(a='oops ]}, "bar": 1 // gone
}"#),
r#"{"foo": ${[ fn(a='oops ]}, "bar": 1
}"#
);
}
#[test]
fn test_multiple_comments() {
assert_eq!(
+114
View File
@@ -0,0 +1,114 @@
use std::iter::Peekable;
/// Reads the single-quoted strings inside template tags, moving left to right through one text.
///
/// Kept as a struct rather than a free function because it remembers one thing between calls;
/// see [`TagStrings::take`].
#[derive(Default)]
pub(crate) struct TagStrings {
/// Set once a scan has run off the end of the text, after which no string can close.
none_left: bool,
}
impl TagStrings {
/// Consumes the body of a single-quoted string, assuming the opening quote has already been
/// read, and returns it (including the closing quote).
///
/// Returns `None` when the quote is never closed, leaving the iterator untouched. The parser
/// and the editor's Lezer grammar both only recognize a terminated `'…'`, so an unterminated
/// quote has to stay an ordinary character and let the tag close at the next `]}`.
///
/// Scanning forward from every quote would be quadratic on text full of unpaired quotes, so
/// a failed scan is remembered: once one runs off the end, no later quote can open a
/// terminated string either. Any unescaped quote after it would have closed the failed scan,
/// so every remaining quote must be one the failed scan consumed as an escaped character —
/// and a scan starting there continues from the same place, in the same state, to the same
/// end. With that, every character of the text is read a bounded number of times.
pub(crate) fn take<I>(&mut self, chars: &mut Peekable<I>) -> Option<String>
where
I: Iterator<Item = char> + Clone,
{
if self.none_left {
return None;
}
let mut lookahead = chars.clone();
let mut consumed = String::new();
while let Some(c) = lookahead.next() {
consumed.push(c);
match c {
// A backslash escapes whatever follows it, including a quote
'\\' => match lookahead.next() {
Some(escaped) => consumed.push(escaped),
None => break,
},
'\'' => {
*chars = lookahead;
return Some(consumed);
}
_ => {}
}
}
self.none_left = true;
None
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::{Duration, Instant};
fn take(text: &str) -> (Option<String>, String) {
let mut chars = text.chars().peekable();
let taken = TagStrings::default().take(&mut chars);
(taken, chars.collect())
}
#[test]
fn takes_a_terminated_string() {
assert_eq!(take("abc' rest"), (Some("abc'".to_string()), " rest".to_string()));
}
#[test]
fn takes_a_string_containing_a_tag_close() {
assert_eq!(take("a]}b') ]}"), (Some("a]}b'".to_string()), ") ]}".to_string()));
}
#[test]
fn takes_a_string_with_an_escaped_quote() {
assert_eq!(take(r"it\'s' rest"), (Some(r"it\'s'".to_string()), " rest".to_string()));
}
#[test]
fn leaves_an_unterminated_string_alone() {
assert_eq!(take("oops ]}"), (None, "oops ]}".to_string()));
}
#[test]
fn leaves_a_trailing_backslash_alone() {
assert_eq!(take(r"oops\"), (None, r"oops\".to_string()));
}
#[test]
fn stays_linear_on_many_unpaired_quotes() {
// Every quote here is escaped, so none of them can ever close a string. Rescanning to
// the end from each one would be quadratic, so this should finish in no time.
let text = r"\'".repeat(20_000);
let mut strings = TagStrings::default();
let mut chars = text.chars().peekable();
let started = Instant::now();
let mut quotes = 0;
while let Some(c) = chars.next() {
if c == '\'' {
quotes += 1;
assert_eq!(strings.take(&mut chars), None);
}
}
assert_eq!(quotes, 20_000);
assert!(started.elapsed() < Duration::from_secs(1), "took {:?}", started.elapsed());
}
}
+107 -2
View File
@@ -1,4 +1,5 @@
use crate::Result;
use crate::import_templates::remap_request_references;
use chrono::Utc;
use log::info;
use serde_json::Value;
@@ -312,11 +313,14 @@ pub fn plan_import_resources(
})
.collect();
let mut request_ids = BTreeMap::new();
let http_requests = resources
.http_requests
.into_iter()
.map(|mut request| {
request.id = HttpRequest::generate_id();
let new_id = HttpRequest::generate_id();
request_ids.insert(request.id.clone(), new_id.clone());
request.id = new_id;
request.workspace_id = resolve_workspace_id(&request.workspace_id);
request.folder_id = resolve_folder_id(request.folder_id);
request
@@ -428,7 +432,7 @@ pub fn plan_import_resources(
));
}
let resources = BatchUpsertResult {
let mut resources = BatchUpsertResult {
workspaces,
environments,
folders,
@@ -436,6 +440,7 @@ pub fn plan_import_resources(
grpc_requests,
websocket_requests,
};
remap_request_references(&mut resources, &request_ids)?;
let mut plan = ImportPlan {
importer,
@@ -961,6 +966,13 @@ fn merge_with_linked_source(
}
}
let request_remap = plan
.resources
.http_requests
.iter()
.filter_map(|v| remap.get(&v.id).map(|id| (v.id.clone(), id.clone())))
.collect();
remap_request_references(&mut plan.resources, &request_remap)?;
let remap_ref = |id: Option<String>| id.map(|v| remap.get(&v).cloned().unwrap_or(v));
for v in &mut plan.resources.folders {
if let Some(existing_id) = remap.get(&v.id) {
@@ -2600,6 +2612,99 @@ mod tests {
])
}
#[tokio::test]
async fn insomnia_plugin_output_produces_sendable_graphql_and_linked_requests() {
// The JS fixture suite asserts this exact payload against the real importer.
let imported: yaak_plugins::events::ImportResponse = serde_json::from_str(include_str!(
"../../../plugins/importer-insomnia/tests/fixtures/chained.output.json"
))
.expect("parse plugin output");
let (query_manager, _blob_manager, _rx) =
yaak_models::init_in_memory().expect("initialize database");
let plan = plan_import_resources(
&query_manager,
"Insomnia".into(),
ImportDestination::NewWorkspace,
imported.resources,
imported.source_keys,
None,
)
.expect("plan Insomnia import");
let committed = commit_import_plan(&query_manager, plan).expect("commit Insomnia import");
let query = &committed.http_requests[0];
let login = &committed.http_requests[1];
assert!(query.authentication["token"].as_str().unwrap().contains(&login.id));
assert!(!query.authentication["token"].as_str().unwrap().contains("GENERATE_ID"));
let sendable =
yaak_http::types::SendableHttpRequest::from_http_request(query, Default::default())
.await
.unwrap();
let Some(yaak_http::types::SendableBody::Bytes(body)) = sendable.body else {
panic!("expected GraphQL JSON body");
};
assert_eq!(
serde_json::from_slice::<Value>(&body).unwrap(),
json!({
"query": "query Get($id: ID!) { node(id: $id) { id } }",
"variables": { "id": "123" },
"operationName": "Get",
})
);
assert!(
sendable.headers.iter().any(|(name, value)| name.eq_ignore_ascii_case("content-type")
&& value == "application/json")
);
}
#[test]
fn request_references_follow_import_and_reimport_ids_without_touching_literals() {
let (query_manager, _blob_manager, _rx) =
yaak_models::init_in_memory().expect("initialize database");
let mut resources = imported_resources();
let template =
"${[ response.body.path(request='rq_root', path='$.token', result='first') ]}";
resources.http_requests[1].authentication =
BTreeMap::from([("token".into(), json!(template))]);
resources.http_requests[1].body =
BTreeMap::from([("text".into(), json!("rq_root is literal text"))]);
resources.environments[0].variables[0].value = template.into();
resources.folders[0].authentication = BTreeMap::from([("token".into(), json!(template))]);
resources.workspaces[0].authentication =
BTreeMap::from([("token".into(), json!(template))]);
let first = first_import_with(&query_manager, resources.clone());
let workspace_id = &first.workspaces[0].id;
let first_id = &first.http_requests[0].id;
let expected = format!(
"${{[ response.body.path(request='{first_id}', path='$.token', result='first') ]}}"
);
assert_eq!(first.http_requests[1].authentication["token"], json!(expected));
assert_eq!(first.environments[0].variables[0].value, expected);
assert_eq!(first.folders[0].authentication["token"], json!(expected));
assert_eq!(first.workspaces[0].authentication["token"], json!(expected));
assert_eq!(first.http_requests[1].body["text"], json!("rq_root is literal text"));
// Reimport allocates temporary IDs, then reuses the previously linked IDs. Neither
// phase may leave a reference behind, or spuriously flag a source/local conflict.
let plan = replan(&query_manager, workspace_id, resources.clone());
assert_eq!(&plan.resources.http_requests[0].id, first_id);
assert_eq!(plan.resources.http_requests[1].authentication["token"], json!(expected));
assert!(plan.items.iter().all(|item| item.action == ImportPlanAction::Unchanged));
commit_import_plan(&query_manager, plan).expect("commit reimport");
assert_eq!(query_manager.connect().list_http_requests(workspace_id).unwrap().len(), 2);
// A separate import must point at its own new request, not at the first workspace.
let second = first_import_with(&query_manager, resources);
let second_id = &second.http_requests[0].id;
assert_ne!(first_id, second_id);
assert!(
second.http_requests[1].authentication["token"].as_str().unwrap().contains(second_id)
);
assert!(
!second.http_requests[1].authentication["token"].as_str().unwrap().contains(first_id)
);
}
fn first_import(query_manager: &QueryManager) -> BatchUpsertResult {
first_import_with(query_manager, imported_resources())
}
+172
View File
@@ -0,0 +1,172 @@
use serde_json::Value;
use std::collections::BTreeMap;
use yaak_models::util::BatchUpsertResult;
use yaak_templates::{Parser, Token, Val};
/// Rewrite request references at each ID-assignment step (initial import and linked reimport).
/// Literal text, missing targets and arguments to unrelated functions must not be rewritten.
pub(crate) fn remap_request_references(
resources: &mut BatchUpsertResult,
ids: &BTreeMap<String, String>,
) -> serde_json::Result<()> {
if ids.is_empty() {
return Ok(());
}
let mut value = serde_json::to_value(&resources)?;
remap_value(&mut value, ids);
*resources = serde_json::from_value(value)?;
Ok(())
}
fn remap_value(value: &mut Value, ids: &BTreeMap<String, String>) {
match value {
Value::String(text) => *text = remap_text(text, ids),
Value::Array(values) => values.iter_mut().for_each(|v| remap_value(v, ids)),
Value::Object(values) => values.values_mut().for_each(|v| remap_value(v, ids)),
_ => {}
}
}
fn remap_argument(value: &mut Val, ids: &BTreeMap<String, String>) -> bool {
let Val::Fn { name, args } = value else {
return false;
};
let request_arg = match name.as_str() {
"response" | "response.body.raw" | "response.body.path" | "response.header" => "request",
"request.body" | "request.body.raw" | "request.body.path" | "request.header"
| "request.param" | "request.name" => "requestId",
_ => "",
};
let mut changed = false;
for arg in args {
if arg.name == request_arg
&& let Val::Str { text } = &mut arg.value
&& let Some(id) = ids.get(text)
&& id != text
{
*text = id.clone();
changed = true;
}
changed |= remap_argument(&mut arg.value, ids);
}
changed
}
fn remap_text(text: &str, ids: &BTreeMap<String, String>) -> String {
// Parse individual unescaped tags. Parsing/reprinting the whole string would unescape raw
// template-looking text, and change formatting even when no reference needs updating.
let bytes = text.as_bytes();
let mut output = String::new();
let mut copied = 0;
let mut cursor = 0;
while let Some(relative) = text[cursor..].find("${[") {
let start = cursor + relative;
cursor = start + 3;
let escapes = bytes[..start].iter().rev().take_while(|&&b| b == b'\\').count();
if escapes % 2 != 0 {
continue;
}
let mut end = cursor;
let mut quoted = false;
while end < bytes.len() {
if quoted && bytes[end] == b'\\' {
end += 2;
continue;
}
if bytes[end] == b'\'' {
quoted = !quoted;
}
if !quoted && bytes[end..].starts_with(b"]}") {
end += 2;
break;
}
end += 1;
}
if end > bytes.len() || !text[start..end].ends_with("]}") {
continue;
}
if let Ok(mut tokens) = Parser::new(&text[start..end]).parse()
&& let [Token::Tag { val }, Token::Eof] = tokens.tokens.as_mut_slice()
&& remap_argument(val, ids)
{
output.push_str(&text[copied..start]);
output.push_str(&tokens.to_string());
copied = end;
}
cursor = end;
}
output.push_str(&text[copied..]);
output
}
#[cfg(test)]
mod tests {
use super::*;
fn ids() -> BTreeMap<String, String> {
BTreeMap::from([("req_old".into(), "rq_new".into())])
}
#[test]
fn only_remaps_known_literal_request_arguments() {
let text = "${[ response.header(request='req_old', header='req_old') ]}";
assert_eq!(
remap_text(text, &ids()),
"${[ response.header(request='rq_new', header='req_old') ]}"
);
for text in [
"req_old",
"${[ custom(request='req_old') ]}",
"${[ response.header(request=req_old) ]}",
"${[ response.header(request='req_missing') ]}",
"${[ response.header(request='req_old' ]}",
] {
assert_eq!(remap_text(text, &ids()), text);
}
}
#[test]
fn handles_encoded_nested_and_escaped_tags_without_changing_surrounding_text() {
let text = r#"café \${[ response.header(request='req_old') ]} ${[ json.jsonpath(input=response.body.raw(request=b64'cmVxX29sZA')) ]} end"#;
assert_eq!(
remap_text(text, &ids()),
r#"café \${[ response.header(request='req_old') ]} ${[ json.jsonpath(input=response.body.raw(request='rq_new')) ]} end"#
);
let text = "${[ response.header(request='req_old', header=']}' ) ]}";
assert!(remap_text(text, &ids()).contains("request='rq_new'"));
}
/// Only the request ID may change. Every other argument keeps its name, order, spacing and
/// plain single-quoted form, so a printer that falls back to base64 for values like
/// `X-Api-Key` or `$.token` fails here.
#[test]
fn remapping_a_request_keeps_the_other_arguments_verbatim() {
assert_eq!(
remap_text("${[ response.header(request='req_old', header='X-Api-Key') ]}", &ids()),
"${[ response.header(request='rq_new', header='X-Api-Key') ]}"
);
assert_eq!(
remap_text(
"${[ response.body.path(request='req_old', path='$.token', behavior='never') ]}",
&ids()
),
"${[ response.body.path(request='rq_new', path='$.token', behavior='never') ]}"
);
}
/// An Insomnia tag that never got converted is plain text, even though it names a request
/// that is being remapped. It must survive byte-for-byte, including alongside a real tag.
#[test]
fn unconverted_nunjucks_text_containing_a_request_id_is_untouched() {
let text = "{% response 'raw', 'req_old', '', 'never', 60 %}";
assert_eq!(remap_text(text, &ids()), text);
assert_eq!(
remap_text(
"${[ response.body.path(request='req_old', path='$.a') ]} and {% response 'raw', 'req_old' %}",
&ids()
),
"${[ response.body.path(request='rq_new', path='$.a') ]} and {% response 'raw', 'req_old' %}"
);
}
}
+1
View File
@@ -1,5 +1,6 @@
pub mod error;
pub mod import;
mod import_templates;
pub mod plugin_events;
pub mod response_body;
pub mod send;
+1 -1
View File
@@ -1 +1 @@
24.11.1
24.22.2
+136 -19
View File
@@ -34,13 +34,15 @@ export function importHttpBodyAndHeaders(obj: any) {
const { headers } = importHeaders(obj);
const { body, bodyType } = importHttpBody(obj.body);
const mimeType = typeof obj.body?.mimeType === "string" ? obj.body.mimeType.trim() : "";
// Insomnia uses application/graphql as an editor marker, but sends a JSON envelope.
const contentType = bodyType === "graphql" ? "application/json" : mimeType;
if (
bodyType != null &&
mimeType !== "" &&
!headers.some((header: { name: string }) => header.name.toLowerCase() === "content-type")
) {
headers.push({ enabled: true, name: "Content-Type", value: mimeType });
headers.push({ enabled: true, name: "Content-Type", value: contentType });
}
return { body, bodyType, headers };
@@ -93,7 +95,36 @@ function importHttpBody(rawBody: any) {
}
if (normalizedMimeType === "application/graphql") {
return { bodyType: "graphql", body: { text: rawBody.text ?? "" } };
const text = typeof rawBody.text === "string" ? rawBody.text : "";
try {
const parsed = JSON.parse(text);
if (typeof parsed?.query === "string") {
return {
bodyType: "graphql",
body: {
query: parsed.query,
variables:
typeof parsed.variables === "string"
? parsed.variables
: parsed.variables == null
? ""
: JSON.stringify(parsed.variables, null, 2),
...(typeof parsed.operationName === "string"
? { operationName: parsed.operationName }
: {}),
},
};
}
} catch {
// A raw GraphQL document is also valid; preserve it verbatim. An envelope that
// failed to parse is usually one with template tags in it, so recover its parts
// rather than dropping the whole envelope into the query editor.
const recovered = recoverGraphQLEnvelope(text);
if (recovered != null) {
return { bodyType: "graphql", body: recovered };
}
}
return { bodyType: "graphql", body: { query: text, variables: "" } };
}
if (normalizedMimeType === "application/json" || normalizedMimeType.endsWith("+json")) {
@@ -115,6 +146,109 @@ function importHttpBody(rawBody: any) {
return { bodyType: null, body: {} };
}
/**
* Pull the parts out of a GraphQL envelope that isn't valid JSON.
*
* Insomnia lets template tags be written straight into the envelope
* ({"query":"{me{id}}","variables":{"id":{{ _.id }}}}), which makes the text
* unparseable. Members are read as raw text so the tags survive the trip.
*/
function recoverGraphQLEnvelope(
text: string,
): { query: string; variables: string; operationName?: string } | null {
const members = readTopLevelMembers(text);
if (members == null) return null;
const query = parseJSONString(members.get("query"));
if (query == null) return null;
const variables = members.get("variables") ?? "";
const operationName = parseJSONString(members.get("operationName"));
return {
query,
variables: variables === "null" ? "" : variables,
...(operationName == null ? {} : { operationName }),
};
}
function parseJSONString(raw: string | undefined): string | null {
if (raw == null || !raw.startsWith('"')) return null;
try {
const parsed = JSON.parse(raw);
return typeof parsed === "string" ? parsed : null;
} catch {
return null;
}
}
/**
* The raw text of each member of the outermost object, without parsing the values.
* Returns null unless the whole object is well-formed enough to walk.
*/
function readTopLevelMembers(text: string): Map<string, string> | null {
const trimmed = text.trim();
if (!trimmed.startsWith("{")) return null;
const members = new Map<string, string>();
let offset = 1;
while (offset < trimmed.length) {
while (/[\s,]/.test(trimmed[offset] ?? "")) offset++;
if (trimmed[offset] === "}") return members;
const keyEnd = scanJSONString(trimmed, offset);
if (keyEnd == null) return null;
const key = parseJSONString(trimmed.slice(offset, keyEnd));
if (key == null) return null;
offset = keyEnd;
while (/\s/.test(trimmed[offset] ?? "")) offset++;
if (trimmed[offset] !== ":") return null;
offset++;
while (/\s/.test(trimmed[offset] ?? "")) offset++;
const valueEnd = scanJSONValue(trimmed, offset);
if (valueEnd == null) return null;
members.set(key, trimmed.slice(offset, valueEnd).trim());
offset = valueEnd;
}
return null;
}
/** Offset just past the string starting at `start`, or null if it never closes. */
function scanJSONString(text: string, start: number): number | null {
if (text[start] !== '"') return null;
for (let i = start + 1; i < text.length; i++) {
if (text[i] === "\\") i++;
else if (text[i] === '"') return i + 1;
}
return null;
}
/**
* Offset just past the value starting at `start`. Nesting is tracked so template
* tags come back whole, but the value itself is not validated.
*/
function scanJSONValue(text: string, start: number): number | null {
let depth = 0;
for (let i = start; i < text.length; i++) {
const char = text[i]!;
if (char === '"') {
const end = scanJSONString(text, i);
if (end == null) return null;
i = end - 1;
} else if (char === "{" || char === "[") {
depth++;
} else if (char === "}" || char === "]") {
if (depth === 0) return i; // The member ended with the enclosing object
depth--;
if (depth === 0) return i + 1;
} else if (depth === 0 && char === ",") {
return i;
}
}
return null;
}
export function deleteUndefinedAttrs<T>(obj: T): T {
if (Array.isArray(obj) && obj != null) {
return obj.map(deleteUndefinedAttrs) as T;
@@ -128,20 +262,3 @@ export function deleteUndefinedAttrs<T>(obj: T): T {
}
return obj;
}
/** Recursively render all nested object properties */
export function convertTemplateSyntax<T>(obj: T): T {
if (typeof obj === "string") {
// oxlint-disable-next-line no-template-curly-in-string -- Yaak template syntax
return obj.replaceAll(/{{\s*(_\.)?([^}]+)\s*}}/g, "${[$2]}") as T;
}
if (Array.isArray(obj) && obj != null) {
return obj.map(convertTemplateSyntax) as T;
}
if (typeof obj === "object" && obj != null) {
return Object.fromEntries(
Object.entries(obj).map(([k, v]) => [k, convertTemplateSyntax(v)]),
) as T;
}
return obj;
}
+254
View File
@@ -0,0 +1,254 @@
import { Buffer } from "node:buffer";
import { convertId } from "./common";
// Convert only syntax with a native equivalent. Leave filters, expressions and third-party
// tags intact rather than importing something that looks valid but means something different.
const identifier = /^[\p{L}\p{N}_][\p{L}\p{N}_.-]*$/u;
function argument(value: string): string {
return `b64'${Buffer.from(value).toString("base64url")}'`;
}
function decodeArgument(value: string): string | null {
if (!value.startsWith("b64::")) return value;
const match = /^b64::([A-Za-z0-9+/_-]*={0,2})(?:::[^:]*)?$/.exec(value);
if (!match) return null;
const bytes = Buffer.from(match[1]!, "base64");
const normalized = match[1]!.replace(/=+$/, "").replaceAll("+", "-").replaceAll("/", "_");
if (bytes.toString("base64url") !== normalized) return null;
const decoded = bytes.toString("utf8");
return Buffer.from(decoded).equals(bytes) ? decoded : null;
}
/** Literal arguments only; never evaluate Nunjucks or silently skip unexpected tokens. */
function parseArgs(input: string): string[] | null {
const args: string[] = [];
const token =
/\s*(?:'((?:[^'\\]|\\['"\\nrt])*)'|"((?:[^"\\]|\\['"\\nrt])*)"|(-?\d+(?:\.\d+)?)|(true|false))\s*/y;
let offset = 0;
while (offset < input.length) {
token.lastIndex = offset;
const match = token.exec(input);
if (!match) return input.trim() === "" ? [] : null;
const literal = (match[1] ?? match[2] ?? match[3] ?? match[4]!).replace(
/\\(['"\\nrt])/g,
(_, char: string) => ({ n: "\n", r: "\r", t: "\t" })[char] ?? char,
);
const decoded = decodeArgument(literal);
if (decoded == null) return null;
args.push(decoded);
offset = token.lastIndex;
if (offset === input.length) break;
if (input[offset] !== ",") return null;
offset++;
if (input.slice(offset).trim() === "") return null;
}
return args;
}
/**
* Insomnia's dropdown arguments fall back to the first option of their list when a tag leaves
* them out, so every converter below mirrors that first option as its default.
*/
function convertUuid(args: string[]): string | null {
// Version 4 is listed first, so a bare tag generates one.
if (args.length === 0) return "uuid.v4()";
if (args.length === 1 && ["v1", "v4"].includes(args[0]!)) return `uuid.${args[0]}()`;
return null;
}
function convertNow(args: string[]): string | null {
if (args.length > 2) return null;
const [kind = "iso-8601", format = ""] = args;
switch (kind.toLowerCase()) {
case "iso-8601":
return "timestamp.iso8601()";
case "millis":
case "ms":
return "timestamp.unixMillis()";
case "unix":
case "seconds":
case "s":
return "timestamp.unix()";
case "custom":
// The format string is passed through untranslated. Insomnia formats with date-fns
// today, like Yaak does, but older versions used Moment and those token sets differ
// (eg. Moment's DD/YYYY against date-fns' dd/yyyy). A token that no longer means the
// same thing shows up as a tag the user can correct.
return `timestamp.format(format=${argument(format)})`;
default:
return null;
}
}
/** The legacy timestamp tag takes no arguments and returns milliseconds. */
function convertTimestamp(args: string[]): string | null {
return args.length === 0 ? "timestamp.unixMillis()" : null;
}
const base64Encodings: Record<string, string> = {
normal: "base64",
url: "base64url",
// Insomnia's hex mode reads/writes hex rather than text. Yaak has no such mode, so this
// falls back to plain base64, which at least imports as a tag instead of raw text.
hex: "base64",
};
function convertBase64(args: string[]): string | null {
if (args.length < 2 || args.length > 3) return null;
// Assumed: exports written before the Kind argument existed pass (action, value), so a
// lone second argument that names no kind is the value.
const legacy = args.length === 2 && !Object.hasOwn(base64Encodings, args[1]!);
const encoding = base64Encodings[legacy ? "normal" : args[1]!];
const value = legacy ? args[1]! : (args[2] ?? "");
if (encoding == null) return null;
if (args[0] === "encode") {
return `base64.encode(encoding='${encoding}', value=${argument(value)})`;
}
// Yaak's decoder reads both alphabets, so the kind doesn't change the call.
if (args[0] === "decode") return `base64.decode(value=${argument(value)})`;
return null;
}
const hashAlgorithms = ["md5", "sha1", "sha256", "sha512"];
function convertHash(args: string[]): string | null {
if (args.length > 3) return null;
const [algorithm = "md5", encoding = "hex", value = ""] = args;
// Insomnia's own tag falls back to SHA-256 for algorithms outside its list.
const name = hashAlgorithms.includes(algorithm.toLowerCase())
? algorithm.toLowerCase()
: "sha256";
// Insomnia offers only hex and base64, the same two digests Yaak does.
const digest = encoding === "base64" ? "base64" : "hex";
return `hash.${name}(input=${argument(value)}, encoding='${digest}')`;
}
function convertCookie(args: string[]): string | null {
if (args.length !== 2) return null;
const [url = "", name = ""] = args;
if (name === "") return null;
const domain = cookieDomain(url);
const filter = domain === "" ? "" : `, domain=${argument(domain)}`;
return `cookie.value(name=${argument(name)}${filter})`;
}
/** Insomnia matches cookies against a full URL, while Yaak filters the jar by domain. */
function cookieDomain(url: string): string {
try {
return new URL(url).hostname;
} catch {
// Not a URL we can read, usually because it holds a template tag. Keep the text so the
// user can see and fix it rather than dropping the filter.
return url.trim();
}
}
function convertPrompt(args: string[]): string | null {
if (args.length > 6) return null;
const [title = "", label = "", defaultValue = "", storageKey = "", mask = "false"] = args;
// Insomnia requires a title and shows the label above the input, falling back to the title.
if (title === "" && label === "") return null;
const parts = [`label=${argument(label === "" ? title : label)}`];
const masked = mask === "true";
if (storageKey !== "" && !masked) {
// Insomnia keeps a stored value only until the app closes. Yaak's nearest option is to
// store it forever, which is close enough for plain values. Masked values are left
// unstored, which is Yaak's default, so a password never outlives the session it was
// typed in. Storing needs a namespace, and the workspace is what Yaak's editor defaults to.
// oxlint-disable-next-line no-template-curly-in-string -- Yaak template syntax
const namespace = argument("${[ctx.workspace()]}");
parts.push("store='forever'", `namespace=${namespace}`, `key=${argument(storageKey)}`);
}
if (title !== "") parts.push(`title=${argument(title)}`);
if (defaultValue !== "") parts.push(`defaultValue=${argument(defaultValue)}`);
if (masked) parts.push("password=true");
return `prompt.text(${parts.join(", ")})`;
}
function convertResponse(args: string[], requestIds: Set<string>): string | null {
if (args.length < 2 || args.length > 5) return null;
const [attribute, request, filter = "", behavior = "never", maxAge = "0"] = args;
if (!request || !requestIds.has(convertId(request))) return null;
const behaviors: Record<string, string> = {
never: "never",
"no-history": "smart",
always: "always",
"when-expired": "ttl",
};
const convertedBehavior = Object.hasOwn(behaviors, behavior) ? behaviors[behavior] : null;
if (!convertedBehavior || !/^\d+$/.test(maxAge)) return null;
// Insomnia treats zero as immediately expired; Yaak uses zero for never expiring.
if (convertedBehavior === "ttl" && Number(maxAge) === 0) return null;
const common = `request=${argument(convertId(request))}, behavior='${convertedBehavior}', ttl='${maxAge}'`;
if (attribute === "raw") {
return `response.body.raw(${common})`;
}
if (attribute === "body") {
// Insomnia returns a scalar for one match, but an array for several. Yaak's first/all
// modes cannot express that automatically. Only convert definite paths for now.
if (!/^\$(?:\.[\p{L}\p{N}_-]+|\[(?:0|[1-9]\d*)\])*$/u.test(filter.trim())) return null;
return `response.body.path(${common}, path=${argument(filter.trim())}, result='first')`;
}
if (attribute === "header" && filter !== "") {
return `response.header(${common}, header=${argument(filter.trim())})`;
}
// Insomnia's url attribute has no counterpart, since Yaak has no response URL function.
return null;
}
function convertTag(name: string, input: string, requestIds: Set<string>): string | null {
const args = parseArgs(input);
if (args == null) return null;
switch (name) {
case "base64":
return convertBase64(args);
case "cookie":
return convertCookie(args);
case "faker":
return args.length === 1 && args[0] === "randomUUID" ? "uuid.v4()" : null;
case "hash":
return convertHash(args);
case "now":
return convertNow(args);
case "prompt":
return convertPrompt(args);
case "response":
return convertResponse(args, requestIds);
case "timestamp":
return convertTimestamp(args);
case "uuid":
return convertUuid(args);
default:
return null;
}
}
export function convertTemplateSyntax<T>(obj: T, requestIds = new Set<string>()): T {
if (typeof obj === "string") {
return obj
.replace(/{{\s*([\s\S]*?)\s*}}/g, (full, expression: string) => {
const trimmed = expression.trim();
const bracket = /^_\s*\[\s*(['"])([^'"\\]+)\1\s*\]$/.exec(trimmed);
const name = bracket?.[2] ?? trimmed.replace(/^_\./, "");
return identifier.test(name) && !["true", "false", "null"].includes(name)
? `\${[ ${name} ]}`
: full;
})
.replace(
/{%\s*(\w+)\s*((?:'(?:[^'\\]|\\.)*'|"(?:[^"\\]|\\.)*"|[^'"])*?)%}/g,
(full, name: string, input: string) => {
const converted = convertTag(name, input, requestIds);
return converted == null ? full : `\${[ ${converted} ]}`;
},
) as T;
}
if (Array.isArray(obj)) return obj.map((value) => convertTemplateSyntax(value, requestIds)) as T;
if (typeof obj === "object" && obj != null) {
return Object.fromEntries(
Object.entries(obj).map(([key, value]) => [key, convertTemplateSyntax(value, requestIds)]),
) as T;
}
return obj;
}
+7 -4
View File
@@ -2,12 +2,12 @@
import type { PartialImportResources } from "@yaakapp/api";
import {
convertId,
convertTemplateSyntax,
createSourceKeys,
importHttpBodyAndHeaders,
isJSObject,
type SourceKeys,
} from "./common";
import { convertTemplateSyntax } from "./templates";
export function convertInsomniaV4(parsed: any) {
if (!Array.isArray(parsed.resources)) return null;
@@ -68,7 +68,10 @@ export function convertInsomniaV4(parsed: any) {
resources.environments = resources.environments.filter(Boolean);
resources.workspaces = resources.workspaces.filter(Boolean);
return { resources: convertTemplateSyntax(resources), sourceKeys: keys.all() };
return {
resources: convertTemplateSyntax(resources, new Set(resources.httpRequests.map((r) => r.id))),
sourceKeys: keys.all(),
};
}
function importHttpRequest(
@@ -78,12 +81,12 @@ function importHttpRequest(
): PartialImportResources["httpRequests"][0] {
let authenticationType: string | null = null;
let authentication = {};
if (r.authentication.type === "bearer") {
if (r.authentication?.type === "bearer") {
authenticationType = "bearer";
authentication = {
token: r.authentication.token,
};
} else if (r.authentication.type === "basic") {
} else if (r.authentication?.type === "basic") {
authenticationType = "basic";
authentication = {
username: r.authentication.username,
+5 -2
View File
@@ -2,13 +2,13 @@
import type { PartialImportResources } from "@yaakapp/api";
import {
convertId,
convertTemplateSyntax,
createSourceKeys,
importHeaders,
importHttpBodyAndHeaders,
isJSObject,
type SourceKeys,
} from "./common";
import { convertTemplateSyntax } from "./templates";
export function convertInsomniaV5(parsed: any) {
// Assert parsed is object
@@ -80,7 +80,10 @@ export function convertInsomniaV5(parsed: any) {
resources.environments = resources.environments.filter(Boolean);
resources.workspaces = resources.workspaces.filter(Boolean);
return { resources: convertTemplateSyntax(resources), sourceKeys: keys.all() };
return {
resources: convertTemplateSyntax(resources, new Set(resources.httpRequests.map((r) => r.id))),
sourceKeys: keys.all(),
};
}
function importHttpRequest(
@@ -2,6 +2,78 @@ import { describe, expect, test } from "vite-plus/test";
import { importHttpBodyAndHeaders } from "../src/common";
describe("importHttpBodyAndHeaders", () => {
test.each([{ id: "123" }, '{"id":"123"}', '{"id":{{ _.id }}}'])(
"imports GraphQL variables into the sendable body shape: %j",
(variables) => {
const result = importHttpBodyAndHeaders({
body: {
mimeType: "application/graphql",
text: JSON.stringify({
query: "query Get($id: ID!) { node(id: $id) { id } }",
variables,
operationName: "Get",
}),
},
});
expect(result.bodyType).toBe("graphql");
expect(result.body).toEqual({
query: "query Get($id: ID!) { node(id: $id) { id } }",
variables: typeof variables === "string" ? variables : JSON.stringify(variables, null, 2),
operationName: "Get",
});
expect(result.headers).toEqual([
{ name: "Content-Type", value: "application/json", enabled: true },
]);
},
);
test.each(["query Get { viewer { id } }", "", "{ malformed"])(
"preserves raw or malformed GraphQL text rather than discarding it: %s",
(query) => {
expect(
importHttpBodyAndHeaders({ body: { mimeType: "application/graphql", text: query } }).body,
).toEqual({ query, variables: "" });
},
);
test("preserves GraphQL operation and defaults absent variables", () => {
expect(
importHttpBodyAndHeaders({
body: { mimeType: "application/graphql; charset=utf-8", text: '{"query":"{ me { id } }"}' },
}).body,
).toEqual({ query: "{ me { id } }", variables: "" });
});
test("recovers a GraphQL envelope holding template tags that break JSON parsing", () => {
expect(
importHttpBodyAndHeaders({
body: {
mimeType: "application/graphql",
text: '{"query":"{me{id}}","operationName":"Get","variables":{"id":{{ _.id }}}}',
},
}).body,
).toEqual({
query: "{me{id}}",
variables: '{"id":{{ _.id }}}',
operationName: "Get",
});
});
test("defaults variables to empty when an unparseable envelope has none", () => {
expect(
importHttpBodyAndHeaders({
body: { mimeType: "application/graphql", text: '{"query":"{me{id}}","meta":{{ _.meta }}}' },
}).body,
).toEqual({ query: "{me{id}}", variables: "" });
});
test("keeps an envelope-looking document in the query when no query member is readable", () => {
const text = "{ query { me { id } } }";
expect(
importHttpBodyAndHeaders({ body: { mimeType: "application/graphql", text } }).body,
).toEqual({ query: text, variables: "" });
});
test("imports XML text using the native XML body type", () => {
const result = importHttpBodyAndHeaders({
body: {
+1 -1
View File
@@ -112,7 +112,7 @@
"method": "GET",
"model": "http_request",
"name": "New Request",
"url": "${[BASE_URL ]}/foo/:id",
"url": "${[ BASE_URL ]}/foo/:id",
"urlParameters": [
{
"name": "query",
@@ -0,0 +1,29 @@
# Synthetic regression based on the migration cases described by
# https://github.com/w33ble/insomnia-yaak-converter (no customer data).
type: collection.insomnia.rest/5.0
name: Chained GraphQL
meta:
id: wrk_chained
environments:
name: Base
meta:
id: env_base
data:
host: https://example.com
collection:
- name: GraphQL
meta:
id: req_query
method: POST
url: "{{ _['host'] }}/graphql"
authentication:
type: bearer
token: "{% response 'body', 'req_login', 'b64::JC50b2tlbg==::nonce', 'when-expired', 60 %}"
body:
mimeType: application/graphql
text: '{"query":"query Get($id: ID!) { node(id: $id) { id } }","variables":"{\"id\":\"123\"}","operationName":"Get"}'
- name: Login
meta:
id: req_login
method: POST
url: "{{ _.host }}/login"
@@ -0,0 +1,85 @@
{
"resources": {
"environments": [
{
"id": "GENERATE_ID::env_base",
"workspaceId": "GENERATE_ID::wrk_chained",
"public": true,
"parentModel": "workspace",
"parentId": null,
"model": "environment",
"name": "Base",
"variables": [
{
"enabled": true,
"name": "host",
"value": "https://example.com"
}
]
}
],
"folders": [],
"grpcRequests": [],
"httpRequests": [
{
"id": "GENERATE_ID::req_query",
"workspaceId": "GENERATE_ID::wrk_chained",
"folderId": null,
"model": "http_request",
"name": "GraphQL",
"url": "${[ host ]}/graphql",
"urlParameters": [],
"body": {
"query": "query Get($id: ID!) { node(id: $id) { id } }",
"variables": "{\"id\":\"123\"}",
"operationName": "Get"
},
"bodyType": "graphql",
"headers": [
{
"enabled": true,
"name": "Content-Type",
"value": "application/json"
}
],
"method": "POST",
"authenticationType": "bearer",
"authentication": {
"token": "${[ response.body.path(request=b64'R0VORVJBVEVfSUQ6OnJlcV9sb2dpbg', behavior='ttl', ttl='60', path=b64'JC50b2tlbg', result='first') ]}"
}
},
{
"id": "GENERATE_ID::req_login",
"workspaceId": "GENERATE_ID::wrk_chained",
"folderId": null,
"model": "http_request",
"name": "Login",
"url": "${[ host ]}/login",
"urlParameters": [],
"body": {},
"bodyType": null,
"headers": [],
"method": "POST",
"authenticationType": null,
"authentication": {}
}
],
"websocketRequests": [],
"workspaces": [
{
"id": "GENERATE_ID::wrk_chained",
"model": "workspace",
"name": "Chained GraphQL",
"headers": [],
"authenticationType": null,
"authentication": {}
}
]
},
"sourceKeys": {
"GENERATE_ID::wrk_chained": "wrk_chained",
"GENERATE_ID::env_base": "env_base",
"GENERATE_ID::req_query": "req_query",
"GENERATE_ID::req_login": "req_login"
}
}
@@ -130,8 +130,8 @@
},
{
"enabled": true,
"name": "${[ApiHeaderName ]}",
"value": "${[ApiKey ]}"
"name": "${[ ApiHeaderName ]}",
"value": "${[ ApiKey ]}"
}
],
"id": "GENERATE_ID::req_d72fff2a6b104b91a2ebe9de9edd2785",
@@ -139,7 +139,7 @@
"model": "http_request",
"name": "New Request",
"sortPriority": -1736781406672,
"url": "${[BASE_URL ]}/foo/:id",
"url": "${[ BASE_URL ]}/foo/:id",
"urlParameters": [
{
"name": "query",
@@ -0,0 +1,95 @@
import { Buffer } from "node:buffer";
import { describe, expect, test } from "vite-plus/test";
import YAML from "yaml";
import { convertInsomnia } from "../src";
describe.each([4, 5])("Insomnia v%i migration", (version) => {
test("imports GraphQL, credentials and forward response references together", () => {
const requests = [
{
_id: "req_query",
meta: { id: "req_query" },
_type: "request",
parentId: "wrk_test",
name: "Query",
method: "POST",
url: "{{ _['host'] }}/graphql",
authentication: {
type: "bearer",
token: "{% response 'body', 'req_login', 'b64::JC50b2tlbg==::abc', 'when-expired', 60 %}",
},
headers: [{ name: "X-Request-ID", value: "{% uuid 'v4' %}" }],
parameters: [{ name: "id", value: '{{ _["id"] }}' }],
body: {
mimeType: "application/graphql",
text: JSON.stringify({
query: "query Get($id: ID!) { node(id: $id) { id } }",
variables: '{"id":"{{ _.id }}"}',
operationName: "Get",
}),
},
},
{
_id: "req_login",
meta: { id: "req_login" },
_type: "request",
parentId: "wrk_test",
name: "Login",
method: "POST",
url: "{{ _.host }}/login",
// Missing authentication is normal in minimal exports.
},
];
const source =
version === 5
? YAML.stringify({
type: "collection.insomnia.rest/5.0",
name: "Migration",
meta: { id: "wrk_test" },
environments: {
meta: { id: "env_base" },
data: { host: "https://example.com", id: "123" },
},
collection: requests,
})
: JSON.stringify({
_type: "export",
__export_format: 4,
resources: [
{ _type: "workspace", _id: "wrk_test", name: "Migration" },
{
_type: "environment",
_id: "env_base",
parentId: "wrk_test",
data: { host: "https://example.com", id: "123" },
},
...requests,
],
});
const imported = convertInsomnia(source)!;
const [query, login] = imported.resources.httpRequests;
expect(query?.body).toEqual({
query: "query Get($id: ID!) { node(id: $id) { id } }",
variables: '{"id":"${[ id ]}"}',
operationName: "Get",
});
expect(query?.url).toBe("${[ host ]}/graphql");
expect(query?.urlParameters?.[0]?.value).toBe("${[ id ]}");
expect(query?.headers).toContainEqual({
name: "X-Request-ID",
value: "${[ uuid.v4() ]}",
enabled: true,
});
expect(query?.headers).toContainEqual({
name: "Content-Type",
value: "application/json",
enabled: true,
});
const ref = `b64'${Buffer.from(login!.id).toString("base64url")}'`;
expect(query?.authentication?.token).toBe(
`\${[ response.body.path(request=${ref}, behavior='ttl', ttl='60', path=b64'JC50b2tlbg', result='first') ]}`,
);
expect(imported.sourceKeys[login!.id]).toBe("req_login");
expect(imported.sourceKeys[query!.id]).toBe("req_query");
});
});
@@ -0,0 +1,231 @@
import { Buffer } from "node:buffer";
import { describe, expect, test } from "vite-plus/test";
import { convertTemplateSyntax } from "../src/templates";
const requestIds = new Set(["GENERATE_ID::req_login"]);
const convert = (text: string) => convertTemplateSyntax(text, requestIds);
const encoded = (text: string) => `b64'${Buffer.from(text).toString("base64url")}'`;
describe("Insomnia templates", () => {
test.each(["{{ _.token }}", "{{ token }}", "{{ _['token'] }}", '{{ _ [ "token" ] }}'])(
"converts %s",
(input) => expect(convert(input)).toBe("${[ token ]}"),
);
test.each([
"{{ _.token | default('fallback') }}",
"{{ _['two words'] }}",
"{{ _['a']['b'] }}",
"{{ _.a + _.b }}",
"{{ true }}",
"{% uuid 'v7' %}",
"{% faker 'name' %}",
"{% dopplersecret 'project', 'config', 'secret' %}",
"{% response 'body', 'req_missing', '$.token', 'always', 60 %}",
"{% response 'body', 'req_login', '$.token', 'unknown', 60 %}",
"{% response 'body', 'req_login', '$.token', 'always', -1 %}",
"{% response 'body' garbage, 'req_login', '$.token', 'always', 60 %}",
"{% response 'body', 'req_login', 'b64::!!::nonce', 'always', 60 %}",
"{% response 'body', 'req_login', '$.token', 'always', 60, %}",
"{% response 'body', 'req_login', '$.tokens[*]', 'always', 60 %}",
"{% response 'body', 'req_login', '$..token', 'always', 60 %}",
"{% response 'body', 'req_login', '/root/token', 'always', 60 %}",
"{% response 'body', 'req_login', '', 'never', 0 %}",
"{% response 'body', 'req_login', '$.token', 'when-expired', 0 %}",
"{% response 'url', 'req_login', '', 'never', 0 %}",
"{% os 'arch' %}",
"{% file '/tmp/token.txt' %}",
"{% jsonpath '{}', '$.a' %}",
"{% request 'url' %}",
"{% now 'quarter' %}",
"{% uuid 'v4', 'extra' %}",
"{% base64 'encode' %}",
"{% cookie 'https://example.com' %}",
"{% prompt %}",
"{% timestamp 'ms' %}",
])("preserves unsupported or malformed syntax: %s", (input) => {
expect(convert(input)).toBe(input);
});
test.each(["v1", "v4"])("converts UUID %s", (version) => {
expect(convert(`{% uuid '${version}' %}`)).toBe(`\${[ uuid.${version}() ]}`);
});
test("converts only the equivalent Faker UUID function", () => {
expect(convert("{% faker 'randomUUID' %}")).toBe("${[ uuid.v4() ]}");
});
test.each([
["never", "never"],
["no-history", "smart"],
["always", "always"],
["when-expired", "ttl"],
])("preserves %s sending behavior", (source, target) => {
expect(convert(`{% response 'body', 'req_login', '$.token', '${source}', 60 %}`)).toBe(
`\${[ response.body.path(request=${encoded("GENERATE_ID::req_login")}, behavior='${target}', ttl='60', path=${encoded("$.token")}, result='first') ]}`,
);
});
test("decodes Insomnia's UTF-8 base64 filters and emits URL-safe Yaak arguments", () => {
const path = "$.clé[0].token";
const base64 = Buffer.from(path).toString("base64");
expect(
convert(`{% response 'body', 'req_login', 'b64::${base64}::nonce', 'never', 0 %}`),
).toContain(`path=${encoded(path)}`);
expect(convert(`{% response 'body', 'req_login', '${path}', 'never', 0 %}`)).toContain(
`path=${encoded(path)}`,
);
});
test("maps raw body and headers, including quotes/commas in literal arguments", () => {
expect(convert("{% response 'raw', 'req_login', '', 'never', 0 %}")).toContain(
"response.body.raw(",
);
expect(
convert("{% response 'header', 'req_login', 'x-\\'quoted,header', 'never', 0 %}"),
).toContain(`header=${encoded("x-'quoted,header")}`);
});
test("converts a bare UUID tag to v4, the version Insomnia lists first", () => {
expect(convert("{% uuid %}")).toBe("${[ uuid.v4() ]}");
});
test.each([
["{% now %}", "timestamp.iso8601()"],
["{% now 'iso-8601' %}", "timestamp.iso8601()"],
["{% now 'ISO-8601' %}", "timestamp.iso8601()"],
["{% now 'unix' %}", "timestamp.unix()"],
["{% now 'seconds' %}", "timestamp.unix()"],
["{% now 's' %}", "timestamp.unix()"],
["{% now 'millis' %}", "timestamp.unixMillis()"],
["{% now 'ms' %}", "timestamp.unixMillis()"],
["{% timestamp %}", "timestamp.unixMillis()"],
])("converts %s", (input, output) => {
expect(convert(input)).toBe(`\${[ ${output} ]}`);
});
test("passes custom date formats through without translating tokens", () => {
const format = "YYYY-MM-DD HH:mm:ss";
expect(convert(`{% now 'custom', '${format}' %}`)).toBe(
`\${[ timestamp.format(format=${encoded(format)}) ]}`,
);
expect(convert("{% now 'custom' %}")).toBe(`\${[ timestamp.format(format=${encoded("")}) ]}`);
});
test.each([
["{% base64 'encode', 'normal', 'hello' %}", "base64.encode(encoding='base64', value="],
["{% base64 'encode', 'url', 'hello' %}", "base64.encode(encoding='base64url', value="],
// Yaak has no hex mode, so it imports as the plain alphabet.
["{% base64 'encode', 'hex', 'hello' %}", "base64.encode(encoding='base64', value="],
["{% base64 'encode', 'hello' %}", "base64.encode(encoding='base64', value="],
["{% base64 'decode', 'normal', 'hello' %}", "base64.decode(value="],
["{% base64 'decode', 'hello' %}", "base64.decode(value="],
])("converts %s", (input, prefix) => {
expect(convert(input)).toBe(`\${[ ${prefix}${encoded("hello")}) ]}`);
});
test("defaults an omitted base64 value to the empty string", () => {
expect(convert("{% base64 'encode', 'normal' %}")).toBe(
`\${[ base64.encode(encoding='base64', value=${encoded("")}) ]}`,
);
});
test("converts variables nested inside tag arguments", () => {
expect(convert("{% base64 'encode', 'normal', '{{ _.secret }}' %}")).toBe(
`\${[ base64.encode(encoding='base64', value=${encoded("${[ secret ]}")}) ]}`,
);
});
test.each(["md5", "sha1", "sha256", "sha512"])("converts the %s hash", (algorithm) => {
expect(convert(`{% hash '${algorithm}', 'hex', 'text' %}`)).toBe(
`\${[ hash.${algorithm}(input=${encoded("text")}, encoding='hex') ]}`,
);
});
test("keeps the base64 digest encoding", () => {
expect(convert("{% hash 'sha256', 'base64', 'text' %}")).toBe(
`\${[ hash.sha256(input=${encoded("text")}, encoding='base64') ]}`,
);
});
test("defaults omitted hash arguments to MD5 and hex, like Insomnia's dropdowns", () => {
expect(convert("{% hash %}")).toBe(`\${[ hash.md5(input=${encoded("")}, encoding='hex') ]}`);
expect(convert("{% hash 'sha1' %}")).toBe(
`\${[ hash.sha1(input=${encoded("")}, encoding='hex') ]}`,
);
});
test("falls back to SHA-256 for algorithms outside Insomnia's list, as Insomnia does", () => {
expect(convert("{% hash 'sha384', 'hex', 'text' %}")).toBe(
`\${[ hash.sha256(input=${encoded("text")}, encoding='hex') ]}`,
);
});
test("converts a cookie URL to the domain Yaak filters by", () => {
expect(convert("{% cookie 'https://api.example.com/login', 'session' %}")).toBe(
`\${[ cookie.value(name=${encoded("session")}, domain=${encoded("api.example.com")}) ]}`,
);
});
test("keeps an unreadable cookie URL so it stays visible in the tag", () => {
expect(convert("{% cookie '{{ _.base_url }}', 'session' %}")).toBe(
`\${[ cookie.value(name=${encoded("session")}, domain=${encoded("${[ base_url ]}")}) ]}`,
);
});
test("searches the whole jar when the cookie URL is empty", () => {
expect(convert("{% cookie '', 'session' %}")).toBe(
`\${[ cookie.value(name=${encoded("session")}) ]}`,
);
});
test("converts a prompt with only a title", () => {
expect(convert("{% prompt 'Password' %}")).toBe(
`\${[ prompt.text(label=${encoded("Password")}, title=${encoded("Password")}) ]}`,
);
});
test("converts every prompt argument with a Yaak counterpart", () => {
expect(convert("{% prompt 'Login', 'Username', 'admin', 'user-key', false, true %}")).toBe(
`\${[ prompt.text(label=${encoded("Username")}, store='forever', ` +
`namespace=${encoded("${[ctx.workspace()]}")}, key=${encoded("user-key")}, ` +
`title=${encoded("Login")}, defaultValue=${encoded("admin")}) ]}`,
);
});
test("never stores a masked prompt, even with a storage key", () => {
expect(convert("{% prompt 'Login', 'Password', 'hunter2', 'pw-key', true, true %}")).toBe(
`\${[ prompt.text(label=${encoded("Password")}, title=${encoded("Login")}, ` +
`defaultValue=${encoded("hunter2")}, password=true) ]}`,
);
});
test("masks a prompt without a storage key", () => {
expect(convert("{% prompt 'Login', 'Password', '', '', true %}")).toBe(
`\${[ prompt.text(label=${encoded("Password")}, title=${encoded("Login")}, password=true) ]}`,
);
});
test("leaves the prompt unmasked and unstored by default", () => {
expect(convert("{% prompt 'Login', 'Password', '', '', false, true %}")).toBe(
`\${[ prompt.text(label=${encoded("Password")}, title=${encoded("Login")}) ]}`,
);
});
test("converts the two-argument response form", () => {
expect(convert("{% response 'raw', 'req_login' %}")).toBe(
`\${[ response.body.raw(request=${encoded("GENERATE_ID::req_login")}, behavior='never', ttl='0') ]}`,
);
});
test("converts nested resource fields without changing the input", () => {
const input = {
authentication: { token: "{{ _['token'] }}" },
body: { form: [{ value: "{% uuid 'v4' %}" }] },
};
expect(convertTemplateSyntax(input)).toEqual({
authentication: { token: "${[ token ]}" },
body: { form: [{ value: "${[ uuid.v4() ]}" }] },
});
expect(input.authentication.token).toBe("{{ _['token'] }}");
});
});
+80 -1
View File
@@ -252,6 +252,85 @@ export const plugin: PluginDefinition = {
],
};
const QUOTE = 0x27; // '
const BACKSLASH = 0x5c; // \
const CLOSE_BRACKET = 0x5d; // ]
const CLOSE_BRACE = 0x7d; // }
/**
* For every index `i`, the index of the `'` that closes a string whose body starts at `i`, or
* `-1` when nothing does. One right-to-left pass, so skipping a string costs one lookup.
*/
function closingQuoteTable(text: string): Int32Array {
// Two extra slots so a `\` in the last position can read past the end and find `-1`
const table = new Int32Array(text.length + 2).fill(-1);
for (let i = text.length - 1; i >= 0; i--) {
const c = text.charCodeAt(i);
// A backslash escapes whatever follows it, including a quote
const next = c === BACKSLASH ? table[i + 2] : table[i + 1];
table[i] = c === QUOTE ? i : (next ?? -1);
}
return table;
}
/**
* Rewrites every template tag in `text` with whatever `replace` returns for its inner content.
*
* A quoted argument may contain `]}`, so the scan steps over strings instead of stopping at the
* first one, and an unterminated quote stays an ordinary character. Kept in sync by hand with
* `apps/yaak-client/lib/templateTags.ts`, which carries the full explanation — including why
* this is a linear hand-written scan rather than a regex.
*/
function replaceTemplateTags(text: string, replace: (inner: string) => string): string {
let closingQuotes: Int32Array | null = null;
let deadEnds: Set<number> | null = null;
let result = "";
let copiedTo = 0;
let searchFrom = 0;
while (searchFrom < text.length) {
const start = text.indexOf("${[", searchFrom);
if (start < 0) break;
const visited: number[] = [];
let end = -1;
let i = start + 3;
while (i < text.length) {
if (deadEnds?.has(i)) break;
visited.push(i);
const c = text.charCodeAt(i);
if (c === QUOTE) {
closingQuotes ??= closingQuoteTable(text);
const close = closingQuotes[i + 1] ?? -1;
i = close < 0 ? i + 1 : close + 1;
continue;
}
if (c === CLOSE_BRACKET && text.charCodeAt(i + 1) === CLOSE_BRACE) {
end = i + 2;
break;
}
i += 1;
}
if (end < 0) {
// Not a tag, but a real one may start inside a string this scan stepped over
deadEnds ??= new Set();
for (const position of visited) deadEnds.add(position);
searchFrom = start + 3;
continue;
}
result += text.slice(copiedTo, start) + replace(text.slice(start + 3, end - 2));
copiedTo = end;
searchFrom = end;
}
return result + text.slice(copiedTo);
}
// TODO: Use a common function for this, but it fails to build on windows during CI if I try importing it here
export function resolvedModelName(r: AnyModel | null): string {
if (r == null) return "";
@@ -266,7 +345,7 @@ export function resolvedModelName(r: AnyModel | null): string {
}
// Replace variable syntax with variable name
const withoutVariables = r.url.replace(/\$\{\[\s*([^\]\s]+)\s*]}/g, "$1");
const withoutVariables = replaceTemplateTags(r.url, (inner) => inner.trim());
if (withoutVariables.trim() === "") {
return r.model === "http_request"
? r.bodyType && r.bodyType === "graphql"
@@ -0,0 +1,42 @@
/*
* This plugin carries its own copy of the template-tag scanner, because importing the client's
* copy into the plugin bundle fails to build on Windows in CI.
*
* Rather than copying the expectations too — copied expectations can't detect drift — this
* imports the client's `resolvedModelName.cases.ts` and runs the plugin's copy against it, then
* asserts the plugin's output matches the client's own `resolvedModelName` for every case. A
* change to the client scanner or its cases therefore fails here until the copy is updated. The
* import is test-only and never reaches the plugin bundle, so the Windows problem doesn't apply.
*/
import type { HttpRequest } from "@yaakapp-internal/models";
import { describe, expect, test, vi } from "vite-plus/test";
import { resolvedModelNameCases } from "../../../apps/yaak-client/lib/resolvedModelName.cases";
// `resolvedModelName` reads plain model fields, but the two modules under test pull in sibling
// plugins that only resolve once they're built, plus the client's platform singleton, which
// needs a browser. Stub them all out to keep this a pure unit test.
vi.mock("../../template-function-json", () => ({ filterJSONPath: () => null }));
vi.mock("../../template-function-xml", () => ({ filterXPath: () => null }));
vi.mock("@yaakapp-internal/models", () => ({ foldersAtom: {} }));
vi.mock("../../../apps/yaak-client/lib/jotai", () => ({ jotaiStore: { get: () => [] } }));
const { resolvedModelName } = await import("../src");
const { resolvedModelName: clientResolvedModelName } = await import(
"../../../apps/yaak-client/lib/resolvedModelName"
);
function httpRequest(url: string, name = ""): HttpRequest {
return { id: "rq_test", model: "http_request", name, url } as HttpRequest;
}
describe("resolvedModelName", () => {
test.each(resolvedModelNameCases)("$name", ({ url, requestName, expected }) => {
expect(resolvedModelName(httpRequest(url, requestName))).toEqual(expected);
});
test.each(resolvedModelNameCases)("matches the client for: $name", ({ url, requestName }) => {
const request = httpRequest(url, requestName);
expect(resolvedModelName(request)).toEqual(clientResolvedModelName(request));
});
});
@@ -16,6 +16,7 @@ import { filterXPath } from "../../template-function-xml";
const BEHAVIOR_TTL = "ttl";
const BEHAVIOR_ALWAYS = "always";
const BEHAVIOR_SMART = "smart";
const BEHAVIOR_NEVER = "never";
const RETURN_FIRST = "first";
const RETURN_ALL = "all";
@@ -33,6 +34,7 @@ const behaviorArgs: DynamicTemplateFunctionArg = {
{ label: "When no responses", value: BEHAVIOR_SMART },
{ label: "Always", value: BEHAVIOR_ALWAYS },
{ label: "When expired", value: BEHAVIOR_TTL },
{ label: "Never", value: BEHAVIOR_NEVER },
],
},
{
@@ -300,7 +302,7 @@ async function getResponse(
const responses = await ctx.httpResponse.find({ requestId: httpRequest.id, limit: 1 });
if (behavior === "never" && responses.length === 0) {
if (behavior === BEHAVIOR_NEVER && responses.length === 0) {
return null;
}
@@ -308,12 +310,13 @@ async function getResponse(
// Previews happen a ton, and we don't want to send too many times on "always," so treat
// it as "smart" during preview.
const finalBehavior = behavior === "always" && purpose === "preview" ? "smart" : behavior;
const finalBehavior =
behavior === BEHAVIOR_ALWAYS && purpose === "preview" ? BEHAVIOR_SMART : behavior;
// Send if no responses and "smart," or "always"
if (
(finalBehavior === "smart" && response == null) ||
finalBehavior === "always" ||
(finalBehavior === BEHAVIOR_SMART && response == null) ||
finalBehavior === BEHAVIOR_ALWAYS ||
(finalBehavior === BEHAVIOR_TTL && shouldSendExpired(response, ttl))
) {
// Explicitly render the request before send (instead of relying on send() to render) so that we can
+2 -1
View File
@@ -8,7 +8,8 @@
"types": "src/index.ts",
"scripts": {
"build": "yaakcli build",
"dev": "yaakcli dev"
"dev": "yaakcli dev",
"test": "vp test --run tests"
},
"dependencies": {
"@xmldom/xmldom": "^0.9.12",
+10 -2
View File
@@ -77,11 +77,19 @@ export function filterXPath(
return String(items);
}
if (!Array.isArray(items) || result === "first") {
return items[0] != null ? String(items[0].firstChild ?? "") : "";
return items[0] != null ? nodeText(items[0]) : "";
}
if (result === "join") {
return items.map((item) => String(item.firstChild ?? "")).join(join ?? "");
return items.map(nodeText).join(join ?? "");
}
// Not sure what cases this happens in (?)
return String(items);
}
function nodeText(node: Node): string {
// Attribute, text and CDATA nodes have no children, so their value is on the node itself
if (xpath.isAttribute(node) || xpath.isTextNode(node) || xpath.isCDATASection(node)) {
return node.nodeValue ?? "";
}
return String(node.firstChild ?? "");
}
@@ -0,0 +1,36 @@
import type { Context } from "@yaakapp/api";
import { describe, expect, it } from "vite-plus/test";
import { filterXPath, plugin } from "../src";
const USERS =
'<users><user id="42"><name>Alice</name></user><user id="7"><name>Bob</name></user></users>';
describe("xml.xpath", () => {
const xpathFunction = plugin.templateFunctions?.find((f) => f.name === "xml.xpath");
const select = async (query: string, result = "first", join = ", ") =>
await xpathFunction?.onRender({} as Context, {
values: { input: USERS, query, result, join },
purpose: "send",
});
it("returns the text of a selected element", async () => {
expect(await select("//user/name")).toBe("Alice");
});
it("returns the value of a selected attribute", async () => {
expect(await select("//user/@id")).toBe("42");
});
it("joins the values of selected attributes", async () => {
expect(await select("//user/@id", "join")).toBe("42, 7");
});
it("returns the value of a selected text node", async () => {
expect(await select("//user/name/text()")).toBe("Alice");
});
it("returns the value of a selected CDATA section", () => {
expect(filterXPath("<a><![CDATA[x<y]]></a>", "/a/text()", "first", null)).toBe("x<y");
});
});
+6 -6
View File
@@ -45,12 +45,12 @@ const DST_BIN_MAP = {
};
const SHA256_MAP = {
[MAC_ARM]: "b05aa3a66efe680023f930bd5af3fdbbd542794da5644ca2ad711d68cbd4dc35",
[MAC_X64]: "096081b6d6fcdd3f5ba0f5f1d44a47e83037ad2e78eada26671c252fe64dd111",
[LNX_ARM]: "0dc93ec5c798b0d347f068db6d205d03dea9a71765e6a53922b682b91265d71f",
[LNX_X64]: "58a5ff5cc8f2200e458bea22e329d5c1994aa1b111d499ca46ec2411d58239ca",
[WIN_X64]: "5355ae6d7c49eddcfde7d34ac3486820600a831bf81dc3bdca5c8db6a9bb0e76",
[WIN_ARM]: "ce9ee4e547ebdff355beb48e309b166c24df6be0291c9eaf103ce15f3de9e5b4",
[MAC_ARM]: "aa9549349e47dffaa42dcf95d6e06ade8c4bdafc980979883b88767670df385a",
[MAC_X64]: "d5e336d588c45be0c37dbd52cc29895593009ca4997a77a1fd02815131ed0e52",
[LNX_ARM]: "95c264150aceb7f3437a29fd6984924f5f8d1511ea3cbd6c508b83f6c555dae2",
[LNX_X64]: "361718c444e49af2ca5a110c83ba6a8901143f23ab285ebbef0341f391f482a9",
[WIN_X64]: "ca1bff5d8ead13c9f2ea404a176db46bd430704d5b5113734049fa365e7799e2",
[WIN_ARM]: "f47968c213105077f64c6510f007632c0ca2574802a0b5067c327c49abe1c468",
};
const key = `${process.platform}_${process.env.YAAK_TARGET_ARCH ?? process.arch}`;
+7 -3
View File
@@ -93,9 +93,13 @@ test.each(["http://tauri.localhost", "tauri://localhost"])(
const pdfjsDir = path.dirname(reactPdfRequire.resolve("pdfjs-dist/package.json"));
const installed = JSON.parse(await readFile(path.join(pdfjsDir, "package.json"), "utf8"));
expect(version).toBe(installed.version);
expect(Buffer.from(worker.source)).toEqual(
await readFile(path.join(pdfjsDir, "build/pdf.worker.min.mjs")),
);
// The worker is bundled with the polyfills older WebKit needs, ahead of PDF.js itself.
const source = Buffer.from(worker.source).toString("utf8");
const escapedVersion = installed.version.replaceAll(".", "\\.");
expect(source).toMatch(new RegExp(`["'\`]${escapedVersion}["'\`]`));
const polyfillAt = source.indexOf("getOrInsertComputed callback is not a function");
expect(polyfillAt).toBeGreaterThanOrEqual(0);
expect(polyfillAt).toBeLessThan(source.indexOf("pdfjs_internal_editor_"));
},
30_000,
);