From 0e4c355e8bc0432cdb1abef6e4298772f414d6cf Mon Sep 17 00:00:00 2001 From: Gregory Schier Date: Thu, 13 Aug 2026 08:40:04 -0700 Subject: [PATCH] Collapse very long lines in response viewers (#532) --- .../components/core/Editor/Editor.css | 10 + .../components/core/Editor/Editor.tsx | 14 +- .../components/core/Editor/extensions.ts | 3 + .../core/Editor/largeValues.test.ts | 204 +++++++++++++ .../components/core/Editor/largeValues.ts | 281 ++++++++++++++++++ apps/yaak-client/lib/docFingerprint.test.ts | 67 +++++ apps/yaak-client/lib/docFingerprint.ts | 33 ++ 7 files changed, 606 insertions(+), 6 deletions(-) create mode 100644 apps/yaak-client/components/core/Editor/largeValues.test.ts create mode 100644 apps/yaak-client/components/core/Editor/largeValues.ts create mode 100644 apps/yaak-client/lib/docFingerprint.test.ts create mode 100644 apps/yaak-client/lib/docFingerprint.ts diff --git a/apps/yaak-client/components/core/Editor/Editor.css b/apps/yaak-client/components/core/Editor/Editor.css index 4a37a4cd..ebe82f12 100644 --- a/apps/yaak-client/components/core/Editor/Editor.css +++ b/apps/yaak-client/components/core/Editor/Editor.css @@ -121,6 +121,16 @@ @apply cursor-default; } + /* An icon button any tag can carry, like the copy on a large value */ + .tag-action { + @apply inline-flex items-center align-middle ml-1 cursor-pointer!; + @apply text-text-subtlest hover:text-text; + + svg { + @apply w-[0.9em] h-[0.9em]; + } + } + .fn { @apply inline-block; .fn-inner { diff --git a/apps/yaak-client/components/core/Editor/Editor.tsx b/apps/yaak-client/components/core/Editor/Editor.tsx index 2dc1c203..73870048 100644 --- a/apps/yaak-client/components/core/Editor/Editor.tsx +++ b/apps/yaak-client/components/core/Editor/Editor.tsx @@ -15,7 +15,6 @@ import { HStack } from "@yaakapp-internal/ui"; import classNames from "classnames"; import type { GraphQLSchema } from "graphql"; import { useAtomValue } from "jotai"; -import { md5 } from "js-md5"; import type { ReactNode, RefObject } from "react"; import { Children, @@ -33,6 +32,7 @@ import { useEnvironmentVariables } from "../../../hooks/useEnvironmentVariables" import { eventMatchesHotkey } from "../../../hooks/useHotKey"; import { useRequestEditor } from "../../../hooks/useRequestEditor"; import { useTemplateFunctionCompletionOptions } from "../../../hooks/useTemplateFunctions"; +import { docFingerprint } from "../../../lib/docFingerprint"; import { editEnvironment } from "../../../lib/editEnvironment"; import { tryFormatJson, tryFormatXml } from "../../../lib/formatters"; import { jotaiStore } from "../../../lib/jotai"; @@ -430,7 +430,7 @@ function EditorInner({ : []), ]; - const cachedJsonState = getCachedEditorState(defaultValue ?? "", stateKey); + const cachedJsonState = getCachedEditorState(defaultValue ?? "", stateKey, !!readOnly); const doc = `${defaultValue ?? ""}`; const config: EditorStateConfig = { extensions, doc }; @@ -656,9 +656,11 @@ function saveCachedEditorState(stateKey: string | null, state: EditorState | nul if (!stateKey || state == null) return; const stateObj = state.toJSON(stateFields); - // Save state in sessionStorage by removing doc and saving the hash of it instead. + // Save state in sessionStorage by removing doc and saving a fingerprint of it instead. // This will be checked on restore and put back in if it matches. - stateObj.docHash = md5(stateObj.doc); + // Editable documents get the exact hash, so a collision can never restore undo history + // belonging to other content + stateObj.docHash = docFingerprint(stateObj.doc, { exact: !state.readOnly }); stateObj.doc = undefined; try { @@ -668,7 +670,7 @@ function saveCachedEditorState(stateKey: string | null, state: EditorState | nul } } -function getCachedEditorState(doc: string, stateKey: string | null) { +function getCachedEditorState(doc: string, stateKey: string | null, readOnly: boolean) { if (stateKey == null) return; try { @@ -678,7 +680,7 @@ function getCachedEditorState(doc: string, stateKey: string | null) { const { docHash, ...state } = JSON.parse(stateStr); // Ensure the doc matches the one that was used to save the state - if (docHash !== md5(doc)) { + if (docHash !== docFingerprint(doc, { exact: !readOnly })) { return null; } diff --git a/apps/yaak-client/components/core/Editor/extensions.ts b/apps/yaak-client/components/core/Editor/extensions.ts index ec9d46ce..95904089 100644 --- a/apps/yaak-client/components/core/Editor/extensions.ts +++ b/apps/yaak-client/components/core/Editor/extensions.ts @@ -59,6 +59,7 @@ import { pluralizeCount } from "../../../lib/pluralize"; import { showGraphQLDocExplorerAtom } from "../../graphql/graphqlAtoms"; import type { EditorProps } from "./Editor"; import { jsonParseLinter } from "./json-lint"; +import { largeValues } from "./largeValues"; import { pairs } from "./pairs/extension"; import { searchMatchCount } from "./searchMatchCount"; import { text } from "./text/extension"; @@ -258,6 +259,8 @@ export const baseExtensions = [ export const readonlyExtensions = [ EditorState.readOnly.of(true), EditorView.contentAttributes.of({ tabindex: "-1" }), + // Read-only only, so we never hide part of a document someone is editing + largeValues, ]; export const multiLineExtensions = ({ hideGutter }: { hideGutter?: boolean }) => [ diff --git a/apps/yaak-client/components/core/Editor/largeValues.test.ts b/apps/yaak-client/components/core/Editor/largeValues.test.ts new file mode 100644 index 00000000..3eeac361 --- /dev/null +++ b/apps/yaak-client/components/core/Editor/largeValues.test.ts @@ -0,0 +1,204 @@ +import { EditorState } from "@codemirror/state"; +import { jsonc } from "@shopify/lang-jsonc"; +import { text } from "./text/extension"; +import { describe, expect, test, vi } from "vite-plus/test"; +import { + COLLAPSE_TOKEN_CHARS, + largeValueField, + largeValues, + MAX_VISIBLE_LINE_CHARS, +} from "./largeValues"; + +vi.mock("../../../lib/copy", () => ({ copyToClipboard: () => {} })); + +const BIG = "A".repeat(1_000_000); + +/** With a grammar, so tokens can be collapsed individually */ +const jsonState = (doc: string) => EditorState.create({ doc, extensions: [jsonc(), largeValues] }); + +/** Without a grammar, so only the column rule applies */ +const plainState = (doc: string) => EditorState.create({ doc, extensions: largeValues }); + +function collapsedRanges(state: EditorState) { + const ranges: { from: number; to: number }[] = []; + const iter = state.field(largeValueField).decorations.iter(); + while (iter.value != null) { + ranges.push({ from: iter.from, to: iter.to }); + iter.next(); + } + return ranges; +} + +/** How much of each line is still rendered */ +function visibleLineLengths(state: EditorState) { + const hidden = collapsedRanges(state); + const lengths: number[] = []; + for (let n = 1; n <= state.doc.lines; n++) { + const line = state.doc.line(n); + const covered = hidden + .filter((h) => h.from >= line.from && h.to <= line.to) + .reduce((sum, h) => sum + (h.to - h.from), 0); + lengths.push(line.length - covered); + } + return lengths; +} + +describe("collapsing", () => { + test("leaves an ordinary body alone", () => { + expect(collapsedRanges(jsonState('{"hello":"world"}'))).toEqual([]); + }); + + test("leaves a large body of short lines alone", () => { + const doc = Array.from({ length: 20_000 }, (_, i) => ` { "id": ${i} },`).join("\n"); + expect(doc.length).toBeGreaterThan(MAX_VISIBLE_LINE_CHARS); + expect(collapsedRanges(jsonState(doc))).toEqual([]); + }); + + test("leaves a line just under the column limit alone", () => { + expect(collapsedRanges(plainState("x".repeat(MAX_VISIBLE_LINE_CHARS)))).toEqual([]); + }); + + test("never renders more than the column limit per line", () => { + for (const state of [jsonState(`{"image":"${BIG}"}`), plainState(BIG)]) { + for (const length of visibleLineLengths(state)) { + expect(length).toBeLessThanOrEqual(MAX_VISIBLE_LINE_CHARS); + } + } + }); + + test("keeps the document text intact", () => { + const doc = `{"image":"${BIG}"}`; + expect(jsonState(doc).sliceDoc()).toBe(doc); + expect(jsonState(doc).doc.length).toBe(doc.length); + }); +}); + +describe("token collapsing, with a grammar", () => { + test("hides the whole value, leaving its quotes visible", () => { + const doc = `{"name":"a.png","image":"${BIG}","size":12}`; + const state = jsonState(doc); + const ranges = collapsedRanges(state); + + expect(ranges).toHaveLength(1); + // Exactly the text between the quotes, so the line reads as `"image":""` + expect(state.sliceDoc(ranges[0]!.from, ranges[0]!.to)).toBe(BIG); + expect(doc[ranges[0]!.from - 1]).toBe('"'); + expect(doc[ranges[0]!.to]).toBe('"'); + + // Everything after the value is still rendered, unlike a plain column cut + expect(doc.slice(ranges[0]!.to)).toContain('"size":12}'); + }); + + // Small enough that the parse always finishes inside PARSE_TIMEOUT_MS, even on a slow + // machine. With a bigger body this falls back to the column cut, which is by design but + // makes the assertion depend on how fast the runner is. + test("keeps every key visible in a minified body with several large values", () => { + const chunk = "B".repeat(20_000); + const doc = `{${["a", "b", "c", "d", "e"].map((k) => `"${k}":"${chunk}"`).join(",")}}`; + const state = jsonState(doc); + const ranges = collapsedRanges(state); + + expect(ranges).toHaveLength(5); + for (const key of ["a", "b", "c", "d", "e"]) { + // No collapse swallows the key + const at = doc.indexOf(`"${key}":`); + expect(ranges.some((r) => r.from <= at && r.to > at)).toBe(false); + } + expect(visibleLineLengths(state)[0]).toBeLessThanOrEqual(MAX_VISIBLE_LINE_CHARS); + }); + + test("collapses a value on a pretty-printed line", () => { + const doc = `{\n "name": "a.png",\n "image": "${BIG}"\n}`; + const state = jsonState(doc); + const ranges = collapsedRanges(state); + + expect(ranges).toHaveLength(1); + expect(state.sliceDoc(ranges[0]!.from, ranges[0]!.to)).toBe(BIG); + // Only the long line is touched, and all that is left of it is ` "image": ""` + expect(visibleLineLengths(state)).toEqual([1, 18, 13, 1]); + expect(state.doc.line(2).text).toBe(' "name": "a.png",'); + }); + + test("ignores tokens under the collapse threshold", () => { + // Under the threshold once the surrounding quotes are counted + const short = "C".repeat(COLLAPSE_TOKEN_CHARS - 10); + const doc = `{${Array.from({ length: 4 }, (_, i) => `"k${i}":"${short}"`).join(",")}}`; + expect(doc.length).toBeGreaterThan(MAX_VISIBLE_LINE_CHARS); + + // Nothing is big enough to collapse on its own, so the column rule takes over + const ranges = collapsedRanges(jsonState(doc)); + expect(ranges).toHaveLength(1); + expect(ranges[0]!.to).toBe(doc.length); + }); +}); + +describe("column collapsing, without a grammar", () => { + test("collapses everything past the limit", () => { + const ranges = collapsedRanges(plainState(BIG)); + expect(ranges).toHaveLength(1); + expect(ranges[0]!.from).toBe(MAX_VISIBLE_LINE_CHARS); + expect(ranges[0]!.to).toBe(BIG.length); + }); + + test("handles a long line of many short tokens", () => { + // A single-line CSV row: no token is long enough to collapse on its own + const row = Array.from({ length: 40_000 }, (_, i) => `value ${i}`).join(", "); + const ranges = collapsedRanges(plainState(row)); + expect(ranges).toHaveLength(1); + expect(ranges[0]!.from).toBe(MAX_VISIBLE_LINE_CHARS); + }); + + test("collapses each long line independently", () => { + const doc = `${BIG}\nshort\n${BIG}`; + const ranges = collapsedRanges(plainState(doc)); + expect(ranges).toHaveLength(2); + for (const length of visibleLineLengths(plainState(doc))) { + expect(length).toBeLessThanOrEqual(MAX_VISIBLE_LINE_CHARS); + } + }); + + test("never hides a line break", () => { + const doc = `${BIG}\nshort`; + const state = plainState(doc); + for (const { from, to } of collapsedRanges(state)) { + expect(state.sliceDoc(from, to)).not.toContain("\n"); + } + expect(state.doc.lines).toBe(2); + }); +}); + +describe("undelimited tokens", () => { + // The text grammar parses a whole line as one token. Collapsing it whole would leave the + // line with nothing on it but a tag, so the column cut handles it instead. + const textState = (doc: string) => EditorState.create({ doc, extensions: [text(), largeValues] }); + + test("falls back to the column cut for a long line of plain text", () => { + const doc = Array.from({ length: 90_000 }, (_, i) => `word${i}`).join(" "); + const ranges = collapsedRanges(textState(doc)); + + expect(ranges).toHaveLength(1); + expect(ranges[0]!.from).toBe(MAX_VISIBLE_LINE_CHARS); + expect(ranges[0]!.to).toBe(doc.length); + }); + + test("leaves the start of the line readable", () => { + const doc = `IMPORTANT-PREFIX ${"z".repeat(500_000)}`; + const state = textState(doc); + const ranges = collapsedRanges(state); + + expect(ranges[0]!.from).toBe(MAX_VISIBLE_LINE_CHARS); + expect(state.sliceDoc(0, 16)).toBe("IMPORTANT-PREFIX"); + }); +}); + +describe("recomputing", () => { + test("updates when the document changes", () => { + const state = plainState('{"image":"short"}'); + expect(collapsedRanges(state)).toEqual([]); + + const next = state.update({ + changes: { from: 0, to: state.doc.length, insert: BIG }, + }).state; + expect(collapsedRanges(next)).toHaveLength(1); + }); +}); diff --git a/apps/yaak-client/components/core/Editor/largeValues.ts b/apps/yaak-client/components/core/Editor/largeValues.ts new file mode 100644 index 00000000..564e86a6 --- /dev/null +++ b/apps/yaak-client/components/core/Editor/largeValues.ts @@ -0,0 +1,281 @@ +import { ensureSyntaxTree, syntaxTree } from "@codemirror/language"; +import { formatSize } from "@yaakapp-internal/lib/formatSize"; +import type { EditorState, Extension, Range } from "@codemirror/state"; +import { StateField } from "@codemirror/state"; +import type { Tree as SyntaxTree } from "@lezer/common"; +import type { DecorationSet } from "@codemirror/view"; +import { Decoration, EditorView, WidgetType } from "@codemirror/view"; +import { copyToClipboard } from "../../../lib/copy"; + +/** + * How much of a line may be rendered before the rest is collapsed. + * + * VS Code draws nothing past column 10,000 (`editor.stopRenderingLineAfter`) for the same + * reason. It can afford to be blunt about it because it doesn't soft wrap by default; we + * collapse to a placeholder that can be opened instead. + */ +export const MAX_VISIBLE_LINE_CHARS = 10_000; + +/** + * A quoted value longer than this on an over-long line is collapsed whole, ahead of the column + * cut, so the structure around it stays visible. Needs a grammar to find. + */ +export const COLLAPSE_TOKEN_CHARS = 5_000; + +/** + * Keeps over-long lines from reaching layout, which is what makes the editor stall on a + * base64 blob or a minified payload. + * + * Cost tracks the length of the longest line, not the size of the document. A 1 MB response of + * ordinary multi-line JSON renders fine, while the same 1 MB on a single line stalls the UI, + * because soft wrap has to measure the whole line end to end to find its break points. + * Measured in WKWebView, the engine macOS ships: 221 ms per render pass at 1 MB and 968 ms at + * 3 MB, against 51 ms and 138 ms once collapsed, with soft wrap left on. + * + * Two rules, applied only to lines over {@link MAX_VISIBLE_LINE_CHARS}, so ordinary documents + * are untouched: + * + * 1. Collapse quoted values over {@link COLLAPSE_TOKEN_CHARS} whole, so a base64 string reads + * as `"image": "999.8 KB hidden…"` with its key intact. A minified body with several large values + * keeps every one of its keys. Needs a grammar to find the value. + * 2. Collapse whatever is still past the column limit. This needs no grammar, so it covers + * plain text, undelimited tokens, and any line that is simply long. + * + * Nothing leaves the document. Copy, filter and save all still see the full text, and the tag + * carries a button that copies exactly what it hides. + * + * Read-only editors only. Hiding part of a document someone is editing would mean editing + * text they can't see. + */ + +/** A hidden range. The tag stands in for exactly this text, and the dialog shows it. */ +interface Collapse { + from: number; + to: number; +} + +class LargeValueWidget extends WidgetType { + constructor( + private readonly from: number, + private readonly to: number, + ) { + super(); + } + + eq(other: LargeValueWidget) { + return other.from === this.from && other.to === this.to; + } + + toDOM(view: EditorView) { + const length = this.to - this.from; + + const el = document.createElement("span"); + // The same neutral tag styling a path parameter uses. The theme scope matters: these + // tokens resolve against the tag palette, not the editor's ambient one, where a + // tag-sized border is meant to be near invisible. + el.className = "x-theme-templateTag x-theme-templateTag--secondary template-tag"; + + const label = document.createElement("span"); + label.textContent = `${formatSize(length)} hidden…`; + el.appendChild(label); + + // A span rather than a button: a button's box model makes the line taller. It keeps the + // role and name so assistive tech still sees it as an action. + const copy = document.createElement("span"); + copy.role = "button"; + copy.className = "tag-action"; + copy.title = `Copy hidden text (${length.toLocaleString()} characters)`; + copy.ariaLabel = copy.title; + // Lucide's `copy` icon, inlined because the widget builds its DOM synchronously and + // rendering React here would leave it empty while CodeMirror measures line heights + copy.innerHTML = + '' + + ''; + // Keep the editor from moving the cursor when the button is pressed + copy.addEventListener("mousedown", (e) => { + e.preventDefault(); + e.stopPropagation(); + }); + copy.addEventListener("click", (e) => { + e.preventDefault(); + e.stopPropagation(); + copyToClipboard(view.state.sliceDoc(this.from, this.to)); + }); + el.appendChild(copy); + + return el; + } + + ignoreEvent() { + return false; + } +} + +interface Line { + from: number; + to: number; +} + +/** Lines long enough to be a problem. Most documents have none, and we stop there. */ +function findLongLines(text: string): Line[] { + if (text.length <= MAX_VISIBLE_LINE_CHARS) { + return []; // No line can be longer than the whole text + } + + const lines: Line[] = []; + let from = 0; + for (;;) { + const newline = text.indexOf("\n", from); + const to = newline < 0 ? text.length : newline; + if (to - from > MAX_VISIBLE_LINE_CHARS) { + lines.push({ from, to }); + } + if (newline < 0) { + return lines; + } + from = newline + 1; + } +} + +/** + * How long to spend parsing before falling back to the column rule. + * + * The initial parse is budgeted by time, so it stops partway through a document with several + * large values, and we'd only find the first one. Parsing the rest of a 1 MB body costs about + * 12 ms, against the 171 ms of layout it saves. + */ +const PARSE_TIMEOUT_MS = 100; + +/** The parsed tree covering the long lines, as far as parsing got in the time allowed. */ +function treeForLongLines(state: EditorState, longLines: Line[]): SyntaxTree { + const lastLine = longLines[longLines.length - 1]; + if (lastLine == null) { + return syntaxTree(state); + } + return ensureSyntaxTree(state, lastLine.to, PARSE_TIMEOUT_MS) ?? syntaxTree(state); +} + +const QUOTES = ['"', "'", "`"]; + +/** + * Quoted values on this line big enough to collapse whole, in document order. + * + * Everything between the quotes goes, so a long string reads as `"image": ""` and the key + * still tells you what it is. Undelimited tokens are left to the column cut instead: replacing + * one whole would leave the line with nothing on it but a tag. + */ +function findLargeTokens(state: EditorState, tree: SyntaxTree, line: Line): Collapse[] { + const collapses: Collapse[] = []; + + tree.iterate({ + from: line.from, + to: line.to, + enter: (node) => { + // A node this small can't contain anything worth collapsing + if (node.to - node.from < COLLAPSE_TOKEN_CHARS) return false; + // Only leaves, so we collapse the string itself rather than the object holding it + if (node.node.firstChild != null) return true; + + const from = Math.max(node.from, line.from); + const to = Math.min(node.to, line.to); + const open = state.sliceDoc(from, from + 1); + const quoted = to - from >= 2 && QUOTES.includes(open) && state.sliceDoc(to - 1, to) === open; + + if (quoted && to - 1 > from + 1) { + collapses.push({ from: from + 1, to: to - 1 }); + } + return false; + }, + }); + + return collapses; +} + +/** + * Where the line runs past the column limit, counting only what is still visible after the + * token collapses, or -1 if it fits. + */ +function findColumnCut(line: Line, tokens: Collapse[]): number { + let visible = 0; + let pos = line.from; + + for (const token of [...tokens, null]) { + const segmentEnd = token == null ? line.to : token.from; + if (segmentEnd > pos) { + if (visible + (segmentEnd - pos) > MAX_VISIBLE_LINE_CHARS) { + return pos + (MAX_VISIBLE_LINE_CHARS - visible); + } + visible += segmentEnd - pos; + } + if (token != null) { + pos = token.to; + } + } + + return -1; +} + +function collapsesForLine(state: EditorState, tree: SyntaxTree, line: Line): Collapse[] { + const tokens = findLargeTokens(state, tree, line); + const cut = findColumnCut(line, tokens); + if (cut < 0) { + return tokens; + } + + // The cut always lands in a visible stretch, so it never splits a token collapse + const kept = tokens.filter((t) => t.to <= cut); + kept.push({ from: cut, to: line.to }); + return kept; +} + +function buildDecorations(state: EditorState, longLines: Line[]): DecorationSet { + if (longLines.length === 0) { + return Decoration.none; + } + + const tree = treeForLongLines(state, longLines); + const ranges: Range[] = []; + for (const line of longLines) { + for (const { from, to } of collapsesForLine(state, tree, line)) { + ranges.push(Decoration.replace({ widget: new LargeValueWidget(from, to) }).range(from, to)); + } + } + return Decoration.set(ranges); +} + +interface LargeValueState { + longLines: Line[]; + decorations: DecorationSet; +} + +export const largeValueField = StateField.define({ + create(state) { + const longLines = findLongLines(state.doc.toString()); + return { longLines, decorations: buildDecorations(state, longLines) }; + }, + + update(value, tr) { + if (tr.docChanged) { + const longLines = findLongLines(tr.state.doc.toString()); + return { longLines, decorations: buildDecorations(tr.state, longLines) }; + } + // Parsing is incremental, so a long line may only become a known token later. Documents + // with no long line can never gain a collapse, so they skip this entirely. + if (value.longLines.length > 0 && syntaxTree(tr.startState) !== syntaxTree(tr.state)) { + return { ...value, decorations: buildDecorations(tr.state, value.longLines) }; + } + return value; + }, + + provide: (f) => [ + EditorView.decorations.from(f, (v) => v.decorations), + // Step the cursor over a placeholder instead of stranding it inside + EditorView.atomicRanges.of( + (view) => view.state.field(f, false)?.decorations ?? Decoration.none, + ), + ], +}); + +export const largeValues: Extension = [largeValueField]; diff --git a/apps/yaak-client/lib/docFingerprint.test.ts b/apps/yaak-client/lib/docFingerprint.test.ts new file mode 100644 index 00000000..51b335ee --- /dev/null +++ b/apps/yaak-client/lib/docFingerprint.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, test } from "vite-plus/test"; +import { docFingerprint } from "./docFingerprint"; + +const sampled = (text: string) => docFingerprint(text, { exact: false }); +const exact = (text: string) => docFingerprint(text, { exact: true }); + +/** Same length as `text`, with one character changed at `at` */ +const changeAt = (text: string, at: number) => + `${text.slice(0, at)}${text[at] === "b" ? "c" : "b"}${text.slice(at + 1)}`; + +describe("docFingerprint", () => { + test("is stable for the same text", () => { + expect(sampled("hello")).toBe(sampled("hello")); + expect(exact("hello")).toBe(exact("hello")); + }); + + test("differs on different short text", () => { + expect(sampled("hello")).not.toBe(sampled("world")); + }); + + test("differs on length alone", () => { + expect(sampled("a".repeat(1_000_000))).not.toBe(sampled("a".repeat(1_000_001))); + }); + + test("hashes small documents in full, so any change is caught", () => { + const doc = "a".repeat(100); + for (let i = 0; i < doc.length; i++) { + expect(sampled(doc)).not.toBe(sampled(changeAt(doc, i))); + } + }); + + describe("sampled", () => { + const doc = "a".repeat(1_000_000); + + test("notices a change at the start", () => { + expect(sampled(doc)).not.toBe(sampled(changeAt(doc, 0))); + }); + + test("notices a change at the end", () => { + expect(sampled(doc)).not.toBe(sampled(changeAt(doc, doc.length - 1))); + }); + + test("notices a change in the middle", () => { + expect(sampled(doc)).not.toBe(sampled(changeAt(doc, doc.length / 2))); + }); + + test("misses a same-length change between the samples", () => { + // The documented cost of sampling. Read-only editors accept it because the worst a + // collision can do there is restore a fold or the cursor to the wrong place. + expect(sampled(doc)).toBe(sampled(changeAt(doc, 250_000))); + }); + }); + + describe("exact", () => { + const doc = "a".repeat(1_000_000); + + test("catches a same-length change anywhere, which is why editable docs use it", () => { + for (const at of [0, 250_000, doc.length / 2, 750_000, doc.length - 1]) { + expect(exact(doc)).not.toBe(exact(changeAt(doc, at))); + } + }); + + test("does not collide with the sampled fingerprint of the same text", () => { + expect(exact(doc)).not.toBe(sampled(doc)); + }); + }); +}); diff --git a/apps/yaak-client/lib/docFingerprint.ts b/apps/yaak-client/lib/docFingerprint.ts new file mode 100644 index 00000000..3e555c7d --- /dev/null +++ b/apps/yaak-client/lib/docFingerprint.ts @@ -0,0 +1,33 @@ +import { md5 } from "js-md5"; + +/** How much of each end and the middle to hash */ +const SAMPLE_CHARS = 512; + +/** + * Identifies a document, so a cached editor state is never restored onto different content. + * + * Hashing the whole document costs about 4 ms per megabyte and is paid on every editor update + * as well as on restore, which is a lot of work for a document nobody can edit. + * + * `exact` decides how much certainty that buys. Editable documents get a full hash, because a + * collision there would restore undo history belonging to other content and let an undo write + * nonsense into the body. Read-only documents get a sampled one: they hold the megabyte-sized + * responses this exists for, their history can never be applied, and the worst a collision can + * do is put a fold or the cursor in the wrong place. + * + * Sampling still pins the exact length plus three windows, so a colliding pair has to agree on + * all four and differ only in between. Documents small enough to hash outright still are. + */ +export function docFingerprint(text: string, { exact }: { exact: boolean }): string { + if (exact || text.length <= SAMPLE_CHARS * 3) { + return `${text.length}:${md5(text)}`; + } + + const middle = Math.floor((text.length - SAMPLE_CHARS) / 2); + return [ + text.length, + md5(text.slice(0, SAMPLE_CHARS)), + md5(text.slice(middle, middle + SAMPLE_CHARS)), + md5(text.slice(-SAMPLE_CHARS)), + ].join(":"); +}