diff --git a/Cargo.lock b/Cargo.lock
index a286d1852..9ff5f085a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -11852,6 +11852,7 @@ dependencies = [
"base64 0.23.1",
"bytes",
"chrono",
+ "encoding_rs",
"flate2",
"http-body-util",
"hyper",
diff --git a/apps/yaak-client/components/WebsocketResponsePane.tsx b/apps/yaak-client/components/WebsocketResponsePane.tsx
index 46cebd3a8..418d5ae5e 100644
--- a/apps/yaak-client/components/WebsocketResponsePane.tsx
+++ b/apps/yaak-client/components/WebsocketResponsePane.tsx
@@ -1,13 +1,14 @@
import type { WebsocketEvent, WebsocketRequest } from "@yaakapp-internal/models";
import {
HStack,
+ hexdump,
+ HexdumpViewer,
Icon,
LoadingIcon,
VStack,
useStateWithDeps,
languageFromContentType,
} from "@yaakapp-internal/ui";
-import { hexy } from "hexy";
import { useAtomValue } from "jotai";
import { useMemo, useState } from "react";
import { useFormatText } from "../hooks/useFormatText";
@@ -19,6 +20,7 @@ import {
} from "../hooks/usePinnedWebsocketConnection";
import { Button } from "./core/Button";
import { Editor } from "./core/Editor/LazyEditor";
+import { renderEditor } from "./core/Editor/renderEditor";
import { type EventDetailAction, EventDetailHeader, EventViewer } from "./core/EventViewer";
import { EventViewerRow } from "./core/EventViewerRow";
import { HotkeyList } from "./core/HotkeyList";
@@ -122,10 +124,10 @@ function WebsocketEventRow({
messageType === "error"
? "alert_triangle"
: messageType === "close" || messageType === "open"
- ? "info"
- : isServer
- ? "arrow_big_down_dash"
- : "arrow_big_up_dash";
+ ? "info"
+ : isServer
+ ? "arrow_big_down_dash"
+ : "arrow_big_up_dash";
const content =
messageType === "close" ? (
@@ -172,7 +174,7 @@ function WebsocketEventDetail({
}) {
const message = useMemo(() => {
if (hexDump) {
- return event.message ? hexy(event.message) : "";
+ return event.message ? hexdump(event.message) : "";
}
return event.message ? new TextDecoder("utf-8").decode(Uint8Array.from(event.message)) : "";
}, [event.message, hexDump]);
@@ -232,6 +234,8 @@ function WebsocketEventDetail({
) : event.message.length === 0 ? (
No Content
+ ) : hexDump ? (
+
) : (
h.name.toLowerCase() === "content-type")?.value ?? null;
}
-export function getCharsetFromContentType(headers: HttpResponseHeader[]): string | null {
- const contentType = getContentTypeFromHeaders(headers);
- if (contentType == null) return null;
-
- const mimeType = getMimeTypeFromContentType(contentType);
- return mimeType.parameters.get("charset") ?? null;
-}
-
export function isBaseEnvironment(environment: Environment): boolean {
return environment.parentModel === "workspace";
}
diff --git a/apps/yaak-client/lib/responseBody.ts b/apps/yaak-client/lib/responseBody.ts
index bc0fe23b6..3bfdfa8b8 100644
--- a/apps/yaak-client/lib/responseBody.ts
+++ b/apps/yaak-client/lib/responseBody.ts
@@ -3,7 +3,9 @@ import type { FilterResponse } from "@yaakapp-internal/plugins";
import type { ServerSentEvent, SseSummary } from "@yaakapp-internal/sse";
import { candidateJsonPayloadsFromSseText, computeSseSummary } from "@yaakapp-internal/sse";
import { rpc } from "./rpc";
+import { decodeBodyText } from "@yaakapp-internal/lib/decodeBodyText";
import { platform } from "@yaakapp-internal/platform";
+import { getContentTypeFromHeaders } from "./model_util";
/**
* Reading a response body means naming the response, never the file it lives
@@ -74,5 +76,6 @@ export async function getResponseBodyBytes(
async function getResponseBodyDecoded(response: HttpResponse): Promise {
const bytes = await getResponseBodyBytes(response);
- return bytes == null ? null : new TextDecoder("utf-8").decode(bytes);
+ if (bytes == null) return null;
+ return decodeBodyText(bytes, getContentTypeFromHeaders(response.headers));
}
diff --git a/apps/yaak-client/package.json b/apps/yaak-client/package.json
index 26bfaba2a..3edfa72a9 100644
--- a/apps/yaak-client/package.json
+++ b/apps/yaak-client/package.json
@@ -48,7 +48,6 @@
"focus-trap-react": "^12.0.3",
"fuzzbunny": "^1.0.1",
"graphql": "^17.0.2",
- "hexy": "^0.4.0",
"history": "^5.3.0",
"jotai": "^2.18.0",
"jotai-family": "^1.0.1",
diff --git a/apps/yaak-proxy/components/FlowBodyViewer.tsx b/apps/yaak-proxy/components/FlowBodyViewer.tsx
index 133ff4c96..b86c7e2b8 100644
--- a/apps/yaak-proxy/components/FlowBodyViewer.tsx
+++ b/apps/yaak-proxy/components/FlowBodyViewer.tsx
@@ -3,7 +3,10 @@ import { formatSize } from "@yaakapp-internal/lib/formatSize";
import type { Flow, FlowBody } from "@yaakapp-internal/proxy-lib";
import {
Banner,
+ Button,
CsvViewer,
+ HexdumpViewer,
+ Icon,
ImageViewer,
languageFromContentType,
LoadingIcon,
@@ -14,16 +17,17 @@ import {
} from "@yaakapp-internal/ui";
import type { CodeEditorProps } from "@yaakapp-internal/ui/editor";
import type { ReactNode } from "react";
-import { useCallback, useEffect, useState } from "react";
+import { useCallback, useEffect, useMemo, useState } from "react";
import {
- base64ToBytes,
- bytesToText,
+ bodyAsText,
+ bodyImageType,
filterText,
findHeader,
+ flowBodyQueryOptions,
formatText,
mimeEssence,
} from "../lib/body";
-import { rpc } from "../lib/rpc";
+import { useSaveFlowBody } from "../lib/flowActions";
interface Props {
flow: Flow;
@@ -32,18 +36,7 @@ interface Props {
}
export function useFlowBody(flow: Flow, which: FlowBody, enabled = true) {
- return useQuery({
- enabled,
- queryKey: ["get_flow_body", flow.id, which, flow.updatedAt],
- queryFn: async () => {
- const res = await rpc("get_flow_body", { flowId: flow.id, which });
- return {
- bytes: base64ToBytes(res.data),
- truncated: res.truncated,
- storedSize: res.storedSize,
- };
- },
- });
+ return useQuery({ ...flowBodyQueryOptions(flow, which), enabled });
}
export function FlowBodyViewer({ flow, which, pretty }: Props) {
@@ -60,8 +53,8 @@ export function FlowBodyViewer({ flow, which, pretty }: Props) {
return {body.error.message};
}
- const { bytes, truncated, storedSize } = body.data;
- if (bytes.length === 0) {
+ const { bytes, truncated, storedSize, decodeError } = body.data;
+ if (bytes.length === 0 && decodeError == null) {
return {which === "request" ? "No request body" : "Empty"};
}
@@ -70,12 +63,16 @@ export function FlowBodyViewer({ flow, which, pretty }: Props) {
const mime = mimeEssence(contentType) ?? "";
const wireSize = which === "request" ? flow.requestSize : flow.responseSize;
const wireTruncated = which === "request" ? flow.requestTruncated : flow.responseTruncated;
+ // An image by its bytes or type, else text whatever the content type says, so a page sent with
+ // an odd one still reads: `viewerText`'s rule.
+ const imageType = decodeError == null ? bodyImageType(bytes, truncated, contentType) : null;
+ const text =
+ decodeError == null && imageType == null ? bodyAsText(bytes, truncated, contentType) : null;
let viewer: ReactNode;
- if (mime.startsWith("image/svg")) {
- viewer = ;
- } else if (mime.startsWith("image/")) {
- viewer = ;
+ if (decodeError != null) {
+ // What failed to decode is as it was on the wire, compressed, so never text.
+ viewer = ;
} else if (mime.startsWith("multipart/") && pretty) {
const boundary = contentType?.match(/;\s*boundary="?([^";]+)"?/i)?.[1] ?? "unknown";
viewer = (
@@ -86,14 +83,17 @@ export function FlowBodyViewer({ flow, which, pretty }: Props) {
textViewerProps={{ formatText }}
/>
);
+ } else if (imageType != null) {
+ viewer = ;
+ } else if (text == null) {
+ viewer = ;
+ } else if (mime.startsWith("image/svg")) {
+ viewer = ;
} else if (/csv|tab-separated/.test(mime) && pretty) {
- viewer = ;
+ viewer = ;
} else {
- const text = bytesToText(bytes, truncated);
const language = languageFromContentType(contentType, text);
- if (text == null) {
- viewer = Binary body, {formatSize(bytes.length)};
- } else if (language === "html" && pretty && which === "response") {
+ if (language === "html" && pretty && which === "response") {
viewer = ;
} else {
viewer = (
@@ -102,16 +102,60 @@ export function FlowBodyViewer({ flow, which, pretty }: Props) {
}
}
- if (!truncated) return viewer;
+ if (!truncated && decodeError == null) return viewer;
return (
-
-
- {wireTruncated
- ? `Showing the first ${formatSize(storedSize)} of ${formatSize(wireSize)}, the capture limit. The app received all of it.`
- : `Showing the first ${formatSize(bytes.length)} of the decoded body, the capture limit. The app received all of it.`}
-
-
{viewer}
+
+ {decodeError != null && (
+
+ {decodeError}. Showing the body as it was on the wire.
+
+ )}
+ {truncated && (
+
+ {wireTruncated || decodeError != null
+ ? `Showing the first ${formatSize(storedSize)} of ${formatSize(wireSize)}, the capture limit. The app received all of it.`
+ : `Showing the first ${formatSize(bytes.length)} of the decoded body, the capture limit. The app received all of it.`}
+
+ )}
+
{viewer}
+
+ );
+}
+
+/** Bytes shown in the hexdump; the rest are in the saved file */
+const HEXDUMP_LIMIT = 256 * 1024;
+
+/** A body that isn't text: a hexdump, and a way to save it */
+function BinaryBody({
+ flow,
+ which,
+ bytes,
+}: {
+ flow: Flow;
+ which: FlowBody;
+ bytes: Uint8Array;
+}) {
+ const saveBody = useSaveFlowBody();
+ const shown = useMemo(() => bytes.subarray(0, HEXDUMP_LIMIT), [bytes]);
+
+ return (
+
+
+
+ Binary body, {formatSize(bytes.length)}
+ {shown.length < bytes.length && `, first ${formatSize(shown.length)} shown`}
+
+ }
+ onClick={() => saveBody(flow, which)}
+ >
+ Save to File
+
+
+
);
}
diff --git a/apps/yaak-proxy/lib/body.test.ts b/apps/yaak-proxy/lib/body.test.ts
index 48c83d61c..804d3d683 100644
--- a/apps/yaak-proxy/lib/body.test.ts
+++ b/apps/yaak-proxy/lib/body.test.ts
@@ -1,5 +1,17 @@
-import { describe, expect, test } from "vite-plus/test";
-import { bytesToText } from "./body";
+import { QueryClient, QueryObserver } from "@tanstack/react-query";
+import type { Flow } from "@yaakapp-internal/proxy-lib";
+import { describe, expect, test, vi } from "vite-plus/test";
+import {
+ bodyAsText,
+ bodyFileName,
+ bodyImageType,
+ bytesToText,
+ flowBodyQueryOptions,
+ viewerText,
+} from "./body";
+import { rpc } from "./rpc";
+
+vi.mock("./rpc", () => ({ rpc: vi.fn() }));
const encode = (text: string) => new TextEncoder().encode(text);
@@ -18,3 +30,198 @@ describe("bytesToText", () => {
expect(bytesToText(cut, true)).toBe("abc");
});
});
+
+describe("bodyAsText", () => {
+ test("reads UTF-8 as text", () => {
+ expect(bodyAsText(encode("
Forbidden
\n\tcafé"))).toBe("
Forbidden
\n\tcafé");
+ });
+
+ test("reads unlabeled Latin-1 text as text, as windows-1252", () => {
+ const page = "
Accès refusé : vous n'avez pas l'autorisation d'accéder à cette page.
";
+ const latin1 = Uint8Array.from(page, (c) => c.charCodeAt(0));
+ expect(bodyAsText(latin1)).toBe(page);
+ });
+
+ test("reads bytes UTF-8 can't read as binary, though windows-1252 reads every one", () => {
+ let seed = 7;
+ const random = Uint8Array.from({ length: 600 }, () => {
+ seed = (seed * 1103515245 + 12345) & 0x7fffffff;
+ return seed >> 16;
+ });
+ expect(bodyAsText(random)).toBeNull();
+ // All high bytes: letters and symbols in windows-1252, none of them UTF-8
+ const high = Uint8Array.from({ length: 600 }, (_, i) => 0xa0 + (i % 96));
+ expect(bodyAsText(high)).toBeNull();
+ expect(bodyAsText(high, false, "text/plain")).toBeNull();
+ });
+
+ test("reads protobuf and other binary as binary, even when it is valid UTF-8", () => {
+ // Field tags, lengths and varints: control characters between short strings.
+ const protobuf = new Uint8Array([
+ 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x31, 0x12, 0x02, 0x08, 0x01, 0x1a, 0x03, 0x10, 0x96, 0x01,
+ 0x22, 0x00, 0x2a, 0x02, 0x18, 0x07,
+ ]);
+ expect(bodyAsText(protobuf)).toBeNull();
+ expect(bodyAsText(new Uint8Array([0x01, 0x02, 0x03, 0x04, 0x41, 0x42]))).toBeNull();
+ expect(bodyAsText(new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))).toBeNull();
+ });
+});
+
+describe("viewerText", () => {
+ // A PNG signature, then a tEXt chunk of printable metadata far longer than the image
+ const png = new Uint8Array([
+ 0x89,
+ ...encode("PNG\r\n"),
+ 0x1a,
+ 0x0a,
+ 0x00,
+ 0x00,
+ 0x12,
+ 0xc0,
+ ...encode(`tEXtDescription\0${"A long printable description. ".repeat(160)}`),
+ ]);
+
+ test("keeps an image with printable metadata an image", () => {
+ expect(bodyAsText(png)).not.toBeNull();
+ expect(viewerText(png, false, "image/png")).toBeNull();
+ });
+
+ test("reads text sent as an image as text, and other types text first", () => {
+ expect(viewerText(encode("
Forbidden
"), false, "image/png")).toBe("
Forbidden
");
+ expect(bodyImageType(encode("
Forbidden
"), false, "image/png")).toBeNull();
+ expect(viewerText(encode(""), false, "image/svg+xml")).toBe("");
+ expect(bodyImageType(encode(""), false, "image/svg+xml")).toBeNull();
+ const bmwText = encode("BMW sells cars.\n");
+ expect(viewerText(bmwText, false, "text/plain")).toBe("BMW sells cars.\n");
+ });
+
+ // A 1×1 black 24-bit BMP: every byte below 0x80, so valid UTF-8
+ // File header, then the 40-byte DIB header, then one pixel padded to four bytes
+ // prettier-ignore
+ const bmp = new Uint8Array([
+ 0x42, 0x4d, 58, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0,
+ 40, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 24, 0, 0, 0, 0, 0, 4, 0, 0, 0,
+ 0x13, 0x0b, 0, 0, 0x13, 0x0b, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ ]);
+
+ test("keeps an image that is valid UTF-8 an image", () => {
+ expect(bytesToText(bmp)).not.toBeNull();
+ expect(bodyImageType(bmp, false, "image/bmp")).toBe("image/bmp");
+ expect(viewerText(bmp, false, "image/bmp")).toBeNull();
+ });
+
+ test("previews an image by its signature whatever its content type", () => {
+ expect(bodyImageType(bmp, false, "application/octet-stream")).toBe("image/bmp");
+ expect(bodyImageType(png, false, "application/octet-stream")).toBe("image/png");
+ expect(bodyImageType(png, false, null)).toBe("image/png");
+ expect(viewerText(png, false, "application/octet-stream")).toBeNull();
+ const gif = encode("GIF89a\x01\0\x01\0");
+ expect(bodyImageType(gif, false, "text/plain")).toBe("image/gif");
+ });
+
+ test("previews formats not every webview draws only when their content type says so", () => {
+ const tiff = new Uint8Array([0x49, 0x49, 0x2a, 0x00, 0x08, 0x00, 0x00, 0x00, 0xff, 0x80]);
+ const heic = new Uint8Array([0, 0, 0, 0x18, ...encode("ftypheic"), 0, 0, 0, 0, 0xff, 0x80]);
+ for (const image of [tiff, heic]) {
+ expect(bodyImageType(image, false, "application/octet-stream")).toBeNull();
+ expect(viewerText(image, false, "application/octet-stream")).toBeNull();
+ }
+ expect(bodyImageType(tiff, false, "image/tiff")).toBe("image/tiff");
+ expect(bodyImageType(heic, false, "image/heic")).toBe("image/heic");
+ });
+
+ test("reads text in the charset its content type names", () => {
+ const page = "Accès refusé : vous n'avez pas l'autorisation d'accéder à cette page.";
+ const latin1 = Uint8Array.from(page, (c) => c.charCodeAt(0));
+ expect(viewerText(latin1, false, "text/html; charset=iso-8859-1")).toBe(page);
+ // "こんにちは、世界" in Shift_JIS
+ const shiftJis = new Uint8Array([
+ 0x82, 0xb1, 0x82, 0xf1, 0x82, 0xc9, 0x82, 0xbf, 0x82, 0xcd, 0x81, 0x41, 0x90, 0xa2, 0x8a,
+ 0x45,
+ ]);
+ expect(viewerText(shiftJis, false, "text/plain; charset=shift_jis")).toBe("こんにちは、世界");
+ });
+
+ test("reads unlabeled Latin-1 as windows-1252, and UTF-8 as UTF-8", () => {
+ const page = "Accès refusé : vous n'avez pas l'autorisation d'accéder à cette page.";
+ const latin1 = Uint8Array.from(page, (c) => c.charCodeAt(0));
+ expect(viewerText(latin1, false, "application/octet-stream")).toBe(page);
+ expect(viewerText(encode(page), false, "application/octet-stream")).toBe(page);
+ expect(viewerText(encode(page), false, "text/plain; charset=utf-8")).toBe(page);
+ });
+});
+
+const flow = (url: string, contentType: string | null) =>
+ ({
+ id: "fl_1",
+ url,
+ updatedAt: "2026-11-04T00:00:00",
+ requestHeaders: contentType == null ? [] : [{ name: "Content-Type", value: contentType }],
+ responseHeaders: contentType == null ? [] : [{ name: "content-type", value: contentType }],
+ }) as unknown as Flow;
+
+describe("bodyFileName", () => {
+ test("uses the URL's file name for a response", () => {
+ expect(bodyFileName(flow("https://a.test/img/logo%20big.png?v=2", null), "response")).toBe(
+ "logo big.png",
+ );
+ });
+
+ test("names a body from its content type otherwise", () => {
+ const protobuf = flow("https://a.test/v1:GetModels", "application/x-protobuf");
+ expect(bodyFileName(protobuf, "request")).toBe("request.protobuf");
+ expect(bodyFileName(protobuf, "response")).toBe("response.protobuf");
+ expect(bodyFileName(flow("https://a.test/", "text/plain; charset=utf-8"), "response")).toBe(
+ "response.txt",
+ );
+ expect(bodyFileName(flow("https://a.test/api", "application/problem+json"), "response")).toBe(
+ "response.json",
+ );
+ expect(bodyFileName(flow("https://a.test/data.json", null), "request")).toBe("request.bin");
+ expect(bodyFileName(flow("not a url", "application/octet-stream"), "response")).toBe(
+ "response.bin",
+ );
+ });
+});
+
+describe("flowBodyQueryOptions", () => {
+ test("shows a body that fails to read at once, without retrying", async () => {
+ vi.mocked(rpc).mockRejectedValue(new Error("No flow fl_1"));
+ // React Query retries three times in a browser, and not at all where there is no window.
+ const client = new QueryClient({ defaultOptions: { queries: { retry: 3 } } });
+ const observer = new QueryObserver(
+ client,
+ flowBodyQueryOptions(flow("https://a.test/", null), "response"),
+ );
+ const failed = new Promise((resolve) => {
+ const unsubscribe = observer.subscribe((result) => {
+ if (result.status === "error") {
+ unsubscribe();
+ resolve(result.error.message);
+ }
+ });
+ });
+ // React Query's default retries would keep it pending for seconds.
+ const timeout = new Promise((resolve) => setTimeout(() => resolve("pending"), 500));
+ expect(await Promise.race([failed, timeout])).toBe("No flow fl_1");
+ expect(rpc).toHaveBeenCalledTimes(1);
+ client.clear();
+ });
+
+ test("passes on why a body failed to decode, with its bytes as they were", async () => {
+ vi.mocked(rpc).mockResolvedValue({
+ data: btoa("\x78\x9c\x01"),
+ truncated: false,
+ storedSize: 3,
+ decodeError: "Failed to decode Deflate body: deflate decompression error",
+ });
+ const client = new QueryClient();
+ const body = await client.fetchQuery(
+ flowBodyQueryOptions(flow("https://a.test/", null), "request"),
+ );
+ expect(Array.from(body.bytes)).toEqual([0x78, 0x9c, 0x01]);
+ expect(body.decodeError).toBe("Failed to decode Deflate body: deflate decompression error");
+ client.clear();
+ });
+});
diff --git a/apps/yaak-proxy/lib/body.ts b/apps/yaak-proxy/lib/body.ts
index b3129dfb3..ae8d896ca 100644
--- a/apps/yaak-proxy/lib/body.ts
+++ b/apps/yaak-proxy/lib/body.ts
@@ -1,8 +1,32 @@
-import type { FilterLanguage, FlowBody } from "@yaakapp-internal/proxy-lib";
+import { queryOptions } from "@tanstack/react-query";
+import {
+ contentTypeCharset,
+ decodeBodyText,
+ strictUtf8Text,
+} from "@yaakapp-internal/lib/decodeBodyText";
+import type { FilterLanguage, Flow, FlowBody } from "@yaakapp-internal/proxy-lib";
import type { CodeEditorProps } from "@yaakapp-internal/ui/editor";
import vkBeautify from "vkbeautify";
import { rpc } from "./rpc";
+/** A flow's body for the viewer: decoded, or as it was on the wire when `decodeError` is set */
+export function flowBodyQueryOptions(flow: Flow, which: FlowBody) {
+ return queryOptions({
+ queryKey: ["get_flow_body", flow.id, which, flow.updatedAt],
+ queryFn: async () => {
+ const res = await rpc("get_flow_body", { flowId: flow.id, which });
+ return {
+ bytes: base64ToBytes(res.data),
+ truncated: res.truncated,
+ storedSize: res.storedSize,
+ decodeError: res.decodeError ?? null,
+ };
+ },
+ // Reading a stored body fails the same way every time, so the error shows at once.
+ retry: false,
+ });
+}
+
export function base64ToBytes(data: string): Uint8Array {
const binary = atob(data);
const bytes = new Uint8Array(binary.length);
@@ -16,19 +40,104 @@ export function base64ToBytes(data: string): Uint8Array {
* The text of a body, or null when it isn't valid UTF-8. A truncated body may end partway
* through a character, so up to three trailing bytes are dropped before giving up on it.
*/
-export function bytesToText(bytes: Uint8Array, truncated = false): string | null {
- const decoder = new TextDecoder("utf-8", { fatal: true });
- const maxTrim = truncated ? Math.min(3, bytes.length) : 0;
- for (let trim = 0; trim <= maxTrim; trim++) {
- try {
- return decoder.decode(bytes.subarray(0, bytes.length - trim));
- } catch {
- // Try again without the last byte
- }
+export const bytesToText = strictUtf8Text;
+
+/** Characters looked at to tell text from binary */
+const TEXT_SAMPLE = 64 * 1024;
+
+/** The share of those that may be control characters or undecodable bytes in text */
+const MAX_UNPRINTABLE = 0.1;
+
+/**
+ * The body as text, whatever its content type says, or null when it is binary. It is decoded in
+ * its content type's charset, or else as UTF-8, or windows-1252 when it isn't valid UTF-8. It is
+ * binary when it is full of control characters, as protobuf is, or, without a charset, of bytes
+ * UTF-8 can't read: windows-1252 reads every byte as something, compressed data included.
+ */
+export function bodyAsText(
+ bytes: Uint8Array,
+ truncated = false,
+ contentType: string | null = null,
+): string | null {
+ const text = decodeBodyText(bytes, contentType, { truncated });
+ const judged =
+ contentTypeCharset(contentType) != null
+ ? text
+ : (strictUtf8Text(bytes, truncated) ?? new TextDecoder("utf-8").decode(bytes));
+ let total = 0;
+ let unprintable = 0;
+ for (const char of judged) {
+ if (total === TEXT_SAMPLE) break;
+ total++;
+ const code = char.codePointAt(0) ?? 0;
+ const control = code < 0x20 && code !== 0x09 && code !== 0x0a && code !== 0x0d;
+ if (control || code === 0x7f || code === 0xfffd) unprintable++;
+ }
+ return unprintable <= total * MAX_UNPRINTABLE ? text : null;
+}
+
+const ascii = (bytes: Uint8Array, start: number, text: string) =>
+ Array.from(text).every((c, i) => bytes[start + i] === c.charCodeAt(0));
+
+/**
+ * The image type a body's first bytes name, or null when they name none. Only types every
+ * platform's webview draws, so a body sniffed as an image always previews: TIFF, HEIC and AVIF
+ * preview only when their content type says so.
+ */
+export function imageSignature(bytes: Uint8Array): string | null {
+ const b = bytes;
+ if (b.length < 4) return null;
+ if (b[0] === 0x89 && ascii(b, 1, "PNG\r\n\x1a\n")) return "image/png";
+ if (b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff) return "image/jpeg";
+ if (ascii(b, 0, "GIF87a") || ascii(b, 0, "GIF89a")) return "image/gif";
+ if (ascii(b, 0, "RIFF") && ascii(b, 8, "WEBP")) return "image/webp";
+ if (ascii(b, 0, "\0\0\x01\0") && b.length >= 6 && (b[4] ?? 0) + (b[5] ?? 0) > 0) {
+ return "image/x-icon";
+ }
+ // "BM" alone starts plenty of text, so the reserved bytes and DIB header size must fit too.
+ const dibSize = (b[14] ?? 0) | ((b[15] ?? 0) << 8) | ((b[16] ?? 0) << 16) | ((b[17] ?? 0) << 24);
+ if (
+ ascii(b, 0, "BM") &&
+ ascii(b, 6, "\0\0\0\0") &&
+ [12, 40, 52, 56, 64, 108, 124].includes(dibSize)
+ ) {
+ return "image/bmp";
}
return null;
}
+/**
+ * The image type to preview a body as, or null when it isn't one: what its first bytes name,
+ * whatever its content type, or else an `image/*` content type other than SVG, unless the body
+ * is genuinely text, as an error page sent with the image's type is.
+ */
+export function bodyImageType(
+ bytes: Uint8Array,
+ truncated: boolean,
+ contentType: string | null,
+): string | null {
+ const signed = imageSignature(bytes);
+ if (signed != null) return signed;
+ const mime = mimeEssence(contentType) ?? "";
+ if (!mime.startsWith("image/") || mime.startsWith("image/svg")) return null;
+ const genuineText =
+ strictUtf8Text(bytes, truncated) != null && bodyAsText(bytes, truncated, contentType) != null;
+ return genuineText ? null : mime;
+}
+
+/**
+ * The body as the viewer reads it: text first, whatever its content type, except an image, which
+ * is never text.
+ */
+export function viewerText(
+ bytes: Uint8Array,
+ truncated: boolean,
+ contentType: string | null,
+): string | null {
+ if (bodyImageType(bytes, truncated, contentType) != null) return null;
+ return bodyAsText(bytes, truncated, contentType);
+}
+
export async function formatText(
text: string,
language: CodeEditorProps["language"],
@@ -81,6 +190,33 @@ export function shortContentType(contentType: string | null): string | null {
return suffix.replace(/^x-/, "");
}
+/** Extensions for media types whose subtype isn't one */
+const EXTENSIONS: Record = {
+ "application/javascript": "js",
+ "application/octet-stream": "bin",
+ "image/jpeg": "jpg",
+ "image/svg+xml": "svg",
+ "text/javascript": "js",
+ "text/markdown": "md",
+ "text/plain": "txt",
+};
+
+/** A name to offer for a saved body: the URL's file name, or one from the content type */
+export function bodyFileName(flow: Flow, which: FlowBody): string {
+ let last = "";
+ try {
+ last = decodeURIComponent(new URL(flow.url).pathname.split("/").pop() ?? "");
+ } catch {
+ // Not a URL, or not encoded properly
+ }
+ if (which === "response" && /^[^\\/:*?"<>|]+\.[a-z0-9]+$/i.test(last)) return last;
+ const headers = which === "request" ? flow.requestHeaders : flow.responseHeaders;
+ const contentType = findHeader(headers, "content-type");
+ const essence = mimeEssence(contentType);
+ const extension = (essence == null ? null : EXTENSIONS[essence]) ?? shortContentType(contentType);
+ return `${which}.${extension?.replace(/[^a-z0-9-]/gi, "") || "bin"}`;
+}
+
export function findHeader(headers: { name: string; value: string }[], name: string) {
const lower = name.toLowerCase();
return headers.find((h) => h.name.toLowerCase() === lower)?.value ?? null;
diff --git a/apps/yaak-proxy/lib/flowActions.ts b/apps/yaak-proxy/lib/flowActions.ts
index 61d64848c..c019c6937 100644
--- a/apps/yaak-proxy/lib/flowActions.ts
+++ b/apps/yaak-proxy/lib/flowActions.ts
@@ -1,9 +1,10 @@
import type { ExportHarRequest, Flow, FlowBody } from "@yaakapp-internal/proxy-lib";
import type { DropdownItem } from "@yaakapp-internal/ui";
import { save } from "@tauri-apps/plugin-dialog";
+import { formatSize } from "@yaakapp-internal/lib/formatSize";
import { atom, useSetAtom } from "jotai";
import { useCallback } from "react";
-import { base64ToBytes } from "./body";
+import { base64ToBytes, bodyFileName } from "./body";
import { curlBodyText, toCurl } from "./curl";
import { rpc } from "./rpc";
import { useShowToast } from "./toast";
@@ -14,10 +15,35 @@ export const saveDialogFlowAtom = atom(null);
/** A body as text, decoding what is not UTF-8 lossily, or null when there is none */
async function bodyText(flow: Flow, which: FlowBody): Promise {
const res = await rpc("get_flow_body", { flowId: flow.id, which });
+ if (res.decodeError != null) throw new Error(res.decodeError);
const bytes = base64ToBytes(res.data);
return bytes.length === 0 ? null : new TextDecoder().decode(bytes);
}
+/** Writes a flow's body to a file the person picks, as the viewer shows it */
+export function useSaveFlowBody(): (flow: Flow, which: FlowBody) => Promise {
+ const showToast = useShowToast();
+ return useCallback(
+ async (flow: Flow, which: FlowBody) => {
+ try {
+ const path = await save({ defaultPath: bodyFileName(flow, which) });
+ if (path == null) return;
+ const saved = await rpc("save_flow_body", { flowId: flow.id, which, path });
+ const notes = [
+ saved.raw ? "as it was on the wire, since it failed to decode" : null,
+ saved.truncated ? "cut at the capture limit" : null,
+ ].filter((n) => n != null);
+ showToast({
+ message: `Saved ${formatSize(saved.size)}${notes.length > 0 ? `, ${notes.join(", ")}` : ""}`,
+ });
+ } catch (err) {
+ showToast({ message: `Failed to save the ${which} body: ${String(err)}`, color: "danger" });
+ }
+ },
+ [showToast],
+ );
+}
+
/**
* Writes flows to a HAR file the person picks: `ids`, or every flow `filter` covers. Returns how
* many were written, or null when the save panel was cancelled.
@@ -38,6 +64,7 @@ export async function exportHar(
export function useFlowActions(): (flow: Flow) => DropdownItem[] {
const showToast = useShowToast();
const openSave = useSetAtom(saveDialogFlowAtom);
+ const saveBody = useSaveFlowBody();
return useCallback(
(flow: Flow) => {
@@ -75,6 +102,10 @@ export function useFlowActions(): (flow: Flow) => DropdownItem[] {
showToast({ message: `Failed to copy cURL: ${String(err)}`, color: "danger" });
return;
}
+ if (res.decodeError != null) {
+ showToast({ message: `Failed to copy cURL: ${res.decodeError}`, color: "danger" });
+ return;
+ }
body = curlBodyText(base64ToBytes(res.data), res.truncated);
// A command that sends other bytes than the app did would mislead.
if (body == null) {
@@ -102,6 +133,18 @@ export function useFlowActions(): (flow: Flow) => DropdownItem[] {
onSelect: () => copy("response body", () => bodyText(flow, "response")),
},
{ type: "separator" },
+ {
+ label: "Save Request Body to File",
+ icon: "download",
+ disabled: flow.requestSize === 0,
+ onSelect: () => saveBody(flow, "request"),
+ },
+ {
+ label: "Save Response Body to File",
+ icon: "download",
+ disabled: noResponse || flow.responseSize === 0,
+ onSelect: () => saveBody(flow, "response"),
+ },
{
label: "Export HAR",
icon: "download",
@@ -116,6 +159,6 @@ export function useFlowActions(): (flow: Flow) => DropdownItem[] {
},
];
},
- [openSave, showToast],
+ [openSave, saveBody, showToast],
);
}
diff --git a/crates-proxy/yaak-proxy-lib/Cargo.toml b/crates-proxy/yaak-proxy-lib/Cargo.toml
index fb6145358..05845b2c7 100644
--- a/crates-proxy/yaak-proxy-lib/Cargo.toml
+++ b/crates-proxy/yaak-proxy-lib/Cargo.toml
@@ -9,6 +9,7 @@ publish = false
base64 = "0.23.1"
bytes = "1"
chrono = { workspace = true, features = ["serde"] }
+encoding_rs = "0.8"
log = { workspace = true }
http-body-util = "0.1"
hyper = { version = "1", features = ["http1", "server"] }
diff --git a/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts b/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts
index 0ed90397d..629597ae7 100644
--- a/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts
+++ b/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts
@@ -133,6 +133,7 @@ export type GetFlowBodyRequest = { flowId: string, which: FlowBody, };
export type GetFlowBodyResponse = {
/**
* The body decoded with its Content-Encoding, at most the capture limit of it, in base64.
+ * When it fails to decode, the bytes as they were on the wire instead.
*/
data: string,
/**
@@ -142,7 +143,11 @@ truncated: boolean,
/**
* Bytes stored for the body, as they were on the wire.
*/
-storedSize: number, };
+storedSize: number,
+/**
+ * Why the body failed to decode, in which case `data` is as it was on the wire.
+ */
+decodeError?: string, };
export type GetProxyStateRequest = Record;
@@ -238,7 +243,27 @@ export type ProxyStatePayload = { state: ProxyState, };
export type RpcEventSchema = { flows_cleared: FlowsClearedPayload, model_write: ModelPayload, proxy_state_changed: ProxyStatePayload, };
-export type RpcSchema = { execute_action: [ActionInvocation, boolean], format_body: [FormatBodyRequest, string], list_actions: [ListActionsRequest, ListActionsResponse], list_flows: [FlowFilter, ListFlowsResponse], set_recording: [SetRecordingRequest, boolean], set_system_proxy_at_launch: [SetSystemProxyAtLaunchRequest, boolean], set_http_version: [SetHttpVersionRequest, HttpVersion], app_icon: [AppIconRequest, string | null], ca_status: [CaStatusRequest, CaStatus], ca_trust: [CaTrustRequest, CaStatus], clear_flows: [ClearFlowsRequest, number], export_har: [ExportHarRequest, number], filter_body: [FilterBodyRequest, FilterBodyResponse], get_flow_body: [GetFlowBodyRequest, GetFlowBodyResponse], get_proxy_state: [GetProxyStateRequest, GetProxyStateResponse], helper_register: [HelperRegisterRequest, SystemProxyStatus], helper_unregister: [HelperUnregisterRequest, SystemProxyStatus], list_workspaces: [ListWorkspacesRequest, ListWorkspacesResponse], preview_save: [PreviewSaveRequest, SavePreview], save_flow: [SaveFlowToYaakRequest, SavedFlow], system_proxy_set: [SystemProxySetRequest, SystemProxyStatus], system_proxy_status: [SystemProxyStatusRequest, SystemProxyStatus], take_over: [TakeOverRequest, boolean], };
+export type RpcSchema = { execute_action: [ActionInvocation, boolean], format_body: [FormatBodyRequest, string], list_actions: [ListActionsRequest, ListActionsResponse], list_flows: [FlowFilter, ListFlowsResponse], set_recording: [SetRecordingRequest, boolean], set_system_proxy_at_launch: [SetSystemProxyAtLaunchRequest, boolean], set_http_version: [SetHttpVersionRequest, HttpVersion], app_icon: [AppIconRequest, string | null], ca_status: [CaStatusRequest, CaStatus], ca_trust: [CaTrustRequest, CaStatus], clear_flows: [ClearFlowsRequest, number], export_har: [ExportHarRequest, number], filter_body: [FilterBodyRequest, FilterBodyResponse], get_flow_body: [GetFlowBodyRequest, GetFlowBodyResponse], get_proxy_state: [GetProxyStateRequest, GetProxyStateResponse], helper_register: [HelperRegisterRequest, SystemProxyStatus], helper_unregister: [HelperUnregisterRequest, SystemProxyStatus], list_workspaces: [ListWorkspacesRequest, ListWorkspacesResponse], preview_save: [PreviewSaveRequest, SavePreview], save_flow: [SaveFlowToYaakRequest, SavedFlow], save_flow_body: [SaveFlowBodyRequest, SaveFlowBodyResponse], system_proxy_set: [SystemProxySetRequest, SystemProxyStatus], system_proxy_status: [SystemProxyStatusRequest, SystemProxyStatus], take_over: [TakeOverRequest, boolean], };
+
+export type SaveFlowBodyRequest = { flowId: string, which: FlowBody,
+/**
+ * The file to write.
+ */
+path: string, };
+
+export type SaveFlowBodyResponse = {
+/**
+ * Bytes written.
+ */
+size: number,
+/**
+ * What was written is incomplete, as `GetFlowBodyResponse.truncated` is.
+ */
+truncated: boolean,
+/**
+ * The body failed to decode, so the file has it as it was on the wire.
+ */
+raw: boolean, };
export type SaveFlowToYaakRequest = { flowId: string, workspaceId: string, folderId?: string,
/**
diff --git a/crates-proxy/yaak-proxy-lib/src/bodies.rs b/crates-proxy/yaak-proxy-lib/src/bodies.rs
index ed3f07abf..cd249a6fd 100644
--- a/crates-proxy/yaak-proxy-lib/src/bodies.rs
+++ b/crates-proxy/yaak-proxy-lib/src/bodies.rs
@@ -67,6 +67,61 @@ pub async fn read_body(
Ok(DecodedBody { data, stored_size, truncated, decode_error })
}
+/// A body's text as the app's viewer reads it, as `decodeBodyText` in `@yaakapp-internal/lib`
+/// does: in the charset its content type names, when it names one the WHATWG Encoding Standard
+/// knows, or else as UTF-8, or windows-1252 when it isn't valid UTF-8, as browsers read text.
+pub fn body_text(data: &[u8], content_type: Option<&str>, truncated: bool) -> String {
+ let encoding = content_type
+ .and_then(charset_label)
+ .and_then(|label| encoding_rs::Encoding::for_label(label.as_bytes()))
+ // A label the standard maps to "replacement" is one `TextDecoder` refuses.
+ .filter(|encoding| *encoding != encoding_rs::REPLACEMENT);
+ if let Some(encoding) = encoding {
+ return encoding.decode_with_bom_removal(data).0.into_owned();
+ }
+ if let Some(text) = strict_utf8(data, truncated) {
+ return text;
+ }
+ encoding_rs::WINDOWS_1252.decode_without_bom_handling(data).0.into_owned()
+}
+
+/// The `charset` parameter of a content type
+fn charset_label(content_type: &str) -> Option<&str> {
+ content_type.split(';').skip(1).find_map(|param| {
+ let (name, value) = param.split_once('=')?;
+ name.trim().eq_ignore_ascii_case("charset").then(|| value.trim().trim_matches('"').trim())
+ })
+}
+
+/// The bytes as UTF-8 without a byte order mark, or None when they aren't valid UTF-8. A
+/// truncated body may end partway through a character, so up to three trailing bytes are dropped
+/// before giving up on it.
+fn strict_utf8(data: &[u8], truncated: bool) -> Option {
+ let max_trim = if truncated { data.len().min(3) } else { 0 };
+ (0..=max_trim).find_map(|trim| {
+ let text = std::str::from_utf8(&data[..data.len() - trim]).ok()?;
+ Some(text.strip_prefix('\u{feff}').unwrap_or(text).to_string())
+ })
+}
+
+/// A flow's body decoded as `read_body` does, or, when it fails to decode, the bytes as they were
+/// on the wire, with `decode_error` saying why.
+pub async fn read_body_or_raw(
+ blobs: &BlobManager,
+ flow: &Flow,
+ body: FlowBody,
+ limit: usize,
+) -> Result {
+ let decoded = read_body(blobs, flow, body, limit).await?;
+ if decoded.decode_error.is_none() {
+ return Ok(decoded);
+ }
+ let mut data = read_raw_body(blobs, &flow.id, body)?;
+ let truncated = flow.truncated(body) || data.len() > limit;
+ data.truncate(limit);
+ Ok(DecodedBody { data, truncated, ..decoded })
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -127,4 +182,94 @@ mod tests {
assert!(body.truncated);
assert_eq!(body.decode_error, None);
}
+
+ /// The cases `decodeBodyText`'s tests in `@yaakapp-internal/lib` cover, read the same way
+ #[test]
+ fn body_text_reads_as_the_viewer_does() {
+ let french = "Accès refusé : vous n'avez pas l'autorisation d'accéder à cette page.";
+ let latin1: Vec = french.chars().map(|c| c as u8).collect();
+ assert_eq!(body_text(&latin1, Some("text/html; charset=iso-8859-1"), false), french);
+ assert_eq!(body_text(&latin1, Some("text/plain; CHARSET=\"ISO-8859-1\""), false), french);
+ let shift_jis = [
+ 0x82, 0xb1, 0x82, 0xf1, 0x82, 0xc9, 0x82, 0xbf, 0x82, 0xcd, 0x81, 0x41, 0x90, 0xa2,
+ 0x8a, 0x45,
+ ];
+ assert_eq!(
+ body_text(&shift_jis, Some("text/plain; charset=Shift_JIS"), false),
+ "こんにちは、世界"
+ );
+
+ // Unlabeled and not UTF-8: windows-1252
+ assert_eq!(body_text(&latin1, Some("text/html"), false), french);
+ assert_eq!(body_text(&latin1, None, false), french);
+ assert_eq!(body_text(&[0x93, b'h', b'i', 0x94], None, false), "\u{201c}hi\u{201d}");
+ // A label `TextDecoder` refuses is no label.
+ assert_eq!(body_text(&latin1, Some("text/plain; charset=not-a-charset"), false), french);
+ assert_eq!(body_text(&latin1, Some("text/plain; charset=iso-2022-kr"), false), french);
+
+ // UTF-8, labeled or not; labeled, what isn't UTF-8 is replaced
+ let json = r#"{"city":"Zürich","greeting":"こんにちは"}"#;
+ assert_eq!(body_text(json.as_bytes(), None, false), json);
+ assert_eq!(
+ body_text(json.as_bytes(), Some("application/json; charset=utf-8"), false),
+ json
+ );
+ assert_eq!(body_text(b"a\xff", Some("text/plain; charset=utf-8"), false), "a\u{fffd}");
+
+ // A UTF-8 body cut partway through a character stays UTF-8
+ assert_eq!(body_text(&"abc€".as_bytes()[..5], None, true), "abc");
+ }
+
+ fn deflate_request(blobs: &BlobManager, id: &str, stored: &[u8]) -> Flow {
+ let mut writer = BodyChunkWriter::new(blobs.clone(), Flow::body_id(id, FlowBody::Request));
+ writer.write(stored).unwrap();
+ writer.finish().unwrap();
+ Flow {
+ id: id.into(),
+ request_size: stored.len() as i64,
+ request_headers: vec![crate::models::ProxyHeader {
+ name: "Content-Encoding".into(),
+ value: "deflate".into(),
+ }],
+ ..Default::default()
+ }
+ }
+
+ /// Browser SDKs send `deflate` both ways: zlib-wrapped, as the spec says, and raw.
+ #[tokio::test]
+ async fn deflate_request_bodies_decode_wrapped_or_raw() {
+ use std::io::Write;
+ let (_db, blobs, _rx) = yaak_models::init_in_memory().unwrap();
+ let logs = br#"[{"message":"hello","service":"browser"}]"#;
+ let mut zlib = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default());
+ zlib.write_all(logs).unwrap();
+ let mut raw =
+ flate2::write::DeflateEncoder::new(Vec::new(), flate2::Compression::default());
+ raw.write_all(logs).unwrap();
+
+ for (id, stored) in [
+ ("fl_zlib", zlib.finish().unwrap()),
+ ("fl_raw", raw.finish().unwrap()),
+ ] {
+ let flow = deflate_request(&blobs, id, &stored);
+ let body = read_body(&blobs, &flow, FlowBody::Request, 1024).await.unwrap();
+ assert_eq!(body.decode_error, None, "{id}");
+ assert_eq!((body.data.as_slice(), body.truncated), (&logs[..], false), "{id}");
+ }
+ }
+
+ #[tokio::test]
+ async fn undecodable_body_reads_as_it_was_on_the_wire() {
+ let (_db, blobs, _rx) = yaak_models::init_in_memory().unwrap();
+ let flow = deflate_request(&blobs, "fl_bad", b"\x00\x01 not deflate");
+
+ let decoded = read_body(&blobs, &flow, FlowBody::Request, 1024).await.unwrap();
+ assert!(decoded.decode_error.is_some());
+ let raw = read_body_or_raw(&blobs, &flow, FlowBody::Request, 1024).await.unwrap();
+ assert_eq!(raw.data, b"\x00\x01 not deflate");
+ assert!(!raw.truncated);
+ assert_eq!(raw.decode_error, decoded.decode_error);
+ let cut = read_body_or_raw(&blobs, &flow, FlowBody::Request, 4).await.unwrap();
+ assert_eq!((cut.data.as_slice(), cut.truncated), (&b"\x00\x01 n"[..], true));
+ }
}
diff --git a/crates-proxy/yaak-proxy-lib/src/lib.rs b/crates-proxy/yaak-proxy-lib/src/lib.rs
index 6ccf2a9ad..f27f7277c 100644
--- a/crates-proxy/yaak-proxy-lib/src/lib.rs
+++ b/crates-proxy/yaak-proxy-lib/src/lib.rs
@@ -851,12 +851,40 @@ pub struct GetFlowBodyRequest {
#[serde(rename_all = "camelCase")]
pub struct GetFlowBodyResponse {
/// The body decoded with its Content-Encoding, at most the capture limit of it, in base64.
+ /// When it fails to decode, the bytes as they were on the wire instead.
pub data: String,
/// The body is incomplete: cut on the wire, or decoded past the capture limit.
pub truncated: bool,
/// Bytes stored for the body, as they were on the wire.
#[ts(type = "number")]
pub stored_size: usize,
+ /// Why the body failed to decode, in which case `data` is as it was on the wire.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ #[ts(optional)]
+ pub decode_error: Option,
+}
+
+#[derive(Deserialize, TS)]
+#[ts(export, export_to = "gen_rpc.ts")]
+#[serde(rename_all = "camelCase")]
+pub struct SaveFlowBodyRequest {
+ pub flow_id: String,
+ pub which: FlowBody,
+ /// The file to write.
+ pub path: String,
+}
+
+#[derive(Serialize, TS)]
+#[ts(export, export_to = "gen_rpc.ts")]
+#[serde(rename_all = "camelCase")]
+pub struct SaveFlowBodyResponse {
+ /// Bytes written.
+ #[ts(type = "number")]
+ pub size: usize,
+ /// What was written is incomplete, as `GetFlowBodyResponse.truncated` is.
+ pub truncated: bool,
+ /// The body failed to decode, so the file has it as it was on the wire.
+ pub raw: bool,
}
#[derive(Deserialize, TS)]
@@ -1082,16 +1110,42 @@ async fn get_flow_body(
) -> Result {
let flow = find_flow(&ctx, &req.flow_id)?;
let limit = settings(&ctx)?.capture_body_limit;
- let body = viewer_body(&ctx, &flow, req.which, limit).await?;
+ let body =
+ bodies::read_body_or_raw(&ctx.blobs, &flow, req.which, limit).await.map_err(rpc_error)?;
Ok(GetFlowBodyResponse {
data: BASE64.encode(&body.data),
truncated: body.truncated,
stored_size: body.stored_size,
+ decode_error: body.decode_error,
})
}
-/// A body for the app's viewer, which shows a body that failed to decode as its error, as it
-/// always has, rather than as what decoded before the failure.
+/// Writes a body to a file the person picked: decoded, as the viewer shows it, or as it was on
+/// the wire when it fails to decode.
+async fn save_flow_body(
+ ctx: ProxyCtx,
+ req: SaveFlowBodyRequest,
+) -> Result {
+ let flow = find_flow(&ctx, &req.flow_id)?;
+ let limit = settings(&ctx)?.capture_body_limit;
+ let body =
+ bodies::read_body_or_raw(&ctx.blobs, &flow, req.which, limit).await.map_err(rpc_error)?;
+ let saved = SaveFlowBodyResponse {
+ size: body.data.len(),
+ truncated: body.truncated,
+ raw: body.decode_error.is_some(),
+ };
+ let path = PathBuf::from(req.path);
+ blocking(move || {
+ std::fs::write(&path, &body.data)
+ .map_err(|e| format!("Failed to write {}: {e}", path.display()))
+ })
+ .await?;
+ Ok(saved)
+}
+
+/// A body to filter, which fails when the body failed to decode rather than filtering what
+/// decoded before the failure.
async fn viewer_body(
ctx: &ProxyCtx,
flow: &Flow,
@@ -1112,14 +1166,13 @@ async fn filter_body(
let flow = find_flow(&ctx, &req.flow_id)?;
let limit = settings(&ctx)?.capture_body_limit;
let body = viewer_body(&ctx, &flow, req.which, limit).await?;
- let text = String::from_utf8_lossy(&body.data);
+ let content_type = find_header(flow.headers(req.which), "content-type");
+ // The text the viewer shows, so the filter's results match it
+ let text = bodies::body_text(&body.data, content_type, body.truncated);
let language = match req.language {
Some(FilterLanguage::JsonPath) => yaak_filter::Language::JsonPath,
Some(FilterLanguage::XPath) => yaak_filter::Language::XPath,
- None => {
- let content_type = find_header(flow.headers(req.which), "content-type");
- yaak_filter::Language::for_content_type(content_type.unwrap_or_default())
- }
+ None => yaak_filter::Language::for_content_type(content_type.unwrap_or_default()),
};
let result = yaak_filter::filter(language, &text, &req.expression);
Ok(FilterBodyResponse { content: result.content, error: result.error })
@@ -1437,6 +1490,7 @@ define_rpc! {
list_workspaces(ListWorkspacesRequest) -> ListWorkspacesResponse,
preview_save(PreviewSaveRequest) -> SavePreview,
save_flow(SaveFlowToYaakRequest) -> SavedFlow,
+ save_flow_body(SaveFlowBodyRequest) -> SaveFlowBodyResponse,
system_proxy_set(SystemProxySetRequest) -> SystemProxyStatus,
system_proxy_status(SystemProxyStatusRequest) -> SystemProxyStatus,
take_over(TakeOverRequest) -> bool,
@@ -1491,6 +1545,57 @@ mod tests {
assert!(log.was_on_at(SystemTime::now()));
}
+ /// A flow whose response is `body` under `content_type`, stored in `ctx`
+ fn flow_with_response(ctx: &ProxyCtx, id: &str, content_type: &str, body: &[u8]) {
+ let flow = Flow {
+ id: id.into(),
+ status: Some(200),
+ response_size: body.len() as i64,
+ response_headers: vec![ProxyHeader {
+ name: "Content-Type".into(),
+ value: content_type.into(),
+ }],
+ ..Default::default()
+ };
+ ctx.db.with_tx(|tx| tx.upsert(&flow, &UpdateSource::Background)).unwrap();
+ let mut writer =
+ BodyChunkWriter::new(ctx.blobs.clone(), Flow::body_id(id, FlowBody::Response));
+ writer.write(body).unwrap();
+ writer.finish().unwrap();
+ }
+
+ /// The filter reads the text the viewer shows: in the body's charset, not UTF-8 alone.
+ #[tokio::test]
+ async fn filter_reads_the_body_in_its_charset() {
+ let dir = tempfile::tempdir().unwrap();
+ let (events, _rx) = RpcEventEmitter::new();
+ let ctx = ProxyCtx::new(dir.path(), events).unwrap();
+
+ let xml = "ZürichKraków";
+ let utf16: Vec = xml.encode_utf16().flat_map(u16::to_le_bytes).collect();
+ flow_with_response(&ctx, "fl_utf16", "application/xml; charset=utf-16le", &utf16);
+ let json = r#"{"items":[{"name":"Café"},{"name":"Crème brûlée"}]}"#;
+ let latin1: Vec = json.chars().map(|c| c as u8).collect();
+ flow_with_response(&ctx, "fl_latin1", "application/json; charset=iso-8859-1", &latin1);
+ flow_with_response(&ctx, "fl_unlabeled", "application/json", &latin1);
+
+ let filter = |flow_id: &str, expression: &str, language| FilterBodyRequest {
+ flow_id: flow_id.into(),
+ which: FlowBody::Response,
+ expression: expression.into(),
+ language: Some(language),
+ };
+ let xpath = filter("fl_utf16", "//city[2]", FilterLanguage::XPath);
+ let result = filter_body(ctx.clone(), xpath).await.unwrap();
+ assert_eq!((result.content.as_str(), result.error), ("Kraków", None));
+ for flow_id in ["fl_latin1", "fl_unlabeled"] {
+ let names = filter(flow_id, "$.items[*].name", FilterLanguage::JsonPath);
+ let result = filter_body(ctx.clone(), names).await.unwrap();
+ assert_eq!(result.error, None, "{flow_id}");
+ assert_eq!(result.content, "[\n \"Café\",\n \"Crème brûlée\"\n]", "{flow_id}");
+ }
+ }
+
#[tokio::test]
async fn the_app_still_hears_why_a_body_failed_to_decode() {
let dir = tempfile::tempdir().unwrap();
@@ -1511,9 +1616,25 @@ mod tests {
writer.write(b"not a gzip stream").unwrap();
writer.finish().unwrap();
+ // The viewer gets the bytes as they were on the wire, and why they didn't decode.
let req = GetFlowBodyRequest { flow_id: "fl_bad".into(), which: FlowBody::Response };
- let err = get_flow_body(ctx.clone(), req).await.err().unwrap();
- assert!(err.message.starts_with("Failed to decode"), "{}", err.message);
+ let body = get_flow_body(ctx.clone(), req).await.unwrap();
+ let error = body.decode_error.unwrap();
+ assert!(error.starts_with("Failed to decode Gzip body"), "{error}");
+ assert_eq!(BASE64.decode(body.data).unwrap(), b"not a gzip stream");
+ assert!(!body.truncated);
+
+ // Saving writes those same bytes.
+ let path = dir.path().join("body.bin");
+ let req = SaveFlowBodyRequest {
+ flow_id: "fl_bad".into(),
+ which: FlowBody::Response,
+ path: path.to_string_lossy().into_owned(),
+ };
+ let saved = save_flow_body(ctx.clone(), req).await.unwrap();
+ assert_eq!((saved.size, saved.truncated, saved.raw), (17, false, true));
+ assert_eq!(std::fs::read(&path).unwrap(), b"not a gzip stream");
+
let req = FilterBodyRequest {
flow_id: "fl_bad".into(),
which: FlowBody::Response,
diff --git a/crates/yaak-http/Cargo.toml b/crates/yaak-http/Cargo.toml
index b26abd39d..ddd751f8f 100644
--- a/crates/yaak-http/Cargo.toml
+++ b/crates/yaak-http/Cargo.toml
@@ -5,7 +5,7 @@ edition = "2024"
publish = false
[dependencies]
-async-compression = { version = "0.4", features = ["tokio", "gzip", "deflate", "brotli", "zstd"] }
+async-compression = { version = "0.4", features = ["tokio", "gzip", "deflate", "zlib", "brotli", "zstd"] }
async-trait = "0.1"
brotli = "9"
bytes = "1.11.1"
diff --git a/crates/yaak-http/src/decompress.rs b/crates/yaak-http/src/decompress.rs
index e3764ea6f..80812c961 100644
--- a/crates/yaak-http/src/decompress.rs
+++ b/crates/yaak-http/src/decompress.rs
@@ -1,11 +1,13 @@
use crate::error::{Error, Result};
use async_compression::tokio::bufread::{
BrotliDecoder, DeflateDecoder as AsyncDeflateDecoder, GzipDecoder,
- ZstdDecoder as AsyncZstdDecoder,
+ ZlibDecoder as AsyncZlibDecoder, ZstdDecoder as AsyncZstdDecoder,
};
-use flate2::read::{DeflateDecoder, GzDecoder};
+use flate2::read::{DeflateDecoder, GzDecoder, ZlibDecoder};
use std::io::Read;
-use tokio::io::{AsyncBufRead, AsyncRead};
+use std::pin::Pin;
+use std::task::{Context, Poll, ready};
+use tokio::io::{AsyncBufRead, AsyncRead, BufReader, ReadBuf};
/// Supported compression encodings
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -66,13 +68,77 @@ fn decompress_gzip(data: &[u8]) -> Result> {
Ok(decompressed)
}
+/// `deflate` names zlib-wrapped data (RFC 1950), but plenty of senders send it raw (RFC 1951).
fn decompress_deflate(data: &[u8]) -> Result> {
- let mut decoder = DeflateDecoder::new(data);
- let mut decompressed = Vec::new();
- decoder
- .read_to_end(&mut decompressed)
- .map_err(|e| Error::DecompressionError(format!("deflate decompression failed: {}", e)))?;
- Ok(decompressed)
+ let read = |zlib: bool| {
+ let mut decompressed = Vec::new();
+ let result = if zlib {
+ ZlibDecoder::new(data).read_to_end(&mut decompressed)
+ } else {
+ DeflateDecoder::new(data).read_to_end(&mut decompressed)
+ };
+ result.map(|_| decompressed)
+ };
+ match deflate_variant(data) {
+ DeflateVariant::Zlib => read(true),
+ DeflateVariant::Raw => read(false),
+ // Zlib only when it ends whole, its checksum verified, or else raw.
+ DeflateVariant::Either => inflate_zlib_whole(data).or_else(|_| read(false)),
+ }
+ .map_err(|e| Error::DecompressionError(format!("deflate decompression failed: {}", e)))
+}
+
+/// Zlib data decoded through to its end and checksum, failing on any less
+fn inflate_zlib_whole(data: &[u8]) -> std::io::Result> {
+ let mut inflater = flate2::Decompress::new(true);
+ let mut out = Vec::new();
+ loop {
+ // Always room to write, so a call that makes no progress has run out of input.
+ out.reserve(64 * 1024);
+ let (read, written) = (inflater.total_in(), inflater.total_out());
+ let rest = &data[usize::try_from(read).unwrap_or(data.len()).min(data.len())..];
+ // Not `Finish`, which fails for good when the output outgrows its buffer.
+ let status = inflater
+ .decompress_vec(rest, &mut out, flate2::FlushDecompress::None)
+ .map_err(std::io::Error::other)?;
+ if status == flate2::Status::StreamEnd {
+ return Ok(out);
+ }
+ if inflater.total_in() == read && inflater.total_out() == written {
+ return Err(std::io::ErrorKind::UnexpectedEof.into());
+ }
+ }
+}
+
+/// Which `deflate` a body is, from its first five bytes
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum DeflateVariant {
+ Zlib,
+ Raw,
+ /// A zlib header that is also the start of a raw stored block: only decoding tells.
+ Either,
+}
+
+/// Zlib data opens with a header whose first byte names deflate with a window of at most 32 KB,
+/// and whose two bytes are a multiple of 31. Raw deflate passes that only when it opens with a
+/// stored block that isn't the last, whose length follows in the next two bytes and its
+/// complement in the two after.
+fn deflate_variant(data: &[u8]) -> DeflateVariant {
+ let zlib = match data {
+ [cmf, flg, ..] => {
+ cmf & 0x0f == 8 && cmf >> 4 <= 7 && (u16::from(*cmf) << 8 | u16::from(*flg)) % 31 == 0
+ }
+ _ => false,
+ };
+ match data {
+ _ if !zlib => DeflateVariant::Raw,
+ [_, l0, l1, n0, n1, ..]
+ if u16::from_le_bytes([*l0, *l1]) != !u16::from_le_bytes([*n0, *n1]) =>
+ {
+ DeflateVariant::Zlib
+ }
+ _ => DeflateVariant::Either,
+ }
}
fn decompress_brotli(data: &[u8]) -> Result> {
@@ -96,12 +162,262 @@ pub fn streaming_decoder(
match encoding {
ContentEncoding::Identity => Box::new(reader),
ContentEncoding::Gzip => Box::new(GzipDecoder::new(reader)),
- ContentEncoding::Deflate => Box::new(AsyncDeflateDecoder::new(reader)),
+ ContentEncoding::Deflate => Box::new(StreamingDeflateDecoder::new(reader)),
ContentEncoding::Brotli => Box::new(BrotliDecoder::new(reader)),
ContentEncoding::Zstd => Box::new(AsyncZstdDecoder::new(reader)),
}
}
+/// Decodes `deflate`, zlib-wrapped or raw, as its first five bytes say. When they could be
+/// either, it decodes both ways at once until one fails, and goes on with the other.
+struct StreamingDeflateDecoder {
+ state: DeflateState,
+}
+
+enum DeflateState {
+ Sniffing(Replay),
+ Zlib(AsyncZlibDecoder>>),
+ Raw(AsyncDeflateDecoder>>),
+ Either(EitherInflate>>),
+ /// Only while switching from sniffing to one of the others
+ Switching,
+}
+
+impl StreamingDeflateDecoder {
+ fn new(reader: R) -> Self {
+ let source = Replay { inner: reader, header: Vec::new(), pos: 0, sniffing: true };
+ Self { state: DeflateState::Sniffing(source) }
+ }
+}
+
+impl AsyncRead for StreamingDeflateDecoder {
+ fn poll_read(
+ self: Pin<&mut Self>,
+ cx: &mut Context<'_>,
+ buf: &mut ReadBuf<'_>,
+ ) -> Poll> {
+ let this = self.get_mut();
+ loop {
+ match &mut this.state {
+ DeflateState::Sniffing(source) => {
+ // Read until the header is in hand, or the input ends short of it.
+ while source.header.len() < 5 {
+ let mut header = [0u8; 5];
+ let mut header = ReadBuf::new(&mut header[source.header.len()..]);
+ ready!(Pin::new(&mut *source).poll_read(cx, &mut header))?;
+ if header.filled().is_empty() {
+ break;
+ }
+ }
+ let DeflateState::Sniffing(mut source) =
+ std::mem::replace(&mut this.state, DeflateState::Switching)
+ else {
+ unreachable!()
+ };
+ source.sniffing = false;
+ let variant = deflate_variant(&source.header);
+ let reader = BufReader::new(source);
+ this.state = match variant {
+ DeflateVariant::Zlib => DeflateState::Zlib(AsyncZlibDecoder::new(reader)),
+ DeflateVariant::Raw => DeflateState::Raw(AsyncDeflateDecoder::new(reader)),
+ DeflateVariant::Either => DeflateState::Either(EitherInflate::new(reader)),
+ };
+ }
+ DeflateState::Zlib(d) => return Pin::new(d).poll_read(cx, buf),
+ DeflateState::Raw(d) => return Pin::new(d).poll_read(cx, buf),
+ DeflateState::Either(d) => return Pin::new(d).poll_read(cx, buf),
+ DeflateState::Switching => {
+ return Poll::Ready(Err(std::io::Error::other("deflate decoder is broken")));
+ }
+ }
+ }
+ }
+}
+
+/// Reads `inner`, keeping what it read while sniffing, then reads that again before the rest.
+struct Replay {
+ inner: R,
+ header: Vec,
+ /// Where reading resumes in `header` once sniffing is over
+ pos: usize,
+ sniffing: bool,
+}
+
+impl AsyncRead for Replay {
+ fn poll_read(
+ self: Pin<&mut Self>,
+ cx: &mut Context<'_>,
+ buf: &mut ReadBuf<'_>,
+ ) -> Poll> {
+ let this = self.get_mut();
+ if this.sniffing {
+ let before = buf.filled().len();
+ ready!(Pin::new(&mut this.inner).poll_read(cx, buf))?;
+ this.header.extend_from_slice(&buf.filled()[before..]);
+ return Poll::Ready(Ok(()));
+ }
+ if this.pos < this.header.len() {
+ let n = buf.remaining().min(this.header.len() - this.pos);
+ buf.put_slice(&this.header[this.pos..this.pos + n]);
+ this.pos += n;
+ return Poll::Ready(Ok(()));
+ }
+ Pin::new(&mut this.inner).poll_read(cx, buf)
+ }
+}
+
+/// Input fed to the readings of an ambiguous deflate body at a time, which deflate grows to at
+/// most about a megabyte
+const DEFLATE_FEED: usize = 1024;
+
+/// Output held while both readings of an ambiguous deflate body are still valid, past which it is
+/// taken to be zlib, as its header says. A wrong reading fails within a few bytes in practice.
+const DEFLATE_HELD_LIMIT: usize = 1024 * 1024;
+
+/// Both readings of a deflate body that could be zlib or raw, fed the same input. Their output
+/// is held back while both are still valid, and the one left streams on: a wrong reading fails
+/// within a few bytes in practice, so little is held. If both end, zlib's checksum verified it.
+struct EitherInflate {
+ input: B,
+ /// Zlib, then raw
+ readings: [Option; 2],
+ /// Why the last reading to fail failed
+ error: Option,
+}
+
+struct Inflate {
+ inflater: flate2::Decompress,
+ out: Vec,
+ /// Where reading resumes in `out`
+ pos: usize,
+ ended: bool,
+}
+
+impl EitherInflate {
+ fn new(input: B) -> Self {
+ let reading = |zlib| Inflate {
+ inflater: flate2::Decompress::new(zlib),
+ out: Vec::new(),
+ pos: 0,
+ ended: false,
+ };
+ Self { input, readings: [Some(reading(true)), Some(reading(false))], error: None }
+ }
+}
+
+impl Inflate {
+ /// Decodes all of `input` onto `out`, or up to the end of the stream
+ fn feed(&mut self, input: &[u8]) -> std::io::Result<()> {
+ let mut consumed = 0;
+ while !self.ended {
+ if self.out.capacity() - self.out.len() < 32 * 1024 {
+ self.out.reserve(64 * 1024);
+ }
+ let (read, written) = (self.inflater.total_in(), self.inflater.total_out());
+ let status = self
+ .inflater
+ .decompress_vec(&input[consumed..], &mut self.out, flate2::FlushDecompress::None)
+ .map_err(std::io::Error::other)?;
+ let progress = (self.inflater.total_in() - read, self.inflater.total_out() - written);
+ consumed += usize::try_from(progress.0).unwrap_or(usize::MAX).min(input.len());
+ if status == flate2::Status::StreamEnd {
+ self.ended = true;
+ } else if consumed == input.len() && self.out.len() < self.out.capacity() {
+ // All of it in, and all that it decodes to out
+ break;
+ } else if progress == (0, 0) {
+ return Err(std::io::Error::other("deflate decoding stalled"));
+ }
+ }
+ Ok(())
+ }
+}
+
+impl AsyncRead for EitherInflate {
+ fn poll_read(
+ self: Pin<&mut Self>,
+ cx: &mut Context<'_>,
+ buf: &mut ReadBuf<'_>,
+ ) -> Poll> {
+ let this = self.get_mut();
+ loop {
+ let [zlib, raw] = &mut this.readings;
+ match (zlib.as_mut(), raw.as_mut()) {
+ (None, None) => {
+ let error = this.error.take();
+ return Poll::Ready(Err(
+ error.unwrap_or_else(|| std::io::Error::other("deflate decoding failed"))
+ ));
+ }
+ // Both ended: zlib's checksum verified it.
+ (Some(z), Some(r)) if z.ended && r.ended => *raw = None,
+ (Some(only), None) | (None, Some(only)) => {
+ if only.pos < only.out.len() {
+ let n = buf.remaining().min(only.out.len() - only.pos);
+ buf.put_slice(&only.out[only.pos..only.pos + n]);
+ only.pos += n;
+ if only.pos == only.out.len() {
+ only.out.clear();
+ only.pos = 0;
+ }
+ return Poll::Ready(Ok(()));
+ }
+ if only.ended {
+ return Poll::Ready(Ok(()));
+ }
+ ready!(this.feed(cx))?;
+ }
+ (Some(_), Some(_)) => ready!(this.feed(cx))?,
+ }
+ }
+ }
+}
+
+impl EitherInflate {
+ /// Feeds the next input to each reading still going, dropping those that fail on it. Pending
+ /// until there is input; at its end, a reading that hasn't ended was cut short.
+ fn feed(&mut self, cx: &mut Context<'_>) -> Poll> {
+ let chunk = ready!(Pin::new(&mut self.input).poll_fill_buf(cx))?;
+ // A little input at a time, since deflate can grow a thousandfold.
+ let len = chunk.len().min(DEFLATE_FEED);
+ let chunk = &chunk[..len];
+ let [zlib, raw] = &mut self.readings;
+ if let (Some(z), Some(r)) = (zlib.as_ref(), raw.as_ref()) {
+ let (z_held, r_held) = (z.out.len() - z.pos, r.out.len() - r.pos);
+ let cut_short = len == 0 && !z.ended && !r.ended;
+ if cut_short {
+ // Both were cut short: keep what decoded furthest, so a cut body shows its start.
+ if r_held > z_held {
+ *zlib = None
+ } else {
+ *raw = None
+ }
+ // The one left gives what it decoded, then fails on the end of input.
+ return Poll::Ready(Ok(()));
+ } else if z_held.max(r_held) > DEFLATE_HELD_LIMIT {
+ // Holding no more for a raw body zlib still reads validly: it is taken to be zlib.
+ *raw = None;
+ }
+ }
+ for slot in &mut self.readings {
+ let Some(reading) = slot.as_mut().filter(|r| !r.ended) else {
+ continue;
+ };
+ let result = if len == 0 {
+ Err(std::io::ErrorKind::UnexpectedEof.into())
+ } else {
+ reading.feed(chunk)
+ };
+ if let Err(e) = result {
+ *slot = None;
+ self.error = Some(e);
+ }
+ }
+ Pin::new(&mut self.input).consume(len);
+ Poll::Ready(Ok(()))
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -159,6 +475,281 @@ mod tests {
assert_eq!(result.decompressed_size, original.len() as u64);
}
+ fn zlib(data: &[u8]) -> Vec {
+ let mut encoder = flate2::write::ZlibEncoder::new(Vec::new(), Compression::default());
+ encoder.write_all(data).unwrap();
+ encoder.finish().unwrap()
+ }
+
+ fn raw_deflate(data: &[u8]) -> Vec {
+ let mut encoder = flate2::write::DeflateEncoder::new(Vec::new(), Compression::default());
+ encoder.write_all(data).unwrap();
+ encoder.finish().unwrap()
+ }
+
+ /// Raw deflate whose first two bytes also pass as a zlib header: a stored block of 29 bytes
+ /// (`08 1d`, with `0x081d` a multiple of 31), then an empty final one.
+ fn raw_deflate_like_zlib(data: &[u8; 29]) -> Vec {
+ let mut out = vec![0x08, 0x1d, 0x00, 0xe2, 0xff];
+ out.extend_from_slice(data);
+ out.extend_from_slice(&[0x01, 0x00, 0x00, 0xff, 0xff]);
+ assert_eq!(deflate_variant(&out), DeflateVariant::Either);
+ out
+ }
+
+ /// Zlib that opens as `raw_deflate_like_zlib` does: header `08 1d`, then a stored block of
+ /// 0xffe2 bytes, which `1d 00 e2 ff` also reads as the start of, and a final stored block of
+ /// `extra` bytes.
+ fn zlib_like_raw_deflate(extra: u16) -> (Vec, Vec) {
+ let data: Vec = (0..0xffe2 + u32::from(extra)).map(|i| (i % 251) as u8).collect();
+ let mut out = vec![0x08, 0x1d, 0x00, 0xe2, 0xff, 0x1d, 0x00];
+ out.extend_from_slice(&data[..0xffe2]);
+ out.push(0x01);
+ out.extend_from_slice(&extra.to_le_bytes());
+ out.extend_from_slice(&(!extra).to_le_bytes());
+ out.extend_from_slice(&data[0xffe2..]);
+ let (mut a, mut b) = (1u32, 0u32);
+ for byte in &data {
+ a = (a + u32::from(*byte)) % 65521;
+ b = (b + a) % 65521;
+ }
+ out.extend_from_slice(&((b << 16) | a).to_be_bytes());
+ assert_eq!(deflate_variant(&out), DeflateVariant::Either);
+ (out, data)
+ }
+
+ async fn stream_decode(data: Vec, encoding: ContentEncoding) -> std::io::Result> {
+ use tokio::io::AsyncReadExt;
+ let mut out = Vec::new();
+ // One byte at a time, to read the header across polls as a slow connection would.
+ let reader = BufReader::with_capacity(1, std::io::Cursor::new(data));
+ streaming_decoder(reader, encoding).read_to_end(&mut out).await?;
+ Ok(out)
+ }
+
+ /// What a body decodes to before it fails, and why it failed
+ async fn stream_decode_until_error(data: Vec) -> (Vec, Option) {
+ use tokio::io::AsyncReadExt;
+ let mut out = Vec::new();
+ let mut decoder =
+ streaming_decoder(BufReader::new(std::io::Cursor::new(data)), ContentEncoding::Deflate);
+ let mut buf = [0u8; 4096];
+ loop {
+ match decoder.read(&mut buf).await {
+ Ok(0) => return (out, None),
+ Ok(n) => out.extend_from_slice(&buf[..n]),
+ Err(e) => return (out, Some(e)),
+ }
+ }
+ }
+
+ #[tokio::test]
+ async fn streaming_deflate_keeps_what_decoded_of_a_cut_ambiguous_body() {
+ // Bytes both readings still accept when the input ends: zlib in a stored block too
+ let text = *b"\x1d\0xxxxxxxxxxxxxxxxxxxxxxxxxxx";
+ let mut cut = raw_deflate_like_zlib(&text);
+ // Without its final block, as a body cut at the capture limit is
+ cut.truncate(cut.len() - 5);
+ let (decoded, error) = stream_decode_until_error(cut).await;
+ assert_eq!(decoded, text);
+ assert_eq!(error.map(|e| e.kind()), Some(std::io::ErrorKind::UnexpectedEof));
+ }
+
+ /// Zlib of `megabytes` of zeros whose raw reading is a stored block of 29 bytes and an empty
+ /// final one, both inside zlib's first stored block: raw ends at once, zlib goes on.
+ fn zlib_whose_raw_reading_ends_early(megabytes: usize) -> (Vec, Vec) {
+ let mut stored = vec![b'z'; 0xffe2];
+ stored[27..32].copy_from_slice(&[0x01, 0x00, 0x00, 0xff, 0xff]);
+ let mut expected = stored.clone();
+ expected.resize(stored.len() + megabytes * 1024 * 1024, 0);
+ let mut data = vec![0x08, 0x1d, 0x00, 0xe2, 0xff, 0x1d, 0x00];
+ data.extend_from_slice(&stored);
+ data.extend_from_slice(&raw_deflate(&expected[stored.len()..]));
+ let (mut a, mut b) = (1u32, 0u32);
+ for byte in &expected {
+ a = (a + u32::from(*byte)) % 65521;
+ b = (b + a) % 65521;
+ }
+ data.extend_from_slice(&((b << 16) | a).to_be_bytes());
+ assert_eq!(deflate_variant(&data), DeflateVariant::Either);
+ (data, expected)
+ }
+
+ #[tokio::test]
+ async fn streaming_deflate_holds_little_while_both_readings_go() {
+ use tokio::io::AsyncReadExt;
+ let (data, expected) = zlib_whose_raw_reading_ends_early(8);
+ let mut decoder =
+ StreamingDeflateDecoder::new(BufReader::new(std::io::Cursor::new(data.clone())));
+ let mut first = [0u8; 1024];
+ decoder.read_exact(&mut first).await.unwrap();
+ assert_eq!(first[..], expected[..1024]);
+ let DeflateState::Either(either) = &decoder.state else {
+ panic!("not read both ways")
+ };
+ // The limit, what one feed of input adds past it (deflate grows at most about 1,032
+ // times), and the raw reading's 64 KB: far short of the 8 MB body.
+ let held: usize = either.readings.iter().flatten().map(|r| r.out.len() - r.pos).sum();
+ let bound = 3 * 1024 * 1024;
+ assert!(held <= bound, "held {held} bytes for a 1 KB read");
+
+ let decoded = stream_decode(data, ContentEncoding::Deflate).await.unwrap();
+ assert!(decoded == expected);
+ }
+
+ const DEFLATE_TEXT: &[u8] = b"{\"service\":\"browser\",\"message\":\"hello deflate\"}";
+
+ #[test]
+ fn test_decompress_deflate_zlib_wrapped() {
+ let result = decompress(zlib(DEFLATE_TEXT), ContentEncoding::Deflate).unwrap();
+ assert_eq!(result.data, DEFLATE_TEXT);
+ let like_zlib = raw_deflate_like_zlib(b"twenty-nine bytes of raw text");
+ let result = decompress(like_zlib, ContentEncoding::Deflate).unwrap();
+ assert_eq!(result.data, b"twenty-nine bytes of raw text");
+ let both = *b"\x1d\0xxxxxxxxxxxxxxxxxxxxxxxxxxx";
+ let result = decompress(raw_deflate_like_zlib(&both), ContentEncoding::Deflate).unwrap();
+ assert_eq!(result.data, both);
+ for extra in [0, 10_000] {
+ let (zlib_like_raw, data) = zlib_like_raw_deflate(extra);
+ let result = decompress(zlib_like_raw, ContentEncoding::Deflate).unwrap();
+ assert!(result.data == data, "{extra} extra bytes");
+ }
+ let (data, expected) = large_raw_like_zlib();
+ assert!(decompress(data, ContentEncoding::Deflate).unwrap().data == expected);
+ }
+
+ /// Ambiguous raw deflate of more than 64 KB: a stored block that reads as zlib's header and
+ /// the start of a stored block of 0xffe2 bytes, then incompressible data, which zlib reads
+ /// as a block header 7 + 0xffe2 bytes in and fails on.
+ fn large_raw_like_zlib() -> (Vec, Vec) {
+ let mut expected = b"\x1d\0xxxxxxxxxxxxxxxxxxxxxxxxxxx".to_vec();
+ let mut seed = 1u32;
+ expected.extend((0..70_000).map(|_| {
+ seed = seed.wrapping_mul(1_103_515_245).wrapping_add(12_345);
+ (seed >> 16) as u8
+ }));
+ let mut data = vec![0x08, 0x1d, 0x00, 0xe2, 0xff];
+ data.extend_from_slice(&expected[..29]);
+ data.extend_from_slice(&raw_deflate(&expected[29..]));
+ assert!(data.len() > 64 * 1024);
+ (data, expected)
+ }
+
+ /// Greptile's case on #812: raw deflate that zlib reads validly past 64 KB. After the first
+ /// stored block, a final stored block of 65,535 bytes, in which zlib finds a valid stored
+ /// block header at 65,513 (`00 28 00 d7 ff`) and an invalid block type only at 65,558.
+ fn raw_valid_as_zlib_past_64k() -> (Vec, Vec) {
+ let mut block: Vec = (0..65_535u32).map(|i| b'a' + (i % 26) as u8).collect();
+ let first = 5 + 29 + 5;
+ block[65_513 - first..65_518 - first].copy_from_slice(&[0x00, 0x28, 0x00, 0xd7, 0xff]);
+ block[65_558 - first] = 0x07;
+ let mut expected = b"\x1d\0xxxxxxxxxxxxxxxxxxxxxxxxxxx".to_vec();
+ let mut data = raw_deflate_like_zlib(&expected.clone().try_into().unwrap());
+ // Make the first block not the last, and follow it with the large final one.
+ data.truncate(data.len() - 5);
+ data.extend_from_slice(&[0x01, 0xff, 0xff, 0x00, 0x00]);
+ data.extend_from_slice(&block);
+ expected.extend_from_slice(&block);
+ (data, expected)
+ }
+
+ #[test]
+ fn test_decompress_deflate_raw_valid_as_zlib_past_64k() {
+ let (data, expected) = raw_valid_as_zlib_past_64k();
+ // Raw alone decodes it; zlib alone gets past 64 KB before failing.
+ let mut raw = Vec::new();
+ DeflateDecoder::new(data.as_slice()).read_to_end(&mut raw).unwrap();
+ assert!(raw == expected);
+ let mut zlib = flate2::Decompress::new(true);
+ let mut out = Vec::with_capacity(200 * 1024);
+ let failed = zlib.decompress_vec(&data, &mut out, flate2::FlushDecompress::None);
+ assert!(failed.is_err() && zlib.total_in() > 65_536, "zlib read {}", zlib.total_in());
+ assert!(decompress(data, ContentEncoding::Deflate).unwrap().data == expected);
+ }
+
+ #[tokio::test]
+ async fn streaming_deflate_decodes_raw_valid_as_zlib_past_64k() {
+ use tokio::io::AsyncReadExt;
+ let (data, expected) = raw_valid_as_zlib_past_64k();
+ for capacity in [1, 8 * 1024, 256 * 1024] {
+ let mut out = Vec::new();
+ let reader = BufReader::with_capacity(capacity, std::io::Cursor::new(data.clone()));
+ streaming_decoder(reader, ContentEncoding::Deflate)
+ .read_to_end(&mut out)
+ .await
+ .unwrap();
+ assert!(out == expected, "read {capacity} bytes at a time");
+ }
+ }
+
+ #[tokio::test]
+ async fn streaming_deflate_decodes_zlib_and_raw() {
+ let zlib_wrapped = stream_decode(zlib(DEFLATE_TEXT), ContentEncoding::Deflate).await;
+ assert_eq!(zlib_wrapped.unwrap(), DEFLATE_TEXT);
+ let raw = stream_decode(raw_deflate(DEFLATE_TEXT), ContentEncoding::Deflate).await;
+ assert_eq!(raw.unwrap(), DEFLATE_TEXT);
+ let large: Vec = (0..200_000u32).flat_map(|i| (i % 251).to_le_bytes()).collect();
+ let large_zlib = stream_decode(zlib(&large), ContentEncoding::Deflate).await;
+ assert_eq!(large_zlib.unwrap(), large);
+ let large_raw = stream_decode(raw_deflate(&large), ContentEncoding::Deflate).await;
+ assert_eq!(large_raw.unwrap(), large);
+ }
+
+ #[tokio::test]
+ async fn streaming_deflate_retries_raw_that_looks_like_zlib() {
+ let data = raw_deflate_like_zlib(b"twenty-nine bytes of raw text");
+ // A zlib decoder alone rejects it.
+ assert!(ZlibDecoder::new(data.as_slice()).read_to_end(&mut Vec::new()).is_err());
+ let decoded = stream_decode(data, ContentEncoding::Deflate).await.unwrap();
+ assert_eq!(decoded, b"twenty-nine bytes of raw text");
+
+ // Read as zlib, this one is a stored block that starts decoding, and only fails later.
+ let both = *b"\x1d\0xxxxxxxxxxxxxxxxxxxxxxxxxxx";
+ let data = raw_deflate_like_zlib(&both);
+ let decoded = stream_decode(data, ContentEncoding::Deflate).await.unwrap();
+ assert_eq!(decoded, both);
+ }
+
+ #[tokio::test]
+ async fn streaming_deflate_decodes_zlib_that_looks_like_raw() {
+ for extra in [0, 10_000] {
+ let (data, expected) = zlib_like_raw_deflate(extra);
+ let decoded = stream_decode(data, ContentEncoding::Deflate).await.unwrap();
+ assert!(decoded == expected, "{extra} extra bytes");
+ }
+ }
+
+ /// However the reader reads ahead, the wrong reading is dropped and the right one kept.
+ #[tokio::test]
+ async fn streaming_deflate_retries_large_raw_whatever_the_chunking() {
+ use tokio::io::AsyncReadExt;
+ let (data, expected) = large_raw_like_zlib();
+ for capacity in [1, 8 * 1024, 256 * 1024] {
+ let mut out = Vec::new();
+ let reader = BufReader::with_capacity(capacity, std::io::Cursor::new(data.clone()));
+ streaming_decoder(reader, ContentEncoding::Deflate)
+ .read_to_end(&mut out)
+ .await
+ .unwrap();
+ assert!(out == expected, "read {capacity} bytes at a time");
+ }
+ }
+
+ #[tokio::test]
+ async fn streaming_deflate_still_fails_on_garbage() {
+ let garbage = b"\x78\x9c this is not deflate in any form".to_vec();
+ assert!(stream_decode(garbage, ContentEncoding::Deflate).await.is_err());
+ assert!(stream_decode(b"\xff".to_vec(), ContentEncoding::Deflate).await.is_err());
+ }
+
+ #[tokio::test]
+ async fn streaming_gzip_is_unchanged() {
+ let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
+ encoder.write_all(DEFLATE_TEXT).unwrap();
+ let gzip = encoder.finish().unwrap();
+ assert_eq!(stream_decode(gzip, ContentEncoding::Gzip).await.unwrap(), DEFLATE_TEXT);
+ }
+
#[test]
fn test_decompress_brotli() {
// Compress some data with brotli
diff --git a/crates/yaak-proxy/tests/proxy.rs b/crates/yaak-proxy/tests/proxy.rs
index 2e4062ba2..9e911d16f 100644
--- a/crates/yaak-proxy/tests/proxy.rs
+++ b/crates/yaak-proxy/tests/proxy.rs
@@ -100,6 +100,11 @@ async fn route(req: Request) -> Result, Infallible>
req.headers().get(HOST).map_or("none", |h| h.to_str().unwrap()),
))),
"/big" => Response::new(full(vec![b'x'; BIG_BODY])),
+ // No body: `Content-Length: 0`, or a stream with nothing in it, chunked on HTTP/1.1.
+ "/empty" => Response::new(full("")),
+ "/empty-stream" => {
+ Response::new(BodyExt::boxed(StreamBody::new(futures_util::stream::empty())))
+ }
"/late-end" => late_end(5),
// The stream ends with half the promised bytes, so the upstream resets it mid-body.
"/cut" => late_end(10),
@@ -1361,6 +1366,64 @@ async fn h2_large_bodies_pass_flow_control() {
}
}
+/// A flow ends when its response does, not when a connection closes: the upstream answers at
+/// once and keeps its connection open, as does the client, and the flow's time is the answer's.
+#[tokio::test(flavor = "multi_thread")]
+async fn keep_alive_connections_do_not_lengthen_flows() {
+ let h = Harness::start(1024);
+ let upstreams = [
+ ("plain HTTP/1.1", spawn_upstream(None).await, false),
+ ("HTTP/1.1", spawn_upstream(Some(h1_only(&h.pki.server))).await, true),
+ ("HTTP/2", spawn_upstream(Some(h.pki.server.clone())).await, true),
+ ];
+ // Every connection stays open to the end of the test, each reused across requests.
+ let h1_client = h.client();
+ let mut h2_clients = Vec::new();
+
+ for (upstream_name, upstream, tls) in upstreams {
+ let port = upstream.port();
+ let mut h2_client = if tls { Some(h.h2_client(port).await) } else { None };
+ for path in ["/empty", "/empty-stream", "/hello"] {
+ for client_version in ["HTTP/1.1", "HTTP/2"] {
+ let case = format!("{client_version} client, {upstream_name} upstream, {path}");
+ let scheme = if tls { "https" } else { "http" };
+ let url = format!("{scheme}://localhost:{port}{path}");
+ let body = match (client_version, &mut h2_client) {
+ ("HTTP/1.1", _) => {
+ let resp = h1_client.post(&url).body("x=1").send().await.unwrap();
+ resp.bytes().await.unwrap()
+ }
+ (_, Some(client)) => {
+ let req = h2_request("POST", port, path, full("x=1"));
+ let resp = client.send_request(req).await.unwrap();
+ resp.into_body().collect().await.unwrap().to_bytes()
+ }
+ // HTTP/2 needs TLS here.
+ (_, None) => continue,
+ };
+ assert_eq!(body, if path == "/hello" { "hello" } else { "" }, "{case}");
+
+ let events = h.flow_events();
+ let version =
+ find!(events, ProxyEvent::RequestStart { http_version, .. } => http_version);
+ assert_eq!(version, client_version, "{case}");
+ let headers_ms =
+ find!(events, ProxyEvent::ResponseStart { elapsed_ms, .. } => *elapsed_ms);
+ let (end_ms, error, cancelled) = find!(events,
+ ProxyEvent::ResponseBodyComplete { elapsed_ms, error, cancelled, .. } =>
+ (*elapsed_ms, error.clone(), *cancelled));
+ assert_eq!((error, cancelled), (None, false), "{case}");
+ // Generous for a slow CI runner; a flow held to the connection's close would
+ // not end at all while every connection here stays open.
+ assert!(end_ms - headers_ms < 2_000, "{case}: headers {headers_ms}, end {end_ms}");
+ assert!(end_ms < 5_000, "{case}: took {end_ms} ms");
+ }
+ }
+ h2_clients.extend(h2_client);
+ }
+ drop((h1_client, h2_clients));
+}
+
/// With HTTP/2 off, a client that offers both gets HTTP/1.1, from the next connection on.
#[tokio::test(flavor = "multi_thread")]
async fn http1_only_makes_h2_clients_fall_back() {
diff --git a/package-lock.json b/package-lock.json
index d21746759..ed8af433b 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -145,7 +145,6 @@
"focus-trap-react": "^12.0.3",
"fuzzbunny": "^1.0.1",
"graphql": "^17.0.2",
- "hexy": "^0.4.0",
"history": "^5.3.0",
"jotai": "^2.18.0",
"jotai-family": "^1.0.1",
@@ -15019,6 +15018,7 @@
"name": "@yaakapp-internal/ui",
"version": "1.0.0",
"dependencies": {
+ "hexy": "^0.4.0",
"jotai-family": "^1.0.1"
}
},
diff --git a/packages/common-lib/decodeBodyText.test.ts b/packages/common-lib/decodeBodyText.test.ts
new file mode 100644
index 000000000..60d2098fd
--- /dev/null
+++ b/packages/common-lib/decodeBodyText.test.ts
@@ -0,0 +1,45 @@
+import { describe, expect, test } from "vitest";
+import { contentTypeCharset, decodeBodyText, strictUtf8Text } from "./decodeBodyText";
+
+const latin1 = (text: string) => Uint8Array.from(text, (c) => c.charCodeAt(0));
+// "こんにちは、世界" in Shift_JIS
+const shiftJis = new Uint8Array([
+ 0x82, 0xb1, 0x82, 0xf1, 0x82, 0xc9, 0x82, 0xbf, 0x82, 0xcd, 0x81, 0x41, 0x90, 0xa2, 0x8a, 0x45,
+]);
+const french = "Accès refusé : vous n'avez pas l'autorisation d'accéder à cette page.";
+
+describe("decodeBodyText", () => {
+ test("decodes in the charset the content type names", () => {
+ const body = latin1(french);
+ expect(decodeBodyText(body, "text/html; charset=iso-8859-1")).toBe(french);
+ expect(decodeBodyText(body, 'text/plain; CHARSET="ISO-8859-1"')).toBe(french);
+ expect(decodeBodyText(shiftJis, "text/plain; charset=Shift_JIS")).toBe("こんにちは、世界");
+ });
+
+ test("reads an unlabeled body that isn't UTF-8 as windows-1252", () => {
+ expect(decodeBodyText(latin1(french), "text/html")).toBe(french);
+ expect(decodeBodyText(latin1(french), null)).toBe(french);
+ expect(decodeBodyText(new Uint8Array([0x93, 0x68, 0x69, 0x94]), null)).toBe("“hi”");
+ });
+
+ test("reads UTF-8 as UTF-8, labeled or not", () => {
+ const utf8 = new TextEncoder().encode(`{"city":"Zürich","greeting":"こんにちは"}`);
+ const text = `{"city":"Zürich","greeting":"こんにちは"}`;
+ expect(decodeBodyText(utf8, null)).toBe(text);
+ expect(decodeBodyText(utf8, "application/json")).toBe(text);
+ expect(decodeBodyText(utf8, "application/json; charset=utf-8")).toBe(text);
+ // A body labeled UTF-8 stays UTF-8, with what isn't replaced, as before
+ expect(decodeBodyText(new Uint8Array([0x61, 0xff]), "text/plain; charset=utf-8")).toBe("a�");
+ });
+
+ test("treats a charset TextDecoder doesn't know as no charset", () => {
+ expect(contentTypeCharset("text/plain; charset=not-a-charset")).toBeNull();
+ expect(decodeBodyText(latin1(french), "text/plain; charset=not-a-charset")).toBe(french);
+ });
+
+ test("keeps a truncated UTF-8 body UTF-8 when it ends partway through a character", () => {
+ const cut = new TextEncoder().encode("abc€").subarray(0, 5);
+ expect(strictUtf8Text(cut)).toBeNull();
+ expect(decodeBodyText(cut, null, { truncated: true })).toBe("abc");
+ });
+});
diff --git a/packages/common-lib/decodeBodyText.ts b/packages/common-lib/decodeBodyText.ts
new file mode 100644
index 000000000..dd6645b37
--- /dev/null
+++ b/packages/common-lib/decodeBodyText.ts
@@ -0,0 +1,44 @@
+/**
+ * The charset a content type names, as `TextDecoder` calls it (`windows-1252` for
+ * `iso-8859-1`), or null when it names none or one `TextDecoder` doesn't know.
+ */
+export function contentTypeCharset(contentType: string | null): string | null {
+ const label = contentType?.match(/;\s*charset\s*=\s*"?([^";\s]+)"?/i)?.[1];
+ if (label == null) return null;
+ try {
+ return new TextDecoder(label).encoding;
+ } catch {
+ return null;
+ }
+}
+
+/**
+ * The bytes as UTF-8, or null when they aren't valid UTF-8. A truncated body may end partway
+ * through a character, so up to three trailing bytes are dropped before giving up on it.
+ */
+export function strictUtf8Text(bytes: Uint8Array, truncated = false): string | null {
+ const decoder = new TextDecoder("utf-8", { fatal: true });
+ const maxTrim = truncated ? Math.min(3, bytes.length) : 0;
+ for (let trim = 0; trim <= maxTrim; trim++) {
+ try {
+ return decoder.decode(bytes.subarray(0, bytes.length - trim));
+ } catch {
+ // Try again without the last byte
+ }
+ }
+ return null;
+}
+
+/**
+ * A body's text in the charset its content type names. Without one it is UTF-8, unless the
+ * bytes aren't valid UTF-8, when it is windows-1252, as browsers read unlabeled text.
+ */
+export function decodeBodyText(
+ bytes: Uint8Array,
+ contentType: string | null,
+ { truncated = false }: { truncated?: boolean } = {},
+): string {
+ const charset = contentTypeCharset(contentType);
+ if (charset != null) return new TextDecoder(charset).decode(bytes);
+ return strictUtf8Text(bytes, truncated) ?? new TextDecoder("windows-1252").decode(bytes);
+}
diff --git a/packages/common-lib/index.ts b/packages/common-lib/index.ts
index 245e43a46..999fbae9b 100644
--- a/packages/common-lib/index.ts
+++ b/packages/common-lib/index.ts
@@ -1,4 +1,5 @@
export * from "./debounce";
+export * from "./decodeBodyText";
export * from "./eagerDebounceAsync";
export * from "./formatSize";
export * from "./templateFunction";
diff --git a/packages/ui/package.json b/packages/ui/package.json
index 8ed94e9cd..849778086 100644
--- a/packages/ui/package.json
+++ b/packages/ui/package.json
@@ -16,6 +16,7 @@
"*.css"
],
"dependencies": {
+ "hexy": "^0.4.0",
"jotai-family": "^1.0.1"
}
}
diff --git a/packages/ui/src/components/viewers/HexdumpViewer.test.ts b/packages/ui/src/components/viewers/HexdumpViewer.test.ts
new file mode 100644
index 000000000..209bd49f7
--- /dev/null
+++ b/packages/ui/src/components/viewers/HexdumpViewer.test.ts
@@ -0,0 +1,20 @@
+import { hexy } from "hexy";
+import { describe, expect, test, vi } from "vite-plus/test";
+import { hexdump } from "./HexdumpViewer";
+
+vi.mock("hexy", () => ({ hexy: vi.fn((data: number[]) => `dump of ${data.length} bytes`) }));
+
+describe("hexdump", () => {
+ test("makes the hexdump of the same bytes once", () => {
+ const message = [0x01, 0x02, 0x03];
+ expect(hexdump(message)).toBe("dump of 3 bytes");
+ expect(hexdump(message)).toBe("dump of 3 bytes");
+ expect(hexy).toHaveBeenCalledTimes(1);
+
+ // Other bytes, even equal ones, get their own.
+ expect(hexdump([0x01, 0x02, 0x03])).toBe("dump of 3 bytes");
+ expect(hexy).toHaveBeenCalledTimes(2);
+ expect(hexdump([])).toBe("");
+ expect(hexy).toHaveBeenCalledTimes(2);
+ });
+});
diff --git a/packages/ui/src/components/viewers/HexdumpViewer.tsx b/packages/ui/src/components/viewers/HexdumpViewer.tsx
new file mode 100644
index 000000000..3e3b94303
--- /dev/null
+++ b/packages/ui/src/components/viewers/HexdumpViewer.tsx
@@ -0,0 +1,50 @@
+import { hexy } from "hexy";
+import type { ReactNode } from "react";
+import { useMemo } from "react";
+import type { CodeEditorProps } from "../editor/CodeEditor";
+import { LazyCodeEditor } from "../editor/LazyCodeEditor";
+
+/** Hexdumps already made, by the bytes they were made of, for as long as those bytes live */
+const hexdumps = new WeakMap();
+
+/**
+ * Bytes as a hexdump: offset, hex and ASCII columns, one line per 16 bytes. Made once for the
+ * same bytes, so a viewer and its copy action share one.
+ */
+export function hexdump(data: Uint8Array | number[]): string {
+ if (data.length === 0) return "";
+ let dump = hexdumps.get(data);
+ if (dump == null) {
+ dump = hexy(data);
+ hexdumps.set(data, dump);
+ }
+ return dump;
+}
+
+export interface HexdumpViewerProps {
+ data: Uint8Array | number[];
+ className?: string;
+ /** Renders the editor, for a host whose editor builds on CodeEditor */
+ renderEditor?: (props: CodeEditorProps) => ReactNode;
+}
+
+/** A read-only hexdump of binary data */
+export function HexdumpViewer({
+ data,
+ className,
+ renderEditor = renderCodeEditor,
+}: HexdumpViewerProps) {
+ const text = useMemo(() => hexdump(data), [data]);
+ return renderEditor({
+ language: "text",
+ defaultValue: text,
+ wrapLines: false,
+ readOnly: true,
+ stateKey: null,
+ className,
+ });
+}
+
+function renderCodeEditor(props: CodeEditorProps) {
+ return ;
+}
diff --git a/packages/ui/src/components/viewers/TextViewer.tsx b/packages/ui/src/components/viewers/TextViewer.tsx
index 6e25defe9..854a15566 100644
--- a/packages/ui/src/components/viewers/TextViewer.tsx
+++ b/packages/ui/src/components/viewers/TextViewer.tsx
@@ -89,6 +89,8 @@ export function TextViewer({
],
queryFn: () => (filterText ?? rejectFilter)(filter.appliedFilter ?? ""),
enabled: hasFilter && filter.appliedFilter != null,
+ // An expression that fails, fails every time, so the error shows at once.
+ retry: false,
});
const canFilter = hasFilter && (language === "json" || language === "xml" || language === "html");
@@ -195,6 +197,7 @@ export function TextViewer({
}
return formatText(text, language);
},
+ retry: false,
}).data;
if (formattedBody == null) {
return null;
diff --git a/packages/ui/src/index.ts b/packages/ui/src/index.ts
index 2718478d6..3c46e5edd 100644
--- a/packages/ui/src/index.ts
+++ b/packages/ui/src/index.ts
@@ -90,6 +90,8 @@ export { Tree } from "./components/tree/Tree";
export type { TreeItemProps } from "./components/tree/TreeItem";
export { AudioViewer } from "./components/viewers/AudioViewer";
export { CsvViewer, CsvViewerInner } from "./components/viewers/CsvViewer";
+export type { HexdumpViewerProps } from "./components/viewers/HexdumpViewer";
+export { hexdump, HexdumpViewer } from "./components/viewers/HexdumpViewer";
export { ImageViewer } from "./components/viewers/ImageViewer";
export { JsonAttributeTree } from "./components/viewers/JsonAttributeTree";
export { JsonViewer } from "./components/viewers/JsonViewer";