From 4ad053390160a3d29ad5b1c4c702c0fc41998c77 Mon Sep 17 00:00:00 2001 From: Gregory Schier Date: Wed, 4 Nov 2026 21:21:05 -0700 Subject: [PATCH] Body viewer and decode fixes (#812) Co-authored-by: Claude Opus 5.5 --- Cargo.lock | 1 + .../components/WebsocketResponsePane.tsx | 16 +- apps/yaak-client/hooks/useHttpRequestBody.ts | 4 +- apps/yaak-client/lib/model_util.ts | 9 - apps/yaak-client/lib/responseBody.ts | 5 +- apps/yaak-client/package.json | 1 - apps/yaak-proxy/components/FlowBodyViewer.tsx | 114 +++- apps/yaak-proxy/lib/body.test.ts | 211 +++++- apps/yaak-proxy/lib/body.ts | 156 ++++- apps/yaak-proxy/lib/flowActions.ts | 47 +- crates-proxy/yaak-proxy-lib/Cargo.toml | 1 + .../yaak-proxy-lib/bindings/gen_rpc.ts | 29 +- crates-proxy/yaak-proxy-lib/src/bodies.rs | 145 +++++ crates-proxy/yaak-proxy-lib/src/lib.rs | 141 +++- crates/yaak-http/Cargo.toml | 2 +- crates/yaak-http/src/decompress.rs | 611 +++++++++++++++++- crates/yaak-proxy/tests/proxy.rs | 63 ++ package-lock.json | 2 +- packages/common-lib/decodeBodyText.test.ts | 45 ++ packages/common-lib/decodeBodyText.ts | 44 ++ packages/common-lib/index.ts | 1 + packages/ui/package.json | 1 + .../components/viewers/HexdumpViewer.test.ts | 20 + .../src/components/viewers/HexdumpViewer.tsx | 50 ++ .../ui/src/components/viewers/TextViewer.tsx | 3 + packages/ui/src/index.ts | 2 + 26 files changed, 1633 insertions(+), 91 deletions(-) create mode 100644 packages/common-lib/decodeBodyText.test.ts create mode 100644 packages/common-lib/decodeBodyText.ts create mode 100644 packages/ui/src/components/viewers/HexdumpViewer.test.ts create mode 100644 packages/ui/src/components/viewers/HexdumpViewer.tsx diff --git a/Cargo.lock b/Cargo.lock index a286d1852..9ff5f085a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11852,6 +11852,7 @@ dependencies = [ "base64 0.23.1", "bytes", "chrono", + "encoding_rs", "flate2", "http-body-util", "hyper", diff --git a/apps/yaak-client/components/WebsocketResponsePane.tsx b/apps/yaak-client/components/WebsocketResponsePane.tsx index 46cebd3a8..418d5ae5e 100644 --- a/apps/yaak-client/components/WebsocketResponsePane.tsx +++ b/apps/yaak-client/components/WebsocketResponsePane.tsx @@ -1,13 +1,14 @@ import type { WebsocketEvent, WebsocketRequest } from "@yaakapp-internal/models"; import { HStack, + hexdump, + HexdumpViewer, Icon, LoadingIcon, VStack, useStateWithDeps, languageFromContentType, } from "@yaakapp-internal/ui"; -import { hexy } from "hexy"; import { useAtomValue } from "jotai"; import { useMemo, useState } from "react"; import { useFormatText } from "../hooks/useFormatText"; @@ -19,6 +20,7 @@ import { } from "../hooks/usePinnedWebsocketConnection"; import { Button } from "./core/Button"; import { Editor } from "./core/Editor/LazyEditor"; +import { renderEditor } from "./core/Editor/renderEditor"; import { type EventDetailAction, EventDetailHeader, EventViewer } from "./core/EventViewer"; import { EventViewerRow } from "./core/EventViewerRow"; import { HotkeyList } from "./core/HotkeyList"; @@ -122,10 +124,10 @@ function WebsocketEventRow({ messageType === "error" ? "alert_triangle" : messageType === "close" || messageType === "open" - ? "info" - : isServer - ? "arrow_big_down_dash" - : "arrow_big_up_dash"; + ? "info" + : isServer + ? "arrow_big_down_dash" + : "arrow_big_up_dash"; const content = messageType === "close" ? ( @@ -172,7 +174,7 @@ function WebsocketEventDetail({ }) { const message = useMemo(() => { if (hexDump) { - return event.message ? hexy(event.message) : ""; + return event.message ? hexdump(event.message) : ""; } return event.message ? new TextDecoder("utf-8").decode(Uint8Array.from(event.message)) : ""; }, [event.message, hexDump]); @@ -232,6 +234,8 @@ function WebsocketEventDetail({ ) : event.message.length === 0 ? ( No Content + ) : hexDump ? ( + ) : ( h.name.toLowerCase() === "content-type")?.value ?? null; } -export function getCharsetFromContentType(headers: HttpResponseHeader[]): string | null { - const contentType = getContentTypeFromHeaders(headers); - if (contentType == null) return null; - - const mimeType = getMimeTypeFromContentType(contentType); - return mimeType.parameters.get("charset") ?? null; -} - export function isBaseEnvironment(environment: Environment): boolean { return environment.parentModel === "workspace"; } diff --git a/apps/yaak-client/lib/responseBody.ts b/apps/yaak-client/lib/responseBody.ts index bc0fe23b6..3bfdfa8b8 100644 --- a/apps/yaak-client/lib/responseBody.ts +++ b/apps/yaak-client/lib/responseBody.ts @@ -3,7 +3,9 @@ import type { FilterResponse } from "@yaakapp-internal/plugins"; import type { ServerSentEvent, SseSummary } from "@yaakapp-internal/sse"; import { candidateJsonPayloadsFromSseText, computeSseSummary } from "@yaakapp-internal/sse"; import { rpc } from "./rpc"; +import { decodeBodyText } from "@yaakapp-internal/lib/decodeBodyText"; import { platform } from "@yaakapp-internal/platform"; +import { getContentTypeFromHeaders } from "./model_util"; /** * Reading a response body means naming the response, never the file it lives @@ -74,5 +76,6 @@ export async function getResponseBodyBytes( async function getResponseBodyDecoded(response: HttpResponse): Promise { const bytes = await getResponseBodyBytes(response); - return bytes == null ? null : new TextDecoder("utf-8").decode(bytes); + if (bytes == null) return null; + return decodeBodyText(bytes, getContentTypeFromHeaders(response.headers)); } diff --git a/apps/yaak-client/package.json b/apps/yaak-client/package.json index 26bfaba2a..3edfa72a9 100644 --- a/apps/yaak-client/package.json +++ b/apps/yaak-client/package.json @@ -48,7 +48,6 @@ "focus-trap-react": "^12.0.3", "fuzzbunny": "^1.0.1", "graphql": "^17.0.2", - "hexy": "^0.4.0", "history": "^5.3.0", "jotai": "^2.18.0", "jotai-family": "^1.0.1", diff --git a/apps/yaak-proxy/components/FlowBodyViewer.tsx b/apps/yaak-proxy/components/FlowBodyViewer.tsx index 133ff4c96..b86c7e2b8 100644 --- a/apps/yaak-proxy/components/FlowBodyViewer.tsx +++ b/apps/yaak-proxy/components/FlowBodyViewer.tsx @@ -3,7 +3,10 @@ import { formatSize } from "@yaakapp-internal/lib/formatSize"; import type { Flow, FlowBody } from "@yaakapp-internal/proxy-lib"; import { Banner, + Button, CsvViewer, + HexdumpViewer, + Icon, ImageViewer, languageFromContentType, LoadingIcon, @@ -14,16 +17,17 @@ import { } from "@yaakapp-internal/ui"; import type { CodeEditorProps } from "@yaakapp-internal/ui/editor"; import type { ReactNode } from "react"; -import { useCallback, useEffect, useState } from "react"; +import { useCallback, useEffect, useMemo, useState } from "react"; import { - base64ToBytes, - bytesToText, + bodyAsText, + bodyImageType, filterText, findHeader, + flowBodyQueryOptions, formatText, mimeEssence, } from "../lib/body"; -import { rpc } from "../lib/rpc"; +import { useSaveFlowBody } from "../lib/flowActions"; interface Props { flow: Flow; @@ -32,18 +36,7 @@ interface Props { } export function useFlowBody(flow: Flow, which: FlowBody, enabled = true) { - return useQuery({ - enabled, - queryKey: ["get_flow_body", flow.id, which, flow.updatedAt], - queryFn: async () => { - const res = await rpc("get_flow_body", { flowId: flow.id, which }); - return { - bytes: base64ToBytes(res.data), - truncated: res.truncated, - storedSize: res.storedSize, - }; - }, - }); + return useQuery({ ...flowBodyQueryOptions(flow, which), enabled }); } export function FlowBodyViewer({ flow, which, pretty }: Props) { @@ -60,8 +53,8 @@ export function FlowBodyViewer({ flow, which, pretty }: Props) { return {body.error.message}; } - const { bytes, truncated, storedSize } = body.data; - if (bytes.length === 0) { + const { bytes, truncated, storedSize, decodeError } = body.data; + if (bytes.length === 0 && decodeError == null) { return {which === "request" ? "No request body" : "Empty"}; } @@ -70,12 +63,16 @@ export function FlowBodyViewer({ flow, which, pretty }: Props) { const mime = mimeEssence(contentType) ?? ""; const wireSize = which === "request" ? flow.requestSize : flow.responseSize; const wireTruncated = which === "request" ? flow.requestTruncated : flow.responseTruncated; + // An image by its bytes or type, else text whatever the content type says, so a page sent with + // an odd one still reads: `viewerText`'s rule. + const imageType = decodeError == null ? bodyImageType(bytes, truncated, contentType) : null; + const text = + decodeError == null && imageType == null ? bodyAsText(bytes, truncated, contentType) : null; let viewer: ReactNode; - if (mime.startsWith("image/svg")) { - viewer = ; - } else if (mime.startsWith("image/")) { - viewer = ; + if (decodeError != null) { + // What failed to decode is as it was on the wire, compressed, so never text. + viewer = ; } else if (mime.startsWith("multipart/") && pretty) { const boundary = contentType?.match(/;\s*boundary="?([^";]+)"?/i)?.[1] ?? "unknown"; viewer = ( @@ -86,14 +83,17 @@ export function FlowBodyViewer({ flow, which, pretty }: Props) { textViewerProps={{ formatText }} /> ); + } else if (imageType != null) { + viewer = ; + } else if (text == null) { + viewer = ; + } else if (mime.startsWith("image/svg")) { + viewer = ; } else if (/csv|tab-separated/.test(mime) && pretty) { - viewer = ; + viewer = ; } else { - const text = bytesToText(bytes, truncated); const language = languageFromContentType(contentType, text); - if (text == null) { - viewer = Binary body, {formatSize(bytes.length)}; - } else if (language === "html" && pretty && which === "response") { + if (language === "html" && pretty && which === "response") { viewer = ; } else { viewer = ( @@ -102,16 +102,60 @@ export function FlowBodyViewer({ flow, which, pretty }: Props) { } } - if (!truncated) return viewer; + if (!truncated && decodeError == null) return viewer; return ( -
- - {wireTruncated - ? `Showing the first ${formatSize(storedSize)} of ${formatSize(wireSize)}, the capture limit. The app received all of it.` - : `Showing the first ${formatSize(bytes.length)} of the decoded body, the capture limit. The app received all of it.`} - -
{viewer}
+
+ {decodeError != null && ( + + {decodeError}. Showing the body as it was on the wire. + + )} + {truncated && ( + + {wireTruncated || decodeError != null + ? `Showing the first ${formatSize(storedSize)} of ${formatSize(wireSize)}, the capture limit. The app received all of it.` + : `Showing the first ${formatSize(bytes.length)} of the decoded body, the capture limit. The app received all of it.`} + + )} +
{viewer}
+
+ ); +} + +/** Bytes shown in the hexdump; the rest are in the saved file */ +const HEXDUMP_LIMIT = 256 * 1024; + +/** A body that isn't text: a hexdump, and a way to save it */ +function BinaryBody({ + flow, + which, + bytes, +}: { + flow: Flow; + which: FlowBody; + bytes: Uint8Array; +}) { + const saveBody = useSaveFlowBody(); + const shown = useMemo(() => bytes.subarray(0, HEXDUMP_LIMIT), [bytes]); + + return ( +
+
+ + Binary body, {formatSize(bytes.length)} + {shown.length < bytes.length && `, first ${formatSize(shown.length)} shown`} + + +
+
); } diff --git a/apps/yaak-proxy/lib/body.test.ts b/apps/yaak-proxy/lib/body.test.ts index 48c83d61c..804d3d683 100644 --- a/apps/yaak-proxy/lib/body.test.ts +++ b/apps/yaak-proxy/lib/body.test.ts @@ -1,5 +1,17 @@ -import { describe, expect, test } from "vite-plus/test"; -import { bytesToText } from "./body"; +import { QueryClient, QueryObserver } from "@tanstack/react-query"; +import type { Flow } from "@yaakapp-internal/proxy-lib"; +import { describe, expect, test, vi } from "vite-plus/test"; +import { + bodyAsText, + bodyFileName, + bodyImageType, + bytesToText, + flowBodyQueryOptions, + viewerText, +} from "./body"; +import { rpc } from "./rpc"; + +vi.mock("./rpc", () => ({ rpc: vi.fn() })); const encode = (text: string) => new TextEncoder().encode(text); @@ -18,3 +30,198 @@ describe("bytesToText", () => { expect(bytesToText(cut, true)).toBe("abc"); }); }); + +describe("bodyAsText", () => { + test("reads UTF-8 as text", () => { + expect(bodyAsText(encode("

Forbidden

\n\tcafé"))).toBe("

Forbidden

\n\tcafé"); + }); + + test("reads unlabeled Latin-1 text as text, as windows-1252", () => { + const page = "

Accès refusé : vous n'avez pas l'autorisation d'accéder à cette page.

"; + const latin1 = Uint8Array.from(page, (c) => c.charCodeAt(0)); + expect(bodyAsText(latin1)).toBe(page); + }); + + test("reads bytes UTF-8 can't read as binary, though windows-1252 reads every one", () => { + let seed = 7; + const random = Uint8Array.from({ length: 600 }, () => { + seed = (seed * 1103515245 + 12345) & 0x7fffffff; + return seed >> 16; + }); + expect(bodyAsText(random)).toBeNull(); + // All high bytes: letters and symbols in windows-1252, none of them UTF-8 + const high = Uint8Array.from({ length: 600 }, (_, i) => 0xa0 + (i % 96)); + expect(bodyAsText(high)).toBeNull(); + expect(bodyAsText(high, false, "text/plain")).toBeNull(); + }); + + test("reads protobuf and other binary as binary, even when it is valid UTF-8", () => { + // Field tags, lengths and varints: control characters between short strings. + const protobuf = new Uint8Array([ + 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x31, 0x12, 0x02, 0x08, 0x01, 0x1a, 0x03, 0x10, 0x96, 0x01, + 0x22, 0x00, 0x2a, 0x02, 0x18, 0x07, + ]); + expect(bodyAsText(protobuf)).toBeNull(); + expect(bodyAsText(new Uint8Array([0x01, 0x02, 0x03, 0x04, 0x41, 0x42]))).toBeNull(); + expect(bodyAsText(new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))).toBeNull(); + }); +}); + +describe("viewerText", () => { + // A PNG signature, then a tEXt chunk of printable metadata far longer than the image + const png = new Uint8Array([ + 0x89, + ...encode("PNG\r\n"), + 0x1a, + 0x0a, + 0x00, + 0x00, + 0x12, + 0xc0, + ...encode(`tEXtDescription\0${"A long printable description. ".repeat(160)}`), + ]); + + test("keeps an image with printable metadata an image", () => { + expect(bodyAsText(png)).not.toBeNull(); + expect(viewerText(png, false, "image/png")).toBeNull(); + }); + + test("reads text sent as an image as text, and other types text first", () => { + expect(viewerText(encode("

Forbidden

"), false, "image/png")).toBe("

Forbidden

"); + expect(bodyImageType(encode("

Forbidden

"), false, "image/png")).toBeNull(); + expect(viewerText(encode(""), false, "image/svg+xml")).toBe(""); + expect(bodyImageType(encode(""), false, "image/svg+xml")).toBeNull(); + const bmwText = encode("BMW sells cars.\n"); + expect(viewerText(bmwText, false, "text/plain")).toBe("BMW sells cars.\n"); + }); + + // A 1×1 black 24-bit BMP: every byte below 0x80, so valid UTF-8 + // File header, then the 40-byte DIB header, then one pixel padded to four bytes + // prettier-ignore + const bmp = new Uint8Array([ + 0x42, 0x4d, 58, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, + 40, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 24, 0, 0, 0, 0, 0, 4, 0, 0, 0, + 0x13, 0x0b, 0, 0, 0x13, 0x0b, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, + ]); + + test("keeps an image that is valid UTF-8 an image", () => { + expect(bytesToText(bmp)).not.toBeNull(); + expect(bodyImageType(bmp, false, "image/bmp")).toBe("image/bmp"); + expect(viewerText(bmp, false, "image/bmp")).toBeNull(); + }); + + test("previews an image by its signature whatever its content type", () => { + expect(bodyImageType(bmp, false, "application/octet-stream")).toBe("image/bmp"); + expect(bodyImageType(png, false, "application/octet-stream")).toBe("image/png"); + expect(bodyImageType(png, false, null)).toBe("image/png"); + expect(viewerText(png, false, "application/octet-stream")).toBeNull(); + const gif = encode("GIF89a\x01\0\x01\0"); + expect(bodyImageType(gif, false, "text/plain")).toBe("image/gif"); + }); + + test("previews formats not every webview draws only when their content type says so", () => { + const tiff = new Uint8Array([0x49, 0x49, 0x2a, 0x00, 0x08, 0x00, 0x00, 0x00, 0xff, 0x80]); + const heic = new Uint8Array([0, 0, 0, 0x18, ...encode("ftypheic"), 0, 0, 0, 0, 0xff, 0x80]); + for (const image of [tiff, heic]) { + expect(bodyImageType(image, false, "application/octet-stream")).toBeNull(); + expect(viewerText(image, false, "application/octet-stream")).toBeNull(); + } + expect(bodyImageType(tiff, false, "image/tiff")).toBe("image/tiff"); + expect(bodyImageType(heic, false, "image/heic")).toBe("image/heic"); + }); + + test("reads text in the charset its content type names", () => { + const page = "Accès refusé : vous n'avez pas l'autorisation d'accéder à cette page."; + const latin1 = Uint8Array.from(page, (c) => c.charCodeAt(0)); + expect(viewerText(latin1, false, "text/html; charset=iso-8859-1")).toBe(page); + // "こんにちは、世界" in Shift_JIS + const shiftJis = new Uint8Array([ + 0x82, 0xb1, 0x82, 0xf1, 0x82, 0xc9, 0x82, 0xbf, 0x82, 0xcd, 0x81, 0x41, 0x90, 0xa2, 0x8a, + 0x45, + ]); + expect(viewerText(shiftJis, false, "text/plain; charset=shift_jis")).toBe("こんにちは、世界"); + }); + + test("reads unlabeled Latin-1 as windows-1252, and UTF-8 as UTF-8", () => { + const page = "Accès refusé : vous n'avez pas l'autorisation d'accéder à cette page."; + const latin1 = Uint8Array.from(page, (c) => c.charCodeAt(0)); + expect(viewerText(latin1, false, "application/octet-stream")).toBe(page); + expect(viewerText(encode(page), false, "application/octet-stream")).toBe(page); + expect(viewerText(encode(page), false, "text/plain; charset=utf-8")).toBe(page); + }); +}); + +const flow = (url: string, contentType: string | null) => + ({ + id: "fl_1", + url, + updatedAt: "2026-11-04T00:00:00", + requestHeaders: contentType == null ? [] : [{ name: "Content-Type", value: contentType }], + responseHeaders: contentType == null ? [] : [{ name: "content-type", value: contentType }], + }) as unknown as Flow; + +describe("bodyFileName", () => { + test("uses the URL's file name for a response", () => { + expect(bodyFileName(flow("https://a.test/img/logo%20big.png?v=2", null), "response")).toBe( + "logo big.png", + ); + }); + + test("names a body from its content type otherwise", () => { + const protobuf = flow("https://a.test/v1:GetModels", "application/x-protobuf"); + expect(bodyFileName(protobuf, "request")).toBe("request.protobuf"); + expect(bodyFileName(protobuf, "response")).toBe("response.protobuf"); + expect(bodyFileName(flow("https://a.test/", "text/plain; charset=utf-8"), "response")).toBe( + "response.txt", + ); + expect(bodyFileName(flow("https://a.test/api", "application/problem+json"), "response")).toBe( + "response.json", + ); + expect(bodyFileName(flow("https://a.test/data.json", null), "request")).toBe("request.bin"); + expect(bodyFileName(flow("not a url", "application/octet-stream"), "response")).toBe( + "response.bin", + ); + }); +}); + +describe("flowBodyQueryOptions", () => { + test("shows a body that fails to read at once, without retrying", async () => { + vi.mocked(rpc).mockRejectedValue(new Error("No flow fl_1")); + // React Query retries three times in a browser, and not at all where there is no window. + const client = new QueryClient({ defaultOptions: { queries: { retry: 3 } } }); + const observer = new QueryObserver( + client, + flowBodyQueryOptions(flow("https://a.test/", null), "response"), + ); + const failed = new Promise((resolve) => { + const unsubscribe = observer.subscribe((result) => { + if (result.status === "error") { + unsubscribe(); + resolve(result.error.message); + } + }); + }); + // React Query's default retries would keep it pending for seconds. + const timeout = new Promise((resolve) => setTimeout(() => resolve("pending"), 500)); + expect(await Promise.race([failed, timeout])).toBe("No flow fl_1"); + expect(rpc).toHaveBeenCalledTimes(1); + client.clear(); + }); + + test("passes on why a body failed to decode, with its bytes as they were", async () => { + vi.mocked(rpc).mockResolvedValue({ + data: btoa("\x78\x9c\x01"), + truncated: false, + storedSize: 3, + decodeError: "Failed to decode Deflate body: deflate decompression error", + }); + const client = new QueryClient(); + const body = await client.fetchQuery( + flowBodyQueryOptions(flow("https://a.test/", null), "request"), + ); + expect(Array.from(body.bytes)).toEqual([0x78, 0x9c, 0x01]); + expect(body.decodeError).toBe("Failed to decode Deflate body: deflate decompression error"); + client.clear(); + }); +}); diff --git a/apps/yaak-proxy/lib/body.ts b/apps/yaak-proxy/lib/body.ts index b3129dfb3..ae8d896ca 100644 --- a/apps/yaak-proxy/lib/body.ts +++ b/apps/yaak-proxy/lib/body.ts @@ -1,8 +1,32 @@ -import type { FilterLanguage, FlowBody } from "@yaakapp-internal/proxy-lib"; +import { queryOptions } from "@tanstack/react-query"; +import { + contentTypeCharset, + decodeBodyText, + strictUtf8Text, +} from "@yaakapp-internal/lib/decodeBodyText"; +import type { FilterLanguage, Flow, FlowBody } from "@yaakapp-internal/proxy-lib"; import type { CodeEditorProps } from "@yaakapp-internal/ui/editor"; import vkBeautify from "vkbeautify"; import { rpc } from "./rpc"; +/** A flow's body for the viewer: decoded, or as it was on the wire when `decodeError` is set */ +export function flowBodyQueryOptions(flow: Flow, which: FlowBody) { + return queryOptions({ + queryKey: ["get_flow_body", flow.id, which, flow.updatedAt], + queryFn: async () => { + const res = await rpc("get_flow_body", { flowId: flow.id, which }); + return { + bytes: base64ToBytes(res.data), + truncated: res.truncated, + storedSize: res.storedSize, + decodeError: res.decodeError ?? null, + }; + }, + // Reading a stored body fails the same way every time, so the error shows at once. + retry: false, + }); +} + export function base64ToBytes(data: string): Uint8Array { const binary = atob(data); const bytes = new Uint8Array(binary.length); @@ -16,19 +40,104 @@ export function base64ToBytes(data: string): Uint8Array { * The text of a body, or null when it isn't valid UTF-8. A truncated body may end partway * through a character, so up to three trailing bytes are dropped before giving up on it. */ -export function bytesToText(bytes: Uint8Array, truncated = false): string | null { - const decoder = new TextDecoder("utf-8", { fatal: true }); - const maxTrim = truncated ? Math.min(3, bytes.length) : 0; - for (let trim = 0; trim <= maxTrim; trim++) { - try { - return decoder.decode(bytes.subarray(0, bytes.length - trim)); - } catch { - // Try again without the last byte - } +export const bytesToText = strictUtf8Text; + +/** Characters looked at to tell text from binary */ +const TEXT_SAMPLE = 64 * 1024; + +/** The share of those that may be control characters or undecodable bytes in text */ +const MAX_UNPRINTABLE = 0.1; + +/** + * The body as text, whatever its content type says, or null when it is binary. It is decoded in + * its content type's charset, or else as UTF-8, or windows-1252 when it isn't valid UTF-8. It is + * binary when it is full of control characters, as protobuf is, or, without a charset, of bytes + * UTF-8 can't read: windows-1252 reads every byte as something, compressed data included. + */ +export function bodyAsText( + bytes: Uint8Array, + truncated = false, + contentType: string | null = null, +): string | null { + const text = decodeBodyText(bytes, contentType, { truncated }); + const judged = + contentTypeCharset(contentType) != null + ? text + : (strictUtf8Text(bytes, truncated) ?? new TextDecoder("utf-8").decode(bytes)); + let total = 0; + let unprintable = 0; + for (const char of judged) { + if (total === TEXT_SAMPLE) break; + total++; + const code = char.codePointAt(0) ?? 0; + const control = code < 0x20 && code !== 0x09 && code !== 0x0a && code !== 0x0d; + if (control || code === 0x7f || code === 0xfffd) unprintable++; + } + return unprintable <= total * MAX_UNPRINTABLE ? text : null; +} + +const ascii = (bytes: Uint8Array, start: number, text: string) => + Array.from(text).every((c, i) => bytes[start + i] === c.charCodeAt(0)); + +/** + * The image type a body's first bytes name, or null when they name none. Only types every + * platform's webview draws, so a body sniffed as an image always previews: TIFF, HEIC and AVIF + * preview only when their content type says so. + */ +export function imageSignature(bytes: Uint8Array): string | null { + const b = bytes; + if (b.length < 4) return null; + if (b[0] === 0x89 && ascii(b, 1, "PNG\r\n\x1a\n")) return "image/png"; + if (b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff) return "image/jpeg"; + if (ascii(b, 0, "GIF87a") || ascii(b, 0, "GIF89a")) return "image/gif"; + if (ascii(b, 0, "RIFF") && ascii(b, 8, "WEBP")) return "image/webp"; + if (ascii(b, 0, "\0\0\x01\0") && b.length >= 6 && (b[4] ?? 0) + (b[5] ?? 0) > 0) { + return "image/x-icon"; + } + // "BM" alone starts plenty of text, so the reserved bytes and DIB header size must fit too. + const dibSize = (b[14] ?? 0) | ((b[15] ?? 0) << 8) | ((b[16] ?? 0) << 16) | ((b[17] ?? 0) << 24); + if ( + ascii(b, 0, "BM") && + ascii(b, 6, "\0\0\0\0") && + [12, 40, 52, 56, 64, 108, 124].includes(dibSize) + ) { + return "image/bmp"; } return null; } +/** + * The image type to preview a body as, or null when it isn't one: what its first bytes name, + * whatever its content type, or else an `image/*` content type other than SVG, unless the body + * is genuinely text, as an error page sent with the image's type is. + */ +export function bodyImageType( + bytes: Uint8Array, + truncated: boolean, + contentType: string | null, +): string | null { + const signed = imageSignature(bytes); + if (signed != null) return signed; + const mime = mimeEssence(contentType) ?? ""; + if (!mime.startsWith("image/") || mime.startsWith("image/svg")) return null; + const genuineText = + strictUtf8Text(bytes, truncated) != null && bodyAsText(bytes, truncated, contentType) != null; + return genuineText ? null : mime; +} + +/** + * The body as the viewer reads it: text first, whatever its content type, except an image, which + * is never text. + */ +export function viewerText( + bytes: Uint8Array, + truncated: boolean, + contentType: string | null, +): string | null { + if (bodyImageType(bytes, truncated, contentType) != null) return null; + return bodyAsText(bytes, truncated, contentType); +} + export async function formatText( text: string, language: CodeEditorProps["language"], @@ -81,6 +190,33 @@ export function shortContentType(contentType: string | null): string | null { return suffix.replace(/^x-/, ""); } +/** Extensions for media types whose subtype isn't one */ +const EXTENSIONS: Record = { + "application/javascript": "js", + "application/octet-stream": "bin", + "image/jpeg": "jpg", + "image/svg+xml": "svg", + "text/javascript": "js", + "text/markdown": "md", + "text/plain": "txt", +}; + +/** A name to offer for a saved body: the URL's file name, or one from the content type */ +export function bodyFileName(flow: Flow, which: FlowBody): string { + let last = ""; + try { + last = decodeURIComponent(new URL(flow.url).pathname.split("/").pop() ?? ""); + } catch { + // Not a URL, or not encoded properly + } + if (which === "response" && /^[^\\/:*?"<>|]+\.[a-z0-9]+$/i.test(last)) return last; + const headers = which === "request" ? flow.requestHeaders : flow.responseHeaders; + const contentType = findHeader(headers, "content-type"); + const essence = mimeEssence(contentType); + const extension = (essence == null ? null : EXTENSIONS[essence]) ?? shortContentType(contentType); + return `${which}.${extension?.replace(/[^a-z0-9-]/gi, "") || "bin"}`; +} + export function findHeader(headers: { name: string; value: string }[], name: string) { const lower = name.toLowerCase(); return headers.find((h) => h.name.toLowerCase() === lower)?.value ?? null; diff --git a/apps/yaak-proxy/lib/flowActions.ts b/apps/yaak-proxy/lib/flowActions.ts index 61d64848c..c019c6937 100644 --- a/apps/yaak-proxy/lib/flowActions.ts +++ b/apps/yaak-proxy/lib/flowActions.ts @@ -1,9 +1,10 @@ import type { ExportHarRequest, Flow, FlowBody } from "@yaakapp-internal/proxy-lib"; import type { DropdownItem } from "@yaakapp-internal/ui"; import { save } from "@tauri-apps/plugin-dialog"; +import { formatSize } from "@yaakapp-internal/lib/formatSize"; import { atom, useSetAtom } from "jotai"; import { useCallback } from "react"; -import { base64ToBytes } from "./body"; +import { base64ToBytes, bodyFileName } from "./body"; import { curlBodyText, toCurl } from "./curl"; import { rpc } from "./rpc"; import { useShowToast } from "./toast"; @@ -14,10 +15,35 @@ export const saveDialogFlowAtom = atom(null); /** A body as text, decoding what is not UTF-8 lossily, or null when there is none */ async function bodyText(flow: Flow, which: FlowBody): Promise { const res = await rpc("get_flow_body", { flowId: flow.id, which }); + if (res.decodeError != null) throw new Error(res.decodeError); const bytes = base64ToBytes(res.data); return bytes.length === 0 ? null : new TextDecoder().decode(bytes); } +/** Writes a flow's body to a file the person picks, as the viewer shows it */ +export function useSaveFlowBody(): (flow: Flow, which: FlowBody) => Promise { + const showToast = useShowToast(); + return useCallback( + async (flow: Flow, which: FlowBody) => { + try { + const path = await save({ defaultPath: bodyFileName(flow, which) }); + if (path == null) return; + const saved = await rpc("save_flow_body", { flowId: flow.id, which, path }); + const notes = [ + saved.raw ? "as it was on the wire, since it failed to decode" : null, + saved.truncated ? "cut at the capture limit" : null, + ].filter((n) => n != null); + showToast({ + message: `Saved ${formatSize(saved.size)}${notes.length > 0 ? `, ${notes.join(", ")}` : ""}`, + }); + } catch (err) { + showToast({ message: `Failed to save the ${which} body: ${String(err)}`, color: "danger" }); + } + }, + [showToast], + ); +} + /** * Writes flows to a HAR file the person picks: `ids`, or every flow `filter` covers. Returns how * many were written, or null when the save panel was cancelled. @@ -38,6 +64,7 @@ export async function exportHar( export function useFlowActions(): (flow: Flow) => DropdownItem[] { const showToast = useShowToast(); const openSave = useSetAtom(saveDialogFlowAtom); + const saveBody = useSaveFlowBody(); return useCallback( (flow: Flow) => { @@ -75,6 +102,10 @@ export function useFlowActions(): (flow: Flow) => DropdownItem[] { showToast({ message: `Failed to copy cURL: ${String(err)}`, color: "danger" }); return; } + if (res.decodeError != null) { + showToast({ message: `Failed to copy cURL: ${res.decodeError}`, color: "danger" }); + return; + } body = curlBodyText(base64ToBytes(res.data), res.truncated); // A command that sends other bytes than the app did would mislead. if (body == null) { @@ -102,6 +133,18 @@ export function useFlowActions(): (flow: Flow) => DropdownItem[] { onSelect: () => copy("response body", () => bodyText(flow, "response")), }, { type: "separator" }, + { + label: "Save Request Body to File", + icon: "download", + disabled: flow.requestSize === 0, + onSelect: () => saveBody(flow, "request"), + }, + { + label: "Save Response Body to File", + icon: "download", + disabled: noResponse || flow.responseSize === 0, + onSelect: () => saveBody(flow, "response"), + }, { label: "Export HAR", icon: "download", @@ -116,6 +159,6 @@ export function useFlowActions(): (flow: Flow) => DropdownItem[] { }, ]; }, - [openSave, showToast], + [openSave, saveBody, showToast], ); } diff --git a/crates-proxy/yaak-proxy-lib/Cargo.toml b/crates-proxy/yaak-proxy-lib/Cargo.toml index fb6145358..05845b2c7 100644 --- a/crates-proxy/yaak-proxy-lib/Cargo.toml +++ b/crates-proxy/yaak-proxy-lib/Cargo.toml @@ -9,6 +9,7 @@ publish = false base64 = "0.23.1" bytes = "1" chrono = { workspace = true, features = ["serde"] } +encoding_rs = "0.8" log = { workspace = true } http-body-util = "0.1" hyper = { version = "1", features = ["http1", "server"] } diff --git a/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts b/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts index 0ed90397d..629597ae7 100644 --- a/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts +++ b/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts @@ -133,6 +133,7 @@ export type GetFlowBodyRequest = { flowId: string, which: FlowBody, }; export type GetFlowBodyResponse = { /** * The body decoded with its Content-Encoding, at most the capture limit of it, in base64. + * When it fails to decode, the bytes as they were on the wire instead. */ data: string, /** @@ -142,7 +143,11 @@ truncated: boolean, /** * Bytes stored for the body, as they were on the wire. */ -storedSize: number, }; +storedSize: number, +/** + * Why the body failed to decode, in which case `data` is as it was on the wire. + */ +decodeError?: string, }; export type GetProxyStateRequest = Record; @@ -238,7 +243,27 @@ export type ProxyStatePayload = { state: ProxyState, }; export type RpcEventSchema = { flows_cleared: FlowsClearedPayload, model_write: ModelPayload, proxy_state_changed: ProxyStatePayload, }; -export type RpcSchema = { execute_action: [ActionInvocation, boolean], format_body: [FormatBodyRequest, string], list_actions: [ListActionsRequest, ListActionsResponse], list_flows: [FlowFilter, ListFlowsResponse], set_recording: [SetRecordingRequest, boolean], set_system_proxy_at_launch: [SetSystemProxyAtLaunchRequest, boolean], set_http_version: [SetHttpVersionRequest, HttpVersion], app_icon: [AppIconRequest, string | null], ca_status: [CaStatusRequest, CaStatus], ca_trust: [CaTrustRequest, CaStatus], clear_flows: [ClearFlowsRequest, number], export_har: [ExportHarRequest, number], filter_body: [FilterBodyRequest, FilterBodyResponse], get_flow_body: [GetFlowBodyRequest, GetFlowBodyResponse], get_proxy_state: [GetProxyStateRequest, GetProxyStateResponse], helper_register: [HelperRegisterRequest, SystemProxyStatus], helper_unregister: [HelperUnregisterRequest, SystemProxyStatus], list_workspaces: [ListWorkspacesRequest, ListWorkspacesResponse], preview_save: [PreviewSaveRequest, SavePreview], save_flow: [SaveFlowToYaakRequest, SavedFlow], system_proxy_set: [SystemProxySetRequest, SystemProxyStatus], system_proxy_status: [SystemProxyStatusRequest, SystemProxyStatus], take_over: [TakeOverRequest, boolean], }; +export type RpcSchema = { execute_action: [ActionInvocation, boolean], format_body: [FormatBodyRequest, string], list_actions: [ListActionsRequest, ListActionsResponse], list_flows: [FlowFilter, ListFlowsResponse], set_recording: [SetRecordingRequest, boolean], set_system_proxy_at_launch: [SetSystemProxyAtLaunchRequest, boolean], set_http_version: [SetHttpVersionRequest, HttpVersion], app_icon: [AppIconRequest, string | null], ca_status: [CaStatusRequest, CaStatus], ca_trust: [CaTrustRequest, CaStatus], clear_flows: [ClearFlowsRequest, number], export_har: [ExportHarRequest, number], filter_body: [FilterBodyRequest, FilterBodyResponse], get_flow_body: [GetFlowBodyRequest, GetFlowBodyResponse], get_proxy_state: [GetProxyStateRequest, GetProxyStateResponse], helper_register: [HelperRegisterRequest, SystemProxyStatus], helper_unregister: [HelperUnregisterRequest, SystemProxyStatus], list_workspaces: [ListWorkspacesRequest, ListWorkspacesResponse], preview_save: [PreviewSaveRequest, SavePreview], save_flow: [SaveFlowToYaakRequest, SavedFlow], save_flow_body: [SaveFlowBodyRequest, SaveFlowBodyResponse], system_proxy_set: [SystemProxySetRequest, SystemProxyStatus], system_proxy_status: [SystemProxyStatusRequest, SystemProxyStatus], take_over: [TakeOverRequest, boolean], }; + +export type SaveFlowBodyRequest = { flowId: string, which: FlowBody, +/** + * The file to write. + */ +path: string, }; + +export type SaveFlowBodyResponse = { +/** + * Bytes written. + */ +size: number, +/** + * What was written is incomplete, as `GetFlowBodyResponse.truncated` is. + */ +truncated: boolean, +/** + * The body failed to decode, so the file has it as it was on the wire. + */ +raw: boolean, }; export type SaveFlowToYaakRequest = { flowId: string, workspaceId: string, folderId?: string, /** diff --git a/crates-proxy/yaak-proxy-lib/src/bodies.rs b/crates-proxy/yaak-proxy-lib/src/bodies.rs index ed3f07abf..cd249a6fd 100644 --- a/crates-proxy/yaak-proxy-lib/src/bodies.rs +++ b/crates-proxy/yaak-proxy-lib/src/bodies.rs @@ -67,6 +67,61 @@ pub async fn read_body( Ok(DecodedBody { data, stored_size, truncated, decode_error }) } +/// A body's text as the app's viewer reads it, as `decodeBodyText` in `@yaakapp-internal/lib` +/// does: in the charset its content type names, when it names one the WHATWG Encoding Standard +/// knows, or else as UTF-8, or windows-1252 when it isn't valid UTF-8, as browsers read text. +pub fn body_text(data: &[u8], content_type: Option<&str>, truncated: bool) -> String { + let encoding = content_type + .and_then(charset_label) + .and_then(|label| encoding_rs::Encoding::for_label(label.as_bytes())) + // A label the standard maps to "replacement" is one `TextDecoder` refuses. + .filter(|encoding| *encoding != encoding_rs::REPLACEMENT); + if let Some(encoding) = encoding { + return encoding.decode_with_bom_removal(data).0.into_owned(); + } + if let Some(text) = strict_utf8(data, truncated) { + return text; + } + encoding_rs::WINDOWS_1252.decode_without_bom_handling(data).0.into_owned() +} + +/// The `charset` parameter of a content type +fn charset_label(content_type: &str) -> Option<&str> { + content_type.split(';').skip(1).find_map(|param| { + let (name, value) = param.split_once('=')?; + name.trim().eq_ignore_ascii_case("charset").then(|| value.trim().trim_matches('"').trim()) + }) +} + +/// The bytes as UTF-8 without a byte order mark, or None when they aren't valid UTF-8. A +/// truncated body may end partway through a character, so up to three trailing bytes are dropped +/// before giving up on it. +fn strict_utf8(data: &[u8], truncated: bool) -> Option { + let max_trim = if truncated { data.len().min(3) } else { 0 }; + (0..=max_trim).find_map(|trim| { + let text = std::str::from_utf8(&data[..data.len() - trim]).ok()?; + Some(text.strip_prefix('\u{feff}').unwrap_or(text).to_string()) + }) +} + +/// A flow's body decoded as `read_body` does, or, when it fails to decode, the bytes as they were +/// on the wire, with `decode_error` saying why. +pub async fn read_body_or_raw( + blobs: &BlobManager, + flow: &Flow, + body: FlowBody, + limit: usize, +) -> Result { + let decoded = read_body(blobs, flow, body, limit).await?; + if decoded.decode_error.is_none() { + return Ok(decoded); + } + let mut data = read_raw_body(blobs, &flow.id, body)?; + let truncated = flow.truncated(body) || data.len() > limit; + data.truncate(limit); + Ok(DecodedBody { data, truncated, ..decoded }) +} + #[cfg(test)] mod tests { use super::*; @@ -127,4 +182,94 @@ mod tests { assert!(body.truncated); assert_eq!(body.decode_error, None); } + + /// The cases `decodeBodyText`'s tests in `@yaakapp-internal/lib` cover, read the same way + #[test] + fn body_text_reads_as_the_viewer_does() { + let french = "Accès refusé : vous n'avez pas l'autorisation d'accéder à cette page."; + let latin1: Vec = french.chars().map(|c| c as u8).collect(); + assert_eq!(body_text(&latin1, Some("text/html; charset=iso-8859-1"), false), french); + assert_eq!(body_text(&latin1, Some("text/plain; CHARSET=\"ISO-8859-1\""), false), french); + let shift_jis = [ + 0x82, 0xb1, 0x82, 0xf1, 0x82, 0xc9, 0x82, 0xbf, 0x82, 0xcd, 0x81, 0x41, 0x90, 0xa2, + 0x8a, 0x45, + ]; + assert_eq!( + body_text(&shift_jis, Some("text/plain; charset=Shift_JIS"), false), + "こんにちは、世界" + ); + + // Unlabeled and not UTF-8: windows-1252 + assert_eq!(body_text(&latin1, Some("text/html"), false), french); + assert_eq!(body_text(&latin1, None, false), french); + assert_eq!(body_text(&[0x93, b'h', b'i', 0x94], None, false), "\u{201c}hi\u{201d}"); + // A label `TextDecoder` refuses is no label. + assert_eq!(body_text(&latin1, Some("text/plain; charset=not-a-charset"), false), french); + assert_eq!(body_text(&latin1, Some("text/plain; charset=iso-2022-kr"), false), french); + + // UTF-8, labeled or not; labeled, what isn't UTF-8 is replaced + let json = r#"{"city":"Zürich","greeting":"こんにちは"}"#; + assert_eq!(body_text(json.as_bytes(), None, false), json); + assert_eq!( + body_text(json.as_bytes(), Some("application/json; charset=utf-8"), false), + json + ); + assert_eq!(body_text(b"a\xff", Some("text/plain; charset=utf-8"), false), "a\u{fffd}"); + + // A UTF-8 body cut partway through a character stays UTF-8 + assert_eq!(body_text(&"abc€".as_bytes()[..5], None, true), "abc"); + } + + fn deflate_request(blobs: &BlobManager, id: &str, stored: &[u8]) -> Flow { + let mut writer = BodyChunkWriter::new(blobs.clone(), Flow::body_id(id, FlowBody::Request)); + writer.write(stored).unwrap(); + writer.finish().unwrap(); + Flow { + id: id.into(), + request_size: stored.len() as i64, + request_headers: vec![crate::models::ProxyHeader { + name: "Content-Encoding".into(), + value: "deflate".into(), + }], + ..Default::default() + } + } + + /// Browser SDKs send `deflate` both ways: zlib-wrapped, as the spec says, and raw. + #[tokio::test] + async fn deflate_request_bodies_decode_wrapped_or_raw() { + use std::io::Write; + let (_db, blobs, _rx) = yaak_models::init_in_memory().unwrap(); + let logs = br#"[{"message":"hello","service":"browser"}]"#; + let mut zlib = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + zlib.write_all(logs).unwrap(); + let mut raw = + flate2::write::DeflateEncoder::new(Vec::new(), flate2::Compression::default()); + raw.write_all(logs).unwrap(); + + for (id, stored) in [ + ("fl_zlib", zlib.finish().unwrap()), + ("fl_raw", raw.finish().unwrap()), + ] { + let flow = deflate_request(&blobs, id, &stored); + let body = read_body(&blobs, &flow, FlowBody::Request, 1024).await.unwrap(); + assert_eq!(body.decode_error, None, "{id}"); + assert_eq!((body.data.as_slice(), body.truncated), (&logs[..], false), "{id}"); + } + } + + #[tokio::test] + async fn undecodable_body_reads_as_it_was_on_the_wire() { + let (_db, blobs, _rx) = yaak_models::init_in_memory().unwrap(); + let flow = deflate_request(&blobs, "fl_bad", b"\x00\x01 not deflate"); + + let decoded = read_body(&blobs, &flow, FlowBody::Request, 1024).await.unwrap(); + assert!(decoded.decode_error.is_some()); + let raw = read_body_or_raw(&blobs, &flow, FlowBody::Request, 1024).await.unwrap(); + assert_eq!(raw.data, b"\x00\x01 not deflate"); + assert!(!raw.truncated); + assert_eq!(raw.decode_error, decoded.decode_error); + let cut = read_body_or_raw(&blobs, &flow, FlowBody::Request, 4).await.unwrap(); + assert_eq!((cut.data.as_slice(), cut.truncated), (&b"\x00\x01 n"[..], true)); + } } diff --git a/crates-proxy/yaak-proxy-lib/src/lib.rs b/crates-proxy/yaak-proxy-lib/src/lib.rs index 6ccf2a9ad..f27f7277c 100644 --- a/crates-proxy/yaak-proxy-lib/src/lib.rs +++ b/crates-proxy/yaak-proxy-lib/src/lib.rs @@ -851,12 +851,40 @@ pub struct GetFlowBodyRequest { #[serde(rename_all = "camelCase")] pub struct GetFlowBodyResponse { /// The body decoded with its Content-Encoding, at most the capture limit of it, in base64. + /// When it fails to decode, the bytes as they were on the wire instead. pub data: String, /// The body is incomplete: cut on the wire, or decoded past the capture limit. pub truncated: bool, /// Bytes stored for the body, as they were on the wire. #[ts(type = "number")] pub stored_size: usize, + /// Why the body failed to decode, in which case `data` is as it was on the wire. + #[serde(skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub decode_error: Option, +} + +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +#[serde(rename_all = "camelCase")] +pub struct SaveFlowBodyRequest { + pub flow_id: String, + pub which: FlowBody, + /// The file to write. + pub path: String, +} + +#[derive(Serialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +#[serde(rename_all = "camelCase")] +pub struct SaveFlowBodyResponse { + /// Bytes written. + #[ts(type = "number")] + pub size: usize, + /// What was written is incomplete, as `GetFlowBodyResponse.truncated` is. + pub truncated: bool, + /// The body failed to decode, so the file has it as it was on the wire. + pub raw: bool, } #[derive(Deserialize, TS)] @@ -1082,16 +1110,42 @@ async fn get_flow_body( ) -> Result { let flow = find_flow(&ctx, &req.flow_id)?; let limit = settings(&ctx)?.capture_body_limit; - let body = viewer_body(&ctx, &flow, req.which, limit).await?; + let body = + bodies::read_body_or_raw(&ctx.blobs, &flow, req.which, limit).await.map_err(rpc_error)?; Ok(GetFlowBodyResponse { data: BASE64.encode(&body.data), truncated: body.truncated, stored_size: body.stored_size, + decode_error: body.decode_error, }) } -/// A body for the app's viewer, which shows a body that failed to decode as its error, as it -/// always has, rather than as what decoded before the failure. +/// Writes a body to a file the person picked: decoded, as the viewer shows it, or as it was on +/// the wire when it fails to decode. +async fn save_flow_body( + ctx: ProxyCtx, + req: SaveFlowBodyRequest, +) -> Result { + let flow = find_flow(&ctx, &req.flow_id)?; + let limit = settings(&ctx)?.capture_body_limit; + let body = + bodies::read_body_or_raw(&ctx.blobs, &flow, req.which, limit).await.map_err(rpc_error)?; + let saved = SaveFlowBodyResponse { + size: body.data.len(), + truncated: body.truncated, + raw: body.decode_error.is_some(), + }; + let path = PathBuf::from(req.path); + blocking(move || { + std::fs::write(&path, &body.data) + .map_err(|e| format!("Failed to write {}: {e}", path.display())) + }) + .await?; + Ok(saved) +} + +/// A body to filter, which fails when the body failed to decode rather than filtering what +/// decoded before the failure. async fn viewer_body( ctx: &ProxyCtx, flow: &Flow, @@ -1112,14 +1166,13 @@ async fn filter_body( let flow = find_flow(&ctx, &req.flow_id)?; let limit = settings(&ctx)?.capture_body_limit; let body = viewer_body(&ctx, &flow, req.which, limit).await?; - let text = String::from_utf8_lossy(&body.data); + let content_type = find_header(flow.headers(req.which), "content-type"); + // The text the viewer shows, so the filter's results match it + let text = bodies::body_text(&body.data, content_type, body.truncated); let language = match req.language { Some(FilterLanguage::JsonPath) => yaak_filter::Language::JsonPath, Some(FilterLanguage::XPath) => yaak_filter::Language::XPath, - None => { - let content_type = find_header(flow.headers(req.which), "content-type"); - yaak_filter::Language::for_content_type(content_type.unwrap_or_default()) - } + None => yaak_filter::Language::for_content_type(content_type.unwrap_or_default()), }; let result = yaak_filter::filter(language, &text, &req.expression); Ok(FilterBodyResponse { content: result.content, error: result.error }) @@ -1437,6 +1490,7 @@ define_rpc! { list_workspaces(ListWorkspacesRequest) -> ListWorkspacesResponse, preview_save(PreviewSaveRequest) -> SavePreview, save_flow(SaveFlowToYaakRequest) -> SavedFlow, + save_flow_body(SaveFlowBodyRequest) -> SaveFlowBodyResponse, system_proxy_set(SystemProxySetRequest) -> SystemProxyStatus, system_proxy_status(SystemProxyStatusRequest) -> SystemProxyStatus, take_over(TakeOverRequest) -> bool, @@ -1491,6 +1545,57 @@ mod tests { assert!(log.was_on_at(SystemTime::now())); } + /// A flow whose response is `body` under `content_type`, stored in `ctx` + fn flow_with_response(ctx: &ProxyCtx, id: &str, content_type: &str, body: &[u8]) { + let flow = Flow { + id: id.into(), + status: Some(200), + response_size: body.len() as i64, + response_headers: vec![ProxyHeader { + name: "Content-Type".into(), + value: content_type.into(), + }], + ..Default::default() + }; + ctx.db.with_tx(|tx| tx.upsert(&flow, &UpdateSource::Background)).unwrap(); + let mut writer = + BodyChunkWriter::new(ctx.blobs.clone(), Flow::body_id(id, FlowBody::Response)); + writer.write(body).unwrap(); + writer.finish().unwrap(); + } + + /// The filter reads the text the viewer shows: in the body's charset, not UTF-8 alone. + #[tokio::test] + async fn filter_reads_the_body_in_its_charset() { + let dir = tempfile::tempdir().unwrap(); + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events).unwrap(); + + let xml = "ZürichKraków"; + let utf16: Vec = xml.encode_utf16().flat_map(u16::to_le_bytes).collect(); + flow_with_response(&ctx, "fl_utf16", "application/xml; charset=utf-16le", &utf16); + let json = r#"{"items":[{"name":"Café"},{"name":"Crème brûlée"}]}"#; + let latin1: Vec = json.chars().map(|c| c as u8).collect(); + flow_with_response(&ctx, "fl_latin1", "application/json; charset=iso-8859-1", &latin1); + flow_with_response(&ctx, "fl_unlabeled", "application/json", &latin1); + + let filter = |flow_id: &str, expression: &str, language| FilterBodyRequest { + flow_id: flow_id.into(), + which: FlowBody::Response, + expression: expression.into(), + language: Some(language), + }; + let xpath = filter("fl_utf16", "//city[2]", FilterLanguage::XPath); + let result = filter_body(ctx.clone(), xpath).await.unwrap(); + assert_eq!((result.content.as_str(), result.error), ("Kraków", None)); + for flow_id in ["fl_latin1", "fl_unlabeled"] { + let names = filter(flow_id, "$.items[*].name", FilterLanguage::JsonPath); + let result = filter_body(ctx.clone(), names).await.unwrap(); + assert_eq!(result.error, None, "{flow_id}"); + assert_eq!(result.content, "[\n \"Café\",\n \"Crème brûlée\"\n]", "{flow_id}"); + } + } + #[tokio::test] async fn the_app_still_hears_why_a_body_failed_to_decode() { let dir = tempfile::tempdir().unwrap(); @@ -1511,9 +1616,25 @@ mod tests { writer.write(b"not a gzip stream").unwrap(); writer.finish().unwrap(); + // The viewer gets the bytes as they were on the wire, and why they didn't decode. let req = GetFlowBodyRequest { flow_id: "fl_bad".into(), which: FlowBody::Response }; - let err = get_flow_body(ctx.clone(), req).await.err().unwrap(); - assert!(err.message.starts_with("Failed to decode"), "{}", err.message); + let body = get_flow_body(ctx.clone(), req).await.unwrap(); + let error = body.decode_error.unwrap(); + assert!(error.starts_with("Failed to decode Gzip body"), "{error}"); + assert_eq!(BASE64.decode(body.data).unwrap(), b"not a gzip stream"); + assert!(!body.truncated); + + // Saving writes those same bytes. + let path = dir.path().join("body.bin"); + let req = SaveFlowBodyRequest { + flow_id: "fl_bad".into(), + which: FlowBody::Response, + path: path.to_string_lossy().into_owned(), + }; + let saved = save_flow_body(ctx.clone(), req).await.unwrap(); + assert_eq!((saved.size, saved.truncated, saved.raw), (17, false, true)); + assert_eq!(std::fs::read(&path).unwrap(), b"not a gzip stream"); + let req = FilterBodyRequest { flow_id: "fl_bad".into(), which: FlowBody::Response, diff --git a/crates/yaak-http/Cargo.toml b/crates/yaak-http/Cargo.toml index b26abd39d..ddd751f8f 100644 --- a/crates/yaak-http/Cargo.toml +++ b/crates/yaak-http/Cargo.toml @@ -5,7 +5,7 @@ edition = "2024" publish = false [dependencies] -async-compression = { version = "0.4", features = ["tokio", "gzip", "deflate", "brotli", "zstd"] } +async-compression = { version = "0.4", features = ["tokio", "gzip", "deflate", "zlib", "brotli", "zstd"] } async-trait = "0.1" brotli = "9" bytes = "1.11.1" diff --git a/crates/yaak-http/src/decompress.rs b/crates/yaak-http/src/decompress.rs index e3764ea6f..80812c961 100644 --- a/crates/yaak-http/src/decompress.rs +++ b/crates/yaak-http/src/decompress.rs @@ -1,11 +1,13 @@ use crate::error::{Error, Result}; use async_compression::tokio::bufread::{ BrotliDecoder, DeflateDecoder as AsyncDeflateDecoder, GzipDecoder, - ZstdDecoder as AsyncZstdDecoder, + ZlibDecoder as AsyncZlibDecoder, ZstdDecoder as AsyncZstdDecoder, }; -use flate2::read::{DeflateDecoder, GzDecoder}; +use flate2::read::{DeflateDecoder, GzDecoder, ZlibDecoder}; use std::io::Read; -use tokio::io::{AsyncBufRead, AsyncRead}; +use std::pin::Pin; +use std::task::{Context, Poll, ready}; +use tokio::io::{AsyncBufRead, AsyncRead, BufReader, ReadBuf}; /// Supported compression encodings #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -66,13 +68,77 @@ fn decompress_gzip(data: &[u8]) -> Result> { Ok(decompressed) } +/// `deflate` names zlib-wrapped data (RFC 1950), but plenty of senders send it raw (RFC 1951). fn decompress_deflate(data: &[u8]) -> Result> { - let mut decoder = DeflateDecoder::new(data); - let mut decompressed = Vec::new(); - decoder - .read_to_end(&mut decompressed) - .map_err(|e| Error::DecompressionError(format!("deflate decompression failed: {}", e)))?; - Ok(decompressed) + let read = |zlib: bool| { + let mut decompressed = Vec::new(); + let result = if zlib { + ZlibDecoder::new(data).read_to_end(&mut decompressed) + } else { + DeflateDecoder::new(data).read_to_end(&mut decompressed) + }; + result.map(|_| decompressed) + }; + match deflate_variant(data) { + DeflateVariant::Zlib => read(true), + DeflateVariant::Raw => read(false), + // Zlib only when it ends whole, its checksum verified, or else raw. + DeflateVariant::Either => inflate_zlib_whole(data).or_else(|_| read(false)), + } + .map_err(|e| Error::DecompressionError(format!("deflate decompression failed: {}", e))) +} + +/// Zlib data decoded through to its end and checksum, failing on any less +fn inflate_zlib_whole(data: &[u8]) -> std::io::Result> { + let mut inflater = flate2::Decompress::new(true); + let mut out = Vec::new(); + loop { + // Always room to write, so a call that makes no progress has run out of input. + out.reserve(64 * 1024); + let (read, written) = (inflater.total_in(), inflater.total_out()); + let rest = &data[usize::try_from(read).unwrap_or(data.len()).min(data.len())..]; + // Not `Finish`, which fails for good when the output outgrows its buffer. + let status = inflater + .decompress_vec(rest, &mut out, flate2::FlushDecompress::None) + .map_err(std::io::Error::other)?; + if status == flate2::Status::StreamEnd { + return Ok(out); + } + if inflater.total_in() == read && inflater.total_out() == written { + return Err(std::io::ErrorKind::UnexpectedEof.into()); + } + } +} + +/// Which `deflate` a body is, from its first five bytes +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DeflateVariant { + Zlib, + Raw, + /// A zlib header that is also the start of a raw stored block: only decoding tells. + Either, +} + +/// Zlib data opens with a header whose first byte names deflate with a window of at most 32 KB, +/// and whose two bytes are a multiple of 31. Raw deflate passes that only when it opens with a +/// stored block that isn't the last, whose length follows in the next two bytes and its +/// complement in the two after. +fn deflate_variant(data: &[u8]) -> DeflateVariant { + let zlib = match data { + [cmf, flg, ..] => { + cmf & 0x0f == 8 && cmf >> 4 <= 7 && (u16::from(*cmf) << 8 | u16::from(*flg)) % 31 == 0 + } + _ => false, + }; + match data { + _ if !zlib => DeflateVariant::Raw, + [_, l0, l1, n0, n1, ..] + if u16::from_le_bytes([*l0, *l1]) != !u16::from_le_bytes([*n0, *n1]) => + { + DeflateVariant::Zlib + } + _ => DeflateVariant::Either, + } } fn decompress_brotli(data: &[u8]) -> Result> { @@ -96,12 +162,262 @@ pub fn streaming_decoder( match encoding { ContentEncoding::Identity => Box::new(reader), ContentEncoding::Gzip => Box::new(GzipDecoder::new(reader)), - ContentEncoding::Deflate => Box::new(AsyncDeflateDecoder::new(reader)), + ContentEncoding::Deflate => Box::new(StreamingDeflateDecoder::new(reader)), ContentEncoding::Brotli => Box::new(BrotliDecoder::new(reader)), ContentEncoding::Zstd => Box::new(AsyncZstdDecoder::new(reader)), } } +/// Decodes `deflate`, zlib-wrapped or raw, as its first five bytes say. When they could be +/// either, it decodes both ways at once until one fails, and goes on with the other. +struct StreamingDeflateDecoder { + state: DeflateState, +} + +enum DeflateState { + Sniffing(Replay), + Zlib(AsyncZlibDecoder>>), + Raw(AsyncDeflateDecoder>>), + Either(EitherInflate>>), + /// Only while switching from sniffing to one of the others + Switching, +} + +impl StreamingDeflateDecoder { + fn new(reader: R) -> Self { + let source = Replay { inner: reader, header: Vec::new(), pos: 0, sniffing: true }; + Self { state: DeflateState::Sniffing(source) } + } +} + +impl AsyncRead for StreamingDeflateDecoder { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + let this = self.get_mut(); + loop { + match &mut this.state { + DeflateState::Sniffing(source) => { + // Read until the header is in hand, or the input ends short of it. + while source.header.len() < 5 { + let mut header = [0u8; 5]; + let mut header = ReadBuf::new(&mut header[source.header.len()..]); + ready!(Pin::new(&mut *source).poll_read(cx, &mut header))?; + if header.filled().is_empty() { + break; + } + } + let DeflateState::Sniffing(mut source) = + std::mem::replace(&mut this.state, DeflateState::Switching) + else { + unreachable!() + }; + source.sniffing = false; + let variant = deflate_variant(&source.header); + let reader = BufReader::new(source); + this.state = match variant { + DeflateVariant::Zlib => DeflateState::Zlib(AsyncZlibDecoder::new(reader)), + DeflateVariant::Raw => DeflateState::Raw(AsyncDeflateDecoder::new(reader)), + DeflateVariant::Either => DeflateState::Either(EitherInflate::new(reader)), + }; + } + DeflateState::Zlib(d) => return Pin::new(d).poll_read(cx, buf), + DeflateState::Raw(d) => return Pin::new(d).poll_read(cx, buf), + DeflateState::Either(d) => return Pin::new(d).poll_read(cx, buf), + DeflateState::Switching => { + return Poll::Ready(Err(std::io::Error::other("deflate decoder is broken"))); + } + } + } + } +} + +/// Reads `inner`, keeping what it read while sniffing, then reads that again before the rest. +struct Replay { + inner: R, + header: Vec, + /// Where reading resumes in `header` once sniffing is over + pos: usize, + sniffing: bool, +} + +impl AsyncRead for Replay { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + let this = self.get_mut(); + if this.sniffing { + let before = buf.filled().len(); + ready!(Pin::new(&mut this.inner).poll_read(cx, buf))?; + this.header.extend_from_slice(&buf.filled()[before..]); + return Poll::Ready(Ok(())); + } + if this.pos < this.header.len() { + let n = buf.remaining().min(this.header.len() - this.pos); + buf.put_slice(&this.header[this.pos..this.pos + n]); + this.pos += n; + return Poll::Ready(Ok(())); + } + Pin::new(&mut this.inner).poll_read(cx, buf) + } +} + +/// Input fed to the readings of an ambiguous deflate body at a time, which deflate grows to at +/// most about a megabyte +const DEFLATE_FEED: usize = 1024; + +/// Output held while both readings of an ambiguous deflate body are still valid, past which it is +/// taken to be zlib, as its header says. A wrong reading fails within a few bytes in practice. +const DEFLATE_HELD_LIMIT: usize = 1024 * 1024; + +/// Both readings of a deflate body that could be zlib or raw, fed the same input. Their output +/// is held back while both are still valid, and the one left streams on: a wrong reading fails +/// within a few bytes in practice, so little is held. If both end, zlib's checksum verified it. +struct EitherInflate { + input: B, + /// Zlib, then raw + readings: [Option; 2], + /// Why the last reading to fail failed + error: Option, +} + +struct Inflate { + inflater: flate2::Decompress, + out: Vec, + /// Where reading resumes in `out` + pos: usize, + ended: bool, +} + +impl EitherInflate { + fn new(input: B) -> Self { + let reading = |zlib| Inflate { + inflater: flate2::Decompress::new(zlib), + out: Vec::new(), + pos: 0, + ended: false, + }; + Self { input, readings: [Some(reading(true)), Some(reading(false))], error: None } + } +} + +impl Inflate { + /// Decodes all of `input` onto `out`, or up to the end of the stream + fn feed(&mut self, input: &[u8]) -> std::io::Result<()> { + let mut consumed = 0; + while !self.ended { + if self.out.capacity() - self.out.len() < 32 * 1024 { + self.out.reserve(64 * 1024); + } + let (read, written) = (self.inflater.total_in(), self.inflater.total_out()); + let status = self + .inflater + .decompress_vec(&input[consumed..], &mut self.out, flate2::FlushDecompress::None) + .map_err(std::io::Error::other)?; + let progress = (self.inflater.total_in() - read, self.inflater.total_out() - written); + consumed += usize::try_from(progress.0).unwrap_or(usize::MAX).min(input.len()); + if status == flate2::Status::StreamEnd { + self.ended = true; + } else if consumed == input.len() && self.out.len() < self.out.capacity() { + // All of it in, and all that it decodes to out + break; + } else if progress == (0, 0) { + return Err(std::io::Error::other("deflate decoding stalled")); + } + } + Ok(()) + } +} + +impl AsyncRead for EitherInflate { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + let this = self.get_mut(); + loop { + let [zlib, raw] = &mut this.readings; + match (zlib.as_mut(), raw.as_mut()) { + (None, None) => { + let error = this.error.take(); + return Poll::Ready(Err( + error.unwrap_or_else(|| std::io::Error::other("deflate decoding failed")) + )); + } + // Both ended: zlib's checksum verified it. + (Some(z), Some(r)) if z.ended && r.ended => *raw = None, + (Some(only), None) | (None, Some(only)) => { + if only.pos < only.out.len() { + let n = buf.remaining().min(only.out.len() - only.pos); + buf.put_slice(&only.out[only.pos..only.pos + n]); + only.pos += n; + if only.pos == only.out.len() { + only.out.clear(); + only.pos = 0; + } + return Poll::Ready(Ok(())); + } + if only.ended { + return Poll::Ready(Ok(())); + } + ready!(this.feed(cx))?; + } + (Some(_), Some(_)) => ready!(this.feed(cx))?, + } + } + } +} + +impl EitherInflate { + /// Feeds the next input to each reading still going, dropping those that fail on it. Pending + /// until there is input; at its end, a reading that hasn't ended was cut short. + fn feed(&mut self, cx: &mut Context<'_>) -> Poll> { + let chunk = ready!(Pin::new(&mut self.input).poll_fill_buf(cx))?; + // A little input at a time, since deflate can grow a thousandfold. + let len = chunk.len().min(DEFLATE_FEED); + let chunk = &chunk[..len]; + let [zlib, raw] = &mut self.readings; + if let (Some(z), Some(r)) = (zlib.as_ref(), raw.as_ref()) { + let (z_held, r_held) = (z.out.len() - z.pos, r.out.len() - r.pos); + let cut_short = len == 0 && !z.ended && !r.ended; + if cut_short { + // Both were cut short: keep what decoded furthest, so a cut body shows its start. + if r_held > z_held { + *zlib = None + } else { + *raw = None + } + // The one left gives what it decoded, then fails on the end of input. + return Poll::Ready(Ok(())); + } else if z_held.max(r_held) > DEFLATE_HELD_LIMIT { + // Holding no more for a raw body zlib still reads validly: it is taken to be zlib. + *raw = None; + } + } + for slot in &mut self.readings { + let Some(reading) = slot.as_mut().filter(|r| !r.ended) else { + continue; + }; + let result = if len == 0 { + Err(std::io::ErrorKind::UnexpectedEof.into()) + } else { + reading.feed(chunk) + }; + if let Err(e) = result { + *slot = None; + self.error = Some(e); + } + } + Pin::new(&mut self.input).consume(len); + Poll::Ready(Ok(())) + } +} + #[cfg(test)] mod tests { use super::*; @@ -159,6 +475,281 @@ mod tests { assert_eq!(result.decompressed_size, original.len() as u64); } + fn zlib(data: &[u8]) -> Vec { + let mut encoder = flate2::write::ZlibEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(data).unwrap(); + encoder.finish().unwrap() + } + + fn raw_deflate(data: &[u8]) -> Vec { + let mut encoder = flate2::write::DeflateEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(data).unwrap(); + encoder.finish().unwrap() + } + + /// Raw deflate whose first two bytes also pass as a zlib header: a stored block of 29 bytes + /// (`08 1d`, with `0x081d` a multiple of 31), then an empty final one. + fn raw_deflate_like_zlib(data: &[u8; 29]) -> Vec { + let mut out = vec![0x08, 0x1d, 0x00, 0xe2, 0xff]; + out.extend_from_slice(data); + out.extend_from_slice(&[0x01, 0x00, 0x00, 0xff, 0xff]); + assert_eq!(deflate_variant(&out), DeflateVariant::Either); + out + } + + /// Zlib that opens as `raw_deflate_like_zlib` does: header `08 1d`, then a stored block of + /// 0xffe2 bytes, which `1d 00 e2 ff` also reads as the start of, and a final stored block of + /// `extra` bytes. + fn zlib_like_raw_deflate(extra: u16) -> (Vec, Vec) { + let data: Vec = (0..0xffe2 + u32::from(extra)).map(|i| (i % 251) as u8).collect(); + let mut out = vec![0x08, 0x1d, 0x00, 0xe2, 0xff, 0x1d, 0x00]; + out.extend_from_slice(&data[..0xffe2]); + out.push(0x01); + out.extend_from_slice(&extra.to_le_bytes()); + out.extend_from_slice(&(!extra).to_le_bytes()); + out.extend_from_slice(&data[0xffe2..]); + let (mut a, mut b) = (1u32, 0u32); + for byte in &data { + a = (a + u32::from(*byte)) % 65521; + b = (b + a) % 65521; + } + out.extend_from_slice(&((b << 16) | a).to_be_bytes()); + assert_eq!(deflate_variant(&out), DeflateVariant::Either); + (out, data) + } + + async fn stream_decode(data: Vec, encoding: ContentEncoding) -> std::io::Result> { + use tokio::io::AsyncReadExt; + let mut out = Vec::new(); + // One byte at a time, to read the header across polls as a slow connection would. + let reader = BufReader::with_capacity(1, std::io::Cursor::new(data)); + streaming_decoder(reader, encoding).read_to_end(&mut out).await?; + Ok(out) + } + + /// What a body decodes to before it fails, and why it failed + async fn stream_decode_until_error(data: Vec) -> (Vec, Option) { + use tokio::io::AsyncReadExt; + let mut out = Vec::new(); + let mut decoder = + streaming_decoder(BufReader::new(std::io::Cursor::new(data)), ContentEncoding::Deflate); + let mut buf = [0u8; 4096]; + loop { + match decoder.read(&mut buf).await { + Ok(0) => return (out, None), + Ok(n) => out.extend_from_slice(&buf[..n]), + Err(e) => return (out, Some(e)), + } + } + } + + #[tokio::test] + async fn streaming_deflate_keeps_what_decoded_of_a_cut_ambiguous_body() { + // Bytes both readings still accept when the input ends: zlib in a stored block too + let text = *b"\x1d\0xxxxxxxxxxxxxxxxxxxxxxxxxxx"; + let mut cut = raw_deflate_like_zlib(&text); + // Without its final block, as a body cut at the capture limit is + cut.truncate(cut.len() - 5); + let (decoded, error) = stream_decode_until_error(cut).await; + assert_eq!(decoded, text); + assert_eq!(error.map(|e| e.kind()), Some(std::io::ErrorKind::UnexpectedEof)); + } + + /// Zlib of `megabytes` of zeros whose raw reading is a stored block of 29 bytes and an empty + /// final one, both inside zlib's first stored block: raw ends at once, zlib goes on. + fn zlib_whose_raw_reading_ends_early(megabytes: usize) -> (Vec, Vec) { + let mut stored = vec![b'z'; 0xffe2]; + stored[27..32].copy_from_slice(&[0x01, 0x00, 0x00, 0xff, 0xff]); + let mut expected = stored.clone(); + expected.resize(stored.len() + megabytes * 1024 * 1024, 0); + let mut data = vec![0x08, 0x1d, 0x00, 0xe2, 0xff, 0x1d, 0x00]; + data.extend_from_slice(&stored); + data.extend_from_slice(&raw_deflate(&expected[stored.len()..])); + let (mut a, mut b) = (1u32, 0u32); + for byte in &expected { + a = (a + u32::from(*byte)) % 65521; + b = (b + a) % 65521; + } + data.extend_from_slice(&((b << 16) | a).to_be_bytes()); + assert_eq!(deflate_variant(&data), DeflateVariant::Either); + (data, expected) + } + + #[tokio::test] + async fn streaming_deflate_holds_little_while_both_readings_go() { + use tokio::io::AsyncReadExt; + let (data, expected) = zlib_whose_raw_reading_ends_early(8); + let mut decoder = + StreamingDeflateDecoder::new(BufReader::new(std::io::Cursor::new(data.clone()))); + let mut first = [0u8; 1024]; + decoder.read_exact(&mut first).await.unwrap(); + assert_eq!(first[..], expected[..1024]); + let DeflateState::Either(either) = &decoder.state else { + panic!("not read both ways") + }; + // The limit, what one feed of input adds past it (deflate grows at most about 1,032 + // times), and the raw reading's 64 KB: far short of the 8 MB body. + let held: usize = either.readings.iter().flatten().map(|r| r.out.len() - r.pos).sum(); + let bound = 3 * 1024 * 1024; + assert!(held <= bound, "held {held} bytes for a 1 KB read"); + + let decoded = stream_decode(data, ContentEncoding::Deflate).await.unwrap(); + assert!(decoded == expected); + } + + const DEFLATE_TEXT: &[u8] = b"{\"service\":\"browser\",\"message\":\"hello deflate\"}"; + + #[test] + fn test_decompress_deflate_zlib_wrapped() { + let result = decompress(zlib(DEFLATE_TEXT), ContentEncoding::Deflate).unwrap(); + assert_eq!(result.data, DEFLATE_TEXT); + let like_zlib = raw_deflate_like_zlib(b"twenty-nine bytes of raw text"); + let result = decompress(like_zlib, ContentEncoding::Deflate).unwrap(); + assert_eq!(result.data, b"twenty-nine bytes of raw text"); + let both = *b"\x1d\0xxxxxxxxxxxxxxxxxxxxxxxxxxx"; + let result = decompress(raw_deflate_like_zlib(&both), ContentEncoding::Deflate).unwrap(); + assert_eq!(result.data, both); + for extra in [0, 10_000] { + let (zlib_like_raw, data) = zlib_like_raw_deflate(extra); + let result = decompress(zlib_like_raw, ContentEncoding::Deflate).unwrap(); + assert!(result.data == data, "{extra} extra bytes"); + } + let (data, expected) = large_raw_like_zlib(); + assert!(decompress(data, ContentEncoding::Deflate).unwrap().data == expected); + } + + /// Ambiguous raw deflate of more than 64 KB: a stored block that reads as zlib's header and + /// the start of a stored block of 0xffe2 bytes, then incompressible data, which zlib reads + /// as a block header 7 + 0xffe2 bytes in and fails on. + fn large_raw_like_zlib() -> (Vec, Vec) { + let mut expected = b"\x1d\0xxxxxxxxxxxxxxxxxxxxxxxxxxx".to_vec(); + let mut seed = 1u32; + expected.extend((0..70_000).map(|_| { + seed = seed.wrapping_mul(1_103_515_245).wrapping_add(12_345); + (seed >> 16) as u8 + })); + let mut data = vec![0x08, 0x1d, 0x00, 0xe2, 0xff]; + data.extend_from_slice(&expected[..29]); + data.extend_from_slice(&raw_deflate(&expected[29..])); + assert!(data.len() > 64 * 1024); + (data, expected) + } + + /// Greptile's case on #812: raw deflate that zlib reads validly past 64 KB. After the first + /// stored block, a final stored block of 65,535 bytes, in which zlib finds a valid stored + /// block header at 65,513 (`00 28 00 d7 ff`) and an invalid block type only at 65,558. + fn raw_valid_as_zlib_past_64k() -> (Vec, Vec) { + let mut block: Vec = (0..65_535u32).map(|i| b'a' + (i % 26) as u8).collect(); + let first = 5 + 29 + 5; + block[65_513 - first..65_518 - first].copy_from_slice(&[0x00, 0x28, 0x00, 0xd7, 0xff]); + block[65_558 - first] = 0x07; + let mut expected = b"\x1d\0xxxxxxxxxxxxxxxxxxxxxxxxxxx".to_vec(); + let mut data = raw_deflate_like_zlib(&expected.clone().try_into().unwrap()); + // Make the first block not the last, and follow it with the large final one. + data.truncate(data.len() - 5); + data.extend_from_slice(&[0x01, 0xff, 0xff, 0x00, 0x00]); + data.extend_from_slice(&block); + expected.extend_from_slice(&block); + (data, expected) + } + + #[test] + fn test_decompress_deflate_raw_valid_as_zlib_past_64k() { + let (data, expected) = raw_valid_as_zlib_past_64k(); + // Raw alone decodes it; zlib alone gets past 64 KB before failing. + let mut raw = Vec::new(); + DeflateDecoder::new(data.as_slice()).read_to_end(&mut raw).unwrap(); + assert!(raw == expected); + let mut zlib = flate2::Decompress::new(true); + let mut out = Vec::with_capacity(200 * 1024); + let failed = zlib.decompress_vec(&data, &mut out, flate2::FlushDecompress::None); + assert!(failed.is_err() && zlib.total_in() > 65_536, "zlib read {}", zlib.total_in()); + assert!(decompress(data, ContentEncoding::Deflate).unwrap().data == expected); + } + + #[tokio::test] + async fn streaming_deflate_decodes_raw_valid_as_zlib_past_64k() { + use tokio::io::AsyncReadExt; + let (data, expected) = raw_valid_as_zlib_past_64k(); + for capacity in [1, 8 * 1024, 256 * 1024] { + let mut out = Vec::new(); + let reader = BufReader::with_capacity(capacity, std::io::Cursor::new(data.clone())); + streaming_decoder(reader, ContentEncoding::Deflate) + .read_to_end(&mut out) + .await + .unwrap(); + assert!(out == expected, "read {capacity} bytes at a time"); + } + } + + #[tokio::test] + async fn streaming_deflate_decodes_zlib_and_raw() { + let zlib_wrapped = stream_decode(zlib(DEFLATE_TEXT), ContentEncoding::Deflate).await; + assert_eq!(zlib_wrapped.unwrap(), DEFLATE_TEXT); + let raw = stream_decode(raw_deflate(DEFLATE_TEXT), ContentEncoding::Deflate).await; + assert_eq!(raw.unwrap(), DEFLATE_TEXT); + let large: Vec = (0..200_000u32).flat_map(|i| (i % 251).to_le_bytes()).collect(); + let large_zlib = stream_decode(zlib(&large), ContentEncoding::Deflate).await; + assert_eq!(large_zlib.unwrap(), large); + let large_raw = stream_decode(raw_deflate(&large), ContentEncoding::Deflate).await; + assert_eq!(large_raw.unwrap(), large); + } + + #[tokio::test] + async fn streaming_deflate_retries_raw_that_looks_like_zlib() { + let data = raw_deflate_like_zlib(b"twenty-nine bytes of raw text"); + // A zlib decoder alone rejects it. + assert!(ZlibDecoder::new(data.as_slice()).read_to_end(&mut Vec::new()).is_err()); + let decoded = stream_decode(data, ContentEncoding::Deflate).await.unwrap(); + assert_eq!(decoded, b"twenty-nine bytes of raw text"); + + // Read as zlib, this one is a stored block that starts decoding, and only fails later. + let both = *b"\x1d\0xxxxxxxxxxxxxxxxxxxxxxxxxxx"; + let data = raw_deflate_like_zlib(&both); + let decoded = stream_decode(data, ContentEncoding::Deflate).await.unwrap(); + assert_eq!(decoded, both); + } + + #[tokio::test] + async fn streaming_deflate_decodes_zlib_that_looks_like_raw() { + for extra in [0, 10_000] { + let (data, expected) = zlib_like_raw_deflate(extra); + let decoded = stream_decode(data, ContentEncoding::Deflate).await.unwrap(); + assert!(decoded == expected, "{extra} extra bytes"); + } + } + + /// However the reader reads ahead, the wrong reading is dropped and the right one kept. + #[tokio::test] + async fn streaming_deflate_retries_large_raw_whatever_the_chunking() { + use tokio::io::AsyncReadExt; + let (data, expected) = large_raw_like_zlib(); + for capacity in [1, 8 * 1024, 256 * 1024] { + let mut out = Vec::new(); + let reader = BufReader::with_capacity(capacity, std::io::Cursor::new(data.clone())); + streaming_decoder(reader, ContentEncoding::Deflate) + .read_to_end(&mut out) + .await + .unwrap(); + assert!(out == expected, "read {capacity} bytes at a time"); + } + } + + #[tokio::test] + async fn streaming_deflate_still_fails_on_garbage() { + let garbage = b"\x78\x9c this is not deflate in any form".to_vec(); + assert!(stream_decode(garbage, ContentEncoding::Deflate).await.is_err()); + assert!(stream_decode(b"\xff".to_vec(), ContentEncoding::Deflate).await.is_err()); + } + + #[tokio::test] + async fn streaming_gzip_is_unchanged() { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(DEFLATE_TEXT).unwrap(); + let gzip = encoder.finish().unwrap(); + assert_eq!(stream_decode(gzip, ContentEncoding::Gzip).await.unwrap(), DEFLATE_TEXT); + } + #[test] fn test_decompress_brotli() { // Compress some data with brotli diff --git a/crates/yaak-proxy/tests/proxy.rs b/crates/yaak-proxy/tests/proxy.rs index 2e4062ba2..9e911d16f 100644 --- a/crates/yaak-proxy/tests/proxy.rs +++ b/crates/yaak-proxy/tests/proxy.rs @@ -100,6 +100,11 @@ async fn route(req: Request) -> Result, Infallible> req.headers().get(HOST).map_or("none", |h| h.to_str().unwrap()), ))), "/big" => Response::new(full(vec![b'x'; BIG_BODY])), + // No body: `Content-Length: 0`, or a stream with nothing in it, chunked on HTTP/1.1. + "/empty" => Response::new(full("")), + "/empty-stream" => { + Response::new(BodyExt::boxed(StreamBody::new(futures_util::stream::empty()))) + } "/late-end" => late_end(5), // The stream ends with half the promised bytes, so the upstream resets it mid-body. "/cut" => late_end(10), @@ -1361,6 +1366,64 @@ async fn h2_large_bodies_pass_flow_control() { } } +/// A flow ends when its response does, not when a connection closes: the upstream answers at +/// once and keeps its connection open, as does the client, and the flow's time is the answer's. +#[tokio::test(flavor = "multi_thread")] +async fn keep_alive_connections_do_not_lengthen_flows() { + let h = Harness::start(1024); + let upstreams = [ + ("plain HTTP/1.1", spawn_upstream(None).await, false), + ("HTTP/1.1", spawn_upstream(Some(h1_only(&h.pki.server))).await, true), + ("HTTP/2", spawn_upstream(Some(h.pki.server.clone())).await, true), + ]; + // Every connection stays open to the end of the test, each reused across requests. + let h1_client = h.client(); + let mut h2_clients = Vec::new(); + + for (upstream_name, upstream, tls) in upstreams { + let port = upstream.port(); + let mut h2_client = if tls { Some(h.h2_client(port).await) } else { None }; + for path in ["/empty", "/empty-stream", "/hello"] { + for client_version in ["HTTP/1.1", "HTTP/2"] { + let case = format!("{client_version} client, {upstream_name} upstream, {path}"); + let scheme = if tls { "https" } else { "http" }; + let url = format!("{scheme}://localhost:{port}{path}"); + let body = match (client_version, &mut h2_client) { + ("HTTP/1.1", _) => { + let resp = h1_client.post(&url).body("x=1").send().await.unwrap(); + resp.bytes().await.unwrap() + } + (_, Some(client)) => { + let req = h2_request("POST", port, path, full("x=1")); + let resp = client.send_request(req).await.unwrap(); + resp.into_body().collect().await.unwrap().to_bytes() + } + // HTTP/2 needs TLS here. + (_, None) => continue, + }; + assert_eq!(body, if path == "/hello" { "hello" } else { "" }, "{case}"); + + let events = h.flow_events(); + let version = + find!(events, ProxyEvent::RequestStart { http_version, .. } => http_version); + assert_eq!(version, client_version, "{case}"); + let headers_ms = + find!(events, ProxyEvent::ResponseStart { elapsed_ms, .. } => *elapsed_ms); + let (end_ms, error, cancelled) = find!(events, + ProxyEvent::ResponseBodyComplete { elapsed_ms, error, cancelled, .. } => + (*elapsed_ms, error.clone(), *cancelled)); + assert_eq!((error, cancelled), (None, false), "{case}"); + // Generous for a slow CI runner; a flow held to the connection's close would + // not end at all while every connection here stays open. + assert!(end_ms - headers_ms < 2_000, "{case}: headers {headers_ms}, end {end_ms}"); + assert!(end_ms < 5_000, "{case}: took {end_ms} ms"); + } + } + h2_clients.extend(h2_client); + } + drop((h1_client, h2_clients)); +} + /// With HTTP/2 off, a client that offers both gets HTTP/1.1, from the next connection on. #[tokio::test(flavor = "multi_thread")] async fn http1_only_makes_h2_clients_fall_back() { diff --git a/package-lock.json b/package-lock.json index d21746759..ed8af433b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -145,7 +145,6 @@ "focus-trap-react": "^12.0.3", "fuzzbunny": "^1.0.1", "graphql": "^17.0.2", - "hexy": "^0.4.0", "history": "^5.3.0", "jotai": "^2.18.0", "jotai-family": "^1.0.1", @@ -15019,6 +15018,7 @@ "name": "@yaakapp-internal/ui", "version": "1.0.0", "dependencies": { + "hexy": "^0.4.0", "jotai-family": "^1.0.1" } }, diff --git a/packages/common-lib/decodeBodyText.test.ts b/packages/common-lib/decodeBodyText.test.ts new file mode 100644 index 000000000..60d2098fd --- /dev/null +++ b/packages/common-lib/decodeBodyText.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, test } from "vitest"; +import { contentTypeCharset, decodeBodyText, strictUtf8Text } from "./decodeBodyText"; + +const latin1 = (text: string) => Uint8Array.from(text, (c) => c.charCodeAt(0)); +// "こんにちは、世界" in Shift_JIS +const shiftJis = new Uint8Array([ + 0x82, 0xb1, 0x82, 0xf1, 0x82, 0xc9, 0x82, 0xbf, 0x82, 0xcd, 0x81, 0x41, 0x90, 0xa2, 0x8a, 0x45, +]); +const french = "Accès refusé : vous n'avez pas l'autorisation d'accéder à cette page."; + +describe("decodeBodyText", () => { + test("decodes in the charset the content type names", () => { + const body = latin1(french); + expect(decodeBodyText(body, "text/html; charset=iso-8859-1")).toBe(french); + expect(decodeBodyText(body, 'text/plain; CHARSET="ISO-8859-1"')).toBe(french); + expect(decodeBodyText(shiftJis, "text/plain; charset=Shift_JIS")).toBe("こんにちは、世界"); + }); + + test("reads an unlabeled body that isn't UTF-8 as windows-1252", () => { + expect(decodeBodyText(latin1(french), "text/html")).toBe(french); + expect(decodeBodyText(latin1(french), null)).toBe(french); + expect(decodeBodyText(new Uint8Array([0x93, 0x68, 0x69, 0x94]), null)).toBe("“hi”"); + }); + + test("reads UTF-8 as UTF-8, labeled or not", () => { + const utf8 = new TextEncoder().encode(`{"city":"Zürich","greeting":"こんにちは"}`); + const text = `{"city":"Zürich","greeting":"こんにちは"}`; + expect(decodeBodyText(utf8, null)).toBe(text); + expect(decodeBodyText(utf8, "application/json")).toBe(text); + expect(decodeBodyText(utf8, "application/json; charset=utf-8")).toBe(text); + // A body labeled UTF-8 stays UTF-8, with what isn't replaced, as before + expect(decodeBodyText(new Uint8Array([0x61, 0xff]), "text/plain; charset=utf-8")).toBe("a�"); + }); + + test("treats a charset TextDecoder doesn't know as no charset", () => { + expect(contentTypeCharset("text/plain; charset=not-a-charset")).toBeNull(); + expect(decodeBodyText(latin1(french), "text/plain; charset=not-a-charset")).toBe(french); + }); + + test("keeps a truncated UTF-8 body UTF-8 when it ends partway through a character", () => { + const cut = new TextEncoder().encode("abc€").subarray(0, 5); + expect(strictUtf8Text(cut)).toBeNull(); + expect(decodeBodyText(cut, null, { truncated: true })).toBe("abc"); + }); +}); diff --git a/packages/common-lib/decodeBodyText.ts b/packages/common-lib/decodeBodyText.ts new file mode 100644 index 000000000..dd6645b37 --- /dev/null +++ b/packages/common-lib/decodeBodyText.ts @@ -0,0 +1,44 @@ +/** + * The charset a content type names, as `TextDecoder` calls it (`windows-1252` for + * `iso-8859-1`), or null when it names none or one `TextDecoder` doesn't know. + */ +export function contentTypeCharset(contentType: string | null): string | null { + const label = contentType?.match(/;\s*charset\s*=\s*"?([^";\s]+)"?/i)?.[1]; + if (label == null) return null; + try { + return new TextDecoder(label).encoding; + } catch { + return null; + } +} + +/** + * The bytes as UTF-8, or null when they aren't valid UTF-8. A truncated body may end partway + * through a character, so up to three trailing bytes are dropped before giving up on it. + */ +export function strictUtf8Text(bytes: Uint8Array, truncated = false): string | null { + const decoder = new TextDecoder("utf-8", { fatal: true }); + const maxTrim = truncated ? Math.min(3, bytes.length) : 0; + for (let trim = 0; trim <= maxTrim; trim++) { + try { + return decoder.decode(bytes.subarray(0, bytes.length - trim)); + } catch { + // Try again without the last byte + } + } + return null; +} + +/** + * A body's text in the charset its content type names. Without one it is UTF-8, unless the + * bytes aren't valid UTF-8, when it is windows-1252, as browsers read unlabeled text. + */ +export function decodeBodyText( + bytes: Uint8Array, + contentType: string | null, + { truncated = false }: { truncated?: boolean } = {}, +): string { + const charset = contentTypeCharset(contentType); + if (charset != null) return new TextDecoder(charset).decode(bytes); + return strictUtf8Text(bytes, truncated) ?? new TextDecoder("windows-1252").decode(bytes); +} diff --git a/packages/common-lib/index.ts b/packages/common-lib/index.ts index 245e43a46..999fbae9b 100644 --- a/packages/common-lib/index.ts +++ b/packages/common-lib/index.ts @@ -1,4 +1,5 @@ export * from "./debounce"; +export * from "./decodeBodyText"; export * from "./eagerDebounceAsync"; export * from "./formatSize"; export * from "./templateFunction"; diff --git a/packages/ui/package.json b/packages/ui/package.json index 8ed94e9cd..849778086 100644 --- a/packages/ui/package.json +++ b/packages/ui/package.json @@ -16,6 +16,7 @@ "*.css" ], "dependencies": { + "hexy": "^0.4.0", "jotai-family": "^1.0.1" } } diff --git a/packages/ui/src/components/viewers/HexdumpViewer.test.ts b/packages/ui/src/components/viewers/HexdumpViewer.test.ts new file mode 100644 index 000000000..209bd49f7 --- /dev/null +++ b/packages/ui/src/components/viewers/HexdumpViewer.test.ts @@ -0,0 +1,20 @@ +import { hexy } from "hexy"; +import { describe, expect, test, vi } from "vite-plus/test"; +import { hexdump } from "./HexdumpViewer"; + +vi.mock("hexy", () => ({ hexy: vi.fn((data: number[]) => `dump of ${data.length} bytes`) })); + +describe("hexdump", () => { + test("makes the hexdump of the same bytes once", () => { + const message = [0x01, 0x02, 0x03]; + expect(hexdump(message)).toBe("dump of 3 bytes"); + expect(hexdump(message)).toBe("dump of 3 bytes"); + expect(hexy).toHaveBeenCalledTimes(1); + + // Other bytes, even equal ones, get their own. + expect(hexdump([0x01, 0x02, 0x03])).toBe("dump of 3 bytes"); + expect(hexy).toHaveBeenCalledTimes(2); + expect(hexdump([])).toBe(""); + expect(hexy).toHaveBeenCalledTimes(2); + }); +}); diff --git a/packages/ui/src/components/viewers/HexdumpViewer.tsx b/packages/ui/src/components/viewers/HexdumpViewer.tsx new file mode 100644 index 000000000..3e3b94303 --- /dev/null +++ b/packages/ui/src/components/viewers/HexdumpViewer.tsx @@ -0,0 +1,50 @@ +import { hexy } from "hexy"; +import type { ReactNode } from "react"; +import { useMemo } from "react"; +import type { CodeEditorProps } from "../editor/CodeEditor"; +import { LazyCodeEditor } from "../editor/LazyCodeEditor"; + +/** Hexdumps already made, by the bytes they were made of, for as long as those bytes live */ +const hexdumps = new WeakMap(); + +/** + * Bytes as a hexdump: offset, hex and ASCII columns, one line per 16 bytes. Made once for the + * same bytes, so a viewer and its copy action share one. + */ +export function hexdump(data: Uint8Array | number[]): string { + if (data.length === 0) return ""; + let dump = hexdumps.get(data); + if (dump == null) { + dump = hexy(data); + hexdumps.set(data, dump); + } + return dump; +} + +export interface HexdumpViewerProps { + data: Uint8Array | number[]; + className?: string; + /** Renders the editor, for a host whose editor builds on CodeEditor */ + renderEditor?: (props: CodeEditorProps) => ReactNode; +} + +/** A read-only hexdump of binary data */ +export function HexdumpViewer({ + data, + className, + renderEditor = renderCodeEditor, +}: HexdumpViewerProps) { + const text = useMemo(() => hexdump(data), [data]); + return renderEditor({ + language: "text", + defaultValue: text, + wrapLines: false, + readOnly: true, + stateKey: null, + className, + }); +} + +function renderCodeEditor(props: CodeEditorProps) { + return ; +} diff --git a/packages/ui/src/components/viewers/TextViewer.tsx b/packages/ui/src/components/viewers/TextViewer.tsx index 6e25defe9..854a15566 100644 --- a/packages/ui/src/components/viewers/TextViewer.tsx +++ b/packages/ui/src/components/viewers/TextViewer.tsx @@ -89,6 +89,8 @@ export function TextViewer({ ], queryFn: () => (filterText ?? rejectFilter)(filter.appliedFilter ?? ""), enabled: hasFilter && filter.appliedFilter != null, + // An expression that fails, fails every time, so the error shows at once. + retry: false, }); const canFilter = hasFilter && (language === "json" || language === "xml" || language === "html"); @@ -195,6 +197,7 @@ export function TextViewer({ } return formatText(text, language); }, + retry: false, }).data; if (formattedBody == null) { return null; diff --git a/packages/ui/src/index.ts b/packages/ui/src/index.ts index 2718478d6..3c46e5edd 100644 --- a/packages/ui/src/index.ts +++ b/packages/ui/src/index.ts @@ -90,6 +90,8 @@ export { Tree } from "./components/tree/Tree"; export type { TreeItemProps } from "./components/tree/TreeItem"; export { AudioViewer } from "./components/viewers/AudioViewer"; export { CsvViewer, CsvViewerInner } from "./components/viewers/CsvViewer"; +export type { HexdumpViewerProps } from "./components/viewers/HexdumpViewer"; +export { hexdump, HexdumpViewer } from "./components/viewers/HexdumpViewer"; export { ImageViewer } from "./components/viewers/ImageViewer"; export { JsonAttributeTree } from "./components/viewers/JsonAttributeTree"; export { JsonViewer } from "./components/viewers/JsonViewer";