Body viewer and decode fixes (#812)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Gregory SchierandClaude Opus 5.5 authored and GitHub committed 2026-11-04 21:21:05 -07:00
1 parent 87a30200f9
commit 4ad0533901
26 files changed
+1633 -91

No files matched your search

Generated
+1
View File
@@ -11852,6 +11852,7 @@ dependencies = [
"base64 0.23.1",
"bytes",
"chrono",
"encoding_rs",
"flate2",
"http-body-util",
"hyper",
@@ -1,13 +1,14 @@
import type { WebsocketEvent, WebsocketRequest } from "@yaakapp-internal/models";
import {
HStack,
hexdump,
HexdumpViewer,
Icon,
LoadingIcon,
VStack,
useStateWithDeps,
languageFromContentType,
} from "@yaakapp-internal/ui";
import { hexy } from "hexy";
import { useAtomValue } from "jotai";
import { useMemo, useState } from "react";
import { useFormatText } from "../hooks/useFormatText";
@@ -19,6 +20,7 @@ import {
} from "../hooks/usePinnedWebsocketConnection";
import { Button } from "./core/Button";
import { Editor } from "./core/Editor/LazyEditor";
import { renderEditor } from "./core/Editor/renderEditor";
import { type EventDetailAction, EventDetailHeader, EventViewer } from "./core/EventViewer";
import { EventViewerRow } from "./core/EventViewerRow";
import { HotkeyList } from "./core/HotkeyList";
@@ -122,10 +124,10 @@ function WebsocketEventRow({
messageType === "error"
? "alert_triangle"
: messageType === "close" || messageType === "open"
? "info"
: isServer
? "arrow_big_down_dash"
: "arrow_big_up_dash";
? "info"
: isServer
? "arrow_big_down_dash"
: "arrow_big_up_dash";
const content =
messageType === "close" ? (
@@ -172,7 +174,7 @@ function WebsocketEventDetail({
}) {
const message = useMemo(() => {
if (hexDump) {
return event.message ? hexy(event.message) : "";
return event.message ? hexdump(event.message) : "";
}
return event.message ? new TextDecoder("utf-8").decode(Uint8Array.from(event.message)) : "";
}, [event.message, hexDump]);
@@ -232,6 +234,8 @@ function WebsocketEventDetail({
</VStack>
) : event.message.length === 0 ? (
<EmptyStateText>No Content</EmptyStateText>
) : hexDump ? (
<HexdumpViewer data={event.message} renderEditor={renderEditor} />
) : (
<Editor
language={language}
+3 -1
View File
@@ -1,5 +1,7 @@
import { useQuery } from "@tanstack/react-query";
import type { HttpResponse } from "@yaakapp-internal/models";
import { decodeBodyText } from "@yaakapp-internal/lib/decodeBodyText";
import { getContentTypeFromHeaders } from "../lib/model_util";
import { rpc } from "../lib/rpc";
export function useHttpRequestBody(response: HttpResponse | null) {
@@ -27,6 +29,6 @@ export async function getRequestBodyText(response: HttpResponse | null) {
}
const body = new Uint8Array(data);
const bodyText = new TextDecoder("utf-8", { fatal: false }).decode(body);
const bodyText = decodeBodyText(body, getContentTypeFromHeaders(response.requestHeaders));
return { body, bodyText };
}
-9
View File
@@ -5,7 +5,6 @@ import type {
HttpResponseEvent,
HttpResponseHeader,
} from "@yaakapp-internal/models";
import { getMimeTypeFromContentType } from "./contentType";
export const BODY_TYPE_NONE = null;
export const BODY_TYPE_GRAPHQL = "graphql";
@@ -46,14 +45,6 @@ export function getContentTypeFromHeaders(headers: HttpResponseHeader[] | null):
return headers?.find((h) => h.name.toLowerCase() === "content-type")?.value ?? null;
}
export function getCharsetFromContentType(headers: HttpResponseHeader[]): string | null {
const contentType = getContentTypeFromHeaders(headers);
if (contentType == null) return null;
const mimeType = getMimeTypeFromContentType(contentType);
return mimeType.parameters.get("charset") ?? null;
}
export function isBaseEnvironment(environment: Environment): boolean {
return environment.parentModel === "workspace";
}
+4 -1
View File
@@ -3,7 +3,9 @@ import type { FilterResponse } from "@yaakapp-internal/plugins";
import type { ServerSentEvent, SseSummary } from "@yaakapp-internal/sse";
import { candidateJsonPayloadsFromSseText, computeSseSummary } from "@yaakapp-internal/sse";
import { rpc } from "./rpc";
import { decodeBodyText } from "@yaakapp-internal/lib/decodeBodyText";
import { platform } from "@yaakapp-internal/platform";
import { getContentTypeFromHeaders } from "./model_util";
/**
* Reading a response body means naming the response, never the file it lives
@@ -74,5 +76,6 @@ export async function getResponseBodyBytes(
async function getResponseBodyDecoded(response: HttpResponse): Promise<string | null> {
const bytes = await getResponseBodyBytes(response);
return bytes == null ? null : new TextDecoder("utf-8").decode(bytes);
if (bytes == null) return null;
return decodeBodyText(bytes, getContentTypeFromHeaders(response.headers));
}
-1
View File
@@ -48,7 +48,6 @@
"focus-trap-react": "^12.0.3",
"fuzzbunny": "^1.0.1",
"graphql": "^17.0.2",
"hexy": "^0.4.0",
"history": "^5.3.0",
"jotai": "^2.18.0",
"jotai-family": "^1.0.1",
+79 -35
View File
@@ -3,7 +3,10 @@ import { formatSize } from "@yaakapp-internal/lib/formatSize";
import type { Flow, FlowBody } from "@yaakapp-internal/proxy-lib";
import {
Banner,
Button,
CsvViewer,
HexdumpViewer,
Icon,
ImageViewer,
languageFromContentType,
LoadingIcon,
@@ -14,16 +17,17 @@ import {
} from "@yaakapp-internal/ui";
import type { CodeEditorProps } from "@yaakapp-internal/ui/editor";
import type { ReactNode } from "react";
import { useCallback, useEffect, useState } from "react";
import { useCallback, useEffect, useMemo, useState } from "react";
import {
base64ToBytes,
bytesToText,
bodyAsText,
bodyImageType,
filterText,
findHeader,
flowBodyQueryOptions,
formatText,
mimeEssence,
} from "../lib/body";
import { rpc } from "../lib/rpc";
import { useSaveFlowBody } from "../lib/flowActions";
interface Props {
flow: Flow;
@@ -32,18 +36,7 @@ interface Props {
}
export function useFlowBody(flow: Flow, which: FlowBody, enabled = true) {
return useQuery({
enabled,
queryKey: ["get_flow_body", flow.id, which, flow.updatedAt],
queryFn: async () => {
const res = await rpc("get_flow_body", { flowId: flow.id, which });
return {
bytes: base64ToBytes(res.data),
truncated: res.truncated,
storedSize: res.storedSize,
};
},
});
return useQuery({ ...flowBodyQueryOptions(flow, which), enabled });
}
export function FlowBodyViewer({ flow, which, pretty }: Props) {
@@ -60,8 +53,8 @@ export function FlowBodyViewer({ flow, which, pretty }: Props) {
return <Banner color="danger">{body.error.message}</Banner>;
}
const { bytes, truncated, storedSize } = body.data;
if (bytes.length === 0) {
const { bytes, truncated, storedSize, decodeError } = body.data;
if (bytes.length === 0 && decodeError == null) {
return <EmptyState>{which === "request" ? "No request body" : "Empty"}</EmptyState>;
}
@@ -70,12 +63,16 @@ export function FlowBodyViewer({ flow, which, pretty }: Props) {
const mime = mimeEssence(contentType) ?? "";
const wireSize = which === "request" ? flow.requestSize : flow.responseSize;
const wireTruncated = which === "request" ? flow.requestTruncated : flow.responseTruncated;
// An image by its bytes or type, else text whatever the content type says, so a page sent with
// an odd one still reads: `viewerText`'s rule.
const imageType = decodeError == null ? bodyImageType(bytes, truncated, contentType) : null;
const text =
decodeError == null && imageType == null ? bodyAsText(bytes, truncated, contentType) : null;
let viewer: ReactNode;
if (mime.startsWith("image/svg")) {
viewer = <SvgViewer text={bytesToText(bytes, truncated) ?? ""} />;
} else if (mime.startsWith("image/")) {
viewer = <ImageBody bytes={bytes} mime={mime} />;
if (decodeError != null) {
// What failed to decode is as it was on the wire, compressed, so never text.
viewer = <BinaryBody flow={flow} which={which} bytes={bytes} />;
} else if (mime.startsWith("multipart/") && pretty) {
const boundary = contentType?.match(/;\s*boundary="?([^";]+)"?/i)?.[1] ?? "unknown";
viewer = (
@@ -86,14 +83,17 @@ export function FlowBodyViewer({ flow, which, pretty }: Props) {
textViewerProps={{ formatText }}
/>
);
} else if (imageType != null) {
viewer = <ImageBody bytes={bytes} mime={imageType} />;
} else if (text == null) {
viewer = <BinaryBody flow={flow} which={which} bytes={bytes} />;
} else if (mime.startsWith("image/svg")) {
viewer = <SvgViewer text={text} />;
} else if (/csv|tab-separated/.test(mime) && pretty) {
viewer = <CsvViewer text={bytesToText(bytes, truncated)} className="pb-2" />;
viewer = <CsvViewer text={text} className="pb-2" />;
} else {
const text = bytesToText(bytes, truncated);
const language = languageFromContentType(contentType, text);
if (text == null) {
viewer = <EmptyState>Binary body, {formatSize(bytes.length)}</EmptyState>;
} else if (language === "html" && pretty && which === "response") {
if (language === "html" && pretty && which === "response") {
viewer = <WebPageViewer html={text} baseUrl={flow.url} />;
} else {
viewer = (
@@ -102,16 +102,60 @@ export function FlowBodyViewer({ flow, which, pretty }: Props) {
}
}
if (!truncated) return viewer;
if (!truncated && decodeError == null) return viewer;
return (
<div className="h-full grid grid-rows-[auto_minmax(0,1fr)] gap-2">
<Banner color="notice" className="text-sm">
{wireTruncated
? `Showing the first ${formatSize(storedSize)} of ${formatSize(wireSize)}, the capture limit. The app received all of it.`
: `Showing the first ${formatSize(bytes.length)} of the decoded body, the capture limit. The app received all of it.`}
</Banner>
<div className="min-h-0">{viewer}</div>
<div className="h-full grid grid-rows-[auto_auto_minmax(0,1fr)]">
{decodeError != null && (
<Banner color="danger" className="text-sm mb-2">
{decodeError}. Showing the body as it was on the wire.
</Banner>
)}
{truncated && (
<Banner color="notice" className="text-sm mb-2">
{wireTruncated || decodeError != null
? `Showing the first ${formatSize(storedSize)} of ${formatSize(wireSize)}, the capture limit. The app received all of it.`
: `Showing the first ${formatSize(bytes.length)} of the decoded body, the capture limit. The app received all of it.`}
</Banner>
)}
<div className="min-h-0 row-start-3">{viewer}</div>
</div>
);
}
/** Bytes shown in the hexdump; the rest are in the saved file */
const HEXDUMP_LIMIT = 256 * 1024;
/** A body that isn't text: a hexdump, and a way to save it */
function BinaryBody({
flow,
which,
bytes,
}: {
flow: Flow;
which: FlowBody;
bytes: Uint8Array<ArrayBuffer>;
}) {
const saveBody = useSaveFlowBody();
const shown = useMemo(() => bytes.subarray(0, HEXDUMP_LIMIT), [bytes]);
return (
<div className="h-full grid grid-rows-[auto_minmax(0,1fr)] gap-1">
<div className="flex items-center justify-between gap-2 text-sm text-text-subtle">
<span>
Binary body, {formatSize(bytes.length)}
{shown.length < bytes.length && `, first ${formatSize(shown.length)} shown`}
</span>
<Button
size="2xs"
variant="border"
leftSlot={<Icon icon="download" size="xs" />}
onClick={() => saveBody(flow, which)}
>
Save to File
</Button>
</div>
<HexdumpViewer data={shown} className="-mr-2 bg-surface" />
</div>
);
}
+209 -2
View File
@@ -1,5 +1,17 @@
import { describe, expect, test } from "vite-plus/test";
import { bytesToText } from "./body";
import { QueryClient, QueryObserver } from "@tanstack/react-query";
import type { Flow } from "@yaakapp-internal/proxy-lib";
import { describe, expect, test, vi } from "vite-plus/test";
import {
bodyAsText,
bodyFileName,
bodyImageType,
bytesToText,
flowBodyQueryOptions,
viewerText,
} from "./body";
import { rpc } from "./rpc";
vi.mock("./rpc", () => ({ rpc: vi.fn() }));
const encode = (text: string) => new TextEncoder().encode(text);
@@ -18,3 +30,198 @@ describe("bytesToText", () => {
expect(bytesToText(cut, true)).toBe("abc");
});
});
describe("bodyAsText", () => {
test("reads UTF-8 as text", () => {
expect(bodyAsText(encode("<h1>Forbidden</h1>\n\tcafé"))).toBe("<h1>Forbidden</h1>\n\tcafé");
});
test("reads unlabeled Latin-1 text as text, as windows-1252", () => {
const page = "<p>Accès refusé : vous n'avez pas l'autorisation d'accéder à cette page.</p>";
const latin1 = Uint8Array.from(page, (c) => c.charCodeAt(0));
expect(bodyAsText(latin1)).toBe(page);
});
test("reads bytes UTF-8 can't read as binary, though windows-1252 reads every one", () => {
let seed = 7;
const random = Uint8Array.from({ length: 600 }, () => {
seed = (seed * 1103515245 + 12345) & 0x7fffffff;
return seed >> 16;
});
expect(bodyAsText(random)).toBeNull();
// All high bytes: letters and symbols in windows-1252, none of them UTF-8
const high = Uint8Array.from({ length: 600 }, (_, i) => 0xa0 + (i % 96));
expect(bodyAsText(high)).toBeNull();
expect(bodyAsText(high, false, "text/plain")).toBeNull();
});
test("reads protobuf and other binary as binary, even when it is valid UTF-8", () => {
// Field tags, lengths and varints: control characters between short strings.
const protobuf = new Uint8Array([
0x0a, 0x04, 0x6b, 0x65, 0x79, 0x31, 0x12, 0x02, 0x08, 0x01, 0x1a, 0x03, 0x10, 0x96, 0x01,
0x22, 0x00, 0x2a, 0x02, 0x18, 0x07,
]);
expect(bodyAsText(protobuf)).toBeNull();
expect(bodyAsText(new Uint8Array([0x01, 0x02, 0x03, 0x04, 0x41, 0x42]))).toBeNull();
expect(bodyAsText(new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))).toBeNull();
});
});
describe("viewerText", () => {
// A PNG signature, then a tEXt chunk of printable metadata far longer than the image
const png = new Uint8Array([
0x89,
...encode("PNG\r\n"),
0x1a,
0x0a,
0x00,
0x00,
0x12,
0xc0,
...encode(`tEXtDescription\0${"A long printable description. ".repeat(160)}`),
]);
test("keeps an image with printable metadata an image", () => {
expect(bodyAsText(png)).not.toBeNull();
expect(viewerText(png, false, "image/png")).toBeNull();
});
test("reads text sent as an image as text, and other types text first", () => {
expect(viewerText(encode("<h1>Forbidden</h1>"), false, "image/png")).toBe("<h1>Forbidden</h1>");
expect(bodyImageType(encode("<h1>Forbidden</h1>"), false, "image/png")).toBeNull();
expect(viewerText(encode("<svg/>"), false, "image/svg+xml")).toBe("<svg/>");
expect(bodyImageType(encode("<svg/>"), false, "image/svg+xml")).toBeNull();
const bmwText = encode("BMW sells cars.\n");
expect(viewerText(bmwText, false, "text/plain")).toBe("BMW sells cars.\n");
});
// A 1×1 black 24-bit BMP: every byte below 0x80, so valid UTF-8
// File header, then the 40-byte DIB header, then one pixel padded to four bytes
// prettier-ignore
const bmp = new Uint8Array([
0x42, 0x4d, 58, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0,
40, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 24, 0, 0, 0, 0, 0, 4, 0, 0, 0,
0x13, 0x0b, 0, 0, 0x13, 0x0b, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0,
]);
test("keeps an image that is valid UTF-8 an image", () => {
expect(bytesToText(bmp)).not.toBeNull();
expect(bodyImageType(bmp, false, "image/bmp")).toBe("image/bmp");
expect(viewerText(bmp, false, "image/bmp")).toBeNull();
});
test("previews an image by its signature whatever its content type", () => {
expect(bodyImageType(bmp, false, "application/octet-stream")).toBe("image/bmp");
expect(bodyImageType(png, false, "application/octet-stream")).toBe("image/png");
expect(bodyImageType(png, false, null)).toBe("image/png");
expect(viewerText(png, false, "application/octet-stream")).toBeNull();
const gif = encode("GIF89a\x01\0\x01\0");
expect(bodyImageType(gif, false, "text/plain")).toBe("image/gif");
});
test("previews formats not every webview draws only when their content type says so", () => {
const tiff = new Uint8Array([0x49, 0x49, 0x2a, 0x00, 0x08, 0x00, 0x00, 0x00, 0xff, 0x80]);
const heic = new Uint8Array([0, 0, 0, 0x18, ...encode("ftypheic"), 0, 0, 0, 0, 0xff, 0x80]);
for (const image of [tiff, heic]) {
expect(bodyImageType(image, false, "application/octet-stream")).toBeNull();
expect(viewerText(image, false, "application/octet-stream")).toBeNull();
}
expect(bodyImageType(tiff, false, "image/tiff")).toBe("image/tiff");
expect(bodyImageType(heic, false, "image/heic")).toBe("image/heic");
});
test("reads text in the charset its content type names", () => {
const page = "Accès refusé : vous n'avez pas l'autorisation d'accéder à cette page.";
const latin1 = Uint8Array.from(page, (c) => c.charCodeAt(0));
expect(viewerText(latin1, false, "text/html; charset=iso-8859-1")).toBe(page);
// "こんにちは、世界" in Shift_JIS
const shiftJis = new Uint8Array([
0x82, 0xb1, 0x82, 0xf1, 0x82, 0xc9, 0x82, 0xbf, 0x82, 0xcd, 0x81, 0x41, 0x90, 0xa2, 0x8a,
0x45,
]);
expect(viewerText(shiftJis, false, "text/plain; charset=shift_jis")).toBe("こんにちは、世界");
});
test("reads unlabeled Latin-1 as windows-1252, and UTF-8 as UTF-8", () => {
const page = "Accès refusé : vous n'avez pas l'autorisation d'accéder à cette page.";
const latin1 = Uint8Array.from(page, (c) => c.charCodeAt(0));
expect(viewerText(latin1, false, "application/octet-stream")).toBe(page);
expect(viewerText(encode(page), false, "application/octet-stream")).toBe(page);
expect(viewerText(encode(page), false, "text/plain; charset=utf-8")).toBe(page);
});
});
const flow = (url: string, contentType: string | null) =>
({
id: "fl_1",
url,
updatedAt: "2026-11-04T00:00:00",
requestHeaders: contentType == null ? [] : [{ name: "Content-Type", value: contentType }],
responseHeaders: contentType == null ? [] : [{ name: "content-type", value: contentType }],
}) as unknown as Flow;
describe("bodyFileName", () => {
test("uses the URL's file name for a response", () => {
expect(bodyFileName(flow("https://a.test/img/logo%20big.png?v=2", null), "response")).toBe(
"logo big.png",
);
});
test("names a body from its content type otherwise", () => {
const protobuf = flow("https://a.test/v1:GetModels", "application/x-protobuf");
expect(bodyFileName(protobuf, "request")).toBe("request.protobuf");
expect(bodyFileName(protobuf, "response")).toBe("response.protobuf");
expect(bodyFileName(flow("https://a.test/", "text/plain; charset=utf-8"), "response")).toBe(
"response.txt",
);
expect(bodyFileName(flow("https://a.test/api", "application/problem+json"), "response")).toBe(
"response.json",
);
expect(bodyFileName(flow("https://a.test/data.json", null), "request")).toBe("request.bin");
expect(bodyFileName(flow("not a url", "application/octet-stream"), "response")).toBe(
"response.bin",
);
});
});
describe("flowBodyQueryOptions", () => {
test("shows a body that fails to read at once, without retrying", async () => {
vi.mocked(rpc).mockRejectedValue(new Error("No flow fl_1"));
// React Query retries three times in a browser, and not at all where there is no window.
const client = new QueryClient({ defaultOptions: { queries: { retry: 3 } } });
const observer = new QueryObserver(
client,
flowBodyQueryOptions(flow("https://a.test/", null), "response"),
);
const failed = new Promise<string>((resolve) => {
const unsubscribe = observer.subscribe((result) => {
if (result.status === "error") {
unsubscribe();
resolve(result.error.message);
}
});
});
// React Query's default retries would keep it pending for seconds.
const timeout = new Promise<string>((resolve) => setTimeout(() => resolve("pending"), 500));
expect(await Promise.race([failed, timeout])).toBe("No flow fl_1");
expect(rpc).toHaveBeenCalledTimes(1);
client.clear();
});
test("passes on why a body failed to decode, with its bytes as they were", async () => {
vi.mocked(rpc).mockResolvedValue({
data: btoa("\x78\x9c\x01"),
truncated: false,
storedSize: 3,
decodeError: "Failed to decode Deflate body: deflate decompression error",
});
const client = new QueryClient();
const body = await client.fetchQuery(
flowBodyQueryOptions(flow("https://a.test/", null), "request"),
);
expect(Array.from(body.bytes)).toEqual([0x78, 0x9c, 0x01]);
expect(body.decodeError).toBe("Failed to decode Deflate body: deflate decompression error");
client.clear();
});
});
+146 -10
View File
@@ -1,8 +1,32 @@
import type { FilterLanguage, FlowBody } from "@yaakapp-internal/proxy-lib";
import { queryOptions } from "@tanstack/react-query";
import {
contentTypeCharset,
decodeBodyText,
strictUtf8Text,
} from "@yaakapp-internal/lib/decodeBodyText";
import type { FilterLanguage, Flow, FlowBody } from "@yaakapp-internal/proxy-lib";
import type { CodeEditorProps } from "@yaakapp-internal/ui/editor";
import vkBeautify from "vkbeautify";
import { rpc } from "./rpc";
/** A flow's body for the viewer: decoded, or as it was on the wire when `decodeError` is set */
export function flowBodyQueryOptions(flow: Flow, which: FlowBody) {
return queryOptions({
queryKey: ["get_flow_body", flow.id, which, flow.updatedAt],
queryFn: async () => {
const res = await rpc("get_flow_body", { flowId: flow.id, which });
return {
bytes: base64ToBytes(res.data),
truncated: res.truncated,
storedSize: res.storedSize,
decodeError: res.decodeError ?? null,
};
},
// Reading a stored body fails the same way every time, so the error shows at once.
retry: false,
});
}
export function base64ToBytes(data: string): Uint8Array<ArrayBuffer> {
const binary = atob(data);
const bytes = new Uint8Array(binary.length);
@@ -16,19 +40,104 @@ export function base64ToBytes(data: string): Uint8Array<ArrayBuffer> {
* The text of a body, or null when it isn't valid UTF-8. A truncated body may end partway
* through a character, so up to three trailing bytes are dropped before giving up on it.
*/
export function bytesToText(bytes: Uint8Array, truncated = false): string | null {
const decoder = new TextDecoder("utf-8", { fatal: true });
const maxTrim = truncated ? Math.min(3, bytes.length) : 0;
for (let trim = 0; trim <= maxTrim; trim++) {
try {
return decoder.decode(bytes.subarray(0, bytes.length - trim));
} catch {
// Try again without the last byte
}
export const bytesToText = strictUtf8Text;
/** Characters looked at to tell text from binary */
const TEXT_SAMPLE = 64 * 1024;
/** The share of those that may be control characters or undecodable bytes in text */
const MAX_UNPRINTABLE = 0.1;
/**
* The body as text, whatever its content type says, or null when it is binary. It is decoded in
* its content type's charset, or else as UTF-8, or windows-1252 when it isn't valid UTF-8. It is
* binary when it is full of control characters, as protobuf is, or, without a charset, of bytes
* UTF-8 can't read: windows-1252 reads every byte as something, compressed data included.
*/
export function bodyAsText(
bytes: Uint8Array,
truncated = false,
contentType: string | null = null,
): string | null {
const text = decodeBodyText(bytes, contentType, { truncated });
const judged =
contentTypeCharset(contentType) != null
? text
: (strictUtf8Text(bytes, truncated) ?? new TextDecoder("utf-8").decode(bytes));
let total = 0;
let unprintable = 0;
for (const char of judged) {
if (total === TEXT_SAMPLE) break;
total++;
const code = char.codePointAt(0) ?? 0;
const control = code < 0x20 && code !== 0x09 && code !== 0x0a && code !== 0x0d;
if (control || code === 0x7f || code === 0xfffd) unprintable++;
}
return unprintable <= total * MAX_UNPRINTABLE ? text : null;
}
const ascii = (bytes: Uint8Array, start: number, text: string) =>
Array.from(text).every((c, i) => bytes[start + i] === c.charCodeAt(0));
/**
* The image type a body's first bytes name, or null when they name none. Only types every
* platform's webview draws, so a body sniffed as an image always previews: TIFF, HEIC and AVIF
* preview only when their content type says so.
*/
export function imageSignature(bytes: Uint8Array): string | null {
const b = bytes;
if (b.length < 4) return null;
if (b[0] === 0x89 && ascii(b, 1, "PNG\r\n\x1a\n")) return "image/png";
if (b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff) return "image/jpeg";
if (ascii(b, 0, "GIF87a") || ascii(b, 0, "GIF89a")) return "image/gif";
if (ascii(b, 0, "RIFF") && ascii(b, 8, "WEBP")) return "image/webp";
if (ascii(b, 0, "\0\0\x01\0") && b.length >= 6 && (b[4] ?? 0) + (b[5] ?? 0) > 0) {
return "image/x-icon";
}
// "BM" alone starts plenty of text, so the reserved bytes and DIB header size must fit too.
const dibSize = (b[14] ?? 0) | ((b[15] ?? 0) << 8) | ((b[16] ?? 0) << 16) | ((b[17] ?? 0) << 24);
if (
ascii(b, 0, "BM") &&
ascii(b, 6, "\0\0\0\0") &&
[12, 40, 52, 56, 64, 108, 124].includes(dibSize)
) {
return "image/bmp";
}
return null;
}
/**
* The image type to preview a body as, or null when it isn't one: what its first bytes name,
* whatever its content type, or else an `image/*` content type other than SVG, unless the body
* is genuinely text, as an error page sent with the image's type is.
*/
export function bodyImageType(
bytes: Uint8Array,
truncated: boolean,
contentType: string | null,
): string | null {
const signed = imageSignature(bytes);
if (signed != null) return signed;
const mime = mimeEssence(contentType) ?? "";
if (!mime.startsWith("image/") || mime.startsWith("image/svg")) return null;
const genuineText =
strictUtf8Text(bytes, truncated) != null && bodyAsText(bytes, truncated, contentType) != null;
return genuineText ? null : mime;
}
/**
* The body as the viewer reads it: text first, whatever its content type, except an image, which
* is never text.
*/
export function viewerText(
bytes: Uint8Array,
truncated: boolean,
contentType: string | null,
): string | null {
if (bodyImageType(bytes, truncated, contentType) != null) return null;
return bodyAsText(bytes, truncated, contentType);
}
export async function formatText(
text: string,
language: CodeEditorProps["language"],
@@ -81,6 +190,33 @@ export function shortContentType(contentType: string | null): string | null {
return suffix.replace(/^x-/, "");
}
/** Extensions for media types whose subtype isn't one */
const EXTENSIONS: Record<string, string> = {
"application/javascript": "js",
"application/octet-stream": "bin",
"image/jpeg": "jpg",
"image/svg+xml": "svg",
"text/javascript": "js",
"text/markdown": "md",
"text/plain": "txt",
};
/** A name to offer for a saved body: the URL's file name, or one from the content type */
export function bodyFileName(flow: Flow, which: FlowBody): string {
let last = "";
try {
last = decodeURIComponent(new URL(flow.url).pathname.split("/").pop() ?? "");
} catch {
// Not a URL, or not encoded properly
}
if (which === "response" && /^[^\\/:*?"<>|]+\.[a-z0-9]+$/i.test(last)) return last;
const headers = which === "request" ? flow.requestHeaders : flow.responseHeaders;
const contentType = findHeader(headers, "content-type");
const essence = mimeEssence(contentType);
const extension = (essence == null ? null : EXTENSIONS[essence]) ?? shortContentType(contentType);
return `${which}.${extension?.replace(/[^a-z0-9-]/gi, "") || "bin"}`;
}
export function findHeader(headers: { name: string; value: string }[], name: string) {
const lower = name.toLowerCase();
return headers.find((h) => h.name.toLowerCase() === lower)?.value ?? null;
+45 -2
View File
@@ -1,9 +1,10 @@
import type { ExportHarRequest, Flow, FlowBody } from "@yaakapp-internal/proxy-lib";
import type { DropdownItem } from "@yaakapp-internal/ui";
import { save } from "@tauri-apps/plugin-dialog";
import { formatSize } from "@yaakapp-internal/lib/formatSize";
import { atom, useSetAtom } from "jotai";
import { useCallback } from "react";
import { base64ToBytes } from "./body";
import { base64ToBytes, bodyFileName } from "./body";
import { curlBodyText, toCurl } from "./curl";
import { rpc } from "./rpc";
import { useShowToast } from "./toast";
@@ -14,10 +15,35 @@ export const saveDialogFlowAtom = atom<Flow | null>(null);
/** A body as text, decoding what is not UTF-8 lossily, or null when there is none */
async function bodyText(flow: Flow, which: FlowBody): Promise<string | null> {
const res = await rpc("get_flow_body", { flowId: flow.id, which });
if (res.decodeError != null) throw new Error(res.decodeError);
const bytes = base64ToBytes(res.data);
return bytes.length === 0 ? null : new TextDecoder().decode(bytes);
}
/** Writes a flow's body to a file the person picks, as the viewer shows it */
export function useSaveFlowBody(): (flow: Flow, which: FlowBody) => Promise<void> {
const showToast = useShowToast();
return useCallback(
async (flow: Flow, which: FlowBody) => {
try {
const path = await save({ defaultPath: bodyFileName(flow, which) });
if (path == null) return;
const saved = await rpc("save_flow_body", { flowId: flow.id, which, path });
const notes = [
saved.raw ? "as it was on the wire, since it failed to decode" : null,
saved.truncated ? "cut at the capture limit" : null,
].filter((n) => n != null);
showToast({
message: `Saved ${formatSize(saved.size)}${notes.length > 0 ? `, ${notes.join(", ")}` : ""}`,
});
} catch (err) {
showToast({ message: `Failed to save the ${which} body: ${String(err)}`, color: "danger" });
}
},
[showToast],
);
}
/**
* Writes flows to a HAR file the person picks: `ids`, or every flow `filter` covers. Returns how
* many were written, or null when the save panel was cancelled.
@@ -38,6 +64,7 @@ export async function exportHar(
export function useFlowActions(): (flow: Flow) => DropdownItem[] {
const showToast = useShowToast();
const openSave = useSetAtom(saveDialogFlowAtom);
const saveBody = useSaveFlowBody();
return useCallback(
(flow: Flow) => {
@@ -75,6 +102,10 @@ export function useFlowActions(): (flow: Flow) => DropdownItem[] {
showToast({ message: `Failed to copy cURL: ${String(err)}`, color: "danger" });
return;
}
if (res.decodeError != null) {
showToast({ message: `Failed to copy cURL: ${res.decodeError}`, color: "danger" });
return;
}
body = curlBodyText(base64ToBytes(res.data), res.truncated);
// A command that sends other bytes than the app did would mislead.
if (body == null) {
@@ -102,6 +133,18 @@ export function useFlowActions(): (flow: Flow) => DropdownItem[] {
onSelect: () => copy("response body", () => bodyText(flow, "response")),
},
{ type: "separator" },
{
label: "Save Request Body to File",
icon: "download",
disabled: flow.requestSize === 0,
onSelect: () => saveBody(flow, "request"),
},
{
label: "Save Response Body to File",
icon: "download",
disabled: noResponse || flow.responseSize === 0,
onSelect: () => saveBody(flow, "response"),
},
{
label: "Export HAR",
icon: "download",
@@ -116,6 +159,6 @@ export function useFlowActions(): (flow: Flow) => DropdownItem[] {
},
];
},
[openSave, showToast],
[openSave, saveBody, showToast],
);
}
+1
View File
@@ -9,6 +9,7 @@ publish = false
base64 = "0.23.1"
bytes = "1"
chrono = { workspace = true, features = ["serde"] }
encoding_rs = "0.8"
log = { workspace = true }
http-body-util = "0.1"
hyper = { version = "1", features = ["http1", "server"] }
+27 -2
View File
@@ -133,6 +133,7 @@ export type GetFlowBodyRequest = { flowId: string, which: FlowBody, };
export type GetFlowBodyResponse = {
/**
* The body decoded with its Content-Encoding, at most the capture limit of it, in base64.
* When it fails to decode, the bytes as they were on the wire instead.
*/
data: string,
/**
@@ -142,7 +143,11 @@ truncated: boolean,
/**
* Bytes stored for the body, as they were on the wire.
*/
storedSize: number, };
storedSize: number,
/**
* Why the body failed to decode, in which case `data` is as it was on the wire.
*/
decodeError?: string, };
export type GetProxyStateRequest = Record<string, never>;
@@ -238,7 +243,27 @@ export type ProxyStatePayload = { state: ProxyState, };
export type RpcEventSchema = { flows_cleared: FlowsClearedPayload, model_write: ModelPayload, proxy_state_changed: ProxyStatePayload, };
export type RpcSchema = { execute_action: [ActionInvocation, boolean], format_body: [FormatBodyRequest, string], list_actions: [ListActionsRequest, ListActionsResponse], list_flows: [FlowFilter, ListFlowsResponse], set_recording: [SetRecordingRequest, boolean], set_system_proxy_at_launch: [SetSystemProxyAtLaunchRequest, boolean], set_http_version: [SetHttpVersionRequest, HttpVersion], app_icon: [AppIconRequest, string | null], ca_status: [CaStatusRequest, CaStatus], ca_trust: [CaTrustRequest, CaStatus], clear_flows: [ClearFlowsRequest, number], export_har: [ExportHarRequest, number], filter_body: [FilterBodyRequest, FilterBodyResponse], get_flow_body: [GetFlowBodyRequest, GetFlowBodyResponse], get_proxy_state: [GetProxyStateRequest, GetProxyStateResponse], helper_register: [HelperRegisterRequest, SystemProxyStatus], helper_unregister: [HelperUnregisterRequest, SystemProxyStatus], list_workspaces: [ListWorkspacesRequest, ListWorkspacesResponse], preview_save: [PreviewSaveRequest, SavePreview], save_flow: [SaveFlowToYaakRequest, SavedFlow], system_proxy_set: [SystemProxySetRequest, SystemProxyStatus], system_proxy_status: [SystemProxyStatusRequest, SystemProxyStatus], take_over: [TakeOverRequest, boolean], };
export type RpcSchema = { execute_action: [ActionInvocation, boolean], format_body: [FormatBodyRequest, string], list_actions: [ListActionsRequest, ListActionsResponse], list_flows: [FlowFilter, ListFlowsResponse], set_recording: [SetRecordingRequest, boolean], set_system_proxy_at_launch: [SetSystemProxyAtLaunchRequest, boolean], set_http_version: [SetHttpVersionRequest, HttpVersion], app_icon: [AppIconRequest, string | null], ca_status: [CaStatusRequest, CaStatus], ca_trust: [CaTrustRequest, CaStatus], clear_flows: [ClearFlowsRequest, number], export_har: [ExportHarRequest, number], filter_body: [FilterBodyRequest, FilterBodyResponse], get_flow_body: [GetFlowBodyRequest, GetFlowBodyResponse], get_proxy_state: [GetProxyStateRequest, GetProxyStateResponse], helper_register: [HelperRegisterRequest, SystemProxyStatus], helper_unregister: [HelperUnregisterRequest, SystemProxyStatus], list_workspaces: [ListWorkspacesRequest, ListWorkspacesResponse], preview_save: [PreviewSaveRequest, SavePreview], save_flow: [SaveFlowToYaakRequest, SavedFlow], save_flow_body: [SaveFlowBodyRequest, SaveFlowBodyResponse], system_proxy_set: [SystemProxySetRequest, SystemProxyStatus], system_proxy_status: [SystemProxyStatusRequest, SystemProxyStatus], take_over: [TakeOverRequest, boolean], };
export type SaveFlowBodyRequest = { flowId: string, which: FlowBody,
/**
* The file to write.
*/
path: string, };
export type SaveFlowBodyResponse = {
/**
* Bytes written.
*/
size: number,
/**
* What was written is incomplete, as `GetFlowBodyResponse.truncated` is.
*/
truncated: boolean,
/**
* The body failed to decode, so the file has it as it was on the wire.
*/
raw: boolean, };
export type SaveFlowToYaakRequest = { flowId: string, workspaceId: string, folderId?: string,
/**
+145
View File
@@ -67,6 +67,61 @@ pub async fn read_body(
Ok(DecodedBody { data, stored_size, truncated, decode_error })
}
/// A body's text as the app's viewer reads it, as `decodeBodyText` in `@yaakapp-internal/lib`
/// does: in the charset its content type names, when it names one the WHATWG Encoding Standard
/// knows, or else as UTF-8, or windows-1252 when it isn't valid UTF-8, as browsers read text.
pub fn body_text(data: &[u8], content_type: Option<&str>, truncated: bool) -> String {
let encoding = content_type
.and_then(charset_label)
.and_then(|label| encoding_rs::Encoding::for_label(label.as_bytes()))
// A label the standard maps to "replacement" is one `TextDecoder` refuses.
.filter(|encoding| *encoding != encoding_rs::REPLACEMENT);
if let Some(encoding) = encoding {
return encoding.decode_with_bom_removal(data).0.into_owned();
}
if let Some(text) = strict_utf8(data, truncated) {
return text;
}
encoding_rs::WINDOWS_1252.decode_without_bom_handling(data).0.into_owned()
}
/// The `charset` parameter of a content type
fn charset_label(content_type: &str) -> Option<&str> {
content_type.split(';').skip(1).find_map(|param| {
let (name, value) = param.split_once('=')?;
name.trim().eq_ignore_ascii_case("charset").then(|| value.trim().trim_matches('"').trim())
})
}
/// The bytes as UTF-8 without a byte order mark, or None when they aren't valid UTF-8. A
/// truncated body may end partway through a character, so up to three trailing bytes are dropped
/// before giving up on it.
fn strict_utf8(data: &[u8], truncated: bool) -> Option<String> {
let max_trim = if truncated { data.len().min(3) } else { 0 };
(0..=max_trim).find_map(|trim| {
let text = std::str::from_utf8(&data[..data.len() - trim]).ok()?;
Some(text.strip_prefix('\u{feff}').unwrap_or(text).to_string())
})
}
/// A flow's body decoded as `read_body` does, or, when it fails to decode, the bytes as they were
/// on the wire, with `decode_error` saying why.
pub async fn read_body_or_raw(
blobs: &BlobManager,
flow: &Flow,
body: FlowBody,
limit: usize,
) -> Result<DecodedBody, String> {
let decoded = read_body(blobs, flow, body, limit).await?;
if decoded.decode_error.is_none() {
return Ok(decoded);
}
let mut data = read_raw_body(blobs, &flow.id, body)?;
let truncated = flow.truncated(body) || data.len() > limit;
data.truncate(limit);
Ok(DecodedBody { data, truncated, ..decoded })
}
#[cfg(test)]
mod tests {
use super::*;
@@ -127,4 +182,94 @@ mod tests {
assert!(body.truncated);
assert_eq!(body.decode_error, None);
}
/// The cases `decodeBodyText`'s tests in `@yaakapp-internal/lib` cover, read the same way
#[test]
fn body_text_reads_as_the_viewer_does() {
let french = "Accès refusé : vous n'avez pas l'autorisation d'accéder à cette page.";
let latin1: Vec<u8> = french.chars().map(|c| c as u8).collect();
assert_eq!(body_text(&latin1, Some("text/html; charset=iso-8859-1"), false), french);
assert_eq!(body_text(&latin1, Some("text/plain; CHARSET=\"ISO-8859-1\""), false), french);
let shift_jis = [
0x82, 0xb1, 0x82, 0xf1, 0x82, 0xc9, 0x82, 0xbf, 0x82, 0xcd, 0x81, 0x41, 0x90, 0xa2,
0x8a, 0x45,
];
assert_eq!(
body_text(&shift_jis, Some("text/plain; charset=Shift_JIS"), false),
"こんにちは、世界"
);
// Unlabeled and not UTF-8: windows-1252
assert_eq!(body_text(&latin1, Some("text/html"), false), french);
assert_eq!(body_text(&latin1, None, false), french);
assert_eq!(body_text(&[0x93, b'h', b'i', 0x94], None, false), "\u{201c}hi\u{201d}");
// A label `TextDecoder` refuses is no label.
assert_eq!(body_text(&latin1, Some("text/plain; charset=not-a-charset"), false), french);
assert_eq!(body_text(&latin1, Some("text/plain; charset=iso-2022-kr"), false), french);
// UTF-8, labeled or not; labeled, what isn't UTF-8 is replaced
let json = r#"{"city":"Zürich","greeting":"こんにちは"}"#;
assert_eq!(body_text(json.as_bytes(), None, false), json);
assert_eq!(
body_text(json.as_bytes(), Some("application/json; charset=utf-8"), false),
json
);
assert_eq!(body_text(b"a\xff", Some("text/plain; charset=utf-8"), false), "a\u{fffd}");
// A UTF-8 body cut partway through a character stays UTF-8
assert_eq!(body_text(&"abc€".as_bytes()[..5], None, true), "abc");
}
fn deflate_request(blobs: &BlobManager, id: &str, stored: &[u8]) -> Flow {
let mut writer = BodyChunkWriter::new(blobs.clone(), Flow::body_id(id, FlowBody::Request));
writer.write(stored).unwrap();
writer.finish().unwrap();
Flow {
id: id.into(),
request_size: stored.len() as i64,
request_headers: vec![crate::models::ProxyHeader {
name: "Content-Encoding".into(),
value: "deflate".into(),
}],
..Default::default()
}
}
/// Browser SDKs send `deflate` both ways: zlib-wrapped, as the spec says, and raw.
#[tokio::test]
async fn deflate_request_bodies_decode_wrapped_or_raw() {
use std::io::Write;
let (_db, blobs, _rx) = yaak_models::init_in_memory().unwrap();
let logs = br#"[{"message":"hello","service":"browser"}]"#;
let mut zlib = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default());
zlib.write_all(logs).unwrap();
let mut raw =
flate2::write::DeflateEncoder::new(Vec::new(), flate2::Compression::default());
raw.write_all(logs).unwrap();
for (id, stored) in [
("fl_zlib", zlib.finish().unwrap()),
("fl_raw", raw.finish().unwrap()),
] {
let flow = deflate_request(&blobs, id, &stored);
let body = read_body(&blobs, &flow, FlowBody::Request, 1024).await.unwrap();
assert_eq!(body.decode_error, None, "{id}");
assert_eq!((body.data.as_slice(), body.truncated), (&logs[..], false), "{id}");
}
}
#[tokio::test]
async fn undecodable_body_reads_as_it_was_on_the_wire() {
let (_db, blobs, _rx) = yaak_models::init_in_memory().unwrap();
let flow = deflate_request(&blobs, "fl_bad", b"\x00\x01 not deflate");
let decoded = read_body(&blobs, &flow, FlowBody::Request, 1024).await.unwrap();
assert!(decoded.decode_error.is_some());
let raw = read_body_or_raw(&blobs, &flow, FlowBody::Request, 1024).await.unwrap();
assert_eq!(raw.data, b"\x00\x01 not deflate");
assert!(!raw.truncated);
assert_eq!(raw.decode_error, decoded.decode_error);
let cut = read_body_or_raw(&blobs, &flow, FlowBody::Request, 4).await.unwrap();
assert_eq!((cut.data.as_slice(), cut.truncated), (&b"\x00\x01 n"[..], true));
}
}
+131 -10
View File
@@ -851,12 +851,40 @@ pub struct GetFlowBodyRequest {
#[serde(rename_all = "camelCase")]
pub struct GetFlowBodyResponse {
/// The body decoded with its Content-Encoding, at most the capture limit of it, in base64.
/// When it fails to decode, the bytes as they were on the wire instead.
pub data: String,
/// The body is incomplete: cut on the wire, or decoded past the capture limit.
pub truncated: bool,
/// Bytes stored for the body, as they were on the wire.
#[ts(type = "number")]
pub stored_size: usize,
/// Why the body failed to decode, in which case `data` is as it was on the wire.
#[serde(skip_serializing_if = "Option::is_none")]
#[ts(optional)]
pub decode_error: Option<String>,
}
#[derive(Deserialize, TS)]
#[ts(export, export_to = "gen_rpc.ts")]
#[serde(rename_all = "camelCase")]
pub struct SaveFlowBodyRequest {
pub flow_id: String,
pub which: FlowBody,
/// The file to write.
pub path: String,
}
#[derive(Serialize, TS)]
#[ts(export, export_to = "gen_rpc.ts")]
#[serde(rename_all = "camelCase")]
pub struct SaveFlowBodyResponse {
/// Bytes written.
#[ts(type = "number")]
pub size: usize,
/// What was written is incomplete, as `GetFlowBodyResponse.truncated` is.
pub truncated: bool,
/// The body failed to decode, so the file has it as it was on the wire.
pub raw: bool,
}
#[derive(Deserialize, TS)]
@@ -1082,16 +1110,42 @@ async fn get_flow_body(
) -> Result<GetFlowBodyResponse, RpcError> {
let flow = find_flow(&ctx, &req.flow_id)?;
let limit = settings(&ctx)?.capture_body_limit;
let body = viewer_body(&ctx, &flow, req.which, limit).await?;
let body =
bodies::read_body_or_raw(&ctx.blobs, &flow, req.which, limit).await.map_err(rpc_error)?;
Ok(GetFlowBodyResponse {
data: BASE64.encode(&body.data),
truncated: body.truncated,
stored_size: body.stored_size,
decode_error: body.decode_error,
})
}
/// A body for the app's viewer, which shows a body that failed to decode as its error, as it
/// always has, rather than as what decoded before the failure.
/// Writes a body to a file the person picked: decoded, as the viewer shows it, or as it was on
/// the wire when it fails to decode.
async fn save_flow_body(
ctx: ProxyCtx,
req: SaveFlowBodyRequest,
) -> Result<SaveFlowBodyResponse, RpcError> {
let flow = find_flow(&ctx, &req.flow_id)?;
let limit = settings(&ctx)?.capture_body_limit;
let body =
bodies::read_body_or_raw(&ctx.blobs, &flow, req.which, limit).await.map_err(rpc_error)?;
let saved = SaveFlowBodyResponse {
size: body.data.len(),
truncated: body.truncated,
raw: body.decode_error.is_some(),
};
let path = PathBuf::from(req.path);
blocking(move || {
std::fs::write(&path, &body.data)
.map_err(|e| format!("Failed to write {}: {e}", path.display()))
})
.await?;
Ok(saved)
}
/// A body to filter, which fails when the body failed to decode rather than filtering what
/// decoded before the failure.
async fn viewer_body(
ctx: &ProxyCtx,
flow: &Flow,
@@ -1112,14 +1166,13 @@ async fn filter_body(
let flow = find_flow(&ctx, &req.flow_id)?;
let limit = settings(&ctx)?.capture_body_limit;
let body = viewer_body(&ctx, &flow, req.which, limit).await?;
let text = String::from_utf8_lossy(&body.data);
let content_type = find_header(flow.headers(req.which), "content-type");
// The text the viewer shows, so the filter's results match it
let text = bodies::body_text(&body.data, content_type, body.truncated);
let language = match req.language {
Some(FilterLanguage::JsonPath) => yaak_filter::Language::JsonPath,
Some(FilterLanguage::XPath) => yaak_filter::Language::XPath,
None => {
let content_type = find_header(flow.headers(req.which), "content-type");
yaak_filter::Language::for_content_type(content_type.unwrap_or_default())
}
None => yaak_filter::Language::for_content_type(content_type.unwrap_or_default()),
};
let result = yaak_filter::filter(language, &text, &req.expression);
Ok(FilterBodyResponse { content: result.content, error: result.error })
@@ -1437,6 +1490,7 @@ define_rpc! {
list_workspaces(ListWorkspacesRequest) -> ListWorkspacesResponse,
preview_save(PreviewSaveRequest) -> SavePreview,
save_flow(SaveFlowToYaakRequest) -> SavedFlow,
save_flow_body(SaveFlowBodyRequest) -> SaveFlowBodyResponse,
system_proxy_set(SystemProxySetRequest) -> SystemProxyStatus,
system_proxy_status(SystemProxyStatusRequest) -> SystemProxyStatus,
take_over(TakeOverRequest) -> bool,
@@ -1491,6 +1545,57 @@ mod tests {
assert!(log.was_on_at(SystemTime::now()));
}
/// A flow whose response is `body` under `content_type`, stored in `ctx`
fn flow_with_response(ctx: &ProxyCtx, id: &str, content_type: &str, body: &[u8]) {
let flow = Flow {
id: id.into(),
status: Some(200),
response_size: body.len() as i64,
response_headers: vec![ProxyHeader {
name: "Content-Type".into(),
value: content_type.into(),
}],
..Default::default()
};
ctx.db.with_tx(|tx| tx.upsert(&flow, &UpdateSource::Background)).unwrap();
let mut writer =
BodyChunkWriter::new(ctx.blobs.clone(), Flow::body_id(id, FlowBody::Response));
writer.write(body).unwrap();
writer.finish().unwrap();
}
/// The filter reads the text the viewer shows: in the body's charset, not UTF-8 alone.
#[tokio::test]
async fn filter_reads_the_body_in_its_charset() {
let dir = tempfile::tempdir().unwrap();
let (events, _rx) = RpcEventEmitter::new();
let ctx = ProxyCtx::new(dir.path(), events).unwrap();
let xml = "<cities><city>Zürich</city><city>Kraków</city></cities>";
let utf16: Vec<u8> = xml.encode_utf16().flat_map(u16::to_le_bytes).collect();
flow_with_response(&ctx, "fl_utf16", "application/xml; charset=utf-16le", &utf16);
let json = r#"{"items":[{"name":"Café"},{"name":"Crème brûlée"}]}"#;
let latin1: Vec<u8> = json.chars().map(|c| c as u8).collect();
flow_with_response(&ctx, "fl_latin1", "application/json; charset=iso-8859-1", &latin1);
flow_with_response(&ctx, "fl_unlabeled", "application/json", &latin1);
let filter = |flow_id: &str, expression: &str, language| FilterBodyRequest {
flow_id: flow_id.into(),
which: FlowBody::Response,
expression: expression.into(),
language: Some(language),
};
let xpath = filter("fl_utf16", "//city[2]", FilterLanguage::XPath);
let result = filter_body(ctx.clone(), xpath).await.unwrap();
assert_eq!((result.content.as_str(), result.error), ("<city>Kraków</city>", None));
for flow_id in ["fl_latin1", "fl_unlabeled"] {
let names = filter(flow_id, "$.items[*].name", FilterLanguage::JsonPath);
let result = filter_body(ctx.clone(), names).await.unwrap();
assert_eq!(result.error, None, "{flow_id}");
assert_eq!(result.content, "[\n \"Café\",\n \"Crème brûlée\"\n]", "{flow_id}");
}
}
#[tokio::test]
async fn the_app_still_hears_why_a_body_failed_to_decode() {
let dir = tempfile::tempdir().unwrap();
@@ -1511,9 +1616,25 @@ mod tests {
writer.write(b"not a gzip stream").unwrap();
writer.finish().unwrap();
// The viewer gets the bytes as they were on the wire, and why they didn't decode.
let req = GetFlowBodyRequest { flow_id: "fl_bad".into(), which: FlowBody::Response };
let err = get_flow_body(ctx.clone(), req).await.err().unwrap();
assert!(err.message.starts_with("Failed to decode"), "{}", err.message);
let body = get_flow_body(ctx.clone(), req).await.unwrap();
let error = body.decode_error.unwrap();
assert!(error.starts_with("Failed to decode Gzip body"), "{error}");
assert_eq!(BASE64.decode(body.data).unwrap(), b"not a gzip stream");
assert!(!body.truncated);
// Saving writes those same bytes.
let path = dir.path().join("body.bin");
let req = SaveFlowBodyRequest {
flow_id: "fl_bad".into(),
which: FlowBody::Response,
path: path.to_string_lossy().into_owned(),
};
let saved = save_flow_body(ctx.clone(), req).await.unwrap();
assert_eq!((saved.size, saved.truncated, saved.raw), (17, false, true));
assert_eq!(std::fs::read(&path).unwrap(), b"not a gzip stream");
let req = FilterBodyRequest {
flow_id: "fl_bad".into(),
which: FlowBody::Response,
+1 -1
View File
@@ -5,7 +5,7 @@ edition = "2024"
publish = false
[dependencies]
async-compression = { version = "0.4", features = ["tokio", "gzip", "deflate", "brotli", "zstd"] }
async-compression = { version = "0.4", features = ["tokio", "gzip", "deflate", "zlib", "brotli", "zstd"] }
async-trait = "0.1"
brotli = "9"
bytes = "1.11.1"
+601 -10
View File
@@ -1,11 +1,13 @@
use crate::error::{Error, Result};
use async_compression::tokio::bufread::{
BrotliDecoder, DeflateDecoder as AsyncDeflateDecoder, GzipDecoder,
ZstdDecoder as AsyncZstdDecoder,
ZlibDecoder as AsyncZlibDecoder, ZstdDecoder as AsyncZstdDecoder,
};
use flate2::read::{DeflateDecoder, GzDecoder};
use flate2::read::{DeflateDecoder, GzDecoder, ZlibDecoder};
use std::io::Read;
use tokio::io::{AsyncBufRead, AsyncRead};
use std::pin::Pin;
use std::task::{Context, Poll, ready};
use tokio::io::{AsyncBufRead, AsyncRead, BufReader, ReadBuf};
/// Supported compression encodings
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -66,13 +68,77 @@ fn decompress_gzip(data: &[u8]) -> Result<Vec<u8>> {
Ok(decompressed)
}
/// `deflate` names zlib-wrapped data (RFC 1950), but plenty of senders send it raw (RFC 1951).
fn decompress_deflate(data: &[u8]) -> Result<Vec<u8>> {
let mut decoder = DeflateDecoder::new(data);
let mut decompressed = Vec::new();
decoder
.read_to_end(&mut decompressed)
.map_err(|e| Error::DecompressionError(format!("deflate decompression failed: {}", e)))?;
Ok(decompressed)
let read = |zlib: bool| {
let mut decompressed = Vec::new();
let result = if zlib {
ZlibDecoder::new(data).read_to_end(&mut decompressed)
} else {
DeflateDecoder::new(data).read_to_end(&mut decompressed)
};
result.map(|_| decompressed)
};
match deflate_variant(data) {
DeflateVariant::Zlib => read(true),
DeflateVariant::Raw => read(false),
// Zlib only when it ends whole, its checksum verified, or else raw.
DeflateVariant::Either => inflate_zlib_whole(data).or_else(|_| read(false)),
}
.map_err(|e| Error::DecompressionError(format!("deflate decompression failed: {}", e)))
}
/// Zlib data decoded through to its end and checksum, failing on any less
fn inflate_zlib_whole(data: &[u8]) -> std::io::Result<Vec<u8>> {
let mut inflater = flate2::Decompress::new(true);
let mut out = Vec::new();
loop {
// Always room to write, so a call that makes no progress has run out of input.
out.reserve(64 * 1024);
let (read, written) = (inflater.total_in(), inflater.total_out());
let rest = &data[usize::try_from(read).unwrap_or(data.len()).min(data.len())..];
// Not `Finish`, which fails for good when the output outgrows its buffer.
let status = inflater
.decompress_vec(rest, &mut out, flate2::FlushDecompress::None)
.map_err(std::io::Error::other)?;
if status == flate2::Status::StreamEnd {
return Ok(out);
}
if inflater.total_in() == read && inflater.total_out() == written {
return Err(std::io::ErrorKind::UnexpectedEof.into());
}
}
}
/// Which `deflate` a body is, from its first five bytes
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum DeflateVariant {
Zlib,
Raw,
/// A zlib header that is also the start of a raw stored block: only decoding tells.
Either,
}
/// Zlib data opens with a header whose first byte names deflate with a window of at most 32 KB,
/// and whose two bytes are a multiple of 31. Raw deflate passes that only when it opens with a
/// stored block that isn't the last, whose length follows in the next two bytes and its
/// complement in the two after.
fn deflate_variant(data: &[u8]) -> DeflateVariant {
let zlib = match data {
[cmf, flg, ..] => {
cmf & 0x0f == 8 && cmf >> 4 <= 7 && (u16::from(*cmf) << 8 | u16::from(*flg)) % 31 == 0
}
_ => false,
};
match data {
_ if !zlib => DeflateVariant::Raw,
[_, l0, l1, n0, n1, ..]
if u16::from_le_bytes([*l0, *l1]) != !u16::from_le_bytes([*n0, *n1]) =>
{
DeflateVariant::Zlib
}
_ => DeflateVariant::Either,
}
}
fn decompress_brotli(data: &[u8]) -> Result<Vec<u8>> {
@@ -96,12 +162,262 @@ pub fn streaming_decoder<R: AsyncBufRead + Unpin + Send + 'static>(
match encoding {
ContentEncoding::Identity => Box::new(reader),
ContentEncoding::Gzip => Box::new(GzipDecoder::new(reader)),
ContentEncoding::Deflate => Box::new(AsyncDeflateDecoder::new(reader)),
ContentEncoding::Deflate => Box::new(StreamingDeflateDecoder::new(reader)),
ContentEncoding::Brotli => Box::new(BrotliDecoder::new(reader)),
ContentEncoding::Zstd => Box::new(AsyncZstdDecoder::new(reader)),
}
}
/// Decodes `deflate`, zlib-wrapped or raw, as its first five bytes say. When they could be
/// either, it decodes both ways at once until one fails, and goes on with the other.
struct StreamingDeflateDecoder<R> {
state: DeflateState<R>,
}
enum DeflateState<R> {
Sniffing(Replay<R>),
Zlib(AsyncZlibDecoder<BufReader<Replay<R>>>),
Raw(AsyncDeflateDecoder<BufReader<Replay<R>>>),
Either(EitherInflate<BufReader<Replay<R>>>),
/// Only while switching from sniffing to one of the others
Switching,
}
impl<R> StreamingDeflateDecoder<R> {
fn new(reader: R) -> Self {
let source = Replay { inner: reader, header: Vec::new(), pos: 0, sniffing: true };
Self { state: DeflateState::Sniffing(source) }
}
}
impl<R: AsyncRead + Unpin> AsyncRead for StreamingDeflateDecoder<R> {
fn poll_read(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &mut ReadBuf<'_>,
) -> Poll<std::io::Result<()>> {
let this = self.get_mut();
loop {
match &mut this.state {
DeflateState::Sniffing(source) => {
// Read until the header is in hand, or the input ends short of it.
while source.header.len() < 5 {
let mut header = [0u8; 5];
let mut header = ReadBuf::new(&mut header[source.header.len()..]);
ready!(Pin::new(&mut *source).poll_read(cx, &mut header))?;
if header.filled().is_empty() {
break;
}
}
let DeflateState::Sniffing(mut source) =
std::mem::replace(&mut this.state, DeflateState::Switching)
else {
unreachable!()
};
source.sniffing = false;
let variant = deflate_variant(&source.header);
let reader = BufReader::new(source);
this.state = match variant {
DeflateVariant::Zlib => DeflateState::Zlib(AsyncZlibDecoder::new(reader)),
DeflateVariant::Raw => DeflateState::Raw(AsyncDeflateDecoder::new(reader)),
DeflateVariant::Either => DeflateState::Either(EitherInflate::new(reader)),
};
}
DeflateState::Zlib(d) => return Pin::new(d).poll_read(cx, buf),
DeflateState::Raw(d) => return Pin::new(d).poll_read(cx, buf),
DeflateState::Either(d) => return Pin::new(d).poll_read(cx, buf),
DeflateState::Switching => {
return Poll::Ready(Err(std::io::Error::other("deflate decoder is broken")));
}
}
}
}
}
/// Reads `inner`, keeping what it read while sniffing, then reads that again before the rest.
struct Replay<R> {
inner: R,
header: Vec<u8>,
/// Where reading resumes in `header` once sniffing is over
pos: usize,
sniffing: bool,
}
impl<R: AsyncRead + Unpin> AsyncRead for Replay<R> {
fn poll_read(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &mut ReadBuf<'_>,
) -> Poll<std::io::Result<()>> {
let this = self.get_mut();
if this.sniffing {
let before = buf.filled().len();
ready!(Pin::new(&mut this.inner).poll_read(cx, buf))?;
this.header.extend_from_slice(&buf.filled()[before..]);
return Poll::Ready(Ok(()));
}
if this.pos < this.header.len() {
let n = buf.remaining().min(this.header.len() - this.pos);
buf.put_slice(&this.header[this.pos..this.pos + n]);
this.pos += n;
return Poll::Ready(Ok(()));
}
Pin::new(&mut this.inner).poll_read(cx, buf)
}
}
/// Input fed to the readings of an ambiguous deflate body at a time, which deflate grows to at
/// most about a megabyte
const DEFLATE_FEED: usize = 1024;
/// Output held while both readings of an ambiguous deflate body are still valid, past which it is
/// taken to be zlib, as its header says. A wrong reading fails within a few bytes in practice.
const DEFLATE_HELD_LIMIT: usize = 1024 * 1024;
/// Both readings of a deflate body that could be zlib or raw, fed the same input. Their output
/// is held back while both are still valid, and the one left streams on: a wrong reading fails
/// within a few bytes in practice, so little is held. If both end, zlib's checksum verified it.
struct EitherInflate<B> {
input: B,
/// Zlib, then raw
readings: [Option<Inflate>; 2],
/// Why the last reading to fail failed
error: Option<std::io::Error>,
}
struct Inflate {
inflater: flate2::Decompress,
out: Vec<u8>,
/// Where reading resumes in `out`
pos: usize,
ended: bool,
}
impl<B> EitherInflate<B> {
fn new(input: B) -> Self {
let reading = |zlib| Inflate {
inflater: flate2::Decompress::new(zlib),
out: Vec::new(),
pos: 0,
ended: false,
};
Self { input, readings: [Some(reading(true)), Some(reading(false))], error: None }
}
}
impl Inflate {
/// Decodes all of `input` onto `out`, or up to the end of the stream
fn feed(&mut self, input: &[u8]) -> std::io::Result<()> {
let mut consumed = 0;
while !self.ended {
if self.out.capacity() - self.out.len() < 32 * 1024 {
self.out.reserve(64 * 1024);
}
let (read, written) = (self.inflater.total_in(), self.inflater.total_out());
let status = self
.inflater
.decompress_vec(&input[consumed..], &mut self.out, flate2::FlushDecompress::None)
.map_err(std::io::Error::other)?;
let progress = (self.inflater.total_in() - read, self.inflater.total_out() - written);
consumed += usize::try_from(progress.0).unwrap_or(usize::MAX).min(input.len());
if status == flate2::Status::StreamEnd {
self.ended = true;
} else if consumed == input.len() && self.out.len() < self.out.capacity() {
// All of it in, and all that it decodes to out
break;
} else if progress == (0, 0) {
return Err(std::io::Error::other("deflate decoding stalled"));
}
}
Ok(())
}
}
impl<B: AsyncBufRead + Unpin> AsyncRead for EitherInflate<B> {
fn poll_read(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &mut ReadBuf<'_>,
) -> Poll<std::io::Result<()>> {
let this = self.get_mut();
loop {
let [zlib, raw] = &mut this.readings;
match (zlib.as_mut(), raw.as_mut()) {
(None, None) => {
let error = this.error.take();
return Poll::Ready(Err(
error.unwrap_or_else(|| std::io::Error::other("deflate decoding failed"))
));
}
// Both ended: zlib's checksum verified it.
(Some(z), Some(r)) if z.ended && r.ended => *raw = None,
(Some(only), None) | (None, Some(only)) => {
if only.pos < only.out.len() {
let n = buf.remaining().min(only.out.len() - only.pos);
buf.put_slice(&only.out[only.pos..only.pos + n]);
only.pos += n;
if only.pos == only.out.len() {
only.out.clear();
only.pos = 0;
}
return Poll::Ready(Ok(()));
}
if only.ended {
return Poll::Ready(Ok(()));
}
ready!(this.feed(cx))?;
}
(Some(_), Some(_)) => ready!(this.feed(cx))?,
}
}
}
}
impl<B: AsyncBufRead + Unpin> EitherInflate<B> {
/// Feeds the next input to each reading still going, dropping those that fail on it. Pending
/// until there is input; at its end, a reading that hasn't ended was cut short.
fn feed(&mut self, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
let chunk = ready!(Pin::new(&mut self.input).poll_fill_buf(cx))?;
// A little input at a time, since deflate can grow a thousandfold.
let len = chunk.len().min(DEFLATE_FEED);
let chunk = &chunk[..len];
let [zlib, raw] = &mut self.readings;
if let (Some(z), Some(r)) = (zlib.as_ref(), raw.as_ref()) {
let (z_held, r_held) = (z.out.len() - z.pos, r.out.len() - r.pos);
let cut_short = len == 0 && !z.ended && !r.ended;
if cut_short {
// Both were cut short: keep what decoded furthest, so a cut body shows its start.
if r_held > z_held {
*zlib = None
} else {
*raw = None
}
// The one left gives what it decoded, then fails on the end of input.
return Poll::Ready(Ok(()));
} else if z_held.max(r_held) > DEFLATE_HELD_LIMIT {
// Holding no more for a raw body zlib still reads validly: it is taken to be zlib.
*raw = None;
}
}
for slot in &mut self.readings {
let Some(reading) = slot.as_mut().filter(|r| !r.ended) else {
continue;
};
let result = if len == 0 {
Err(std::io::ErrorKind::UnexpectedEof.into())
} else {
reading.feed(chunk)
};
if let Err(e) = result {
*slot = None;
self.error = Some(e);
}
}
Pin::new(&mut self.input).consume(len);
Poll::Ready(Ok(()))
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -159,6 +475,281 @@ mod tests {
assert_eq!(result.decompressed_size, original.len() as u64);
}
fn zlib(data: &[u8]) -> Vec<u8> {
let mut encoder = flate2::write::ZlibEncoder::new(Vec::new(), Compression::default());
encoder.write_all(data).unwrap();
encoder.finish().unwrap()
}
fn raw_deflate(data: &[u8]) -> Vec<u8> {
let mut encoder = flate2::write::DeflateEncoder::new(Vec::new(), Compression::default());
encoder.write_all(data).unwrap();
encoder.finish().unwrap()
}
/// Raw deflate whose first two bytes also pass as a zlib header: a stored block of 29 bytes
/// (`08 1d`, with `0x081d` a multiple of 31), then an empty final one.
fn raw_deflate_like_zlib(data: &[u8; 29]) -> Vec<u8> {
let mut out = vec![0x08, 0x1d, 0x00, 0xe2, 0xff];
out.extend_from_slice(data);
out.extend_from_slice(&[0x01, 0x00, 0x00, 0xff, 0xff]);
assert_eq!(deflate_variant(&out), DeflateVariant::Either);
out
}
/// Zlib that opens as `raw_deflate_like_zlib` does: header `08 1d`, then a stored block of
/// 0xffe2 bytes, which `1d 00 e2 ff` also reads as the start of, and a final stored block of
/// `extra` bytes.
fn zlib_like_raw_deflate(extra: u16) -> (Vec<u8>, Vec<u8>) {
let data: Vec<u8> = (0..0xffe2 + u32::from(extra)).map(|i| (i % 251) as u8).collect();
let mut out = vec![0x08, 0x1d, 0x00, 0xe2, 0xff, 0x1d, 0x00];
out.extend_from_slice(&data[..0xffe2]);
out.push(0x01);
out.extend_from_slice(&extra.to_le_bytes());
out.extend_from_slice(&(!extra).to_le_bytes());
out.extend_from_slice(&data[0xffe2..]);
let (mut a, mut b) = (1u32, 0u32);
for byte in &data {
a = (a + u32::from(*byte)) % 65521;
b = (b + a) % 65521;
}
out.extend_from_slice(&((b << 16) | a).to_be_bytes());
assert_eq!(deflate_variant(&out), DeflateVariant::Either);
(out, data)
}
async fn stream_decode(data: Vec<u8>, encoding: ContentEncoding) -> std::io::Result<Vec<u8>> {
use tokio::io::AsyncReadExt;
let mut out = Vec::new();
// One byte at a time, to read the header across polls as a slow connection would.
let reader = BufReader::with_capacity(1, std::io::Cursor::new(data));
streaming_decoder(reader, encoding).read_to_end(&mut out).await?;
Ok(out)
}
/// What a body decodes to before it fails, and why it failed
async fn stream_decode_until_error(data: Vec<u8>) -> (Vec<u8>, Option<std::io::Error>) {
use tokio::io::AsyncReadExt;
let mut out = Vec::new();
let mut decoder =
streaming_decoder(BufReader::new(std::io::Cursor::new(data)), ContentEncoding::Deflate);
let mut buf = [0u8; 4096];
loop {
match decoder.read(&mut buf).await {
Ok(0) => return (out, None),
Ok(n) => out.extend_from_slice(&buf[..n]),
Err(e) => return (out, Some(e)),
}
}
}
#[tokio::test]
async fn streaming_deflate_keeps_what_decoded_of_a_cut_ambiguous_body() {
// Bytes both readings still accept when the input ends: zlib in a stored block too
let text = *b"\x1d\0xxxxxxxxxxxxxxxxxxxxxxxxxxx";
let mut cut = raw_deflate_like_zlib(&text);
// Without its final block, as a body cut at the capture limit is
cut.truncate(cut.len() - 5);
let (decoded, error) = stream_decode_until_error(cut).await;
assert_eq!(decoded, text);
assert_eq!(error.map(|e| e.kind()), Some(std::io::ErrorKind::UnexpectedEof));
}
/// Zlib of `megabytes` of zeros whose raw reading is a stored block of 29 bytes and an empty
/// final one, both inside zlib's first stored block: raw ends at once, zlib goes on.
fn zlib_whose_raw_reading_ends_early(megabytes: usize) -> (Vec<u8>, Vec<u8>) {
let mut stored = vec![b'z'; 0xffe2];
stored[27..32].copy_from_slice(&[0x01, 0x00, 0x00, 0xff, 0xff]);
let mut expected = stored.clone();
expected.resize(stored.len() + megabytes * 1024 * 1024, 0);
let mut data = vec![0x08, 0x1d, 0x00, 0xe2, 0xff, 0x1d, 0x00];
data.extend_from_slice(&stored);
data.extend_from_slice(&raw_deflate(&expected[stored.len()..]));
let (mut a, mut b) = (1u32, 0u32);
for byte in &expected {
a = (a + u32::from(*byte)) % 65521;
b = (b + a) % 65521;
}
data.extend_from_slice(&((b << 16) | a).to_be_bytes());
assert_eq!(deflate_variant(&data), DeflateVariant::Either);
(data, expected)
}
#[tokio::test]
async fn streaming_deflate_holds_little_while_both_readings_go() {
use tokio::io::AsyncReadExt;
let (data, expected) = zlib_whose_raw_reading_ends_early(8);
let mut decoder =
StreamingDeflateDecoder::new(BufReader::new(std::io::Cursor::new(data.clone())));
let mut first = [0u8; 1024];
decoder.read_exact(&mut first).await.unwrap();
assert_eq!(first[..], expected[..1024]);
let DeflateState::Either(either) = &decoder.state else {
panic!("not read both ways")
};
// The limit, what one feed of input adds past it (deflate grows at most about 1,032
// times), and the raw reading's 64 KB: far short of the 8 MB body.
let held: usize = either.readings.iter().flatten().map(|r| r.out.len() - r.pos).sum();
let bound = 3 * 1024 * 1024;
assert!(held <= bound, "held {held} bytes for a 1 KB read");
let decoded = stream_decode(data, ContentEncoding::Deflate).await.unwrap();
assert!(decoded == expected);
}
const DEFLATE_TEXT: &[u8] = b"{\"service\":\"browser\",\"message\":\"hello deflate\"}";
#[test]
fn test_decompress_deflate_zlib_wrapped() {
let result = decompress(zlib(DEFLATE_TEXT), ContentEncoding::Deflate).unwrap();
assert_eq!(result.data, DEFLATE_TEXT);
let like_zlib = raw_deflate_like_zlib(b"twenty-nine bytes of raw text");
let result = decompress(like_zlib, ContentEncoding::Deflate).unwrap();
assert_eq!(result.data, b"twenty-nine bytes of raw text");
let both = *b"\x1d\0xxxxxxxxxxxxxxxxxxxxxxxxxxx";
let result = decompress(raw_deflate_like_zlib(&both), ContentEncoding::Deflate).unwrap();
assert_eq!(result.data, both);
for extra in [0, 10_000] {
let (zlib_like_raw, data) = zlib_like_raw_deflate(extra);
let result = decompress(zlib_like_raw, ContentEncoding::Deflate).unwrap();
assert!(result.data == data, "{extra} extra bytes");
}
let (data, expected) = large_raw_like_zlib();
assert!(decompress(data, ContentEncoding::Deflate).unwrap().data == expected);
}
/// Ambiguous raw deflate of more than 64 KB: a stored block that reads as zlib's header and
/// the start of a stored block of 0xffe2 bytes, then incompressible data, which zlib reads
/// as a block header 7 + 0xffe2 bytes in and fails on.
fn large_raw_like_zlib() -> (Vec<u8>, Vec<u8>) {
let mut expected = b"\x1d\0xxxxxxxxxxxxxxxxxxxxxxxxxxx".to_vec();
let mut seed = 1u32;
expected.extend((0..70_000).map(|_| {
seed = seed.wrapping_mul(1_103_515_245).wrapping_add(12_345);
(seed >> 16) as u8
}));
let mut data = vec![0x08, 0x1d, 0x00, 0xe2, 0xff];
data.extend_from_slice(&expected[..29]);
data.extend_from_slice(&raw_deflate(&expected[29..]));
assert!(data.len() > 64 * 1024);
(data, expected)
}
/// Greptile's case on #812: raw deflate that zlib reads validly past 64 KB. After the first
/// stored block, a final stored block of 65,535 bytes, in which zlib finds a valid stored
/// block header at 65,513 (`00 28 00 d7 ff`) and an invalid block type only at 65,558.
fn raw_valid_as_zlib_past_64k() -> (Vec<u8>, Vec<u8>) {
let mut block: Vec<u8> = (0..65_535u32).map(|i| b'a' + (i % 26) as u8).collect();
let first = 5 + 29 + 5;
block[65_513 - first..65_518 - first].copy_from_slice(&[0x00, 0x28, 0x00, 0xd7, 0xff]);
block[65_558 - first] = 0x07;
let mut expected = b"\x1d\0xxxxxxxxxxxxxxxxxxxxxxxxxxx".to_vec();
let mut data = raw_deflate_like_zlib(&expected.clone().try_into().unwrap());
// Make the first block not the last, and follow it with the large final one.
data.truncate(data.len() - 5);
data.extend_from_slice(&[0x01, 0xff, 0xff, 0x00, 0x00]);
data.extend_from_slice(&block);
expected.extend_from_slice(&block);
(data, expected)
}
#[test]
fn test_decompress_deflate_raw_valid_as_zlib_past_64k() {
let (data, expected) = raw_valid_as_zlib_past_64k();
// Raw alone decodes it; zlib alone gets past 64 KB before failing.
let mut raw = Vec::new();
DeflateDecoder::new(data.as_slice()).read_to_end(&mut raw).unwrap();
assert!(raw == expected);
let mut zlib = flate2::Decompress::new(true);
let mut out = Vec::with_capacity(200 * 1024);
let failed = zlib.decompress_vec(&data, &mut out, flate2::FlushDecompress::None);
assert!(failed.is_err() && zlib.total_in() > 65_536, "zlib read {}", zlib.total_in());
assert!(decompress(data, ContentEncoding::Deflate).unwrap().data == expected);
}
#[tokio::test]
async fn streaming_deflate_decodes_raw_valid_as_zlib_past_64k() {
use tokio::io::AsyncReadExt;
let (data, expected) = raw_valid_as_zlib_past_64k();
for capacity in [1, 8 * 1024, 256 * 1024] {
let mut out = Vec::new();
let reader = BufReader::with_capacity(capacity, std::io::Cursor::new(data.clone()));
streaming_decoder(reader, ContentEncoding::Deflate)
.read_to_end(&mut out)
.await
.unwrap();
assert!(out == expected, "read {capacity} bytes at a time");
}
}
#[tokio::test]
async fn streaming_deflate_decodes_zlib_and_raw() {
let zlib_wrapped = stream_decode(zlib(DEFLATE_TEXT), ContentEncoding::Deflate).await;
assert_eq!(zlib_wrapped.unwrap(), DEFLATE_TEXT);
let raw = stream_decode(raw_deflate(DEFLATE_TEXT), ContentEncoding::Deflate).await;
assert_eq!(raw.unwrap(), DEFLATE_TEXT);
let large: Vec<u8> = (0..200_000u32).flat_map(|i| (i % 251).to_le_bytes()).collect();
let large_zlib = stream_decode(zlib(&large), ContentEncoding::Deflate).await;
assert_eq!(large_zlib.unwrap(), large);
let large_raw = stream_decode(raw_deflate(&large), ContentEncoding::Deflate).await;
assert_eq!(large_raw.unwrap(), large);
}
#[tokio::test]
async fn streaming_deflate_retries_raw_that_looks_like_zlib() {
let data = raw_deflate_like_zlib(b"twenty-nine bytes of raw text");
// A zlib decoder alone rejects it.
assert!(ZlibDecoder::new(data.as_slice()).read_to_end(&mut Vec::new()).is_err());
let decoded = stream_decode(data, ContentEncoding::Deflate).await.unwrap();
assert_eq!(decoded, b"twenty-nine bytes of raw text");
// Read as zlib, this one is a stored block that starts decoding, and only fails later.
let both = *b"\x1d\0xxxxxxxxxxxxxxxxxxxxxxxxxxx";
let data = raw_deflate_like_zlib(&both);
let decoded = stream_decode(data, ContentEncoding::Deflate).await.unwrap();
assert_eq!(decoded, both);
}
#[tokio::test]
async fn streaming_deflate_decodes_zlib_that_looks_like_raw() {
for extra in [0, 10_000] {
let (data, expected) = zlib_like_raw_deflate(extra);
let decoded = stream_decode(data, ContentEncoding::Deflate).await.unwrap();
assert!(decoded == expected, "{extra} extra bytes");
}
}
/// However the reader reads ahead, the wrong reading is dropped and the right one kept.
#[tokio::test]
async fn streaming_deflate_retries_large_raw_whatever_the_chunking() {
use tokio::io::AsyncReadExt;
let (data, expected) = large_raw_like_zlib();
for capacity in [1, 8 * 1024, 256 * 1024] {
let mut out = Vec::new();
let reader = BufReader::with_capacity(capacity, std::io::Cursor::new(data.clone()));
streaming_decoder(reader, ContentEncoding::Deflate)
.read_to_end(&mut out)
.await
.unwrap();
assert!(out == expected, "read {capacity} bytes at a time");
}
}
#[tokio::test]
async fn streaming_deflate_still_fails_on_garbage() {
let garbage = b"\x78\x9c this is not deflate in any form".to_vec();
assert!(stream_decode(garbage, ContentEncoding::Deflate).await.is_err());
assert!(stream_decode(b"\xff".to_vec(), ContentEncoding::Deflate).await.is_err());
}
#[tokio::test]
async fn streaming_gzip_is_unchanged() {
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
encoder.write_all(DEFLATE_TEXT).unwrap();
let gzip = encoder.finish().unwrap();
assert_eq!(stream_decode(gzip, ContentEncoding::Gzip).await.unwrap(), DEFLATE_TEXT);
}
#[test]
fn test_decompress_brotli() {
// Compress some data with brotli
+63
View File
@@ -100,6 +100,11 @@ async fn route(req: Request<Incoming>) -> Result<Response<TestBody>, Infallible>
req.headers().get(HOST).map_or("none", |h| h.to_str().unwrap()),
))),
"/big" => Response::new(full(vec![b'x'; BIG_BODY])),
// No body: `Content-Length: 0`, or a stream with nothing in it, chunked on HTTP/1.1.
"/empty" => Response::new(full("")),
"/empty-stream" => {
Response::new(BodyExt::boxed(StreamBody::new(futures_util::stream::empty())))
}
"/late-end" => late_end(5),
// The stream ends with half the promised bytes, so the upstream resets it mid-body.
"/cut" => late_end(10),
@@ -1361,6 +1366,64 @@ async fn h2_large_bodies_pass_flow_control() {
}
}
/// A flow ends when its response does, not when a connection closes: the upstream answers at
/// once and keeps its connection open, as does the client, and the flow's time is the answer's.
#[tokio::test(flavor = "multi_thread")]
async fn keep_alive_connections_do_not_lengthen_flows() {
let h = Harness::start(1024);
let upstreams = [
("plain HTTP/1.1", spawn_upstream(None).await, false),
("HTTP/1.1", spawn_upstream(Some(h1_only(&h.pki.server))).await, true),
("HTTP/2", spawn_upstream(Some(h.pki.server.clone())).await, true),
];
// Every connection stays open to the end of the test, each reused across requests.
let h1_client = h.client();
let mut h2_clients = Vec::new();
for (upstream_name, upstream, tls) in upstreams {
let port = upstream.port();
let mut h2_client = if tls { Some(h.h2_client(port).await) } else { None };
for path in ["/empty", "/empty-stream", "/hello"] {
for client_version in ["HTTP/1.1", "HTTP/2"] {
let case = format!("{client_version} client, {upstream_name} upstream, {path}");
let scheme = if tls { "https" } else { "http" };
let url = format!("{scheme}://localhost:{port}{path}");
let body = match (client_version, &mut h2_client) {
("HTTP/1.1", _) => {
let resp = h1_client.post(&url).body("x=1").send().await.unwrap();
resp.bytes().await.unwrap()
}
(_, Some(client)) => {
let req = h2_request("POST", port, path, full("x=1"));
let resp = client.send_request(req).await.unwrap();
resp.into_body().collect().await.unwrap().to_bytes()
}
// HTTP/2 needs TLS here.
(_, None) => continue,
};
assert_eq!(body, if path == "/hello" { "hello" } else { "" }, "{case}");
let events = h.flow_events();
let version =
find!(events, ProxyEvent::RequestStart { http_version, .. } => http_version);
assert_eq!(version, client_version, "{case}");
let headers_ms =
find!(events, ProxyEvent::ResponseStart { elapsed_ms, .. } => *elapsed_ms);
let (end_ms, error, cancelled) = find!(events,
ProxyEvent::ResponseBodyComplete { elapsed_ms, error, cancelled, .. } =>
(*elapsed_ms, error.clone(), *cancelled));
assert_eq!((error, cancelled), (None, false), "{case}");
// Generous for a slow CI runner; a flow held to the connection's close would
// not end at all while every connection here stays open.
assert!(end_ms - headers_ms < 2_000, "{case}: headers {headers_ms}, end {end_ms}");
assert!(end_ms < 5_000, "{case}: took {end_ms} ms");
}
}
h2_clients.extend(h2_client);
}
drop((h1_client, h2_clients));
}
/// With HTTP/2 off, a client that offers both gets HTTP/1.1, from the next connection on.
#[tokio::test(flavor = "multi_thread")]
async fn http1_only_makes_h2_clients_fall_back() {
+1 -1
View File
@@ -145,7 +145,6 @@
"focus-trap-react": "^12.0.3",
"fuzzbunny": "^1.0.1",
"graphql": "^17.0.2",
"hexy": "^0.4.0",
"history": "^5.3.0",
"jotai": "^2.18.0",
"jotai-family": "^1.0.1",
@@ -15019,6 +15018,7 @@
"name": "@yaakapp-internal/ui",
"version": "1.0.0",
"dependencies": {
"hexy": "^0.4.0",
"jotai-family": "^1.0.1"
}
},
@@ -0,0 +1,45 @@
import { describe, expect, test } from "vitest";
import { contentTypeCharset, decodeBodyText, strictUtf8Text } from "./decodeBodyText";
const latin1 = (text: string) => Uint8Array.from(text, (c) => c.charCodeAt(0));
// "こんにちは、世界" in Shift_JIS
const shiftJis = new Uint8Array([
0x82, 0xb1, 0x82, 0xf1, 0x82, 0xc9, 0x82, 0xbf, 0x82, 0xcd, 0x81, 0x41, 0x90, 0xa2, 0x8a, 0x45,
]);
const french = "Accès refusé : vous n'avez pas l'autorisation d'accéder à cette page.";
describe("decodeBodyText", () => {
test("decodes in the charset the content type names", () => {
const body = latin1(french);
expect(decodeBodyText(body, "text/html; charset=iso-8859-1")).toBe(french);
expect(decodeBodyText(body, 'text/plain; CHARSET="ISO-8859-1"')).toBe(french);
expect(decodeBodyText(shiftJis, "text/plain; charset=Shift_JIS")).toBe("こんにちは、世界");
});
test("reads an unlabeled body that isn't UTF-8 as windows-1252", () => {
expect(decodeBodyText(latin1(french), "text/html")).toBe(french);
expect(decodeBodyText(latin1(french), null)).toBe(french);
expect(decodeBodyText(new Uint8Array([0x93, 0x68, 0x69, 0x94]), null)).toBe("“hi”");
});
test("reads UTF-8 as UTF-8, labeled or not", () => {
const utf8 = new TextEncoder().encode(`{"city":"Zürich","greeting":"こんにちは"}`);
const text = `{"city":"Zürich","greeting":"こんにちは"}`;
expect(decodeBodyText(utf8, null)).toBe(text);
expect(decodeBodyText(utf8, "application/json")).toBe(text);
expect(decodeBodyText(utf8, "application/json; charset=utf-8")).toBe(text);
// A body labeled UTF-8 stays UTF-8, with what isn't replaced, as before
expect(decodeBodyText(new Uint8Array([0x61, 0xff]), "text/plain; charset=utf-8")).toBe("a�");
});
test("treats a charset TextDecoder doesn't know as no charset", () => {
expect(contentTypeCharset("text/plain; charset=not-a-charset")).toBeNull();
expect(decodeBodyText(latin1(french), "text/plain; charset=not-a-charset")).toBe(french);
});
test("keeps a truncated UTF-8 body UTF-8 when it ends partway through a character", () => {
const cut = new TextEncoder().encode("abc€").subarray(0, 5);
expect(strictUtf8Text(cut)).toBeNull();
expect(decodeBodyText(cut, null, { truncated: true })).toBe("abc");
});
});
+44
View File
@@ -0,0 +1,44 @@
/**
* The charset a content type names, as `TextDecoder` calls it (`windows-1252` for
* `iso-8859-1`), or null when it names none or one `TextDecoder` doesn't know.
*/
export function contentTypeCharset(contentType: string | null): string | null {
const label = contentType?.match(/;\s*charset\s*=\s*"?([^";\s]+)"?/i)?.[1];
if (label == null) return null;
try {
return new TextDecoder(label).encoding;
} catch {
return null;
}
}
/**
* The bytes as UTF-8, or null when they aren't valid UTF-8. A truncated body may end partway
* through a character, so up to three trailing bytes are dropped before giving up on it.
*/
export function strictUtf8Text(bytes: Uint8Array, truncated = false): string | null {
const decoder = new TextDecoder("utf-8", { fatal: true });
const maxTrim = truncated ? Math.min(3, bytes.length) : 0;
for (let trim = 0; trim <= maxTrim; trim++) {
try {
return decoder.decode(bytes.subarray(0, bytes.length - trim));
} catch {
// Try again without the last byte
}
}
return null;
}
/**
* A body's text in the charset its content type names. Without one it is UTF-8, unless the
* bytes aren't valid UTF-8, when it is windows-1252, as browsers read unlabeled text.
*/
export function decodeBodyText(
bytes: Uint8Array,
contentType: string | null,
{ truncated = false }: { truncated?: boolean } = {},
): string {
const charset = contentTypeCharset(contentType);
if (charset != null) return new TextDecoder(charset).decode(bytes);
return strictUtf8Text(bytes, truncated) ?? new TextDecoder("windows-1252").decode(bytes);
}
+1
View File
@@ -1,4 +1,5 @@
export * from "./debounce";
export * from "./decodeBodyText";
export * from "./eagerDebounceAsync";
export * from "./formatSize";
export * from "./templateFunction";
+1
View File
@@ -16,6 +16,7 @@
"*.css"
],
"dependencies": {
"hexy": "^0.4.0",
"jotai-family": "^1.0.1"
}
}
@@ -0,0 +1,20 @@
import { hexy } from "hexy";
import { describe, expect, test, vi } from "vite-plus/test";
import { hexdump } from "./HexdumpViewer";
vi.mock("hexy", () => ({ hexy: vi.fn((data: number[]) => `dump of ${data.length} bytes`) }));
describe("hexdump", () => {
test("makes the hexdump of the same bytes once", () => {
const message = [0x01, 0x02, 0x03];
expect(hexdump(message)).toBe("dump of 3 bytes");
expect(hexdump(message)).toBe("dump of 3 bytes");
expect(hexy).toHaveBeenCalledTimes(1);
// Other bytes, even equal ones, get their own.
expect(hexdump([0x01, 0x02, 0x03])).toBe("dump of 3 bytes");
expect(hexy).toHaveBeenCalledTimes(2);
expect(hexdump([])).toBe("");
expect(hexy).toHaveBeenCalledTimes(2);
});
});
@@ -0,0 +1,50 @@
import { hexy } from "hexy";
import type { ReactNode } from "react";
import { useMemo } from "react";
import type { CodeEditorProps } from "../editor/CodeEditor";
import { LazyCodeEditor } from "../editor/LazyCodeEditor";
/** Hexdumps already made, by the bytes they were made of, for as long as those bytes live */
const hexdumps = new WeakMap<Uint8Array | number[], string>();
/**
* Bytes as a hexdump: offset, hex and ASCII columns, one line per 16 bytes. Made once for the
* same bytes, so a viewer and its copy action share one.
*/
export function hexdump(data: Uint8Array | number[]): string {
if (data.length === 0) return "";
let dump = hexdumps.get(data);
if (dump == null) {
dump = hexy(data);
hexdumps.set(data, dump);
}
return dump;
}
export interface HexdumpViewerProps {
data: Uint8Array | number[];
className?: string;
/** Renders the editor, for a host whose editor builds on CodeEditor */
renderEditor?: (props: CodeEditorProps) => ReactNode;
}
/** A read-only hexdump of binary data */
export function HexdumpViewer({
data,
className,
renderEditor = renderCodeEditor,
}: HexdumpViewerProps) {
const text = useMemo(() => hexdump(data), [data]);
return renderEditor({
language: "text",
defaultValue: text,
wrapLines: false,
readOnly: true,
stateKey: null,
className,
});
}
function renderCodeEditor(props: CodeEditorProps) {
return <LazyCodeEditor {...props} />;
}
@@ -89,6 +89,8 @@ export function TextViewer({
],
queryFn: () => (filterText ?? rejectFilter)(filter.appliedFilter ?? ""),
enabled: hasFilter && filter.appliedFilter != null,
// An expression that fails, fails every time, so the error shows at once.
retry: false,
});
const canFilter = hasFilter && (language === "json" || language === "xml" || language === "html");
@@ -195,6 +197,7 @@ export function TextViewer({
}
return formatText(text, language);
},
retry: false,
}).data;
if (formattedBody == null) {
return null;
+2
View File
@@ -90,6 +90,8 @@ export { Tree } from "./components/tree/Tree";
export type { TreeItemProps } from "./components/tree/TreeItem";
export { AudioViewer } from "./components/viewers/AudioViewer";
export { CsvViewer, CsvViewerInner } from "./components/viewers/CsvViewer";
export type { HexdumpViewerProps } from "./components/viewers/HexdumpViewer";
export { hexdump, HexdumpViewer } from "./components/viewers/HexdumpViewer";
export { ImageViewer } from "./components/viewers/ImageViewer";
export { JsonAttributeTree } from "./components/viewers/JsonAttributeTree";
export { JsonViewer } from "./components/viewers/JsonViewer";