mirror of
https://github.com/mountain-loop/yaak.git
synced 2026-11-17 22:21:26 +01:00
Body viewer and decode fixes (#812)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
87a30200f9
commit
4ad0533901
26 files changed
+1633
-91
No files matched your search
Generated
+1
@@ -11852,6 +11852,7 @@ dependencies = [
|
||||
"base64 0.23.1",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"encoding_rs",
|
||||
"flate2",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
import type { WebsocketEvent, WebsocketRequest } from "@yaakapp-internal/models";
|
||||
import {
|
||||
HStack,
|
||||
hexdump,
|
||||
HexdumpViewer,
|
||||
Icon,
|
||||
LoadingIcon,
|
||||
VStack,
|
||||
useStateWithDeps,
|
||||
languageFromContentType,
|
||||
} from "@yaakapp-internal/ui";
|
||||
import { hexy } from "hexy";
|
||||
import { useAtomValue } from "jotai";
|
||||
import { useMemo, useState } from "react";
|
||||
import { useFormatText } from "../hooks/useFormatText";
|
||||
@@ -19,6 +20,7 @@ import {
|
||||
} from "../hooks/usePinnedWebsocketConnection";
|
||||
import { Button } from "./core/Button";
|
||||
import { Editor } from "./core/Editor/LazyEditor";
|
||||
import { renderEditor } from "./core/Editor/renderEditor";
|
||||
import { type EventDetailAction, EventDetailHeader, EventViewer } from "./core/EventViewer";
|
||||
import { EventViewerRow } from "./core/EventViewerRow";
|
||||
import { HotkeyList } from "./core/HotkeyList";
|
||||
@@ -122,10 +124,10 @@ function WebsocketEventRow({
|
||||
messageType === "error"
|
||||
? "alert_triangle"
|
||||
: messageType === "close" || messageType === "open"
|
||||
? "info"
|
||||
: isServer
|
||||
? "arrow_big_down_dash"
|
||||
: "arrow_big_up_dash";
|
||||
? "info"
|
||||
: isServer
|
||||
? "arrow_big_down_dash"
|
||||
: "arrow_big_up_dash";
|
||||
|
||||
const content =
|
||||
messageType === "close" ? (
|
||||
@@ -172,7 +174,7 @@ function WebsocketEventDetail({
|
||||
}) {
|
||||
const message = useMemo(() => {
|
||||
if (hexDump) {
|
||||
return event.message ? hexy(event.message) : "";
|
||||
return event.message ? hexdump(event.message) : "";
|
||||
}
|
||||
return event.message ? new TextDecoder("utf-8").decode(Uint8Array.from(event.message)) : "";
|
||||
}, [event.message, hexDump]);
|
||||
@@ -232,6 +234,8 @@ function WebsocketEventDetail({
|
||||
</VStack>
|
||||
) : event.message.length === 0 ? (
|
||||
<EmptyStateText>No Content</EmptyStateText>
|
||||
) : hexDump ? (
|
||||
<HexdumpViewer data={event.message} renderEditor={renderEditor} />
|
||||
) : (
|
||||
<Editor
|
||||
language={language}
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
import type { HttpResponse } from "@yaakapp-internal/models";
|
||||
import { decodeBodyText } from "@yaakapp-internal/lib/decodeBodyText";
|
||||
import { getContentTypeFromHeaders } from "../lib/model_util";
|
||||
import { rpc } from "../lib/rpc";
|
||||
|
||||
export function useHttpRequestBody(response: HttpResponse | null) {
|
||||
@@ -27,6 +29,6 @@ export async function getRequestBodyText(response: HttpResponse | null) {
|
||||
}
|
||||
|
||||
const body = new Uint8Array(data);
|
||||
const bodyText = new TextDecoder("utf-8", { fatal: false }).decode(body);
|
||||
const bodyText = decodeBodyText(body, getContentTypeFromHeaders(response.requestHeaders));
|
||||
return { body, bodyText };
|
||||
}
|
||||
@@ -5,7 +5,6 @@ import type {
|
||||
HttpResponseEvent,
|
||||
HttpResponseHeader,
|
||||
} from "@yaakapp-internal/models";
|
||||
import { getMimeTypeFromContentType } from "./contentType";
|
||||
|
||||
export const BODY_TYPE_NONE = null;
|
||||
export const BODY_TYPE_GRAPHQL = "graphql";
|
||||
@@ -46,14 +45,6 @@ export function getContentTypeFromHeaders(headers: HttpResponseHeader[] | null):
|
||||
return headers?.find((h) => h.name.toLowerCase() === "content-type")?.value ?? null;
|
||||
}
|
||||
|
||||
export function getCharsetFromContentType(headers: HttpResponseHeader[]): string | null {
|
||||
const contentType = getContentTypeFromHeaders(headers);
|
||||
if (contentType == null) return null;
|
||||
|
||||
const mimeType = getMimeTypeFromContentType(contentType);
|
||||
return mimeType.parameters.get("charset") ?? null;
|
||||
}
|
||||
|
||||
export function isBaseEnvironment(environment: Environment): boolean {
|
||||
return environment.parentModel === "workspace";
|
||||
}
|
||||
|
||||
@@ -3,7 +3,9 @@ import type { FilterResponse } from "@yaakapp-internal/plugins";
|
||||
import type { ServerSentEvent, SseSummary } from "@yaakapp-internal/sse";
|
||||
import { candidateJsonPayloadsFromSseText, computeSseSummary } from "@yaakapp-internal/sse";
|
||||
import { rpc } from "./rpc";
|
||||
import { decodeBodyText } from "@yaakapp-internal/lib/decodeBodyText";
|
||||
import { platform } from "@yaakapp-internal/platform";
|
||||
import { getContentTypeFromHeaders } from "./model_util";
|
||||
|
||||
/**
|
||||
* Reading a response body means naming the response, never the file it lives
|
||||
@@ -74,5 +76,6 @@ export async function getResponseBodyBytes(
|
||||
|
||||
async function getResponseBodyDecoded(response: HttpResponse): Promise<string | null> {
|
||||
const bytes = await getResponseBodyBytes(response);
|
||||
return bytes == null ? null : new TextDecoder("utf-8").decode(bytes);
|
||||
if (bytes == null) return null;
|
||||
return decodeBodyText(bytes, getContentTypeFromHeaders(response.headers));
|
||||
}
|
||||
@@ -48,7 +48,6 @@
|
||||
"focus-trap-react": "^12.0.3",
|
||||
"fuzzbunny": "^1.0.1",
|
||||
"graphql": "^17.0.2",
|
||||
"hexy": "^0.4.0",
|
||||
"history": "^5.3.0",
|
||||
"jotai": "^2.18.0",
|
||||
"jotai-family": "^1.0.1",
|
||||
|
||||
@@ -3,7 +3,10 @@ import { formatSize } from "@yaakapp-internal/lib/formatSize";
|
||||
import type { Flow, FlowBody } from "@yaakapp-internal/proxy-lib";
|
||||
import {
|
||||
Banner,
|
||||
Button,
|
||||
CsvViewer,
|
||||
HexdumpViewer,
|
||||
Icon,
|
||||
ImageViewer,
|
||||
languageFromContentType,
|
||||
LoadingIcon,
|
||||
@@ -14,16 +17,17 @@ import {
|
||||
} from "@yaakapp-internal/ui";
|
||||
import type { CodeEditorProps } from "@yaakapp-internal/ui/editor";
|
||||
import type { ReactNode } from "react";
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import { useCallback, useEffect, useMemo, useState } from "react";
|
||||
import {
|
||||
base64ToBytes,
|
||||
bytesToText,
|
||||
bodyAsText,
|
||||
bodyImageType,
|
||||
filterText,
|
||||
findHeader,
|
||||
flowBodyQueryOptions,
|
||||
formatText,
|
||||
mimeEssence,
|
||||
} from "../lib/body";
|
||||
import { rpc } from "../lib/rpc";
|
||||
import { useSaveFlowBody } from "../lib/flowActions";
|
||||
|
||||
interface Props {
|
||||
flow: Flow;
|
||||
@@ -32,18 +36,7 @@ interface Props {
|
||||
}
|
||||
|
||||
export function useFlowBody(flow: Flow, which: FlowBody, enabled = true) {
|
||||
return useQuery({
|
||||
enabled,
|
||||
queryKey: ["get_flow_body", flow.id, which, flow.updatedAt],
|
||||
queryFn: async () => {
|
||||
const res = await rpc("get_flow_body", { flowId: flow.id, which });
|
||||
return {
|
||||
bytes: base64ToBytes(res.data),
|
||||
truncated: res.truncated,
|
||||
storedSize: res.storedSize,
|
||||
};
|
||||
},
|
||||
});
|
||||
return useQuery({ ...flowBodyQueryOptions(flow, which), enabled });
|
||||
}
|
||||
|
||||
export function FlowBodyViewer({ flow, which, pretty }: Props) {
|
||||
@@ -60,8 +53,8 @@ export function FlowBodyViewer({ flow, which, pretty }: Props) {
|
||||
return <Banner color="danger">{body.error.message}</Banner>;
|
||||
}
|
||||
|
||||
const { bytes, truncated, storedSize } = body.data;
|
||||
if (bytes.length === 0) {
|
||||
const { bytes, truncated, storedSize, decodeError } = body.data;
|
||||
if (bytes.length === 0 && decodeError == null) {
|
||||
return <EmptyState>{which === "request" ? "No request body" : "Empty"}</EmptyState>;
|
||||
}
|
||||
|
||||
@@ -70,12 +63,16 @@ export function FlowBodyViewer({ flow, which, pretty }: Props) {
|
||||
const mime = mimeEssence(contentType) ?? "";
|
||||
const wireSize = which === "request" ? flow.requestSize : flow.responseSize;
|
||||
const wireTruncated = which === "request" ? flow.requestTruncated : flow.responseTruncated;
|
||||
// An image by its bytes or type, else text whatever the content type says, so a page sent with
|
||||
// an odd one still reads: `viewerText`'s rule.
|
||||
const imageType = decodeError == null ? bodyImageType(bytes, truncated, contentType) : null;
|
||||
const text =
|
||||
decodeError == null && imageType == null ? bodyAsText(bytes, truncated, contentType) : null;
|
||||
|
||||
let viewer: ReactNode;
|
||||
if (mime.startsWith("image/svg")) {
|
||||
viewer = <SvgViewer text={bytesToText(bytes, truncated) ?? ""} />;
|
||||
} else if (mime.startsWith("image/")) {
|
||||
viewer = <ImageBody bytes={bytes} mime={mime} />;
|
||||
if (decodeError != null) {
|
||||
// What failed to decode is as it was on the wire, compressed, so never text.
|
||||
viewer = <BinaryBody flow={flow} which={which} bytes={bytes} />;
|
||||
} else if (mime.startsWith("multipart/") && pretty) {
|
||||
const boundary = contentType?.match(/;\s*boundary="?([^";]+)"?/i)?.[1] ?? "unknown";
|
||||
viewer = (
|
||||
@@ -86,14 +83,17 @@ export function FlowBodyViewer({ flow, which, pretty }: Props) {
|
||||
textViewerProps={{ formatText }}
|
||||
/>
|
||||
);
|
||||
} else if (imageType != null) {
|
||||
viewer = <ImageBody bytes={bytes} mime={imageType} />;
|
||||
} else if (text == null) {
|
||||
viewer = <BinaryBody flow={flow} which={which} bytes={bytes} />;
|
||||
} else if (mime.startsWith("image/svg")) {
|
||||
viewer = <SvgViewer text={text} />;
|
||||
} else if (/csv|tab-separated/.test(mime) && pretty) {
|
||||
viewer = <CsvViewer text={bytesToText(bytes, truncated)} className="pb-2" />;
|
||||
viewer = <CsvViewer text={text} className="pb-2" />;
|
||||
} else {
|
||||
const text = bytesToText(bytes, truncated);
|
||||
const language = languageFromContentType(contentType, text);
|
||||
if (text == null) {
|
||||
viewer = <EmptyState>Binary body, {formatSize(bytes.length)}</EmptyState>;
|
||||
} else if (language === "html" && pretty && which === "response") {
|
||||
if (language === "html" && pretty && which === "response") {
|
||||
viewer = <WebPageViewer html={text} baseUrl={flow.url} />;
|
||||
} else {
|
||||
viewer = (
|
||||
@@ -102,16 +102,60 @@ export function FlowBodyViewer({ flow, which, pretty }: Props) {
|
||||
}
|
||||
}
|
||||
|
||||
if (!truncated) return viewer;
|
||||
if (!truncated && decodeError == null) return viewer;
|
||||
|
||||
return (
|
||||
<div className="h-full grid grid-rows-[auto_minmax(0,1fr)] gap-2">
|
||||
<Banner color="notice" className="text-sm">
|
||||
{wireTruncated
|
||||
? `Showing the first ${formatSize(storedSize)} of ${formatSize(wireSize)}, the capture limit. The app received all of it.`
|
||||
: `Showing the first ${formatSize(bytes.length)} of the decoded body, the capture limit. The app received all of it.`}
|
||||
</Banner>
|
||||
<div className="min-h-0">{viewer}</div>
|
||||
<div className="h-full grid grid-rows-[auto_auto_minmax(0,1fr)]">
|
||||
{decodeError != null && (
|
||||
<Banner color="danger" className="text-sm mb-2">
|
||||
{decodeError}. Showing the body as it was on the wire.
|
||||
</Banner>
|
||||
)}
|
||||
{truncated && (
|
||||
<Banner color="notice" className="text-sm mb-2">
|
||||
{wireTruncated || decodeError != null
|
||||
? `Showing the first ${formatSize(storedSize)} of ${formatSize(wireSize)}, the capture limit. The app received all of it.`
|
||||
: `Showing the first ${formatSize(bytes.length)} of the decoded body, the capture limit. The app received all of it.`}
|
||||
</Banner>
|
||||
)}
|
||||
<div className="min-h-0 row-start-3">{viewer}</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/** Bytes shown in the hexdump; the rest are in the saved file */
|
||||
const HEXDUMP_LIMIT = 256 * 1024;
|
||||
|
||||
/** A body that isn't text: a hexdump, and a way to save it */
|
||||
function BinaryBody({
|
||||
flow,
|
||||
which,
|
||||
bytes,
|
||||
}: {
|
||||
flow: Flow;
|
||||
which: FlowBody;
|
||||
bytes: Uint8Array<ArrayBuffer>;
|
||||
}) {
|
||||
const saveBody = useSaveFlowBody();
|
||||
const shown = useMemo(() => bytes.subarray(0, HEXDUMP_LIMIT), [bytes]);
|
||||
|
||||
return (
|
||||
<div className="h-full grid grid-rows-[auto_minmax(0,1fr)] gap-1">
|
||||
<div className="flex items-center justify-between gap-2 text-sm text-text-subtle">
|
||||
<span>
|
||||
Binary body, {formatSize(bytes.length)}
|
||||
{shown.length < bytes.length && `, first ${formatSize(shown.length)} shown`}
|
||||
</span>
|
||||
<Button
|
||||
size="2xs"
|
||||
variant="border"
|
||||
leftSlot={<Icon icon="download" size="xs" />}
|
||||
onClick={() => saveBody(flow, which)}
|
||||
>
|
||||
Save to File
|
||||
</Button>
|
||||
</div>
|
||||
<HexdumpViewer data={shown} className="-mr-2 bg-surface" />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,5 +1,17 @@
|
||||
import { describe, expect, test } from "vite-plus/test";
|
||||
import { bytesToText } from "./body";
|
||||
import { QueryClient, QueryObserver } from "@tanstack/react-query";
|
||||
import type { Flow } from "@yaakapp-internal/proxy-lib";
|
||||
import { describe, expect, test, vi } from "vite-plus/test";
|
||||
import {
|
||||
bodyAsText,
|
||||
bodyFileName,
|
||||
bodyImageType,
|
||||
bytesToText,
|
||||
flowBodyQueryOptions,
|
||||
viewerText,
|
||||
} from "./body";
|
||||
import { rpc } from "./rpc";
|
||||
|
||||
vi.mock("./rpc", () => ({ rpc: vi.fn() }));
|
||||
|
||||
const encode = (text: string) => new TextEncoder().encode(text);
|
||||
|
||||
@@ -18,3 +30,198 @@ describe("bytesToText", () => {
|
||||
expect(bytesToText(cut, true)).toBe("abc");
|
||||
});
|
||||
});
|
||||
|
||||
describe("bodyAsText", () => {
|
||||
test("reads UTF-8 as text", () => {
|
||||
expect(bodyAsText(encode("<h1>Forbidden</h1>\n\tcafé"))).toBe("<h1>Forbidden</h1>\n\tcafé");
|
||||
});
|
||||
|
||||
test("reads unlabeled Latin-1 text as text, as windows-1252", () => {
|
||||
const page = "<p>Accès refusé : vous n'avez pas l'autorisation d'accéder à cette page.</p>";
|
||||
const latin1 = Uint8Array.from(page, (c) => c.charCodeAt(0));
|
||||
expect(bodyAsText(latin1)).toBe(page);
|
||||
});
|
||||
|
||||
test("reads bytes UTF-8 can't read as binary, though windows-1252 reads every one", () => {
|
||||
let seed = 7;
|
||||
const random = Uint8Array.from({ length: 600 }, () => {
|
||||
seed = (seed * 1103515245 + 12345) & 0x7fffffff;
|
||||
return seed >> 16;
|
||||
});
|
||||
expect(bodyAsText(random)).toBeNull();
|
||||
// All high bytes: letters and symbols in windows-1252, none of them UTF-8
|
||||
const high = Uint8Array.from({ length: 600 }, (_, i) => 0xa0 + (i % 96));
|
||||
expect(bodyAsText(high)).toBeNull();
|
||||
expect(bodyAsText(high, false, "text/plain")).toBeNull();
|
||||
});
|
||||
|
||||
test("reads protobuf and other binary as binary, even when it is valid UTF-8", () => {
|
||||
// Field tags, lengths and varints: control characters between short strings.
|
||||
const protobuf = new Uint8Array([
|
||||
0x0a, 0x04, 0x6b, 0x65, 0x79, 0x31, 0x12, 0x02, 0x08, 0x01, 0x1a, 0x03, 0x10, 0x96, 0x01,
|
||||
0x22, 0x00, 0x2a, 0x02, 0x18, 0x07,
|
||||
]);
|
||||
expect(bodyAsText(protobuf)).toBeNull();
|
||||
expect(bodyAsText(new Uint8Array([0x01, 0x02, 0x03, 0x04, 0x41, 0x42]))).toBeNull();
|
||||
expect(bodyAsText(new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("viewerText", () => {
|
||||
// A PNG signature, then a tEXt chunk of printable metadata far longer than the image
|
||||
const png = new Uint8Array([
|
||||
0x89,
|
||||
...encode("PNG\r\n"),
|
||||
0x1a,
|
||||
0x0a,
|
||||
0x00,
|
||||
0x00,
|
||||
0x12,
|
||||
0xc0,
|
||||
...encode(`tEXtDescription\0${"A long printable description. ".repeat(160)}`),
|
||||
]);
|
||||
|
||||
test("keeps an image with printable metadata an image", () => {
|
||||
expect(bodyAsText(png)).not.toBeNull();
|
||||
expect(viewerText(png, false, "image/png")).toBeNull();
|
||||
});
|
||||
|
||||
test("reads text sent as an image as text, and other types text first", () => {
|
||||
expect(viewerText(encode("<h1>Forbidden</h1>"), false, "image/png")).toBe("<h1>Forbidden</h1>");
|
||||
expect(bodyImageType(encode("<h1>Forbidden</h1>"), false, "image/png")).toBeNull();
|
||||
expect(viewerText(encode("<svg/>"), false, "image/svg+xml")).toBe("<svg/>");
|
||||
expect(bodyImageType(encode("<svg/>"), false, "image/svg+xml")).toBeNull();
|
||||
const bmwText = encode("BMW sells cars.\n");
|
||||
expect(viewerText(bmwText, false, "text/plain")).toBe("BMW sells cars.\n");
|
||||
});
|
||||
|
||||
// A 1×1 black 24-bit BMP: every byte below 0x80, so valid UTF-8
|
||||
// File header, then the 40-byte DIB header, then one pixel padded to four bytes
|
||||
// prettier-ignore
|
||||
const bmp = new Uint8Array([
|
||||
0x42, 0x4d, 58, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0,
|
||||
40, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 24, 0, 0, 0, 0, 0, 4, 0, 0, 0,
|
||||
0x13, 0x0b, 0, 0, 0x13, 0x0b, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0,
|
||||
]);
|
||||
|
||||
test("keeps an image that is valid UTF-8 an image", () => {
|
||||
expect(bytesToText(bmp)).not.toBeNull();
|
||||
expect(bodyImageType(bmp, false, "image/bmp")).toBe("image/bmp");
|
||||
expect(viewerText(bmp, false, "image/bmp")).toBeNull();
|
||||
});
|
||||
|
||||
test("previews an image by its signature whatever its content type", () => {
|
||||
expect(bodyImageType(bmp, false, "application/octet-stream")).toBe("image/bmp");
|
||||
expect(bodyImageType(png, false, "application/octet-stream")).toBe("image/png");
|
||||
expect(bodyImageType(png, false, null)).toBe("image/png");
|
||||
expect(viewerText(png, false, "application/octet-stream")).toBeNull();
|
||||
const gif = encode("GIF89a\x01\0\x01\0");
|
||||
expect(bodyImageType(gif, false, "text/plain")).toBe("image/gif");
|
||||
});
|
||||
|
||||
test("previews formats not every webview draws only when their content type says so", () => {
|
||||
const tiff = new Uint8Array([0x49, 0x49, 0x2a, 0x00, 0x08, 0x00, 0x00, 0x00, 0xff, 0x80]);
|
||||
const heic = new Uint8Array([0, 0, 0, 0x18, ...encode("ftypheic"), 0, 0, 0, 0, 0xff, 0x80]);
|
||||
for (const image of [tiff, heic]) {
|
||||
expect(bodyImageType(image, false, "application/octet-stream")).toBeNull();
|
||||
expect(viewerText(image, false, "application/octet-stream")).toBeNull();
|
||||
}
|
||||
expect(bodyImageType(tiff, false, "image/tiff")).toBe("image/tiff");
|
||||
expect(bodyImageType(heic, false, "image/heic")).toBe("image/heic");
|
||||
});
|
||||
|
||||
test("reads text in the charset its content type names", () => {
|
||||
const page = "Accès refusé : vous n'avez pas l'autorisation d'accéder à cette page.";
|
||||
const latin1 = Uint8Array.from(page, (c) => c.charCodeAt(0));
|
||||
expect(viewerText(latin1, false, "text/html; charset=iso-8859-1")).toBe(page);
|
||||
// "こんにちは、世界" in Shift_JIS
|
||||
const shiftJis = new Uint8Array([
|
||||
0x82, 0xb1, 0x82, 0xf1, 0x82, 0xc9, 0x82, 0xbf, 0x82, 0xcd, 0x81, 0x41, 0x90, 0xa2, 0x8a,
|
||||
0x45,
|
||||
]);
|
||||
expect(viewerText(shiftJis, false, "text/plain; charset=shift_jis")).toBe("こんにちは、世界");
|
||||
});
|
||||
|
||||
test("reads unlabeled Latin-1 as windows-1252, and UTF-8 as UTF-8", () => {
|
||||
const page = "Accès refusé : vous n'avez pas l'autorisation d'accéder à cette page.";
|
||||
const latin1 = Uint8Array.from(page, (c) => c.charCodeAt(0));
|
||||
expect(viewerText(latin1, false, "application/octet-stream")).toBe(page);
|
||||
expect(viewerText(encode(page), false, "application/octet-stream")).toBe(page);
|
||||
expect(viewerText(encode(page), false, "text/plain; charset=utf-8")).toBe(page);
|
||||
});
|
||||
});
|
||||
|
||||
const flow = (url: string, contentType: string | null) =>
|
||||
({
|
||||
id: "fl_1",
|
||||
url,
|
||||
updatedAt: "2026-11-04T00:00:00",
|
||||
requestHeaders: contentType == null ? [] : [{ name: "Content-Type", value: contentType }],
|
||||
responseHeaders: contentType == null ? [] : [{ name: "content-type", value: contentType }],
|
||||
}) as unknown as Flow;
|
||||
|
||||
describe("bodyFileName", () => {
|
||||
test("uses the URL's file name for a response", () => {
|
||||
expect(bodyFileName(flow("https://a.test/img/logo%20big.png?v=2", null), "response")).toBe(
|
||||
"logo big.png",
|
||||
);
|
||||
});
|
||||
|
||||
test("names a body from its content type otherwise", () => {
|
||||
const protobuf = flow("https://a.test/v1:GetModels", "application/x-protobuf");
|
||||
expect(bodyFileName(protobuf, "request")).toBe("request.protobuf");
|
||||
expect(bodyFileName(protobuf, "response")).toBe("response.protobuf");
|
||||
expect(bodyFileName(flow("https://a.test/", "text/plain; charset=utf-8"), "response")).toBe(
|
||||
"response.txt",
|
||||
);
|
||||
expect(bodyFileName(flow("https://a.test/api", "application/problem+json"), "response")).toBe(
|
||||
"response.json",
|
||||
);
|
||||
expect(bodyFileName(flow("https://a.test/data.json", null), "request")).toBe("request.bin");
|
||||
expect(bodyFileName(flow("not a url", "application/octet-stream"), "response")).toBe(
|
||||
"response.bin",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("flowBodyQueryOptions", () => {
|
||||
test("shows a body that fails to read at once, without retrying", async () => {
|
||||
vi.mocked(rpc).mockRejectedValue(new Error("No flow fl_1"));
|
||||
// React Query retries three times in a browser, and not at all where there is no window.
|
||||
const client = new QueryClient({ defaultOptions: { queries: { retry: 3 } } });
|
||||
const observer = new QueryObserver(
|
||||
client,
|
||||
flowBodyQueryOptions(flow("https://a.test/", null), "response"),
|
||||
);
|
||||
const failed = new Promise<string>((resolve) => {
|
||||
const unsubscribe = observer.subscribe((result) => {
|
||||
if (result.status === "error") {
|
||||
unsubscribe();
|
||||
resolve(result.error.message);
|
||||
}
|
||||
});
|
||||
});
|
||||
// React Query's default retries would keep it pending for seconds.
|
||||
const timeout = new Promise<string>((resolve) => setTimeout(() => resolve("pending"), 500));
|
||||
expect(await Promise.race([failed, timeout])).toBe("No flow fl_1");
|
||||
expect(rpc).toHaveBeenCalledTimes(1);
|
||||
client.clear();
|
||||
});
|
||||
|
||||
test("passes on why a body failed to decode, with its bytes as they were", async () => {
|
||||
vi.mocked(rpc).mockResolvedValue({
|
||||
data: btoa("\x78\x9c\x01"),
|
||||
truncated: false,
|
||||
storedSize: 3,
|
||||
decodeError: "Failed to decode Deflate body: deflate decompression error",
|
||||
});
|
||||
const client = new QueryClient();
|
||||
const body = await client.fetchQuery(
|
||||
flowBodyQueryOptions(flow("https://a.test/", null), "request"),
|
||||
);
|
||||
expect(Array.from(body.bytes)).toEqual([0x78, 0x9c, 0x01]);
|
||||
expect(body.decodeError).toBe("Failed to decode Deflate body: deflate decompression error");
|
||||
client.clear();
|
||||
});
|
||||
});
|
||||
+146
-10
@@ -1,8 +1,32 @@
|
||||
import type { FilterLanguage, FlowBody } from "@yaakapp-internal/proxy-lib";
|
||||
import { queryOptions } from "@tanstack/react-query";
|
||||
import {
|
||||
contentTypeCharset,
|
||||
decodeBodyText,
|
||||
strictUtf8Text,
|
||||
} from "@yaakapp-internal/lib/decodeBodyText";
|
||||
import type { FilterLanguage, Flow, FlowBody } from "@yaakapp-internal/proxy-lib";
|
||||
import type { CodeEditorProps } from "@yaakapp-internal/ui/editor";
|
||||
import vkBeautify from "vkbeautify";
|
||||
import { rpc } from "./rpc";
|
||||
|
||||
/** A flow's body for the viewer: decoded, or as it was on the wire when `decodeError` is set */
|
||||
export function flowBodyQueryOptions(flow: Flow, which: FlowBody) {
|
||||
return queryOptions({
|
||||
queryKey: ["get_flow_body", flow.id, which, flow.updatedAt],
|
||||
queryFn: async () => {
|
||||
const res = await rpc("get_flow_body", { flowId: flow.id, which });
|
||||
return {
|
||||
bytes: base64ToBytes(res.data),
|
||||
truncated: res.truncated,
|
||||
storedSize: res.storedSize,
|
||||
decodeError: res.decodeError ?? null,
|
||||
};
|
||||
},
|
||||
// Reading a stored body fails the same way every time, so the error shows at once.
|
||||
retry: false,
|
||||
});
|
||||
}
|
||||
|
||||
export function base64ToBytes(data: string): Uint8Array<ArrayBuffer> {
|
||||
const binary = atob(data);
|
||||
const bytes = new Uint8Array(binary.length);
|
||||
@@ -16,19 +40,104 @@ export function base64ToBytes(data: string): Uint8Array<ArrayBuffer> {
|
||||
* The text of a body, or null when it isn't valid UTF-8. A truncated body may end partway
|
||||
* through a character, so up to three trailing bytes are dropped before giving up on it.
|
||||
*/
|
||||
export function bytesToText(bytes: Uint8Array, truncated = false): string | null {
|
||||
const decoder = new TextDecoder("utf-8", { fatal: true });
|
||||
const maxTrim = truncated ? Math.min(3, bytes.length) : 0;
|
||||
for (let trim = 0; trim <= maxTrim; trim++) {
|
||||
try {
|
||||
return decoder.decode(bytes.subarray(0, bytes.length - trim));
|
||||
} catch {
|
||||
// Try again without the last byte
|
||||
}
|
||||
export const bytesToText = strictUtf8Text;
|
||||
|
||||
/** Characters looked at to tell text from binary */
|
||||
const TEXT_SAMPLE = 64 * 1024;
|
||||
|
||||
/** The share of those that may be control characters or undecodable bytes in text */
|
||||
const MAX_UNPRINTABLE = 0.1;
|
||||
|
||||
/**
|
||||
* The body as text, whatever its content type says, or null when it is binary. It is decoded in
|
||||
* its content type's charset, or else as UTF-8, or windows-1252 when it isn't valid UTF-8. It is
|
||||
* binary when it is full of control characters, as protobuf is, or, without a charset, of bytes
|
||||
* UTF-8 can't read: windows-1252 reads every byte as something, compressed data included.
|
||||
*/
|
||||
export function bodyAsText(
|
||||
bytes: Uint8Array,
|
||||
truncated = false,
|
||||
contentType: string | null = null,
|
||||
): string | null {
|
||||
const text = decodeBodyText(bytes, contentType, { truncated });
|
||||
const judged =
|
||||
contentTypeCharset(contentType) != null
|
||||
? text
|
||||
: (strictUtf8Text(bytes, truncated) ?? new TextDecoder("utf-8").decode(bytes));
|
||||
let total = 0;
|
||||
let unprintable = 0;
|
||||
for (const char of judged) {
|
||||
if (total === TEXT_SAMPLE) break;
|
||||
total++;
|
||||
const code = char.codePointAt(0) ?? 0;
|
||||
const control = code < 0x20 && code !== 0x09 && code !== 0x0a && code !== 0x0d;
|
||||
if (control || code === 0x7f || code === 0xfffd) unprintable++;
|
||||
}
|
||||
return unprintable <= total * MAX_UNPRINTABLE ? text : null;
|
||||
}
|
||||
|
||||
const ascii = (bytes: Uint8Array, start: number, text: string) =>
|
||||
Array.from(text).every((c, i) => bytes[start + i] === c.charCodeAt(0));
|
||||
|
||||
/**
|
||||
* The image type a body's first bytes name, or null when they name none. Only types every
|
||||
* platform's webview draws, so a body sniffed as an image always previews: TIFF, HEIC and AVIF
|
||||
* preview only when their content type says so.
|
||||
*/
|
||||
export function imageSignature(bytes: Uint8Array): string | null {
|
||||
const b = bytes;
|
||||
if (b.length < 4) return null;
|
||||
if (b[0] === 0x89 && ascii(b, 1, "PNG\r\n\x1a\n")) return "image/png";
|
||||
if (b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff) return "image/jpeg";
|
||||
if (ascii(b, 0, "GIF87a") || ascii(b, 0, "GIF89a")) return "image/gif";
|
||||
if (ascii(b, 0, "RIFF") && ascii(b, 8, "WEBP")) return "image/webp";
|
||||
if (ascii(b, 0, "\0\0\x01\0") && b.length >= 6 && (b[4] ?? 0) + (b[5] ?? 0) > 0) {
|
||||
return "image/x-icon";
|
||||
}
|
||||
// "BM" alone starts plenty of text, so the reserved bytes and DIB header size must fit too.
|
||||
const dibSize = (b[14] ?? 0) | ((b[15] ?? 0) << 8) | ((b[16] ?? 0) << 16) | ((b[17] ?? 0) << 24);
|
||||
if (
|
||||
ascii(b, 0, "BM") &&
|
||||
ascii(b, 6, "\0\0\0\0") &&
|
||||
[12, 40, 52, 56, 64, 108, 124].includes(dibSize)
|
||||
) {
|
||||
return "image/bmp";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The image type to preview a body as, or null when it isn't one: what its first bytes name,
|
||||
* whatever its content type, or else an `image/*` content type other than SVG, unless the body
|
||||
* is genuinely text, as an error page sent with the image's type is.
|
||||
*/
|
||||
export function bodyImageType(
|
||||
bytes: Uint8Array,
|
||||
truncated: boolean,
|
||||
contentType: string | null,
|
||||
): string | null {
|
||||
const signed = imageSignature(bytes);
|
||||
if (signed != null) return signed;
|
||||
const mime = mimeEssence(contentType) ?? "";
|
||||
if (!mime.startsWith("image/") || mime.startsWith("image/svg")) return null;
|
||||
const genuineText =
|
||||
strictUtf8Text(bytes, truncated) != null && bodyAsText(bytes, truncated, contentType) != null;
|
||||
return genuineText ? null : mime;
|
||||
}
|
||||
|
||||
/**
|
||||
* The body as the viewer reads it: text first, whatever its content type, except an image, which
|
||||
* is never text.
|
||||
*/
|
||||
export function viewerText(
|
||||
bytes: Uint8Array,
|
||||
truncated: boolean,
|
||||
contentType: string | null,
|
||||
): string | null {
|
||||
if (bodyImageType(bytes, truncated, contentType) != null) return null;
|
||||
return bodyAsText(bytes, truncated, contentType);
|
||||
}
|
||||
|
||||
export async function formatText(
|
||||
text: string,
|
||||
language: CodeEditorProps["language"],
|
||||
@@ -81,6 +190,33 @@ export function shortContentType(contentType: string | null): string | null {
|
||||
return suffix.replace(/^x-/, "");
|
||||
}
|
||||
|
||||
/** Extensions for media types whose subtype isn't one */
|
||||
const EXTENSIONS: Record<string, string> = {
|
||||
"application/javascript": "js",
|
||||
"application/octet-stream": "bin",
|
||||
"image/jpeg": "jpg",
|
||||
"image/svg+xml": "svg",
|
||||
"text/javascript": "js",
|
||||
"text/markdown": "md",
|
||||
"text/plain": "txt",
|
||||
};
|
||||
|
||||
/** A name to offer for a saved body: the URL's file name, or one from the content type */
|
||||
export function bodyFileName(flow: Flow, which: FlowBody): string {
|
||||
let last = "";
|
||||
try {
|
||||
last = decodeURIComponent(new URL(flow.url).pathname.split("/").pop() ?? "");
|
||||
} catch {
|
||||
// Not a URL, or not encoded properly
|
||||
}
|
||||
if (which === "response" && /^[^\\/:*?"<>|]+\.[a-z0-9]+$/i.test(last)) return last;
|
||||
const headers = which === "request" ? flow.requestHeaders : flow.responseHeaders;
|
||||
const contentType = findHeader(headers, "content-type");
|
||||
const essence = mimeEssence(contentType);
|
||||
const extension = (essence == null ? null : EXTENSIONS[essence]) ?? shortContentType(contentType);
|
||||
return `${which}.${extension?.replace(/[^a-z0-9-]/gi, "") || "bin"}`;
|
||||
}
|
||||
|
||||
export function findHeader(headers: { name: string; value: string }[], name: string) {
|
||||
const lower = name.toLowerCase();
|
||||
return headers.find((h) => h.name.toLowerCase() === lower)?.value ?? null;
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import type { ExportHarRequest, Flow, FlowBody } from "@yaakapp-internal/proxy-lib";
|
||||
import type { DropdownItem } from "@yaakapp-internal/ui";
|
||||
import { save } from "@tauri-apps/plugin-dialog";
|
||||
import { formatSize } from "@yaakapp-internal/lib/formatSize";
|
||||
import { atom, useSetAtom } from "jotai";
|
||||
import { useCallback } from "react";
|
||||
import { base64ToBytes } from "./body";
|
||||
import { base64ToBytes, bodyFileName } from "./body";
|
||||
import { curlBodyText, toCurl } from "./curl";
|
||||
import { rpc } from "./rpc";
|
||||
import { useShowToast } from "./toast";
|
||||
@@ -14,10 +15,35 @@ export const saveDialogFlowAtom = atom<Flow | null>(null);
|
||||
/** A body as text, decoding what is not UTF-8 lossily, or null when there is none */
|
||||
async function bodyText(flow: Flow, which: FlowBody): Promise<string | null> {
|
||||
const res = await rpc("get_flow_body", { flowId: flow.id, which });
|
||||
if (res.decodeError != null) throw new Error(res.decodeError);
|
||||
const bytes = base64ToBytes(res.data);
|
||||
return bytes.length === 0 ? null : new TextDecoder().decode(bytes);
|
||||
}
|
||||
|
||||
/** Writes a flow's body to a file the person picks, as the viewer shows it */
|
||||
export function useSaveFlowBody(): (flow: Flow, which: FlowBody) => Promise<void> {
|
||||
const showToast = useShowToast();
|
||||
return useCallback(
|
||||
async (flow: Flow, which: FlowBody) => {
|
||||
try {
|
||||
const path = await save({ defaultPath: bodyFileName(flow, which) });
|
||||
if (path == null) return;
|
||||
const saved = await rpc("save_flow_body", { flowId: flow.id, which, path });
|
||||
const notes = [
|
||||
saved.raw ? "as it was on the wire, since it failed to decode" : null,
|
||||
saved.truncated ? "cut at the capture limit" : null,
|
||||
].filter((n) => n != null);
|
||||
showToast({
|
||||
message: `Saved ${formatSize(saved.size)}${notes.length > 0 ? `, ${notes.join(", ")}` : ""}`,
|
||||
});
|
||||
} catch (err) {
|
||||
showToast({ message: `Failed to save the ${which} body: ${String(err)}`, color: "danger" });
|
||||
}
|
||||
},
|
||||
[showToast],
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes flows to a HAR file the person picks: `ids`, or every flow `filter` covers. Returns how
|
||||
* many were written, or null when the save panel was cancelled.
|
||||
@@ -38,6 +64,7 @@ export async function exportHar(
|
||||
export function useFlowActions(): (flow: Flow) => DropdownItem[] {
|
||||
const showToast = useShowToast();
|
||||
const openSave = useSetAtom(saveDialogFlowAtom);
|
||||
const saveBody = useSaveFlowBody();
|
||||
|
||||
return useCallback(
|
||||
(flow: Flow) => {
|
||||
@@ -75,6 +102,10 @@ export function useFlowActions(): (flow: Flow) => DropdownItem[] {
|
||||
showToast({ message: `Failed to copy cURL: ${String(err)}`, color: "danger" });
|
||||
return;
|
||||
}
|
||||
if (res.decodeError != null) {
|
||||
showToast({ message: `Failed to copy cURL: ${res.decodeError}`, color: "danger" });
|
||||
return;
|
||||
}
|
||||
body = curlBodyText(base64ToBytes(res.data), res.truncated);
|
||||
// A command that sends other bytes than the app did would mislead.
|
||||
if (body == null) {
|
||||
@@ -102,6 +133,18 @@ export function useFlowActions(): (flow: Flow) => DropdownItem[] {
|
||||
onSelect: () => copy("response body", () => bodyText(flow, "response")),
|
||||
},
|
||||
{ type: "separator" },
|
||||
{
|
||||
label: "Save Request Body to File",
|
||||
icon: "download",
|
||||
disabled: flow.requestSize === 0,
|
||||
onSelect: () => saveBody(flow, "request"),
|
||||
},
|
||||
{
|
||||
label: "Save Response Body to File",
|
||||
icon: "download",
|
||||
disabled: noResponse || flow.responseSize === 0,
|
||||
onSelect: () => saveBody(flow, "response"),
|
||||
},
|
||||
{
|
||||
label: "Export HAR",
|
||||
icon: "download",
|
||||
@@ -116,6 +159,6 @@ export function useFlowActions(): (flow: Flow) => DropdownItem[] {
|
||||
},
|
||||
];
|
||||
},
|
||||
[openSave, showToast],
|
||||
[openSave, saveBody, showToast],
|
||||
);
|
||||
}
|
||||
@@ -9,6 +9,7 @@ publish = false
|
||||
base64 = "0.23.1"
|
||||
bytes = "1"
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
encoding_rs = "0.8"
|
||||
log = { workspace = true }
|
||||
http-body-util = "0.1"
|
||||
hyper = { version = "1", features = ["http1", "server"] }
|
||||
|
||||
+27
-2
@@ -133,6 +133,7 @@ export type GetFlowBodyRequest = { flowId: string, which: FlowBody, };
|
||||
export type GetFlowBodyResponse = {
|
||||
/**
|
||||
* The body decoded with its Content-Encoding, at most the capture limit of it, in base64.
|
||||
* When it fails to decode, the bytes as they were on the wire instead.
|
||||
*/
|
||||
data: string,
|
||||
/**
|
||||
@@ -142,7 +143,11 @@ truncated: boolean,
|
||||
/**
|
||||
* Bytes stored for the body, as they were on the wire.
|
||||
*/
|
||||
storedSize: number, };
|
||||
storedSize: number,
|
||||
/**
|
||||
* Why the body failed to decode, in which case `data` is as it was on the wire.
|
||||
*/
|
||||
decodeError?: string, };
|
||||
|
||||
export type GetProxyStateRequest = Record<string, never>;
|
||||
|
||||
@@ -238,7 +243,27 @@ export type ProxyStatePayload = { state: ProxyState, };
|
||||
|
||||
export type RpcEventSchema = { flows_cleared: FlowsClearedPayload, model_write: ModelPayload, proxy_state_changed: ProxyStatePayload, };
|
||||
|
||||
export type RpcSchema = { execute_action: [ActionInvocation, boolean], format_body: [FormatBodyRequest, string], list_actions: [ListActionsRequest, ListActionsResponse], list_flows: [FlowFilter, ListFlowsResponse], set_recording: [SetRecordingRequest, boolean], set_system_proxy_at_launch: [SetSystemProxyAtLaunchRequest, boolean], set_http_version: [SetHttpVersionRequest, HttpVersion], app_icon: [AppIconRequest, string | null], ca_status: [CaStatusRequest, CaStatus], ca_trust: [CaTrustRequest, CaStatus], clear_flows: [ClearFlowsRequest, number], export_har: [ExportHarRequest, number], filter_body: [FilterBodyRequest, FilterBodyResponse], get_flow_body: [GetFlowBodyRequest, GetFlowBodyResponse], get_proxy_state: [GetProxyStateRequest, GetProxyStateResponse], helper_register: [HelperRegisterRequest, SystemProxyStatus], helper_unregister: [HelperUnregisterRequest, SystemProxyStatus], list_workspaces: [ListWorkspacesRequest, ListWorkspacesResponse], preview_save: [PreviewSaveRequest, SavePreview], save_flow: [SaveFlowToYaakRequest, SavedFlow], system_proxy_set: [SystemProxySetRequest, SystemProxyStatus], system_proxy_status: [SystemProxyStatusRequest, SystemProxyStatus], take_over: [TakeOverRequest, boolean], };
|
||||
export type RpcSchema = { execute_action: [ActionInvocation, boolean], format_body: [FormatBodyRequest, string], list_actions: [ListActionsRequest, ListActionsResponse], list_flows: [FlowFilter, ListFlowsResponse], set_recording: [SetRecordingRequest, boolean], set_system_proxy_at_launch: [SetSystemProxyAtLaunchRequest, boolean], set_http_version: [SetHttpVersionRequest, HttpVersion], app_icon: [AppIconRequest, string | null], ca_status: [CaStatusRequest, CaStatus], ca_trust: [CaTrustRequest, CaStatus], clear_flows: [ClearFlowsRequest, number], export_har: [ExportHarRequest, number], filter_body: [FilterBodyRequest, FilterBodyResponse], get_flow_body: [GetFlowBodyRequest, GetFlowBodyResponse], get_proxy_state: [GetProxyStateRequest, GetProxyStateResponse], helper_register: [HelperRegisterRequest, SystemProxyStatus], helper_unregister: [HelperUnregisterRequest, SystemProxyStatus], list_workspaces: [ListWorkspacesRequest, ListWorkspacesResponse], preview_save: [PreviewSaveRequest, SavePreview], save_flow: [SaveFlowToYaakRequest, SavedFlow], save_flow_body: [SaveFlowBodyRequest, SaveFlowBodyResponse], system_proxy_set: [SystemProxySetRequest, SystemProxyStatus], system_proxy_status: [SystemProxyStatusRequest, SystemProxyStatus], take_over: [TakeOverRequest, boolean], };
|
||||
|
||||
export type SaveFlowBodyRequest = { flowId: string, which: FlowBody,
|
||||
/**
|
||||
* The file to write.
|
||||
*/
|
||||
path: string, };
|
||||
|
||||
export type SaveFlowBodyResponse = {
|
||||
/**
|
||||
* Bytes written.
|
||||
*/
|
||||
size: number,
|
||||
/**
|
||||
* What was written is incomplete, as `GetFlowBodyResponse.truncated` is.
|
||||
*/
|
||||
truncated: boolean,
|
||||
/**
|
||||
* The body failed to decode, so the file has it as it was on the wire.
|
||||
*/
|
||||
raw: boolean, };
|
||||
|
||||
export type SaveFlowToYaakRequest = { flowId: string, workspaceId: string, folderId?: string,
|
||||
/**
|
||||
|
||||
@@ -67,6 +67,61 @@ pub async fn read_body(
|
||||
Ok(DecodedBody { data, stored_size, truncated, decode_error })
|
||||
}
|
||||
|
||||
/// A body's text as the app's viewer reads it, as `decodeBodyText` in `@yaakapp-internal/lib`
|
||||
/// does: in the charset its content type names, when it names one the WHATWG Encoding Standard
|
||||
/// knows, or else as UTF-8, or windows-1252 when it isn't valid UTF-8, as browsers read text.
|
||||
pub fn body_text(data: &[u8], content_type: Option<&str>, truncated: bool) -> String {
|
||||
let encoding = content_type
|
||||
.and_then(charset_label)
|
||||
.and_then(|label| encoding_rs::Encoding::for_label(label.as_bytes()))
|
||||
// A label the standard maps to "replacement" is one `TextDecoder` refuses.
|
||||
.filter(|encoding| *encoding != encoding_rs::REPLACEMENT);
|
||||
if let Some(encoding) = encoding {
|
||||
return encoding.decode_with_bom_removal(data).0.into_owned();
|
||||
}
|
||||
if let Some(text) = strict_utf8(data, truncated) {
|
||||
return text;
|
||||
}
|
||||
encoding_rs::WINDOWS_1252.decode_without_bom_handling(data).0.into_owned()
|
||||
}
|
||||
|
||||
/// The `charset` parameter of a content type
|
||||
fn charset_label(content_type: &str) -> Option<&str> {
|
||||
content_type.split(';').skip(1).find_map(|param| {
|
||||
let (name, value) = param.split_once('=')?;
|
||||
name.trim().eq_ignore_ascii_case("charset").then(|| value.trim().trim_matches('"').trim())
|
||||
})
|
||||
}
|
||||
|
||||
/// The bytes as UTF-8 without a byte order mark, or None when they aren't valid UTF-8. A
|
||||
/// truncated body may end partway through a character, so up to three trailing bytes are dropped
|
||||
/// before giving up on it.
|
||||
fn strict_utf8(data: &[u8], truncated: bool) -> Option<String> {
|
||||
let max_trim = if truncated { data.len().min(3) } else { 0 };
|
||||
(0..=max_trim).find_map(|trim| {
|
||||
let text = std::str::from_utf8(&data[..data.len() - trim]).ok()?;
|
||||
Some(text.strip_prefix('\u{feff}').unwrap_or(text).to_string())
|
||||
})
|
||||
}
|
||||
|
||||
/// A flow's body decoded as `read_body` does, or, when it fails to decode, the bytes as they were
|
||||
/// on the wire, with `decode_error` saying why.
|
||||
pub async fn read_body_or_raw(
|
||||
blobs: &BlobManager,
|
||||
flow: &Flow,
|
||||
body: FlowBody,
|
||||
limit: usize,
|
||||
) -> Result<DecodedBody, String> {
|
||||
let decoded = read_body(blobs, flow, body, limit).await?;
|
||||
if decoded.decode_error.is_none() {
|
||||
return Ok(decoded);
|
||||
}
|
||||
let mut data = read_raw_body(blobs, &flow.id, body)?;
|
||||
let truncated = flow.truncated(body) || data.len() > limit;
|
||||
data.truncate(limit);
|
||||
Ok(DecodedBody { data, truncated, ..decoded })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -127,4 +182,94 @@ mod tests {
|
||||
assert!(body.truncated);
|
||||
assert_eq!(body.decode_error, None);
|
||||
}
|
||||
|
||||
/// The cases `decodeBodyText`'s tests in `@yaakapp-internal/lib` cover, read the same way
|
||||
#[test]
|
||||
fn body_text_reads_as_the_viewer_does() {
|
||||
let french = "Accès refusé : vous n'avez pas l'autorisation d'accéder à cette page.";
|
||||
let latin1: Vec<u8> = french.chars().map(|c| c as u8).collect();
|
||||
assert_eq!(body_text(&latin1, Some("text/html; charset=iso-8859-1"), false), french);
|
||||
assert_eq!(body_text(&latin1, Some("text/plain; CHARSET=\"ISO-8859-1\""), false), french);
|
||||
let shift_jis = [
|
||||
0x82, 0xb1, 0x82, 0xf1, 0x82, 0xc9, 0x82, 0xbf, 0x82, 0xcd, 0x81, 0x41, 0x90, 0xa2,
|
||||
0x8a, 0x45,
|
||||
];
|
||||
assert_eq!(
|
||||
body_text(&shift_jis, Some("text/plain; charset=Shift_JIS"), false),
|
||||
"こんにちは、世界"
|
||||
);
|
||||
|
||||
// Unlabeled and not UTF-8: windows-1252
|
||||
assert_eq!(body_text(&latin1, Some("text/html"), false), french);
|
||||
assert_eq!(body_text(&latin1, None, false), french);
|
||||
assert_eq!(body_text(&[0x93, b'h', b'i', 0x94], None, false), "\u{201c}hi\u{201d}");
|
||||
// A label `TextDecoder` refuses is no label.
|
||||
assert_eq!(body_text(&latin1, Some("text/plain; charset=not-a-charset"), false), french);
|
||||
assert_eq!(body_text(&latin1, Some("text/plain; charset=iso-2022-kr"), false), french);
|
||||
|
||||
// UTF-8, labeled or not; labeled, what isn't UTF-8 is replaced
|
||||
let json = r#"{"city":"Zürich","greeting":"こんにちは"}"#;
|
||||
assert_eq!(body_text(json.as_bytes(), None, false), json);
|
||||
assert_eq!(
|
||||
body_text(json.as_bytes(), Some("application/json; charset=utf-8"), false),
|
||||
json
|
||||
);
|
||||
assert_eq!(body_text(b"a\xff", Some("text/plain; charset=utf-8"), false), "a\u{fffd}");
|
||||
|
||||
// A UTF-8 body cut partway through a character stays UTF-8
|
||||
assert_eq!(body_text(&"abc€".as_bytes()[..5], None, true), "abc");
|
||||
}
|
||||
|
||||
fn deflate_request(blobs: &BlobManager, id: &str, stored: &[u8]) -> Flow {
|
||||
let mut writer = BodyChunkWriter::new(blobs.clone(), Flow::body_id(id, FlowBody::Request));
|
||||
writer.write(stored).unwrap();
|
||||
writer.finish().unwrap();
|
||||
Flow {
|
||||
id: id.into(),
|
||||
request_size: stored.len() as i64,
|
||||
request_headers: vec![crate::models::ProxyHeader {
|
||||
name: "Content-Encoding".into(),
|
||||
value: "deflate".into(),
|
||||
}],
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Browser SDKs send `deflate` both ways: zlib-wrapped, as the spec says, and raw.
|
||||
#[tokio::test]
|
||||
async fn deflate_request_bodies_decode_wrapped_or_raw() {
|
||||
use std::io::Write;
|
||||
let (_db, blobs, _rx) = yaak_models::init_in_memory().unwrap();
|
||||
let logs = br#"[{"message":"hello","service":"browser"}]"#;
|
||||
let mut zlib = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default());
|
||||
zlib.write_all(logs).unwrap();
|
||||
let mut raw =
|
||||
flate2::write::DeflateEncoder::new(Vec::new(), flate2::Compression::default());
|
||||
raw.write_all(logs).unwrap();
|
||||
|
||||
for (id, stored) in [
|
||||
("fl_zlib", zlib.finish().unwrap()),
|
||||
("fl_raw", raw.finish().unwrap()),
|
||||
] {
|
||||
let flow = deflate_request(&blobs, id, &stored);
|
||||
let body = read_body(&blobs, &flow, FlowBody::Request, 1024).await.unwrap();
|
||||
assert_eq!(body.decode_error, None, "{id}");
|
||||
assert_eq!((body.data.as_slice(), body.truncated), (&logs[..], false), "{id}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn undecodable_body_reads_as_it_was_on_the_wire() {
|
||||
let (_db, blobs, _rx) = yaak_models::init_in_memory().unwrap();
|
||||
let flow = deflate_request(&blobs, "fl_bad", b"\x00\x01 not deflate");
|
||||
|
||||
let decoded = read_body(&blobs, &flow, FlowBody::Request, 1024).await.unwrap();
|
||||
assert!(decoded.decode_error.is_some());
|
||||
let raw = read_body_or_raw(&blobs, &flow, FlowBody::Request, 1024).await.unwrap();
|
||||
assert_eq!(raw.data, b"\x00\x01 not deflate");
|
||||
assert!(!raw.truncated);
|
||||
assert_eq!(raw.decode_error, decoded.decode_error);
|
||||
let cut = read_body_or_raw(&blobs, &flow, FlowBody::Request, 4).await.unwrap();
|
||||
assert_eq!((cut.data.as_slice(), cut.truncated), (&b"\x00\x01 n"[..], true));
|
||||
}
|
||||
}
|
||||
@@ -851,12 +851,40 @@ pub struct GetFlowBodyRequest {
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct GetFlowBodyResponse {
|
||||
/// The body decoded with its Content-Encoding, at most the capture limit of it, in base64.
|
||||
/// When it fails to decode, the bytes as they were on the wire instead.
|
||||
pub data: String,
|
||||
/// The body is incomplete: cut on the wire, or decoded past the capture limit.
|
||||
pub truncated: bool,
|
||||
/// Bytes stored for the body, as they were on the wire.
|
||||
#[ts(type = "number")]
|
||||
pub stored_size: usize,
|
||||
/// Why the body failed to decode, in which case `data` is as it was on the wire.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[ts(optional)]
|
||||
pub decode_error: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, TS)]
|
||||
#[ts(export, export_to = "gen_rpc.ts")]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct SaveFlowBodyRequest {
|
||||
pub flow_id: String,
|
||||
pub which: FlowBody,
|
||||
/// The file to write.
|
||||
pub path: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, TS)]
|
||||
#[ts(export, export_to = "gen_rpc.ts")]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct SaveFlowBodyResponse {
|
||||
/// Bytes written.
|
||||
#[ts(type = "number")]
|
||||
pub size: usize,
|
||||
/// What was written is incomplete, as `GetFlowBodyResponse.truncated` is.
|
||||
pub truncated: bool,
|
||||
/// The body failed to decode, so the file has it as it was on the wire.
|
||||
pub raw: bool,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, TS)]
|
||||
@@ -1082,16 +1110,42 @@ async fn get_flow_body(
|
||||
) -> Result<GetFlowBodyResponse, RpcError> {
|
||||
let flow = find_flow(&ctx, &req.flow_id)?;
|
||||
let limit = settings(&ctx)?.capture_body_limit;
|
||||
let body = viewer_body(&ctx, &flow, req.which, limit).await?;
|
||||
let body =
|
||||
bodies::read_body_or_raw(&ctx.blobs, &flow, req.which, limit).await.map_err(rpc_error)?;
|
||||
Ok(GetFlowBodyResponse {
|
||||
data: BASE64.encode(&body.data),
|
||||
truncated: body.truncated,
|
||||
stored_size: body.stored_size,
|
||||
decode_error: body.decode_error,
|
||||
})
|
||||
}
|
||||
|
||||
/// A body for the app's viewer, which shows a body that failed to decode as its error, as it
|
||||
/// always has, rather than as what decoded before the failure.
|
||||
/// Writes a body to a file the person picked: decoded, as the viewer shows it, or as it was on
|
||||
/// the wire when it fails to decode.
|
||||
async fn save_flow_body(
|
||||
ctx: ProxyCtx,
|
||||
req: SaveFlowBodyRequest,
|
||||
) -> Result<SaveFlowBodyResponse, RpcError> {
|
||||
let flow = find_flow(&ctx, &req.flow_id)?;
|
||||
let limit = settings(&ctx)?.capture_body_limit;
|
||||
let body =
|
||||
bodies::read_body_or_raw(&ctx.blobs, &flow, req.which, limit).await.map_err(rpc_error)?;
|
||||
let saved = SaveFlowBodyResponse {
|
||||
size: body.data.len(),
|
||||
truncated: body.truncated,
|
||||
raw: body.decode_error.is_some(),
|
||||
};
|
||||
let path = PathBuf::from(req.path);
|
||||
blocking(move || {
|
||||
std::fs::write(&path, &body.data)
|
||||
.map_err(|e| format!("Failed to write {}: {e}", path.display()))
|
||||
})
|
||||
.await?;
|
||||
Ok(saved)
|
||||
}
|
||||
|
||||
/// A body to filter, which fails when the body failed to decode rather than filtering what
|
||||
/// decoded before the failure.
|
||||
async fn viewer_body(
|
||||
ctx: &ProxyCtx,
|
||||
flow: &Flow,
|
||||
@@ -1112,14 +1166,13 @@ async fn filter_body(
|
||||
let flow = find_flow(&ctx, &req.flow_id)?;
|
||||
let limit = settings(&ctx)?.capture_body_limit;
|
||||
let body = viewer_body(&ctx, &flow, req.which, limit).await?;
|
||||
let text = String::from_utf8_lossy(&body.data);
|
||||
let content_type = find_header(flow.headers(req.which), "content-type");
|
||||
// The text the viewer shows, so the filter's results match it
|
||||
let text = bodies::body_text(&body.data, content_type, body.truncated);
|
||||
let language = match req.language {
|
||||
Some(FilterLanguage::JsonPath) => yaak_filter::Language::JsonPath,
|
||||
Some(FilterLanguage::XPath) => yaak_filter::Language::XPath,
|
||||
None => {
|
||||
let content_type = find_header(flow.headers(req.which), "content-type");
|
||||
yaak_filter::Language::for_content_type(content_type.unwrap_or_default())
|
||||
}
|
||||
None => yaak_filter::Language::for_content_type(content_type.unwrap_or_default()),
|
||||
};
|
||||
let result = yaak_filter::filter(language, &text, &req.expression);
|
||||
Ok(FilterBodyResponse { content: result.content, error: result.error })
|
||||
@@ -1437,6 +1490,7 @@ define_rpc! {
|
||||
list_workspaces(ListWorkspacesRequest) -> ListWorkspacesResponse,
|
||||
preview_save(PreviewSaveRequest) -> SavePreview,
|
||||
save_flow(SaveFlowToYaakRequest) -> SavedFlow,
|
||||
save_flow_body(SaveFlowBodyRequest) -> SaveFlowBodyResponse,
|
||||
system_proxy_set(SystemProxySetRequest) -> SystemProxyStatus,
|
||||
system_proxy_status(SystemProxyStatusRequest) -> SystemProxyStatus,
|
||||
take_over(TakeOverRequest) -> bool,
|
||||
@@ -1491,6 +1545,57 @@ mod tests {
|
||||
assert!(log.was_on_at(SystemTime::now()));
|
||||
}
|
||||
|
||||
/// A flow whose response is `body` under `content_type`, stored in `ctx`
|
||||
fn flow_with_response(ctx: &ProxyCtx, id: &str, content_type: &str, body: &[u8]) {
|
||||
let flow = Flow {
|
||||
id: id.into(),
|
||||
status: Some(200),
|
||||
response_size: body.len() as i64,
|
||||
response_headers: vec![ProxyHeader {
|
||||
name: "Content-Type".into(),
|
||||
value: content_type.into(),
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
ctx.db.with_tx(|tx| tx.upsert(&flow, &UpdateSource::Background)).unwrap();
|
||||
let mut writer =
|
||||
BodyChunkWriter::new(ctx.blobs.clone(), Flow::body_id(id, FlowBody::Response));
|
||||
writer.write(body).unwrap();
|
||||
writer.finish().unwrap();
|
||||
}
|
||||
|
||||
/// The filter reads the text the viewer shows: in the body's charset, not UTF-8 alone.
|
||||
#[tokio::test]
|
||||
async fn filter_reads_the_body_in_its_charset() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (events, _rx) = RpcEventEmitter::new();
|
||||
let ctx = ProxyCtx::new(dir.path(), events).unwrap();
|
||||
|
||||
let xml = "<cities><city>Zürich</city><city>Kraków</city></cities>";
|
||||
let utf16: Vec<u8> = xml.encode_utf16().flat_map(u16::to_le_bytes).collect();
|
||||
flow_with_response(&ctx, "fl_utf16", "application/xml; charset=utf-16le", &utf16);
|
||||
let json = r#"{"items":[{"name":"Café"},{"name":"Crème brûlée"}]}"#;
|
||||
let latin1: Vec<u8> = json.chars().map(|c| c as u8).collect();
|
||||
flow_with_response(&ctx, "fl_latin1", "application/json; charset=iso-8859-1", &latin1);
|
||||
flow_with_response(&ctx, "fl_unlabeled", "application/json", &latin1);
|
||||
|
||||
let filter = |flow_id: &str, expression: &str, language| FilterBodyRequest {
|
||||
flow_id: flow_id.into(),
|
||||
which: FlowBody::Response,
|
||||
expression: expression.into(),
|
||||
language: Some(language),
|
||||
};
|
||||
let xpath = filter("fl_utf16", "//city[2]", FilterLanguage::XPath);
|
||||
let result = filter_body(ctx.clone(), xpath).await.unwrap();
|
||||
assert_eq!((result.content.as_str(), result.error), ("<city>Kraków</city>", None));
|
||||
for flow_id in ["fl_latin1", "fl_unlabeled"] {
|
||||
let names = filter(flow_id, "$.items[*].name", FilterLanguage::JsonPath);
|
||||
let result = filter_body(ctx.clone(), names).await.unwrap();
|
||||
assert_eq!(result.error, None, "{flow_id}");
|
||||
assert_eq!(result.content, "[\n \"Café\",\n \"Crème brûlée\"\n]", "{flow_id}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_app_still_hears_why_a_body_failed_to_decode() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
@@ -1511,9 +1616,25 @@ mod tests {
|
||||
writer.write(b"not a gzip stream").unwrap();
|
||||
writer.finish().unwrap();
|
||||
|
||||
// The viewer gets the bytes as they were on the wire, and why they didn't decode.
|
||||
let req = GetFlowBodyRequest { flow_id: "fl_bad".into(), which: FlowBody::Response };
|
||||
let err = get_flow_body(ctx.clone(), req).await.err().unwrap();
|
||||
assert!(err.message.starts_with("Failed to decode"), "{}", err.message);
|
||||
let body = get_flow_body(ctx.clone(), req).await.unwrap();
|
||||
let error = body.decode_error.unwrap();
|
||||
assert!(error.starts_with("Failed to decode Gzip body"), "{error}");
|
||||
assert_eq!(BASE64.decode(body.data).unwrap(), b"not a gzip stream");
|
||||
assert!(!body.truncated);
|
||||
|
||||
// Saving writes those same bytes.
|
||||
let path = dir.path().join("body.bin");
|
||||
let req = SaveFlowBodyRequest {
|
||||
flow_id: "fl_bad".into(),
|
||||
which: FlowBody::Response,
|
||||
path: path.to_string_lossy().into_owned(),
|
||||
};
|
||||
let saved = save_flow_body(ctx.clone(), req).await.unwrap();
|
||||
assert_eq!((saved.size, saved.truncated, saved.raw), (17, false, true));
|
||||
assert_eq!(std::fs::read(&path).unwrap(), b"not a gzip stream");
|
||||
|
||||
let req = FilterBodyRequest {
|
||||
flow_id: "fl_bad".into(),
|
||||
which: FlowBody::Response,
|
||||
|
||||
@@ -5,7 +5,7 @@ edition = "2024"
|
||||
publish = false
|
||||
|
||||
[dependencies]
|
||||
async-compression = { version = "0.4", features = ["tokio", "gzip", "deflate", "brotli", "zstd"] }
|
||||
async-compression = { version = "0.4", features = ["tokio", "gzip", "deflate", "zlib", "brotli", "zstd"] }
|
||||
async-trait = "0.1"
|
||||
brotli = "9"
|
||||
bytes = "1.11.1"
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
use crate::error::{Error, Result};
|
||||
use async_compression::tokio::bufread::{
|
||||
BrotliDecoder, DeflateDecoder as AsyncDeflateDecoder, GzipDecoder,
|
||||
ZstdDecoder as AsyncZstdDecoder,
|
||||
ZlibDecoder as AsyncZlibDecoder, ZstdDecoder as AsyncZstdDecoder,
|
||||
};
|
||||
use flate2::read::{DeflateDecoder, GzDecoder};
|
||||
use flate2::read::{DeflateDecoder, GzDecoder, ZlibDecoder};
|
||||
use std::io::Read;
|
||||
use tokio::io::{AsyncBufRead, AsyncRead};
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll, ready};
|
||||
use tokio::io::{AsyncBufRead, AsyncRead, BufReader, ReadBuf};
|
||||
|
||||
/// Supported compression encodings
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -66,13 +68,77 @@ fn decompress_gzip(data: &[u8]) -> Result<Vec<u8>> {
|
||||
Ok(decompressed)
|
||||
}
|
||||
|
||||
/// `deflate` names zlib-wrapped data (RFC 1950), but plenty of senders send it raw (RFC 1951).
|
||||
fn decompress_deflate(data: &[u8]) -> Result<Vec<u8>> {
|
||||
let mut decoder = DeflateDecoder::new(data);
|
||||
let mut decompressed = Vec::new();
|
||||
decoder
|
||||
.read_to_end(&mut decompressed)
|
||||
.map_err(|e| Error::DecompressionError(format!("deflate decompression failed: {}", e)))?;
|
||||
Ok(decompressed)
|
||||
let read = |zlib: bool| {
|
||||
let mut decompressed = Vec::new();
|
||||
let result = if zlib {
|
||||
ZlibDecoder::new(data).read_to_end(&mut decompressed)
|
||||
} else {
|
||||
DeflateDecoder::new(data).read_to_end(&mut decompressed)
|
||||
};
|
||||
result.map(|_| decompressed)
|
||||
};
|
||||
match deflate_variant(data) {
|
||||
DeflateVariant::Zlib => read(true),
|
||||
DeflateVariant::Raw => read(false),
|
||||
// Zlib only when it ends whole, its checksum verified, or else raw.
|
||||
DeflateVariant::Either => inflate_zlib_whole(data).or_else(|_| read(false)),
|
||||
}
|
||||
.map_err(|e| Error::DecompressionError(format!("deflate decompression failed: {}", e)))
|
||||
}
|
||||
|
||||
/// Zlib data decoded through to its end and checksum, failing on any less
|
||||
fn inflate_zlib_whole(data: &[u8]) -> std::io::Result<Vec<u8>> {
|
||||
let mut inflater = flate2::Decompress::new(true);
|
||||
let mut out = Vec::new();
|
||||
loop {
|
||||
// Always room to write, so a call that makes no progress has run out of input.
|
||||
out.reserve(64 * 1024);
|
||||
let (read, written) = (inflater.total_in(), inflater.total_out());
|
||||
let rest = &data[usize::try_from(read).unwrap_or(data.len()).min(data.len())..];
|
||||
// Not `Finish`, which fails for good when the output outgrows its buffer.
|
||||
let status = inflater
|
||||
.decompress_vec(rest, &mut out, flate2::FlushDecompress::None)
|
||||
.map_err(std::io::Error::other)?;
|
||||
if status == flate2::Status::StreamEnd {
|
||||
return Ok(out);
|
||||
}
|
||||
if inflater.total_in() == read && inflater.total_out() == written {
|
||||
return Err(std::io::ErrorKind::UnexpectedEof.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Which `deflate` a body is, from its first five bytes
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum DeflateVariant {
|
||||
Zlib,
|
||||
Raw,
|
||||
/// A zlib header that is also the start of a raw stored block: only decoding tells.
|
||||
Either,
|
||||
}
|
||||
|
||||
/// Zlib data opens with a header whose first byte names deflate with a window of at most 32 KB,
|
||||
/// and whose two bytes are a multiple of 31. Raw deflate passes that only when it opens with a
|
||||
/// stored block that isn't the last, whose length follows in the next two bytes and its
|
||||
/// complement in the two after.
|
||||
fn deflate_variant(data: &[u8]) -> DeflateVariant {
|
||||
let zlib = match data {
|
||||
[cmf, flg, ..] => {
|
||||
cmf & 0x0f == 8 && cmf >> 4 <= 7 && (u16::from(*cmf) << 8 | u16::from(*flg)) % 31 == 0
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
match data {
|
||||
_ if !zlib => DeflateVariant::Raw,
|
||||
[_, l0, l1, n0, n1, ..]
|
||||
if u16::from_le_bytes([*l0, *l1]) != !u16::from_le_bytes([*n0, *n1]) =>
|
||||
{
|
||||
DeflateVariant::Zlib
|
||||
}
|
||||
_ => DeflateVariant::Either,
|
||||
}
|
||||
}
|
||||
|
||||
fn decompress_brotli(data: &[u8]) -> Result<Vec<u8>> {
|
||||
@@ -96,12 +162,262 @@ pub fn streaming_decoder<R: AsyncBufRead + Unpin + Send + 'static>(
|
||||
match encoding {
|
||||
ContentEncoding::Identity => Box::new(reader),
|
||||
ContentEncoding::Gzip => Box::new(GzipDecoder::new(reader)),
|
||||
ContentEncoding::Deflate => Box::new(AsyncDeflateDecoder::new(reader)),
|
||||
ContentEncoding::Deflate => Box::new(StreamingDeflateDecoder::new(reader)),
|
||||
ContentEncoding::Brotli => Box::new(BrotliDecoder::new(reader)),
|
||||
ContentEncoding::Zstd => Box::new(AsyncZstdDecoder::new(reader)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Decodes `deflate`, zlib-wrapped or raw, as its first five bytes say. When they could be
|
||||
/// either, it decodes both ways at once until one fails, and goes on with the other.
|
||||
struct StreamingDeflateDecoder<R> {
|
||||
state: DeflateState<R>,
|
||||
}
|
||||
|
||||
enum DeflateState<R> {
|
||||
Sniffing(Replay<R>),
|
||||
Zlib(AsyncZlibDecoder<BufReader<Replay<R>>>),
|
||||
Raw(AsyncDeflateDecoder<BufReader<Replay<R>>>),
|
||||
Either(EitherInflate<BufReader<Replay<R>>>),
|
||||
/// Only while switching from sniffing to one of the others
|
||||
Switching,
|
||||
}
|
||||
|
||||
impl<R> StreamingDeflateDecoder<R> {
|
||||
fn new(reader: R) -> Self {
|
||||
let source = Replay { inner: reader, header: Vec::new(), pos: 0, sniffing: true };
|
||||
Self { state: DeflateState::Sniffing(source) }
|
||||
}
|
||||
}
|
||||
|
||||
impl<R: AsyncRead + Unpin> AsyncRead for StreamingDeflateDecoder<R> {
|
||||
fn poll_read(
|
||||
self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
buf: &mut ReadBuf<'_>,
|
||||
) -> Poll<std::io::Result<()>> {
|
||||
let this = self.get_mut();
|
||||
loop {
|
||||
match &mut this.state {
|
||||
DeflateState::Sniffing(source) => {
|
||||
// Read until the header is in hand, or the input ends short of it.
|
||||
while source.header.len() < 5 {
|
||||
let mut header = [0u8; 5];
|
||||
let mut header = ReadBuf::new(&mut header[source.header.len()..]);
|
||||
ready!(Pin::new(&mut *source).poll_read(cx, &mut header))?;
|
||||
if header.filled().is_empty() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let DeflateState::Sniffing(mut source) =
|
||||
std::mem::replace(&mut this.state, DeflateState::Switching)
|
||||
else {
|
||||
unreachable!()
|
||||
};
|
||||
source.sniffing = false;
|
||||
let variant = deflate_variant(&source.header);
|
||||
let reader = BufReader::new(source);
|
||||
this.state = match variant {
|
||||
DeflateVariant::Zlib => DeflateState::Zlib(AsyncZlibDecoder::new(reader)),
|
||||
DeflateVariant::Raw => DeflateState::Raw(AsyncDeflateDecoder::new(reader)),
|
||||
DeflateVariant::Either => DeflateState::Either(EitherInflate::new(reader)),
|
||||
};
|
||||
}
|
||||
DeflateState::Zlib(d) => return Pin::new(d).poll_read(cx, buf),
|
||||
DeflateState::Raw(d) => return Pin::new(d).poll_read(cx, buf),
|
||||
DeflateState::Either(d) => return Pin::new(d).poll_read(cx, buf),
|
||||
DeflateState::Switching => {
|
||||
return Poll::Ready(Err(std::io::Error::other("deflate decoder is broken")));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads `inner`, keeping what it read while sniffing, then reads that again before the rest.
|
||||
struct Replay<R> {
|
||||
inner: R,
|
||||
header: Vec<u8>,
|
||||
/// Where reading resumes in `header` once sniffing is over
|
||||
pos: usize,
|
||||
sniffing: bool,
|
||||
}
|
||||
|
||||
impl<R: AsyncRead + Unpin> AsyncRead for Replay<R> {
|
||||
fn poll_read(
|
||||
self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
buf: &mut ReadBuf<'_>,
|
||||
) -> Poll<std::io::Result<()>> {
|
||||
let this = self.get_mut();
|
||||
if this.sniffing {
|
||||
let before = buf.filled().len();
|
||||
ready!(Pin::new(&mut this.inner).poll_read(cx, buf))?;
|
||||
this.header.extend_from_slice(&buf.filled()[before..]);
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
if this.pos < this.header.len() {
|
||||
let n = buf.remaining().min(this.header.len() - this.pos);
|
||||
buf.put_slice(&this.header[this.pos..this.pos + n]);
|
||||
this.pos += n;
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
Pin::new(&mut this.inner).poll_read(cx, buf)
|
||||
}
|
||||
}
|
||||
|
||||
/// Input fed to the readings of an ambiguous deflate body at a time, which deflate grows to at
|
||||
/// most about a megabyte
|
||||
const DEFLATE_FEED: usize = 1024;
|
||||
|
||||
/// Output held while both readings of an ambiguous deflate body are still valid, past which it is
|
||||
/// taken to be zlib, as its header says. A wrong reading fails within a few bytes in practice.
|
||||
const DEFLATE_HELD_LIMIT: usize = 1024 * 1024;
|
||||
|
||||
/// Both readings of a deflate body that could be zlib or raw, fed the same input. Their output
|
||||
/// is held back while both are still valid, and the one left streams on: a wrong reading fails
|
||||
/// within a few bytes in practice, so little is held. If both end, zlib's checksum verified it.
|
||||
struct EitherInflate<B> {
|
||||
input: B,
|
||||
/// Zlib, then raw
|
||||
readings: [Option<Inflate>; 2],
|
||||
/// Why the last reading to fail failed
|
||||
error: Option<std::io::Error>,
|
||||
}
|
||||
|
||||
struct Inflate {
|
||||
inflater: flate2::Decompress,
|
||||
out: Vec<u8>,
|
||||
/// Where reading resumes in `out`
|
||||
pos: usize,
|
||||
ended: bool,
|
||||
}
|
||||
|
||||
impl<B> EitherInflate<B> {
|
||||
fn new(input: B) -> Self {
|
||||
let reading = |zlib| Inflate {
|
||||
inflater: flate2::Decompress::new(zlib),
|
||||
out: Vec::new(),
|
||||
pos: 0,
|
||||
ended: false,
|
||||
};
|
||||
Self { input, readings: [Some(reading(true)), Some(reading(false))], error: None }
|
||||
}
|
||||
}
|
||||
|
||||
impl Inflate {
|
||||
/// Decodes all of `input` onto `out`, or up to the end of the stream
|
||||
fn feed(&mut self, input: &[u8]) -> std::io::Result<()> {
|
||||
let mut consumed = 0;
|
||||
while !self.ended {
|
||||
if self.out.capacity() - self.out.len() < 32 * 1024 {
|
||||
self.out.reserve(64 * 1024);
|
||||
}
|
||||
let (read, written) = (self.inflater.total_in(), self.inflater.total_out());
|
||||
let status = self
|
||||
.inflater
|
||||
.decompress_vec(&input[consumed..], &mut self.out, flate2::FlushDecompress::None)
|
||||
.map_err(std::io::Error::other)?;
|
||||
let progress = (self.inflater.total_in() - read, self.inflater.total_out() - written);
|
||||
consumed += usize::try_from(progress.0).unwrap_or(usize::MAX).min(input.len());
|
||||
if status == flate2::Status::StreamEnd {
|
||||
self.ended = true;
|
||||
} else if consumed == input.len() && self.out.len() < self.out.capacity() {
|
||||
// All of it in, and all that it decodes to out
|
||||
break;
|
||||
} else if progress == (0, 0) {
|
||||
return Err(std::io::Error::other("deflate decoding stalled"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl<B: AsyncBufRead + Unpin> AsyncRead for EitherInflate<B> {
|
||||
fn poll_read(
|
||||
self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
buf: &mut ReadBuf<'_>,
|
||||
) -> Poll<std::io::Result<()>> {
|
||||
let this = self.get_mut();
|
||||
loop {
|
||||
let [zlib, raw] = &mut this.readings;
|
||||
match (zlib.as_mut(), raw.as_mut()) {
|
||||
(None, None) => {
|
||||
let error = this.error.take();
|
||||
return Poll::Ready(Err(
|
||||
error.unwrap_or_else(|| std::io::Error::other("deflate decoding failed"))
|
||||
));
|
||||
}
|
||||
// Both ended: zlib's checksum verified it.
|
||||
(Some(z), Some(r)) if z.ended && r.ended => *raw = None,
|
||||
(Some(only), None) | (None, Some(only)) => {
|
||||
if only.pos < only.out.len() {
|
||||
let n = buf.remaining().min(only.out.len() - only.pos);
|
||||
buf.put_slice(&only.out[only.pos..only.pos + n]);
|
||||
only.pos += n;
|
||||
if only.pos == only.out.len() {
|
||||
only.out.clear();
|
||||
only.pos = 0;
|
||||
}
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
if only.ended {
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
ready!(this.feed(cx))?;
|
||||
}
|
||||
(Some(_), Some(_)) => ready!(this.feed(cx))?,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<B: AsyncBufRead + Unpin> EitherInflate<B> {
|
||||
/// Feeds the next input to each reading still going, dropping those that fail on it. Pending
|
||||
/// until there is input; at its end, a reading that hasn't ended was cut short.
|
||||
fn feed(&mut self, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
let chunk = ready!(Pin::new(&mut self.input).poll_fill_buf(cx))?;
|
||||
// A little input at a time, since deflate can grow a thousandfold.
|
||||
let len = chunk.len().min(DEFLATE_FEED);
|
||||
let chunk = &chunk[..len];
|
||||
let [zlib, raw] = &mut self.readings;
|
||||
if let (Some(z), Some(r)) = (zlib.as_ref(), raw.as_ref()) {
|
||||
let (z_held, r_held) = (z.out.len() - z.pos, r.out.len() - r.pos);
|
||||
let cut_short = len == 0 && !z.ended && !r.ended;
|
||||
if cut_short {
|
||||
// Both were cut short: keep what decoded furthest, so a cut body shows its start.
|
||||
if r_held > z_held {
|
||||
*zlib = None
|
||||
} else {
|
||||
*raw = None
|
||||
}
|
||||
// The one left gives what it decoded, then fails on the end of input.
|
||||
return Poll::Ready(Ok(()));
|
||||
} else if z_held.max(r_held) > DEFLATE_HELD_LIMIT {
|
||||
// Holding no more for a raw body zlib still reads validly: it is taken to be zlib.
|
||||
*raw = None;
|
||||
}
|
||||
}
|
||||
for slot in &mut self.readings {
|
||||
let Some(reading) = slot.as_mut().filter(|r| !r.ended) else {
|
||||
continue;
|
||||
};
|
||||
let result = if len == 0 {
|
||||
Err(std::io::ErrorKind::UnexpectedEof.into())
|
||||
} else {
|
||||
reading.feed(chunk)
|
||||
};
|
||||
if let Err(e) = result {
|
||||
*slot = None;
|
||||
self.error = Some(e);
|
||||
}
|
||||
}
|
||||
Pin::new(&mut self.input).consume(len);
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -159,6 +475,281 @@ mod tests {
|
||||
assert_eq!(result.decompressed_size, original.len() as u64);
|
||||
}
|
||||
|
||||
fn zlib(data: &[u8]) -> Vec<u8> {
|
||||
let mut encoder = flate2::write::ZlibEncoder::new(Vec::new(), Compression::default());
|
||||
encoder.write_all(data).unwrap();
|
||||
encoder.finish().unwrap()
|
||||
}
|
||||
|
||||
fn raw_deflate(data: &[u8]) -> Vec<u8> {
|
||||
let mut encoder = flate2::write::DeflateEncoder::new(Vec::new(), Compression::default());
|
||||
encoder.write_all(data).unwrap();
|
||||
encoder.finish().unwrap()
|
||||
}
|
||||
|
||||
/// Raw deflate whose first two bytes also pass as a zlib header: a stored block of 29 bytes
|
||||
/// (`08 1d`, with `0x081d` a multiple of 31), then an empty final one.
|
||||
fn raw_deflate_like_zlib(data: &[u8; 29]) -> Vec<u8> {
|
||||
let mut out = vec![0x08, 0x1d, 0x00, 0xe2, 0xff];
|
||||
out.extend_from_slice(data);
|
||||
out.extend_from_slice(&[0x01, 0x00, 0x00, 0xff, 0xff]);
|
||||
assert_eq!(deflate_variant(&out), DeflateVariant::Either);
|
||||
out
|
||||
}
|
||||
|
||||
/// Zlib that opens as `raw_deflate_like_zlib` does: header `08 1d`, then a stored block of
|
||||
/// 0xffe2 bytes, which `1d 00 e2 ff` also reads as the start of, and a final stored block of
|
||||
/// `extra` bytes.
|
||||
fn zlib_like_raw_deflate(extra: u16) -> (Vec<u8>, Vec<u8>) {
|
||||
let data: Vec<u8> = (0..0xffe2 + u32::from(extra)).map(|i| (i % 251) as u8).collect();
|
||||
let mut out = vec![0x08, 0x1d, 0x00, 0xe2, 0xff, 0x1d, 0x00];
|
||||
out.extend_from_slice(&data[..0xffe2]);
|
||||
out.push(0x01);
|
||||
out.extend_from_slice(&extra.to_le_bytes());
|
||||
out.extend_from_slice(&(!extra).to_le_bytes());
|
||||
out.extend_from_slice(&data[0xffe2..]);
|
||||
let (mut a, mut b) = (1u32, 0u32);
|
||||
for byte in &data {
|
||||
a = (a + u32::from(*byte)) % 65521;
|
||||
b = (b + a) % 65521;
|
||||
}
|
||||
out.extend_from_slice(&((b << 16) | a).to_be_bytes());
|
||||
assert_eq!(deflate_variant(&out), DeflateVariant::Either);
|
||||
(out, data)
|
||||
}
|
||||
|
||||
async fn stream_decode(data: Vec<u8>, encoding: ContentEncoding) -> std::io::Result<Vec<u8>> {
|
||||
use tokio::io::AsyncReadExt;
|
||||
let mut out = Vec::new();
|
||||
// One byte at a time, to read the header across polls as a slow connection would.
|
||||
let reader = BufReader::with_capacity(1, std::io::Cursor::new(data));
|
||||
streaming_decoder(reader, encoding).read_to_end(&mut out).await?;
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// What a body decodes to before it fails, and why it failed
|
||||
async fn stream_decode_until_error(data: Vec<u8>) -> (Vec<u8>, Option<std::io::Error>) {
|
||||
use tokio::io::AsyncReadExt;
|
||||
let mut out = Vec::new();
|
||||
let mut decoder =
|
||||
streaming_decoder(BufReader::new(std::io::Cursor::new(data)), ContentEncoding::Deflate);
|
||||
let mut buf = [0u8; 4096];
|
||||
loop {
|
||||
match decoder.read(&mut buf).await {
|
||||
Ok(0) => return (out, None),
|
||||
Ok(n) => out.extend_from_slice(&buf[..n]),
|
||||
Err(e) => return (out, Some(e)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_deflate_keeps_what_decoded_of_a_cut_ambiguous_body() {
|
||||
// Bytes both readings still accept when the input ends: zlib in a stored block too
|
||||
let text = *b"\x1d\0xxxxxxxxxxxxxxxxxxxxxxxxxxx";
|
||||
let mut cut = raw_deflate_like_zlib(&text);
|
||||
// Without its final block, as a body cut at the capture limit is
|
||||
cut.truncate(cut.len() - 5);
|
||||
let (decoded, error) = stream_decode_until_error(cut).await;
|
||||
assert_eq!(decoded, text);
|
||||
assert_eq!(error.map(|e| e.kind()), Some(std::io::ErrorKind::UnexpectedEof));
|
||||
}
|
||||
|
||||
/// Zlib of `megabytes` of zeros whose raw reading is a stored block of 29 bytes and an empty
|
||||
/// final one, both inside zlib's first stored block: raw ends at once, zlib goes on.
|
||||
fn zlib_whose_raw_reading_ends_early(megabytes: usize) -> (Vec<u8>, Vec<u8>) {
|
||||
let mut stored = vec![b'z'; 0xffe2];
|
||||
stored[27..32].copy_from_slice(&[0x01, 0x00, 0x00, 0xff, 0xff]);
|
||||
let mut expected = stored.clone();
|
||||
expected.resize(stored.len() + megabytes * 1024 * 1024, 0);
|
||||
let mut data = vec![0x08, 0x1d, 0x00, 0xe2, 0xff, 0x1d, 0x00];
|
||||
data.extend_from_slice(&stored);
|
||||
data.extend_from_slice(&raw_deflate(&expected[stored.len()..]));
|
||||
let (mut a, mut b) = (1u32, 0u32);
|
||||
for byte in &expected {
|
||||
a = (a + u32::from(*byte)) % 65521;
|
||||
b = (b + a) % 65521;
|
||||
}
|
||||
data.extend_from_slice(&((b << 16) | a).to_be_bytes());
|
||||
assert_eq!(deflate_variant(&data), DeflateVariant::Either);
|
||||
(data, expected)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_deflate_holds_little_while_both_readings_go() {
|
||||
use tokio::io::AsyncReadExt;
|
||||
let (data, expected) = zlib_whose_raw_reading_ends_early(8);
|
||||
let mut decoder =
|
||||
StreamingDeflateDecoder::new(BufReader::new(std::io::Cursor::new(data.clone())));
|
||||
let mut first = [0u8; 1024];
|
||||
decoder.read_exact(&mut first).await.unwrap();
|
||||
assert_eq!(first[..], expected[..1024]);
|
||||
let DeflateState::Either(either) = &decoder.state else {
|
||||
panic!("not read both ways")
|
||||
};
|
||||
// The limit, what one feed of input adds past it (deflate grows at most about 1,032
|
||||
// times), and the raw reading's 64 KB: far short of the 8 MB body.
|
||||
let held: usize = either.readings.iter().flatten().map(|r| r.out.len() - r.pos).sum();
|
||||
let bound = 3 * 1024 * 1024;
|
||||
assert!(held <= bound, "held {held} bytes for a 1 KB read");
|
||||
|
||||
let decoded = stream_decode(data, ContentEncoding::Deflate).await.unwrap();
|
||||
assert!(decoded == expected);
|
||||
}
|
||||
|
||||
const DEFLATE_TEXT: &[u8] = b"{\"service\":\"browser\",\"message\":\"hello deflate\"}";
|
||||
|
||||
#[test]
|
||||
fn test_decompress_deflate_zlib_wrapped() {
|
||||
let result = decompress(zlib(DEFLATE_TEXT), ContentEncoding::Deflate).unwrap();
|
||||
assert_eq!(result.data, DEFLATE_TEXT);
|
||||
let like_zlib = raw_deflate_like_zlib(b"twenty-nine bytes of raw text");
|
||||
let result = decompress(like_zlib, ContentEncoding::Deflate).unwrap();
|
||||
assert_eq!(result.data, b"twenty-nine bytes of raw text");
|
||||
let both = *b"\x1d\0xxxxxxxxxxxxxxxxxxxxxxxxxxx";
|
||||
let result = decompress(raw_deflate_like_zlib(&both), ContentEncoding::Deflate).unwrap();
|
||||
assert_eq!(result.data, both);
|
||||
for extra in [0, 10_000] {
|
||||
let (zlib_like_raw, data) = zlib_like_raw_deflate(extra);
|
||||
let result = decompress(zlib_like_raw, ContentEncoding::Deflate).unwrap();
|
||||
assert!(result.data == data, "{extra} extra bytes");
|
||||
}
|
||||
let (data, expected) = large_raw_like_zlib();
|
||||
assert!(decompress(data, ContentEncoding::Deflate).unwrap().data == expected);
|
||||
}
|
||||
|
||||
/// Ambiguous raw deflate of more than 64 KB: a stored block that reads as zlib's header and
|
||||
/// the start of a stored block of 0xffe2 bytes, then incompressible data, which zlib reads
|
||||
/// as a block header 7 + 0xffe2 bytes in and fails on.
|
||||
fn large_raw_like_zlib() -> (Vec<u8>, Vec<u8>) {
|
||||
let mut expected = b"\x1d\0xxxxxxxxxxxxxxxxxxxxxxxxxxx".to_vec();
|
||||
let mut seed = 1u32;
|
||||
expected.extend((0..70_000).map(|_| {
|
||||
seed = seed.wrapping_mul(1_103_515_245).wrapping_add(12_345);
|
||||
(seed >> 16) as u8
|
||||
}));
|
||||
let mut data = vec![0x08, 0x1d, 0x00, 0xe2, 0xff];
|
||||
data.extend_from_slice(&expected[..29]);
|
||||
data.extend_from_slice(&raw_deflate(&expected[29..]));
|
||||
assert!(data.len() > 64 * 1024);
|
||||
(data, expected)
|
||||
}
|
||||
|
||||
/// Greptile's case on #812: raw deflate that zlib reads validly past 64 KB. After the first
|
||||
/// stored block, a final stored block of 65,535 bytes, in which zlib finds a valid stored
|
||||
/// block header at 65,513 (`00 28 00 d7 ff`) and an invalid block type only at 65,558.
|
||||
fn raw_valid_as_zlib_past_64k() -> (Vec<u8>, Vec<u8>) {
|
||||
let mut block: Vec<u8> = (0..65_535u32).map(|i| b'a' + (i % 26) as u8).collect();
|
||||
let first = 5 + 29 + 5;
|
||||
block[65_513 - first..65_518 - first].copy_from_slice(&[0x00, 0x28, 0x00, 0xd7, 0xff]);
|
||||
block[65_558 - first] = 0x07;
|
||||
let mut expected = b"\x1d\0xxxxxxxxxxxxxxxxxxxxxxxxxxx".to_vec();
|
||||
let mut data = raw_deflate_like_zlib(&expected.clone().try_into().unwrap());
|
||||
// Make the first block not the last, and follow it with the large final one.
|
||||
data.truncate(data.len() - 5);
|
||||
data.extend_from_slice(&[0x01, 0xff, 0xff, 0x00, 0x00]);
|
||||
data.extend_from_slice(&block);
|
||||
expected.extend_from_slice(&block);
|
||||
(data, expected)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_decompress_deflate_raw_valid_as_zlib_past_64k() {
|
||||
let (data, expected) = raw_valid_as_zlib_past_64k();
|
||||
// Raw alone decodes it; zlib alone gets past 64 KB before failing.
|
||||
let mut raw = Vec::new();
|
||||
DeflateDecoder::new(data.as_slice()).read_to_end(&mut raw).unwrap();
|
||||
assert!(raw == expected);
|
||||
let mut zlib = flate2::Decompress::new(true);
|
||||
let mut out = Vec::with_capacity(200 * 1024);
|
||||
let failed = zlib.decompress_vec(&data, &mut out, flate2::FlushDecompress::None);
|
||||
assert!(failed.is_err() && zlib.total_in() > 65_536, "zlib read {}", zlib.total_in());
|
||||
assert!(decompress(data, ContentEncoding::Deflate).unwrap().data == expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_deflate_decodes_raw_valid_as_zlib_past_64k() {
|
||||
use tokio::io::AsyncReadExt;
|
||||
let (data, expected) = raw_valid_as_zlib_past_64k();
|
||||
for capacity in [1, 8 * 1024, 256 * 1024] {
|
||||
let mut out = Vec::new();
|
||||
let reader = BufReader::with_capacity(capacity, std::io::Cursor::new(data.clone()));
|
||||
streaming_decoder(reader, ContentEncoding::Deflate)
|
||||
.read_to_end(&mut out)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(out == expected, "read {capacity} bytes at a time");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_deflate_decodes_zlib_and_raw() {
|
||||
let zlib_wrapped = stream_decode(zlib(DEFLATE_TEXT), ContentEncoding::Deflate).await;
|
||||
assert_eq!(zlib_wrapped.unwrap(), DEFLATE_TEXT);
|
||||
let raw = stream_decode(raw_deflate(DEFLATE_TEXT), ContentEncoding::Deflate).await;
|
||||
assert_eq!(raw.unwrap(), DEFLATE_TEXT);
|
||||
let large: Vec<u8> = (0..200_000u32).flat_map(|i| (i % 251).to_le_bytes()).collect();
|
||||
let large_zlib = stream_decode(zlib(&large), ContentEncoding::Deflate).await;
|
||||
assert_eq!(large_zlib.unwrap(), large);
|
||||
let large_raw = stream_decode(raw_deflate(&large), ContentEncoding::Deflate).await;
|
||||
assert_eq!(large_raw.unwrap(), large);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_deflate_retries_raw_that_looks_like_zlib() {
|
||||
let data = raw_deflate_like_zlib(b"twenty-nine bytes of raw text");
|
||||
// A zlib decoder alone rejects it.
|
||||
assert!(ZlibDecoder::new(data.as_slice()).read_to_end(&mut Vec::new()).is_err());
|
||||
let decoded = stream_decode(data, ContentEncoding::Deflate).await.unwrap();
|
||||
assert_eq!(decoded, b"twenty-nine bytes of raw text");
|
||||
|
||||
// Read as zlib, this one is a stored block that starts decoding, and only fails later.
|
||||
let both = *b"\x1d\0xxxxxxxxxxxxxxxxxxxxxxxxxxx";
|
||||
let data = raw_deflate_like_zlib(&both);
|
||||
let decoded = stream_decode(data, ContentEncoding::Deflate).await.unwrap();
|
||||
assert_eq!(decoded, both);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_deflate_decodes_zlib_that_looks_like_raw() {
|
||||
for extra in [0, 10_000] {
|
||||
let (data, expected) = zlib_like_raw_deflate(extra);
|
||||
let decoded = stream_decode(data, ContentEncoding::Deflate).await.unwrap();
|
||||
assert!(decoded == expected, "{extra} extra bytes");
|
||||
}
|
||||
}
|
||||
|
||||
/// However the reader reads ahead, the wrong reading is dropped and the right one kept.
|
||||
#[tokio::test]
|
||||
async fn streaming_deflate_retries_large_raw_whatever_the_chunking() {
|
||||
use tokio::io::AsyncReadExt;
|
||||
let (data, expected) = large_raw_like_zlib();
|
||||
for capacity in [1, 8 * 1024, 256 * 1024] {
|
||||
let mut out = Vec::new();
|
||||
let reader = BufReader::with_capacity(capacity, std::io::Cursor::new(data.clone()));
|
||||
streaming_decoder(reader, ContentEncoding::Deflate)
|
||||
.read_to_end(&mut out)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(out == expected, "read {capacity} bytes at a time");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_deflate_still_fails_on_garbage() {
|
||||
let garbage = b"\x78\x9c this is not deflate in any form".to_vec();
|
||||
assert!(stream_decode(garbage, ContentEncoding::Deflate).await.is_err());
|
||||
assert!(stream_decode(b"\xff".to_vec(), ContentEncoding::Deflate).await.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_gzip_is_unchanged() {
|
||||
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
|
||||
encoder.write_all(DEFLATE_TEXT).unwrap();
|
||||
let gzip = encoder.finish().unwrap();
|
||||
assert_eq!(stream_decode(gzip, ContentEncoding::Gzip).await.unwrap(), DEFLATE_TEXT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_decompress_brotli() {
|
||||
// Compress some data with brotli
|
||||
|
||||
@@ -100,6 +100,11 @@ async fn route(req: Request<Incoming>) -> Result<Response<TestBody>, Infallible>
|
||||
req.headers().get(HOST).map_or("none", |h| h.to_str().unwrap()),
|
||||
))),
|
||||
"/big" => Response::new(full(vec![b'x'; BIG_BODY])),
|
||||
// No body: `Content-Length: 0`, or a stream with nothing in it, chunked on HTTP/1.1.
|
||||
"/empty" => Response::new(full("")),
|
||||
"/empty-stream" => {
|
||||
Response::new(BodyExt::boxed(StreamBody::new(futures_util::stream::empty())))
|
||||
}
|
||||
"/late-end" => late_end(5),
|
||||
// The stream ends with half the promised bytes, so the upstream resets it mid-body.
|
||||
"/cut" => late_end(10),
|
||||
@@ -1361,6 +1366,64 @@ async fn h2_large_bodies_pass_flow_control() {
|
||||
}
|
||||
}
|
||||
|
||||
/// A flow ends when its response does, not when a connection closes: the upstream answers at
|
||||
/// once and keeps its connection open, as does the client, and the flow's time is the answer's.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn keep_alive_connections_do_not_lengthen_flows() {
|
||||
let h = Harness::start(1024);
|
||||
let upstreams = [
|
||||
("plain HTTP/1.1", spawn_upstream(None).await, false),
|
||||
("HTTP/1.1", spawn_upstream(Some(h1_only(&h.pki.server))).await, true),
|
||||
("HTTP/2", spawn_upstream(Some(h.pki.server.clone())).await, true),
|
||||
];
|
||||
// Every connection stays open to the end of the test, each reused across requests.
|
||||
let h1_client = h.client();
|
||||
let mut h2_clients = Vec::new();
|
||||
|
||||
for (upstream_name, upstream, tls) in upstreams {
|
||||
let port = upstream.port();
|
||||
let mut h2_client = if tls { Some(h.h2_client(port).await) } else { None };
|
||||
for path in ["/empty", "/empty-stream", "/hello"] {
|
||||
for client_version in ["HTTP/1.1", "HTTP/2"] {
|
||||
let case = format!("{client_version} client, {upstream_name} upstream, {path}");
|
||||
let scheme = if tls { "https" } else { "http" };
|
||||
let url = format!("{scheme}://localhost:{port}{path}");
|
||||
let body = match (client_version, &mut h2_client) {
|
||||
("HTTP/1.1", _) => {
|
||||
let resp = h1_client.post(&url).body("x=1").send().await.unwrap();
|
||||
resp.bytes().await.unwrap()
|
||||
}
|
||||
(_, Some(client)) => {
|
||||
let req = h2_request("POST", port, path, full("x=1"));
|
||||
let resp = client.send_request(req).await.unwrap();
|
||||
resp.into_body().collect().await.unwrap().to_bytes()
|
||||
}
|
||||
// HTTP/2 needs TLS here.
|
||||
(_, None) => continue,
|
||||
};
|
||||
assert_eq!(body, if path == "/hello" { "hello" } else { "" }, "{case}");
|
||||
|
||||
let events = h.flow_events();
|
||||
let version =
|
||||
find!(events, ProxyEvent::RequestStart { http_version, .. } => http_version);
|
||||
assert_eq!(version, client_version, "{case}");
|
||||
let headers_ms =
|
||||
find!(events, ProxyEvent::ResponseStart { elapsed_ms, .. } => *elapsed_ms);
|
||||
let (end_ms, error, cancelled) = find!(events,
|
||||
ProxyEvent::ResponseBodyComplete { elapsed_ms, error, cancelled, .. } =>
|
||||
(*elapsed_ms, error.clone(), *cancelled));
|
||||
assert_eq!((error, cancelled), (None, false), "{case}");
|
||||
// Generous for a slow CI runner; a flow held to the connection's close would
|
||||
// not end at all while every connection here stays open.
|
||||
assert!(end_ms - headers_ms < 2_000, "{case}: headers {headers_ms}, end {end_ms}");
|
||||
assert!(end_ms < 5_000, "{case}: took {end_ms} ms");
|
||||
}
|
||||
}
|
||||
h2_clients.extend(h2_client);
|
||||
}
|
||||
drop((h1_client, h2_clients));
|
||||
}
|
||||
|
||||
/// With HTTP/2 off, a client that offers both gets HTTP/1.1, from the next connection on.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn http1_only_makes_h2_clients_fall_back() {
|
||||
|
||||
Generated
+1
-1
@@ -145,7 +145,6 @@
|
||||
"focus-trap-react": "^12.0.3",
|
||||
"fuzzbunny": "^1.0.1",
|
||||
"graphql": "^17.0.2",
|
||||
"hexy": "^0.4.0",
|
||||
"history": "^5.3.0",
|
||||
"jotai": "^2.18.0",
|
||||
"jotai-family": "^1.0.1",
|
||||
@@ -15019,6 +15018,7 @@
|
||||
"name": "@yaakapp-internal/ui",
|
||||
"version": "1.0.0",
|
||||
"dependencies": {
|
||||
"hexy": "^0.4.0",
|
||||
"jotai-family": "^1.0.1"
|
||||
}
|
||||
},
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import { describe, expect, test } from "vitest";
|
||||
import { contentTypeCharset, decodeBodyText, strictUtf8Text } from "./decodeBodyText";
|
||||
|
||||
const latin1 = (text: string) => Uint8Array.from(text, (c) => c.charCodeAt(0));
|
||||
// "こんにちは、世界" in Shift_JIS
|
||||
const shiftJis = new Uint8Array([
|
||||
0x82, 0xb1, 0x82, 0xf1, 0x82, 0xc9, 0x82, 0xbf, 0x82, 0xcd, 0x81, 0x41, 0x90, 0xa2, 0x8a, 0x45,
|
||||
]);
|
||||
const french = "Accès refusé : vous n'avez pas l'autorisation d'accéder à cette page.";
|
||||
|
||||
describe("decodeBodyText", () => {
|
||||
test("decodes in the charset the content type names", () => {
|
||||
const body = latin1(french);
|
||||
expect(decodeBodyText(body, "text/html; charset=iso-8859-1")).toBe(french);
|
||||
expect(decodeBodyText(body, 'text/plain; CHARSET="ISO-8859-1"')).toBe(french);
|
||||
expect(decodeBodyText(shiftJis, "text/plain; charset=Shift_JIS")).toBe("こんにちは、世界");
|
||||
});
|
||||
|
||||
test("reads an unlabeled body that isn't UTF-8 as windows-1252", () => {
|
||||
expect(decodeBodyText(latin1(french), "text/html")).toBe(french);
|
||||
expect(decodeBodyText(latin1(french), null)).toBe(french);
|
||||
expect(decodeBodyText(new Uint8Array([0x93, 0x68, 0x69, 0x94]), null)).toBe("“hi”");
|
||||
});
|
||||
|
||||
test("reads UTF-8 as UTF-8, labeled or not", () => {
|
||||
const utf8 = new TextEncoder().encode(`{"city":"Zürich","greeting":"こんにちは"}`);
|
||||
const text = `{"city":"Zürich","greeting":"こんにちは"}`;
|
||||
expect(decodeBodyText(utf8, null)).toBe(text);
|
||||
expect(decodeBodyText(utf8, "application/json")).toBe(text);
|
||||
expect(decodeBodyText(utf8, "application/json; charset=utf-8")).toBe(text);
|
||||
// A body labeled UTF-8 stays UTF-8, with what isn't replaced, as before
|
||||
expect(decodeBodyText(new Uint8Array([0x61, 0xff]), "text/plain; charset=utf-8")).toBe("a�");
|
||||
});
|
||||
|
||||
test("treats a charset TextDecoder doesn't know as no charset", () => {
|
||||
expect(contentTypeCharset("text/plain; charset=not-a-charset")).toBeNull();
|
||||
expect(decodeBodyText(latin1(french), "text/plain; charset=not-a-charset")).toBe(french);
|
||||
});
|
||||
|
||||
test("keeps a truncated UTF-8 body UTF-8 when it ends partway through a character", () => {
|
||||
const cut = new TextEncoder().encode("abc€").subarray(0, 5);
|
||||
expect(strictUtf8Text(cut)).toBeNull();
|
||||
expect(decodeBodyText(cut, null, { truncated: true })).toBe("abc");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,44 @@
|
||||
/**
|
||||
* The charset a content type names, as `TextDecoder` calls it (`windows-1252` for
|
||||
* `iso-8859-1`), or null when it names none or one `TextDecoder` doesn't know.
|
||||
*/
|
||||
export function contentTypeCharset(contentType: string | null): string | null {
|
||||
const label = contentType?.match(/;\s*charset\s*=\s*"?([^";\s]+)"?/i)?.[1];
|
||||
if (label == null) return null;
|
||||
try {
|
||||
return new TextDecoder(label).encoding;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The bytes as UTF-8, or null when they aren't valid UTF-8. A truncated body may end partway
|
||||
* through a character, so up to three trailing bytes are dropped before giving up on it.
|
||||
*/
|
||||
export function strictUtf8Text(bytes: Uint8Array, truncated = false): string | null {
|
||||
const decoder = new TextDecoder("utf-8", { fatal: true });
|
||||
const maxTrim = truncated ? Math.min(3, bytes.length) : 0;
|
||||
for (let trim = 0; trim <= maxTrim; trim++) {
|
||||
try {
|
||||
return decoder.decode(bytes.subarray(0, bytes.length - trim));
|
||||
} catch {
|
||||
// Try again without the last byte
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* A body's text in the charset its content type names. Without one it is UTF-8, unless the
|
||||
* bytes aren't valid UTF-8, when it is windows-1252, as browsers read unlabeled text.
|
||||
*/
|
||||
export function decodeBodyText(
|
||||
bytes: Uint8Array,
|
||||
contentType: string | null,
|
||||
{ truncated = false }: { truncated?: boolean } = {},
|
||||
): string {
|
||||
const charset = contentTypeCharset(contentType);
|
||||
if (charset != null) return new TextDecoder(charset).decode(bytes);
|
||||
return strictUtf8Text(bytes, truncated) ?? new TextDecoder("windows-1252").decode(bytes);
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
export * from "./debounce";
|
||||
export * from "./decodeBodyText";
|
||||
export * from "./eagerDebounceAsync";
|
||||
export * from "./formatSize";
|
||||
export * from "./templateFunction";
|
||||
@@ -16,6 +16,7 @@
|
||||
"*.css"
|
||||
],
|
||||
"dependencies": {
|
||||
"hexy": "^0.4.0",
|
||||
"jotai-family": "^1.0.1"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import { hexy } from "hexy";
|
||||
import { describe, expect, test, vi } from "vite-plus/test";
|
||||
import { hexdump } from "./HexdumpViewer";
|
||||
|
||||
vi.mock("hexy", () => ({ hexy: vi.fn((data: number[]) => `dump of ${data.length} bytes`) }));
|
||||
|
||||
describe("hexdump", () => {
|
||||
test("makes the hexdump of the same bytes once", () => {
|
||||
const message = [0x01, 0x02, 0x03];
|
||||
expect(hexdump(message)).toBe("dump of 3 bytes");
|
||||
expect(hexdump(message)).toBe("dump of 3 bytes");
|
||||
expect(hexy).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Other bytes, even equal ones, get their own.
|
||||
expect(hexdump([0x01, 0x02, 0x03])).toBe("dump of 3 bytes");
|
||||
expect(hexy).toHaveBeenCalledTimes(2);
|
||||
expect(hexdump([])).toBe("");
|
||||
expect(hexy).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,50 @@
|
||||
import { hexy } from "hexy";
|
||||
import type { ReactNode } from "react";
|
||||
import { useMemo } from "react";
|
||||
import type { CodeEditorProps } from "../editor/CodeEditor";
|
||||
import { LazyCodeEditor } from "../editor/LazyCodeEditor";
|
||||
|
||||
/** Hexdumps already made, by the bytes they were made of, for as long as those bytes live */
|
||||
const hexdumps = new WeakMap<Uint8Array | number[], string>();
|
||||
|
||||
/**
|
||||
* Bytes as a hexdump: offset, hex and ASCII columns, one line per 16 bytes. Made once for the
|
||||
* same bytes, so a viewer and its copy action share one.
|
||||
*/
|
||||
export function hexdump(data: Uint8Array | number[]): string {
|
||||
if (data.length === 0) return "";
|
||||
let dump = hexdumps.get(data);
|
||||
if (dump == null) {
|
||||
dump = hexy(data);
|
||||
hexdumps.set(data, dump);
|
||||
}
|
||||
return dump;
|
||||
}
|
||||
|
||||
export interface HexdumpViewerProps {
|
||||
data: Uint8Array | number[];
|
||||
className?: string;
|
||||
/** Renders the editor, for a host whose editor builds on CodeEditor */
|
||||
renderEditor?: (props: CodeEditorProps) => ReactNode;
|
||||
}
|
||||
|
||||
/** A read-only hexdump of binary data */
|
||||
export function HexdumpViewer({
|
||||
data,
|
||||
className,
|
||||
renderEditor = renderCodeEditor,
|
||||
}: HexdumpViewerProps) {
|
||||
const text = useMemo(() => hexdump(data), [data]);
|
||||
return renderEditor({
|
||||
language: "text",
|
||||
defaultValue: text,
|
||||
wrapLines: false,
|
||||
readOnly: true,
|
||||
stateKey: null,
|
||||
className,
|
||||
});
|
||||
}
|
||||
|
||||
function renderCodeEditor(props: CodeEditorProps) {
|
||||
return <LazyCodeEditor {...props} />;
|
||||
}
|
||||
@@ -89,6 +89,8 @@ export function TextViewer({
|
||||
],
|
||||
queryFn: () => (filterText ?? rejectFilter)(filter.appliedFilter ?? ""),
|
||||
enabled: hasFilter && filter.appliedFilter != null,
|
||||
// An expression that fails, fails every time, so the error shows at once.
|
||||
retry: false,
|
||||
});
|
||||
|
||||
const canFilter = hasFilter && (language === "json" || language === "xml" || language === "html");
|
||||
@@ -195,6 +197,7 @@ export function TextViewer({
|
||||
}
|
||||
return formatText(text, language);
|
||||
},
|
||||
retry: false,
|
||||
}).data;
|
||||
if (formattedBody == null) {
|
||||
return null;
|
||||
|
||||
@@ -90,6 +90,8 @@ export { Tree } from "./components/tree/Tree";
|
||||
export type { TreeItemProps } from "./components/tree/TreeItem";
|
||||
export { AudioViewer } from "./components/viewers/AudioViewer";
|
||||
export { CsvViewer, CsvViewerInner } from "./components/viewers/CsvViewer";
|
||||
export type { HexdumpViewerProps } from "./components/viewers/HexdumpViewer";
|
||||
export { hexdump, HexdumpViewer } from "./components/viewers/HexdumpViewer";
|
||||
export { ImageViewer } from "./components/viewers/ImageViewer";
|
||||
export { JsonAttributeTree } from "./components/viewers/JsonAttributeTree";
|
||||
export { JsonViewer } from "./components/viewers/JsonViewer";
|
||||
|
||||
Reference in new issue
Block a user