diff --git a/apps/yaak-proxy/components/AddColumnDialog.tsx b/apps/yaak-proxy/components/AddColumnDialog.tsx new file mode 100644 index 000000000..2cba433ad --- /dev/null +++ b/apps/yaak-proxy/components/AddColumnDialog.tsx @@ -0,0 +1,97 @@ +import { Compartment, type Extension } from "@codemirror/state"; +import type { Schema } from "@yaakapp-internal/query"; +import { + Dialog, + DialogFooter, + Input, + type InputHandle, + LazyCodeEditor, +} from "@yaakapp-internal/ui"; +import { queryFilter } from "@yaakapp-internal/ui/editor"; +import { atom, useAtom, useSetAtom } from "jotai"; +import { useEffect, useMemo, useRef, useState } from "react"; +import { useRpcQuery } from "../hooks/useRpcQuery"; +import { addCustomColumn, customPath } from "../lib/columns"; +import { loadSeenValues } from "../lib/searchBox"; +import { columnPrefsAtom } from "../lib/store"; + +/** The Add Column dialog is open */ +export const addColumnDialogAtom = atom(false); + +const FORM_ID = "add-column"; + +/** + * Adds a column for a header or query parameter, named by its field as the search names it, + * `header.x-request-id` or `query.page`, completed as the search box completes it + */ +export function AddColumnDialog() { + const [open, setOpen] = useAtom(addColumnDialogAtom); + if (!open) return null; + return setOpen(false)} />; +} + +function AddColumnForm({ onClose }: { onClose: () => void }) { + const [text, setText] = useState(""); + const setPrefs = useSetAtom(columnPrefsAtom); + const handleRef = useRef(null); + const schema = useRpcQuery("query_schema", {}, { staleTime: Infinity }).data?.schema; + // Only the fields a column can show: the maps of headers and query parameters. It goes in + // once the schema is read, through a compartment, as the search box's does. + const compartment = useRef(new Compartment()); + const filter = useMemo(() => { + if (schema == null) return []; + const maps: Schema = { ...schema, fields: schema.fields.filter((f) => f.map) }; + return queryFilter({ schema: maps, loadValues: loadSeenValues }); + }, [schema]); + const extensions = useMemo(() => [compartment.current.of(filter)], [filter]); + useEffect(() => { + handleRef.current?.dispatch({ effects: compartment.current.reconfigure(filter) }); + }, [filter]); + const path = customPath(text); + + const submit = () => { + if (path == null) return; + setPrefs((p) => addCustomColumn(p, path)); + onClose(); + }; + + return ( + +
{ + e.preventDefault(); + submit(); + }} + > + } + setRef={(h) => { + handleRef.current = h; + }} + label="Field" + autoFocus + className="font-mono text-xs" + stateKey={null} + language={null} + placeholder="@header.x-request-id" + defaultValue="" + onChange={setText} + extensions={extensions} + onKeyDown={(e) => { + if (e.key === "Enter" && !e.defaultPrevented) { + e.preventDefault(); + submit(); + } + }} + /> + + +
+ ); +} diff --git a/apps/yaak-proxy/components/AppIcon.tsx b/apps/yaak-proxy/components/AppIcon.tsx index e132ee021..c90b0a824 100644 --- a/apps/yaak-proxy/components/AppIcon.tsx +++ b/apps/yaak-proxy/components/AppIcon.tsx @@ -24,8 +24,8 @@ function loadIcon(path: string): Promise { return promise; } -/** An app's icon, read once per path for the whole session */ -export function AppIcon({ path, className }: { path: string; className?: string }) { +/** An app's icon, read once per path for the whole session; `className` sets its size */ +export function AppIcon({ path, className = "size-4" }: { path: string; className?: string }) { const [url, setUrl] = useState(() => icons.get(path)); useEffect(() => { if (icons.has(path)) { @@ -42,8 +42,8 @@ export function AppIcon({ path, className }: { path: string; className?: string }, [path]); return url == null ? ( - + ) : ( - + ); } diff --git a/apps/yaak-proxy/components/FlowInfo.tsx b/apps/yaak-proxy/components/FlowInfo.tsx index 601f997fd..7286729d9 100644 --- a/apps/yaak-proxy/components/FlowInfo.tsx +++ b/apps/yaak-proxy/components/FlowInfo.tsx @@ -35,7 +35,7 @@ export function FlowInfo({ flow }: { flow: Flow }) { "Name", flow.clientName == null ? null : ( - {flow.clientPath != null && } + {flow.clientPath != null && } {flow.clientName} {folder != null && {folder}} diff --git a/apps/yaak-proxy/components/FlowList.tsx b/apps/yaak-proxy/components/FlowList.tsx index b4f42b7bb..949261c9d 100644 --- a/apps/yaak-proxy/components/FlowList.tsx +++ b/apps/yaak-proxy/components/FlowList.tsx @@ -1,10 +1,11 @@ import { useVirtualizer } from "@tanstack/react-virtual"; import { formatSize } from "@yaakapp-internal/lib/formatSize"; -import type { Flow } from "@yaakapp-internal/proxy-lib"; +import type { Flow, FlowSortKey, SortColumn } from "@yaakapp-internal/proxy-lib"; import { ContextMenu, CONTROL_TOOLTIP_DELAY, CountBadge, + type DropdownItem, formatMillis, HttpMethodTagRaw, HttpStatusTagRaw, @@ -12,7 +13,7 @@ import { Tooltip, } from "@yaakapp-internal/ui"; import classNames from "classnames"; -import { useAtom, useAtomValue, useSetAtom } from "jotai"; +import { useAtom, useAtomValue, useSetAtom, useStore } from "jotai"; import type { CSSProperties, KeyboardEvent, MouseEvent, ReactElement, WheelEvent } from "react"; import { memo, useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from "react"; import { selectedFlowAtom, selectedFlowIdAtom } from "../lib/atoms"; @@ -21,33 +22,49 @@ import { utcTimestamp } from "../lib/flowViews"; import { type Column, type ColumnId, + cellFilterTerm, + customValue, gridTemplate, gridWidth, hostOf, + isCustomColumn, nextSort, + operationOf, + showOperationFor, shownColumns, + tlsVersionOf, + toggleSortKey, + unavailableColumns, + urlParts, } from "../lib/columns"; import { useExportShownHar, useFlowActions } from "../lib/flowActions"; import { compareFlows } from "../lib/flowWindow"; -import { countNewFlows } from "../lib/follow"; +import { countNewFlows, isAtEnd, resumesFollowing } from "../lib/follow"; +import { asOneTerm } from "../lib/query"; import { folderOf } from "../lib/search"; import { columnPrefsAtom, hasAppsAtom, flowSortAtom, followAtom, + loadMoreSortedAtom, loadOlderFlowsAtom, newestStartedAtAtom, oldestFlowAtom, proxyStateAtom, + searchLoadingAtom, + searchTextAtom, + setSearchTextAtom, + flowsAtom, shownFlowsAtom, } from "../lib/store"; +import { useShowToast } from "../lib/toast"; import { AppIcon } from "./AppIcon"; import { FlowListEmpty, PausedRow } from "./FlowListEmpty"; import { FlowListHeader } from "./FlowListHeader"; const BOTTOM_PADDING = 6; -/** Rows from the top at which older flows start loading */ +/** Rows from the top at which older flows start loading, or from the bottom, sorted, more */ const LOAD_OLDER_ROWS = 20; /** How long a row that arrives while following stays tinted */ const ARRIVAL_MS = 1400; @@ -68,20 +85,37 @@ interface Props { /** * Follows new traffic until a flow is selected or the list is scrolled, then holds. Scrolling to * the top loads older flows from the database. Catching up leaves a divider where the hold - * began. Sorting by a column sorts the flows loaded and stops following. The columns keep their - * widths, and the list scrolls sideways when they are wider than it, the header with it. + * began. Sorting by columns reads the flows from the database in that order, a page at a time on + * scrolling down, and stops following. The columns keep their widths, and the list scrolls + * sideways when they are wider than it, the header with it. */ export function FlowList({ style, className }: Props) { const flows = useAtomValue(shownFlowsAtom); - const [menu, setMenu] = useState<{ flow: Flow; x: number; y: number } | null>(null); + const [menu, setMenu] = useState<{ + flow: Flow; + column: ColumnId | null; + x: number; + y: number; + } | null>(null); const flowActions = useFlowActions(); + const cellActions = useCellActions(); const exportShown = useExportShownHar(); const selectedId = useAtomValue(selectedFlowIdAtom); const setSelectedFlow = useSetAtom(selectedFlowAtom); const loadOlder = useSetAtom(loadOlderFlowsAtom); + const loadMoreSorted = useSetAtom(loadMoreSortedAtom); const oldestFlow = useAtomValue(oldestFlowAtom); - const prefs = useAtomValue(columnPrefsAtom); + const [prefs, setPrefs] = useAtom(columnPrefsAtom); const [sort, setSort] = useAtom(flowSortAtom); + const sorted = sort.length > 0; + const allFlows = useAtomValue(flowsAtom); + + // Operation shows itself the first time a flow has one, unless the person set it already. + useEffect(() => { + if (prefs.visible.operation !== undefined) return; + const next = showOperationFor(prefs, allFlows); + if (next !== prefs) setPrefs(next); + }, [prefs, allFlows, setPrefs]); const proxyState = useAtomValue(proxyStateAtom); const paused = proxyState?.state === "running" && proxyState.recording === false; const scrollRef = useRef(null); @@ -89,6 +123,8 @@ export function FlowList({ style, className }: Props) { const lastScrollTop = useRef(0); // The list's vertical scrollbar, which the header leaves room for so the two scroll as far const [scrollbarWidth, setScrollbarWidth] = useState(0); + // The list's visible height, so a resize that leaves it at its end follows again + const [viewportHeight, setViewportHeight] = useState(0); const ignoreScroll = useRef(false); const [following, setFollowing] = useAtom(followAtom); const [heldAt, setHeldAt] = useState(null); @@ -105,14 +141,11 @@ export function FlowList({ style, className }: Props) { // Where the platform cannot tell which app sent a flow, there is no App column. const hasApps = useAtomValue(hasAppsAtom); - const columns = useMemo( - () => shownColumns(prefs).filter((c) => hasApps || c.id !== "app"), - [prefs, hasApps], - ); + const columns = useMemo(() => shownColumns(prefs, unavailableColumns(hasApps)), [prefs, hasApps]); const template = useMemo(() => gridTemplate(columns), [columns]); // A row's grid, its padding and the list's margins: `px-1.5` and `left-1.5 right-1.5` const contentWidth = useMemo(() => `calc(${gridWidth(columns)} + 1.5rem)`, [columns]); - const arrivals = useArrivals(flows, following && sort == null); + const arrivals = useArrivals(flows, following && !sorted); const virtualizer = useVirtualizer({ count: flows.length, @@ -142,48 +175,83 @@ export function FlowList({ style, className }: Props) { const follow = useCallback(() => { const { flows, heldAt, following, sort } = latest.current; // Catching up from a hold marks where it began. - if (!following && sort == null) { + if (!following && sort.length === 0) { const count = countNewFlows(flows, heldAt); const first = flows[flows.length - count]; setDivider(count > 0 && first != null ? { id: first.id, count } : null); } - setSort(null); + setSort([]); setFollowing(true); setHeldAt(null); setScrollTop(Number.MAX_SAFE_INTEGER); }, [setFollowing, setScrollTop, setSort]); + // A sorted list never follows, whether the sort was just set, kept from the last launch, or + // kept through a clear. useLayoutEffect(() => { - if (following) setScrollTop(Number.MAX_SAFE_INTEGER); - }, [following, flows, paused, setScrollTop]); + if (sorted && following) setFollowing(false); + }, [sorted, following, setFollowing]); + + // A held list follows again once it is at its end, not only when scrolled there: a sidebar row, + // search or resize can leave it there with no scroll to make, as one that fits its viewport + // always is, and its flows would otherwise count as new while all are on screen. Judged when + // which flows are listed changes, or the list's height, not when one listed is updated or + // `following` itself changes, so selecting a row at the end still holds; and not while a + // search's first page loads, as its empty list is at its end only until the page is in. + const searchLoading = useAtomValue(searchLoadingAtom); + const listed = `${flows.length}:${flows[0]?.id ?? ""}:${flows.at(-1)?.id ?? ""}`; + useLayoutEffect(() => { + const el = scrollRef.current; + if (el == null || searchLoading) return; + if (!resumesFollowing({ following: latest.current.following, sorted, el })) return; + setFollowing(true); + setHeldAt(null); + }, [listed, sorted, searchLoading, viewportHeight, setFollowing]); + + useLayoutEffect(() => { + if (following && !sorted) setScrollTop(Number.MAX_SAFE_INTEGER); + }, [following, sorted, flows, paused, setScrollTop]); + + // Sorted, rows that leave, deleted or moved past what is read, can leave the end of the list + // in view with no scroll to load the next page; it loads as they go. + useEffect(() => { + const el = scrollRef.current; + if (!sorted || el == null) return; + const toEnd = el.scrollHeight - el.scrollTop - el.clientHeight; + if (toEnd < LOAD_OLDER_ROWS * ROW_HEIGHT) void loadMoreSorted(); + }, [sorted, flows, loadMoreSorted, ROW_HEIGHT]); // A new sort starts from its top. + const sortKey = JSON.stringify(sort); useLayoutEffect(() => { - if (sort != null) setScrollTop(0); - }, [sort, setScrollTop]); + if (sortKey !== "[]") setScrollTop(0); + }, [sortKey, setScrollTop]); // Older flows that loaded in above push the rows down; scroll by as much, so they stay put. const anchor = useRef(null); useLayoutEffect(() => { const previous = anchor.current; anchor.current = oldestFlow; - if (following || sort != null || previous == null || oldestFlow == null) return; + if (following || sorted || previous == null || oldestFlow == null) return; if (compareFlows(oldestFlow, previous) >= 0) return; const added = flows.findIndex((f) => compareFlows(f, previous) >= 0); const el = scrollRef.current; if (added > 0 && el != null) setScrollTop(el.scrollTop + added * ROW_HEIGHT); - }, [oldestFlow, flows, following, sort, setScrollTop, ROW_HEIGHT]); + }, [oldestFlow, flows, following, sorted, setScrollTop, ROW_HEIGHT]); // Sorted, the flows loaded are what is sorted; older ones would land anywhere. const loadOlderHeld = useCallback(() => { hold(); - if (latest.current.sort == null) void loadOlder(); + if (latest.current.sort.length === 0) void loadOlder(); }, [hold, loadOlder]); useEffect(() => { const el = scrollRef.current; if (el == null) return; - const measure = () => setScrollbarWidth(el.offsetWidth - el.clientWidth); + const measure = () => { + setScrollbarWidth(el.offsetWidth - el.clientWidth); + setViewportHeight(el.clientHeight); + }; measure(); const observer = new ResizeObserver(measure); observer.observe(el); @@ -203,17 +271,20 @@ export function FlowList({ style, className }: Props) { return; } if (!moved) return; - const atEnd = el.scrollHeight - el.scrollTop - el.clientHeight < 2; - if (atEnd && latest.current.sort == null) { + const sortedNow = latest.current.sort.length > 0; + if (isAtEnd(el) && !sortedNow) { setFollowing(true); setHeldAt(null); } else { hold(); - if (el.scrollTop < LOAD_OLDER_ROWS * ROW_HEIGHT && latest.current.sort == null) { + if (el.scrollTop < LOAD_OLDER_ROWS * ROW_HEIGHT && !sortedNow) { void loadOlder(); } } - }, [hold, loadOlder, setFollowing, ROW_HEIGHT]); + // Sorted, the next page of the sort loads near the bottom. + const toEnd = el.scrollHeight - el.scrollTop - el.clientHeight; + if (sortedNow && toEnd < LOAD_OLDER_ROWS * ROW_HEIGHT) void loadMoreSorted(); + }, [hold, loadOlder, loadMoreSorted, setFollowing, ROW_HEIGHT]); // Focus moving to a sort button out of view scrolls the header itself; the rows follow. const handleHeaderScroll = useCallback(() => { @@ -269,9 +340,12 @@ export function FlowList({ style, className }: Props) { const index = flows.findIndex((f) => f.id === selectedId); if (index === -1) select(e.key === "ArrowDown" ? 0 : flows.length - 1); else if (index === 0 && e.key === "ArrowUp") loadOlderHeld(); - else select(e.key === "ArrowDown" ? index + 1 : index - 1); + else { + if (sorted && index >= flows.length - 2 && e.key === "ArrowDown") void loadMoreSorted(); + select(e.key === "ArrowDown" ? index + 1 : index - 1); + } }, - [flows, follow, loadOlderHeld, select, selectedId], + [flows, follow, loadOlderHeld, loadMoreSorted, select, selectedId, sorted], ); const openMenu = useCallback( @@ -280,16 +354,21 @@ export function FlowList({ style, className }: Props) { if (flow == null) return; e.preventDefault(); select(index); - setMenu({ flow, x: e.clientX, y: e.clientY }); + // The cell under the pointer, for the menu's items on its value + const cellEl = (e.target as HTMLElement).closest("[data-column]"); + const column = (cellEl?.dataset.column as ColumnId | undefined) ?? null; + setMenu({ flow, column, x: e.clientX, y: e.clientY }); }, [select], ); - // A third click on a column goes back to start time, following as Sort by Start Time does. + // A click sorts by the column alone; a third goes back to arrival order, following as Reset + // Sort does. Option-click adds the column to the sort, reverses it, then takes it out. const handleSort = useCallback( - (column: ColumnId) => { - const next = nextSort(latest.current.sort, column); - if (next == null) { + (column: SortColumn, add: boolean) => { + const current = latest.current.sort; + const next = add ? toggleSortKey(current, column) : nextSort(current, column); + if (next.length === 0) { follow(); return; } @@ -299,7 +378,30 @@ export function FlowList({ style, className }: Props) { [follow, hold, setSort], ); - const newCount = following || sort != null ? 0 : countNewFlows(flows, heldAt); + const handleSetSort = useCallback( + (next: FlowSortKey[]) => { + if (next.length === 0) { + follow(); + return; + } + hold(); + setSort(next); + }, + [follow, hold, setSort], + ); + + const menuItems = useMemo((): DropdownItem[] => { + if (menu == null) return []; + // The cell's value first, as it is what was right-clicked + const cell = menu.column == null ? [] : cellActions(menu.flow, menu.column); + return [ + ...(cell.length > 0 ? [...cell, { type: "separator" } as const] : []), + ...flowActions(menu.flow), + exportShown(), + ]; + }, [menu, flowActions, cellActions, exportShown]); + + const newCount = following || sorted ? 0 : countNewFlows(flows, heldAt); const dividerIndex = divider == null ? -1 : flows.findIndex((f) => f.id === divider.id); const totalSize = virtualizer.getTotalSize(); @@ -326,7 +428,8 @@ export function FlowList({ style, className }: Props) { prefs={prefs} sort={sort} onSort={handleSort} - onSortByStart={follow} + onSetSort={handleSetSort} + onResetSort={follow} />
@@ -398,11 +501,7 @@ export function FlowList({ style, className }: Props) { {newCount === 1 ? "1 new flow" : `${newCount} new flows`} )} - setMenu(null)} - /> + setMenu(null)} />
); @@ -475,8 +574,9 @@ const FlowRow = memo(function FlowRow({ // Failed or cancelled before any response; after the headers, the body was cut short mid-way. const noResponse = (failed || flow.cancelled) && flow.status == null; const ids = columns.map((c) => c.id); - // A failure's reason, or Cancelled, takes the type and size cells together. - const spanReason = noResponse && ids.includes("type") && ids.includes("size"); + // A failure's reason, or Cancelled, takes the type and size cells together, side by side. + const typeAt = ids.indexOf("type"); + const spanReason = noResponse && typeAt !== -1 && ids[typeAt + 1] === "size"; const sinceArrival = arrivedAt == null ? null : Date.now() - arrivedAt; return ( @@ -510,7 +610,10 @@ const FlowRow = memo(function FlowRow({ {columns.map((c) => { if (spanReason && c.id === "size") return null; return ( - + // Names the cell's column for its menu, without a box of its own in the row's grid +
+ +
); })} @@ -585,9 +688,7 @@ function FlowCell({ column, flow, span }: { column: ColumnId; flow: Flow; span: switch (column) { case "number": return ( - - {flow.number || ""} - + {flow.number || ""} ); case "app": { if (flow.clientPath == null) return ; @@ -601,7 +702,7 @@ function FlowCell({ column, flow, span }: { column: ColumnId; flow: Flow; span: delay={CONTROL_TOOLTIP_DELAY} > - + ); @@ -632,26 +733,20 @@ function FlowCell({ column, flow, span }: { column: ColumnId; flow: Flow; span: if (flow.tunnelled) { return Not decrypted; } - const { path, query } = splitPath(flow.url); + const path = pathOf(flow.url); if (flow.streamKind == null) { return ( - - - {path} - {query} - + + {path} ); } // A stream's message count rides the end of the path, so it survives the narrow width, // ticking while the stream is open. return ( - + - - {path} - {query} - + {path} ); + case "query": { + const query = flow.tunnelled ? "" : (urlParts(flow.url).query ?? ""); + return ( + + {query} + + ); + } + case "started": { + const at = new Date(utcTimestamp(flow.startedAt)); + if (Number.isNaN(at.getTime())) return ; + return ( + + {clockTime(at)} + + ); + } + case "requestSize": + return ( + + {flow.requestSize === 0 ? "--" : formatSize(flow.requestSize)} + + ); + case "latency": + return ( + + {flow.elapsedHeadersMs == null ? "--" : formatMillis(flow.elapsedHeadersMs)} + + ); + case "tls": + return ( + {tlsVersionOf(flow)} + ); + case "operation": { + const operation = operationOf(flow) ?? ""; + return ( + + {operation} + + ); + } + default: { + if (!isCustomColumn(column)) return ; + const value = customValue(flow, column.slice("field:".length)) ?? ""; + return ( + + {value} + + ); + } } } -function splitPath(url: string): { path: string; query: string } { +/** The URL's path as the browser reads it, `/` for none, without the query */ +function pathOf(url: string): string { try { - const u = new URL(url); - return { path: u.pathname, query: u.search }; + return new URL(url).pathname; } catch { - return { path: url, query: "" }; + return url; + } +} + +/** `14:03:27.481`, in local time */ +function clockTime(at: Date): string { + const two = (n: number) => String(n).padStart(2, "0"); + const ms = String(at.getMilliseconds()).padStart(3, "0"); + return `${two(at.getHours())}:${two(at.getMinutes())}:${two(at.getSeconds())}.${ms}`; +} + +/** + * The items a cell's menu adds for its value: a search term for it, either way, where its column + * has a field to match it by, and copying it + */ +function useCellActions(): (flow: Flow, column: ColumnId) => DropdownItem[] { + const store = useStore(); + const setSearchText = useSetAtom(setSearchTextAtom); + const showToast = useShowToast(); + return useCallback( + (flow: Flow, column: ColumnId) => { + const add = (term: string) => { + // A search with `OR` at its top is grouped first, so the term narrows all of it. + const text = store.get(searchTextAtom).trim(); + setSearchText(text === "" ? term : `${asOneTerm(text)} ${term}`); + }; + const include = cellFilterTerm(flow, column, false); + const exclude = cellFilterTerm(flow, column, true); + const value = cellText(flow, column); + const items: DropdownItem[] = []; + if (include != null && exclude != null) { + items.push( + { label: "Filter by This Value", icon: "filter", onSelect: () => add(include) }, + { label: "Exclude This Value", icon: "minus_circle", onSelect: () => add(exclude) }, + ); + } + if (value !== "") { + items.push({ + label: "Copy Value", + icon: "copy", + onSelect: async () => { + try { + await navigator.clipboard.writeText(value); + showToast({ message: "Copied value" }); + } catch (err) { + showToast({ message: `Failed to copy value: ${String(err)}`, color: "danger" }); + } + }, + }); + } + return items; + }, + [store, setSearchText, showToast], + ); +} + +/** A cell's value as text, as it shows it, for copying */ +function cellText(flow: Flow, column: ColumnId): string { + if (isCustomColumn(column)) return customValue(flow, column.slice("field:".length)) ?? ""; + switch (column) { + case "number": + return flow.number ? String(flow.number) : ""; + case "app": + return flow.clientName ?? flow.clientPath ?? ""; + case "method": + return flow.method; + case "host": + return hostOf(flow.url); + case "path": + return flow.tunnelled ? "" : pathOf(flow.url); + case "query": + return flow.tunnelled ? "" : (urlParts(flow.url).query ?? ""); + case "status": + return flow.status == null ? "" : String(flow.status); + case "type": + return flow.contentType ?? ""; + case "size": + return String(flow.responseSize); + case "requestSize": + return String(flow.requestSize); + case "time": + return flow.elapsedMs == null ? "" : String(flow.elapsedMs); + case "latency": + return flow.elapsedHeadersMs == null ? "" : String(flow.elapsedHeadersMs); + case "started": { + const at = new Date(utcTimestamp(flow.startedAt)); + return Number.isNaN(at.getTime()) ? "" : at.toISOString(); + } + case "remote": + return flow.remoteAddr ?? ""; + case "version": + return flow.upstreamHttpVersion ?? flow.clientHttpVersion; + case "tls": + return tlsVersionOf(flow); + case "operation": + return operationOf(flow) ?? ""; } } diff --git a/apps/yaak-proxy/components/FlowListHeader.tsx b/apps/yaak-proxy/components/FlowListHeader.tsx index 61bcb8847..65033f37c 100644 --- a/apps/yaak-proxy/components/FlowListHeader.tsx +++ b/apps/yaak-proxy/components/FlowListHeader.tsx @@ -1,3 +1,4 @@ +import type { FlowSortKey, SortColumn } from "@yaakapp-internal/proxy-lib"; import { ContextMenu, Dropdown, @@ -7,21 +8,27 @@ import { ResizeHandle, } from "@yaakapp-internal/ui"; import classNames from "classnames"; -import { useSetAtom } from "jotai"; +import { useAtomValue, useSetAtom } from "jotai"; import { memo, type MouseEvent, useCallback, useRef, useState } from "react"; import { + addSortKey, + allColumns, type Column, - COLUMNS, + type ColumnDef, type ColumnId, type ColumnPrefs, - type FlowSort, + canHide, + hideOtherColumns, + isCustomColumn, + removeCustomColumn, resetWidth, resetWidths, resizeColumn, toggleColumn, + unavailableColumns, } from "../lib/columns"; -import { useExportShownHar } from "../lib/flowActions"; -import { columnPrefsAtom } from "../lib/store"; +import { columnPrefsAtom, hasAppsAtom } from "../lib/store"; +import { addColumnDialogAtom } from "./AddColumnDialog"; interface Props { columns: Column[]; @@ -29,16 +36,19 @@ interface Props { /** The rows' width, as CSS, so the header scrolls sideways with them */ width: string; prefs: ColumnPrefs; - sort: FlowSort | null; - onSort: (column: ColumnId) => void; - onSortByStart: () => void; + sort: FlowSortKey[]; + /** A click on a label, or with `add`, an Option-click */ + onSort: (column: SortColumn, add: boolean) => void; + /** Sets the sort from the menus; empty goes back to arrival order */ + onSetSort: (sort: FlowSortKey[]) => void; + onResetSort: () => void; } /** - * The list's only control surface: a click on a label sorts by it, a second reverses it and a - * third goes back to start time. Dragging the edge after a label resizes its column and a - * double-click on it resets it. The picker sits in the empty corner on hover, or anywhere on a - * right-click. + * The list's only control surface: a click on a label sorts by it alone, a second reverses it + * and a third goes back to arrival order; Option-click adds it to the sort as a further key. + * Dragging the edge after a label resizes its column and a double-click on it resets it. A + * right-click on a label opens the column's menu; the picker sits in the empty corner on hover. */ export const FlowListHeader = memo(function FlowListHeader({ columns, @@ -47,49 +57,113 @@ export const FlowListHeader = memo(function FlowListHeader({ prefs, sort, onSort, - onSortByStart, + onSetSort: setSort, + onResetSort, }: Props) { const setPrefs = useSetAtom(columnPrefsAtom); - const exportShown = useExportShownHar(); - const [menu, setMenu] = useState<{ x: number; y: number } | null>(null); + const unavailable = unavailableColumns(useAtomValue(hasAppsAtom)); + const openAddColumn = useSetAtom(addColumnDialogAtom); + const [menu, setMenu] = useState<{ x: number; y: number; column: ColumnDef | null } | null>(null); // Widths when a drag began, so each move sets the width from where it started. const startWidth = useRef(0); - const items: DropdownItem[] = [ + const pickerItems: DropdownItem[] = [ { type: "content", label:
Columns
, }, - ...COLUMNS.map((c): DropdownItem => ({ - label: c.label, - keepOpenOnSelect: true, - disabled: c.id === "path", - leftSlot: - (prefs.visible[c.id] ?? c.defaultVisible) || c.id === "path" ? ( + ...allColumns(prefs).map((c): DropdownItem => { + const shown = prefs.visible[c.id] ?? c.defaultVisible; + return { + label: c.label, + keepOpenOnSelect: true, + disabled: shown && !canHide(prefs, c.id, unavailable), + leftSlot: shown ? ( ) : ( ), - rightSlot: - c.id === "path" ? always : undefined, - onSelect: () => setPrefs((p) => toggleColumn(p, c.id)), - })), + onSelect: () => setPrefs((p) => toggleColumn(p, c.id, unavailable)), + ...(isCustomColumn(c.id) + ? { + submenuTrigger: "button" as const, + submenu: [ + { + label: "Remove Column", + icon: "trash" as const, + color: "danger" as const, + onSelect: () => setPrefs((p) => removeCustomColumn(p, c.id as `field:${string}`)), + }, + ], + } + : {}), + }; + }), + { label: "Add Column…", icon: "plus", onSelect: () => openAddColumn(true) }, { type: "separator" }, - { label: "Sort by Start Time", disabled: sort == null, onSelect: onSortByStart }, + { label: "Reset Sort", disabled: sort.length === 0, onSelect: onResetSort }, { label: "Reset Column Widths", onSelect: () => setPrefs((p) => resetWidths(p)) }, - { type: "separator" }, - exportShown(), ]; - const openMenu = useCallback((e: MouseEvent) => { - e.preventDefault(); - setMenu({ x: e.clientX, y: e.clientY }); - }, []); + const columnItems = (c: ColumnDef): DropdownItem[] => { + const column = c.sortable ? (c.id as SortColumn) : null; + const sortBy = (descending: boolean) => { + if (column != null) setSort([{ column, descending }]); + }; + const addTo = (descending: boolean) => { + if (column != null) setSort(addSortKey(sort, column, descending)); + }; + return [ + ...(column == null + ? [] + : ([ + { label: "Sort Ascending", onSelect: () => sortBy(false) }, + { label: "Sort Descending", onSelect: () => sortBy(true) }, + { label: "Add to Sort Ascending", onSelect: () => addTo(false) }, + { label: "Add to Sort Descending", onSelect: () => addTo(true) }, + ] satisfies DropdownItem[])), + { label: "Clear Sorting", disabled: sort.length === 0, onSelect: onResetSort }, + { type: "separator" }, + { + label: "Hide Column", + disabled: !canHide(prefs, c.id, unavailable), + onSelect: () => setPrefs((p) => toggleColumn(p, c.id, unavailable)), + }, + { + label: "Hide Other Columns", + disabled: columns.length <= 1, + onSelect: () => setPrefs((p) => hideOtherColumns(p, c.id)), + }, + ...(isCustomColumn(c.id) + ? [ + { + label: "Remove Column", + color: "danger" as const, + onSelect: () => setPrefs((p) => removeCustomColumn(p, c.id as `field:${string}`)), + }, + ] + : []), + { type: "separator" }, + { label: "Reset Column Widths", onSelect: () => setPrefs((p) => resetWidths(p)) }, + ]; + }; + + // A right-click on a label is for its column; anywhere else in the header, the picker. + const openMenu = useCallback( + (e: MouseEvent) => { + e.preventDefault(); + const el = (e.target as HTMLElement).closest("[data-column]"); + const id = el?.dataset.column as ColumnId | undefined; + const column = columns.find((c) => c.id === id) ?? null; + setMenu({ x: e.clientX, y: e.clientY, column }); + }, + [columns], + ); return ( // The same margins and padding as the rows, so each label sits over its cells.
- {/* oxlint-disable-next-line jsx-a11y/no-static-element-interactions -- a right-click opens the same menu as the picker */} + {/* oxlint-disable-next-line jsx-a11y/no-static-element-interactions -- a right-click opens the column's menu */}
- + - {columns.map((c) => ( - - - {/* On the column's right edge, the boundary with the next, resizing this one */} - { - startWidth.current = c.current; - }} - onResizeMove={(e) => - setPrefs((p) => resizeColumn(p, c.id, startWidth.current + e.x - e.xStart)) - } - onReset={() => setPrefs((p) => resetWidth(p, c.id))} - /> - - ))} - setMenu(null)} /> + + ); + const labelClass = classNames( + "flex items-center gap-0.5 min-w-0 w-full h-full whitespace-nowrap", + c.align === "right" && "justify-end", + key != null && "text-text-subtle", + ); + return ( + + {c.sortable ? ( + + ) : ( + {label} + )} + {/* On the column's right edge, the boundary with the next, resizing this one */} + { + startWidth.current = c.current; + }} + onResizeMove={(e) => + setPrefs((p) => resizeColumn(p, c.id, startWidth.current + e.x - e.xStart)) + } + onReset={() => setPrefs((p) => resetWidth(p, c.id))} + /> + + ); + })} + setMenu(null)} + />
); diff --git a/apps/yaak-proxy/components/ProxyLayout.tsx b/apps/yaak-proxy/components/ProxyLayout.tsx index 4f4ffd8be..0cd38b228 100644 --- a/apps/yaak-proxy/components/ProxyLayout.tsx +++ b/apps/yaak-proxy/components/ProxyLayout.tsx @@ -12,8 +12,10 @@ import { useState } from "react"; import { useLocalStorage } from "react-use"; import { flowLayoutAtom } from "../lib/atoms"; import { setupIncomplete } from "../lib/firstRun"; +import { useMenuExportHar } from "../lib/flowActions"; import { lockedAtom, noFlowsAtom, systemProxyEverOnAtom } from "../lib/store"; import { getOsType } from "../lib/tauri"; +import { AddColumnDialog } from "./AddColumnDialog"; import { ClearFlowsButton } from "./ClearFlowsButton"; import { FirstRun } from "./FirstRun"; import { FlowDetail } from "./FlowDetail"; @@ -32,6 +34,7 @@ export function ProxyLayout() { const os = getOsType(); const [flowLayout, setFlowLayout] = useAtom(flowLayoutAtom); const openSettings = useOpenSettings(); + useMenuExportHar(); const [sidebarWidth, setSidebarWidth] = useLocalStorage("sidebar_width", 250); const [sidebarHidden, setSidebarHidden] = useLocalStorage("sidebar_hidden", false); const [floatingSidebarHidden, setFloatingSidebarHidden] = useLocalStorage( @@ -164,6 +167,7 @@ export function ProxyLayout() { + diff --git a/apps/yaak-proxy/components/SearchBar.tsx b/apps/yaak-proxy/components/SearchBar.tsx index ff7dba372..fd0a1c0f6 100644 --- a/apps/yaak-proxy/components/SearchBar.tsx +++ b/apps/yaak-proxy/components/SearchBar.tsx @@ -23,6 +23,7 @@ import { useState, } from "react"; import { useRpcQuery } from "../hooks/useRpcQuery"; +import { useExportShownHar } from "../lib/flowActions"; import { loadSeenValues, savedCompletions, searchProblems } from "../lib/searchBox"; import { allTrafficTotalAtom, @@ -54,6 +55,7 @@ export const SearchBar = memo(function SearchBar() { const total = useAtomValue(allTrafficTotalAtom); const store = useStore(); const setSavedDialog = useSetAtom(savedSearchDialogAtom); + const exportShown = useExportShownHar(); // Nothing to search while captured traffic is locked. const locked = useAtomValue(lockedAtom) != null; const schema = useRpcQuery("query_schema", {}, { staleTime: Infinity }).data?.schema; @@ -179,6 +181,7 @@ export const SearchBar = memo(function SearchBar() { leftSlot: , onSelect: () => setSavedDialog({ mode: "save", query: text.trim() }), }, + exportShown(), ]} > } {item.kind === "saved" && } {item.kind === "folder" && } - {item.appPath != null && } + {item.appPath != null && } {item.kind === "folder" && item.path != null ? ( // Named by its last part; the whole path on hover diff --git a/apps/yaak-proxy/harness/FlowPanesHarness.tsx b/apps/yaak-proxy/harness/FlowPanesHarness.tsx index dc01632b0..e24dcada3 100644 --- a/apps/yaak-proxy/harness/FlowPanesHarness.tsx +++ b/apps/yaak-proxy/harness/FlowPanesHarness.tsx @@ -1,5 +1,6 @@ import { useSetAtom } from "jotai"; -import { useLayoutEffect, useState } from "react"; +import { useEffect, useLayoutEffect, useState } from "react"; +import { AddColumnDialog } from "../components/AddColumnDialog"; import { FlowDetail } from "../components/FlowDetail"; import { FlowList } from "../components/FlowList"; import { selectedFlowAtom } from "../lib/atoms"; @@ -34,6 +35,7 @@ export function FlowPanesHarness() { ))} + ); } @@ -54,6 +56,28 @@ function FixtureCard({ fixture }: { fixture: FlowFixture }) { setReady(true); // oxlint-disable-next-line react-hooks/exhaustive-deps -- once per fixture }, [fixture]); + + // `&arrive=3`: that many new flows, one a second, as traffic would bring them + useEffect(() => { + const count = Number(params.get("arrive") ?? 0); + let added = 0; + const timer = setInterval(() => { + if (added >= count) return clearInterval(timer); + added++; + const source = fixtures[added % fixtures.length]?.flow; + if (source == null) return; + const flow = { + ...source, + // The fixture's id first, so the backend finds its bodies, as `manyFlows` does + id: `${source.id}~arrived_${added}`, + number: 1500 + added, + startedAt: new Date().toISOString().slice(0, 23), + }; + setWindow((w) => ({ ...w, flows: [...w.flows, flow], total: w.total + 1 })); + }, 1000); + return () => clearInterval(timer); + // oxlint-disable-next-line react-hooks/exhaustive-deps -- once per fixture + }, [fixture]); if (!ready) return null; if (params.has("list")) { diff --git a/apps/yaak-proxy/harness/backend.ts b/apps/yaak-proxy/harness/backend.ts index 391268241..92f001571 100644 --- a/apps/yaak-proxy/harness/backend.ts +++ b/apps/yaak-proxy/harness/backend.ts @@ -1,5 +1,5 @@ import type { Flow, GetProxyStateResponse } from "@yaakapp-internal/proxy-lib"; -import { fixtures } from "./fixtures"; +import { appIconPng, fixtures } from "./fixtures"; /** * Dev only: answers the engine's RPCs the harness pages need, from fixtures, so the card and @@ -13,8 +13,9 @@ export function installFixtureBackend() { if (cmd.startsWith("plugin:event|")) return 0; throw new Error(`Not in the harness: ${cmd}`); }; - (window as unknown as { __TAURI_OS_PLUGIN_INTERNALS__: Record }) - .__TAURI_OS_PLUGIN_INTERNALS__ = { + ( + window as unknown as { __TAURI_OS_PLUGIN_INTERNALS__: Record } + ).__TAURI_OS_PLUGIN_INTERNALS__ = { os_type: "macos", platform: "macos", family: "unix", @@ -58,7 +59,7 @@ function answer(cmd: string, payload: unknown): unknown { } } case "app_icon": - return null; + return appIconPng; case "get_proxy_state": return HARNESS_STATE; case "get_storage_size": @@ -127,11 +128,29 @@ function answer(cmd: string, payload: unknown): unknown { info: "A request or response header", example: "@header.accept:*json*", }, + { + name: "res.header", + kind: { type: "text" }, + map: true, + info: "A response header", + example: "@res.header.x-cache:hit", + }, + { + name: "query", + kind: { type: "text" }, + map: true, + info: "A query parameter", + example: "@query.page:2", + }, ], }, }; - case "query_values": - return []; + // Header and query parameter names, for completing a map's keys + case "query_values": { + const path = (payload as { path?: string } | undefined)?.path ?? ""; + if (path.endsWith("header")) return ["x-cache", "x-request-id", "content-type"]; + return path === "query" ? ["expand", "limit"] : []; + } case "validate_query": return []; default: diff --git a/apps/yaak-proxy/harness/fixtures.ts b/apps/yaak-proxy/harness/fixtures.ts index cc90cf443..7368f528c 100644 --- a/apps/yaak-proxy/harness/fixtures.ts +++ b/apps/yaak-proxy/harness/fixtures.ts @@ -50,6 +50,7 @@ const base: Flow = { grpcMethod: null, grpcStatus: null, grpcMessage: null, + graphqlOperation: null, tunnelled: false, agentRead: false, clientAddr: "127.0.0.1:54321", @@ -69,6 +70,10 @@ const jsonResponse = JSON.stringify({ status: "succeeded", charges: { data: [{ id: "ch_1", paid: true }] }, }); +const graphqlRequest = JSON.stringify({ + query: "query GetViewer { viewer { login } }", + operationName: "GetViewer", +}); const formRequest = "grant_type=client_credentials&client_id=yaak-proxy&scope=read+write"; const htmlResponse = "

Signed in

You can close this window.

"; @@ -176,4 +181,43 @@ export const fixtures: FlowFixture[] = [ requestBody: body(""), responseBody: body(""), }, + { + label: "GraphQL POST", + flow: { + ...base, + id: "fl_graphql", + method: "POST", + url: "https://api.github.com/graphql", + contentType: "application/json", + requestHeaders: [{ name: "Content-Type", value: "application/json" }], + responseHeaders: [{ name: "X-Cache", value: "MISS" }], + requestSize: graphqlRequest.length, + responseSize: 64, + graphqlOperation: "GetViewer", + }, + requestBody: body(graphqlRequest), + responseBody: body('{"data":{"viewer":{"login":"yaak"}}}'), + }, + { + label: "gRPC call", + flow: { + ...base, + id: "fl_grpc", + method: "POST", + url: "https://grpc.example.com/pkg.Greeter/SayHello", + contentType: "application/grpc", + clientHttpVersion: "HTTP/2", + upstreamHttpVersion: "HTTP/2", + requestHeaders: [{ name: "Content-Type", value: "application/grpc" }], + responseHeaders: [{ name: "Content-Type", value: "application/grpc" }], + grpcService: "pkg.Greeter", + grpcMethod: "SayHello", + grpcStatus: 0, + }, + requestBody: body(""), + responseBody: body(""), + }, ]; + +/** An app icon for every flow in the harness, as base64 PNG */ +export const appIconPng = btoa(String.fromCharCode(...png)); diff --git a/apps/yaak-proxy/harness/main.tsx b/apps/yaak-proxy/harness/main.tsx index 8fa7d599a..e22ce2ceb 100644 --- a/apps/yaak-proxy/harness/main.tsx +++ b/apps/yaak-proxy/harness/main.tsx @@ -1,6 +1,7 @@ import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { applyThemeToDocument, defaultDarkTheme, defaultLightTheme } from "@yaakapp-internal/theme"; import { createStore, Provider } from "jotai"; +import { LazyMotion, MotionConfig } from "motion/react"; import { StrictMode } from "react"; import { createRoot } from "react-dom/client"; import { installFixtureBackend } from "./backend"; @@ -11,6 +12,8 @@ import { SettingsHarness } from "./SettingsHarness"; import "../main.css"; const params = new URLSearchParams(location.search); +// As the app loads them, so dialogs fade in +const motionFeatures = () => import("framer-motion").then((mod) => mod.domAnimation); const dark = params.get("theme") !== "light"; applyThemeToDocument(dark ? defaultDarkTheme : defaultLightTheme); if (params.has("locked")) installLockedBackend(); @@ -20,15 +23,19 @@ createRoot(document.getElementById("root") as HTMLElement).render( - {params.has("locked") || params.has("window") ? ( - - ) : params.has("settings") ? ( - - ) : params.has("stream") ? ( - - ) : ( - - )} + + + {params.has("locked") || params.has("window") ? ( + + ) : params.has("settings") ? ( + + ) : params.has("stream") ? ( + + ) : ( + + )} + + , diff --git a/apps/yaak-proxy/harness/streamFixtures.ts b/apps/yaak-proxy/harness/streamFixtures.ts index 47fa878b7..73d9d29be 100644 --- a/apps/yaak-proxy/harness/streamFixtures.ts +++ b/apps/yaak-proxy/harness/streamFixtures.ts @@ -49,6 +49,7 @@ const flowBase: Flow = { grpcMethod: null, grpcStatus: null, grpcMessage: null, + graphqlOperation: null, tunnelled: false, agentRead: false, clientAddr: null, diff --git a/apps/yaak-proxy/lib/columns.test.ts b/apps/yaak-proxy/lib/columns.test.ts index 6703b589d..5f4120a10 100644 --- a/apps/yaak-proxy/lib/columns.test.ts +++ b/apps/yaak-proxy/lib/columns.test.ts @@ -1,35 +1,59 @@ -import type { Flow } from "@yaakapp-internal/proxy-lib"; +import type { Flow, FlowSortKey } from "@yaakapp-internal/proxy-lib"; import { describe, expect, test } from "vite-plus/test"; import { + addCustomColumn, + addSortKey, + cellFilterTerm, type ColumnPrefs, + customPath, + customValue, defaultColumnPrefs, gridTemplate, gridWidth, + hideOtherColumns, nextSort, + operationOf, parseColumnPrefs, + removeCustomColumn, resetWidth, resetWidths, resizeColumn, + showOperationFor, shownColumns, sortFlows, toggleColumn, + toggleSortKey, + urlParts, } from "./columns"; function flow(number: number, extra: Partial = {}): Flow { return { - id: `fl_${number}`, + id: `fl_${String(number).padStart(3, "0")}`, number, + startedAt: `2026-11-09T12:00:${String(number).padStart(2, "0")}`, url: "https://example.com/", method: "GET", status: 200, error: null, + requestSize: 0, responseSize: 0, elapsedMs: null, + elapsedHeadersMs: null, contentType: null, clientName: null, clientBundleId: null, clientPath: null, clientCwd: null, + clientHttpVersion: "HTTP/1.1", + upstreamHttpVersion: null, + upstreamTlsVersion: null, + remoteAddr: null, + requestHeaders: [], + responseHeaders: [], + grpcService: null, + grpcMethod: null, + graphqlOperation: null, + tunnelled: false, ...extra, } as Flow; } @@ -37,13 +61,14 @@ function flow(number: number, extra: Partial = {}): Flow { const ids = (prefs: ColumnPrefs) => shownColumns(prefs).map((c) => c.id); describe("shownColumns", () => { - test("shows every visible column, however narrow the list", () => { + test("shows every visible column, Query after Path", () => { expect(ids(defaultColumnPrefs)).toEqual([ "number", "app", "method", "host", "path", + "query", "status", "type", "size", @@ -51,29 +76,33 @@ describe("shownColumns", () => { ]); }); - test("follows the picker, and always keeps the path", () => { + test("follows the picker, Path included, but keeps the last column", () => { let prefs = toggleColumn(defaultColumnPrefs, "remote"); prefs = toggleColumn(prefs, "type"); prefs = toggleColumn(prefs, "path"); expect(ids(prefs)).toContain("remote"); expect(ids(prefs)).not.toContain("type"); - expect(ids(prefs)).toContain("path"); + expect(ids(prefs)).not.toContain("path"); + const alone = hideOtherColumns(defaultColumnPrefs, "host"); + expect(ids(alone)).toEqual(["host"]); + expect(toggleColumn(alone, "host")).toBe(alone); + // Where App has nothing to show, the last other column is the last one. + const withApp = hideOtherColumns(defaultColumnPrefs, "app"); + const both = toggleColumn(withApp, "host"); + expect(toggleColumn(both, "host", ["app"])).toBe(both); + expect(ids(both)).toEqual(["app", "host"]); + expect(shownColumns(both, ["app"]).map((c) => c.id)).toEqual(["host"]); }); - test("lays out every column at a fixed width, the path included", () => { + test("lays out every column at a fixed width", () => { let prefs = resizeColumn(defaultColumnPrefs, "host", 240.4); prefs = resizeColumn(prefs, "path", 500); const columns = shownColumns(prefs); const pad = (w: number) => `calc(${w}px + 0.75rem)`; expect(gridTemplate(columns)).toBe( - ["24px", ...[46, 18, 50, 240, 500, 46, 50, 72, 64].map(pad)].join(" "), + ["24px", ...[46, 20, 50, 240, 500, 120, 46, 50, 72, 64].map(pad)].join(" "), ); - expect(gridWidth(columns)).toBe("calc(1110px + 9 * 0.75rem)"); - }); - - test("keeps the host's width in a narrow list", () => { - const host = shownColumns(defaultColumnPrefs).find((c) => c.id === "host"); - expect(host?.current).toBe(190); + expect(gridWidth(columns)).toBe("calc(1232px + 10 * 0.75rem)"); }); }); @@ -81,11 +110,32 @@ describe("column preferences", () => { test("persist through storage and drop what no longer applies", () => { let prefs = resizeColumn(defaultColumnPrefs, "host", 10); prefs = toggleColumn(prefs, "number"); + prefs = addCustomColumn(prefs, "header.x-cache"); + prefs = { ...prefs, sort: [{ column: "status", descending: true }] }; const stored = JSON.parse(JSON.stringify(prefs)); - expect(parseColumnPrefs(stored)).toEqual({ visible: { number: false }, widths: { host: 60 } }); + expect(parseColumnPrefs(stored)).toEqual({ + visible: { number: false, "field:header.x-cache": true }, + widths: { host: 60 }, + custom: ["field:header.x-cache"], + sort: [{ column: "status", descending: true }], + }); expect( - parseColumnPrefs({ visible: { gone: true, size: "yes" }, widths: { time: -1, size: 90 } }), - ).toEqual({ visible: {}, widths: { size: 90 } }); + parseColumnPrefs({ + visible: { gone: true, size: "yes" }, + widths: { time: -1, size: 90 }, + custom: ["field:nope.x", 3, "field:query.page"], + sort: [ + { column: "gone", descending: true }, + { column: "host" }, + { column: "time", descending: false }, + ], + }), + ).toEqual({ + visible: {}, + widths: { size: 90 }, + custom: ["field:query.page"], + sort: [{ column: "time", descending: false }], + }); expect(parseColumnPrefs(null)).toEqual(defaultColumnPrefs); expect(resetWidths(prefs).widths).toEqual({}); }); @@ -95,61 +145,162 @@ describe("column preferences", () => { prefs = resizeColumn(prefs, "path", 400); expect(resetWidth(prefs, "path").widths).toEqual({ host: 300 }); }); + + test("custom columns are added at the end and removed with their settings", () => { + let prefs = addCustomColumn(defaultColumnPrefs, "res.header.x-cache"); + prefs = resizeColumn(prefs, "field:res.header.x-cache", 200); + expect(ids(prefs).at(-1)).toBe("field:res.header.x-cache"); + prefs = removeCustomColumn(prefs, "field:res.header.x-cache"); + expect(prefs).toEqual(defaultColumnPrefs); + }); + + test("Operation shows itself once, unless the person set it", () => { + const grpc = flow(1, { grpcService: "pkg.Greeter", grpcMethod: "SayHello" }); + const anonymous = flow(2, { graphqlOperation: "" }); + expect(showOperationFor(defaultColumnPrefs, [flow(3), anonymous])).toBe(defaultColumnPrefs); + expect(showOperationFor(defaultColumnPrefs, [grpc]).visible.operation).toBe(true); + const hidden = { ...defaultColumnPrefs, visible: { operation: false } }; + expect(showOperationFor(hidden, [grpc])).toBe(hidden); + }); }); describe("sorting", () => { - test("sorts numbers largest first and names from A, reverses, then goes back to start", () => { - expect(nextSort(null, "size")).toEqual({ column: "size", descending: true }); - expect(nextSort(null, "host")).toEqual({ column: "host", descending: false }); - expect(nextSort({ column: "host", descending: false }, "host")).toEqual({ - column: "host", - descending: true, - }); - expect(nextSort({ column: "host", descending: true }, "host")).toBeNull(); - expect(nextSort({ column: "size", descending: true }, "size")).toEqual({ - column: "size", - descending: false, - }); - expect(nextSort({ column: "size", descending: false }, "size")).toBeNull(); - // Another column starts over. - expect(nextSort({ column: "host", descending: true }, "size")).toEqual({ - column: "size", - descending: true, - }); + const one = (column: FlowSortKey["column"], descending: boolean) => [{ column, descending }]; + + test("a click sorts by a column alone, reverses it, then goes back to arrival order", () => { + expect(nextSort([], "size")).toEqual(one("size", true)); + expect(nextSort([], "host")).toEqual(one("host", false)); + expect(nextSort(one("host", false), "host")).toEqual(one("host", true)); + expect(nextSort(one("host", true), "host")).toEqual([]); + expect(nextSort(one("size", true), "size")).toEqual(one("size", false)); + expect(nextSort(one("size", false), "size")).toEqual([]); + // Another column, or a sort on several, starts over. + expect(nextSort(one("host", true), "size")).toEqual(one("size", true)); + const two = [...one("host", false), ...one("size", true)]; + expect(nextSort(two, "host")).toEqual(one("host", false)); }); - test("sorts the loaded flows, those without a value lowest, ties in arrival order", () => { - const a = flow(1, { responseSize: 10, elapsedMs: 5, url: "https://b.com/" }); - const b = flow(2, { responseSize: 300, elapsedMs: null, url: "https://a.com/" }); - const c = flow(3, { responseSize: 10, elapsedMs: 50, url: "https://c.com/" }); - const d = flow(4, { error: "DNS lookup failed", status: null, url: "https://a.com/" }); - const flows = [a, b, c, d]; - const by = (column: Parameters[1], descending: boolean) => - sortFlows(flows, { column, descending }).map((f) => f.number); + test("Option-click adds a key ascending, reverses it, then takes it out", () => { + let sort = toggleSortKey(one("status", true), "host"); + expect(sort).toEqual([...one("status", true), ...one("host", false)]); + sort = toggleSortKey(sort, "host"); + expect(sort).toEqual([...one("status", true), ...one("host", true)]); + expect(toggleSortKey(sort, "host")).toEqual(one("status", true)); + expect(addSortKey(sort, "status", false)).toEqual([ + ...one("status", false), + ...one("host", true), + ]); + expect(addSortKey([], "time", true)).toEqual(one("time", true)); + }); + + // The same flows and orders as `a_sort_on_several_columns_pages_in_its_order` in the lib's + // `flows.rs`, so the app places a flow where the database would. + const flows = [ + flow(0, { url: "https://B.com/x", status: 200 }), + flow(1, { url: "https://a.com/y", status: 404 }), + flow(2, { url: "https://a.com/z", status: null }), + flow(3, { url: "https://c.com/", status: 200 }), + flow(4, { url: "https://a.com/", status: null, error: "boom" }), + flow(5, { url: "https://b.com/q", status: 200 }), + flow(6, { url: "https://A.com/", status: 404 }), + flow(7, { url: "https://c.com/", status: 200 }), + ]; + const numbers = (sort: FlowSortKey[]) => sortFlows(flows, sort).map((f) => f.number); + + test("sorts on several columns as the database does", () => { + const status = [...one("status", true), ...one("host", false)]; + expect(numbers(status)).toEqual([1, 6, 0, 5, 3, 7, 4, 2]); + const ascending = [...one("status", false), ...one("host", false)]; + expect(numbers(ascending)).toEqual([2, 4, 0, 5, 3, 7, 1, 6]); + expect(numbers(one("query", true))).toEqual([0, 1, 2, 3, 4, 5, 6, 7]); + expect(sortFlows(flows, [])).toBe(flows); + }); + + test("sizes, times and the rest put flows without a value lowest", () => { + const a = flow(1, { responseSize: 10, elapsedMs: 5, requestSize: 3, elapsedHeadersMs: 2 }); + const b = flow(2, { responseSize: 300, elapsedMs: null, requestSize: 0 }); + const c = flow(3, { responseSize: 10, elapsedMs: 50, elapsedHeadersMs: 40 }); + const d = flow(4, { error: "DNS lookup failed", status: null, responseSize: 99 }); + const four = [a, b, c, d]; + const by = (column: FlowSortKey["column"], descending: boolean) => + sortFlows(four, one(column, descending)).map((f) => f.number); expect(by("size", true)).toEqual([2, 1, 3, 4]); expect(by("size", false)).toEqual([4, 1, 3, 2]); expect(by("time", true)).toEqual([3, 1, 2, 4]); - expect(by("time", false)).toEqual([2, 4, 1, 3]); - expect(by("host", false)).toEqual([2, 4, 1, 3]); + expect(by("latency", true)).toEqual([3, 1, 2, 4]); + expect(by("requestSize", true)).toEqual([1, 2, 3, 4]); expect(by("number", true)).toEqual([4, 3, 2, 1]); // Failures sort below every status. expect(by("status", true)).toEqual([1, 2, 3, 4]); expect(by("status", false)).toEqual([4, 1, 2, 3]); - expect(sortFlows(flows, null)).toBe(flows); - }); - - test("flips flows with no status, type, size or time to the other end", () => { - const done = flow(1, { status: 200, contentType: "text/html", responseSize: 5, elapsedMs: 9 }); - const pending = flow(2, { status: null, contentType: null, responseSize: 0, elapsedMs: null }); - const cancelled = flow(3, { status: null, cancelled: true, contentType: "" }); - const flows = [done, pending, cancelled]; - for (const column of ["status", "type", "size", "time"] as const) { - expect(sortFlows(flows, { column, descending: true }).map((f) => f.number)).toEqual([ - 1, 2, 3, - ]); - expect(sortFlows(flows, { column, descending: false }).map((f) => f.number)).toEqual([ - 2, 3, 1, - ]); - } + }); +}); + +describe("values", () => { + test("splits URLs as the lib does", () => { + expect(urlParts("https://a.com:8080/v1/x?page=2&q=1#top")).toEqual({ + authority: "a.com:8080", + path: "/v1/x", + query: "page=2&q=1", + }); + expect(urlParts("https://a.com?x")).toEqual({ authority: "a.com", path: "", query: "x" }); + expect(urlParts("bank.example.com:443")).toEqual({ + authority: "bank.example.com:443", + path: "", + query: null, + }); + }); + + test("reads a custom column's path from what was typed", () => { + expect(customPath("@header.X-Request-Id:")).toBe("header.x-request-id"); + expect(customPath("query.page")).toBe("query.page"); + expect(customPath(" res.header.x-cache ")).toBe("res.header.x-cache"); + expect(customPath("@host")).toBeNull(); + expect(customPath("header.")).toBeNull(); + expect(customPath("header.a b")).toBeNull(); + expect(customPath("query.filter[status]")).toBeNull(); + }); + + test("a custom column shows the first value, the response's header before the request's", () => { + const f = flow(1, { + url: "https://a.com/?Page=2&page=3", + requestHeaders: [ + { name: "X-Cache", value: "req" }, + { name: "Accept", value: "a" }, + { name: "accept", value: "b" }, + ], + responseHeaders: [{ name: "x-cache", value: "HIT" }], + }); + expect(customValue(f, "header.x-cache")).toBe("HIT"); + expect(customValue(f, "req.header.x-cache")).toBe("req"); + expect(customValue(f, "header.accept")).toBe("a"); + expect(customValue(f, "query.page")).toBe("2"); + expect(customValue(f, "query.nope")).toBeNull(); + }); + + test("a cell's filter terms, where its column has a field to match", () => { + const f = flow(1, { + url: "https://api.stripe.com:443/v1/charges?x=1", + status: 404, + clientName: "Google Chrome", + contentType: "application/json; charset=utf-8", + responseHeaders: [{ name: "X-Cache", value: "HIT" }], + }); + expect(cellFilterTerm(f, "host", false)).toBe("@host:api.stripe.com"); + expect(cellFilterTerm(f, "status", true)).toBe("-@status:404"); + expect(cellFilterTerm(f, "app", false)).toBe('@app:"Google Chrome"'); + expect(cellFilterTerm(f, "path", false)).toBe("@path:/v1/charges"); + expect(cellFilterTerm(f, "type", false)).toBe("@type:application/json"); + expect(cellFilterTerm(f, "field:header.x-cache", false)).toBe("@header.x-cache:HIT"); + for (const column of ["number", "size", "time", "query", "started"] as const) { + expect(cellFilterTerm(f, column, false)).toBeNull(); + } + const grpc = flow(2, { grpcService: "pkg.Greeter", grpcMethod: "SayHello" }); + expect(operationOf(grpc)).toBe("pkg.Greeter/SayHello"); + expect(cellFilterTerm(grpc, "operation", true)).toBe( + "-(@grpc.service:pkg.Greeter @grpc.method:SayHello)", + ); + const graphql = flow(3, { graphqlOperation: "GetUser" }); + expect(cellFilterTerm(graphql, "operation", false)).toBe("@graphql.operation:GetUser"); }); }); diff --git a/apps/yaak-proxy/lib/columns.ts b/apps/yaak-proxy/lib/columns.ts index 1492b7b07..b7f0e4492 100644 --- a/apps/yaak-proxy/lib/columns.ts +++ b/apps/yaak-proxy/lib/columns.ts @@ -1,18 +1,15 @@ -import type { Flow } from "@yaakapp-internal/proxy-lib"; -import { appKeyOf } from "./search"; +import type { Flow, FlowSortKey, SortColumn } from "@yaakapp-internal/proxy-lib"; +import { compareFlows } from "./flowWindow"; +import { fieldTerm } from "./query"; +import { sidebarPath } from "./search"; -export type ColumnId = - | "number" - | "app" - | "method" - | "host" - | "path" - | "status" - | "type" - | "size" - | "time" - | "remote" - | "version"; +/** The columns the list has built in; each sorts by the `SortColumn` of the same name */ +export type BuiltinColumnId = SortColumn; + +/** A column of a header or query parameter the person added, by its field's path */ +export type CustomColumnId = `field:${string}`; + +export type ColumnId = BuiltinColumnId | CustomColumnId; export interface ColumnDef { id: ColumnId; @@ -24,102 +21,81 @@ export interface ColumnDef { align: "left" | "right"; /** Shown until the person hides it */ defaultVisible: boolean; + /** Sorted by in the database; custom columns are not */ + sortable: boolean; + /** The header or query parameter a custom column shows: `header.x-request-id` */ + path?: string; } +const builtin = ( + id: BuiltinColumnId, + label: string, + width: number, + minWidth: number, + align: "left" | "right", + defaultVisible: boolean, +): ColumnDef => ({ id, label, width, minWidth, align, defaultVisible, sortable: true }); + export const COLUMNS: ColumnDef[] = [ - { - id: "number", - label: "#", - width: 46, - minWidth: 30, - align: "right", - defaultVisible: true, - }, - { - id: "app", - label: "App", - width: 18, - minWidth: 18, - align: "left", - defaultVisible: true, - }, - { - id: "method", - label: "Method", - width: 50, + builtin("number", "#", 46, 30, "left", true), + builtin("app", "App", 20, 20, "left", true), + builtin("method", "Method", 50, 40, "left", true), + builtin("host", "Host", 190, 60, "left", true), + builtin("path", "Path", 300, 60, "left", true), + builtin("query", "Query", 120, 40, "left", true), + builtin("status", "Status", 46, 34, "left", true), + builtin("type", "Type", 50, 34, "left", true), + builtin("requestSize", "Request Size", 80, 44, "right", false), + builtin("size", "Size", 72, 44, "right", true), + builtin("time", "Time", 64, 44, "right", true), + builtin("latency", "Latency", 64, 44, "right", false), + builtin("started", "Started", 92, 60, "left", false), + builtin("remote", "Remote Address", 150, 60, "left", false), + builtin("version", "HTTP Version", 70, 44, "left", false), + builtin("tls", "TLS Version", 70, 34, "left", false), + builtin("operation", "Operation", 160, 60, "left", false), +]; + +const DEFS = new Map(COLUMNS.map((c) => [c.id, c])); + +/** Map fields a custom column can show, as the query language names them */ +const CUSTOM_MAPS = ["req.header.", "res.header.", "header.", "query."]; + +/** `header.x-request-id` from what was typed, `@header.X-Request-Id:` included, or null */ +export function customPath(typed: string): string | null { + const path = typed.trim().replace(/^@/, "").replace(/:$/, "").trim(); + const map = CUSTOM_MAPS.find((m) => path.toLowerCase().startsWith(m)); + if (map == null) return null; + const key = path.slice(map.length); + // As a field path takes it, so the column's filter items make a term the search can run + if (!/^[A-Za-z0-9_-]+$/.test(key)) return null; + return map + key.toLowerCase(); +} + +export function customColumnId(path: string): CustomColumnId { + return `field:${path}`; +} + +export function isCustomColumn(id: ColumnId): id is CustomColumnId { + return id.startsWith("field:"); +} + +function customDef(id: CustomColumnId): ColumnDef { + const path = id.slice("field:".length); + return { + id, + label: path, + width: 140, minWidth: 40, align: "left", defaultVisible: true, - }, - { - id: "host", - label: "Host", - width: 190, - minWidth: 60, - align: "left", - defaultVisible: true, - }, - { - id: "path", - label: "Path", - width: 360, - minWidth: 80, - align: "left", - defaultVisible: true, - }, - { - id: "status", - label: "Status", - width: 46, - minWidth: 34, - align: "left", - defaultVisible: true, - }, - { - id: "type", - label: "Type", - width: 50, - minWidth: 34, - align: "left", - defaultVisible: true, - }, - { - id: "size", - label: "Size", - width: 72, - minWidth: 44, - align: "right", - defaultVisible: true, - }, - { - id: "time", - label: "Time", - width: 64, - minWidth: 44, - align: "right", - defaultVisible: true, - }, - { - id: "remote", - label: "Remote Address", - width: 150, - minWidth: 60, - align: "left", - defaultVisible: false, - }, - { - id: "version", - label: "HTTP Version", - width: 70, - minWidth: 44, - align: "left", - defaultVisible: false, - }, -]; - -const DEFS = new Map(COLUMNS.map((c) => [c.id, c])); + sortable: false, + path, + }; +} export function columnDef(id: ColumnId): ColumnDef { + if (isCustomColumn(id)) return customDef(id); const def = DEFS.get(id); if (def == null) throw new Error(`No column ${id}`); return def; @@ -131,27 +107,48 @@ export const MARK_WIDTH = 24; /** Each cell's padding, both sides together, added to its column's width; `px-1.5` per side */ export const CELL_PADDING = "0.75rem"; -/** What the person chose: which columns show, and the widths they dragged, by id */ +/** + * What the person chose: which columns show, the widths they dragged, by id, the header and + * query parameter columns they added, in order, and the list's sort + */ export interface ColumnPrefs { visible: Partial>; widths: Partial>; + custom: CustomColumnId[]; + /** The keys in order; empty for the order flows arrive in */ + sort: FlowSortKey[]; } -export const defaultColumnPrefs: ColumnPrefs = { visible: {}, widths: {} }; +export const defaultColumnPrefs: ColumnPrefs = { visible: {}, widths: {}, custom: [], sort: [] }; export interface Column extends ColumnDef { /** Its width now, in pixels of content */ current: number; } +/** Every column the picker lists: the built-in ones, then those the person added */ +export function allColumns(prefs: ColumnPrefs): ColumnDef[] { + return [...COLUMNS, ...prefs.custom.map(customDef)]; +} + +function isVisible(prefs: ColumnPrefs, def: ColumnDef): boolean { + return prefs.visible[def.id] ?? def.defaultVisible; +} + /** - * The columns shown, in order: the visible ones, the path always. Each keeps its width however - * narrow the list is; the list scrolls sideways instead. + * The columns shown, in order. Each keeps its width however narrow the list is; the list scrolls + * sideways instead. At least one always shows. */ -export function shownColumns(prefs: ColumnPrefs): Column[] { - return COLUMNS.filter((c) => c.id === "path" || (prefs.visible[c.id] ?? c.defaultVisible)).map( - (c) => ({ ...c, current: columnWidth(prefs, c.id) }), - ); +export function shownColumns(prefs: ColumnPrefs, unavailable: ColumnId[] = []): Column[] { + const all = allColumns(prefs).filter((c) => !unavailable.includes(c.id)); + const shown = all.filter((c) => isVisible(prefs, c)); + const columns = shown.length > 0 ? shown : all.filter((c) => c.id === "path"); + return columns.map((c) => ({ ...c, current: columnWidth(prefs, c.id) })); +} + +/** Columns the platform has nothing for: App, where it cannot tell which app sent a flow */ +export function unavailableColumns(hasApps: boolean): ColumnId[] { + return hasApps ? [] : ["app"]; } export function columnWidth(prefs: ColumnPrefs, id: ColumnId): number { @@ -171,12 +168,42 @@ export function gridWidth(columns: Column[]): string { return `calc(${px}px + ${columns.length} * ${CELL_PADDING})`; } -export function toggleColumn(prefs: ColumnPrefs, id: ColumnId): ColumnPrefs { - if (id === "path") return prefs; - const shown = prefs.visible[id] ?? columnDef(id).defaultVisible; +/** Whether the picker may uncheck `id`: any column but the last one shown */ +export function canHide(prefs: ColumnPrefs, id: ColumnId, unavailable: ColumnId[] = []): boolean { + const shown = allColumns(prefs).filter((c) => !unavailable.includes(c.id) && isVisible(prefs, c)); + return !shown.every((c) => c.id === id); +} + +export function toggleColumn( + prefs: ColumnPrefs, + id: ColumnId, + unavailable: ColumnId[] = [], +): ColumnPrefs { + const shown = isVisible(prefs, columnDef(id)); + if (shown && !canHide(prefs, id, unavailable)) return prefs; return { ...prefs, visible: { ...prefs.visible, [id]: !shown } }; } +/** Shows `id` alone */ +export function hideOtherColumns(prefs: ColumnPrefs, id: ColumnId): ColumnPrefs { + const visible: ColumnPrefs["visible"] = {}; + for (const c of allColumns(prefs)) visible[c.id] = c.id === id; + return { ...prefs, visible }; +} + +/** Adds a column for a header or query parameter's path, shown at the end; already there, shown */ +export function addCustomColumn(prefs: ColumnPrefs, path: string): ColumnPrefs { + const id = customColumnId(path); + const custom = prefs.custom.includes(id) ? prefs.custom : [...prefs.custom, id]; + return { ...prefs, custom, visible: { ...prefs.visible, [id]: true } }; +} + +export function removeCustomColumn(prefs: ColumnPrefs, id: CustomColumnId): ColumnPrefs { + const { [id]: _v, ...visible } = prefs.visible; + const { [id]: _w, ...widths } = prefs.widths; + return { ...prefs, custom: prefs.custom.filter((c) => c !== id), visible, widths }; +} + export function resizeColumn(prefs: ColumnPrefs, id: ColumnId, width: number): ColumnPrefs { const w = Math.max(columnDef(id).minWidth, Math.round(width)); return { ...prefs, widths: { ...prefs.widths, [id]: w } }; @@ -192,109 +219,285 @@ export function resetWidth(prefs: ColumnPrefs, id: ColumnId): ColumnPrefs { return { ...prefs, widths }; } +/** Shows Operation the first time a flow has one, unless the person ever set it themselves */ +export function showOperationFor(prefs: ColumnPrefs, flows: Flow[]): ColumnPrefs { + if (prefs.visible.operation !== undefined) return prefs; + if (!flows.some((f) => (operationOf(f) ?? "") !== "")) return prefs; + return { ...prefs, visible: { ...prefs.visible, operation: true } }; +} + /** * Reads stored preferences, dropping what no longer applies: unknown ids, and values of the * wrong type, so a change to the columns never breaks the list. */ export function parseColumnPrefs(value: unknown): ColumnPrefs { - const prefs: ColumnPrefs = { visible: {}, widths: {} }; + const prefs: ColumnPrefs = { visible: {}, widths: {}, custom: [], sort: [] }; if (value == null || typeof value !== "object") return prefs; - const { visible, widths } = value as Record; - for (const def of COLUMNS) { + const { visible, widths, custom, sort } = value as Record; + if (Array.isArray(custom)) { + for (const id of custom) { + if (typeof id !== "string" || !id.startsWith("field:")) continue; + const path = customPath(id.slice("field:".length)); + if (path != null && !prefs.custom.includes(customColumnId(path))) { + prefs.custom.push(customColumnId(path)); + } + } + } + for (const def of allColumns(prefs)) { const v = (visible as Record | undefined)?.[def.id]; if (typeof v === "boolean") prefs.visible[def.id] = v; const w = (widths as Record | undefined)?.[def.id]; if (typeof w === "number" && Number.isFinite(w) && w > 0) prefs.widths[def.id] = w; } + if (Array.isArray(sort)) { + for (const key of sort) { + if (key == null || typeof key !== "object") continue; + const { column, descending } = key as Record; + if (typeof column !== "string" || typeof descending !== "boolean") continue; + const def = DEFS.get(column as ColumnId); + if (def == null || prefs.sort.some((k) => k.column === column)) continue; + prefs.sort.push({ column: def.id as SortColumn, descending }); + } + } return prefs; } -/** A sort by a column, or by start time when null */ -export interface FlowSort { - column: ColumnId; - descending: boolean; -} +// -- Sorting -- /** Numbers sort largest first on the first click, names from A */ -export function firstDirection(column: ColumnId): boolean { - return column === "number" || column === "size" || column === "time" || column === "status"; -} - -/** - * A click on a column's label: sort by it, reverse the sort it has, then, from the reversed sort, - * go back to start time - */ -export function nextSort(sort: FlowSort | null, column: ColumnId): FlowSort | null { - const first = firstDirection(column); - if (sort?.column !== column) return { column, descending: first }; - if (sort.descending === first) return { column, descending: !first }; - return null; -} - -const collator = new Intl.Collator(undefined, { numeric: true, sensitivity: "base" }); - -type Key = string | number | null; - -function sortKey(flow: Flow, column: ColumnId): Key { +export function firstDirection(column: SortColumn): boolean { switch (column) { case "number": - return flow.number; - case "app": - return appKeyOf(flow); - case "method": - return flow.method; - case "host": - return hostOf(flow.url); - case "path": - return pathOf(flow.url); case "status": - return flow.error != null ? -1 : flow.status; - case "type": - return flow.contentType; case "size": - return flow.error != null && flow.status == null ? null : flow.responseSize; + case "requestSize": case "time": - return flow.elapsedMs; - case "remote": - return flow.remoteAddr; - case "version": - return flow.upstreamHttpVersion ?? flow.clientHttpVersion; + case "latency": + return true; + default: + return false; } } /** - * The flows in `sort`'s order, those without a value (none, empty or zero) lowest, so first - * ascending and last descending, then by start so equal values keep arrival order. Sorts the - * flows given, which are the ones loaded, not the whole database: the database only ever reads - * by start time, so no `ORDER BY` follows the column sort. + * A click on a column's label: sort by it alone, reverse the sort it has alone, then, from the + * reversed sort, go back to the order flows arrive in */ -export function sortFlows(flows: Flow[], sort: FlowSort | null): Flow[] { - if (sort == null) return flows; - const dir = sort.descending ? -1 : 1; - const keyed = flows.map((flow, i) => ({ flow, i, key: sortKey(flow, sort.column) || null })); - keyed.sort((a, b) => { - let c: number; - if (a.key == null || b.key == null) c = Number(b.key == null) - Number(a.key == null); - else if (typeof a.key === "number" && typeof b.key === "number") c = a.key - b.key; - else c = collator.compare(String(a.key), String(b.key)); - return c === 0 ? a.i - b.i : c * dir; - }); - return keyed.map((k) => k.flow); +export function nextSort(sort: FlowSortKey[], column: SortColumn): FlowSortKey[] { + const first = firstDirection(column); + const only = sort.length === 1 ? sort[0] : null; + if (only?.column !== column) return [{ column, descending: first }]; + if (only.descending === first) return [{ column, descending: !first }]; + return []; +} + +/** + * An Option-click on a column's label: adds it as the last key, ascending; again reverses it, and + * a third time takes it out + */ +export function toggleSortKey(sort: FlowSortKey[], column: SortColumn): FlowSortKey[] { + const key = sort.find((k) => k.column === column); + if (key == null) return [...sort, { column, descending: false }]; + if (!key.descending) + return sort.map((k) => (k.column === column ? { column, descending: true } : k)); + return sort.filter((k) => k.column !== column); +} + +/** Adds `column` as a key in `descending` order: the last key, or in its place if it is one */ +export function addSortKey( + sort: FlowSortKey[], + column: SortColumn, + descending: boolean, +): FlowSortKey[] { + if (sort.some((k) => k.column === column)) { + return sort.map((k) => (k.column === column ? { column, descending } : k)); + } + return [...sort, { column, descending }]; +} + +/** What an empty number sorts as, below every value; `EMPTY_NUMBER` in the lib's `flows.rs` */ +const EMPTY_NUMBER = -1e18; + +/** Text as SQLite's `NOCASE` compares it: ASCII letters lowercased, nothing else changed */ +function noCase(text: string | null | undefined): string { + return (text ?? "").replace(/[A-Z]/g, (c) => c.toLowerCase()); +} + +function emptyNumber(n: number | null | undefined): number { + return n == null || n === 0 ? EMPTY_NUMBER : n; +} + +/** + * The value `column` sorts `flow` by, as `SortColumn::sql` in the lib's `flows.rs` makes it in + * the database, so a flow placed here lands where a page read from there would have it + */ +export function columnSortKey(flow: Flow, column: SortColumn): string | number { + switch (column) { + case "number": + return flow.number; + case "app": + return noCase(flow.clientName ?? flow.clientBundleId ?? flow.clientPath); + case "method": + return noCase(flow.method); + case "host": + return noCase(urlParts(flow.url).authority); + case "path": + return noCase(urlParts(flow.url).path); + case "query": + return noCase(urlParts(flow.url).query); + case "status": + return flow.error != null ? -1 : emptyNumber(flow.status); + case "type": + return noCase(flow.contentType); + case "size": + return flow.error != null && flow.status == null + ? EMPTY_NUMBER + : emptyNumber(flow.responseSize); + case "requestSize": + return emptyNumber(flow.requestSize); + case "time": + return emptyNumber(flow.elapsedMs); + case "latency": + return emptyNumber(flow.elapsedHeadersMs); + case "started": + return flow.startedAt; + case "remote": + return noCase(flow.remoteAddr); + case "version": + return noCase(flow.upstreamHttpVersion ?? flow.clientHttpVersion); + case "tls": + return noCase(flow.upstreamTlsVersion); + case "operation": + return noCase( + flow.grpcService != null && flow.grpcMethod != null + ? `${flow.grpcService}/${flow.grpcMethod}` + : flow.graphqlOperation, + ); + } +} + +function compareKeys(a: string | number, b: string | number): number { + if (a === b) return 0; + return a < b ? -1 : 1; +} + +/** + * Orders flows as `list_flows` orders them for `sort`: each key in turn, then by start and id, + * oldest first + */ +export function sortComparator(sort: FlowSortKey[]): (a: Flow, b: Flow) => number { + return (a, b) => { + for (const { column, descending } of sort) { + const c = compareKeys(columnSortKey(a, column), columnSortKey(b, column)); + if (c !== 0) return descending ? -c : c; + } + return compareFlows(a, b); + }; +} + +/** The flows in `sort`'s order, as the database orders them */ +export function sortFlows(flows: Flow[], sort: FlowSortKey[]): Flow[] { + if (sort.length === 0) return flows; + return [...flows].sort(sortComparator(sort)); +} + +// -- Values -- + +/** + * A URL's parts as the lib's `url_parts` reads them: no scheme, all host; otherwise the + * authority up to the first `/`, `?` or `#`, then the path and the query without its `?` + */ +export function urlParts(url: string): { authority: string; path: string; query: string | null } { + const start = url.indexOf("://"); + if (start === -1) return { authority: url, path: "", query: null }; + const rest = url.slice(start + 3); + const end = rest.search(/[/?#]/); + const authority = end === -1 ? rest : rest.slice(0, end); + const tail = (end === -1 ? "" : rest.slice(end)).split("#")[0] ?? ""; + const q = tail.indexOf("?"); + return q === -1 + ? { authority, path: tail, query: null } + : { authority, path: tail.slice(0, q), query: tail.slice(q + 1) }; } /** The URL's host, with its port */ export function hostOf(url: string): string { - const start = url.indexOf("://"); - if (start === -1) return ""; - const rest = url.slice(start + 3); - const end = rest.search(/[/?#]/); - return end === -1 ? rest : rest.slice(0, end); + return url.includes("://") ? urlParts(url).authority : ""; } -function pathOf(url: string): string { - const start = url.indexOf("://"); - if (start === -1) return url; - const rest = url.slice(start + 3); - const end = rest.search(/[/?#]/); - return end === -1 ? "/" : rest.slice(end); +/** `Service/Method` for a gRPC call, the operation's name for GraphQL, or null */ +export function operationOf(flow: Flow): string | null { + if (flow.grpcService != null && flow.grpcMethod != null) { + return `${flow.grpcService}/${flow.grpcMethod}`; + } + return flow.graphqlOperation ?? null; +} + +/** `1.3` from `TLSv1.3` */ +export function tlsVersionOf(flow: Flow): string { + return (flow.upstreamTlsVersion ?? "").replace(/^TLSv?/i, ""); +} + +/** + * The value of a custom column's path for `flow`: a header's first value, the response's before + * the request's for `header.`, or a query parameter's first value + */ +export function customValue(flow: Flow, path: string): string | null { + const header = (headers: Flow["requestHeaders"], name: string) => + headers.find((h) => h.name.toLowerCase() === name)?.value ?? null; + if (path.startsWith("req.header.")) { + return header(flow.requestHeaders, path.slice("req.header.".length)); + } + if (path.startsWith("res.header.")) { + return header(flow.responseHeaders, path.slice("res.header.".length)); + } + if (path.startsWith("header.")) { + const name = path.slice("header.".length); + return header(flow.responseHeaders, name) ?? header(flow.requestHeaders, name); + } + if (path.startsWith("query.")) { + const query = urlParts(flow.url).query; + if (query == null) return null; + const name = path.slice("query.".length); + for (const [key, value] of new URLSearchParams(query)) { + if (key.toLowerCase() === name) return value; + } + } + return null; +} + +/** + * The search term that finds flows with `flow`'s value in `column`, or excludes them, for the + * cell's menu; null for columns with no field to match their value exactly + */ +export function cellFilterTerm(flow: Flow, column: ColumnId, negated: boolean): string | null { + const term = (field: string, value: string | null | undefined) => + value == null || value === "" ? null : fieldTerm(field, value, negated); + if (isCustomColumn(column)) { + const path = column.slice("field:".length); + return term(path, customValue(flow, path)); + } + switch (column) { + case "app": + return term("app", flow.clientName ?? flow.clientBundleId ?? flow.clientPath); + case "method": + return term("method", flow.method); + case "host": + return term("host", sidebarPath(flow.url).split("/")[0]); + case "path": + return flow.tunnelled ? null : term("path", urlParts(flow.url).path); + case "status": + return flow.status == null ? null : term("status", String(flow.status)); + case "type": + return term("type", flow.contentType?.split(";")[0]?.trim()); + case "operation": { + if (flow.grpcService != null && flow.grpcMethod != null) { + const both = `${fieldTerm("grpc.service", flow.grpcService)} ${fieldTerm("grpc.method", flow.grpcMethod)}`; + return negated ? `-(${both})` : both; + } + return term("graphql.operation", flow.graphqlOperation); + } + default: + return null; + } } diff --git a/apps/yaak-proxy/lib/flowActions.ts b/apps/yaak-proxy/lib/flowActions.ts index 985d306fd..a8e4e138b 100644 --- a/apps/yaak-proxy/lib/flowActions.ts +++ b/apps/yaak-proxy/lib/flowActions.ts @@ -3,11 +3,13 @@ import type { DropdownItem } from "@yaakapp-internal/ui"; import { save } from "@tauri-apps/plugin-dialog"; import { formatSize } from "@yaakapp-internal/lib/formatSize"; import { atom, useAtomValue, useSetAtom } from "jotai"; -import { useCallback } from "react"; +import { useCallback, useEffect } from "react"; import { base64ToBytes, bodyFileName } from "./body"; import { curlBodyText, toCurl } from "./curl"; +import type { FlowQuery } from "./query"; import { rpc } from "./rpc"; import { activeQueryAtom, lockedAtom } from "./store"; +import { subscribe } from "./tauri"; import { useShowToast } from "./toast"; /** The flow the save dialog is open for */ @@ -61,12 +63,32 @@ export async function exportHar( return rpc("export_har", { ...req, path }); } +/** Exports every flow `query` covers, or all of them, as HAR, saying how many it wrote */ +function useExportHarOf(): (query: FlowQuery | null) => Promise { + const showToast = useShowToast(); + return useCallback( + async (query: FlowQuery | null) => { + try { + const written = await exportHar(query == null ? {} : { filter: query }, "flows.har"); + if (written != null) { + showToast({ + message: written === 1 ? "Exported 1 flow as HAR" : `Exported ${written} flows as HAR`, + }); + } + } catch (err) { + showToast({ message: `Failed to export HAR: ${String(err)}`, color: "danger" }); + } + }, + [showToast], + ); +} + /** - * Exporting every flow the list covers as HAR, all of them or the search's, for the list's menus: - * it acts on flows, so it lives where they are chosen. + * Exporting every flow the list covers as HAR, all of them or the search's, for the list's and + * the search box's menus: it acts on flows, so it lives where they are chosen. */ export function useExportShownHar(): () => DropdownItem { - const showToast = useShowToast(); + const exportOf = useExportHarOf(); const query = useAtomValue(activeQueryAtom); // Nothing can be read to export while captured traffic is locked. const locked = useAtomValue(lockedAtom) != null; @@ -75,24 +97,28 @@ export function useExportShownHar(): () => DropdownItem { label: query == null ? "Export All Flows as HAR" : "Export Matching Flows as HAR", icon: "download", disabled: locked, - onSelect: async () => { - try { - const written = await exportHar(query == null ? {} : { filter: query }, "flows.har"); - if (written != null) { - showToast({ - message: - written === 1 ? "Exported 1 flow as HAR" : `Exported ${written} flows as HAR`, - }); - } - } catch (err) { - showToast({ message: `Failed to export HAR: ${String(err)}`, color: "danger" }); - } - }, + onSelect: () => exportOf(query), }), - [query, locked, showToast], + [query, locked, exportOf], ); } +/** File's Export All Flows as HAR, which the app menu sends as an event */ +export function useMenuExportHar() { + const exportOf = useExportHarOf(); + const locked = useAtomValue(lockedAtom) != null; + useEffect( + () => + subscribe(MENU_EXPORT_HAR_EVENT, () => { + if (!locked) void exportOf(null); + }), + [exportOf, locked], + ); +} + +/** `EXPORT_HAR_EVENT` in the app's `window_menu.rs` */ +const MENU_EXPORT_HAR_EVENT = "menu_export_har"; + /** What a flow's context menu and the detail pane's menu offer */ export function useFlowActions(): (flow: Flow) => DropdownItem[] { const showToast = useShowToast(); diff --git a/apps/yaak-proxy/lib/flowViews.test.ts b/apps/yaak-proxy/lib/flowViews.test.ts index 6ef8d323c..b8107835b 100644 --- a/apps/yaak-proxy/lib/flowViews.test.ts +++ b/apps/yaak-proxy/lib/flowViews.test.ts @@ -58,6 +58,7 @@ const flow: Flow = { grpcMethod: null, grpcStatus: null, grpcMessage: null, + graphqlOperation: null, tunnelled: false, agentRead: false, clientAddr: null, diff --git a/apps/yaak-proxy/lib/follow.test.ts b/apps/yaak-proxy/lib/follow.test.ts index 9db5bf3a7..30f97236b 100644 --- a/apps/yaak-proxy/lib/follow.test.ts +++ b/apps/yaak-proxy/lib/follow.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "vite-plus/test"; -import { countNewFlows } from "./follow"; +import { countNewFlows, isAtEnd, resumesFollowing } from "./follow"; const at = (...times: string[]) => times.map((t) => ({ startedAt: `2026-10-26T10:00:${t}` })); @@ -26,3 +26,28 @@ describe("countNewFlows", () => { expect(countNewFlows(at("03", "03.25"), "2026-10-26T10:00:03.2")).toBe(1); }); }); + +describe("resumesFollowing", () => { + const fits = { scrollHeight: 300, scrollTop: 0, clientHeight: 300 }; + const scrolledUp = { scrollHeight: 3000, scrollTop: 1200, clientHeight: 300 }; + const atBottom = { scrollHeight: 3000, scrollTop: 2700, clientHeight: 300 }; + + test("follows again when the list fits its viewport, so it never shows new flows as unseen", () => { + expect(isAtEnd(fits)).toBe(true); + expect(resumesFollowing({ following: false, sorted: false, el: fits })).toBe(true); + }); + + test("follows again at the end of a list that overflows", () => { + expect(resumesFollowing({ following: false, sorted: false, el: atBottom })).toBe(true); + }); + + test("stays held while scrolled up in a list that overflows", () => { + expect(isAtEnd(scrolledUp)).toBe(false); + expect(resumesFollowing({ following: false, sorted: false, el: scrolledUp })).toBe(false); + }); + + test("never follows while sorted, and has nothing to do while following", () => { + expect(resumesFollowing({ following: false, sorted: true, el: fits })).toBe(false); + expect(resumesFollowing({ following: true, sorted: false, el: fits })).toBe(false); + }); +}); diff --git a/apps/yaak-proxy/lib/follow.ts b/apps/yaak-proxy/lib/follow.ts index 354e1abb6..4b2c0e9b8 100644 --- a/apps/yaak-proxy/lib/follow.ts +++ b/apps/yaak-proxy/lib/follow.ts @@ -1,5 +1,30 @@ import type { Flow } from "@yaakapp-internal/proxy-lib"; +/** Where a list's scroll is, as its element reports it */ +export interface ScrollPosition { + scrollHeight: number; + scrollTop: number; + clientHeight: number; +} + +/** The list is scrolled to its end, or fits its viewport and has no scroll at all */ +export function isAtEnd(el: ScrollPosition): boolean { + return el.scrollHeight - el.scrollTop - el.clientHeight < 2; +} + +/** + * A held list follows new traffic again once it is at its end, unless it is sorted: after a + * scroll there, or after its flows, filter or size change leave it there, as a list that fits its + * viewport always is. One scrolled up in a list that overflows stays held. + */ +export function resumesFollowing(state: { + following: boolean; + sorted: boolean; + el: ScrollPosition; +}): boolean { + return !state.following && !state.sorted && isAtEnd(state.el); +} + /** * Flows in the list that started after the list was held. `heldAt` is the start of the newest * flow across all traffic at that moment, or null when there was none, so it still holds when diff --git a/apps/yaak-proxy/lib/sortedWindow.test.ts b/apps/yaak-proxy/lib/sortedWindow.test.ts new file mode 100644 index 000000000..d203083ff --- /dev/null +++ b/apps/yaak-proxy/lib/sortedWindow.test.ts @@ -0,0 +1,89 @@ +import type { Flow, ModelPayload } from "@yaakapp-internal/proxy-lib"; +import { describe, expect, test } from "vite-plus/test"; +import { sortComparator } from "./columns"; +import { appendPage, applySortedChanges, type SortedWindow } from "./sortedWindow"; + +function flow(n: number, status: number | null): Flow { + return { + id: `fl_${n}`, + number: n, + startedAt: `2026-11-09T12:00:0${n}`, + status, + error: null, + } as Flow; +} + +const bySize = sortComparator([{ column: "status", descending: true }]); +const upsert = (model: Flow, search?: ModelPayload["search"]): ModelPayload => ({ + model, + change: { type: "upsert", created: true }, + ...(search == null ? {} : { search }), +}); +const remove = (model: Flow): ModelPayload => ({ model, change: { type: "delete" } }); +const numbers = (flows: Flow[]) => flows.map((f) => f.number); + +describe("applySortedChanges", () => { + const held = [flow(1, 500), flow(2, 404), flow(3, 200)]; + + test("places a new or changed flow by the sort, and drops a deleted one", () => { + let flows = applySortedChanges(held, null, [upsert(flow(4, 300))], bySize, null); + expect(numbers(flows)).toEqual([1, 2, 4, 3]); + flows = applySortedChanges(flows, null, [upsert(flow(3, 503))], bySize, null); + expect(numbers(flows)).toEqual([3, 1, 2, 4]); + flows = applySortedChanges(flows, null, [remove(flow(1, 500))], bySize, null); + expect(numbers(flows)).toEqual([3, 2, 4]); + }); + + test("leaves a flow past the last one read for paging to bring", () => { + const boundary = held[2] ?? null; + const flows = applySortedChanges(held, boundary, [upsert(flow(4, 100))], bySize, null); + expect(numbers(flows)).toEqual([1, 2, 3]); + // The last flow read, written again unchanged, stays. + const same = applySortedChanges(held, boundary, [upsert(flow(3, 200))], bySize, null); + expect(numbers(same)).toEqual([1, 2, 3]); + // A held flow that moves past the end leaves until paging reaches it. + const moved = applySortedChanges(held, boundary, [upsert(flow(1, 100))], bySize, null); + expect(numbers(moved)).toEqual([2, 3]); + }); + + test("searching, takes only flows the search matched, and keeps ones it can't tell", () => { + const changes = [ + upsert(flow(4, 300), { key: "k", matches: true }), + upsert(flow(5, 300), { key: "k", matches: false }), + upsert(flow(2, 404), { key: "k", matches: false }), + upsert(flow(3, 999), { key: "other", matches: true }), + ]; + const flows = applySortedChanges(held, null, changes, bySize, "k"); + expect(numbers(flows)).toEqual([1, 4, 3]); + }); +}); + +test("a page adds what it does not already hold", () => { + const win: SortedWindow = { + key: "1", + sort: [], + query: null, + flows: [flow(1, 500), flow(2, 404)], + boundary: flow(2, 404), + cursor: [404, "fl_2"], + loading: false, + buffered: [], + }; + const next = appendPage(win, [flow(2, 404), flow(3, 200)], [200, "fl_3"], 2); + expect(numbers(next.flows)).toEqual([1, 2, 3]); + expect(next.boundary?.id).toBe("fl_3"); + expect(next.cursor).toEqual([200, "fl_3"]); + // A short page is the last: nothing is past it. + const last = appendPage(win, [flow(3, 200)], [200, "fl_3"], 2); + expect([last.boundary, last.cursor]).toEqual([null, null]); + // One the sweep deleted meanwhile is left out, but still marks where the page ended. + const swept = appendPage( + win, + [flow(3, 200), flow(4, 100)], + [100, "fl_4"], + 2, + (f) => f.id !== "fl_4", + ); + expect(numbers(swept.flows)).toEqual([1, 2, 3]); + expect(swept.boundary?.id).toBe("fl_4"); +}); diff --git a/apps/yaak-proxy/lib/sortedWindow.ts b/apps/yaak-proxy/lib/sortedWindow.ts new file mode 100644 index 000000000..f5fcd3d0e --- /dev/null +++ b/apps/yaak-proxy/lib/sortedWindow.ts @@ -0,0 +1,97 @@ +import type { Flow, FlowSortKey, ModelPayload, SortValue } from "@yaakapp-internal/proxy-lib"; +import type { FlowQuery } from "./query"; + +/** + * The flows the list shows while it is sorted: read from the database in the sort's order, a + * page at a time from the top, as `list_flows` orders them for `sort`. Flows that change are + * placed by the same comparison, and only up to where the pages read end, so paging on never + * skips or repeats one. + */ +export interface SortedWindow { + /** Unique to this load, so a page read for an earlier sort or search is dropped */ + key: string; + sort: FlowSortKey[]; + query: FlowQuery | null; + /** In the sort's order */ + flows: Flow[]; + /** + * The last flow of the last page, as it was read, while the database has more after it: where + * the next page starts. Null once every flow is read. + */ + boundary: Flow | null; + /** The next page's `cursor`, from the last page: the boundary's sort values as read */ + cursor: SortValue[] | null; + /** The first page has not arrived yet */ + loading: boolean; + /** Changes that came while the first page loaded, applied once it is in */ + buffered: ModelPayload[]; +} + +/** Where `flow` goes in `flows`, which `compare` orders */ +function insertionIndex(flows: Flow[], flow: Flow, compare: (a: Flow, b: Flow) => number): number { + let lo = 0; + let hi = flows.length; + while (lo < hi) { + const mid = (lo + hi) >>> 1; + const item = flows[mid]; + if (item != null && compare(item, flow) < 0) lo = mid + 1; + else hi = mid; + } + return lo; +} + +/** + * Applies flow events to a sorted window's flows: a changed flow moves to where `compare` puts + * it, a deleted one leaves, and one that sorts after `boundary`, where the pages read end, is + * left for paging. With `searchKey`, only flows the backend matched against that search join or + * stay; a flow matched against another search is left as it is, to be matched again. + */ +export function applySortedChanges( + flows: Flow[], + boundary: Flow | null, + changes: ModelPayload[], + compare: (a: Flow, b: Flow) => number, + searchKey: string | null, +): Flow[] { + if (changes.length === 0) return flows; + const next = flows.slice(); + for (const c of changes) { + const deleted = c.change.type === "delete"; + if (!deleted && searchKey != null && c.search?.key !== searchKey) continue; + const existing = next.findIndex((f) => f.id === c.model.id); + if (existing !== -1) next.splice(existing, 1); + if (deleted || (searchKey != null && c.search?.matches !== true)) continue; + if (boundary != null && compare(c.model, boundary) > 0) continue; + next.splice(insertionIndex(next, c.model, compare), 0, c.model); + } + return next; +} + +/** A page from `list_flows` as a sorted window takes it: where it ends, and where the next starts */ +export function pageBoundary( + page: Flow[], + cursor: SortValue[] | null, + limit: number, +): Pick { + const more = page.length >= limit; + return { boundary: more ? (page.at(-1) ?? null) : null, cursor: more ? cursor : null }; +} + +/** + * Adds the next page from `list_flows`, read after the window's boundary, less any it has and + * any `keep` leaves out + */ +export function appendPage( + win: SortedWindow, + page: Flow[], + cursor: SortValue[] | null, + limit: number, + keep: (flow: Flow) => boolean = () => true, +): SortedWindow { + const held = new Set(win.flows.map((f) => f.id)); + return { + ...win, + flows: [...win.flows, ...page.filter((f) => !held.has(f.id) && keep(f))], + ...pageBoundary(page, cursor, limit), + }; +} diff --git a/apps/yaak-proxy/lib/store.ts b/apps/yaak-proxy/lib/store.ts index fc9655c4d..997372718 100644 --- a/apps/yaak-proxy/lib/store.ts +++ b/apps/yaak-proxy/lib/store.ts @@ -1,8 +1,10 @@ import type { Flow, + FlowSortKey, GetProxyStateResponse, ModelPayload, SavedQuery, + SortValue, } from "@yaakapp-internal/proxy-lib"; import { selectedIdsFamily } from "@yaakapp-internal/ui"; import { atom, type createStore } from "jotai"; @@ -11,10 +13,11 @@ import { followingAtom, SIDEBAR_TREE_ID, selectedFlowAtom } from "./atoms"; import { type ColumnPrefs, defaultColumnPrefs, - type FlowSort, parseColumnPrefs, + sortComparator, sortFlows, } from "./columns"; +import { appendPage, applySortedChanges, pageBoundary, type SortedWindow } from "./sortedWindow"; import { applyChanges, emptyWindow, @@ -158,7 +161,7 @@ const storedColumnPrefsAtom = atomWithStorage( }, ); -/** Which columns the list shows and how wide, as the person left them */ +/** Which columns the list shows, how wide, and its sort, as the person left them */ export const columnPrefsAtom = atom( (get) => parseColumnPrefs(get(storedColumnPrefsAtom)), (get, set, update: ColumnPrefs | ((prev: ColumnPrefs) => ColumnPrefs)) => { @@ -167,8 +170,14 @@ export const columnPrefsAtom = atom( }, ); -/** The list's sort, or null for start time, the order flows arrive in */ -export const flowSortAtom = atom(null); +/** The list's sort keys, in order, kept with the columns; empty for the order flows arrive in */ +export const flowSortAtom = atom( + (get) => get(columnPrefsAtom).sort, + (_get, set, sort: FlowSortKey[]) => set(columnPrefsAtom, (p) => ({ ...p, sort })), +); + +/** The flows the list shows while sorted, read from the database in the sort's order */ +export const sortedWindowAtom = atom(null); /** A window onto the flows a search covers, loaded from the database as the main one is */ export interface SearchWindow { @@ -189,12 +198,26 @@ const shownWindowAtom = atom((get) => get(searchWindowAtom)?.win ?? get(flowWind /** The flows loaded for the list, oldest first */ export const windowFlowsAtom = atom((get) => get(shownWindowAtom).flows); -/** The flows the list shows: oldest first, or in the sort's order, over those loaded */ -export const shownFlowsAtom = atom((get) => sortFlows(get(windowFlowsAtom), get(flowSortAtom))); +/** + * The flows the list shows: oldest first, or in the sort's order from the database. Until a + * sort's first page is in, the flows loaded, sorted here, stand in for it. + */ +export const shownFlowsAtom = atom((get) => { + const sort = get(flowSortAtom); + if (sort.length === 0) return get(windowFlowsAtom); + const sorted = get(sortedWindowAtom); + if (sorted == null || (sorted.loading && sorted.flows.length === 0)) { + return sortFlows(get(windowFlowsAtom), sort); + } + return sorted.flows; +}); /** Flows in the database the list covers: every one, or those the search matches */ export const shownTotalAtom = atom((get) => get(shownWindowAtom).total); +/** The search's first page has not arrived, so the list shows none of its flows yet */ +export const searchLoadingAtom = atom((get) => get(searchWindowAtom)?.loading ?? false); + /** Start and id of the oldest flow shown, which loading older flows moves back */ export const oldestFlowAtom = atom((get) => get(windowFlowsAtom)[0] ?? null); @@ -288,6 +311,54 @@ export const loadOlderFlowsAtom = atom(null, async (get, set) => { } }); +/** Changes applied while a page of a sort loads, applied again over it once it is in */ +let changesDuringSortedPage: ModelPayload[] | null = null; + +/** The key flow events mark matches of the search shown with, or null for all traffic */ +function searchKeyOf(search: SearchWindow | null): string | null { + return search?.key ?? null; +} + +/** Loads the sort's next page, after the last flow shown, on scrolling to the bottom */ +export const loadMoreSortedAtom = atom(null, async (get, set) => { + const win = get(sortedWindowAtom); + const cursor = win?.cursor; + if (win == null || win.loading || win.boundary == null || cursor == null) return; + if (changesDuringSortedPage != null) return; + const changes: ModelPayload[] = []; + changesDuringSortedPage = changes; + pageStarted(); + try { + const res = await rpc("list_flows", { + ...win.query, + sort: win.sort, + cursor, + limit: PAGE_SIZE, + }); + set(sortedWindowAtom, (s) => { + if (s?.key !== win.key || s.cursor !== cursor) return s; + const kept = new Set(withoutRemoved(res.flows).map((f) => f.id)); + const merged = appendPage(s, res.flows, res.cursor, PAGE_SIZE, (f) => kept.has(f.id)); + const compare = sortComparator(s.sort); + const key = searchKeyOf(get(searchWindowAtom)); + return { + ...merged, + flows: applySortedChanges(merged.flows, merged.boundary, changes, compare, key), + }; + }); + } catch (err) { + // Read the sort again from its top. + console.warn("Failed to load more sorted flows", err); + reloadSorted?.(); + } finally { + changesDuringSortedPage = null; + pageSettled(); + } +}); + +/** Reads the sort's first page again, once `syncFlows` has started */ +let reloadSorted: (() => void) | null = null; + /** `changesFor`, with flows matched against another search sent to be matched again */ function changesOf( search: Pick, @@ -320,6 +391,7 @@ function matchLater(search: Pick, changes: ModelP search: { key: search.key, matches: covered.has(c.model.id) }, })); if (marked.length === 0) return; + applySortedChangesTo(store, marked); const following = store.get(followingAtom); store.set(searchWindowAtom, (s) => { if (s?.key !== search.key) return s; @@ -331,6 +403,19 @@ function matchLater(search: Pick, changes: ModelP .catch((err) => console.warn("Failed to match flows", err)); } +/** Applies flow events to the sorted window, or holds them while its first page loads */ +function applySortedChangesTo(store: Store, changes: ModelPayload[]) { + if (store.get(sortedWindowAtom) == null) return; + changesDuringSortedPage?.push(...changes); + const key = searchKeyOf(store.get(searchWindowAtom)); + store.set(sortedWindowAtom, (s) => { + if (s == null) return s; + if (s.loading) return { ...s, buffered: [...s.buffered, ...changes] }; + const flows = applySortedChanges(s.flows, s.boundary, changes, sortComparator(s.sort), key); + return flows === s.flows ? s : { ...s, flows }; + }); +} + /** * Loads the newest flows, then keeps them current with the engine's events, or with reads of the * database while another process owns the engine. Events gather for `FLUSH_MS` and apply @@ -366,6 +451,7 @@ export function syncFlows(store: Store) { const next = applyChanges(s.win, changesOf(s, s.win, changes)); return { ...s, win: following ? trimWindow(next) : next }; }); + applySortedChangesTo(store, changes); const selected = store.get(selectedFlowAtom); const change = changes.findLast((c) => c.model.id === selected?.id); if (change != null) { @@ -477,6 +563,16 @@ export function syncFlows(store: Store) { dropRemovedChanges(buffered, removed); return { ...s, win: drop(s.win), buffered }; }); + if (changesDuringSortedPage != null) dropRemovedChanges(changesDuringSortedPage, removed); + store.set(sortedWindowAtom, (s) => { + if (s == null) return s; + const buffered = [...s.buffered]; + dropRemovedChanges(buffered, removed); + const flows = s.flows.some((f) => removed.has(f.id)) + ? s.flows.filter((f) => !removed.has(f.id)) + : s.flows; + return { ...s, flows, buffered }; + }); if (removed.has(store.get(selectedFlowAtom)?.id ?? "")) store.set(selectedFlowAtom, null); if (recountTimer == null) recountTimer = setTimeout(recount, RECOUNT_MS); }); @@ -503,9 +599,62 @@ export function syncFlows(store: Store) { }) .finally(pageSettled); loadSearch(store.get(activeQueryAtom)); + loadSorted(); } reloadFlows = load; + /** + * Reads the first page of the list's sort from the database, over the search shown, or drops + * the sorted window when there is no sort. Until the page is in, the flows loaded, sorted here, + * stand in for it. + */ + let sortedLoads = 0; + function loadSorted() { + const sort = store.get(flowSortAtom); + if (sort.length === 0) { + store.set(sortedWindowAtom, null); + return; + } + const query = store.get(activeQueryAtom); + const key = `${++sortedLoads}:${JSON.stringify({ sort, query })}`; + store.set(sortedWindowAtom, { + key, + sort, + query, + flows: [], + boundary: null, + cursor: null, + loading: true, + buffered: [], + }); + const settle = (page: Flow[] | null, cursor: SortValue[] | null) => + store.set(sortedWindowAtom, (s) => { + if (s?.key !== key) return s; + const ends = pageBoundary(page ?? [], cursor, PAGE_SIZE); + const base = page == null ? s.flows : withoutRemoved(page); + const search = searchKeyOf(store.get(searchWindowAtom)); + const compare = sortComparator(sort); + const flows = applySortedChanges(base, ends.boundary, s.buffered, compare, search); + return { ...s, ...ends, flows, loading: false, buffered: [] }; + }); + pageStarted(); + void rpc("list_flows", { ...query, sort, limit: PAGE_SIZE }) + .then((res) => settle(res.flows, res.cursor)) + .catch((err) => { + console.warn("Failed to read the sorted flows", err); + settle(null, null); + }) + .finally(pageSettled); + } + reloadSorted = loadSorted; + let sortJson = JSON.stringify(store.get(flowSortAtom)); + store.sub(flowSortAtom, () => { + const json = JSON.stringify(store.get(flowSortAtom)); + if (json === sortJson) return; + sortJson = json; + loadSorted(); + }); + /** * Loads the first page of `query`'s flows into a window of its own, or drops it for null. The * backend matches the flows written from now on against it, and their events say so. @@ -590,6 +739,7 @@ export function syncFlows(store: Store) { if (key === activeKey) return; activeKey = key; loadSearch(query); + loadSorted(); }); // While another process owns the engine no events come, so new flows are read from the @@ -652,9 +802,11 @@ const emptyFlowsAtom = atom(null, (_get, set) => { set(flowWindowAtom, emptyWindow); set(nonSystemTotalAtom, 0); set(searchWindowAtom, (s) => (s == null ? s : { ...s, win: emptyWindow, buffered: [] })); + set(sortedWindowAtom, (s) => + s == null ? s : { ...s, flows: [], boundary: null, cursor: null, buffered: [] }, + ); set(selectedFlowAtom, null); set(followingAtom, true); - set(flowSortAtom, null); }); /** diff --git a/apps/yaak-proxy/lib/tauri.test.ts b/apps/yaak-proxy/lib/tauri.test.ts new file mode 100644 index 000000000..57ea38398 --- /dev/null +++ b/apps/yaak-proxy/lib/tauri.test.ts @@ -0,0 +1,65 @@ +import { beforeEach, describe, expect, test, vi } from "vite-plus/test"; + +type Handler = (e: { payload: unknown }) => void; + +/** Listeners Tauri holds, and the registrations not yet resolved */ +const listeners = new Set(); +let pending: (() => void)[] = []; + +vi.mock("@tauri-apps/api/event", () => ({ + // Registers asynchronously, as Tauri does, resolving only when the test lets it. + listen: (_event: string, handler: Handler) => + new Promise<() => void>((resolve) => { + pending.push(() => { + listeners.add(handler); + resolve(() => listeners.delete(handler)); + }); + }), +})); +vi.mock("@tauri-apps/api/core", () => ({ invoke: vi.fn() })); +vi.mock("@tauri-apps/api/webviewWindow", () => ({ getCurrentWebviewWindow: vi.fn() })); +vi.mock("@tauri-apps/plugin-os", () => ({ type: vi.fn() })); + +const { subscribe } = await import("./tauri"); + +async function register() { + for (const resolve of pending) resolve(); + pending = []; + await Promise.resolve(); + await Promise.resolve(); +} + +function emit(payload: unknown) { + for (const handler of listeners) handler({ payload }); +} + +describe("subscribe", () => { + beforeEach(() => { + listeners.clear(); + pending = []; + }); + + test("an effect cleaned up before its listener registers hears nothing, as in strict mode", async () => { + const heard = vi.fn(); + // Strict mode: subscribe, clean up at once, subscribe again + const first = subscribe("menu_export_har", heard); + first(); + const second = subscribe("menu_export_har", heard); + await register(); + emit(null); + expect(heard).toHaveBeenCalledTimes(1); + expect(listeners.size).toBe(1); + second(); + expect(listeners.size).toBe(0); + }); + + test("delivers events until unsubscribed", async () => { + const heard = vi.fn(); + const unsubscribe = subscribe("menu_export_har", heard); + await register(); + emit(1); + unsubscribe(); + emit(2); + expect(heard.mock.calls).toEqual([[1]]); + }); +}); diff --git a/apps/yaak-proxy/lib/tauri.ts b/apps/yaak-proxy/lib/tauri.ts index e4371eb89..344d0ad3b 100644 --- a/apps/yaak-proxy/lib/tauri.ts +++ b/apps/yaak-proxy/lib/tauri.ts @@ -9,15 +9,19 @@ export function command(cmd: string, args?: Record): Promise return invoke(cmd, args) as Promise; } -/** Subscribe to a Tauri event. Returns an unsubscribe function. */ +/** + * Subscribe to a Tauri event. Returns an unsubscribe function, which also removes a listener + * still being registered once it is, and stops it delivering meanwhile: an effect cleaned up + * that fast, as strict mode does to every one, would otherwise leave its listener attached and + * hear each event twice. + */ export function subscribe(event: string, callback: (payload: T) => void): () => void { - let unsub: (() => void) | null = null; - tauriListen(event, (e) => callback(e.payload)) - .then((fn) => { - unsub = fn; - }) - .catch(console.error); - return () => unsub?.(); + const subscription = unsubscribeWhenReady( + tauriListen(event, (e) => { + if (subscription.active()) callback(e.payload); + }), + ); + return subscription.unsubscribe; } /** Calls `callback` each time the window regains focus. Returns an unsubscribe function. */ diff --git a/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts b/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts index 98ca43b83..eeb878d1c 100644 --- a/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts +++ b/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts @@ -126,6 +126,11 @@ grpcStatus: number | null, * A gRPC call's `grpc-message`, decoded. */ grpcMessage: string | null, +/** + * A GraphQL request's operation, from its body or URL: `GetUser`, `GetUser +2` for a + * batch, empty for an anonymous one. None when the request is not GraphQL. + */ +graphqlOperation: string | null, /** * An HTTPS connection passed through undecrypted, as the decrypt list leaves it out or its * client rejected the proxy's certificate: method `CONNECT`, its URL the host, its sizes diff --git a/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts b/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts index c97e3c210..de0b1e6b3 100644 --- a/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts +++ b/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts @@ -178,7 +178,24 @@ after?: string, * Only flows older than the flow with this id, by start and then id, for paging back * through older traffic. */ -before?: string, }; +before?: string, +/** + * The order to list in, the list's column sort: each key in turn, then by start and id, + * oldest first. Empty lists newest first, as without it. With keys, `before` does not + * apply; `cursor` pages instead. + */ +sort?: Array, +/** + * Only flows after these values in `sort`'s order, for paging through a sort: the `cursor` + * of the page before, the sort values of its last flow as it was read, so a change to that + * flow since does not move where the next page starts. + */ +cursor?: Array, }; + +/** + * One key of a sort. + */ +export type FlowSortKey = { column: SortColumn, descending: boolean, }; export type FlowsClearedPayload = Record; @@ -328,7 +345,12 @@ total: number, /** * Flows the filter covers, whatever the limit, when it narrows to fewer than all of them. */ -matching: number | null, }; +matching: number | null, +/** + * Sorted, the next page's `cursor`: the last flow's sort values, as read. None without a + * sort or a flow. + */ +cursor: Array | null, }; export type ListGrpcSchemasRequest = Record; @@ -574,6 +596,16 @@ export type SetSystemProxyAtLaunchRequest = { enabled: boolean, }; export type SetTrafficRequest = { traffic: TrafficRetention, }; +/** + * A column the list sorts by. + */ +export type SortColumn = "number" | "app" | "method" | "host" | "path" | "query" | "status" | "type" | "size" | "requestSize" | "time" | "latency" | "started" | "remote" | "version" | "tls" | "operation"; + +/** + * One of a flow's sort values, as the database sorted it. + */ +export type SortValue = number | string; + /** * How a change gets administrator rights. */ diff --git a/crates-proxy/yaak-proxy-lib/migrations/00000013_flow_graphql_operation.sql b/crates-proxy/yaak-proxy-lib/migrations/00000013_flow_graphql_operation.sql new file mode 100644 index 000000000..7b463c0aa --- /dev/null +++ b/crates-proxy/yaak-proxy-lib/migrations/00000013_flow_graphql_operation.sql @@ -0,0 +1,3 @@ +-- A GraphQL request's operation name, read from its body or URL as it was captured: empty for an +-- anonymous operation, NULL for a flow that is not GraphQL. +ALTER TABLE flows ADD COLUMN graphql_operation TEXT; diff --git a/crates-proxy/yaak-proxy-lib/src/convert.rs b/crates-proxy/yaak-proxy-lib/src/convert.rs index 12c122ffd..bb3a6e6c5 100644 --- a/crates-proxy/yaak-proxy-lib/src/convert.rs +++ b/crates-proxy/yaak-proxy-lib/src/convert.rs @@ -177,7 +177,7 @@ fn request_body( }; if mime == "application/json" || mime.ends_with("+json") { - if let Some(graphql) = graphql_body(flow, text) { + if let Some(graphql) = graphql_body(flow, content_type, text) { return (Some(BODY_TYPE_GRAPHQL.into()), graphql); } return text_body(BODY_TYPE_JSON, text); @@ -206,26 +206,23 @@ fn text_body(body_type: &str, text: &str) -> (Option, BTreeMap Option> { - let path = flow.url.split('?').next().unwrap_or_default(); - if !path.to_ascii_lowercase().ends_with("/graphql") { - return None; +/// A JSON body that is a GraphQL request, as the client's GraphQL body. Detected as the proxy +/// detects it at capture (`yaak_http::graphql`), which leaves out bodies with fields a GraphQL +/// body cannot hold, such as persisted-query `extensions`. +fn graphql_body(flow: &Flow, content_type: &str, text: &str) -> Option> { + let body = yaak_http::graphql::parse_json_body(Some(content_type), text, &flow.url)?; + let mut fields = BTreeMap::from([("query".to_string(), json!(body.query))]); + match body.variables { + Some(Value::String(variables)) => { + fields.insert("variables".into(), json!(variables)); + } + Some(variables) => { + let pretty = serde_json::to_string_pretty(&variables).ok()?; + fields.insert("variables".into(), json!(pretty)); + } + None => {} } - let Value::Object(body) = serde_json::from_str::(text).ok()? else { - return None; - }; - // Anything else, such as persisted-query `extensions`, would be lost in a GraphQL body. - if body.keys().any(|k| !matches!(k.as_str(), "query" | "variables" | "operationName")) { - return None; - } - let query = body.get("query")?.as_str()?; - let mut fields = BTreeMap::from([("query".to_string(), json!(query))]); - if let Some(variables) = body.get("variables").filter(|v| !v.is_null()) { - let pretty = serde_json::to_string_pretty(variables).ok()?; - fields.insert("variables".into(), json!(pretty)); - } - if let Some(name) = body.get("operationName").and_then(Value::as_str) { + if let Some(name) = body.operation_name { fields.insert("operationName".into(), json!(name)); } Some(fields) @@ -355,10 +352,14 @@ mod tests { let c = flow_to_request(&f, &body(persisted)); assert_eq!(c.request.body_type.as_deref(), Some("application/json")); - // The same body anywhere else is plain JSON. + // A `query` that is no GraphQL document is plain JSON. let f = flow("POST", "https://x.test/api", &[("Content-Type", "application/json")]); let c = flow_to_request(&f, &body(r#"{"query":"q"}"#)); assert_eq!(c.request.body_type.as_deref(), Some("application/json")); + + // A GraphQL document is GraphQL on any path, as capture reads it. + let c = flow_to_request(&f, &body(r#"{"query":"mutation Save { save { id } }"}"#)); + assert_eq!(c.request.body_type.as_deref(), Some("graphql")); } #[test] diff --git a/crates-proxy/yaak-proxy-lib/src/db.rs b/crates-proxy/yaak-proxy-lib/src/db.rs index 4a7c81ac4..cbb0d171f 100644 --- a/crates-proxy/yaak-proxy-lib/src/db.rs +++ b/crates-proxy/yaak-proxy-lib/src/db.rs @@ -543,6 +543,7 @@ mod tests { "grpc_method", "grpc_status", "grpc_message", + "graphql_operation", "agent_read", ] { assert!(flow_columns.contains(&column.to_string()), "{column} in {flow_columns:?}"); @@ -614,7 +615,7 @@ mod tests { assert_eq!( &applied[6..], [ - "00000007", "00000008", "00000009", "00000010", "00000011", "00000012" + "00000007", "00000008", "00000009", "00000010", "00000011", "00000012", "00000013" ] ); let mark_columns = columns(&conn, "flow_clear_mark"); diff --git a/crates-proxy/yaak-proxy-lib/src/flows.rs b/crates-proxy/yaak-proxy-lib/src/flows.rs index a01c4c97d..2885b77e0 100644 --- a/crates-proxy/yaak-proxy-lib/src/flows.rs +++ b/crates-proxy/yaak-proxy-lib/src/flows.rs @@ -34,6 +34,152 @@ pub struct FlowFilter { /// through older traffic. #[ts(optional)] pub before: Option, + /// The order to list in, the list's column sort: each key in turn, then by start and id, + /// oldest first. Empty lists newest first, as without it. With keys, `before` does not + /// apply; `cursor` pages instead. + #[ts(optional, as = "Option>")] + pub sort: Vec, + /// Only flows after these values in `sort`'s order, for paging through a sort: the `cursor` + /// of the page before, the sort values of its last flow as it was read, so a change to that + /// flow since does not move where the next page starts. + #[ts(optional, as = "Option>")] + pub cursor: Option>, +} + +/// One of a flow's sort values, as the database sorted it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(untagged)] +#[ts(export, export_to = "gen_rpc.ts")] +pub enum SortValue { + Int(#[ts(type = "number")] i64), + Text(String), +} + +impl SortValue { + fn from_sql(value: Value) -> Self { + match value { + Value::Integer(i) => Self::Int(i), + Value::Real(f) => Self::Int(f as i64), + Value::Text(t) => Self::Text(t), + Value::Null | Value::Blob(_) => Self::Text(String::new()), + } + } + + fn to_sql(&self) -> Value { + match self { + Self::Int(i) => Value::Integer(*i), + Self::Text(t) => Value::Text(t.clone()), + } + } +} + +/// A column the list sorts by. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_rpc.ts")] +pub enum SortColumn { + Number, + App, + Method, + Host, + Path, + Query, + Status, + Type, + Size, + RequestSize, + Time, + Latency, + Started, + Remote, + Version, + Tls, + Operation, +} + +/// One key of a sort. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct FlowSortKey { + pub column: SortColumn, + pub descending: bool, +} + +/// What an empty number sorts as: below every value, as an empty text's `''` is. +const EMPTY_NUMBER: &str = "-1000000000000000000"; + +impl SortColumn { + /// The value the column sorts by, in SQL, never NULL: empty values (none, `''` or `0`) + /// lowest, text without regard to ASCII case. `columnSortKey` in the app's + /// `lib/columns.ts` makes the same values for flows it places itself. + fn sql(self, columns: &FlowColumns) -> String { + let c = |n: &str| columns.col(n); + let number = |v: &str| format!("coalesce(nullif({v}, 0), {EMPTY_NUMBER})"); + let text = |v: &str| format!("(coalesce({v}, '') COLLATE NOCASE)"); + match self { + Self::Number => "flows.rowid".into(), + Self::App => text(&format!( + "coalesce({}, {}, {})", + c("client_name"), + c("client_bundle_id"), + c("client_path") + )), + Self::Method => text("method"), + Self::Host => text("url_authority(url)"), + Self::Path => text("url_path(url)"), + Self::Query => text("url_query_string(url)"), + Self::Status => { + format!("CASE WHEN error IS NOT NULL THEN -1 ELSE {} END", number("status")) + } + Self::Type => text("content_type"), + Self::Size => format!( + "CASE WHEN error IS NOT NULL AND status IS NULL THEN {EMPTY_NUMBER} ELSE {} END", + number("response_size") + ), + Self::RequestSize => number("request_size"), + Self::Time => number("elapsed_ms"), + Self::Latency => number("elapsed_headers_ms"), + Self::Started => "started_at".into(), + Self::Remote => text("remote_addr"), + Self::Version => text("coalesce(upstream_http_version, client_http_version)"), + Self::Tls => text(&c("upstream_tls_version")), + Self::Operation => { + let (service, method) = (c("grpc_service"), c("grpc_method")); + text(&format!( + "CASE WHEN {service} IS NOT NULL AND {method} IS NOT NULL \ + THEN {service} || '/' || {method} ELSE {} END", + c("graphql_operation") + )) + } + } + } +} + +/// A sort's keys in SQL, each with whether it descends, then start and id, ascending. +fn sort_terms(sort: &[FlowSortKey], columns: &FlowColumns) -> Vec<(String, bool)> { + sort.iter() + .map(|k| (k.column.sql(columns), k.descending)) + .chain([ + ("started_at".to_string(), false), + ("flows.id".to_string(), false), + ]) + .collect() +} + +/// The `ORDER BY` for a sort, and the condition for the flows after `cursor`'s values in it: +/// each key equal up to one that is past the cursor's. +fn sort_after(terms: &[(String, bool)], cursor: &[Value]) -> (String, Vec) { + let mut params = Vec::new(); + let mut alternatives = Vec::new(); + for (i, (term, descending)) in terms.iter().enumerate() { + let mut parts: Vec = terms[..i].iter().map(|(t, _)| format!("{t} = ?")).collect(); + params.extend(cursor[..i].iter().cloned()); + parts.push(format!("{term} {} ?", if *descending { "<" } else { ">" })); + params.push(cursor[i].clone()); + alternatives.push(format!("({})", parts.join(" AND "))); + } + (format!("({})", alternatives.join(" OR ")), params) } impl FlowFilter { @@ -197,20 +343,50 @@ pub fn page( db: &ProxyQueryManager, filter: &FlowFilter, ) -> Result<(Vec, usize, Option), String> { + let page = sorted_page(db, filter)?; + Ok((page.flows, page.total, page.matching)) +} + +/// A page of flows, as [`page`] reads it, with where a sort's next page starts. +pub struct Page { + pub flows: Vec, + pub total: usize, + pub matching: Option, + /// The sort values of the last flow, for the next page's `cursor`. None without a sort. + pub cursor: Option>, +} + +/// [`page`], with the cursor for a sort's next page. +pub fn sorted_page(db: &ProxyQueryManager, filter: &FlowFilter) -> Result { db.with_conn(|c| { let tx = c.conn().resolve().unchecked_transaction().map_err(|e| e.to_string())?; - let flows = list_in(&tx, filter)?; + let (flows, cursor) = list_and_cursor(&tx, filter)?; let total = count_in(&tx)?; let matching = if filter.narrows() { Some(count_matching(&tx, filter)?) } else { None }; tx.commit().map_err(|e| e.to_string())?; - Ok((flows, total, matching)) + Ok(Page { flows, total, matching, cursor }) }) } fn list_in(conn: &rusqlite::Connection, filter: &FlowFilter) -> Result, String> { + list_and_cursor(conn, filter).map(|(flows, _)| flows) +} + +/// The flows `filter` covers and, sorted, the sort values of the last one. +fn list_and_cursor( + conn: &rusqlite::Connection, + filter: &FlowFilter, +) -> Result<(Vec, Option>), String> { let (condition, mut params) = filter.condition_in(conn); - // The rowid, for the flow's number. - let mut sql = format!("SELECT rowid, * FROM flows WHERE {condition}"); + let terms = if filter.sort.is_empty() { + Vec::new() + } else { + sort_terms(&filter.sort, &FlowColumns::of(conn)) + }; + // The rowid, for the flow's number, and a sort's values, for its cursor. + let keys: String = + terms.iter().enumerate().map(|(i, (t, _))| format!(", {t} AS sort_{i}")).collect(); + let mut sql = format!("SELECT rowid, *{keys} FROM flows WHERE {condition}"); push_clear_mark(conn, &mut sql, &mut params); if let Some(id) = &filter.after { // By rowid, which only grows as flows are written. @@ -220,24 +396,53 @@ fn list_in(conn: &rusqlite::Connection, filter: &FlowFilter) -> Result sql.push_str(" AND rowid > ?"); params.push(rowid.into()); } - if let Some(id) = &filter.before { - let started_at: Value = conn - .query_row("SELECT started_at FROM flows WHERE id = ?", [id], |r| r.get(0)) - .map_err(|_| format!("No flow {id}"))?; - sql.push_str(" AND (started_at, id) < (?, ?)"); - params.push(started_at); - params.push(id.clone().into()); + if !terms.is_empty() { + if let Some(cursor) = &filter.cursor { + if cursor.len() != terms.len() { + return Err("The cursor is not for this sort".into()); + } + let cursor: Vec = cursor.iter().map(SortValue::to_sql).collect(); + let (after, after_params) = sort_after(&terms, &cursor); + sql.push_str(" AND "); + sql.push_str(&after); + params.extend(after_params); + } + let order: Vec = terms + .iter() + .map(|(t, descending)| format!("{t} {}", if *descending { "DESC" } else { "ASC" })) + .collect(); + sql.push_str(" ORDER BY "); + sql.push_str(&order.join(", ")); + } else { + if let Some(id) = &filter.before { + let started_at: Value = conn + .query_row("SELECT started_at FROM flows WHERE id = ?", [id], |r| r.get(0)) + .map_err(|_| format!("No flow {id}"))?; + sql.push_str(" AND (started_at, id) < (?, ?)"); + params.push(started_at); + params.push(id.clone().into()); + } + sql.push_str(" ORDER BY started_at DESC, id DESC"); } - sql.push_str(" ORDER BY started_at DESC, id DESC"); if let Some(limit) = filter.limit { sql.push_str(" LIMIT ?"); params.push((limit.min(i64::MAX as usize) as i64).into()); } let mut stmt = conn.prepare(&sql).map_err(|e| e.to_string())?; - let rows = stmt - .query_map(rusqlite::params_from_iter(params), Flow::from_row) - .map_err(|e| e.to_string())?; - rows.collect::, _>>().map_err(|e| e.to_string()) + let mut cursor = None; + let mut flows = Vec::new(); + let mut rows = stmt.query(rusqlite::params_from_iter(params)).map_err(|e| e.to_string())?; + while let Some(row) = rows.next().map_err(|e| e.to_string())? { + flows.push(Flow::from_row(row).map_err(|e| e.to_string())?); + if !terms.is_empty() { + let values = (0..terms.len()) + .map(|i| row.get::<_, Value>(format!("sort_{i}").as_str()).map(SortValue::from_sql)) + .collect::, _>>() + .map_err(|e| e.to_string())?; + cursor = Some(values); + } + } + Ok((flows, cursor)) } fn count_matching(conn: &rusqlite::Connection, filter: &FlowFilter) -> Result { @@ -1035,6 +1240,66 @@ mod tests { assert_eq!(matching, None); } + /// A sort on several columns orders by each in turn, text without regard to case, empty + /// values lowest and start as the last tiebreak, and pages through that order from a cursor. + #[test] + fn a_sort_on_several_columns_pages_in_its_order() { + let (_dir, db) = db_of(vec![ + flow("https://B.com/x", "GET", Some(200), None), + flow("https://a.com/y", "GET", Some(404), None), + flow("https://a.com/z", "GET", None, None), + flow("https://c.com/", "GET", Some(200), None), + flow("https://a.com/", "GET", None, Some("boom")), + flow("https://b.com/q", "GET", Some(200), None), + flow("https://A.com/", "GET", Some(404), None), + flow("https://c.com/", "GET", Some(200), None), + ]); + let key = |column, descending| FlowSortKey { column, descending }; + let read = |sort: &[FlowSortKey], cursor: Option>, limit| { + let filter = FlowFilter { + sort: sort.to_vec(), + cursor, + limit: Some(limit), + ..Default::default() + }; + let page = sorted_page(&db, &filter).unwrap(); + let ids: Vec = page.flows.iter().map(|f| f.id[3..].to_string()).collect(); + (ids, page.cursor) + }; + let sorted = |sort: &[FlowSortKey]| read(sort, None, 100).0; + let by_status = [key(SortColumn::Status, true), key(SortColumn::Host, false)]; + let all = sorted(&by_status); + assert_eq!(all, ["001", "006", "000", "005", "003", "007", "004", "002"]); + let mut paged: Vec = Vec::new(); + let mut cursor = None; + loop { + let (page, next) = read(&by_status, cursor.clone(), 3); + if page.is_empty() { + break; + } + cursor = next; + paged.extend(page); + } + assert_eq!(paged, all); + + // The last flow of a page changing does not move where the next page starts. + let (first, cursor) = read(&by_status, None, 3); + assert_eq!(first, ["001", "006", "000"]); + db.with_conn(|c| { + c.conn().resolve().execute("UPDATE flows SET status = 100 WHERE id = 'fl_000'", []) + }) + .unwrap(); + let (next, _) = read(&by_status, cursor, 3); + assert_eq!(next, ["005", "003", "007"]); + + let ascending = [key(SortColumn::Status, false), key(SortColumn::Host, false)]; + assert_eq!(sorted(&ascending), ["002", "004", "000", "005", "003", "007", "001", "006"]); + + // Query alone: the flows without one first, in start order. + let all = sorted(&[key(SortColumn::Query, true)]); + assert_eq!(all, ["000", "001", "002", "003", "004", "005", "006", "007"]); + } + #[test] fn a_search_reads_the_newest_page_through_the_index() { let (_dir, db) = db_of(vec![]); diff --git a/crates-proxy/yaak-proxy-lib/src/lib.rs b/crates-proxy/yaak-proxy-lib/src/lib.rs index cbec53467..d5cb19938 100644 --- a/crates-proxy/yaak-proxy-lib/src/lib.rs +++ b/crates-proxy/yaak-proxy-lib/src/lib.rs @@ -1655,6 +1655,9 @@ pub struct ListFlowsResponse { /// Flows the filter covers, whatever the limit, when it narrows to fewer than all of them. #[ts(type = "number | null")] pub matching: Option, + /// Sorted, the next page's `cursor`: the last flow's sort values, as read. None without a + /// sort or a flow. + pub cursor: Option>, } #[derive(Deserialize, TS)] @@ -2689,8 +2692,9 @@ fn list_actions( } fn list_flows(ctx: &ProxyCtx, req: flows::FlowFilter) -> Result { - let (flows, total, matching) = flows::page(&ctx.db()?, &req).map_err(rpc_error)?; - Ok(ListFlowsResponse { flows, total, matching }) + let flows::Page { flows, total, matching, cursor } = + flows::sorted_page(&ctx.db()?, &req).map_err(rpc_error)?; + Ok(ListFlowsResponse { flows, total, matching, cursor }) } async fn clear_flows(ctx: ProxyCtx, _req: ClearFlowsRequest) -> Result { @@ -2827,9 +2831,52 @@ struct InFlight { /// Changed since it was last written, by a message or a status. dirty: bool, last_write: Option, + /// The request body's first bytes, while it may be a GraphQL request whose operation is + /// read once it is whole. None once it can't be: not JSON, encoded, or too long. + graphql_scan: Option>, } +/// Most request body bytes read for a GraphQL operation; a longer body is not looked at. +const GRAPHQL_SCAN_LIMIT: usize = 64 * 1024; + impl InFlight { + /// Keeps a request body chunk for reading a GraphQL operation from, when the body may be + /// one: JSON, not encoded, and short. `len` is the chunk's size on the wire, `data` what of + /// it was captured. + fn scan_graphql(&mut self, first: bool, len: usize, data: &[u8]) { + if first { + let headers = &self.flow.request_headers; + let json = find_header(headers, "content-type") + .and_then(|t| t.split(';').next()) + .is_some_and(|t| t.trim().eq_ignore_ascii_case("application/json")); + let encoded = find_header(headers, "content-encoding") + .is_some_and(|e| !e.trim().eq_ignore_ascii_case("identity")); + let stream = self.flow.stream_kind.is_some(); + self.graphql_scan = (json && !encoded && !stream).then(Vec::new); + } + let Some(buf) = &mut self.graphql_scan else { + return; + }; + if data.len() < len || buf.len() + len > GRAPHQL_SCAN_LIMIT { + self.graphql_scan = None; + return; + } + buf.extend_from_slice(data); + } + + /// Reads the GraphQL operation from the request body kept by [`Self::scan_graphql`]. + fn finish_graphql_scan(&mut self) { + let Some(body) = self.graphql_scan.take() else { + return; + }; + let content_type = find_header(&self.flow.request_headers, "content-type"); + if let Some(op) = + yaak_http::graphql::detect(&self.flow.method, &self.flow.url, content_type, Some(&body)) + { + self.flow.graphql_operation = Some(op.label()); + } + } + /// Stops writing the bodies its stream's messages carry, which are read back from them. /// A request's bytes are kept while it uploads, as a call can end before its upload does, /// with a message still partway in: they are dropped once the upload ends, or kept as its @@ -3079,6 +3126,12 @@ impl FlowSink { let flow_id = Flow::generate_id(); let request_body = Flow::body_id(&flow_id, FlowBody::Request); let response_body = Flow::body_id(&flow_id, FlowBody::Response); + // GraphQL over GET is all in its URL; a body's operation is read once it is whole. + let graphql_operation = method + .eq_ignore_ascii_case("GET") + .then(|| yaak_http::graphql::detect(&method, &url, None, None)) + .flatten() + .map(|op| op.label()); let flow = Flow { id: flow_id, started_at: DateTime::::from(started_at).naive_utc(), @@ -3086,6 +3139,7 @@ impl FlowSink { url, client_http_version: http_version, client_addr: Some(client_addr.to_string()), + graphql_operation, ..Default::default() }; in_flight.insert( @@ -3100,6 +3154,7 @@ impl FlowSink { next_seq: 0, dirty: false, last_write: None, + graphql_scan: None, }, ); } @@ -3128,6 +3183,7 @@ impl FlowSink { next_seq: 0, dirty: false, last_write: None, + graphql_scan: None, }; // Shown while it is open, as tunnels often stay open for minutes. self.write_stream_flow(&mut f); @@ -3179,6 +3235,7 @@ impl FlowSink { // upload does. ProxyEvent::RequestBodyChunk { id, len, data } => { if let Some(f) = in_flight.get_mut(&id) { + f.scan_graphql(f.flow.request_size == 0, len, &data); f.flow.request_size += len as i64; if f.keeps_bytes(FlowBody::Request) { f.flow.request_truncated |= data.len() < len; @@ -3191,6 +3248,7 @@ impl FlowSink { if let Some(f) = in_flight.get_mut(&id) { f.flow.request_size = size as i64; f.flow.elapsed_request_ms = Some(clamp_ms(elapsed_ms)); + f.finish_graphql_scan(); if f.keeps_bytes(FlowBody::Request) { f.flow.request_truncated = truncated; finish_body(f.request_body.take()); @@ -4487,6 +4545,96 @@ mod tests { .unwrap(); } + /// A GraphQL request's operation is stored with the flow: from a JSON body sent in chunks, + /// and from a GET's URL. A JSON body that is not GraphQL, and one over the scan limit, store + /// none. + #[test] + fn graphql_operations_are_stored_at_capture() { + let dir = tempfile::tempdir().unwrap(); + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + let (tx, event_loop) = event_loop(&ctx); + let send = |event| tx.send(event).unwrap(); + let post = |id: u64, url: &str, chunks: &[&[u8]]| { + send(ProxyEvent::RequestStart { + id, + started_at: SystemTime::now(), + method: "POST".into(), + url: url.into(), + http_version: "HTTP/1.1".into(), + connection: 1, + client_addr: "127.0.0.1:1".parse().unwrap(), + }); + send(ProxyEvent::RequestHeader { + id, + name: "Content-Type".into(), + value: "application/json; charset=utf-8".into(), + }); + let mut size = 0; + for chunk in chunks { + size += chunk.len(); + send(ProxyEvent::RequestBodyChunk { + id, + len: chunk.len(), + data: bytes::Bytes::copy_from_slice(chunk), + }); + } + send(ProxyEvent::RequestBodyComplete { + id, + size: size as u64, + truncated: false, + elapsed_ms: 1, + }); + send(ProxyEvent::ResponseStart { + id, + status: 200, + http_version: "HTTP/1.1".into(), + remote_addr: None, + elapsed_ms: 2, + }); + send(ProxyEvent::ResponseBodyComplete { + id, + size: 0, + truncated: false, + elapsed_ms: 3, + error: None, + cancelled: false, + }); + }; + post( + 1, + "https://api.example.com/graphql", + &[ + br#"{"query":"query GetUser($id: ID!) "#, + br#"{ user(id: $id) { id } }"}"#, + ], + ); + post(2, "https://api.example.com/users", &[br#"{"query":"red shoes"}"#]); + let long = format!( + r#"{{"query":"query Big {{ a }}","variables":{{"pad":"{}"}}}}"#, + "x".repeat(GRAPHQL_SCAN_LIMIT) + ); + post(3, "https://api.example.com/big/graphql", &[long.as_bytes()]); + send_flow(&tx, 4, "https://api.example.com/graphql?query=query%20Feed%20%7B%20a%20%7D"); + drop(tx); + event_loop.join().unwrap(); + + let flows = flows::list(&ctx.db().unwrap(), &Default::default()).unwrap(); + assert_eq!(flows.len(), 4); + let operation = |url: &str| { + let flow = flows.iter().find(|f| f.url == url).unwrap(); + flow.graphql_operation.clone() + }; + assert_eq!(operation("https://api.example.com/graphql").as_deref(), Some("GetUser")); + assert_eq!(operation("https://api.example.com/users"), None); + assert_eq!(operation("https://api.example.com/big/graphql"), None); + assert_eq!( + operation("https://api.example.com/graphql?query=query%20Feed%20%7B%20a%20%7D") + .as_deref(), + Some("Feed") + ); + } + /// Flows that end together are written together, and every one the engine sent is written /// when it stops, mid-batch or not, each with its body. #[test] @@ -4545,6 +4693,7 @@ mod tests { next_seq: 0, dirty: false, last_write: None, + graphql_scan: None, }; sink.finish_flow(f); assert!(!sink.finished_due(), "the clock stands still"); diff --git a/crates-proxy/yaak-proxy-lib/src/models.rs b/crates-proxy/yaak-proxy-lib/src/models.rs index 2d922fd07..8c03f4aa9 100644 --- a/crates-proxy/yaak-proxy-lib/src/models.rs +++ b/crates-proxy/yaak-proxy-lib/src/models.rs @@ -103,6 +103,9 @@ pub struct Flow { pub grpc_status: Option, /// A gRPC call's `grpc-message`, decoded. pub grpc_message: Option, + /// A GraphQL request's operation, from its body or URL: `GetUser`, `GetUser +2` for a + /// batch, empty for an anonymous one. None when the request is not GraphQL. + pub graphql_operation: Option, /// An HTTPS connection passed through undecrypted, as the decrypt list leaves it out or its /// client rejected the proxy's certificate: method `CONNECT`, its URL the host, its sizes /// the bytes each way, and no headers or bodies. Open until `elapsedMs` or `error` is set. @@ -273,6 +276,7 @@ impl UpsertModelInfo for Flow { (GrpcMethod, self.grpc_method.into()), (GrpcStatus, self.grpc_status.into()), (GrpcMessage, self.grpc_message.into()), + (GraphqlOperation, self.graphql_operation.into()), (Tunnelled, self.tunnelled.into()), (ElapsedConnectMs, self.elapsed_connect_ms.into()), (ElapsedTlsMs, self.elapsed_tls_ms.into()), @@ -316,6 +320,7 @@ impl UpsertModelInfo for Flow { FlowIden::GrpcMethod, FlowIden::GrpcStatus, FlowIden::GrpcMessage, + FlowIden::GraphqlOperation, FlowIden::Tunnelled, FlowIden::ElapsedConnectMs, FlowIden::ElapsedTlsMs, @@ -374,6 +379,7 @@ impl UpsertModelInfo for Flow { grpc_method: added_column(r, "grpc_method")?, grpc_status: added_column(r, "grpc_status")?, grpc_message: added_column(r, "grpc_message")?, + graphql_operation: added_column(r, "graphql_operation")?, tunnelled: added_column(r, "tunnelled")?.unwrap_or_default(), agent_read: added_column(r, "agent_read")?.unwrap_or_default(), }) diff --git a/crates-proxy/yaak-proxy-lib/src/query.rs b/crates-proxy/yaak-proxy-lib/src/query.rs index a91185092..6359c3d0d 100644 --- a/crates-proxy/yaak-proxy-lib/src/query.rs +++ b/crates-proxy/yaak-proxy-lib/src/query.rs @@ -37,6 +37,7 @@ pub const FLAGS: &[&str] = &[ "ws", "sse", "grpc", + "graphql", ]; /// Flags `@is:` will take once flows have what they need. Valid, matching nothing. @@ -100,6 +101,14 @@ pub fn schema() -> &'static Schema { field("id", number(), "Flow number", "@id:>1200"), field("cwd", text(), "Working folder", "@cwd:myapp"), field("pid", number(), "Process id", "@pid:4121"), + field( + "graphql.operation", + text(), + "GraphQL operation name", + "@graphql.operation:GetUser", + ), + field("grpc.service", text(), "gRPC service", "@grpc.service:pkg.Greeter"), + field("grpc.method", text(), "gRPC method", "@grpc.method:SayHello"), field("body", text(), "Not yet", ""), field("req.body", text(), "Not yet", ""), field("res.body", text(), "Not yet", ""), @@ -471,9 +480,10 @@ const SEEN_VALUES: i64 = 200; const HEADER_VALUES: i64 = 20; /// The values seen for a field path, for completing it, most recent first: hosts, methods, -/// statuses, content types, protocols, schemes, apps, working folders, `@is` flags, and header -/// names, or a header's values when it has had few. Nothing for fields whose values are -/// near-unique (URLs, paths, ids, sizes, times, query parameters). +/// statuses, content types, protocols, schemes, apps, working folders, GraphQL operations, gRPC +/// services and methods, `@is` flags, header names, or a header's values when it has had few, +/// and query parameter names. Nothing for fields whose values are near-unique (URLs, paths, ids, +/// sizes, times, query parameter values). pub fn seen_values(db: &crate::db::ProxyQueryManager, path: &str) -> Result, String> { let path = path.to_lowercase(); let column = match path.as_str() { @@ -486,6 +496,10 @@ pub fn seen_values(db: &crate::db::ProxyQueryManager, path: &str) -> Result "url_scheme(url)", "app" => "client_name", "cwd" => "folder_name(client_cwd)", + "graphql.operation" => "graphql_operation", + "grpc.service" => "grpc_service", + "grpc.method" => "grpc_method", + "query" => return seen_query_names(db), "header" | "req.header" | "res.header" => { return seen_headers(db, &path, None); } @@ -524,6 +538,17 @@ fn newest_flows() -> String { ) } +/// The names of the query parameters seen, for completing `@query.`. +fn seen_query_names(db: &crate::db::ProxyQueryManager) -> Result, String> { + let sql = format!( + "SELECT q_lower(json_extract(kv.value, '$.name')) AS name FROM ({}) AS f, \ + json_each(url_query_json(f.url)) AS kv WHERE name IS NOT NULL AND name != '' \ + GROUP BY name ORDER BY max(f.rid) DESC LIMIT {SEEN_VALUES}", + newest_flows() + ); + select_strings(db, &sql, None) +} + /// Header names under a map, or one header's values when it has had few. fn seen_headers( db: &crate::db::ProxyQueryManager, @@ -723,6 +748,16 @@ pub fn url_sidebar_path(url: &str) -> Option { (normalized != path && !path.is_empty()).then_some(normalized) } +/// The authority as written, user info and port included: the list's Host column. +pub fn url_authority(url: &str) -> &str { + url_parts(url).authority +} + +/// The query string as written, without its `?`: the list's Query column. +pub fn url_query_string(url: &str) -> Option<&str> { + url_parts(url).query +} + /// The query's parameters, decoded, their names lowercased. pub fn url_query(url: &str) -> Vec<(String, String)> { let Some(query) = url_parts(url).query else { @@ -784,6 +819,7 @@ fn flags(flow: &Flow) -> Vec<&'static str> { "ws" => stream == Some("websocket"), "sse" => stream == Some("sse"), "grpc" => matches!(stream, Some("grpc" | "grpc_web")), + "graphql" => flow.graphql_operation.is_some(), _ => false, }; FLAGS.iter().copied().filter(|f| set(f)).collect() @@ -862,6 +898,16 @@ pub fn document(flow: &Flow) -> Document { if let Some(pid) = flow.client_pid { add("pid", num(f64::from(pid))); } + // An anonymous operation is GraphQL with no name: `@is:graphql`, not `@graphql.operation:*`. + if let Some(op) = flow.graphql_operation.as_deref().filter(|op| !op.is_empty()) { + add("graphql.operation", text(op)); + } + if let Some(service) = &flow.grpc_service { + add("grpc.service", text(service)); + } + if let Some(method) = &flow.grpc_method { + add("grpc.method", text(method)); + } for (side, headers) in [ ("req", &flow.request_headers), ("res", &flow.response_headers), @@ -919,7 +965,7 @@ impl FlowColumns { } /// `name`, or what the model reads in its place when the table lacks it. - fn col(&self, name: &str) -> String { + pub(crate) fn col(&self, name: &str) -> String { if self.0.contains(name) { return name.to_string(); } @@ -1005,6 +1051,9 @@ impl Sql<'_> { "req.size" => (vec!["request_size".into()], true), "duration" => (vec!["elapsed_ms".into()], true), "pid" => (vec![c("client_pid")], true), + "graphql.operation" => (vec![format!("nullif({}, '')", c("graphql_operation"))], false), + "grpc.service" => (vec![c("grpc_service")], false), + "grpc.method" => (vec![c("grpc_method")], false), "app" => { let (bundle, path, name) = (c("client_bundle_id"), c("client_path"), c("client_name")); @@ -1103,6 +1152,7 @@ impl Sql<'_> { "ws" => format!("{} = 'websocket'", c("stream_kind")), "sse" => format!("{} = 'sse'", c("stream_kind")), "grpc" => format!("{} IN ('grpc', 'grpc_web')", c("stream_kind")), + "graphql" => format!("{} IS NOT NULL", c("graphql_operation")), _ => "0".into(), } } @@ -1214,6 +1264,12 @@ pub(crate) fn add_functions(conn: &rusqlite::Connection) -> rusqlite::Result<()> conn.create_scalar_function("url_path", 1, flags, move |ctx| { Ok(url_path(&text_arg(ctx, 0)?.unwrap_or_default()).map(str::to_string)) })?; + conn.create_scalar_function("url_authority", 1, flags, move |ctx| { + Ok(url_authority(&text_arg(ctx, 0)?.unwrap_or_default()).to_string()) + })?; + conn.create_scalar_function("url_query_string", 1, flags, move |ctx| { + Ok(url_query_string(&text_arg(ctx, 0)?.unwrap_or_default()).map(str::to_string)) + })?; conn.create_scalar_function("url_sidebar_path", 1, flags, move |ctx| { Ok(url_sidebar_path(&text_arg(ctx, 0)?.unwrap_or_default())) })?; diff --git a/crates-proxy/yaak-proxy-lib/src/query/tests.rs b/crates-proxy/yaak-proxy-lib/src/query/tests.rs index 499ecf52a..6e88d6329 100644 --- a/crates-proxy/yaak-proxy-lib/src/query/tests.rs +++ b/crates-proxy/yaak-proxy-lib/src/query/tests.rs @@ -690,6 +690,13 @@ fn sample(count: usize) -> Vec { client_pid: rng.chance(50).then(|| (rng.next() % 1000) as i32), stream_kind: *rng.pick(&streams), grpc_status: *rng.pick(&[None, Some(0), Some(5)]), + grpc_service: rng + .chance(20) + .then(|| rng.pick(&["pkg.Greeter", "Users"]).to_string()), + grpc_method: rng.chance(20).then(|| rng.pick(&["SayHello", "getUser"]).to_string()), + graphql_operation: rng + .chance(25) + .then(|| rng.pick(&["GetUser", "", "Feed +2", "createÉtat"]).to_string()), error: rng.chance(15).then(|| "connection reset".to_string()), cancelled: rng.chance(10), saved_request_id: rng.chance(15).then(|| "rq_x".to_string()), @@ -859,6 +866,18 @@ const BATTERY: &[&str] = &[ "@is:ws", "@is:sse", "@is:grpc", + "@is:graphql", + "-@is:graphql", + "@graphql.operation:getuser", + "@graphql.operation:feed*", + "@graphql.operation:*", + "-@graphql.operation:*", + "@graphql.operation:createétat", + "@grpc.service:pkg.greeter", + "@grpc.service:*", + "@grpc.method:SayHello", + "-@grpc.method:*", + "@grpc.service:users @grpc.method:get*", "@is:*", "-@is:*", "@is:s*", @@ -961,11 +980,58 @@ fn seen_values_for_completion() { "size", "duration", "time", - "query", "query.page", ] { assert!(values(path).is_empty(), "{path}"); } + // A query parameter's names, for `@query.`, but not its values + let mut params = values("query"); + params.sort(); + assert_eq!(params, vec!["page", "sort"]); +} + +/// GraphQL operations and gRPC calls, by name and by flag, and offered for completion. +#[test] +fn graphql_and_grpc_fields() { + let base = |id: &str, url: &str| Flow { + id: id.into(), + url: url.into(), + method: "POST".into(), + started_at: now().naive_utc(), + ..Default::default() + }; + let flows = vec![ + Flow { + graphql_operation: Some("GetUser".into()), + ..base("fl_named", "https://api.example.com/graphql") + }, + Flow { + graphql_operation: Some(String::new()), + ..base("fl_anonymous", "https://api.example.com/graphql") + }, + Flow { + grpc_service: Some("pkg.Greeter".into()), + grpc_method: Some("SayHello".into()), + ..base("fl_call", "https://grpc.example.com/pkg.Greeter/SayHello") + }, + base("fl_plain", "https://example.com/"), + ]; + let (_dir, db, all) = db_of(flows); + let finds = |query: &str| { + let prepared = FlowQuery::at(query, false, now()); + let sql = sql_ids(&db, &prepared); + assert_eq!(sql, memory_ids(&all, &prepared), "the two disagree on {query:?}"); + sql + }; + assert_eq!(finds("@is:graphql"), ids(&["fl_anonymous", "fl_named"])); + assert_eq!(finds("@graphql.operation:getuser"), ids(&["fl_named"])); + assert_eq!(finds("@graphql.operation:*"), ids(&["fl_named"])); + assert_eq!(finds("@grpc.service:pkg.Greeter"), ids(&["fl_call"])); + assert_eq!(finds("@grpc.method:say*"), ids(&["fl_call"])); + assert_eq!(finds("-@grpc.method:*"), ids(&["fl_anonymous", "fl_named", "fl_plain"])); + assert_eq!(seen_values(&db, "graphql.operation").unwrap(), vec!["GetUser"]); + assert_eq!(seen_values(&db, "grpc.service").unwrap(), vec!["pkg.Greeter"]); + assert_eq!(seen_values(&db, "grpc.method").unwrap(), vec!["SayHello"]); } #[test] diff --git a/crates-tauri/yaak-app-proxy/src/lib.rs b/crates-tauri/yaak-app-proxy/src/lib.rs index dc5360ce1..fcebddd1e 100644 --- a/crates-tauri/yaak-app-proxy/src/lib.rs +++ b/crates-tauri/yaak-app-proxy/src/lib.rs @@ -36,6 +36,11 @@ fn setup_window_menu(win: &WebviewWindow) { let _ = w.close(); }); } + window_menu::EXPORT_HAR_ID => { + if let Err(e) = webview_window.emit(window_menu::EXPORT_HAR_EVENT, ()) { + error!("Failed to emit the HAR export: {e}"); + } + } "dev.refresh" => webview_window.eval("location.reload()").unwrap(), "dev.toggle_devtools" => { if webview_window.is_devtools_open() { diff --git a/crates-tauri/yaak-app-proxy/src/window_menu.rs b/crates-tauri/yaak-app-proxy/src/window_menu.rs index b71433af0..c09faed14 100644 --- a/crates-tauri/yaak-app-proxy/src/window_menu.rs +++ b/crates-tauri/yaak-app-proxy/src/window_menu.rs @@ -5,6 +5,12 @@ use tauri::menu::{ WINDOW_SUBMENU_ID, }; +/// File's Export All Flows as HAR, which the window carries out, as the list's menus do. +pub const EXPORT_HAR_ID: &str = "file.export_har"; + +/// The event the window hears File's Export All Flows as HAR by. +pub const EXPORT_HAR_EVENT: &str = "menu_export_har"; + pub fn app_menu(app_handle: &AppHandle) -> tauri::Result> { let pkg_info = app_handle.package_info(); let config = app_handle.config(); @@ -87,6 +93,9 @@ pub fn app_menu(app_handle: &AppHandle) -> tauri::Result> "File", true, &[ + &MenuItemBuilder::with_id(EXPORT_HAR_ID, "Export All Flows as HAR") + .build(app_handle)?, + &PredefinedMenuItem::separator(app_handle)?, &PredefinedMenuItem::close_window(app_handle, None)?, #[cfg(not(target_os = "macos"))] &PredefinedMenuItem::quit(app_handle, None)?, diff --git a/crates/yaak-http/src/graphql.rs b/crates/yaak-http/src/graphql.rs new file mode 100644 index 000000000..e764b15cc --- /dev/null +++ b/crates/yaak-http/src/graphql.rs @@ -0,0 +1,612 @@ +//! GraphQL request detection, ported from the curl importer +//! (`plugins/importer-curl/src/graphql.ts`) so the proxy and the importer agree. + +use serde_json::{Map, Value}; +use url::Url; + +/// A GraphQL request's JSON envelope. +#[derive(Debug, Clone, PartialEq)] +pub struct GraphQlBody { + pub query: String, + /// The `variables` value as sent (object, string, or other), None when absent or null. + pub variables: Option, + pub operation_name: Option, +} + +impl GraphQlBody { + /// `operationName` when present and non-blank, else the first named operation in `query`. + pub fn operation(&self) -> Option { + match self.operation_name.as_deref().map(str::trim) { + Some(name) if !name.is_empty() => Some(name.to_string()), + _ => first_operation_name(&self.query), + } + } +} + +/// A detected GraphQL request, as the proxy labels it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GraphQlOperation { + /// The operation's name; for a batch, the first named operation in it. None when anonymous. + pub name: Option, + /// How many operations the request carries: 1, or the batch's length. + pub count: usize, +} + +impl GraphQlOperation { + /// The name, with ` +N` for the rest of a batch (`GetUser +2`); empty when anonymous. + pub fn label(&self) -> String { + match &self.name { + None => String::new(), + Some(name) if self.count > 1 => format!("{name} +{}", self.count - 1), + Some(name) => name.clone(), + } + } +} + +/// A JSON GraphQL envelope that passes the importer's scored check. +pub fn parse_json_body(content_type: Option<&str>, text: &str, url: &str) -> Option { + if !is_json(content_type) { + return None; + } + match serde_json::from_str::(text).ok()? { + Value::Object(body) => check_envelope(&body, &url_path(url)), + _ => None, + } +} + +/// A non-empty JSON array whose every element passes `parse_json_body`'s check. +pub fn parse_json_batch( + content_type: Option<&str>, + text: &str, + url: &str, +) -> Option> { + if !is_json(content_type) { + return None; + } + let Value::Array(items) = serde_json::from_str::(text).ok()? else { + return None; + }; + if items.is_empty() { + return None; + } + let path = url_path(url); + items + .iter() + .map(|item| match item { + Value::Object(body) => check_envelope(body, &path), + _ => None, + }) + .collect() +} + +/// GraphQL over GET: the URL's `query`, `operationName` and `variables` parameters, scored like a +/// body. +pub fn parse_get(url: &str) -> Option { + let parsed = Url::parse(url).ok()?; + let mut query = None; + let mut operation_name = None; + let mut variables = None; + for (key, value) in parsed.query_pairs() { + match key.as_ref() { + "query" if query.is_none() => query = Some(value.into_owned()), + "operationName" if operation_name.is_none() => { + operation_name = Some(value.into_owned()) + } + "variables" if variables.is_none() && !value.is_empty() => { + variables = Some(Value::String(value.into_owned())) + } + _ => {} + } + } + let body = GraphQlBody { query: query?, variables, operation_name }; + passes(&body, parsed.path()).then_some(body) +} + +/// The proxy's check: GET reads the URL, anything else the body as one envelope or a batch. +pub fn detect( + method: &str, + url: &str, + content_type: Option<&str>, + body: Option<&[u8]>, +) -> Option { + if method.eq_ignore_ascii_case("GET") { + let body = parse_get(url)?; + return Some(GraphQlOperation { name: body.operation(), count: 1 }); + } + let text = std::str::from_utf8(body?).ok()?; + if let Some(body) = parse_json_body(content_type, text, url) { + return Some(GraphQlOperation { name: body.operation(), count: 1 }); + } + let batch = parse_json_batch(content_type, text, url)?; + Some(GraphQlOperation { + name: batch.iter().find_map(GraphQlBody::operation), + count: batch.len(), + }) +} + +fn is_json(content_type: Option<&str>) -> bool { + content_type + .and_then(|ct| ct.split(';').next()) + .is_some_and(|essence| essence.trim().eq_ignore_ascii_case("application/json")) +} + +fn url_path(url: &str) -> String { + match Url::parse(url) { + Ok(parsed) => parsed.path().to_string(), + Err(_) => url.to_string(), + } +} + +fn check_envelope(body: &Map, path: &str) -> Option { + let query = body.get("query")?.as_str()?; + if body.keys().any(|k| !matches!(k.as_str(), "query" | "variables" | "operationName")) { + return None; + } + let parsed = GraphQlBody { + query: query.to_string(), + variables: body.get("variables").filter(|v| !v.is_null()).cloned(), + operation_name: body.get("operationName").and_then(Value::as_str).map(str::to_string), + }; + passes(&parsed, path).then_some(parsed) +} + +/// The importer's scored signals: a document signal and a score of at least 4. +fn passes(body: &GraphQlBody, path: &str) -> bool { + let mut score = 0; + let mut has_document = false; + let query = body.query.as_str(); + // From the document's first token: comments and whitespace before it don't count. + let trimmed = skip_leading_ignored(query).trim_end(); + + if has_graphql_path_word(&path.to_lowercase()) { + score += 2; + } + + if ["query", "mutation", "subscription", "fragment"] + .iter() + .any(|keyword| starts_with_word(trimmed, keyword)) + { + score += 3; + } else if trimmed.len() >= 2 && trimmed.starts_with('{') && trimmed.ends_with('}') { + score += 3; + has_document = true; + } + + if let Some(open) = query.find('{') + && query[open..].contains('}') + { + score += 1; + has_document = true; + } + + if body.operation_name.as_deref().is_some_and(|name| !name.trim().is_empty()) { + score += 1; + } + + if matches!(body.variables, Some(Value::Object(_) | Value::Array(_) | Value::String(_))) { + score += 1; + } + + has_document && score >= 4 +} + +/// `document` from its first token: GraphQL's ignored tokens before it (whitespace, commas, a +/// byte order mark and `#` comments to the end of their line) left out. +fn skip_leading_ignored(document: &str) -> &str { + let mut rest = document; + loop { + let trimmed = + rest.trim_start_matches(|c: char| c.is_whitespace() || c == ',' || c == '\u{feff}'); + match trimmed.strip_prefix('#') { + Some(comment) => rest = comment.find(['\n', '\r']).map_or("", |end| &comment[end..]), + None => return trimmed, + } + } +} + +fn is_word_byte(b: u8) -> bool { + b.is_ascii_alphanumeric() || b == b'_' +} + +/// `^keyword\b` +fn starts_with_word(text: &str, keyword: &str) -> bool { + text.starts_with(keyword) + && !text.as_bytes().get(keyword.len()).copied().is_some_and(is_word_byte) +} + +/// `\b(graphql|gql)\b` +fn has_graphql_path_word(path: &str) -> bool { + let bytes = path.as_bytes(); + ["graphql", "gql"].iter().any(|word| { + path.match_indices(word).any(|(start, _)| { + let end = start + word.len(); + let before = start > 0 && is_word_byte(bytes[start - 1]); + let after = bytes.get(end).copied().is_some_and(is_word_byte); + !before && !after + }) + }) +} + +/// The first named `query`, `mutation` or `subscription` at the top level of a document, by a +/// token scan that skips comments and strings. Fragments are not operations. +pub fn first_operation_name(document: &str) -> Option { + let bytes = document.as_bytes(); + let mut i = 0; + let mut braces = 0usize; + let mut parens = 0usize; + let mut definition_start = true; + // Set after an operation keyword at a definition's start: the next token may be its name. + let mut after_keyword = false; + + while i < bytes.len() { + let b = bytes[i]; + match b { + b'#' => { + while i < bytes.len() && bytes[i] != b'\n' && bytes[i] != b'\r' { + i += 1; + } + continue; + } + b'"' if bytes[i..].starts_with(b"\"\"\"") => { + after_keyword = false; + i += 3; + while i < bytes.len() && !bytes[i..].starts_with(b"\"\"\"") { + i += if bytes[i..].starts_with(b"\\\"\"\"") { 4 } else { 1 }; + } + i += 3; + continue; + } + b'"' => { + after_keyword = false; + i += 1; + while i < bytes.len() && !matches!(bytes[i], b'"' | b'\n' | b'\r') { + i += if bytes[i] == b'\\' { 2 } else { 1 }; + } + i += 1; + continue; + } + b'_' | b'A'..=b'Z' | b'a'..=b'z' => { + let start = i; + while i < bytes.len() && is_word_byte(bytes[i]) { + i += 1; + } + let name = &document[start..i]; + if after_keyword { + return Some(name.to_string()); + } + if definition_start && braces == 0 && parens == 0 { + after_keyword = matches!(name, "query" | "mutation" | "subscription"); + definition_start = false; + } + continue; + } + b' ' | b'\t' | b'\n' | b'\r' | b',' => {} + _ => { + after_keyword = false; + match b { + b'{' => { + braces += 1; + definition_start = false; + } + b'}' => { + braces = braces.saturating_sub(1); + if braces == 0 && parens == 0 { + definition_start = true; + } + } + b'(' => parens += 1, + b')' => parens = parens.saturating_sub(1), + _ => {} + } + } + } + i += 1; + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + const JSON: Option<&str> = Some("application/json"); + + fn is_body(content_type: Option<&str>, body: Value, url: &str) -> bool { + parse_json_body(content_type, &body.to_string(), url).is_some() + } + + // Carried over from plugins/importer-curl/tests/graphql.test.ts. + + #[test] + fn detects_named_query_documents_without_a_graphql_url() { + let text = json!({ + "query": "query Search($id: ID!) { node(id: $id) { id } }", + "variables": { "id": "123" }, + "operationName": "Search", + }) + .to_string(); + assert_eq!( + parse_json_body(JSON, &text, "https://api.example.com/search"), + Some(GraphQlBody { + query: "query Search($id: ID!) { node(id: $id) { id } }".into(), + variables: Some(json!({ "id": "123" })), + operation_name: Some("Search".into()), + }) + ); + } + + #[test] + fn detects_mutation_documents() { + let body = json!({ "query": "mutation Save { saveThing { id } }" }); + assert!(is_body(JSON, body, "https://api.example.com")); + } + + #[test] + fn detects_anonymous_selection_set_documents() { + let body = json!({ "query": "{ viewer { id email } }" }); + assert!(is_body(JSON, body, "https://api.example.com")); + } + + #[test] + fn detects_document_bodies_on_graphql_looking_paths() { + let body = json!({ "query": "query Search { viewer { id } }", "operationName": "Search" }); + assert!(is_body(JSON, body, "https://api.example.com/v1/graphql")); + } + + #[test] + fn rejects_incomplete_operation_documents_on_graphql_paths() { + let body = json!({ "query": "query Search", "operationName": "Search" }); + assert!(!is_body(JSON, body, "https://api.example.com/graphql")); + } + + #[test] + fn rejects_plain_json_query_fields_on_graphql_paths() { + let body = json!({ "query": "SearchQueryInput!" }); + assert!(!is_body(JSON, body, "https://api.example.com/graphql")); + } + + #[test] + fn rejects_variables_and_operation_name_alone() { + let body = json!({ + "query": "SearchQueryInput!", + "variables": { "id": "123" }, + "operationName": "Search", + }); + assert!(!is_body(JSON, body, "https://api.example.com")); + } + + #[test] + fn detects_bodies_with_string_variables_without_parsing_them() { + let text = json!({ + "query": "query Search($id: ID!) { node(id: $id) { id } }", + "variables": "{ \"id\": \"123\" }", + }) + .to_string(); + assert_eq!( + parse_json_body(JSON, &text, "https://api.example.com"), + Some(GraphQlBody { + query: "query Search($id: ID!) { node(id: $id) { id } }".into(), + variables: Some(json!("{ \"id\": \"123\" }")), + operation_name: None, + }) + ); + } + + #[test] + fn rejects_envelopes_with_extra_fields() { + let body = json!({ + "query": "query Search($id: ID!) { node(id: $id) { id } }", + "variables": { "id": "123" }, + "extensions": { "persistedQuery": { "version": 1, "sha256Hash": "abc123" } }, + }); + assert!(!is_body(JSON, body, "https://api.example.com/graphql")); + } + + #[test] + fn ignores_invalid_json_and_non_object_json() { + assert_eq!(parse_json_body(JSON, "not json", "https://api.example.com/graphql"), None); + assert_eq!(parse_json_body(JSON, "[]", "https://api.example.com/graphql"), None); + } + + #[test] + fn ignores_non_json_mime_types() { + let body = json!({ "query": "query Search { viewer { id } }" }); + assert!(!is_body(Some("text/plain"), body, "https://api.example.com/graphql")); + } + + // Additions for the proxy. + + #[test] + fn content_type_essence_ignores_parameters_and_case() { + let body = json!({ "query": "query Search { viewer { id } }" }); + let ct = Some(" Application/JSON ; charset=utf-8"); + assert!(is_body(ct, body.clone(), "https://api.example.com")); + assert!(!is_body(None, body, "https://api.example.com")); + } + + #[test] + fn first_operation_name_reads_named_operations() { + assert_eq!(first_operation_name("query GetUser { user { id } }"), Some("GetUser".into())); + assert_eq!( + first_operation_name("mutation SaveUser($u: In!) { save(u: $u) { id } }"), + Some("SaveUser".into()) + ); + assert_eq!( + first_operation_name("subscription OnMessage { message { body } }"), + Some("OnMessage".into()) + ); + } + + #[test] + fn first_operation_name_skips_anonymous_operations() { + assert_eq!(first_operation_name("{ viewer { id } }"), None); + assert_eq!(first_operation_name("query { viewer { id } }"), None); + assert_eq!(first_operation_name("query($id: ID) { node(id: $id) { id } }"), None); + assert_eq!( + first_operation_name("{ viewer { id } } query Named { viewer { id } }"), + Some("Named".into()) + ); + } + + #[test] + fn first_operation_name_handles_variables_and_directives() { + let doc = r#"query GetUser($id: ID! = "x", $f: In = {a: 1}) @cached(ttl: 60) { + user(id: $id) @include(if: true) { name } + }"#; + assert_eq!(first_operation_name(doc), Some("GetUser".into())); + } + + #[test] + fn first_operation_name_ignores_keywords_in_strings_and_comments() { + let doc = r#" + # query Commented { x } + { search(text: "query Quoted { x }") { id } } + query Real { id } + "#; + assert_eq!(first_operation_name(doc), Some("Real".into())); + } + + /// Comments and whitespace before a document's first token don't hide its keyword or its + /// selection set, from detection or the name scan. + #[test] + fn detects_documents_after_leading_comments() { + let commented = "# viewer query\nquery GetViewer { viewer { id } }"; + let body = json!({ "query": commented }); + assert!(is_body(JSON, body.clone(), "https://api.example.com/graphql")); + let op = detect( + "POST", + "https://api.example.com/graphql", + JSON, + Some(body.to_string().as_bytes()), + ); + assert_eq!(op.map(|o| o.label()).as_deref(), Some("GetViewer")); + + for document in [ + "\n\n # one\r\n# two\n mutation Save { save { id } }", + "#only a comment line\nsubscription OnEvent { event { id } }", + "\u{feff} , # list\n{ viewer { id } }", + ] { + let body = json!({ "query": document }); + assert!(is_body(JSON, body, "https://api.example.com/graphql"), "{document:?}"); + } + // A selection set alone after a comment needs no GraphQL path. + let bare = json!({ "query": "# who\n{ viewer { id } }", "variables": {} }); + assert!(is_body(JSON, bare, "https://api.example.com/v1")); + + assert_eq!( + first_operation_name("# c\n # query Hidden { x }\n query Shown { a }"), + Some("Shown".into()) + ); + // A comment and nothing else is no document. + assert!(!is_body( + JSON, + json!({ "query": "# query X { a }" }), + "https://api.example.com/graphql" + )); + } + + #[test] + fn first_operation_name_skips_fragments() { + let doc = + "fragment UserFields on User { id name } query GetUser { user { ...UserFields } }"; + assert_eq!(first_operation_name(doc), Some("GetUser".into())); + assert_eq!(first_operation_name("fragment query on User { id }"), None); + } + + #[test] + fn first_operation_name_skips_block_strings() { + let doc = r#"{ a(text: """ query Hidden { "x" } \""" still """) } mutation Shown { b }"#; + assert_eq!(first_operation_name(doc), Some("Shown".into())); + } + + #[test] + fn operation_prefers_operation_name() { + let body = GraphQlBody { + query: "query FromDoc { x }".into(), + variables: None, + operation_name: Some("FromParam".into()), + }; + assert_eq!(body.operation(), Some("FromParam".into())); + let body = GraphQlBody { operation_name: Some(" ".into()), ..body }; + assert_eq!(body.operation(), Some("FromDoc".into())); + } + + #[test] + fn parse_get_reads_operation_name_param() { + let url = "https://api.example.com/graphql?query=%7B%20viewer%20%7B%20id%20%7D%20%7D\ + &operationName=Viewer&variables=%7B%22a%22%3A1%7D"; + let body = parse_get(url).unwrap(); + assert_eq!(body.query, "{ viewer { id } }"); + assert_eq!(body.variables, Some(json!("{\"a\":1}"))); + assert_eq!(body.operation(), Some("Viewer".into())); + } + + #[test] + fn parse_get_reads_name_from_document() { + let url = "https://api.example.com/api?query=query+GetUser+%7B+user+%7B+id+%7D+%7D"; + let op = detect("get", url, None, None).unwrap(); + assert_eq!(op, GraphQlOperation { name: Some("GetUser".into()), count: 1 }); + } + + #[test] + fn parse_get_rejects_plain_search_queries() { + assert_eq!(parse_get("https://shop.example.com/search?query=red+shoes"), None); + assert_eq!(parse_get("https://shop.example.com/graphql?q=shoes"), None); + assert_eq!( + detect("GET", "https://shop.example.com/search?query=red+shoes", None, None), + None + ); + } + + #[test] + fn detect_labels_batches() { + let text = json!([ + { "query": "query A { a }" }, + { "query": "query B { b }" }, + { "query": "query C { c }" }, + ]) + .to_string(); + let url = "https://api.example.com/graphql"; + assert_eq!(parse_json_batch(JSON, &text, url).map(|b| b.len()), Some(3)); + let op = detect("POST", url, JSON, Some(text.as_bytes())).unwrap(); + assert_eq!(op, GraphQlOperation { name: Some("A".into()), count: 3 }); + assert_eq!(op.label(), "A +2"); + } + + #[test] + fn detect_labels_batches_by_first_named_operation() { + let text = json!([{ "query": "{ a }" }, { "query": "query B { b }" }]).to_string(); + let url = "https://api.example.com/graphql"; + let op = detect("POST", url, JSON, Some(text.as_bytes())).unwrap(); + assert_eq!(op.label(), "B +1"); + } + + #[test] + fn batch_rejects_empty_or_mixed_arrays() { + let url = "https://api.example.com/graphql"; + assert_eq!(parse_json_batch(JSON, "[]", url), None); + let text = json!([{ "query": "query A { a }" }, { "foo": 1 }]).to_string(); + assert_eq!(parse_json_batch(JSON, &text, url), None); + } + + #[test] + fn detect_ignores_non_json_content_types_and_bad_bodies() { + let url = "https://api.example.com/graphql"; + let text = json!({ "query": "query A { a }" }).to_string(); + assert_eq!(detect("POST", url, Some("text/plain"), Some(text.as_bytes())), None); + assert_eq!(detect("POST", url, JSON, None), None); + assert_eq!(detect("POST", url, JSON, Some(&[0xff, 0xfe])), None); + } + + #[test] + fn label_is_empty_for_anonymous_operations() { + let url = "https://api.example.com/graphql"; + let text = json!({ "query": "{ viewer { id } }" }).to_string(); + let op = detect("POST", url, JSON, Some(text.as_bytes())).unwrap(); + assert_eq!(op, GraphQlOperation { name: None, count: 1 }); + assert_eq!(op.label(), ""); + assert_eq!(GraphQlOperation { name: None, count: 3 }.label(), ""); + assert_eq!(GraphQlOperation { name: Some("One".into()), count: 1 }.label(), "One"); + } +} diff --git a/crates/yaak-http/src/lib.rs b/crates/yaak-http/src/lib.rs index 2a4155c9e..dfbf03d9d 100644 --- a/crates/yaak-http/src/lib.rs +++ b/crates/yaak-http/src/lib.rs @@ -4,6 +4,7 @@ pub mod cookies; pub mod decompress; pub mod dns; pub mod error; +pub mod graphql; pub mod manager; mod proto; pub mod sender;