diff --git a/.gitignore b/.gitignore index 1beb88f89..f4d96a8bc 100644 --- a/.gitignore +++ b/.gitignore @@ -59,3 +59,6 @@ flatpak/node-sources.json # Claude Code local settings .claude/settings.local.json .claude/worktrees/ + +# Headless screenshots taken while working +.shots/ diff --git a/apps/yaak-proxy/components/DecryptListEditor.tsx b/apps/yaak-proxy/components/DecryptListEditor.tsx index 74f956d0a..fa788ee95 100644 --- a/apps/yaak-proxy/components/DecryptListEditor.tsx +++ b/apps/yaak-proxy/components/DecryptListEditor.tsx @@ -1,10 +1,9 @@ import { Prec } from "@codemirror/state"; import { type EditorView, keymap } from "@codemirror/view"; import type { DecryptListEntry, SeenTargets } from "@yaakapp-internal/proxy-lib"; -import { Dialog, Icon, IconButton, Input } from "@yaakapp-internal/ui"; +import { Icon, IconButton, Input } from "@yaakapp-internal/ui"; import { filterAutocomplete } from "@yaakapp-internal/ui/editor"; import classNames from "classnames"; -import { atom, useAtom } from "jotai"; import { useMemo, useRef, useState } from "react"; import { useDecryptList, useEditDecryptList } from "../hooks/useDecryptList"; import { useRpcQuery } from "../hooks/useRpcQuery"; @@ -13,8 +12,8 @@ import { decryptFilterOptions, entryLabel, normalizeEntry } from "../lib/decrypt /** * What HTTPS is decrypted: one row per entry, the user's in order and then the built-in * exclusions, each removable, and an input that adds one, suggesting `host:`, `app:` and `!`, - * then the hosts and apps seen in the capture. Saves as it changes. Placed by Settings; until - * that exists, by the dialog the state pill's More… opens. + * then the hosts and apps seen in the capture. Saves as it changes. Placed by Settings' Proxy + * tab. */ export function DecryptListEditor({ className }: { className?: string }) { const list = useDecryptList(); @@ -167,21 +166,3 @@ function EntryInput({ ); } - -/** Opens the decrypt list from the state pill's More…, until Settings holds it */ -export const decryptListDialogOpenAtom = atom(false); - -export function DecryptListDialog() { - const [open, setOpen] = useAtom(decryptListDialogOpenAtom); - return ( - setOpen(false)} - size="dynamic" - title="Decrypt List" - className="w-[36rem]" - > -
{open && }
- - ); -} diff --git a/apps/yaak-proxy/components/FlowCard.tsx b/apps/yaak-proxy/components/FlowCard.tsx new file mode 100644 index 000000000..977b04326 --- /dev/null +++ b/apps/yaak-proxy/components/FlowCard.tsx @@ -0,0 +1,307 @@ +import { formatSize } from "@yaakapp-internal/lib/formatSize"; +import type { Flow, FlowBody } from "@yaakapp-internal/proxy-lib"; +import { + CountBadge, + Dropdown, + EmptyStateText, + type ExtraPaneTab, + HttpRequestPaneView, + HttpResponsePaneView, + type HttpBodyResult, + Icon, + IconButton, + parseCookieHeaders, + ResponseCookies, + type ResponseViewMode, + SplitLayout, +} from "@yaakapp-internal/ui"; +import classNames from "classnames"; +import { useSetAtom } from "jotai"; +import { useCallback, useMemo, useState } from "react"; +import { BODY_TYPE_BINARY } from "@yaakapp-internal/ui/httpTypes"; +import { findHeader } from "../lib/body"; +import { + browserTabMemory, + defaultTabFor, + type FlowPane, + learnTab, + REQUEST_TAB_AUTH, + REQUEST_TAB_BODY, + REQUEST_TAB_COOKIES, + REQUEST_TAB_HEADERS, + REQUEST_TAB_PARAMS, + RESPONSE_TAB_BODY, + RESPONSE_TAB_COOKIES, + RESPONSE_TAB_HEADERS, + RESPONSE_TAB_INFO, + RESPONSE_TAB_MESSAGES, + RESPONSE_TAB_TIMELINE, +} from "../lib/defaultTab"; +import { saveDialogFlowAtom, useFlowActions } from "../lib/flowActions"; +import { + type FlowBodyData, + flowBodyLoader, + flowCookiesView, + flowRequestToView, + flowResponseToView, +} from "../lib/flowViews"; +import { messagesTabLabel } from "../lib/messages"; +import { FlowBodyViewer, useFlowBody } from "./FlowBodyViewer"; +import { FlowInfo } from "./FlowInfo"; +import { FlowRequestBody } from "./FlowRequestBody"; +import { FlowMessages } from "./FlowMessages"; +import { FlowTimeline } from "./FlowTimeline"; +import { ProxyEditorHost } from "./ProxyEditorHost"; + +/** + * One card for a flow: the request half (its URL line and tabs), a hairline, and the response + * half (its status line and tabs). The halves sit perpendicular to the list: stacked when the + * list is beside the card, side by side when it is above. + */ +export function FlowCard({ flow, listBeside }: { flow: Flow; listBeside: boolean }) { + return ( + + ( +
+ +
+ )} + secondSlot={({ style, orientation }) => ( +
+ +
+ )} + /> + + ); +} + +/** The tab a half opens on, and learning from the ones chosen by hand */ +function useShapeTab(pane: FlowPane, flow: Flow, available: string[]) { + const memory = browserTabMemory(); + // Read again for every flow, so a tab learned on one opens the next of its shape + const defaultTab = defaultTabFor(pane, flow, available, memory); + const onTabChange = useCallback( + (tab: string) => learnTab(pane, flow, tab, memory), + [pane, flow, memory], + ); + return { defaultTab, onTabChange }; +} + +/** A body read through the engine's body RPC, as the shared panes' loaders ask for it */ +function useLoadedBody( + flow: Flow, + which: FlowBody, + enabled = true, +): HttpBodyResult { + const size = which === "request" ? flow.requestSize : flow.responseSize; + const load = enabled && size > 0; + const body = useFlowBody(flow, which, load); + if (!load) return { data: null, isLoading: false, error: null }; + return { data: body.data ?? undefined, isLoading: body.isLoading, error: body.error }; +} + +function FlowRequestHalf({ flow }: { flow: Flow }) { + const body = useLoadedBody(flow, "request"); + const request = useMemo(() => flowRequestToView(flow, body.data ?? null), [flow, body.data]); + const loader = useMemo( + () => + flowBodyLoader({ + useBody: () => useLoadedBody(flow, "request"), + contentType: findHeader(flow.requestHeaders, "content-type"), + }), + [flow], + ); + const cookies = useMemo(() => flowCookiesView(flow), [flow]); + const sentCount = useMemo( + () => parseCookieHeaders(cookies.cookieHeaders).sentCookies.length, + [cookies], + ); + const extraTabs = useMemo( + () => [ + { + tab: { + value: REQUEST_TAB_COOKIES, + label: "Cookies", + empty: sentCount === 0, + rightSlot: , + }, + content: , + }, + ], + [cookies, sentCount], + ); + const available = [ + REQUEST_TAB_BODY, + REQUEST_TAB_PARAMS, + REQUEST_TAB_HEADERS, + ...(request.auth == null ? [] : [REQUEST_TAB_AUTH]), + REQUEST_TAB_COOKIES, + ]; + const { defaultTab, onTabChange } = useShapeTab("request", flow, available); + + return ( + } + renderBody={(editor) => ( + + ) : null + } + requestSize={flow.requestSize} + /> + )} + /> + ); +} + +/** The method and URL, at the status line's height so the two header lines align */ +function UrlLine({ flow }: { flow: Flow }) { + return ( +
+ {flow.method} + + {flow.url} + +
+ ); +} + +function FlowResponseHalf({ flow, className }: { flow: Flow; className: string }) { + const [viewMode, setViewMode] = useState("pretty"); + // A stream's body is its messages, read by the Messages tab; reading it whole again on every + // message would only hold copies of it + const loadBody = flow.streamKind == null; + const body = useLoadedBody(flow, "response", loadBody); + const response = useMemo(() => flowResponseToView(flow, body.data ?? null), [flow, body.data]); + const loader = useMemo( + () => + flowBodyLoader({ + useBody: () => useLoadedBody(flow, "response", loadBody), + contentType: findHeader(flow.responseHeaders, "content-type"), + }), + [flow, loadBody], + ); + const openSave = useSetAtom(saveDialogFlowAtom); + const flowActions = useFlowActions(); + const messagesLabel = messagesTabLabel(flow); + + const extraTabs = useMemo( + () => [ + { + tab: { value: RESPONSE_TAB_TIMELINE, label: "Timeline" }, + content: , + }, + { tab: { value: RESPONSE_TAB_INFO, label: "Info" }, content: }, + ...(messagesLabel == null + ? [] + : [ + { + tab: { + value: RESPONSE_TAB_MESSAGES, + label: messagesLabel, + empty: flow.messageCount === 0, + rightSlot: ( + + ), + }, + content: , + }, + ]), + ], + [flow, messagesLabel], + ); + const available = [ + RESPONSE_TAB_BODY, + RESPONSE_TAB_HEADERS, + RESPONSE_TAB_COOKIES, + RESPONSE_TAB_TIMELINE, + RESPONSE_TAB_INFO, + ...(messagesLabel == null ? [] : [RESPONSE_TAB_MESSAGES]), + ]; + const { defaultTab, onTabChange } = useShapeTab("response", flow, available); + const saved = flow.savedRequestId != null; + + return ( + Waiting for a response} + viewMode={viewMode} + onViewModeChange={setViewMode} + tabsStorageKey="proxy_response_tabs" + activeTabKey={flow.id} + defaultTab={defaultTab} + onTabChange={onTabChange} + extraTabs={extraTabs} + renderBodyViewer={() => ( + + )} + statusLineCenter={ + flow.responseTruncated ? ( + + + {body.data == null ? "Body cut" : `First ${formatSize(body.data.storedSize)} kept`} + + ) : undefined + } + statusLineRight={ +
+ openSave(flow)} + /> + + + +
+ } + /> + ); +} diff --git a/apps/yaak-proxy/components/FlowDetail.tsx b/apps/yaak-proxy/components/FlowDetail.tsx index a17875b2a..5c31ac522 100644 --- a/apps/yaak-proxy/components/FlowDetail.tsx +++ b/apps/yaak-proxy/components/FlowDetail.tsx @@ -1,50 +1,22 @@ -import { formatSize } from "@yaakapp-internal/lib/formatSize"; -import type { Flow } from "@yaakapp-internal/proxy-lib"; -import { - Banner, - Button, - CountBadge, - Dropdown, - type DropdownItem, - formatMillis, - HStack, - HttpStatusTagRaw, - Icon, - IconButton, - LoadingIcon, - SizeTag, - type TabButtonRenderProps, - type TabItem, - TabContent, - Tabs, -} from "@yaakapp-internal/ui"; import classNames from "classnames"; -import { useAtomValue, useSetAtom } from "jotai"; +import { useAtomValue } from "jotai"; import type { CSSProperties } from "react"; -import { useMemo, useState } from "react"; import { selectedFlowAtom } from "../lib/atoms"; -import { findHeader, mimeEssence } from "../lib/body"; -import { saveDialogFlowAtom, useFlowActions } from "../lib/flowActions"; -import { statusReason } from "../lib/statusReason"; -import { messagesTabLabel, streamIsOpen } from "../lib/messages"; -import { EmptyState, FlowBodyViewer, useFlowBody } from "./FlowBodyViewer"; -import { FlowHeaders, FlowQuery, queryParams } from "./FlowHeaders"; -import { FlowMessages } from "./FlowMessages"; +import { EmptyState } from "./FlowBodyViewer"; +import { FlowCard } from "./FlowCard"; import { NotDecrypted } from "./NotDecrypted"; -const TAB_RESPONSE = "response"; -const TAB_REQUEST = "request"; -const TAB_HEADERS = "headers"; -const TAB_QUERY = "query"; -const TAB_MESSAGES = "messages"; - +/** + * The detail pane: the selected flow's card, or what the pane holds before one is selected. Side + * by side, a wider gutter separates the card from the list and it sits a little under the header. + */ export function FlowDetail({ style, sideBySide }: { style?: CSSProperties; sideBySide: boolean }) { const flow = useAtomValue(selectedFlowAtom); return (
) : ( - + )}
); } -function FlowPane({ flow }: { flow: Flow }) { - const [pretty, setPretty] = useState(true); - const responseBody = useFlowBody(flow, "response", flow.responseTruncated); - // Failed or cancelled before any response; after the headers, the body was cut short mid-way. - const noResponse = (flow.error != null || flow.cancelled) && flow.status == null; - const mime = mimeEssence(findHeader(flow.responseHeaders, "content-type")); - const queryCount = useMemo(() => queryParams(flow.url).length, [flow.url]); - const openSave = useSetAtom(saveDialogFlowAtom); - const flowActions = useFlowActions(); - const messagesLabel = messagesTabLabel(flow); - - const tabs = useMemo( - () => [ - ...(messagesLabel == null - ? [] - : [ - { - value: TAB_MESSAGES, - label: messagesLabel, - rightSlot: , - }, - ]), - { - value: TAB_RESPONSE, - renderButton: ({ buttonProps, isActive }) => ( - - ), - }, - { - value: TAB_REQUEST, - label: "Request", - rightSlot: flow.requestSize > 0 ? : null, - }, - { - value: TAB_HEADERS, - label: "Headers", - rightSlot: ( - - ), - }, - { - value: TAB_QUERY, - label: "Query", - rightSlot: , - }, - ], - [ - messagesLabel, - flow.messageCount, - pretty, - mime, - flow.requestSize, - flow.requestHeaders.length, - flow.responseHeaders.length, - queryCount, - ], - ); - - return ( -
-
- - {streamIsOpen(flow) && } - {flow.cancelled && flow.status == null ? ( - Cancelled - ) : ( - - )} - {flow.elapsedMs != null && ( - <> - • - - {formatMillis(flow.elapsedMs)} - - - )} - {!noResponse && ( - <> - • - - - )} - - {flow.responseTruncated ? ( - - - {responseBody.data == null - ? "Body cut" - : `First ${formatSize(responseBody.data.storedSize)} kept`} - - ) : ( - - )} - - - - - - -
- -
- {flow.error != null && ( - - {flow.error} - - )} - {flow.cancelled && ( - - {flow.status == null - ? "The app cancelled the request before a response came back." - : "The app cancelled the request before the response finished."} - - )} - - - {noResponse ? ( - -
- {flow.cancelled ? "Cancelled" : "No response"} -
-
- {flow.cancelled - ? "The app cancelled the request before a response came back." - : "The request never got a response."}{" "} - The Request tab has what the app sent. -
- - ) : ( - - )} - - - - - - - - - - - {messagesLabel != null && ( - - - - )} - -
-
- ); -} - -interface ViewModeButtonProps { - buttonProps: TabButtonRenderProps["buttonProps"]; - isActive: boolean; - pretty: boolean; - onChangePretty: (pretty: boolean) => void; - prettyLabel: string; - allowRaw: boolean; -} - -/** The client's Response tab: a click on the active tab opens its view menu */ -function ViewModeButton({ - buttonProps, - isActive, - pretty, - onChangePretty, - prettyLabel, - allowRaw, -}: ViewModeButtonProps) { - const showPretty = pretty || !allowRaw; - const items: DropdownItem[] = [ - { - label: prettyLabel, - leftSlot: , - onSelect: () => onChangePretty(true), - }, - { - label: `${prettyLabel} (Raw)`, - hidden: !allowRaw, - leftSlot: , - onSelect: () => onChangePretty(false), - }, - ]; - - return ( - - - - ); -} - -function CheckMark({ visible }: { visible: boolean }) { - return ; -} - function Keys({ keys }: { keys: string[] }) { return ( diff --git a/apps/yaak-proxy/components/FlowHeaders.tsx b/apps/yaak-proxy/components/FlowHeaders.tsx deleted file mode 100644 index 01d8c7496..000000000 --- a/apps/yaak-proxy/components/FlowHeaders.tsx +++ /dev/null @@ -1,109 +0,0 @@ -import { formatSize } from "@yaakapp-internal/lib/formatSize"; -import type { Flow } from "@yaakapp-internal/proxy-lib"; -import { - DetailsBanner, - formatMillis, - HeadersTable, - KeyValueRow, - KeyValueRows, -} from "@yaakapp-internal/ui"; -import { useMemo } from "react"; -import { EmptyState } from "./FlowBodyViewer"; - -const startedFormat = new Intl.DateTimeFormat("en-US", { - month: "short", - day: "numeric", - year: "numeric", - hour: "numeric", - minute: "2-digit", - second: "2-digit", - fractionalSecondDigits: 3, - timeZoneName: "short", -}); - -const bytesFormat = new Intl.NumberFormat("en-US"); - -export function FlowHeaders({ flow }: { flow: Flow }) { - const versions = - flow.upstreamHttpVersion == null - ? `${flow.clientHttpVersion} from the app` - : `${flow.clientHttpVersion} from the app, ${flow.upstreamHttpVersion} to the server`; - - return ( -
- Info}> - - - {startedFormat.format(new Date(`${flow.startedAt}Z`))} - - {flow.url} - {flow.remoteAddr ?? } - {versions} - - {flow.elapsedMs == null ? ( - - ) : flow.elapsedHeadersMs == null ? ( - `${formatMillis(flow.elapsedMs)} in all` - ) : ( - `${formatMillis(flow.elapsedMs)} in all, headers after ${formatMillis(flow.elapsedHeadersMs)}` - )} - - - - {formatSize(flow.requestSize)} sent, {formatSize(flow.responseSize)} received{" "} - - ({bytesFormat.format(flow.responseSize)} B) - - - - - - - -
- ); -} - -export function FlowQuery({ flow }: { flow: Flow }) { - const params = useMemo(() => queryParams(flow.url), [flow.url]); - - if (params.length === 0) { - return No query parameters; - } - - return ( -
- - {params.map(([name, value], i) => ( - // oxlint-disable-next-line react/no-array-index-key -- a name can repeat - - {value} - - ))} - -
- ); -} - -export function queryParams(url: string): [string, string][] { - try { - return [...new URL(url).searchParams.entries()]; - } catch { - return []; - } -} - -function Missing() { - return --; -} diff --git a/apps/yaak-proxy/components/FlowInfo.tsx b/apps/yaak-proxy/components/FlowInfo.tsx new file mode 100644 index 000000000..601f997fd --- /dev/null +++ b/apps/yaak-proxy/components/FlowInfo.tsx @@ -0,0 +1,125 @@ +import { formatSize } from "@yaakapp-internal/lib/formatSize"; +import type { Flow } from "@yaakapp-internal/proxy-lib"; +import { DetailsBanner, formatMillis, KeyValueRow, KeyValueRows } from "@yaakapp-internal/ui"; +import { format } from "date-fns"; +import type { HTMLAttributes, ReactElement, ReactNode } from "react"; +import { utcTimestamp } from "../lib/flowViews"; +import { folderOf } from "../lib/search"; +import { AppIcon } from "./AppIcon"; + +/** + * Where a flow came from and how it travelled: in a proxy you didn't send the request, so the + * connection on each leg, the app and its process, timing, sizes and the flow's record are part + * of the answer. + */ +export function FlowInfo({ flow }: { flow: Flow }) { + const folder = folderOf(flow); + const clientPort = portOf(flow.clientAddr); + + return ( +
+
+ {[ + row("remote", "Remote Address", flow.remoteAddr, { copy: true }), + row("client_version", "Client Version", flow.clientHttpVersion), + row("server_version", "Server Version", flow.upstreamHttpVersion), + row("tls", "TLS", flow.upstreamTlsVersion), + row("cipher", "Cipher", flow.upstreamTlsCipher), + row("client_port", "Client Port", clientPort), + ]} +
+
+ {[ + row( + "app", + "Name", + flow.clientName == null ? null : ( + + {flow.clientPath != null && } + {flow.clientName} + {folder != null && {folder}} + + ), + ), + flow.clientBundleId != null + ? row("bundle", "Bundle ID", flow.clientBundleId, { copy: true }) + : row("path", "Path", flow.clientPath, { copy: true }), + row("pid", "PID", flow.clientPid == null ? null : `${flow.clientPid}`), + row("cwd", "Directory", flow.clientCwd, { copy: true }), + ]} +
+
+ {[ + row( + "started", + "Started", + format(utcTimestamp(flow.startedAt), "MMM d, yyyy, h:mm:ss.SSS a"), + ), + row("headers", "Headers", ms(flow.elapsedHeadersMs)), + row("total", "Total", ms(flow.elapsedMs)), + ]} +
+
+ {[ + row("request_size", "Request", formatSize(flow.requestSize)), + row("response_size", "Response", formatSize(flow.responseSize)), + ]} +
+
+ {[ + row("number", "Number", `#${flow.number}`, { copy: true }), + row("id", "ID", flow.id, { copy: true }), + row("saved", "Saved As", flow.savedRequestId, { copy: true }), + ]} +
+
+ ); +} + +/** A row as `KeyValueRows` takes it */ +type Row = ReactElement>; + +function Section({ title, children }: { title: string; children: (Row | null)[] }) { + const rows = children.filter((r): r is Row => r != null); + if (rows.length === 0) return null; + return ( + {title}}> + {rows} + + ); +} + +/** A row, or nothing when there's no value for it */ +function row( + key: string, + label: string, + value: ReactNode | null | undefined, + opts: { copy?: boolean } = {}, +): Row | null { + if (value == null || value === "") return null; + return ( + + {value} + + ); +} + +function ms(value: number | null): string | null { + return value == null ? null : formatMillis(value); +} + +/** The port of `127.0.0.1:54321` or `[::1]:54321` */ +export function portOf(addr: string | null | undefined): string | null { + const port = addr?.match(/:(\d+)$/)?.[1]; + return port ?? null; +} + +function copyText(text: string) { + navigator.clipboard.writeText(text).catch(console.error); +} diff --git a/apps/yaak-proxy/components/FlowList.tsx b/apps/yaak-proxy/components/FlowList.tsx index 36811a46c..451acae63 100644 --- a/apps/yaak-proxy/components/FlowList.tsx +++ b/apps/yaak-proxy/components/FlowList.tsx @@ -329,7 +329,7 @@ export function FlowList({ style, className }: Props) { className={classNames( "absolute bottom-3 left-1/2 -translate-x-1/2", "flex items-center gap-1.5 h-6 px-3 rounded-full", - "bg-surface border border-border shadow-lg text-text text-xs", + "bg-surface border border-border shadow-lg text-text text-xs tabular-nums", "hover:bg-surface-highlight", )} > diff --git a/apps/yaak-proxy/components/FlowListHeader.tsx b/apps/yaak-proxy/components/FlowListHeader.tsx index b65c56fd0..45971fe0c 100644 --- a/apps/yaak-proxy/components/FlowListHeader.tsx +++ b/apps/yaak-proxy/components/FlowListHeader.tsx @@ -107,11 +107,8 @@ export const FlowListHeader = memo(function FlowListHeader({ - ) - } - /> - ); -} - -/** Saves whether the system proxy turns on as the app launches */ -function useSetSystemProxyAtLaunch() { - const store = useStore(); - return useRpcMutation("set_system_proxy_at_launch", { - onSettled: () => void refreshProxyState(store), - }); -} - -/** Saves the HTTP versions offered to clients, which the engine applies to new connections */ -function useSetHttpVersion() { - const store = useStore(); - return useRpcMutation("set_http_version", { - onSettled: () => void refreshProxyState(store), - }); -} - export function useCopyEnvironment() { const proxyState = useAtomValue(proxyStateAtom); const caStatus = useRpcQuery("ca_status", {}); diff --git a/apps/yaak-proxy/components/SearchBar.tsx b/apps/yaak-proxy/components/SearchBar.tsx index 91954ac83..44d1cc045 100644 --- a/apps/yaak-proxy/components/SearchBar.tsx +++ b/apps/yaak-proxy/components/SearchBar.tsx @@ -7,9 +7,7 @@ import { searchTermsAtom, searchTextAtom, searchTextVersionAtom, - searchWindowAtom, setSearchTextAtom, - shownTotalAtom, } from "../lib/store"; /** How long typing settles before the search runs */ @@ -17,8 +15,8 @@ const SEARCH_DEBOUNCE_MS = 200; /** * The header's search over every flow in the database, by URL, method, status and content type, - * and the `@app`, `@host` and `@path` terms the sidebar writes. Its placeholder counts the flows; - * a search's matches show inside it. ⌘F focuses it and Escape clears it. + * and the `@app`, `@host` and `@path` terms the sidebar writes. Its placeholder counts the flows. + * ⌘F focuses it and Escape clears it. */ export const SearchBar = memo(function SearchBar() { const [text, setText] = useAtom(searchTextAtom); @@ -75,26 +73,24 @@ export const SearchBar = memo(function SearchBar() { }} leftSlot={ <> - + } rightSlot={ text === "" ? ( - + ⌘F ) : ( -
- - -
+ ) } /> @@ -118,16 +114,3 @@ function placeholderFor(total: number): string { if (total === 0) return "Search flows"; return `Search ${total.toLocaleString()} ${total === 1 ? "flow" : "flows"}`; } - -/** A search's matches of all traffic, "12 of 4,102", inside the field before its x */ -function MatchCount() { - const matches = useAtomValue(shownTotalAtom); - const total = useAtomValue(allTrafficTotalAtom); - const search = useAtomValue(searchWindowAtom); - if (search == null || search.loading) return null; - return ( - - {matches.toLocaleString()} of {total.toLocaleString()} - - ); -} diff --git a/apps/yaak-proxy/components/Settings.tsx b/apps/yaak-proxy/components/Settings.tsx new file mode 100644 index 000000000..169b5359c --- /dev/null +++ b/apps/yaak-proxy/components/Settings.tsx @@ -0,0 +1,181 @@ +import { useQuery } from "@tanstack/react-query"; +import { getVersion } from "@tauri-apps/api/app"; +import { appDataDir, appLogDir } from "@tauri-apps/api/path"; +import { + Dialog, + EmptyStateText, + SETTINGS_TAB_GENERAL, + SETTINGS_TAB_INTERFACE, + SETTINGS_TAB_LICENSE, + SETTINGS_TAB_THEME, + type SettingsCallbacks, + SettingsDialog, + SettingsGeneral, + SettingsInterface, + SettingsTheme, +} from "@yaakapp-internal/ui"; +import { atom, useAtom, useSetAtom } from "jotai"; +import { useEffect, useMemo, useState } from "react"; +import { settingsValuesAtom } from "../lib/interfaceSettings"; +import { openLink, revealPath } from "../lib/opener"; +import { proxySettingsView } from "../lib/settings"; +import { getOsType } from "../lib/tauri"; +import { useShowToast } from "../lib/toast"; +import { SettingsProxy } from "./SettingsProxy"; + +export const SETTINGS_TAB_PROXY = "proxy"; + +/** The settings window is open, on a tab, or closed */ +export const settingsOpenAtom = atom(null); + +/** + * Yaak's settings, shared: its General, Theme, Interface and License tabs, plus the proxy's own + * Proxy tab. The header's gear and `⌘,` open it. + */ +export function SettingsWindow() { + const [tab, setTab] = useAtom(settingsOpenAtom); + useSettingsHotkey(); + return ( + setTab(null)} + size="lg" + noPadding + noScroll + className="h-[min(44rem,90vh)]" + > + {tab != null && } + + ); +} + +/** The settings themselves, as the window holds them */ +export function SettingsContent({ defaultTab }: { defaultTab: string }) { + const [values, change] = useAtom(settingsValuesAtom); + const app = useAppInfo(); + const systemAppearance = useSystemAppearance(); + const view = useMemo( + () => proxySettingsView(values, app, systemAppearance), + [values, app, systemAppearance], + ); + const showToast = useShowToast(); + const callbacks = useMemo(() => { + const fail = (message: string) => showToast({ message, color: "danger" }); + return { + onChange: (patch) => change(patch), + // The proxy draws its own title bar always; the row isn't shown + onApplyNativeTitlebar: (useNativeTitlebar) => change({ useNativeTitlebar }), + onCheckForUpdates: () => undefined, + onActivateLicense: async () => undefined, + onDeactivateLicense: () => undefined, + onOpenUrl: (url) => void openLink(url, fail), + onRevealPath: (path) => void revealPath(path, fail), + pricingUrl: (intent) => `https://yaak.app/pricing?intent=${encodeURIComponent(intent)}`, + }; + }, [change, showToast]); + const props = { view, callbacks }; + + return ( + , + }, + { + value: SETTINGS_TAB_THEME, + label: "Theme", + icon: "palette", + content: , + }, + { + value: SETTINGS_TAB_INTERFACE, + label: "Interface", + icon: "columns_2", + content: , + }, + { + value: SETTINGS_TAB_PROXY, + label: "Proxy", + icon: "wifi", + content: , + }, + { + value: SETTINGS_TAB_LICENSE, + label: "License", + icon: "key_round", + content: Coming soon, + }, + ]} + /> + ); +} + +/** `⌘,`, or `Ctrl+,` off macOS, opens settings, as in Yaak */ +function useSettingsHotkey() { + const setOpen = useSetAtom(settingsOpenAtom); + useEffect(() => { + const mac = getOsType() === "macos"; + function onKeyDown(e: KeyboardEvent) { + if (e.key !== "," || e.shiftKey || e.altKey) return; + if (mac ? !e.metaKey || e.ctrlKey : !e.ctrlKey || e.metaKey) return; + e.preventDefault(); + setOpen((tab) => tab ?? SETTINGS_TAB_GENERAL); + } + window.addEventListener("keydown", onKeyDown); + return () => window.removeEventListener("keydown", onKeyDown); + }, [setOpen]); +} + +function useAppInfo() { + const os = getOsType(); + const { data } = useQuery({ + queryKey: ["settings_app_info"], + queryFn: async () => { + const [version, dataDir, logDir] = await Promise.all([ + getVersion(), + appDataDir(), + appLogDir(), + ]); + return { version, dataDir, logDir }; + }, + retry: false, + }); + return useMemo( + () => ({ + name: "Yaak Proxy", + version: data?.version ?? "", + dataDir: data?.dataDir ?? "", + logDir: data?.logDir ?? "", + revealPathLabel: + os === "macos" + ? "Reveal in Finder" + : os === "windows" + ? "Show in Explorer" + : "Show in Folder", + }), + [data, os], + ); +} + +function useSystemAppearance(): "light" | "dark" { + const query = "(prefers-color-scheme: dark)"; + const [dark, setDark] = useState(() => window.matchMedia(query).matches); + useEffect(() => { + const media = window.matchMedia(query); + const onChange = () => setDark(media.matches); + media.addEventListener("change", onChange); + return () => media.removeEventListener("change", onChange); + }, []); + return dark ? "dark" : "light"; +} + +/** Opens settings on a tab */ +export function useOpenSettings() { + const setOpen = useSetAtom(settingsOpenAtom); + return (tab: string = SETTINGS_TAB_GENERAL) => setOpen(tab); +} diff --git a/apps/yaak-proxy/components/SettingsProxy.tsx b/apps/yaak-proxy/components/SettingsProxy.tsx new file mode 100644 index 000000000..5ed520827 --- /dev/null +++ b/apps/yaak-proxy/components/SettingsProxy.tsx @@ -0,0 +1,366 @@ +import { useQueryClient } from "@tanstack/react-query"; +import { formatSize } from "@yaakapp-internal/lib/formatSize"; +import type { HttpVersion, SystemProxyStatus } from "@yaakapp-internal/proxy-lib"; +import { + Button, + Dialog, + DialogFooter, + Heading, + Icon, + RadioDropdown, + SettingRow, + SettingRowBoolean, + PlainInput, + SettingRowSelect, + SettingSelectControl, + SettingSnippet, + SettingsList, + SettingsSection, + VStack, +} from "@yaakapp-internal/ui"; +import { useAtomValue, useSetAtom, useStore } from "jotai"; +import { useState } from "react"; +import { useRpcMutation } from "../hooks/useRpcMutation"; +import { useRpcQuery } from "../hooks/useRpcQuery"; +import { helperDialogOpenAtom } from "../lib/atoms"; +import { + defaultShell, + environmentBlock, + environmentVars, + TERMINAL_SHELLS, + type TerminalShell, +} from "../lib/env"; +import { + CAPTURE_LIMIT_OPTIONS, + KEEP_TRAFFIC_OPTIONS, + sizeFromOptionValue, + sizeOptionsFor, + sizeOptionValue, +} from "../lib/settings"; +import { helperRow } from "../lib/systemProxy"; +import { proxyStateAtom, refreshProxyState } from "../lib/store"; +import { getOsType } from "../lib/tauri"; +import { useShowToast } from "../lib/toast"; +import { DecryptListEditor } from "./DecryptListEditor"; + +/** + * The proxy's own settings tab: how it listens, what it decrypts, the system proxy and its + * helper, how much traffic it keeps, and the environment for one terminal. Everything set once + * and left alone, which is what left the state pill's menu. + */ +export function SettingsProxy() { + const proxyState = useAtomValue(proxyStateAtom); + const store = useStore(); + const refresh = { onSettled: () => void refreshProxyState(store) }; + const setHttpVersion = useRpcMutation("set_http_version", refresh); + const setCaptureLimit = useRpcMutation("set_capture_body_limit", refresh); + const setAtLaunch = useRpcMutation("set_system_proxy_at_launch", refresh); + const systemProxy = useRpcQuery("system_proxy_status", {}); + // Another process owning the engine wouldn't see changes until it restarts + const elsewhere = proxyState?.otherOwner != null; + + return ( + +
+ Proxy +

Listening, decryption, storage and terminals

+
+ + + + + name="httpVersion" + title="HTTP version" + description="For new connections" + value={proxyState?.httpVersion ?? "auto"} + disabled={elsewhere} + onChange={(httpVersion) => setHttpVersion.mutate({ httpVersion })} + options={[ + { label: "Auto", value: "auto" }, + { label: "HTTP/1.1 only", value: "http1Only" }, + ]} + /> + { + const bytes = sizeFromOptionValue(v); + if (bytes != null) setCaptureLimit.mutate({ bytes }); + }} + options={sizeOptionsFor(CAPTURE_LIMIT_OPTIONS, proxyState?.captureBodyLimit ?? null) + .filter((o) => o.bytes != null) + .map(({ label, value }) => ({ label, value }))} + /> + + + + + + + {systemProxy.data?.supported === true && ( + + setAtLaunch.mutate({ enabled })} + /> + + + )} + + + + + + + + + + + + ); +} + +/** A port from 1 to 65535, or null */ +function parsePort(text: string): number | null { + if (!/^\d+$/.test(text.trim())) return null; + const port = Number(text.trim()); + return port > 0 && port < 65536 ? port : null; +} + +/** + * The port, saved when the field is left or Enter is pressed rather than on each keystroke: + * every save restarts a running proxy, so typing 8080 must not restart it on 8, 80 and 808. + */ +function PortRow({ disabled }: { disabled: boolean }) { + const proxyState = useAtomValue(proxyStateAtom); + const store = useStore(); + const showToast = useShowToast(); + const saved = proxyState?.configuredPort ?? 9090; + const [draft, setDraft] = useState(null); + // Puts the saved port back in the field after a rejected or failed change + const [resets, setResets] = useState(0); + const reset = () => { + setDraft(null); + setResets((n) => n + 1); + }; + const setPort = useRpcMutation("set_port", { + onError: (err) => showToast({ message: err.message, color: "danger" }), + onSettled: () => { + reset(); + void refreshProxyState(store); + }, + }); + const commit = () => { + if (draft == null || setPort.isPending) return; + const port = parsePort(draft); + if (port == null || port === saved) { + reset(); + return; + } + setPort.mutate({ port }); + }; + return ( + + parsePort(v) != null} + onChange={setDraft} + onBlur={commit} + onKeyDownCapture={(e) => { + if (e.key === "Enter") commit(); + }} + /> + + ); +} + +function HelperRow({ status }: { status: SystemProxyStatus }) { + const queryClient = useQueryClient(); + const openDialog = useSetAtom(helperDialogOpenAtom); + const unregister = useRpcMutation("helper_unregister", { + onSuccess: (status) => + queryClient.setQueryData(["system_proxy_status", {}], status), + }); + const row = helperRow(status.helper, status.backend); + return ( + {unregister.error.message} + ) : ( + `${capitalize(row.title.replace(/^Helper: /, ""))}. ${row.description}` + ) + } + > + {row.action != null && ( + + )} + + ); +} + +function KeepTrafficRow() { + const proxyState = useAtomValue(proxyStateAtom); + const store = useStore(); + const size = useRpcQuery("get_storage_size", {}, { refetchInterval: 10_000 }); + const setKeep = useRpcMutation("set_keep_traffic", { + onSettled: () => void refreshProxyState(store), + }); + const keep = proxyState?.keepTrafficBytes ?? null; + return ( + +
+ {size.data != null && ( + + {formatSize(size.data.bytes)} used + + )} + setKeep.mutate({ bytes: sizeFromOptionValue(v) })} + options={sizeOptionsFor(KEEP_TRAFFIC_OPTIONS, keep).map(({ label, value }) => ({ + label, + value, + }))} + /> +
+ + ); +} + +function ResetRow() { + const [confirming, setConfirming] = useState(false); + const store = useStore(); + const queryClient = useQueryClient(); + const showToast = useShowToast(); + const reset = useRpcMutation("reset_captured_data", { + onSuccess: () => { + setConfirming(false); + void queryClient.invalidateQueries(); + showToast({ message: "Captured data reset" }); + }, + onSettled: () => void refreshProxyState(store), + }); + return ( + + + setConfirming(false)} + size="sm" + title="Reset captured data?" + description="Every captured flow and the keys that encrypt them are deleted." + > + {reset.error != null &&

{reset.error.message}

} + setConfirming(false) }, + { + label: "Reset", + color: "danger", + isLoading: reset.isPending, + onClick: () => reset.mutate({}), + }, + ]} + /> + + + ); +} + +function TerminalRow() { + const proxyState = useAtomValue(proxyStateAtom); + const caStatus = useRpcQuery("ca_status", {}); + const showToast = useShowToast(); + const os = getOsType(); + const [shell, setShell] = useState(() => defaultShell(os)); + const openTerminal = useRpcMutation("open_terminal", { + onError: (err) => showToast({ message: err.message, color: "danger" }), + }); + const port = + proxyState?.port ?? proxyState?.otherOwner?.proxyPort ?? proxyState?.configuredPort ?? 9090; + const caPath = caStatus.data?.certPath; + const shellLabel = TERMINAL_SHELLS.find((s) => s.value === shell)?.label ?? shell; + + return ( +
+
+ Environment + setShell(v as TerminalShell)} + items={TERMINAL_SHELLS.map((s) => ({ label: s.label, value: s.value }))} + > + + + +
+

Proxies one terminal instead of the system

+ {caPath != null && ( + + )} +
+ ); +} + +function capitalize(s: string): string { + return s.charAt(0).toUpperCase() + s.slice(1); +} diff --git a/apps/yaak-proxy/components/Sidebar.tsx b/apps/yaak-proxy/components/Sidebar.tsx index d519b3ab7..5bea89e8a 100644 --- a/apps/yaak-proxy/components/Sidebar.tsx +++ b/apps/yaak-proxy/components/Sidebar.tsx @@ -63,6 +63,7 @@ function ItemInner({ item }: { item: SidebarItem }) {
{item.kind === "all" && } {item.kind === "tunnelled" && } + {item.kind === "folder" && } {item.appPath != null && } {item.label} {item.sublabel != null && ( diff --git a/apps/yaak-proxy/font-size.ts b/apps/yaak-proxy/font-size.ts index ffe5fe96f..37af96871 100644 --- a/apps/yaak-proxy/font-size.ts +++ b/apps/yaak-proxy/font-size.ts @@ -1,3 +1,6 @@ -// Yaak's default interface size, so the two apps read as one family. Later from the shared -// Interface settings. -document.documentElement.style.fontSize = "15px"; +import { applyFontSizes } from "./lib/interfaceSettings"; +import { readSettingsValues, SETTINGS_VALUES_KEY } from "./lib/settings"; + +// Yaak's default sizes unless changed in Settings' Interface tab, so the two apps read as one +// family. +applyFontSizes(readSettingsValues(localStorage.getItem(SETTINGS_VALUES_KEY))); diff --git a/apps/yaak-proxy/harness-settings.html b/apps/yaak-proxy/harness-settings.html deleted file mode 100644 index 14957782e..000000000 --- a/apps/yaak-proxy/harness-settings.html +++ /dev/null @@ -1,212 +0,0 @@ - - - - - - Yaak Proxy, shared settings harness - - - - -
- - - - - diff --git a/apps/yaak-proxy/harness/DragRegionCheck.tsx b/apps/yaak-proxy/harness/DragRegionCheck.tsx new file mode 100644 index 000000000..0d7a99deb --- /dev/null +++ b/apps/yaak-proxy/harness/DragRegionCheck.tsx @@ -0,0 +1,57 @@ +import { useEffect, useState } from "react"; + +interface Probe { + x: number; + y: number; + kind: "drag" | "control" | "dead"; +} + +/** + * Dev only, with `?window&dragcheck`: probes the header across its width and marks what a press + * at each point lands on. Green drags the window (the element hit carries + * `data-tauri-drag-region`, which Tauri reads off the target alone), blue is a control, and red + * is dead space that neither drags nor does anything. + */ +export function DragRegionCheck() { + const [probes, setProbes] = useState([]); + useEffect(() => { + const timer = setTimeout(() => { + const header = document.querySelector(".x-theme-appHeader"); + if (header == null) return; + const rect = header.getBoundingClientRect(); + const out: Probe[] = []; + for (const y of [rect.top + 6, rect.top + rect.height / 2]) { + for (let x = rect.left + 4; x < rect.right; x += 10) { + const el = document.elementFromPoint(x, y); + const kind = + el == null + ? "dead" + : el.hasAttribute("data-tauri-drag-region") + ? "drag" + : el.closest("button, input, [role=button], .cm-editor, [role=combobox]") != null + ? "control" + : "dead"; + out.push({ x, y, kind }); + } + } + setProbes(out); + }, 800); + return () => clearTimeout(timer); + }, []); + return ( +
+ {probes.map((p) => ( + + ))} +
+ ); +} diff --git a/apps/yaak-proxy/harness/FlowPanesHarness.tsx b/apps/yaak-proxy/harness/FlowPanesHarness.tsx index ff15d61ba..dc01632b0 100644 --- a/apps/yaak-proxy/harness/FlowPanesHarness.tsx +++ b/apps/yaak-proxy/harness/FlowPanesHarness.tsx @@ -1,24 +1,17 @@ -import { - HttpRequestPaneView, - HttpResponsePaneView, - type HttpBodyResult, - type ResponseViewMode, -} from "@yaakapp-internal/ui"; -import { useMemo, useState } from "react"; -import { ProxyEditorHost } from "../components/ProxyEditorHost"; -import { findHeader } from "../lib/body"; -import { - type FlowBodyData, - flowBodyFromRpc, - flowBodyLoader, - flowRequestToView, - flowResponseToView, -} from "../lib/flowViews"; +import { useSetAtom } from "jotai"; +import { useLayoutEffect, useState } from "react"; +import { FlowDetail } from "../components/FlowDetail"; +import { FlowList } from "../components/FlowList"; +import { selectedFlowAtom } from "../lib/atoms"; +import { browserTabMemory, learnedTabKey, flowShape } from "../lib/defaultTab"; +import { flowWindowAtom } from "../lib/store"; import { type FlowFixture, fixtures } from "./fixtures"; /** - * Dev only: renders captured flows through the shared request and response panes, read-only, - * side by side, the way the proxy's detail view will. Open `/harness.html?flow=0&theme=dark`. + * Dev only: captured flows in the detail card, over fixtures. Open + * `/harness.html?flow=0&theme=dark`; `&list` puts the flow list beside the card (its halves + * stacked), `&below` puts it above (side by side), and `&tab=info` opens the response half on a + * tab, as though it were chosen by hand for flows of that shape. */ export function FlowPanesHarness() { const params = new URLSearchParams(location.search); @@ -27,87 +20,57 @@ export function FlowPanesHarness() { if (fixture == null) return null; return ( - -
-
- {fixtures.map((f, i) => ( - - ))} -
- +
+
+ {fixtures.map((f, i) => ( + + ))}
- - ); -} - -function FlowPanes({ fixture }: { fixture: FlowFixture }) { - const { flow } = fixture; - const [viewMode, setViewMode] = useState("pretty"); - const requestBody = useMemo(() => flowBodyFromRpc(fixture.requestBody), [fixture]); - const responseBody = useMemo(() => flowBodyFromRpc(fixture.responseBody), [fixture]); - const request = useMemo(() => flowRequestToView(flow, requestBody), [flow, requestBody]); - const response = useMemo(() => flowResponseToView(flow, responseBody), [flow, responseBody]); - - const requestLoader = useMemo( - () => - flowBodyLoader({ - useBody: () => loaded(requestBody), - contentType: findHeader(flow.requestHeaders, "content-type"), - }), - [requestBody, flow], - ); - const responseLoader = useMemo( - () => - flowBodyLoader({ - useBody: () => loaded(responseBody), - contentType: findHeader(flow.responseHeaders, "content-type"), - }), - [responseBody, flow], - ); - - return ( -
-
- -
- Waiting for a response
} - viewMode={viewMode} - onViewModeChange={setViewMode} - tabsStorageKey="flow_response_tabs" - activeTabKey={flow.id} - onCopyAsCurl={() => undefined} - /> +
); } -function loaded(data: FlowBodyData): HttpBodyResult { - return { data, isLoading: false, error: null }; -} +function FixtureCard({ fixture }: { fixture: FlowFixture }) { + const params = new URLSearchParams(location.search); + const setSelected = useSetAtom(selectedFlowAtom); + const setWindow = useSetAtom(flowWindowAtom); + const [ready, setReady] = useState(false); + useLayoutEffect(() => { + const flows = fixtures.map((f, i) => ({ ...f.flow, number: 1401 + i })); + const tab = params.get("tab"); + if (tab != null) { + browserTabMemory()?.setItem(learnedTabKey("response", flowShape(fixture.flow)), tab); + } + setWindow({ flows, hasOlder: false, total: flows.length }); + setSelected(flows.find((f) => f.id === fixture.flow.id) ?? fixture.flow); + setReady(true); + // oxlint-disable-next-line react-hooks/exhaustive-deps -- once per fixture + }, [fixture]); + if (!ready) return null; -/** The engine formats JSON over RPC, which a page outside the app can't reach */ -async function formatLocally(text: string, language: unknown): Promise { - if (language !== "json") return text; - try { - return JSON.stringify(JSON.parse(text), null, 2); - } catch { - return text; + if (params.has("list")) { + return ( +
+ + +
+ ); } + if (params.has("below")) { + return ( +
+ + +
+ ); + } + return ; } diff --git a/apps/yaak-proxy/harness/LockedHarness.tsx b/apps/yaak-proxy/harness/LockedHarness.tsx index c3b2a80be..f8c77a071 100644 --- a/apps/yaak-proxy/harness/LockedHarness.tsx +++ b/apps/yaak-proxy/harness/LockedHarness.tsx @@ -3,6 +3,7 @@ import type { GetProxyStateResponse } from "@yaakapp-internal/proxy-lib"; import { useStore } from "jotai"; import { type ReactNode, useEffect, useState } from "react"; import { ProxyLayout } from "../components/ProxyLayout"; +import { DragRegionCheck } from "./DragRegionCheck"; import { useHotkeyContextValue } from "../lib/hotkeys"; import { syncFlows, syncProxyState, syncSidebarSelection } from "../lib/store"; @@ -21,6 +22,8 @@ const STATE: GetProxyStateResponse = { encrypting: null, locked: { message: LOCKED, keychain: true }, otherOwner: null, + captureBodyLimit: 10 * 1024 * 1024, + keepTrafficBytes: null, }; /** What the backend answers while the keychain refuses: the state, and every read refused */ @@ -55,8 +58,9 @@ export function installLockedBackend() { if (cmd.startsWith("plugin:event|")) return 0; throw new Error(`Not in the harness: ${cmd}`); }; - (window as unknown as { __TAURI_OS_PLUGIN_INTERNALS__: Record }) - .__TAURI_OS_PLUGIN_INTERNALS__ = { + ( + window as unknown as { __TAURI_OS_PLUGIN_INTERNALS__: Record } + ).__TAURI_OS_PLUGIN_INTERNALS__ = { os_type: "macos", platform: "macos", family: "unix", @@ -67,7 +71,10 @@ export function installLockedBackend() { localStorage.setItem("system_proxy_ever_on", "true"); } -/** The whole window, as the app renders it, while captured traffic is locked */ +/** + * The whole window, as the app renders it: while captured traffic is locked, or over the + * fixture backend with `?window` + */ export function LockedHarness() { const store = useStore(); const [synced, setSynced] = useState(false); @@ -81,6 +88,7 @@ export function LockedHarness() { return ( + {new URLSearchParams(location.search).has("dragcheck") && } ); } diff --git a/apps/yaak-proxy/harness/SettingsHarness.tsx b/apps/yaak-proxy/harness/SettingsHarness.tsx new file mode 100644 index 000000000..8b3be070e --- /dev/null +++ b/apps/yaak-proxy/harness/SettingsHarness.tsx @@ -0,0 +1,26 @@ +import { useStore } from "jotai"; +import { useEffect, useState } from "react"; +import { SETTINGS_TAB_PROXY, SettingsContent } from "../components/Settings"; +import { syncProxyState } from "../lib/store"; + +/** + * Dev only: the settings window over the fixture backend, at the dialog's size. Open + * `/harness.html?settings&theme=dark`; `&tab=general` opens another tab than Proxy. + */ +export function SettingsHarness() { + const store = useStore(); + const [synced, setSynced] = useState(false); + useEffect(() => { + syncProxyState(store); + setSynced(true); + }, [store]); + const tab = new URLSearchParams(location.search).get("tab") ?? SETTINGS_TAB_PROXY; + if (!synced) return null; + return ( +
+
+ +
+
+ ); +} diff --git a/apps/yaak-proxy/harness/backend.ts b/apps/yaak-proxy/harness/backend.ts new file mode 100644 index 000000000..14a7c5f05 --- /dev/null +++ b/apps/yaak-proxy/harness/backend.ts @@ -0,0 +1,97 @@ +import type { GetProxyStateResponse } from "@yaakapp-internal/proxy-lib"; +import { fixtures } from "./fixtures"; + +/** + * Dev only: answers the engine's RPCs the harness pages need, from fixtures, so the card and + * settings render outside the app as they do inside it. + */ +export function installFixtureBackend() { + const internals = (window as unknown as { __TAURI_INTERNALS__: Record }) + .__TAURI_INTERNALS__; + internals.invoke = async (cmd: string, args?: { cmd?: string; payload?: unknown }) => { + if (cmd === "rpc") return answer(args?.cmd ?? "", args?.payload); + if (cmd.startsWith("plugin:event|")) return 0; + throw new Error(`Not in the harness: ${cmd}`); + }; + (window as unknown as { __TAURI_OS_PLUGIN_INTERNALS__: Record }) + .__TAURI_OS_PLUGIN_INTERNALS__ = { + os_type: "macos", + platform: "macos", + family: "unix", + version: "15.0", + eol: "\n", + }; +} + +export const HARNESS_STATE: GetProxyStateResponse = { + state: "running", + port: 9090, + configuredPort: 9090, + recording: true, + systemProxyAtLaunch: true, + httpVersion: "auto", + captureBodyLimit: 10 * 1024 * 1024, + keepTrafficBytes: 2 * 1024 * 1024 * 1024, + startError: null, + encrypting: null, + locked: null, + otherOwner: null, +}; + +function answer(cmd: string, payload: unknown): unknown { + const req = (payload ?? {}) as Record; + switch (cmd) { + case "get_flow_body": { + const fixture = fixtures.find((f) => f.flow.id === req.flowId); + if (fixture == null) throw new Error(`No fixture for ${String(req.flowId)}`); + return req.which === "request" ? fixture.requestBody : fixture.responseBody; + } + case "format_body": { + const text = typeof req.text === "string" ? req.text : ""; + try { + return JSON.stringify(JSON.parse(text), null, 2); + } catch { + return text; + } + } + case "app_icon": + return null; + case "get_proxy_state": + return HARNESS_STATE; + case "get_storage_size": + return { bytes: 412 * 1024 * 1024 }; + case "ca_status": + return { + trusted: true, + installed: true, + certPath: "/Users/you/Library/Application Support/app.yaak.proxy/ca.pem", + }; + case "system_proxy_status": + return { + enabled: true, + port: 9090, + services: [], + backend: "helper", + helper: "enabled", + supported: true, + }; + case "get_decrypt_list": + return { + entries: [ + { entry: "host:api.stripe.com", builtIn: false }, + { entry: "app:Google Chrome", builtIn: false }, + { entry: "!host:*.apple.com", builtIn: true }, + ], + }; + case "list_actions": + return { actions: [] }; + case "list_flows": { + const flows = fixtures.map((f, i) => ({ ...f.flow, number: 1401 + i })).reverse(); + return { flows, total: flows.length, matching: null }; + } + case "list_seen": + return { hosts: [], apps: [] }; + default: + throw new Error(`Not in the harness: ${cmd}`); + } +} diff --git a/apps/yaak-proxy/harness/fixtures.ts b/apps/yaak-proxy/harness/fixtures.ts index b58b87f09..2f052dd74 100644 --- a/apps/yaak-proxy/harness/fixtures.ts +++ b/apps/yaak-proxy/harness/fixtures.ts @@ -51,6 +51,12 @@ const base: Flow = { grpcStatus: null, grpcMessage: null, tunnelled: false, + clientAddr: "127.0.0.1:54321", + elapsedConnectMs: 18, + elapsedTlsMs: 41, + elapsedRequestMs: 43, + upstreamTlsVersion: "TLSv1.3", + upstreamTlsCipher: "TLS13_AES_128_GCM_SHA256", }; const jsonRequest = JSON.stringify({ amount: 4200, currency: "usd", capture: true }); @@ -147,4 +153,26 @@ export const fixtures: FlowFixture[] = [ requestBody: body(""), responseBody: body(""), }, + { + label: "Tunnelled", + flow: { + ...base, + id: "fl_tunnel", + method: "CONNECT", + url: "bank.example.com:443", + status: null, + elapsedHeadersMs: null, + elapsedMs: 4_210, + clientHttpVersion: "HTTP/1.1", + upstreamHttpVersion: null, + clientName: "Safari", + clientBundleId: "com.apple.Safari", + clientPath: "/Applications/Safari.app", + requestSize: 3_412, + responseSize: 48_210, + tunnelled: true, + }, + requestBody: body(""), + responseBody: body(""), + }, ]; diff --git a/apps/yaak-proxy/harness/main.tsx b/apps/yaak-proxy/harness/main.tsx index 85a58af3a..8fa7d599a 100644 --- a/apps/yaak-proxy/harness/main.tsx +++ b/apps/yaak-proxy/harness/main.tsx @@ -3,22 +3,27 @@ import { applyThemeToDocument, defaultDarkTheme, defaultLightTheme } from "@yaak import { createStore, Provider } from "jotai"; import { StrictMode } from "react"; import { createRoot } from "react-dom/client"; +import { installFixtureBackend } from "./backend"; import { FlowPanesHarness } from "./FlowPanesHarness"; import { installLockedBackend, LockedHarness } from "./LockedHarness"; import { MessagesHarness } from "./MessagesHarness"; +import { SettingsHarness } from "./SettingsHarness"; import "../main.css"; const params = new URLSearchParams(location.search); const dark = params.get("theme") !== "light"; applyThemeToDocument(dark ? defaultDarkTheme : defaultLightTheme); if (params.has("locked")) installLockedBackend(); +else installFixtureBackend(); createRoot(document.getElementById("root") as HTMLElement).render( - {params.has("locked") ? ( + {params.has("locked") || params.has("window") ? ( + ) : params.has("settings") ? ( + ) : params.has("stream") ? ( ) : ( diff --git a/apps/yaak-proxy/harness/streamFixtures.ts b/apps/yaak-proxy/harness/streamFixtures.ts index 835b5d3ed..9fcbe3c9a 100644 --- a/apps/yaak-proxy/harness/streamFixtures.ts +++ b/apps/yaak-proxy/harness/streamFixtures.ts @@ -50,6 +50,12 @@ const flowBase: Flow = { grpcStatus: null, grpcMessage: null, tunnelled: false, + clientAddr: null, + elapsedConnectMs: null, + elapsedTlsMs: null, + elapsedRequestMs: null, + upstreamTlsVersion: null, + upstreamTlsCipher: null, }; /** Seconds past 21:41:12 as a stored timestamp */ diff --git a/apps/yaak-proxy/lib/atoms.ts b/apps/yaak-proxy/lib/atoms.ts index 45a2bfc2e..291ddef95 100644 --- a/apps/yaak-proxy/lib/atoms.ts +++ b/apps/yaak-proxy/lib/atoms.ts @@ -17,9 +17,6 @@ export const followingAtom = atom(true); /** When the helper was last registered, so its state is read until approval shows */ export const helperRegisteredAtAtom = atom(null); -/** Settings was asked for, by the header's gear; the settings window opens from it once it exists */ -export const settingsRequestedAtom = atom(0); - /** The dialog that walks through allowing the helper in System Settings */ export const helperDialogOpenAtom = atom(false); diff --git a/apps/yaak-proxy/lib/body.ts b/apps/yaak-proxy/lib/body.ts index 08bca1075..3fdb468bf 100644 --- a/apps/yaak-proxy/lib/body.ts +++ b/apps/yaak-proxy/lib/body.ts @@ -21,6 +21,9 @@ export function flowBodyQueryOptions(flow: Flow, which: FlowBody) { queryFn: async () => viewerBody(await rpc("get_flow_body", { flowId: flow.id, which })), // Reading a stored body fails the same way every time, so the error shows at once. retry: false, + // A stream's body changes with each message, each a new key: dropped once unwatched, so a + // long stream left selected doesn't hold a copy per message + ...(flow.streamKind == null ? {} : { gcTime: 0 }), }); } diff --git a/apps/yaak-proxy/lib/defaultTab.test.ts b/apps/yaak-proxy/lib/defaultTab.test.ts new file mode 100644 index 000000000..b0773c294 --- /dev/null +++ b/apps/yaak-proxy/lib/defaultTab.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, test } from "vite-plus/test"; +import { + defaultTabFor, + flowShape, + learnTab, + REQUEST_TAB_BODY, + REQUEST_TAB_HEADERS, + RESPONSE_TAB_BODY, + RESPONSE_TAB_HEADERS, + RESPONSE_TAB_INFO, + RESPONSE_TAB_MESSAGES, + RESPONSE_TAB_TIMELINE, + type TabMemory, +} from "./defaultTab"; + +type ShapeFlow = Parameters[0]; + +const ok: ShapeFlow = { + streamKind: null, + status: 200, + error: null, + cancelled: false, + requestSize: 0, + responseSize: 120, +}; + +function memory(): TabMemory & { items: Map } { + const items = new Map(); + return { + items, + getItem: (k) => items.get(k) ?? null, + setItem: (k, v) => void items.set(k, v), + }; +} + +const RESPONSE_TABS = [ + RESPONSE_TAB_BODY, + RESPONSE_TAB_HEADERS, + "cookies", + RESPONSE_TAB_TIMELINE, + RESPONSE_TAB_INFO, +]; + +describe("flowShape", () => { + test("names what the flow has", () => { + expect(flowShape(ok)).toBe("response_body"); + expect(flowShape({ ...ok, responseSize: 0, requestSize: 20 })).toBe("request_body"); + expect(flowShape({ ...ok, responseSize: 0 })).toBe("neither"); + expect(flowShape({ ...ok, streamKind: "websocket" })).toBe("stream"); + expect(flowShape({ ...ok, status: null, error: "refused" })).toBe("no_response"); + expect(flowShape({ ...ok, status: null, cancelled: true })).toBe("no_response"); + }); + + test("a body cut after the headers still has its response", () => { + expect(flowShape({ ...ok, error: "reset mid-body" })).toBe("response_body"); + }); +}); + +describe("defaultTabFor", () => { + test("follows the table", () => { + const m = memory(); + // A response body + expect(defaultTabFor("request", ok, [], m)).toBe(REQUEST_TAB_HEADERS); + expect(defaultTabFor("request", { ...ok, requestSize: 9 }, [], m)).toBe(REQUEST_TAB_BODY); + expect(defaultTabFor("response", ok, RESPONSE_TABS, m)).toBe(RESPONSE_TAB_BODY); + // A request body, no response body + const post = { ...ok, requestSize: 30, responseSize: 0 }; + expect(defaultTabFor("request", post, [], m)).toBe(REQUEST_TAB_BODY); + expect(defaultTabFor("response", post, RESPONSE_TABS, m)).toBe(RESPONSE_TAB_HEADERS); + // Neither + const empty = { ...ok, responseSize: 0 }; + expect(defaultTabFor("request", empty, [], m)).toBe(REQUEST_TAB_HEADERS); + expect(defaultTabFor("response", empty, RESPONSE_TABS, m)).toBe(RESPONSE_TAB_HEADERS); + // A stream + const ws = { ...ok, streamKind: "websocket" as const }; + expect(defaultTabFor("response", ws, [...RESPONSE_TABS, RESPONSE_TAB_MESSAGES], m)).toBe( + RESPONSE_TAB_MESSAGES, + ); + // No response at all + const failed = { ...ok, status: null, error: "refused", responseSize: 0 }; + expect(defaultTabFor("response", failed, RESPONSE_TABS, m)).toBe(RESPONSE_TAB_TIMELINE); + expect(defaultTabFor("request", { ...failed, requestSize: 4 }, [], m)).toBe(REQUEST_TAB_BODY); + }); + + test("a tab chosen by hand becomes the default for the same shape and pane", () => { + const m = memory(); + learnTab("response", ok, RESPONSE_TAB_INFO, m); + expect(defaultTabFor("response", ok, RESPONSE_TABS, m)).toBe(RESPONSE_TAB_INFO); + // Not for other shapes, nor the other pane + const post = { ...ok, requestSize: 30, responseSize: 0 }; + expect(defaultTabFor("response", post, RESPONSE_TABS, m)).toBe(RESPONSE_TAB_HEADERS); + expect(defaultTabFor("request", ok, ["body", "headers", "info"], m)).toBe(REQUEST_TAB_HEADERS); + }); + + test("a learned tab the flow doesn't have falls back to the table", () => { + const m = memory(); + const ws = { ...ok, streamKind: "websocket" as const }; + learnTab("response", ws, RESPONSE_TAB_MESSAGES, m); + learnTab("response", ok, RESPONSE_TAB_MESSAGES, m); + expect(defaultTabFor("response", ok, RESPONSE_TABS, m)).toBe(RESPONSE_TAB_BODY); + }); + + test("works without storage", () => { + expect(defaultTabFor("response", ok, RESPONSE_TABS, null)).toBe(RESPONSE_TAB_BODY); + }); +}); diff --git a/apps/yaak-proxy/lib/defaultTab.ts b/apps/yaak-proxy/lib/defaultTab.ts new file mode 100644 index 000000000..14c02d83f --- /dev/null +++ b/apps/yaak-proxy/lib/defaultTab.ts @@ -0,0 +1,97 @@ +import type { Flow } from "@yaakapp-internal/proxy-lib"; + +/** The request half's tabs, named as the shared request pane names them */ +export const REQUEST_TAB_BODY = "body"; +export const REQUEST_TAB_PARAMS = "params"; +export const REQUEST_TAB_HEADERS = "headers"; +export const REQUEST_TAB_AUTH = "auth"; +export const REQUEST_TAB_COOKIES = "cookies"; + +/** The response half's tabs, named as the shared response pane names them */ +export const RESPONSE_TAB_BODY = "body"; +export const RESPONSE_TAB_HEADERS = "headers"; +export const RESPONSE_TAB_COOKIES = "cookies"; +export const RESPONSE_TAB_TIMELINE = "timeline"; +export const RESPONSE_TAB_INFO = "info"; +export const RESPONSE_TAB_MESSAGES = "messages"; + +export type FlowPane = "request" | "response"; + +/** What a flow has, which picks the tab each half opens on */ +export type FlowShape = "response_body" | "request_body" | "neither" | "stream" | "no_response"; + +export function flowShape( + flow: Pick< + Flow, + "streamKind" | "status" | "error" | "cancelled" | "requestSize" | "responseSize" + >, +): FlowShape { + if (flow.streamKind != null) return "stream"; + if (flow.status == null && (flow.error != null || flow.cancelled)) return "no_response"; + if (flow.responseSize > 0) return "response_body"; + if (flow.requestSize > 0) return "request_body"; + return "neither"; +} + +/** The tab a half opens on for a shape, before anything was learned */ +export function shapeDefaultTab(pane: FlowPane, shape: FlowShape, hasRequestBody: boolean): string { + if (pane === "request") { + return hasRequestBody ? REQUEST_TAB_BODY : REQUEST_TAB_HEADERS; + } + switch (shape) { + case "response_body": + return RESPONSE_TAB_BODY; + case "stream": + return RESPONSE_TAB_MESSAGES; + case "no_response": + return RESPONSE_TAB_TIMELINE; + case "request_body": + case "neither": + return RESPONSE_TAB_HEADERS; + } +} + +/** Where the tab last chosen by hand for a shape is kept */ +export function learnedTabKey(pane: FlowPane, shape: FlowShape): string { + return `proxy_learned_tab.${pane}.${shape}`; +} + +/** Where learned tabs are read from and written to: local storage in the app */ +export interface TabMemory { + getItem: (key: string) => string | null; + setItem: (key: string, value: string) => void; +} + +/** + * The tab a half opens on for a flow with no tab of its own: the one last chosen by hand for + * flows of its shape, when this flow has that tab, else the shape's default. + */ +export function defaultTabFor( + pane: FlowPane, + flow: Parameters[0], + available: string[], + memory: TabMemory | null, +): string { + const shape = flowShape(flow); + const learned = memory?.getItem(learnedTabKey(pane, shape)) ?? null; + if (learned != null && available.includes(learned)) return learned; + return shapeDefaultTab(pane, shape, flow.requestSize > 0); +} + +/** A tab chosen by hand becomes the default for flows of the same shape */ +export function learnTab( + pane: FlowPane, + flow: Parameters[0], + tab: string, + memory: TabMemory | null, +) { + memory?.setItem(learnedTabKey(pane, flowShape(flow)), tab); +} + +export function browserTabMemory(): TabMemory | null { + try { + return window.localStorage; + } catch { + return null; + } +} diff --git a/apps/yaak-proxy/lib/env.test.ts b/apps/yaak-proxy/lib/env.test.ts index 0bd24dd08..b7708d7f0 100644 --- a/apps/yaak-proxy/lib/env.test.ts +++ b/apps/yaak-proxy/lib/env.test.ts @@ -1,19 +1,75 @@ import { describe, expect, test } from "vite-plus/test"; -import { environmentExports } from "./env"; +import { defaultShell, environmentBlock, environmentExports } from "./env"; + +const MAC_PATH = "/Users/me/Library/Application Support/app.yaak.proxy/ca/ca.pem"; describe("environmentExports", () => { test("points the proxy variables at the port and the CA variables at the certificate", () => { - const path = "/Users/me/Library/Application Support/app.yaak.proxy/ca/ca.pem"; - expect(environmentExports(9090, path)).toBe( + expect(environmentExports(9090, MAC_PATH)).toBe( [ "export HTTP_PROXY=http://127.0.0.1:9090", "export HTTPS_PROXY=http://127.0.0.1:9090", "export http_proxy=http://127.0.0.1:9090", "export https_proxy=http://127.0.0.1:9090", - `export SSL_CERT_FILE='${path}'`, - `export NODE_EXTRA_CA_CERTS='${path}'`, - `export REQUESTS_CA_BUNDLE='${path}'`, + `export SSL_CERT_FILE='${MAC_PATH}'`, + `export NODE_EXTRA_CA_CERTS='${MAC_PATH}'`, + `export REQUESTS_CA_BUNDLE='${MAC_PATH}'`, ].join("\n"), ); }); }); + +describe("environmentBlock", () => { + test("bash and zsh quote only what needs it", () => { + const lines = environmentBlock("posix", 9090, "/tmp/it's.pem").split("\n"); + expect(lines[0]).toBe("export HTTP_PROXY=http://127.0.0.1:9090"); + expect(lines[4]).toBe(`export SSL_CERT_FILE='/tmp/it'\\''s.pem'`); + }); + + test("fish", () => { + const lines = environmentBlock("fish", 8080, MAC_PATH).split("\n"); + expect(lines[0]).toBe("set -gx HTTP_PROXY http://127.0.0.1:8080"); + expect(lines[6]).toBe(`set -gx REQUESTS_CA_BUNDLE '${MAC_PATH}'`); + expect(environmentBlock("fish", 1, "/a b/it's\\x").split("\n")[4]).toBe( + `set -gx SSL_CERT_FILE '/a b/it\\'s\\\\x'`, + ); + }); + + test("PowerShell", () => { + const path = "C:\\Users\\O'Neil\\AppData\\Roaming\\app.yaak.proxy\\ca.pem"; + const lines = environmentBlock("powershell", 9090, path).split("\n"); + expect(lines[1]).toBe("$env:HTTPS_PROXY = 'http://127.0.0.1:9090'"); + expect(lines[5]).toBe( + "$env:NODE_EXTRA_CA_CERTS = 'C:\\Users\\O''Neil\\AppData\\Roaming\\app.yaak.proxy\\ca.pem'", + ); + }); + + test("cmd", () => { + const path = "C:\\Program Files\\Yaak\\ca.pem"; + const lines = environmentBlock("cmd", 9090, path).split("\n"); + expect(lines[0]).toBe(`set "HTTP_PROXY=http://127.0.0.1:9090"`); + expect(lines[4]).toBe(`set "SSL_CERT_FILE=${path}"`); + }); + + test("names every variable a tool reads", () => { + const block = environmentBlock("posix", 9090, "/ca.pem"); + for (const name of [ + "HTTP_PROXY", + "HTTPS_PROXY", + "SSL_CERT_FILE", + "NODE_EXTRA_CA_CERTS", + "REQUESTS_CA_BUNDLE", + ]) { + expect(block).toContain(`export ${name}=`); + } + }); +}); + +describe("defaultShell", () => { + test("by the platform", () => { + expect(defaultShell("macos")).toBe("posix"); + expect(defaultShell("linux")).toBe("posix"); + expect(defaultShell("windows")).toBe("powershell"); + expect(defaultShell(null)).toBe("posix"); + }); +}); diff --git a/apps/yaak-proxy/lib/env.ts b/apps/yaak-proxy/lib/env.ts index 1b7511fa7..042deb724 100644 --- a/apps/yaak-proxy/lib/env.ts +++ b/apps/yaak-proxy/lib/env.ts @@ -1,21 +1,70 @@ import { shellQuote } from "./curl"; -/** - * Shell exports that send one terminal's traffic through the proxy on `port` and trust its CA at - * `caPath`, for tools that read neither the system proxy nor the login keychain: Node, Python, - * and anything built on OpenSSL. - */ -export function environmentExports(port: number, caPath: string): string { - const proxy = `http://127.0.0.1:${port}`; - const ca = shellQuote(caPath); - return [ - `export HTTP_PROXY=${proxy}`, - `export HTTPS_PROXY=${proxy}`, - // curl reads only the lowercase name for plain HTTP. - `export http_proxy=${proxy}`, - `export https_proxy=${proxy}`, - `export SSL_CERT_FILE=${ca}`, - `export NODE_EXTRA_CA_CERTS=${ca}`, - `export REQUESTS_CA_BUNDLE=${ca}`, - ].join("\n"); +/** The shells the terminal environment is written for */ +export type TerminalShell = "posix" | "fish" | "powershell" | "cmd"; + +export const TERMINAL_SHELLS: { value: TerminalShell; label: string }[] = [ + { value: "posix", label: "bash / zsh" }, + { value: "fish", label: "fish" }, + { value: "powershell", label: "PowerShell" }, + { value: "cmd", label: "cmd" }, +]; + +/** The shell a platform's terminal opens by default */ +export function defaultShell(osType: string | null | undefined): TerminalShell { + return osType === "windows" ? "powershell" : "posix"; +} + +/** + * The variables that send one terminal's traffic through the proxy on `port` and trust its CA + * at `caPath`, for tools that read neither the system proxy nor the login keychain: Node, + * Python, and anything built on OpenSSL. + */ +export function environmentVars(port: number, caPath: string): [string, string][] { + const proxy = `http://127.0.0.1:${port}`; + return [ + ["HTTP_PROXY", proxy], + ["HTTPS_PROXY", proxy], + // curl reads only the lowercase name for plain HTTP. + ["http_proxy", proxy], + ["https_proxy", proxy], + ["SSL_CERT_FILE", caPath], + ["NODE_EXTRA_CA_CERTS", caPath], + ["REQUESTS_CA_BUNDLE", caPath], + ]; +} + +/** The variables as `shell` sets them, one a line */ +export function environmentBlock(shell: TerminalShell, port: number, caPath: string): string { + return environmentVars(port, caPath) + .map(([name, value]) => setLine(shell, name, value)) + .join("\n"); +} + +function setLine(shell: TerminalShell, name: string, value: string): string { + switch (shell) { + case "posix": + return `export ${name}=${SAFE.test(value) ? value : shellQuote(value)}`; + case "fish": + return `set -gx ${name} ${fishQuote(value)}`; + case "powershell": + return `$env:${name} = '${value.replaceAll("'", "''")}'`; + case "cmd": + // The quotes around the pair keep spaces and stop at the closing one, unlike `set A="b"` + return `set "${name}=${value}"`; + } +} + +/** Needs no quotes in any POSIX shell or fish */ +const SAFE = /^[\w@%+=:,./-]+$/; + +/** Fish quotes like the POSIX shells, except a backslash and a quote escape inside single quotes */ +function fishQuote(value: string): string { + if (SAFE.test(value)) return value; + return `'${value.replaceAll("\\", "\\\\").replaceAll("'", "\\'")}'`; +} + +/** Shell exports for bash and zsh, as the palette and Settings copy them */ +export function environmentExports(port: number, caPath: string): string { + return environmentBlock("posix", port, caPath); } diff --git a/apps/yaak-proxy/lib/flowViews.test.ts b/apps/yaak-proxy/lib/flowViews.test.ts index 7e8c35c65..5c7daff3c 100644 --- a/apps/yaak-proxy/lib/flowViews.test.ts +++ b/apps/yaak-proxy/lib/flowViews.test.ts @@ -59,6 +59,12 @@ const flow: Flow = { grpcStatus: null, grpcMessage: null, tunnelled: false, + clientAddr: null, + elapsedConnectMs: null, + elapsedTlsMs: null, + elapsedRequestMs: null, + upstreamTlsVersion: null, + upstreamTlsCipher: null, }; function bytes(text: string): FlowBodyData { diff --git a/apps/yaak-proxy/lib/flowViews.ts b/apps/yaak-proxy/lib/flowViews.ts index f4945e04a..3b0110e96 100644 --- a/apps/yaak-proxy/lib/flowViews.ts +++ b/apps/yaak-proxy/lib/flowViews.ts @@ -15,7 +15,7 @@ import { type HttpResponseView, } from "@yaakapp-internal/ui/httpTypes"; import { useEffect, useState } from "react"; -import { base64ToBytes, bytesToText, findHeader, mimeEssence } from "./body"; +import { base64ToBytes, bodyAsText, findHeader, mimeEssence } from "./body"; import { statusReason } from "./statusReason"; /** A captured body as the engine's body RPC returns it, decoded */ @@ -32,7 +32,8 @@ export function flowBodyFromRpc(res: GetFlowBodyResponse): FlowBodyData { /** A captured request as the shared request pane shows it, read-only */ export function flowRequestToView(flow: Flow, body: FlowBodyData | null): HttpRequestView { const contentType = findHeader(flow.requestHeaders, "content-type"); - const text = body == null ? null : bytesToText(body.bytes, body.truncated); + // In its declared charset, as the response viewers read text + const text = body == null ? null : bodyAsText(body.bytes, body.truncated, contentType); const { bodyType, body: bodyView } = requestBodyView(contentType, body, text); return { @@ -84,7 +85,18 @@ export function flowResponseToView( ): HttpResponseView | null { const ended = flow.elapsedMs != null || flow.error != null || flow.cancelled; if (flow.status == null && !ended) return null; + return responseView(flow, body, ended); +} +/** + * A flow as the shared cookie lists read it, whether or not a response came: the request + * half's Cookies tab shows what was sent before anything was received. + */ +export function flowCookiesView(flow: Flow): HttpResponseView { + return responseView(flow, null, flow.elapsedMs != null || flow.error != null || flow.cancelled); +} + +function responseView(flow: Flow, body: FlowBodyData | null, ended: boolean): HttpResponseView { const encoding = findHeader(flow.responseHeaders, "content-encoding")?.trim().toLowerCase(); const encoded = encoding != null && encoding !== "" && encoding !== "identity"; @@ -133,7 +145,10 @@ export function flowBodyLoader({ return { useText: () => { const body = useBody(); - return { ...body, data: mapData(body, (b) => bytesToText(b.bytes, b.truncated)) }; + return { + ...body, + data: mapData(body, (b) => bodyAsText(b.bytes, b.truncated, contentType)), + }; }, useBytes: () => { const body = useBody(); diff --git a/apps/yaak-proxy/lib/interfaceSettings.ts b/apps/yaak-proxy/lib/interfaceSettings.ts new file mode 100644 index 000000000..0bf77df03 --- /dev/null +++ b/apps/yaak-proxy/lib/interfaceSettings.ts @@ -0,0 +1,32 @@ +import type { SettingsPatch, SettingsValues } from "@yaakapp-internal/ui"; +import { atom } from "jotai"; +import { readSettingsValues, SETTINGS_VALUES_KEY } from "./settings"; + +/** The shared settings tabs' values, kept in local storage */ +const valuesAtom = atom( + readSettingsValues(localStorage.getItem(SETTINGS_VALUES_KEY)), +); + +/** Reads the values, and writes a patch of them through to storage and the window */ +export const settingsValuesAtom = atom( + (get) => get(valuesAtom), + (get, set, patch: SettingsPatch) => { + const next = { ...get(valuesAtom), ...patch }; + set(valuesAtom, next); + localStorage.setItem(SETTINGS_VALUES_KEY, JSON.stringify(next)); + applyFontSizes(next); + if (patch.appearance != null) { + // Loaded on demand: the theme module shows the window as it loads + import("./theme").then((m) => m.setChosenAppearance(next.appearance)).catch(console.error); + } + }, +); + +/** The interface and editor sizes, the editor's through the variable the editors read */ +export function applyFontSizes( + values: Pick, +) { + const root = document.documentElement; + root.style.fontSize = `${values.interfaceFontSize}px`; + root.style.setProperty("--editor-font-size", `${values.editorFontSize}px`); +} diff --git a/apps/yaak-proxy/lib/opener.test.ts b/apps/yaak-proxy/lib/opener.test.ts new file mode 100644 index 000000000..a59946036 --- /dev/null +++ b/apps/yaak-proxy/lib/opener.test.ts @@ -0,0 +1,42 @@ +import { beforeEach, describe, expect, test, vi } from "vite-plus/test"; + +const opener = vi.hoisted(() => ({ + openUrl: vi.fn<(url: string) => Promise>(), + revealItemInDir: vi.fn<(path: string) => Promise>(), +})); +vi.mock("@tauri-apps/plugin-opener", () => opener); + +const { openLink, revealPath } = await import("./opener"); + +describe("opener", () => { + beforeEach(() => { + opener.openUrl.mockReset(); + opener.revealItemInDir.mockReset(); + }); + + test("a folder that can't be shown says so", async () => { + opener.revealItemInDir.mockRejectedValue("not allowed: opener.reveal_item_in_dir"); + const report = vi.fn(); + await revealPath("/data/app.yaak.proxy", report); + expect(opener.revealItemInDir).toHaveBeenCalledWith("/data/app.yaak.proxy"); + expect(report).toHaveBeenCalledWith( + "Failed to show /data/app.yaak.proxy: not allowed: opener.reveal_item_in_dir", + ); + }); + + test("nothing to report when it works", async () => { + opener.revealItemInDir.mockResolvedValue(); + opener.openUrl.mockResolvedValue(); + const report = vi.fn(); + await revealPath("/logs", report); + await openLink("https://yaak.app", report); + expect(report).not.toHaveBeenCalled(); + }); + + test("a link that can't be opened says so", async () => { + opener.openUrl.mockRejectedValue(new Error("denied")); + const report = vi.fn(); + await openLink("https://yaak.app", report); + expect(report).toHaveBeenCalledWith("Failed to open https://yaak.app: Error: denied"); + }); +}); diff --git a/apps/yaak-proxy/lib/opener.ts b/apps/yaak-proxy/lib/opener.ts new file mode 100644 index 000000000..7d653095c --- /dev/null +++ b/apps/yaak-proxy/lib/opener.ts @@ -0,0 +1,22 @@ +import { openUrl, revealItemInDir } from "@tauri-apps/plugin-opener"; + +/** Says what failed, as a toast, rather than leaving a button that does nothing */ +export type ReportFailure = (message: string) => void; + +/** Shows a file or folder in the platform's file manager */ +export async function revealPath(path: string, report: ReportFailure): Promise { + try { + await revealItemInDir(path); + } catch (err) { + report(`Failed to show ${path}: ${String(err)}`); + } +} + +/** Opens a link in the default browser */ +export async function openLink(url: string, report: ReportFailure): Promise { + try { + await openUrl(url); + } catch (err) { + report(`Failed to open ${url}: ${String(err)}`); + } +} diff --git a/apps/yaak-proxy/lib/phases.test.ts b/apps/yaak-proxy/lib/phases.test.ts new file mode 100644 index 000000000..edff97e7f --- /dev/null +++ b/apps/yaak-proxy/lib/phases.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, test } from "vite-plus/test"; +import { flowPhases, type PhaseFlow } from "./phases"; + +const done: PhaseFlow = { + elapsedConnectMs: 12, + elapsedTlsMs: 40, + elapsedRequestMs: 41, + elapsedHeadersMs: 90, + elapsedMs: 120, + status: 200, + error: null, + cancelled: false, + streamKind: null, +}; + +describe("flowPhases", () => { + test("lists each phase with how long it took", () => { + expect(flowPhases(done).map((p) => [p.key, p.atMs, p.tookMs])).toEqual([ + ["start", 0, 0], + ["connect", 12, 12], + ["tls", 40, 28], + ["request", 41, 1], + ["headers", 90, 49], + ["end", 120, 30], + ]); + }); + + test("leaves out phases the engine has no time for", () => { + const phases = flowPhases({ + ...done, + elapsedConnectMs: null, + elapsedTlsMs: null, + elapsedRequestMs: null, + }); + expect(phases.map((p) => p.key)).toEqual(["start", "headers", "end"]); + }); + + test("an open flow has no end yet", () => { + const phases = flowPhases({ ...done, elapsedMs: null }); + expect(phases.at(-1)?.key).toBe("headers"); + }); + + test("a failure ends with the engine's error", () => { + const phases = flowPhases({ + ...done, + elapsedTlsMs: null, + elapsedRequestMs: null, + elapsedHeadersMs: null, + elapsedMs: 31, + status: null, + error: "Connection refused", + }); + expect(phases.at(-1)).toMatchObject({ kind: "error", atMs: 31, detail: "Connection refused" }); + }); + + test("a stream closes", () => { + expect(flowPhases({ ...done, streamKind: "websocket" }).at(-1)?.label).toBe("Closed"); + }); +}); diff --git a/apps/yaak-proxy/lib/phases.ts b/apps/yaak-proxy/lib/phases.ts new file mode 100644 index 000000000..051ed78b7 --- /dev/null +++ b/apps/yaak-proxy/lib/phases.ts @@ -0,0 +1,66 @@ +import type { Flow } from "@yaakapp-internal/proxy-lib"; + +export type PhaseKind = "start" | "step" | "end" | "error" | "cancelled"; + +/** One row of a flow's Timeline: a moment in the exchange, and how long it took to reach it */ +export interface FlowPhase { + key: string; + label: string; + kind: PhaseKind; + /** Milliseconds from the start */ + atMs: number; + /** Milliseconds since the phase before */ + tookMs: number; + detail?: string; +} + +export type PhaseFlow = Pick< + Flow, + | "elapsedHeadersMs" + | "elapsedMs" + | "status" + | "error" + | "cancelled" + | "streamKind" + | "elapsedConnectMs" + | "elapsedTlsMs" + | "elapsedRequestMs" +>; + +/** + * The phases the engine recorded for a flow, in order: started, connected, TLS, request sent, + * headers, and how it ended. A phase the engine has no time for is left out; an open flow has + * no end yet. + */ +export function flowPhases(flow: PhaseFlow): FlowPhase[] { + const steps: Omit[] = [ + { key: "start", label: "Started", kind: "start", atMs: 0 }, + ]; + const add = (key: string, label: string, atMs: number | null | undefined) => { + if (atMs != null) steps.push({ key, label, kind: "step", atMs }); + }; + add("connect", "Connected", flow.elapsedConnectMs); + add("tls", "TLS handshake", flow.elapsedTlsMs); + add("request", "Request sent", flow.elapsedRequestMs); + add( + "headers", + flow.status == null ? "Headers received" : `Headers received, ${flow.status}`, + flow.elapsedHeadersMs, + ); + + if (flow.elapsedMs != null || flow.error != null || flow.cancelled) { + const atMs = flow.elapsedMs ?? steps[steps.length - 1]?.atMs ?? 0; + if (flow.error != null) { + steps.push({ key: "end", label: "Failed", kind: "error", atMs, detail: flow.error }); + } else if (flow.cancelled) { + steps.push({ key: "end", label: "Cancelled", kind: "cancelled", atMs }); + } else { + const label = flow.streamKind == null ? "Body received" : "Closed"; + steps.push({ key: "end", label, kind: "end", atMs }); + } + } + + // Times come from different clocks in the engine; keep them in order regardless + steps.sort((a, b) => a.atMs - b.atMs); + return steps.map((s, i) => ({ ...s, tookMs: i === 0 ? 0 : s.atMs - (steps[i - 1]?.atMs ?? 0) })); +} diff --git a/apps/yaak-proxy/lib/search.ts b/apps/yaak-proxy/lib/search.ts index 39207dde6..8cd2f4046 100644 --- a/apps/yaak-proxy/lib/search.ts +++ b/apps/yaak-proxy/lib/search.ts @@ -5,7 +5,10 @@ import { getDomain } from "tldts"; * A search over every flow in the database: the search box's terms, its field terms, and * whether System's flows are hidden, as `list_flows` takes them. */ -export type FlowQuery = Pick; +export type FlowQuery = Pick< + FlowFilter, + "q" | "under" | "app" | "cwd" | "system" | "tunnelled" | "any" +>; /** A word of the search box: a free term, or `@field:value`, maybe negated with `-` */ export interface SearchToken { @@ -75,12 +78,14 @@ export function parseSearch(text: string, hideSystem = false): FlowQuery | null function parseClause(tokens: SearchToken[], hideSystem: boolean): FlowQuery | null { const words: string[] = []; const apps: string[] = []; + const cwds: string[] = []; // Each host with the paths written after it. const hosts: { host: string; paths: string[] }[] = []; let system: boolean | undefined = hideSystem ? false : undefined; let tunnelled: boolean | undefined; for (const t of tokens) { if (t.field === "app" && !t.negated && t.value !== "") apps.push(t.value); + else if (t.field === "cwd" && !t.negated && t.value !== "") cwds.push(t.value); else if (t.field === "host" && !t.negated && t.value !== "") { hosts.push({ host: t.value.replace(/\/+$/, ""), paths: [] }); } else if (t.field === "path" && !t.negated && t.value !== "") { @@ -107,6 +112,7 @@ function parseClause(tokens: SearchToken[], hideSystem: boolean): FlowQuery | nu ...(q === "" ? {} : { q }), ...(under.length > 0 ? { under } : {}), ...(apps.length > 0 ? { app: apps } : {}), + ...(cwds.length > 0 ? { cwd: cwds } : {}), ...(system == null ? {} : { system }), ...(tunnelled == null ? {} : { tunnelled }), }; @@ -136,19 +142,49 @@ export function widerQueries(text: string): string[] { return [...out]; } -/** The folder a program outside an app ran in, which names it better than its executable */ +/** The folder a program outside an app ran in */ export function folderOf(flow: Pick): string | null { - if (flow.clientBundleId != null || flow.clientCwd == null) return null; - const name = flow.clientCwd.replace(/\/+$/, "").split("/").pop() ?? ""; + if (flow.clientBundleId != null) return null; + return folderName(flow.clientCwd); +} + +/** The last part of a path: `fieldwork` of `/src/fieldwork/` */ +function folderName(path: string | null): string | null { + if (path == null) return null; + const name = + path + .replace(/[/\\]+$/, "") + .split(/[/\\]/) + .pop() ?? ""; return name === "" ? null : name; } +/** Runtimes that run anyone's code, so six rows of `node` say nothing without their folder */ +const GENERIC_RUNTIMES = new Set(["node", "python", "bun", "deno", "ruby", "java", "cargo"]); + +/** A program outside an app that runs other code: `node`, `python3.12`, `java` */ +export function isGenericRuntime(flow: Pick): boolean { + if (flow.clientBundleId != null || flow.clientName == null) return false; + const name = flow.clientName.toLowerCase().replace(/\.exe$/, ""); + return GENERIC_RUNTIMES.has(name.replace(/[\d.]+$/, "")); +} + /** - * What `@app` names a flow's app by, and what the sidebar groups its apps by: an app's name, or - * for a program outside an app the folder it ran in, else its name. Null when unknown. + * The folder that names a generic runtime's row, `node` in `fieldwork`; null for anything + * else, whose folders are rows under it + */ +export function runtimeFolderOf( + flow: Pick, +): string | null { + return isGenericRuntime(flow) ? folderOf(flow) : null; +} + +/** + * What `@app` names a flow's app by, and what the sidebar groups its apps by: an app's name, a + * program's name, or for a generic runtime the folder it ran in. Null when unknown. */ export function appKeyOf(flow: Flow): string | null { - return folderOf(flow) ?? flow.clientName ?? flow.clientBundleId ?? flow.clientPath; + return runtimeFolderOf(flow) ?? flow.clientName ?? flow.clientBundleId ?? flow.clientPath; } const SYSTEM_DIRS = ["/System/", "/usr/libexec/", "/usr/sbin/", "/sbin/", "/Library/Apple/"]; @@ -193,6 +229,7 @@ export function matchesQuery(flow: Flow, query: FlowQuery): boolean { } if (query.under != null && !query.under.some((node) => isUnder(flow.url, node))) return false; if (query.app != null && !query.app.some((app) => matchesApp(flow, app))) return false; + if (query.cwd != null && !query.cwd.some((cwd) => matchesCwd(flow, cwd))) return false; if (query.system != null && isSystemFlow(flow) !== query.system) return false; if (query.tunnelled != null && flow.tunnelled !== query.tunnelled) return false; // One of the groups, each judged whole, as the database judges them. @@ -209,6 +246,12 @@ function matchesApp(flow: Flow, app: string): boolean { return flow.clientName?.toLowerCase() === lower || folderOf(flow)?.toLowerCase() === lower; } +/** By the whole working directory exactly, or its folder's name in any case */ +function matchesCwd(flow: Flow, cwd: string): boolean { + if (flow.clientCwd == null) return false; + return flow.clientCwd === cwd || folderName(flow.clientCwd)?.toLowerCase() === cwd.toLowerCase(); +} + /** `404` exactly, `40` as the start of one, or a class such as `4xx` */ function matchesStatus(status: number | null, term: string): boolean { if (status == null) return false; diff --git a/apps/yaak-proxy/lib/settings.test.ts b/apps/yaak-proxy/lib/settings.test.ts new file mode 100644 index 000000000..158c7dea7 --- /dev/null +++ b/apps/yaak-proxy/lib/settings.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, test } from "vite-plus/test"; +import { + CAPTURE_LIMIT_OPTIONS, + DEFAULT_SETTINGS_VALUES, + KEEP_TRAFFIC_OPTIONS, + proxySettingsView, + readSettingsValues, + sizeFromOptionValue, + sizeOptionsFor, + sizeOptionValue, +} from "./settings"; + +const APP = { + name: "Yaak Proxy", + version: "2026.11.0", + dataDir: "/data", + logDir: "/logs", + revealPathLabel: "Reveal in Finder", +}; + +describe("readSettingsValues", () => { + test("Yaak's defaults when nothing is stored", () => { + expect(readSettingsValues(null)).toEqual(DEFAULT_SETTINGS_VALUES); + expect(readSettingsValues("not json")).toEqual(DEFAULT_SETTINGS_VALUES); + expect(DEFAULT_SETTINGS_VALUES.interfaceFontSize).toBe(15); + expect(DEFAULT_SETTINGS_VALUES.editorFontSize).toBe(13); + }); + + test("stored values over the defaults, the wrong types ignored", () => { + const values = readSettingsValues( + JSON.stringify({ appearance: "dark", interfaceFontSize: "big", editorFont: "Menlo", x: 1 }), + ); + expect(values.appearance).toBe("dark"); + expect(values.interfaceFontSize).toBe(15); + expect(values.editorFont).toBe("Menlo"); + expect("x" in values).toBe(false); + }); +}); + +describe("proxySettingsView", () => { + test("resolves the system appearance and the themes", () => { + const view = proxySettingsView(DEFAULT_SETTINGS_VALUES, APP, "dark"); + expect(view.themes?.resolvedAppearance).toBe("dark"); + expect(view.themes?.active.dark).toBe(true); + const light = proxySettingsView( + { ...DEFAULT_SETTINGS_VALUES, appearance: "light" }, + APP, + "dark", + ); + expect(light.themes?.resolvedAppearance).toBe("light"); + expect(light.themes?.active.dark).toBe(false); + }); + + test("a missing theme falls back to Yaak's", () => { + const view = proxySettingsView({ ...DEFAULT_SETTINGS_VALUES, themeDark: "gone" }, APP, "dark"); + expect(view.themes?.dark.id).toBe(DEFAULT_SETTINGS_VALUES.themeDark); + }); + + test("leaves out what the proxy doesn't have yet", () => { + const view = proxySettingsView(DEFAULT_SETTINGS_VALUES, APP, "light"); + expect(view.updates).toBeNull(); + expect(view.license).toBeNull(); + expect(view.hiddenInterfaceRows).toEqual([ + "editorKeymap", + "editorSoftWrap", + "coloredMethods", + "nativeTitlebar", + ]); + expect(view.app.name).toBe("Yaak Proxy"); + }); +}); + +describe("size options", () => { + test("keep traffic is unlimited by default", () => { + expect(KEEP_TRAFFIC_OPTIONS[0]).toMatchObject({ label: "Unlimited", bytes: null }); + expect(sizeOptionValue(null)).toBe("none"); + expect(sizeFromOptionValue("none")).toBeNull(); + }); + + test("labels in the units the limits are set in", () => { + expect(CAPTURE_LIMIT_OPTIONS.map((o) => o.label)).toEqual([ + "1 MB", + "5 MB", + "10 MB", + "50 MB", + "100 MB", + ]); + expect(KEEP_TRAFFIC_OPTIONS.map((o) => o.label)).toEqual([ + "Unlimited", + "500 MB", + "1 GB", + "2 GB", + "5 GB", + "10 GB", + "20 GB", + ]); + }); + + test("values round trip", () => { + for (const o of [...KEEP_TRAFFIC_OPTIONS, ...CAPTURE_LIMIT_OPTIONS]) { + expect(sizeFromOptionValue(sizeOptionValue(o.bytes))).toBe(o.bytes); + } + expect(sizeFromOptionValue("abc")).toBeNull(); + }); + + test("a size set elsewhere gets its own option, in order", () => { + const options = sizeOptionsFor(CAPTURE_LIMIT_OPTIONS, 3 * 1024 * 1024); + expect(options.map((o) => o.bytes)).toEqual([ + 1024 * 1024, + 3 * 1024 * 1024, + 5 * 1024 * 1024, + 10 * 1024 * 1024, + 50 * 1024 * 1024, + 100 * 1024 * 1024, + ]); + expect(sizeOptionsFor(CAPTURE_LIMIT_OPTIONS, 10 * 1024 * 1024)).toBe(CAPTURE_LIMIT_OPTIONS); + }); +}); diff --git a/apps/yaak-proxy/lib/settings.ts b/apps/yaak-proxy/lib/settings.ts new file mode 100644 index 000000000..7c22db897 --- /dev/null +++ b/apps/yaak-proxy/lib/settings.ts @@ -0,0 +1,168 @@ +import { formatSize } from "@yaakapp-internal/lib/formatSize"; +// The themes alone, not the package index, which reaches for the window as it loads +import { defaultDarkTheme, defaultLightTheme } from "@yaakapp-internal/theme/src/defaultThemes"; +import type { + SettingsAppInfo, + SettingsThemeOption, + SettingsValues, + SettingsView, +} from "@yaakapp-internal/ui"; + +/** Yaak's own defaults for the shared tabs, so the two apps start the same */ +export const DEFAULT_SETTINGS_VALUES: SettingsValues = { + appearance: "system", + themeLight: defaultLightTheme.id, + themeDark: defaultDarkTheme.id, + interfaceFont: null, + interfaceFontSize: 15, + editorFont: null, + editorFontSize: 13, + editorKeymap: "default", + editorSoftWrap: true, + coloredMethods: false, + useNativeTitlebar: false, + hideWindowControls: false, + updateChannel: "stable", + autoupdate: true, + autoDownloadUpdates: true, + checkNotifications: true, + promptFeedback: true, +}; + +/** Where the shared tabs' values are kept in the proxy */ +export const SETTINGS_VALUES_KEY = "proxy_settings_values"; + +/** The stored values over the defaults; anything unreadable or of the wrong type is ignored */ +export function readSettingsValues(raw: string | null): SettingsValues { + if (raw == null) return DEFAULT_SETTINGS_VALUES; + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return DEFAULT_SETTINGS_VALUES; + } + if (parsed == null || typeof parsed !== "object") return DEFAULT_SETTINGS_VALUES; + const out: Record = { ...DEFAULT_SETTINGS_VALUES }; + for (const [key, fallback] of Object.entries(DEFAULT_SETTINGS_VALUES)) { + const value = (parsed as Record)[key]; + const nullable = fallback === null; + if (value === undefined) continue; + if ( + (nullable && (value === null || typeof value === "string")) || + typeof value === typeof fallback + ) { + out[key] = value; + } + } + return out as unknown as SettingsValues; +} + +const THEMES: SettingsThemeOption[] = [defaultDarkTheme, defaultLightTheme].map((t) => ({ + id: t.id, + label: t.label, + dark: t.dark, +})); + +/** The appearance showing, with `system` resolved by the OS's */ +export function resolveAppearance(appearance: string, system: "light" | "dark"): "light" | "dark" { + return appearance === "light" || appearance === "dark" ? appearance : system; +} + +/** + * The proxy's settings as the shared tabs render them. The proxy has Yaak's two themes, no + * installed-font list, updater or feedback prompts yet, and its license comes with the license + * work, so those parts are left out, as are the Interface rows it doesn't act on. + */ +export function proxySettingsView( + values: SettingsValues, + app: SettingsAppInfo, + systemAppearance: "light" | "dark", +): SettingsView { + const light = THEMES.find((t) => t.id === values.themeLight && !t.dark) ?? lightOption(); + const dark = THEMES.find((t) => t.id === values.themeDark && t.dark) ?? darkOption(); + const resolvedAppearance = resolveAppearance(values.appearance, systemAppearance); + return { + app, + values, + themes: { + themes: THEMES, + light, + dark, + active: resolvedAppearance === "dark" ? dark : light, + resolvedAppearance, + }, + fonts: null, + windowControlsSetting: false, + updates: null, + feedbackPrompts: false, + license: null, + // Not acted on in the proxy yet: its editors take no keymap or wrap setting, its list draws + // methods uncoloured, and it always draws its own title bar. + hiddenInterfaceRows: ["editorKeymap", "editorSoftWrap", "coloredMethods", "nativeTitlebar"], + }; +} + +function lightOption(): SettingsThemeOption { + return THEMES.find((t) => !t.dark) as SettingsThemeOption; +} + +function darkOption(): SettingsThemeOption { + return THEMES.find((t) => t.dark) as SettingsThemeOption; +} + +/** A size choice in a select, `null` for no limit */ +export interface SizeOption { + label: string; + value: string; + bytes: number | null; +} + +const MB = 1024 * 1024; +const GB = 1024 * MB; + +function sizeOptions(sizes: (number | null)[], none: string): SizeOption[] { + return sizes.map((bytes) => ({ + label: bytes == null ? none : sizeLabel(bytes), + value: bytes == null ? "none" : `${bytes}`, + bytes, + })); +} + +/** A size in the binary units the limits are set in, so 10 MiB reads "10 MB", not "10.5 MB" */ +export function sizeLabel(bytes: number): string { + if (bytes >= GB && bytes % (GB / 2) === 0) return `${bytes / GB} GB`; + if (bytes >= MB && bytes % MB === 0) return `${bytes / MB} MB`; + return formatSize(bytes); +} + +/** How much captured traffic is kept before the oldest flows are dropped */ +export const KEEP_TRAFFIC_OPTIONS = sizeOptions( + [null, 500 * MB, GB, 2 * GB, 5 * GB, 10 * GB, 20 * GB], + "Unlimited", +); + +/** How much of each body is stored */ +export const CAPTURE_LIMIT_OPTIONS = sizeOptions( + [MB, 5 * MB, 10 * MB, 50 * MB, 100 * MB], + "Unlimited", +); + +/** + * The option a stored size selects. A size set elsewhere (the CLI or `settings.json`) that no + * option has is added as its own, so the select still shows it. + */ +export function sizeOptionsFor(options: SizeOption[], bytes: number | null): SizeOption[] { + if (options.some((o) => o.bytes === bytes)) return options; + const extra = sizeOptions([bytes], "Unlimited"); + return [...options, ...extra].sort((a, b) => (a.bytes ?? Infinity) - (b.bytes ?? Infinity)); +} + +export function sizeOptionValue(bytes: number | null): string { + return bytes == null ? "none" : `${bytes}`; +} + +export function sizeFromOptionValue(value: string): number | null { + if (value === "none") return null; + const n = Number(value); + return Number.isFinite(n) && n > 0 ? n : null; +} diff --git a/apps/yaak-proxy/lib/sidebarTree.test.ts b/apps/yaak-proxy/lib/sidebarTree.test.ts index 986a089a6..002a7bb15 100644 --- a/apps/yaak-proxy/lib/sidebarTree.test.ts +++ b/apps/yaak-proxy/lib/sidebarTree.test.ts @@ -103,6 +103,71 @@ describe("buildSidebarTree", () => { }); }); +describe("programs outside an app", () => { + const claude = (cwd: string) => ({ + clientName: "claude", + clientPath: "/usr/local/bin/claude", + clientCwd: cwd, + }); + const python = { + clientName: "python3.12", + clientPath: "/usr/bin/python3.12", + clientCwd: "/src/ml", + }; + const curl = { clientName: "curl", clientPath: "/usr/bin/curl", clientCwd: null }; + const root = buildSidebarTree( + [ + flow("https://api.anthropic.com/v1/messages", claude("/src/yaak")), + flow("https://api.anthropic.com/v1/models", claude("/src/fieldwork")), + flow("https://statsig.anthropic.com/v1", claude("/src/yaak")), + flow("https://pypi.org/simple", python), + flow("https://example.com/", curl), + flow("https://api.github.com/repos", node), + ], + 6, + ); + const items = itemsById(root); + const apps = (root.children ?? []).find((c) => c.item.id === "apps"); + + test("one row per program, its working directories one level down above the hosts", () => { + expect(shape(apps as TreeNode).slice(0, 12)).toEqual([ + "Apps 6", + " claude 3", + " fieldwork 1", + " anthropic.com 1", + " api.anthropic.com 1", + " /v1 1", + " /models 1", + " yaak 2", + " anthropic.com 2", + " api.anthropic.com 1", + " /v1 1", + " /messages 1", + ]); + }); + + test("generic runtimes keep their folder in the row's name", () => { + const labels = (apps?.children ?? []).map((c) => [c.item.label, c.item.sublabel ?? null]); + expect(labels).toEqual([ + ["claude", null], + ["curl", null], + ["node", "fieldwork"], + ["python3.12", "ml"], + ]); + // A program with no working directory has its hosts straight under it. + expect(items.get("app:curl|site:example.com")?.kind).toBe("site"); + }); + + test("a folder row selects that program in that folder", () => { + const query = items.get("app:claude|cwd:yaak")?.query ?? ""; + expect(query).toBe("@app:claude @cwd:yaak"); + expect(items.get("app:claude|cwd:yaak|host:statsig.anthropic.com")?.query).toBe( + "@app:claude @cwd:yaak @host:statsig.anthropic.com", + ); + expect(parseSearch(query)).toEqual({ app: ["claude"], cwd: ["yaak"] }); + }); +}); + describe("decrypting from the sidebar", () => { const tunnel = flow("https://bank.example.com", { ...slack, diff --git a/apps/yaak-proxy/lib/sidebarTree.ts b/apps/yaak-proxy/lib/sidebarTree.ts index c7a99798d..3e1066689 100644 --- a/apps/yaak-proxy/lib/sidebarTree.ts +++ b/apps/yaak-proxy/lib/sidebarTree.ts @@ -6,19 +6,31 @@ import { folderOf, formatFieldFilter, isSystemFlow, + runtimeFolderOf, sidebarPath, siteOf, tokenize, } from "./search"; -export type SidebarKind = "all" | "tunnelled" | "group" | "app" | "site" | "host" | "path"; +export type SidebarKind = + | "all" + | "tunnelled" + | "group" + | "app" + | "folder" + | "site" + | "host" + | "path"; -/** A row of the sidebar: All traffic, a group, an app, or a site, host or path under one */ +/** + * A row of the sidebar: All traffic, a group, an app, a program's working directory, or a site, + * host or path under one + */ export interface SidebarItem { id: string; kind: SidebarKind; label: string; - /** Dim beside the label: a program's folder */ + /** Dim beside the label: a generic runtime's folder */ sublabel?: string; /** Flows under the row, of those loaded */ count: number; @@ -42,7 +54,7 @@ export const TUNNELLED_ID = "tunnelled"; /** A row a flow sits under, with what makes it, so a rebuild only looks it up */ interface PlaceRow { id: string; - kind: "site" | "host" | "path"; + kind: "folder" | "site" | "host" | "path"; label: string; query: string; decrypt?: DecryptSubject; @@ -50,7 +62,8 @@ interface PlaceRow { /** * Where a flow sits, parsed once per flow rather than on every rebuild: its site, host and path - * rows in Domains, and the same rows under its app. + * rows in Domains, and the same rows under its app, below its working directory for a program + * outside an app that isn't a generic runtime (those are named by their folder instead). */ interface Place { /** What `@app` names its app by, or null when unknown */ @@ -99,10 +112,28 @@ function placeOf(flow: Flow): Place { } const system = isSystemFlow(flow); const appQuery = app == null ? "" : appQueryFor(app, system); - const appRows = - app == null - ? [] - : rows.map((r) => ({ ...r, id: `app:${app}|${r.id}`, query: `${appQuery} ${r.query}` })); + let appRows: PlaceRow[] = []; + if (app != null) { + const folder = runtimeFolderOf(flow) == null ? folderOf(flow) : null; + if (folder == null) { + appRows = rows.map((r) => ({ + ...r, + id: `app:${app}|${r.id}`, + query: `${appQuery} ${r.query}`, + })); + } else { + const folderId = `app:${app}|cwd:${folder}`; + const folderQuery = `${appQuery} ${formatFieldFilter("cwd", folder)}`; + appRows = [ + { id: folderId, kind: "folder", label: folder, query: folderQuery }, + ...rows.map((r) => ({ + ...r, + id: `${folderId}|${r.id}`, + query: `${folderQuery} ${r.query}`, + })), + ]; + } + } place = { app, system, rows, appRows }; places.set(flow, place); return place; @@ -164,8 +195,10 @@ function toNodes( }); } +/** By label, a program's working directories before the sites it reached directly */ function byLabel(a: Trie, b: Trie): number { - return collator.compare(a.item.label, b.item.label); + const folders = Number(b.item.kind === "folder") - Number(a.item.kind === "folder"); + return folders || collator.compare(a.item.label, b.item.label); } function group(id: string, label: string): Trie { @@ -203,7 +236,7 @@ export function buildSidebarTree(flows: Flow[], total: number): TreeNode(null); */ let changesDuringPage: ModelPayload[] | null = null; +/** + * Flows the keep-traffic sweep deleted while a page was being read: the page may be a snapshot + * from before, so it is filtered by these as it lands. Forgotten once no page is being read. + */ +const removedWhileLoading = new Set(); +let pagesLoading = 0; + +function pageStarted() { + pagesLoading++; +} + +function pageSettled() { + pagesLoading = Math.max(0, pagesLoading - 1); + if (pagesLoading === 0) removedWhileLoading.clear(); +} + +/** A page's flows less any the sweep deleted since it was asked for */ +function withoutRemoved(flows: Flow[]): Flow[] { + return removedWhileLoading.size === 0 + ? flows + : flows.filter((f) => !removedWhileLoading.has(f.id)); +} + +/** Changes still to apply, less the ones that would bring back a deleted flow */ +function dropRemovedChanges(changes: ModelPayload[], removed: Set) { + for (let i = changes.length - 1; i >= 0; i--) { + const c = changes[i]; + if (c != null && c.change.type !== "delete" && removed.has(c.model.id)) changes.splice(i, 1); + } +} + /** Loads the page of flows before the oldest one shown, from the search's flows when searching */ export const loadOlderFlowsAtom = atom(null, async (get, set) => { const search = get(searchWindowAtom); @@ -206,12 +237,13 @@ export const loadOlderFlowsAtom = atom(null, async (get, set) => { if (changesDuringPage != null || !hasOlder || first == null) return; const changes: ModelPayload[] = []; changesDuringPage = changes; + pageStarted(); try { const res = await rpc("list_flows", { ...search?.query, before: first.id, limit: PAGE_SIZE }); // Dropped when the window moved on meanwhile, trimmed, cleared or searched again. const merge = (win: FlowWindow) => { if (win.flows[0]?.id !== first.id) return win; - const merged = prependOlder(win, res.flows, PAGE_SIZE); + const merged = prependOlder(win, withoutRemoved(res.flows), PAGE_SIZE); const kept = search == null ? changes : changesFor(search.query, changes); // Counted when they first applied. return { ...applyChanges(merged, kept), total: merged.total }; @@ -225,6 +257,7 @@ export const loadOlderFlowsAtom = atom(null, async (get, set) => { console.warn("Failed to load older flows", err); } finally { changesDuringPage = null; + pageSettled(); } }); @@ -319,21 +352,49 @@ export function syncFlows(store: Store) { load(); }); + // The oldest flows, swept to keep the capture under its size limit: dropped wherever they + // are held, and the totals counted again from the database. + listen("flows_removed", ({ ids }) => { + const removed = new Set(ids); + const drop = (win: FlowWindow) => + win.flows.some((f) => removed.has(f.id)) + ? { ...win, flows: win.flows.filter((f) => !removed.has(f.id)) } + : win; + if (pagesLoading > 0) for (const id of ids) removedWhileLoading.add(id); + dropRemovedChanges(pending, removed); + if (changesDuringPage != null) dropRemovedChanges(changesDuringPage, removed); + store.set(flowWindowAtom, drop); + store.set(searchWindowAtom, (s) => { + if (s == null) return s; + const buffered = [...s.buffered]; + dropRemovedChanges(buffered, removed); + return { ...s, win: drop(s.win), buffered }; + }); + if (removed.has(store.get(selectedFlowAtom)?.id ?? "")) store.set(selectedFlowAtom, null); + if (recountTimer == null) recountTimer = setTimeout(recount, RECOUNT_MS); + }); + function load() { loaded = false; const thisLoad = ++loads; - void rpc("list_flows", { limit: WINDOW_SIZE }).then((res) => { - // A clear since asked for a newer page. - if (thisLoad !== loads) return; - store.set(flowWindowAtom, windowFromPage(res.flows, WINDOW_SIZE, res.total)); - seenId = res.flows[0]?.id ?? null; - loaded = true; - store.set(flowsLoadedAtom, true); - // Flows recorded before this install remembered it mean setup was done. - if (res.total > 0) store.set(systemProxyEverOnAtom, true); - flush(); - recount(); - }); + pageStarted(); + void rpc("list_flows", { limit: WINDOW_SIZE }) + .then((res) => { + // A clear since asked for a newer page. + if (thisLoad !== loads) return; + store.set( + flowWindowAtom, + windowFromPage(withoutRemoved(res.flows), WINDOW_SIZE, res.total), + ); + seenId = res.flows[0]?.id ?? null; + loaded = true; + store.set(flowsLoadedAtom, true); + // Flows recorded before this install remembered it mean setup was done. + if (res.total > 0) store.set(systemProxyEverOnAtom, true); + flush(); + recount(); + }) + .finally(pageSettled); loadSearch(store.get(activeQueryAtom)); } reloadFlows = load; @@ -360,12 +421,16 @@ export function syncFlows(store: Store) { buffered: [], }; }); + pageStarted(); void rpc("list_flows", { ...query, limit: WINDOW_SIZE }) - .then((res) => settle(windowFromPage(res.flows, WINDOW_SIZE, res.matching ?? res.total))) + .then((res) => + settle(windowFromPage(withoutRemoved(res.flows), WINDOW_SIZE, res.matching ?? res.total)), + ) .catch((err) => { console.warn("Failed to search flows", err); settle(null); - }); + }) + .finally(pageSettled); } // The sidebar takes the window's flows at once when it empties or first fills, and at most diff --git a/apps/yaak-proxy/lib/theme.ts b/apps/yaak-proxy/lib/theme.ts index ef1e877bd..a4b7d5535 100644 --- a/apps/yaak-proxy/lib/theme.ts +++ b/apps/yaak-proxy/lib/theme.ts @@ -9,24 +9,35 @@ import { subscribeToPreferredAppearance, type Appearance, } from "@yaakapp-internal/theme"; +import { readSettingsValues, resolveAppearance, SETTINGS_VALUES_KEY } from "./settings"; import { showWindow } from "./tauri"; setPlatformOnDocument(platformFromUserAgent(navigator.userAgent)); +// The appearance chosen in Settings, `system` following the OS's +let chosenAppearance = readSettingsValues(localStorage.getItem(SETTINGS_VALUES_KEY)).appearance; + // Apply a quick initial theme based on CSS media query let preferredAppearance: Appearance = getCSSAppearance(); -applyTheme(preferredAppearance); +applyTheme(); // Then subscribe to accurate OS appearance detection and changes subscribeToPreferredAppearance((a) => { preferredAppearance = a; - applyTheme(preferredAppearance); + applyTheme(); }); // Show window after initial theme is applied (window starts hidden to prevent flash) showWindow().catch(console.error); -function applyTheme(appearance: Appearance) { +/** Applies the appearance chosen in Settings: `light`, `dark` or `system` */ +export function setChosenAppearance(appearance: string) { + chosenAppearance = appearance; + applyTheme(); +} + +function applyTheme() { + const appearance = resolveAppearance(chosenAppearance, preferredAppearance); const theme = appearance === "dark" ? defaultDarkTheme : defaultLightTheme; applyThemeToDocument(theme); if (theme.base.surface != null) { diff --git a/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts b/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts index 70fbac00c..7e5b70058 100644 --- a/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts +++ b/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts @@ -48,7 +48,28 @@ elapsedHeadersMs: number | null, /** * Milliseconds from `started_at` to the end of the response. */ -elapsedMs: number | null, error: string | null, +elapsedMs: number | null, +/** + * Milliseconds from `started_at` to the TCP connection to the server, and to the end of + * its TLS handshake. Only for intercepted HTTPS on a connection opened for this flow. + */ +elapsedConnectMs: number | null, elapsedTlsMs: number | null, +/** + * Milliseconds from `started_at` to the request body being sent whole. None without one. + */ +elapsedRequestMs: number | null, +/** + * The client's end of its connection: `127.0.0.1:54321`. + */ +clientAddr: string | null, +/** + * The TLS negotiated with the server for intercepted HTTPS: `TLSv1.3`. + */ +upstreamTlsVersion: string | null, +/** + * `TLS13_AES_128_GCM_SHA256`. + */ +upstreamTlsCipher: string | null, error: string | null, /** * The client gave up before the response ended, by resetting its HTTP/2 stream or closing * the connection. Not an error. diff --git a/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts b/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts index 302fa98d3..2e70e0589 100644 --- a/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts +++ b/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts @@ -170,6 +170,11 @@ under?: Array, * it ran in (`fieldwork`). */ app?: Array, +/** + * Only flows from a process that ran in one of these working directories: the whole path, + * or its folder's name in any case (`fieldwork`). With `app`, a program in a folder. + */ +cwd?: Array, /** * Only flows from macOS's own daemons (true), or none of them (false), as [`is_system`] * tells them by their executable. @@ -212,6 +217,11 @@ before?: string, }; export type FlowsClearedPayload = Record; +/** + * Flows deleted to keep captured traffic under the keep-traffic cap, the oldest first. + */ +export type FlowsRemovedPayload = { ids: Array, }; + export type FolderSummary = { id: string, name: string, /** * The folder this one is in, if any. @@ -282,6 +292,14 @@ systemProxyAtLaunch: boolean, * The HTTP versions offered to clients. */ httpVersion: HttpVersion, +/** + * Bytes of each body stored per flow. + */ +captureBodyLimit: number, +/** + * The most captured traffic kept, in bytes, or null to keep everything. + */ +keepTrafficBytes: number | null, /** * Why the engine last failed to start, until it starts. */ @@ -301,6 +319,15 @@ locked: ProxyLock | null, */ otherOwner: OtherOwner | null, }; +export type GetStorageSizeRequest = Record; + +export type GetStorageSizeResponse = { +/** + * The databases' used pages, as the keep-traffic cap counts them; locked, their files' + * sizes on disk. + */ +bytes: number, }; + export type GlobalAction = "recording_toggle" | "flows_clear"; export type HelperRegisterRequest = Record; @@ -370,6 +397,12 @@ yaakFound: boolean, workspaces: Array, }; */ export type MessagePayload = { message: FlowMessage, }; +export type OpenTerminalRequest = { +/** + * Environment variables the shell starts with, by name: `HTTPS_PROXY`. + */ +vars: Array<[string, string]>, }; + export type OtherOwner = { owner: Owner, pid: number, startedAt: string, /** * The engine's port while it runs. @@ -412,9 +445,11 @@ host: string, }; export type RemoveGrpcSchemaRequest = { id: string, }; -export type RpcEventSchema = { decrypt_list_changed: DecryptListChangedPayload, flows_cleared: FlowsClearedPayload, message_write: MessagePayload, model_write: ModelPayload, proxy_state_changed: ProxyStatePayload, }; +export type ResetCapturedDataRequest = Record; -export type RpcSchema = { execute_action: [ActionInvocation, boolean], format_body: [FormatBodyRequest, string], get_decrypt_list: [GetDecryptListRequest, DecryptList], get_message_payload: [GetMessagePayloadRequest, GetMessagePayloadResponse], list_actions: [ListActionsRequest, ListActionsResponse], list_flows: [FlowFilter, ListFlowsResponse], list_grpc_schemas: [ListGrpcSchemasRequest, Array], list_messages: [ListMessagesRequest, ListMessagesResponse], remove_grpc_schema: [RemoveGrpcSchemaRequest, boolean], set_recording: [SetRecordingRequest, boolean], set_system_proxy_at_launch: [SetSystemProxyAtLaunchRequest, boolean], set_http_version: [SetHttpVersionRequest, HttpVersion], add_grpc_schema: [AddGrpcSchemaRequest, GrpcSchema], app_icon: [AppIconRequest, string | null], ca_status: [CaStatusRequest, CaStatus], ca_trust: [CaTrustRequest, CaStatus], clear_flows: [ClearFlowsRequest, number], decode_grpc_message: [DecodeGrpcMessageRequest, DecodedGrpcMessage], export_har: [ExportHarRequest, number], filter_body: [FilterBodyRequest, FilterBodyResponse], get_flow_body: [GetFlowBodyRequest, GetFlowBodyResponse], get_proxy_state: [GetProxyStateRequest, GetProxyStateResponse], helper_register: [HelperRegisterRequest, SystemProxyStatus], helper_unregister: [HelperUnregisterRequest, SystemProxyStatus], edit_decrypt_list: [EditDecryptListRequest, DecryptList], list_seen: [ListSeenRequest, SeenTargets], set_decrypt_list: [SetDecryptListRequest, DecryptList], list_workspaces: [ListWorkspacesRequest, ListWorkspacesResponse], preview_save: [PreviewSaveRequest, SavePreview], reflect_grpc_host: [ReflectGrpcHostRequest, GrpcSchema], save_flow: [SaveFlowToYaakRequest, SavedFlow], save_flow_body: [SaveFlowBodyRequest, SaveFlowBodyResponse], system_proxy_set: [SystemProxySetRequest, SystemProxyStatus], system_proxy_status: [SystemProxyStatusRequest, SystemProxyStatus], take_over: [TakeOverRequest, boolean], }; +export type RpcEventSchema = { decrypt_list_changed: DecryptListChangedPayload, flows_cleared: FlowsClearedPayload, flows_removed: FlowsRemovedPayload, message_write: MessagePayload, model_write: ModelPayload, proxy_state_changed: ProxyStatePayload, }; + +export type RpcSchema = { execute_action: [ActionInvocation, boolean], format_body: [FormatBodyRequest, string], get_decrypt_list: [GetDecryptListRequest, DecryptList], get_message_payload: [GetMessagePayloadRequest, GetMessagePayloadResponse], list_actions: [ListActionsRequest, ListActionsResponse], list_flows: [FlowFilter, ListFlowsResponse], list_grpc_schemas: [ListGrpcSchemasRequest, Array], list_messages: [ListMessagesRequest, ListMessagesResponse], remove_grpc_schema: [RemoveGrpcSchemaRequest, boolean], set_recording: [SetRecordingRequest, boolean], set_system_proxy_at_launch: [SetSystemProxyAtLaunchRequest, boolean], set_http_version: [SetHttpVersionRequest, HttpVersion], set_capture_body_limit: [SetCaptureBodyLimitRequest, number], set_keep_traffic: [SetKeepTrafficRequest, number | null], add_grpc_schema: [AddGrpcSchemaRequest, GrpcSchema], app_icon: [AppIconRequest, string | null], ca_status: [CaStatusRequest, CaStatus], ca_trust: [CaTrustRequest, CaStatus], clear_flows: [ClearFlowsRequest, number], decode_grpc_message: [DecodeGrpcMessageRequest, DecodedGrpcMessage], export_har: [ExportHarRequest, number], filter_body: [FilterBodyRequest, FilterBodyResponse], get_flow_body: [GetFlowBodyRequest, GetFlowBodyResponse], get_proxy_state: [GetProxyStateRequest, GetProxyStateResponse], helper_register: [HelperRegisterRequest, SystemProxyStatus], helper_unregister: [HelperUnregisterRequest, SystemProxyStatus], edit_decrypt_list: [EditDecryptListRequest, DecryptList], list_seen: [ListSeenRequest, SeenTargets], set_decrypt_list: [SetDecryptListRequest, DecryptList], list_workspaces: [ListWorkspacesRequest, ListWorkspacesResponse], preview_save: [PreviewSaveRequest, SavePreview], reflect_grpc_host: [ReflectGrpcHostRequest, GrpcSchema], save_flow: [SaveFlowToYaakRequest, SavedFlow], save_flow_body: [SaveFlowBodyRequest, SaveFlowBodyResponse], system_proxy_set: [SystemProxySetRequest, SystemProxyStatus], system_proxy_status: [SystemProxyStatusRequest, SystemProxyStatus], take_over: [TakeOverRequest, boolean], set_port: [SetPortRequest, number], get_storage_size: [GetStorageSizeRequest, GetStorageSizeResponse], reset_captured_data: [ResetCapturedDataRequest, boolean], open_terminal: [OpenTerminalRequest, boolean], }; export type SaveFlowBodyRequest = { flowId: string, which: FlowBody, /** @@ -504,6 +539,8 @@ name: string, */ id?: string, http: ProxySetting, https: ProxySetting, }; +export type SetCaptureBodyLimitRequest = { bytes: number, }; + export type SetDecryptListRequest = { /** * Every entry, in order, as typed: a bare hostname is a `host:` entry. A built-in @@ -513,6 +550,14 @@ entries: Array, }; export type SetHttpVersionRequest = { httpVersion: HttpVersion, }; +export type SetKeepTrafficRequest = { +/** + * Null keeps everything. + */ +bytes: number | null, }; + +export type SetPortRequest = { port: number, }; + export type SetRecordingRequest = { enabled: boolean, }; export type SetSystemProxyAtLaunchRequest = { enabled: boolean, }; diff --git a/crates-proxy/yaak-proxy-lib/migrations/00000010_flow_connection.sql b/crates-proxy/yaak-proxy-lib/migrations/00000010_flow_connection.sql new file mode 100644 index 000000000..d530ba0db --- /dev/null +++ b/crates-proxy/yaak-proxy-lib/migrations/00000010_flow_connection.sql @@ -0,0 +1,8 @@ +-- The client's address, when the request was sent and the upstream connection made, and the +-- TLS it negotiated, for the flow's timeline and details. +ALTER TABLE flows ADD COLUMN client_addr TEXT; +ALTER TABLE flows ADD COLUMN elapsed_connect_ms INTEGER; +ALTER TABLE flows ADD COLUMN elapsed_tls_ms INTEGER; +ALTER TABLE flows ADD COLUMN elapsed_request_ms INTEGER; +ALTER TABLE flows ADD COLUMN upstream_tls_version TEXT; +ALTER TABLE flows ADD COLUMN upstream_tls_cipher TEXT; diff --git a/crates-proxy/yaak-proxy-lib/migrations/00000011_flow_sweep_pending.sql b/crates-proxy/yaak-proxy-lib/migrations/00000011_flow_sweep_pending.sql new file mode 100644 index 000000000..cd64ce42e --- /dev/null +++ b/crates-proxy/yaak-proxy-lib/migrations/00000011_flow_sweep_pending.sql @@ -0,0 +1,7 @@ +-- Flows the keep-traffic sweep deleted whose bodies in `blobs.sqlite` may not be deleted yet: +-- written with the rows' deletion and removed once their bodies are, so a sweep that fails +-- between the two finishes the bodies next time. +CREATE TABLE flow_sweep_pending +( + flow_id TEXT NOT NULL PRIMARY KEY +); diff --git a/crates-proxy/yaak-proxy-lib/src/db.rs b/crates-proxy/yaak-proxy-lib/src/db.rs index 5cf01fcb7..e6835aebf 100644 --- a/crates-proxy/yaak-proxy-lib/src/db.rs +++ b/crates-proxy/yaak-proxy-lib/src/db.rs @@ -165,6 +165,9 @@ fn missing_data_key(data_dir: &Path) -> String { pub struct Stores { pub db: ProxyQueryManager, pub blobs: BlobManager, + /// `blobs.sqlite` and its key, for measuring it: its pools are its manager's own. + blobs_path: PathBuf, + key: Option, } impl Stores { @@ -173,9 +176,26 @@ impl Stores { pub fn open(data_dir: &Path, key: &DatabaseKey) -> std::result::Result { let db = ProxyQueryManager::new(&data_dir.join(PROXY_DB), key) .map_err(|e| format!("Failed to open proxy database: {e}"))?; - let blobs = open_blobs(&data_dir.join(BLOBS_DB), key) + let blobs_path = data_dir.join(BLOBS_DB); + let blobs = open_blobs(&blobs_path, key) .map_err(|e| format!("Failed to open proxy blob store: {e}"))?; - Ok(Self { db, blobs }) + Ok(Self { db, blobs, blobs_path, key: Some(key.clone()) }) + } + + /// Bytes the captured traffic takes, as [`used_bytes`] counts them, in both databases. + pub fn used_bytes(&self) -> std::result::Result { + let fail = |e: rusqlite::Error| format!("Failed to measure captured traffic: {e}"); + let flows = self.db.with_conn(|c| used_bytes(c.conn().resolve())).map_err(fail)?; + let conn = rusqlite::Connection::open_with_flags( + &self.blobs_path, + OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX, + ) + .map_err(fail)?; + if let Some(key) = &self.key { + encryption::apply_key(&conn, key).map_err(fail)?; + } + conn.busy_timeout(Duration::from_millis(5000)).map_err(fail)?; + Ok(flows + used_bytes(&conn).map_err(fail)?) } /// The databases in `data_dir` for reading, with the data key the master key in `keys` @@ -198,10 +218,58 @@ impl Stores { .map_err(|e| format!("Failed to open {}: {e}", db_path.display()))?; let blobs = open_blobs_read_only(&blobs_path, key.as_ref()) .map_err(|e| format!("Failed to open the proxy's blob store: {e}"))?; - Ok(Self { db, blobs }) + Ok(Self { db, blobs, blobs_path, key }) } } +/// Bytes the data in a database takes: its pages less the free ones. SQLite reuses free pages +/// but gives none back to the file without a vacuum, so this shrinks as rows are deleted where +/// the file does not. The WAL is left out, as its pages are copies of ones counted here. +pub fn used_bytes(conn: &rusqlite::Connection) -> rusqlite::Result { + use rusqlite::types::Value; + // SQLCipher answers `page_size` with its own as text. + let pragma = |name: &str| { + let value = conn.query_row(&format!("PRAGMA {name}"), [], |r| r.get::<_, Value>(0))?; + let n = match &value { + Value::Integer(n) => Some(*n), + Value::Text(s) => s.trim().parse().ok(), + _ => None, + }; + n.ok_or_else(|| rusqlite::Error::InvalidColumnType(0, name.into(), value.data_type())) + }; + let used = (pragma("page_count")? - pragma("freelist_count")?) * pragma("page_size")?; + Ok(used.max(0) as u64) +} + +/// The databases' files and what goes with them: their WAL and shared memory. +fn database_files(data_dir: &Path) -> impl Iterator + '_ { + [PROXY_DB, BLOBS_DB] + .into_iter() + .flat_map(|name| ["", "-wal", "-shm"].map(|suffix| format!("{name}{suffix}"))) + .map(move |name| data_dir.join(name)) +} + +/// Bytes the databases' files take on disk, free pages and WAL included, for when they are +/// locked and cannot be measured as [`Stores::used_bytes`] does. +pub fn file_bytes(data_dir: &Path) -> u64 { + database_files(data_dir).filter_map(|p| std::fs::metadata(p).ok()).map(|m| m.len()).sum() +} + +/// Deletes the databases and the data key that encrypts them, for good. Nothing may write to +/// them meanwhile. +pub fn remove_captured(data_dir: &Path) -> std::result::Result<(), String> { + let key = data_key_path(data_dir); + for path in database_files(data_dir).chain([key.with_extension("tmp"), key]) { + match std::fs::remove_file(&path) { + Err(e) if e.kind() != std::io::ErrorKind::NotFound => { + return Err(format!("Failed to delete {}: {e}", path.display())); + } + _ => {} + } + } + Ok(()) +} + /// Either database in `data_dir` is plain, left by a build from before encryption. pub fn needs_encrypting(data_dir: &Path) -> bool { [PROXY_DB, BLOBS_DB] @@ -443,6 +511,7 @@ mod tests { "_sqlx_migrations", "flow_clear_mark", "flow_messages", + "flow_sweep_pending", "flows", "grpc_schemas" ] @@ -530,7 +599,7 @@ mod tests { .unwrap() .map(|v| v.unwrap()) .collect(); - assert_eq!(&applied[applied.len() - 3..], ["00000007", "00000008", "00000009"]); + assert_eq!(&applied[6..], ["00000007", "00000008", "00000009", "00000010", "00000011"]); let mark_columns = columns(&conn, "flow_clear_mark"); assert_eq!(mark_columns, ["id", "max_rowid", "cleaner", "cleaner_at"]); drop(conn); diff --git a/crates-proxy/yaak-proxy-lib/src/exit_gate.rs b/crates-proxy/yaak-proxy-lib/src/exit_gate.rs new file mode 100644 index 000000000..254ded054 --- /dev/null +++ b/crates-proxy/yaak-proxy-lib/src/exit_gate.rs @@ -0,0 +1,137 @@ +//! Keeps background database work from running into SQLCipher's teardown as the process exits. +//! +//! SQLCipher frees its global state (the key shield, its private heap) in an `atexit` handler it +//! registers the first time a database is keyed. A thread still reading or writing a database, +//! or closing its last connection, then crashes inside SQLCipher. The threads this crate starts +//! (the event loop writing flows, deleting cleared flows, keeping traffic under its cap) hold +//! [`hold`] around each unit of work. [`install`], called once a database is open and so after +//! SQLCipher's handler, registers one that runs before it, as handlers run last registered +//! first: it stops any more work starting, waits for work in flight to finish, for a few seconds +//! at most, and lets the process exit with that work parked. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Once, RwLock, RwLockReadGuard, TryLockError}; +use std::time::{Duration, Instant}; + +static GATE: RwLock<()> = RwLock::new(()); +/// Set as the process exits: no more work starts. +static EXITING: AtomicBool = AtomicBool::new(false); +static INSTALL: Once = Once::new(); + +/// The longest the exit handler waits for work in flight before letting the process go on +/// exiting regardless, so a stuck thread never hangs a quit. +const EXIT_WAIT: Duration = Duration::from_secs(5); + +/// Held around a unit of background database work, a batch, closing connections included. Once +/// the process is exiting, this parks the thread for good instead: the work never starts. +pub(crate) fn hold() -> RwLockReadGuard<'static, ()> { + if EXITING.load(Ordering::SeqCst) { + loop { + std::thread::park(); + } + } + GATE.read().unwrap_or_else(std::sync::PoisonError::into_inner) +} + +/// Registers the exit handler, once per process. Call after a database is open, so SQLCipher's +/// own handler is registered first and runs after this one. +pub(crate) fn install() { + INSTALL.call_once(|| { + // SAFETY: `close_gate` is an `extern "C" fn()` that never unwinds. + let failed = unsafe { atexit(close_gate) } != 0; + if failed { + log::warn!("Failed to register the database exit handler"); + } + }); +} + +unsafe extern "C" { + fn atexit(callback: extern "C" fn()) -> std::ffi::c_int; +} + +/// Stops more work starting, then waits for the work in flight to finish its batch and keeps +/// the gate shut for the rest of the process's life. +extern "C" fn close_gate() { + EXITING.store(true, Ordering::SeqCst); + wait_for_work(EXIT_WAIT); +} + +/// Takes the gate for good once no work holds it, or gives up after `wait`. Returns whether it +/// was taken. Polled, as a blocking write can't be given up on; with no new work starting, the +/// work in flight can't starve it. +fn wait_for_work(wait: Duration) -> bool { + let deadline = Instant::now() + wait; + loop { + match GATE.try_write() { + Ok(guard) => { + std::mem::forget(guard); + return true; + } + Err(TryLockError::Poisoned(e)) => { + std::mem::forget(e.into_inner()); + return true; + } + Err(TryLockError::WouldBlock) if Instant::now() < deadline => { + std::thread::sleep(Duration::from_millis(1)); + } + Err(TryLockError::WouldBlock) => return false, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + use std::sync::mpsc::channel; + + const SELF: &str = "exit_gate::tests::exiting_waits_for_work_in_flight_and_starts_no_more"; + + /// A batch in flight finishes before the gate shuts, however busy other work keeps it, and + /// no work starts after. Run in a process of its own, as shutting the gate parks every + /// thread that would hold it, which the other tests' threads would be. + #[test] + fn exiting_waits_for_work_in_flight_and_starts_no_more() { + if std::env::var("YAAK_EXIT_GATE_CHILD").is_err() { + let status = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", SELF, "--test-threads", "1"]) + .env("YAAK_EXIT_GATE_CHILD", "1") + .status() + .unwrap(); + assert!(status.success()); + return; + } + + // Busy work, taking the gate again as soon as it lets go, as the event loop does. + let busy = Arc::new(AtomicBool::new(false)); + for _ in 0..4 { + let busy = busy.clone(); + std::thread::spawn(move || { + loop { + let _held = hold(); + busy.store(true, Ordering::SeqCst); + } + }); + } + // A batch in flight, a slow one. + let (holding_tx, holding_rx) = channel(); + let (finished_tx, finished_rx) = channel(); + std::thread::spawn(move || { + let _batch = hold(); + holding_tx.send(()).unwrap(); + std::thread::sleep(Duration::from_millis(200)); + finished_tx.send(Instant::now()).unwrap(); + }); + holding_rx.recv().unwrap(); + while !busy.load(Ordering::SeqCst) { + std::hint::spin_loop(); + } + + EXITING.store(true, Ordering::SeqCst); + assert!(wait_for_work(Duration::from_secs(5)), "not starved by the busy work"); + let finished = finished_rx.try_recv().expect("the batch finished before the gate shut"); + assert!(finished <= Instant::now()); + // Nothing holds the gate after, the busy work included: it is all parked. + assert!(GATE.try_read().is_err(), "shut for good"); + } +} diff --git a/crates-proxy/yaak-proxy-lib/src/flows.rs b/crates-proxy/yaak-proxy-lib/src/flows.rs index 11bae5f43..619fddb66 100644 --- a/crates-proxy/yaak-proxy-lib/src/flows.rs +++ b/crates-proxy/yaak-proxy-lib/src/flows.rs @@ -1,5 +1,5 @@ use crate::bodies; -use crate::db::ProxyQueryManager; +use crate::db::{ProxyQueryManager, Stores}; use crate::models::{Flow, FlowBody}; use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64; @@ -31,6 +31,10 @@ pub struct FlowFilter { /// it ran in (`fieldwork`). #[ts(optional)] pub app: Option>, + /// Only flows from a process that ran in one of these working directories: the whole path, + /// or its folder's name in any case (`fieldwork`). With `app`, a program in a folder. + #[ts(optional)] + pub cwd: Option>, /// Only flows from macOS's own daemons (true), or none of them (false), as [`is_system`] /// tells them by their executable. #[ts(optional)] @@ -65,8 +69,8 @@ pub struct FlowFilter { } impl FlowFilter { - /// The filter from a query string's `q`, `host`, `under`, `app`, `system`, `method`, - /// `status`, `limit`, `after` and `before`. `under` and `app` may repeat. + /// The filter from a query string's `q`, `host`, `under`, `app`, `cwd`, `system`, `method`, + /// `status`, `limit`, `after` and `before`. `under`, `app` and `cwd` may repeat. pub fn from_query(query: &str) -> Result { let mut filter = Self::default(); for (key, value) in url::form_urlencoded::parse(query.as_bytes()) { @@ -76,6 +80,7 @@ impl FlowFilter { "host" => filter.host = value, "under" => filter.under.get_or_insert_default().extend(value), "app" => filter.app.get_or_insert_default().extend(value), + "cwd" => filter.cwd.get_or_insert_default().extend(value), "system" => { filter.system = value .map(|v| v.parse().map_err(|_| format!("Invalid system {v:?}"))) @@ -116,7 +121,11 @@ impl FlowFilter { query.append_pair(key, value); } } - for (key, values) in [("under", &self.under), ("app", &self.app)] { + for (key, values) in [ + ("under", &self.under), + ("app", &self.app), + ("cwd", &self.cwd), + ] { for value in values.iter().flatten() { query.append_pair(key, value); } @@ -144,6 +153,7 @@ impl FlowFilter { || given(&self.status) || self.under.is_some() || self.app.is_some() + || self.cwd.is_some() || self.system.is_some() || self.tunnelled.is_some() || self.any.as_ref().is_some_and(|groups| groups.iter().any(FlowFilter::narrows)) @@ -224,6 +234,19 @@ impl FlowFilter { } sql.push(')'); } + if let Some(cwds) = &self.cwd { + if has_cwd && !cwds.is_empty() { + let marks = vec!["?"; cwds.len()].join(", "); + sql.push_str(&format!( + " AND (client_cwd IN ({marks}) OR lower(folder_name(client_cwd)) IN ({marks}))" + )); + params.extend(cwds.iter().map(|c| c.clone().into())); + params.extend(cwds.iter().map(|c| c.to_lowercase().into())); + } else { + // None, or a database from before the column, which has no folders. + sql.push_str(" AND 0"); + } + } match self.system { Some(true) => sql.push_str(" AND is_system(client_bundle_id, client_path)"), Some(false) => sql.push_str(" AND NOT is_system(client_bundle_id, client_path)"), @@ -665,6 +688,142 @@ pub fn clear_mark_in(db: &ProxyQueryManager) -> i64 { db.with_conn(|c| clear_mark(c.conn().resolve())) } +/// When captured traffic takes more than `cap` bytes, as [`Stores::used_bytes`] counts them, +/// deletes the flows written first, with their bodies and messages: as many as the average flow +/// says would bring it under, from one to [`DELETE_BATCH`]. Returns the ids deleted, none once it +/// is under or no flow is left, so a caller sweeps a batch per call until then. Cleared flows are +/// left to [`delete_cleared`], and a stream still open is never swept: its writer would go on +/// adding messages to it. +pub fn sweep_oldest(stores: &Stores, cap: u64) -> Result, String> { + // Bodies a sweep that failed partway left behind go first: they count against the cap. + let pending = pending_sweeps(stores)?; + delete_swept_bodies(stores, &pending)?; + let used = stores.used_bytes()?; + if used <= cap { + return Ok(Vec::new()); + } + let count = count(&stores.db)? as u64; + if count == 0 { + return Ok(Vec::new()); + } + let average = (used / count).max(1); + let batch = (used - cap).div_ceil(average).clamp(1, DELETE_BATCH as u64); + let rows = sweepable(stores, batch as usize)?; + let deleted = delete_swept_rows(stores, rows)?; + delete_swept_bodies(stores, &deleted)?; + Ok(deleted) +} + +/// Flows whose rows a sweep deleted and whose bodies it may not have, at most a batch of them. +pub(crate) fn pending_sweeps(stores: &Stores) -> Result, String> { + stores + .db + .with_conn(|c| { + let conn = c.conn().resolve(); + let mut stmt = conn.prepare("SELECT flow_id FROM flow_sweep_pending LIMIT ?")?; + stmt.query_map([DELETE_BATCH as i64], |r| r.get(0))?.collect::>() + }) + .map_err(sweep_failed) +} + +/// A stream still open: it has no end yet, and its writer goes on adding messages to it. +const OPEN_STREAM: &str = + "stream_kind IS NOT NULL AND elapsed_ms IS NULL AND error IS NULL AND NOT cancelled"; + +/// Fail messages for sweeping, all of a kind. +fn sweep_failed(e: impl std::fmt::Display) -> String { + format!("Failed to delete the oldest flows: {e}") +} + +/// Up to `limit` of the oldest flows that are neither cleared nor an open stream, by rowid. +pub(crate) fn sweepable(stores: &Stores, limit: usize) -> Result, String> { + stores + .db + .with_conn(|c| { + let conn = c.conn().resolve(); + let mut stmt = conn.prepare(&format!( + "SELECT rowid, id FROM flows WHERE rowid > ? + AND NOT ({OPEN_STREAM}) + ORDER BY rowid LIMIT ?" + ))?; + stmt.query_map(rusqlite::params![clear_mark(conn), limit as i64], |r| { + Ok((r.get(0)?, r.get(1)?)) + })? + .collect::>() + }) + .map_err(sweep_failed) +} + +/// Deletes the flows' rows and their messages' rows in one transaction, first, so nothing +/// written for them after this has an owner left to count under, and records them as pending +/// until their bodies go, so a failure before then is finished by the next sweep. A flow that +/// changed or was cleared since it was chosen is kept. Returns the ids deleted. +pub(crate) fn delete_swept_rows( + stores: &Stores, + rows: Vec<(i64, String)>, +) -> Result, String> { + stores + .db + .with_tx(|c| { + let conn = c.conn().resolve(); + let mut delete = conn.prepare(&format!( + "DELETE FROM flows WHERE rowid = ? AND id = ? + AND NOT ({OPEN_STREAM})" + ))?; + let mut delete_messages = + conn.prepare("DELETE FROM flow_messages WHERE flow_id = ?")?; + let mut pending = + conn.prepare("INSERT OR IGNORE INTO flow_sweep_pending (flow_id) VALUES (?)")?; + let mut deleted = Vec::with_capacity(rows.len()); + for (rowid, id) in rows { + if delete.execute(rusqlite::params![rowid, id])? > 0 { + delete_messages.execute([&id])?; + pending.execute([&id])?; + deleted.push(id); + } + } + Ok::<_, yaak_database::Error>(deleted) + }) + .map_err(sweep_failed) +} + +/// Deletes the bodies and message payloads of flows whose rows are gone, then once more the +/// message rows: anything a writer added for them while the rows went is an orphan, which +/// would otherwise go on counting against the cap with nothing to sweep it by. The flows stop +/// being pending once this is done. +pub(crate) fn delete_swept_bodies(stores: &Stores, ids: &[String]) -> Result<(), String> { + if ids.is_empty() { + return Ok(()); + } + stores + .blobs + .with_tx(|b| { + for id in ids { + b.delete_chunks(&Flow::body_id(id, FlowBody::Request))?; + b.delete_chunks(&Flow::body_id(id, FlowBody::Response))?; + b.delete_chunks_like(&format!("{id}.msg.%"))?; + } + Ok::<_, yaak_models::error::Error>(()) + }) + .map_err(|e| sweep_failed(format!("flow bodies: {e}")))?; + stores + .db + .with_tx(|c| { + let conn = c.conn().resolve(); + let mut orphans = conn.prepare( + "DELETE FROM flow_messages WHERE flow_id = ? + AND NOT EXISTS (SELECT 1 FROM flows WHERE flows.id = flow_messages.flow_id)", + )?; + let mut done = conn.prepare("DELETE FROM flow_sweep_pending WHERE flow_id = ?")?; + for id in ids { + orphans.execute([id])?; + done.execute([id])?; + } + Ok::<_, yaak_database::Error>(()) + }) + .map_err(sweep_failed) +} + /// A flow by id, with its number. None once it is cleared, though its row may not be deleted /// yet. pub fn find(db: &ProxyQueryManager, flow_id: &str) -> Option { @@ -1014,6 +1173,42 @@ mod tests { assert_eq!(app(&["com.google.Chrome", "/usr/bin/curl"]), 3); } + /// A program in a folder: its working directory by the whole path, or its folder's name in + /// any case, along with whatever else the filter asks. + #[test] + fn cwd_matches_a_path_or_a_folder_name() { + let node_in = |cwd: Option<&str>| Flow { + client_name: Some("node".into()), + client_path: Some("/usr/local/bin/node".into()), + client_cwd: cwd.map(String::from), + ..flow("https://example.com/", "GET", Some(200), None) + }; + let (_dir, db) = db_of(vec![ + node_in(Some("/Users/ada/Workspace/fieldwork")), + node_in(Some("/Users/ada/Workspace/fieldwork")), + node_in(Some("/Users/ada/Workspace/Ledger")), + node_in(None), + Flow { client_cwd: Some("/tmp/fieldwork".into()), ..node_in(None) }, + Flow { client_path: Some("/usr/bin/curl".into()), ..node_in(Some("/tmp/ledger")) }, + ]); + let count = |app: Option<&str>, cwds: &[&str]| { + let cwd = Some(cwds.iter().map(|c| c.to_string()).collect()); + let app = app.map(|a| vec![a.to_string()]); + urls(&db, FlowFilter { app, cwd, ..Default::default() }).len() + }; + assert_eq!(count(None, &["/Users/ada/Workspace/fieldwork"]), 2, "the whole path"); + assert_eq!(count(None, &["/users/ada/workspace/fieldwork"]), 0, "a path exactly"); + assert_eq!(count(None, &["FieldWork"]), 3, "a folder name, any case"); + assert_eq!(count(None, &["fieldwork", "ledger"]), 5, "any of them"); + assert_eq!(count(Some("/usr/local/bin/node"), &["ledger"]), 1, "and the program"); + assert_eq!(count(Some("/usr/bin/curl"), &["ledger"]), 1); + assert_eq!(count(None, &[]), 0); + let query = FlowFilter { cwd: Some(vec!["a".into(), "/b".into()]), ..Default::default() }; + assert_eq!(query.to_query(), "cwd=a&cwd=%2Fb"); + assert_eq!(FlowFilter::from_query(&query.to_query()).unwrap(), query); + assert!(query.narrows()); + } + #[test] fn a_search_pages_back_and_counts_what_it_matches() { // Every third flow is a POST, newest last. @@ -1460,4 +1655,54 @@ mod tests { ("/wD+", BodyEncoding::Base64, 3) ); } + + /// Sweeping takes the flows written first, with their bodies, until the traffic is under the + /// cap, and no more; under it, nothing goes. + #[test] + fn sweeping_deletes_the_oldest_until_under_the_cap() { + use yaak_database::UpdateSource; + use yaak_models::blob_manager::BodyChunkWriter; + const BODY: usize = 64 * 1024; + let dir = tempfile::tempdir().unwrap(); + let stores = Stores::open(dir.path(), &crate::db::test_key()).unwrap(); + let empty = stores.used_bytes().unwrap(); + let ids: Vec = (0..20).map(|i| format!("fl_{i:02}")).collect(); + for id in &ids { + let flow = Flow { id: id.clone(), ..Default::default() }; + stores.db.with_tx(|tx| tx.upsert(&flow, &UpdateSource::Background)).unwrap(); + let body_id = Flow::body_id(id, FlowBody::Response); + let mut writer = BodyChunkWriter::new(stores.blobs.clone(), body_id); + writer.write(&[7; BODY]).unwrap(); + writer.finish().unwrap(); + } + let full = stores.used_bytes().unwrap(); + assert!(full > empty + (20 * BODY) as u64, "{empty} {full}"); + + assert_eq!(sweep_oldest(&stores, full).unwrap(), Vec::::new(), "under the cap"); + + let cap = (empty + full) / 2; + let mut removed = Vec::new(); + loop { + let swept = sweep_oldest(&stores, cap).unwrap(); + if swept.is_empty() { + break; + } + removed.extend(swept); + } + assert!(stores.used_bytes().unwrap() <= cap); + // About half went, oldest first, and no more than had to. + assert_eq!(removed, ids[..removed.len()]); + assert!((8..=11).contains(&removed.len()), "{removed:?}"); + let kept: Vec = list(&stores.db, &FlowFilter::default()) + .unwrap() + .into_iter() + .rev() + .map(|f| f.id) + .collect(); + assert_eq!(kept, ids[removed.len()..]); + let bodies = stores.blobs.connect(); + let has_body = |id: &str| bodies.body_exists(&Flow::body_id(id, FlowBody::Response)); + assert!(removed.iter().all(|id| !has_body(id).unwrap())); + assert!(kept.iter().all(|id| has_body(id).unwrap())); + } } diff --git a/crates-proxy/yaak-proxy-lib/src/lib.rs b/crates-proxy/yaak-proxy-lib/src/lib.rs index 5411f408b..80e16fb68 100644 --- a/crates-proxy/yaak-proxy-lib/src/lib.rs +++ b/crates-proxy/yaak-proxy-lib/src/lib.rs @@ -6,6 +6,7 @@ pub mod client; pub mod control; pub mod convert; pub mod db; +mod exit_gate; pub mod flows; pub mod grpc; pub mod har; @@ -15,6 +16,7 @@ pub mod messages; pub mod models; pub mod save; pub mod settings; +pub mod terminal; use crate::actions::{ActionInvocation, ActionMetadata, GlobalAction}; use crate::attribution::Attribution; @@ -40,7 +42,7 @@ use std::io::Write; use std::net::{SocketAddr, TcpStream}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::{Arc, Mutex, OnceLock, PoisonError, RwLock}; +use std::sync::{Arc, Condvar, Mutex, OnceLock, PoisonError, RwLock, Weak}; use std::time::{Duration, SystemTime}; use ts_rs::TS; pub use yaak_crypto::MasterKeySource; @@ -97,6 +99,11 @@ pub struct ProxyCtx { stores: Arc>>, /// Why they could not be unlocked, the last time it was tried. lock_reason: Arc>>, + /// Resets of the captured data so far, so a thread deleting from databases since deleted + /// stops. + resets: Arc, + /// The thread keeping captured traffic under the keep-traffic cap, while there is one. + sweeper: Arc<(Mutex, Condvar)>, pub events: RpcEventEmitter, /// gRPC schemas as decoded, and the protoc that compiles attached protos. pub grpc: GrpcState, @@ -137,6 +144,8 @@ impl ProxyCtx { stops: Arc::new(AtomicU64::new(0)), stores: Arc::new(RwLock::new(None)), lock_reason: Arc::new(Mutex::new(None)), + resets: Arc::new(AtomicU64::new(0)), + sweeper: Default::default(), events, grpc: GrpcState::default(), }; @@ -243,6 +252,8 @@ impl ProxyCtx { converted?; } let stores = Stores::open(&self.data_dir, &db_key)?; + // SQLCipher's exit handler is registered by now, so the gate's runs before it. + exit_gate::install(); *self.stores.write().unwrap_or_else(PoisonError::into_inner) = Some(stores); *unlocked = Some(key.clone()); drop(unlocked); @@ -253,6 +264,7 @@ impl ProxyCtx { if self.db().is_ok_and(|db| db.with_conn(|c| flows::clear_mark(c.conn().resolve())) > 0) { self.delete_cleared(); } + self.sweep_soon(); Ok(key) } @@ -336,7 +348,13 @@ impl ProxyCtx { /// another process owns the engine, so there are never two. A failure is kept until the /// next start, for the app to show. pub fn start_on(&self, port: Option) -> Result<(), String> { - let started = self.start_engine(port); + self.start_routing(port, None) + } + + /// [`Self::start_on`], the system proxy pointed at the engine when `system_proxy` says so, + /// or as the settings ask at launch when it is None. + fn start_routing(&self, port: Option, system_proxy: Option) -> Result<(), String> { + let started = self.start_engine(port, system_proxy); if let Ok(mut start_error) = self.start_error.lock() { *start_error = started.as_ref().err().cloned(); } @@ -352,7 +370,7 @@ impl ProxyCtx { self.start_error.lock().ok()?.clone() } - fn start_engine(&self, port: Option) -> Result<(), String> { + fn start_engine(&self, port: Option, system_proxy: Option) -> Result<(), String> { if let Some(owner) = control::live_owner(&self.data_dir) && owner.pid != std::process::id() { @@ -403,6 +421,7 @@ impl ProxyCtx { attribution, recording: self.recording.clone(), finished: Default::default(), + clock: BatchClock::Real, }; std::thread::spawn(move || run_event_loop(event_rx, sink)); } @@ -414,10 +433,11 @@ impl ProxyCtx { if let Ok(mut last_port) = self.last_port.lock() { *last_port = Some(bound); } - self.system_proxy_on.store(settings.system_proxy_at_launch, Ordering::SeqCst); - if settings.system_proxy_at_launch { + let route = system_proxy.unwrap_or(settings.system_proxy_at_launch); + self.system_proxy_on.store(route, Ordering::SeqCst); + if route { // The password dialog can sit open for a while; the engine is up regardless. - let ctx = self.clone(); + let ctx = self.detached(); std::thread::spawn(move || { if let Err(e) = ctx.change_system_proxy(true, bound, true) { warn!("Failed to turn on the system proxy: {e}"); @@ -463,7 +483,7 @@ impl ProxyCtx { self.system_proxy_on.store(false, Ordering::SeqCst); let port = self.take_handle()?; if let Some(port) = port { - let ctx = self.clone(); + let ctx = self.detached(); std::thread::spawn(move || { if let Err(e) = ctx.change_system_proxy(false, port, true) { warn!("Failed to turn off the system proxy: {e}"); @@ -533,7 +553,7 @@ impl ProxyCtx { let mut ports: Vec = previous.and_then(|p| p.proxy_port).into_iter().collect(); ports.extend(self.settings().ok().map(|s| s.port)); - let ctx = self.clone(); + let ctx = self.detached(); std::thread::spawn(move || { if let Err(e) = ctx.reset_stale_system_proxy(&ports) { warn!("Failed to reset a stale system proxy: {e}"); @@ -686,7 +706,9 @@ impl ProxyCtx { pub fn set_system_proxy_at_launch(&self, enabled: bool) -> Result<(), String> { let mut settings = self.settings()?; settings.system_proxy_at_launch = enabled; - settings.save(&self.settings_path()) + settings.save(&self.settings_path())?; + self.emit_state(); + Ok(()) } /// The HTTP versions offered to clients, saved in `settings.json` and applied to the running @@ -700,9 +722,257 @@ impl ProxyCtx { { handle.set_http2(version == HttpVersion::Auto); } + self.emit_state(); Ok(()) } + /// The port the engine starts on, saved in `settings.json`. A running engine is restarted on + /// it at once, the system proxy following. When it cannot bind there, the port before is put + /// back and the engine restarted on it, and why is returned. Another process's engine takes + /// it at its next start. + pub fn set_port(&self, port: u16) -> Result { + if port == 0 { + return Err("The port must be from 1 to 65535".to_string()); + } + let mut settings = self.settings()?; + let previous = settings.port; + settings.port = port; + settings.save(&self.settings_path())?; + if self.port().is_none_or(|running| running == port) { + self.emit_state(); + return Ok(port); + } + // Read once: stopping clears it, and a failed start would leave the rollback reading + // the cleared value, so the old port would come back without the system proxy. + let proxy_on = self.system_proxy_on.load(Ordering::SeqCst); + if let Err(e) = self.restart(proxy_on) { + settings.port = previous; + settings.save(&self.settings_path())?; + if let Err(e) = self.restart(proxy_on) { + warn!("Failed to restart the proxy on port {previous}: {e}"); + } + return Err(e); + } + Ok(port) + } + + /// Stops the engine and starts it again as `settings.json` has it, routing the system proxy + /// to it only when `proxy_on`, as it was before the first stop: a restart keeps the routing + /// as the user left it rather than applying the launch setting again. + fn restart(&self, proxy_on: bool) -> Result<(), String> { + self.stop()?; + self.start_routing(None, Some(proxy_on)) + } + + /// Bytes of each body stored per flow, saved in `settings.json` and applied to the running + /// engine's bodies from now on. + pub fn set_capture_body_limit(&self, bytes: usize) -> Result { + let mut settings = self.settings()?; + settings.capture_body_limit = bytes; + settings.save(&self.settings_path())?; + if let Ok(handle) = self.handle.lock() + && let Some(handle) = handle.as_ref() + { + handle.set_capture_body_limit(bytes); + } + self.emit_state(); + Ok(bytes) + } + + /// The most captured traffic kept, saved in `settings.json`, or None to keep everything. + /// Swept down to it soon after. + pub fn set_keep_traffic(&self, bytes: Option) -> Result, String> { + if bytes == Some(0) { + return Err("Keep more than 0 bytes of traffic, or all of it".to_string()); + } + let mut settings = self.settings()?; + settings.keep_traffic_bytes = bytes; + settings.save(&self.settings_path())?; + self.sweep_soon(); + self.emit_state(); + Ok(bytes) + } + + /// Bytes the captured traffic takes, as the keep-traffic cap counts them: the databases' + /// used pages, so it shrinks as flows are deleted though the files do not. Locked, the + /// files' sizes instead, as nothing in them can be read. + pub fn storage_size(&self) -> Result { + match self.stores() { + Ok(stores) => stores.used_bytes(), + Err(_) => Ok(db::file_bytes(&self.data_dir)), + } + } + + /// Deletes every flow, body and message for good, with the data key that encrypts them, and + /// opens new databases under a new one, so the app goes on, empty. Works while locked, as + /// the way out of a key the keychain lost. The master key in the keychain stays: it wraps + /// only the data key, and wraps the new one, or a new one is made when it is gone. The CA is + /// kept, its key encrypted again, while the old key can read it; locked, it goes too, and a + /// new one has to be trusted. Refused while another process owns the engine, as it has the + /// databases open. A running engine is restarted on the new ones. + pub fn reset_captured_data(&self) -> Result<(), String> { + if let Some(owner) = control::live_owner(&self.data_dir) + && owner.pid != std::process::id() + { + return Err(format!("{}. Quit it first.", control::already_owned(&owner))); + } + let running = self.port().is_some(); + let proxy_on = self.system_proxy_on.load(Ordering::SeqCst); + if running { + self.stop()?; + } + { + // Held so nothing unlocks the old databases meanwhile. + let mut unlocked = self.unlocked.lock().unwrap_or_else(PoisonError::into_inner); + let old_key = unlocked.take(); + *self.stores.write().unwrap_or_else(PoisonError::into_inner) = None; + self.resets.fetch_add(1, Ordering::SeqCst); + *self.deleting_cleared.lock().unwrap_or_else(PoisonError::into_inner) = false; + + let ca_key = self.ca_dir().join("ca.key"); + let ca_pem = old_key.and_then(|key| match std::fs::read(&ca_key) { + Ok(stored) => key.decrypt(&stored).ok(), + Err(_) => None, + }); + // The event loop and a clear's batch let go of the databases moments after the + // engine stops; until then Windows refuses to delete files still open. + if let Err(e) = remove_captured_when_closed(&self.data_dir) { + drop(unlocked); + // What is left opens again, so the app is not left without its databases. + if let Err(e) = self.unlock() { + warn!("Failed to reopen the proxy's databases: {e}"); + } + if running { + self.start_routing(None, Some(proxy_on))?; + } + return Err(e); + } + let new_key = db::load_or_create_data_key(&self.keys, &self.data_dir); + match (ca_pem, &new_key) { + (Some(pem), Ok(new_key)) => { + let sealed = new_key.encrypt(&pem).map_err(|e| e.to_string())?; + write_private(&ca_key, &sealed)?; + } + // Unreadable without the old key: a new CA is made as the databases open. + _ => { + for name in ["ca.key", "ca.pem"] { + let _ = std::fs::remove_file(self.ca_dir().join(name)); + } + } + } + *self.lock_reason.lock().unwrap_or_else(PoisonError::into_inner) = None; + } + log::info!("Reset the proxy's captured traffic in {}", self.data_dir.display()); + let unlocked = self.unlock(); + // Whatever the app had is gone, opened again or not. + self.events.emit("flows_cleared", &FlowsClearedPayload {}); + self.emit_state(); + unlocked?; + if running { + self.start_routing(None, Some(proxy_on))?; + } + Ok(()) + } + + /// A clone sharing everything but the databases, for a thread that may outlive every + /// context that owns them, like one turning the system proxy off as the engine stops. Were + /// it the last to hold them, it would close them itself, and at a process's exit that races + /// SQLCipher's own teardown and crashes; this way they close where their owner lets go. + fn detached(&self) -> Self { + Self { stores: Arc::new(RwLock::new(None)), ..self.clone() } + } + + /// Has the sweeper look at the captured traffic's size now, started when the keep-traffic + /// cap is set and it is not running. Without a cap nothing runs, and a sweeper running + /// stops. + fn sweep_soon(&self) { + let (state, wake) = &*self.sweeper; + let mut sweeper = state.lock().unwrap_or_else(PoisonError::into_inner); + if sweeper.running { + sweeper.poked = true; + wake.notify_all(); + return; + } + if self.keep_traffic().is_none() { + return; + } + sweeper.running = true; + drop(sweeper); + let ctx = self.detached(); + let stores = Arc::downgrade(&self.stores); + let spawned = std::thread::Builder::new().name("proxy-sweep".into()).spawn(move || { + lower_thread_priority(); + ctx.run_sweeper(&stores); + }); + if let Err(e) = spawned { + warn!("Failed to start keeping captured traffic under its cap: {e}"); + state.lock().unwrap_or_else(PoisonError::into_inner).running = false; + } + } + + /// The keep-traffic cap, as `settings.json` has it. + fn keep_traffic(&self) -> Option { + self.settings().ok().and_then(|s| s.keep_traffic_bytes) + } + + /// Sweeps the oldest flows while the captured traffic is over the cap, then waits for + /// [`SWEEP_INTERVAL`] or a nudge, until the cap is taken away. The cap is read under the + /// sweeper's lock, so one set as this stops finds it stopped and starts another. + fn run_sweeper(&self, stores: &Weak>>) { + let (state, wake) = &*self.sweeper; + loop { + let cap = { + let mut sweeper = state.lock().unwrap_or_else(PoisonError::into_inner); + let Some(cap) = self.keep_traffic() else { + sweeper.running = false; + return; + }; + sweeper.poked = false; + cap + }; + self.sweep_under(cap, stores); + let sweeper = state.lock().unwrap_or_else(PoisonError::into_inner); + if !sweeper.poked { + drop(wake.wait_timeout(sweeper, SWEEP_INTERVAL)); + } + } + } + + /// Deletes the oldest flows a batch at a time, yielding the writer between, until the + /// captured traffic is under `cap`, telling the app of each batch. Stops early when the + /// sweeper is nudged, to read the cap again. `stores` is the context's databases, held + /// weakly so the sweeper never keeps them open past the context that owns them. + fn sweep_under(&self, cap: u64, stores: &Weak>>) { + loop { + // Held through the batch, letting go of the databases included. + let gate = exit_gate::hold(); + // Each batch from the databases open now, which a reset replaces. + let Some(stores) = stores + .upgrade() + .and_then(|s| s.read().unwrap_or_else(PoisonError::into_inner).clone()) + else { + return; + }; + match flows::sweep_oldest(&stores, cap) { + Ok(ids) if ids.is_empty() => return, + Ok(ids) => { + // After the commit, as a clear's is. + self.events.emit("flows_removed", &FlowsRemovedPayload { ids }); + } + Err(e) => { + warn!("{e}"); + return; + } + } + drop(stores); + drop(gate); + if self.sweeper.0.lock().unwrap_or_else(PoisonError::into_inner).poked { + return; + } + std::thread::sleep(DELETE_PAUSE); + } + } + /// What HTTPS is decrypted, as `settings.json` has it. pub fn decrypt_list(&self) -> Result { Ok(self.settings()?.decrypt_list()) @@ -873,18 +1143,46 @@ impl ProxyCtx { /// already doing so. Yields the writer between batches, so new flows are written meanwhile. fn delete_cleared(&self) { // Only ever after a clear or unlocking, so open. - let Ok(Stores { db, blobs }) = self.stores() else { + let Ok(Stores { db, blobs, .. }) = self.stores() else { return; }; if !claim(&self.deleting_cleared) { return; } let (running, cleaner) = (self.deleting_cleared.clone(), self.cleaner_id.clone()); + let (resets, reset) = (self.resets.clone(), self.resets.load(Ordering::SeqCst)); let spawned = std::thread::Builder::new().name("proxy-clear".into()).spawn(move || { lower_thread_priority(); + clean(&db, &blobs, &running, &cleaner, &resets, reset); + // Its last hold on the databases, which may close them, is let go of in a batch too. + let _gate = exit_gate::hold(); + drop((db, blobs)); + }); + if let Err(e) = spawned { + warn!("Failed to start deleting cleared flows: {e}"); + *self.deleting_cleared.lock().unwrap_or_else(PoisonError::into_inner) = false; + } + + /// Deletes cleared flows a batch at a time until none are left, another process takes + /// over, or a reset replaces the databases. + fn clean( + db: &ProxyQueryManager, + blobs: &BlobManager, + running: &Mutex, + cleaner: &str, + resets: &AtomicU64, + reset: u64, + ) { let mut retry = DELETE_PAUSE; loop { - match flows::delete_cleared(&db, &blobs, flows::DELETE_BATCH, &cleaner) { + let gate = exit_gate::hold(); + // Its databases are gone, and the reset has let another start. + if resets.load(Ordering::SeqCst) != reset { + return; + } + let cleaned = flows::delete_cleared(db, blobs, flows::DELETE_BATCH, cleaner); + drop(gate); + match cleaned { Ok(flows::Cleaned::More) => { retry = DELETE_PAUSE; std::thread::sleep(DELETE_PAUSE); @@ -896,7 +1194,11 @@ impl ProxyCtx { *running.lock().unwrap_or_else(PoisonError::into_inner) = false; // A clear since the last batch saw this running and left its flows to // it, so they are deleted here unless another thread has taken over. - if flows::clear_mark_in(&db) == 0 || !claim(&running) { + let more = { + let _gate = exit_gate::hold(); + flows::clear_mark_in(db) != 0 + }; + if !more || !claim(running) { return; } } @@ -909,10 +1211,6 @@ impl ProxyCtx { } } } - }); - if let Err(e) = spawned { - warn!("Failed to start deleting cleared flows: {e}"); - *self.deleting_cleared.lock().unwrap_or_else(PoisonError::into_inner) = false; } } } @@ -923,6 +1221,50 @@ fn claim(flag: &Mutex) -> bool { !std::mem::replace(&mut *set, true) } +/// Whether the sweeper runs, and whether it has been asked to look again since it last read the +/// cap. +#[derive(Debug, Default)] +struct Sweeper { + running: bool, + poked: bool, +} + +/// Deletes the captured databases, trying again for a few seconds while something still has +/// them open, as Windows refuses to delete an open file. +fn remove_captured_when_closed(data_dir: &Path) -> Result<(), String> { + let mut tries = 0; + loop { + match db::remove_captured(data_dir) { + Ok(()) => return Ok(()), + Err(_) if tries < REMOVE_TRIES => { + tries += 1; + std::thread::sleep(Duration::from_millis(100)); + } + Err(e) => return Err(e), + } + } +} + +/// Tries at deleting the captured databases, 100 ms apart, before a reset gives up. +const REMOVE_TRIES: u32 = 50; + +/// How often the sweeper measures the captured traffic against the keep-traffic cap. +const SWEEP_INTERVAL: Duration = Duration::from_secs(30); + +/// Writes `contents` to `path` through a file beside it and a rename, readable by the user +/// alone where the platform has such modes. +fn write_private(path: &Path, contents: &[u8]) -> Result<(), String> { + let fail = |e: std::io::Error| format!("Failed to write {}: {e}", path.display()); + let tmp = path.with_extension("tmp"); + let mut options = std::fs::OpenOptions::new(); + options.write(true).create(true).truncate(true); + #[cfg(unix)] + std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600); + let mut file = options.open(&tmp).map_err(fail)?; + file.write_all(contents).and_then(|_| file.sync_all()).map_err(fail)?; + std::fs::rename(&tmp, path).map_err(fail) +} + /// How long deleting cleared flows rests between batches, leaving the database to new flows. const DELETE_PAUSE: Duration = Duration::from_millis(5); @@ -1051,6 +1393,13 @@ pub struct ClearFlowsRequest {} #[ts(export, export_to = "gen_rpc.ts")] pub struct FlowsClearedPayload {} +/// Flows deleted to keep captured traffic under the keep-traffic cap, the oldest first. +#[derive(Serialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct FlowsRemovedPayload { + pub ids: Vec, +} + #[derive(Deserialize, TS)] #[ts(export, export_to = "gen_rpc.ts")] pub struct GetProxyStateRequest {} @@ -1070,6 +1419,12 @@ pub struct GetProxyStateResponse { pub system_proxy_at_launch: bool, /// The HTTP versions offered to clients. pub http_version: HttpVersion, + /// Bytes of each body stored per flow. + #[ts(type = "number")] + pub capture_body_limit: usize, + /// The most captured traffic kept, in bytes, or null to keep everything. + #[ts(type = "number | null")] + pub keep_traffic_bytes: Option, /// Why the engine last failed to start, until it starts. pub start_error: Option, /// How far encrypting traffic stored before encryption has got, from 0 to 1, while it @@ -1154,6 +1509,51 @@ pub struct SetHttpVersionRequest { pub http_version: HttpVersion, } +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct SetPortRequest { + pub port: u16, +} + +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct SetCaptureBodyLimitRequest { + #[ts(type = "number")] + pub bytes: u64, +} + +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct SetKeepTrafficRequest { + /// Null keeps everything. + #[ts(type = "number | null")] + pub bytes: Option, +} + +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct GetStorageSizeRequest {} + +#[derive(Serialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct GetStorageSizeResponse { + /// The databases' used pages, as the keep-traffic cap counts them; locked, their files' + /// sizes on disk. + #[ts(type = "number")] + pub bytes: u64, +} + +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct ResetCapturedDataRequest {} + +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct OpenTerminalRequest { + /// Environment variables the shell starts with, by name: `HTTPS_PROXY`. + pub vars: Vec<(String, String)>, +} + #[derive(Deserialize, TS)] #[ts(export, export_to = "gen_rpc.ts")] pub struct GetDecryptListRequest {} @@ -1561,6 +1961,8 @@ async fn get_proxy_state( recording: ctx.recording(), system_proxy_at_launch: settings.system_proxy_at_launch, http_version: settings.http_version, + capture_body_limit: settings.capture_body_limit, + keep_traffic_bytes: settings.keep_traffic_bytes, start_error: ctx.start_error(), encrypting: ctx.encrypting(), locked: ctx.locked(), @@ -1588,6 +1990,46 @@ fn set_http_version(ctx: &ProxyCtx, req: SetHttpVersionRequest) -> Result Result { + blocking(move || ctx.set_port(req.port)).await +} + +fn set_capture_body_limit( + ctx: &ProxyCtx, + req: SetCaptureBodyLimitRequest, +) -> Result { + let bytes = usize::try_from(req.bytes).map_err(rpc_error)?; + ctx.set_capture_body_limit(bytes).map_err(rpc_error) +} + +/// The cap as saved, in bytes, or null to keep everything. +fn set_keep_traffic(ctx: &ProxyCtx, req: SetKeepTrafficRequest) -> Result, RpcError> { + let bytes = ctx.set_keep_traffic(req.bytes).map_err(rpc_error)?; + Ok(bytes.map(|b| usize::try_from(b).unwrap_or(usize::MAX))) +} + +async fn get_storage_size( + ctx: ProxyCtx, + _req: GetStorageSizeRequest, +) -> Result { + let bytes = blocking(move || ctx.storage_size()).await?; + Ok(GetStorageSizeResponse { bytes }) +} + +async fn reset_captured_data( + ctx: ProxyCtx, + _req: ResetCapturedDataRequest, +) -> Result { + blocking(move || ctx.reset_captured_data()).await?; + Ok(true) +} + +async fn open_terminal(_ctx: ProxyCtx, req: OpenTerminalRequest) -> Result { + blocking(move || terminal::open(&req.vars)).await?; + Ok(true) +} + fn get_decrypt_list(ctx: &ProxyCtx, _req: GetDecryptListRequest) -> Result { ctx.decrypt_list().map_err(rpc_error) } @@ -1991,6 +2433,28 @@ struct FlowSink { recording: Arc>, /// Flows that have ended, waiting to be written together. finished: std::cell::RefCell, + /// What a batch's wait is timed by. + clock: BatchClock, +} + +/// What the event loop times a batch's wait by: the time, or in tests a time that stands still, +/// so a batch is written only once no events wait or it is full. +#[derive(Debug, Clone, Copy, Default)] +enum BatchClock { + #[default] + Real, + #[cfg(test)] + Stopped(std::time::Instant), +} + +impl BatchClock { + fn now(self) -> std::time::Instant { + match self { + Self::Real => std::time::Instant::now(), + #[cfg(test)] + Self::Stopped(at) => at, + } + } } /// Flows that have ended and are yet to be written, with when the first of them ended. Each @@ -2033,6 +2497,7 @@ fn run_event_loop(rx: std::sync::mpsc::Receiver, sink: FlowSink) { match rx.try_recv() { Ok(event) => event, Err(TryRecvError::Empty) => { + let _gate = exit_gate::hold(); sink.write_messages(&mut in_flight, &mut pending); sink.write_finished(); sink.write_due_flows(&mut in_flight); @@ -2045,6 +2510,7 @@ fn run_event_loop(rx: std::sync::mpsc::Receiver, sink: FlowSink) { match rx.recv_timeout(wait) { Ok(event) => event, Err(RecvTimeoutError::Timeout) => { + let _gate = exit_gate::hold(); sink.write_due_flows(&mut in_flight); continue; } @@ -2056,6 +2522,8 @@ fn run_event_loop(rx: std::sync::mpsc::Receiver, sink: FlowSink) { Err(_) => break, } }; + // Only while writing, not while waiting for an event above. + let _gate = exit_gate::hold(); if let ProxyEvent::Message { id, message } = event { record_message(&mut in_flight, &mut pending, id, &message); if pending.len() >= MESSAGE_BATCH { @@ -2080,6 +2548,7 @@ fn run_event_loop(rx: std::sync::mpsc::Receiver, sink: FlowSink) { } } + let _gate = exit_gate::hold(); sink.write_messages(&mut in_flight, &mut pending); // The engine stopped: a stream still open ends with it. Other flows in flight are dropped, // as they always have been. Every flow that ended is written. @@ -2091,6 +2560,8 @@ fn run_event_loop(rx: std::sync::mpsc::Receiver, sink: FlowSink) { } } sink.write_finished(); + // Its hold on the databases goes before the gate opens again. + drop(sink); } /// When the next changed streaming flow is due to be written. @@ -2174,6 +2645,7 @@ impl FlowSink { method, url, client_http_version: http_version, + client_addr: Some(client_addr.to_string()), ..Default::default() }; in_flight.insert( @@ -2202,6 +2674,7 @@ impl FlowSink { method: "CONNECT".to_string(), url: format!("https://{authority}"), client_http_version: "HTTP/1.1".to_string(), + client_addr: Some(client_addr.to_string()), tunnelled: true, ..Default::default() }; @@ -2274,9 +2747,10 @@ impl FlowSink { f.changed(); } } - ProxyEvent::RequestBodyComplete { id, size, truncated } => { + ProxyEvent::RequestBodyComplete { id, size, truncated, elapsed_ms } => { if let Some(f) = in_flight.get_mut(&id) { f.flow.request_size = size as i64; + f.flow.elapsed_request_ms = Some(clamp_ms(elapsed_ms)); if f.keeps_bytes(FlowBody::Request) { f.flow.request_truncated = truncated; finish_body(f.request_body.take()); @@ -2290,6 +2764,21 @@ impl FlowSink { f.changed(); } } + ProxyEvent::UpstreamConnection { + id, + elapsed_connect_ms, + elapsed_tls_ms, + tls_version, + tls_cipher, + } => { + if let Some(f) = in_flight.get_mut(&id) { + f.flow.elapsed_connect_ms = elapsed_connect_ms.map(clamp_ms); + f.flow.elapsed_tls_ms = elapsed_tls_ms.map(clamp_ms); + f.flow.upstream_tls_version = tls_version; + f.flow.upstream_tls_cipher = tls_cipher; + f.changed(); + } + } ProxyEvent::ResponseStart { id, status, http_version, remote_addr, elapsed_ms } => { if let Some(f) = in_flight.get_mut(&id) { f.flow.status = Some(status as i32); @@ -2496,7 +2985,7 @@ impl FlowSink { flow.client_cwd = app.cwd; } let mut finished = self.finished.borrow_mut(); - finished.since.get_or_insert_with(std::time::Instant::now); + finished.since.get_or_insert_with(|| self.clock.now()); finished.flows.push(FinishedFlow { flow, persisted: f.persisted, chunks }); } @@ -2508,7 +2997,9 @@ impl FlowSink { fn finished_due(&self) -> bool { let finished = self.finished.borrow(); finished.flows.len() >= FLOW_BATCH - || finished.since.is_some_and(|t| t.elapsed() >= FLOW_BATCH_WAIT) + || finished + .since + .is_some_and(|t| self.clock.now().saturating_duration_since(t) >= FLOW_BATCH_WAIT) } /// Writes the flows that have ended: their bodies' last chunks in one transaction, then @@ -2657,6 +3148,8 @@ define_rpc! { set_recording(SetRecordingRequest) -> bool, set_system_proxy_at_launch(SetSystemProxyAtLaunchRequest) -> bool, set_http_version(SetHttpVersionRequest) -> HttpVersion, + set_capture_body_limit(SetCaptureBodyLimitRequest) -> usize, + set_keep_traffic(SetKeepTrafficRequest) -> Option, } async_commands { add_grpc_schema(AddGrpcSchemaRequest) -> GrpcSchema, @@ -2682,10 +3175,15 @@ define_rpc! { system_proxy_set(SystemProxySetRequest) -> SystemProxyStatus, system_proxy_status(SystemProxyStatusRequest) -> SystemProxyStatus, take_over(TakeOverRequest) -> bool, + set_port(SetPortRequest) -> u16, + get_storage_size(GetStorageSizeRequest) -> GetStorageSizeResponse, + reset_captured_data(ResetCapturedDataRequest) -> bool, + open_terminal(OpenTerminalRequest) -> bool, } events { decrypt_list_changed(DecryptListChangedPayload), flows_cleared(FlowsClearedPayload), + flows_removed(FlowsRemovedPayload), message_write(MessagePayload), model_write(ModelPayload), proxy_state_changed(ProxyStatePayload), @@ -2738,17 +3236,31 @@ mod tests { /// An event loop writing to `ctx`'s databases, fed by the returned sender. fn event_loop( ctx: &ProxyCtx, + ) -> (std::sync::mpsc::Sender, std::thread::JoinHandle<()>) { + event_loop_timed(ctx, BatchClock::Real) + } + + /// [`event_loop`], its batches timed by `clock`. + fn event_loop_timed( + ctx: &ProxyCtx, + clock: BatchClock, ) -> (std::sync::mpsc::Sender, std::thread::JoinHandle<()>) { let (tx, rx) = std::sync::mpsc::channel(); - let sink = FlowSink { + let sink = sink(ctx); + let sink = FlowSink { clock, ..sink }; + (tx, std::thread::spawn(move || run_event_loop(rx, sink))) + } + + fn sink(ctx: &ProxyCtx) -> FlowSink { + FlowSink { db: ctx.db().unwrap(), blobs: ctx.blobs().unwrap(), events: ctx.events.clone(), attribution: Attribution::new(0), recording: ctx.recording.clone(), finished: Default::default(), - }; - (tx, std::thread::spawn(move || run_event_loop(rx, sink))) + clock: BatchClock::Real, + } } fn open_websocket(tx: &std::sync::mpsc::Sender, id: u64, url: &str) { @@ -2842,6 +3354,7 @@ mod tests { id: 1, size: request.len() as u64, truncated: false, + elapsed_ms: 1, }); send(ProxyEvent::ResponseStart { id: 1, @@ -3020,7 +3533,7 @@ mod tests { len: 30, data: bytes::Bytes::from_static(b"junk"), }); - send(ProxyEvent::RequestBodyComplete { id: 1, size: 30, truncated: true }); + send(ProxyEvent::RequestBodyComplete { id: 1, size: 30, truncated: true, elapsed_ms: 1 }); send(ProxyEvent::ResponseStart { id: 1, status: 200, @@ -3548,22 +4061,403 @@ mod tests { } } - /// A flow that ends alone is written at once, not after the batch's wait. + /// A flow that ends alone is written at once, not after the batch's wait: with the batch's + /// clock stopped, the wait never ends, so only the queue running dry writes it. #[test] fn a_lone_flow_shows_at_once() { let dir = tempfile::tempdir().unwrap(); let (events, _rx) = RpcEventEmitter::new(); let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); - let (tx, event_loop) = event_loop(&ctx); - let sent = std::time::Instant::now(); + let stopped = BatchClock::Stopped(std::time::Instant::now()); + let (tx, event_loop) = event_loop_timed(&ctx, stopped); send_flow(&tx, 1, "https://example.com/alone"); + // The engine still runs, so it is not the stop that writes it. wait_for_flow(&ctx, |f| f.url == "https://example.com/alone"); - let shown = sent.elapsed(); - assert!(shown < Duration::from_millis(50), "shown after {shown:?}"); drop(tx); event_loop.join().unwrap(); } + /// The batch's wait is what writes a batch while events keep coming. + #[test] + fn a_batch_is_due_once_its_first_flow_has_waited() { + let dir = tempfile::tempdir().unwrap(); + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + let started = std::time::Instant::now(); + let mut sink = FlowSink { clock: BatchClock::Stopped(started), ..sink(&ctx) }; + let f = InFlight { + flow: Flow { id: Flow::generate_id(), ..Default::default() }, + connection: 1, + request_body: None, + response_body: None, + persisted: false, + cleared: false, + next_seq: 0, + dirty: false, + last_write: None, + }; + sink.finish_flow(f); + assert!(!sink.finished_due(), "the clock stands still"); + sink.clock = BatchClock::Stopped(started + FLOW_BATCH_WAIT); + assert!(sink.finished_due()); + sink.write_finished(); + assert_eq!(flows::count(&ctx.db().unwrap()).unwrap(), 1); + } + + /// A flow keeps the client's address, its connection's timings and TLS, and when its + /// request was sent whole, for its timeline and details. + #[test] + fn a_flow_keeps_its_connection_and_timings() { + let dir = tempfile::tempdir().unwrap(); + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + let (tx, event_loop) = event_loop(&ctx); + let send = |event| tx.send(event).unwrap(); + send(ProxyEvent::RequestStart { + id: 1, + started_at: SystemTime::now(), + method: "POST".into(), + url: "https://api.example.com/upload".into(), + http_version: "HTTP/2".into(), + connection: 1, + client_addr: "127.0.0.1:54321".parse().unwrap(), + }); + send(ProxyEvent::UpstreamConnection { + id: 1, + elapsed_connect_ms: Some(3), + elapsed_tls_ms: Some(9), + tls_version: Some("TLSv1.3".into()), + tls_cipher: Some("TLS13_AES_128_GCM_SHA256".into()), + }); + let data = bytes::Bytes::from_static(b"payload"); + send(ProxyEvent::RequestBodyChunk { id: 1, len: data.len(), data }); + send(ProxyEvent::RequestBodyComplete { id: 1, size: 7, truncated: false, elapsed_ms: 12 }); + send(ProxyEvent::ResponseStart { + id: 1, + status: 201, + http_version: "HTTP/2".into(), + remote_addr: Some("93.184.216.34:443".parse().unwrap()), + elapsed_ms: 20, + }); + send(ProxyEvent::ResponseBodyComplete { + id: 1, + size: 0, + truncated: false, + elapsed_ms: 25, + error: None, + cancelled: false, + }); + drop(tx); + event_loop.join().unwrap(); + let flow = flows::list(&ctx.db().unwrap(), &Default::default()).unwrap().remove(0); + assert_eq!(flow.client_addr.as_deref(), Some("127.0.0.1:54321")); + assert_eq!((flow.elapsed_connect_ms, flow.elapsed_tls_ms), (Some(3), Some(9))); + assert_eq!((flow.elapsed_request_ms, flow.elapsed_headers_ms), (Some(12), Some(20))); + assert_eq!(flow.upstream_tls_version.as_deref(), Some("TLSv1.3")); + assert_eq!(flow.upstream_tls_cipher.as_deref(), Some("TLS13_AES_128_GCM_SHA256")); + } + + /// Waits up to 5 seconds for `check`. + fn wait_until(what: &str, check: impl Fn() -> bool) { + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while !check() { + assert!(std::time::Instant::now() < deadline, "{what}"); + std::thread::sleep(Duration::from_millis(10)); + } + } + + fn sweeper_running(ctx: &ProxyCtx) -> bool { + ctx.sweeper.0.lock().unwrap().running + } + + /// Without a cap nothing sweeps. With one, the oldest flows go until the traffic is under + /// it, and the app hears which; taken away, the sweeper stops. + #[test] + fn keeping_traffic_under_a_cap_sweeps_the_oldest() { + let dir = tempfile::tempdir().unwrap(); + let (events, rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + let empty = ctx.storage_size().unwrap(); + let ids: Vec = (0..20).map(|i| format!("fl_{i:02}")).collect(); + for id in &ids { + flow_with_response(&ctx, id, "application/octet-stream", &[3; 64 * 1024]); + } + let full = ctx.storage_size().unwrap(); + + ctx.sweep_soon(); + assert!(!sweeper_running(&ctx), "no thread without a cap"); + assert_eq!(flows::count(&ctx.db().unwrap()).unwrap(), 20); + + let cap = (empty + full) / 2; + // Swept here rather than waited on from the sweeper's thread, which a busy machine may + // not run in time: what goes is the same either way. + ctx.sweep_under(cap, &Arc::downgrade(&ctx.stores)); + let removed: Vec = rx + .try_iter() + .filter(|e| e.event == "flows_removed") + .flat_map(|e| serde_json::from_value::>(e.payload["ids"].clone()).unwrap()) + .collect(); + let kept = flows::count(&ctx.db().unwrap()).unwrap(); + let after = ctx.storage_size().unwrap(); + let sizes = format!("empty {empty}, full {full}, cap {cap}, after {after}"); + assert!(after <= cap, "under the cap: {sizes}"); + assert!(!removed.is_empty() && removed.len() < 20, "some, not all: {removed:?}, {sizes}"); + assert_eq!(removed.len(), 20 - kept, "the app hears of each"); + assert_eq!(removed, ids[..removed.len()], "the oldest, in order"); + + // Set, the cap starts the sweeper, which finds nothing more to do; taken away, it stops. + assert_eq!(ctx.set_keep_traffic(Some(cap)).unwrap(), Some(cap)); + assert_eq!(ctx.settings().unwrap().keep_traffic_bytes, Some(cap)); + assert!(sweeper_running(&ctx)); + assert_eq!(ctx.set_keep_traffic(None).unwrap(), None); + wait_until("the sweeper stops", || !sweeper_running(&ctx)); + assert_eq!(flows::count(&ctx.db().unwrap()).unwrap(), kept, "nothing more went"); + assert!(ctx.set_keep_traffic(Some(0)).is_err()); + } + + /// Writes a body for `body_id` straight to the blob store, as the event loop does. + fn write_blob(ctx: &ProxyCtx, body_id: &str, bytes: &[u8]) { + let mut writer = BodyChunkWriter::new(ctx.blobs().unwrap(), body_id.to_string()); + writer.write(bytes).unwrap(); + writer.finish().unwrap(); + } + + /// The bodies stored under a flow's id: its request and response, and its messages' payloads + /// numbered below `messages`. + fn blobs_of(ctx: &ProxyCtx, id: &str, messages: usize) -> usize { + let blobs = ctx.blobs().unwrap(); + let conn = blobs.connect(); + let bodies = [ + Flow::body_id(id, FlowBody::Request), + Flow::body_id(id, FlowBody::Response), + ]; + let payloads = (0..messages).map(|n| format!("{id}.msg.{n}")); + bodies.into_iter().chain(payloads).filter(|b| conn.body_exists(b).unwrap()).count() + } + + /// A thread turning the system proxy on or off holds none of the databases, so it never + /// closes them itself, as it would at a process's exit, crashing in SQLCipher's teardown. + #[test] + fn threads_that_outlive_the_engine_hold_no_databases() { + let dir = tempfile::tempdir().unwrap(); + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + assert!(ctx.stores().is_ok()); + let detached = ctx.detached(); + assert!(detached.stores().is_err(), "none of its own"); + assert_eq!(Arc::strong_count(&ctx.stores), 1, "the owner's alone"); + // Everything else is shared, as the system proxy's state is. + ctx.system_proxy_on.store(true, Ordering::SeqCst); + assert!(detached.system_proxy_on.load(Ordering::SeqCst)); + } + + /// A message written for a flow while the sweep deletes it, before its row goes or between + /// its row and its bodies, is deleted with it rather than left counting against the cap. + #[test] + fn a_message_written_while_its_flow_is_swept_goes_too() { + let dir = tempfile::tempdir().unwrap(); + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + for id in ["fl_a", "fl_b"] { + flow_with_response(&ctx, id, "text/plain", &[1; 4096]); + } + let stores = ctx.stores().unwrap(); + + let rows = flows::sweepable(&stores, 1).unwrap(); + assert_eq!(rows.iter().map(|(_, id)| id.as_str()).collect::>(), ["fl_a"]); + // Written after it was chosen, before its row is deleted... + write_blob(&ctx, "fl_a.msg.7", b"late"); + let deleted = flows::delete_swept_rows(&stores, rows).unwrap(); + // ...and after its row is gone, before its bodies are. + write_blob(&ctx, "fl_a.msg.8", b"later"); + flows::delete_swept_bodies(&stores, &deleted).unwrap(); + + assert_eq!(deleted, ["fl_a"]); + assert_eq!(blobs_of(&ctx, "fl_a", 10), 0, "no orphans"); + assert_eq!(blobs_of(&ctx, "fl_b", 10), 1, "the other flow kept"); + } + + /// A sweep that deletes the rows and then fails before the bodies, as on a full disk, leaves + /// them pending, and the next sweep deletes them, even with nothing more to sweep. + #[test] + fn bodies_a_failed_sweep_left_go_on_the_next() { + let dir = tempfile::tempdir().unwrap(); + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + for id in ["fl_a", "fl_b"] { + flow_with_response(&ctx, id, "text/plain", &[1; 4096]); + } + let stores = ctx.stores().unwrap(); + + let rows = flows::sweepable(&stores, 1).unwrap(); + // The rows go, and the bodies don't. + assert_eq!(flows::delete_swept_rows(&stores, rows).unwrap(), ["fl_a"]); + assert_eq!(flows::pending_sweeps(&stores).unwrap(), ["fl_a"]); + assert_eq!(blobs_of(&ctx, "fl_a", 0), 1, "left behind"); + + // Under the cap, so only what was left goes. + assert!(flows::sweep_oldest(&stores, u64::MAX).unwrap().is_empty()); + assert_eq!(blobs_of(&ctx, "fl_a", 0), 0); + assert!(flows::pending_sweeps(&stores).unwrap().is_empty()); + assert_eq!(blobs_of(&ctx, "fl_b", 0), 1, "the other flow kept"); + } + + /// A stream still open is never swept, even while its writer adds messages as the sweep + /// runs and it is the oldest flow there is. + #[test] + fn an_open_stream_is_not_swept_while_it_is_written() { + let dir = tempfile::tempdir().unwrap(); + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + let open = Flow { + id: "fl_open".into(), + stream_kind: Some(StreamKind::Websocket), + status: Some(101), + ..Default::default() + }; + ctx.db().unwrap().with_tx(|tx| tx.upsert(&open, &UpdateSource::Background)).unwrap(); + write_blob(&ctx, "fl_open.msg.0", &[2; 16 * 1024]); + let ended: Vec = (0..10).map(|i| format!("fl_{i:02}")).collect(); + for id in &ended { + flow_with_response(&ctx, id, "application/octet-stream", &[3; 32 * 1024]); + } + + let done = Arc::new(AtomicBool::new(false)); + let writer = { + let (ctx, done) = (ctx.clone(), done.clone()); + std::thread::spawn(move || { + let mut seq = 1; + while !done.load(Ordering::SeqCst) { + write_blob(&ctx, &format!("fl_open.msg.{seq}"), &[4; 1024]); + seq += 1; + } + seq + }) + }; + // A cap below anything reachable, so the sweep takes every flow it may. + let stores = ctx.stores().unwrap(); + while !flows::sweep_oldest(&stores, 1).unwrap().is_empty() {} + done.store(true, Ordering::SeqCst); + let written = writer.join().unwrap(); + + let db = ctx.db().unwrap(); + assert!(flows::find(&db, "fl_open").is_some(), "the open stream stays"); + assert_eq!(flows::count(&db).unwrap(), 1, "every ended flow swept"); + assert_eq!(blobs_of(&ctx, "fl_open", written), written, "its messages all kept"); + for id in &ended { + assert_eq!(blobs_of(&ctx, id, 0), 0, "{id} left nothing behind"); + } + } + + /// Resetting deletes the traffic and its data key and opens empty databases under a new + /// one; the CA, readable with the old key, is kept, so it stays trusted. + #[test] + fn resetting_captured_data_starts_empty_and_keeps_the_ca() { + let dir = tempfile::tempdir().unwrap(); + let (events, rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + flow_with_response(&ctx, "fl_old", "text/plain", b"secret"); + let pem = ctx.load_ca().unwrap().pem().to_string(); + let key = std::fs::read(db::data_key_path(dir.path())).unwrap(); + rx.try_iter().count(); + + ctx.reset_captured_data().unwrap(); + assert!(ctx.locked().is_none()); + assert_eq!(flows::count(&ctx.db().unwrap()).unwrap(), 0); + assert!(!ctx.blobs().unwrap().connect().body_exists("fl_old.response").unwrap()); + assert_ne!(std::fs::read(db::data_key_path(dir.path())).unwrap(), key, "a new key"); + assert_eq!(ctx.load_ca().unwrap().pem(), pem, "the same CA"); + let heard: Vec<&str> = rx.try_iter().map(|e| e.event).collect(); + assert!(heard.contains(&"flows_cleared") && heard.contains(&"proxy_state_changed")); + // Capturing goes on into the new databases. + flow_with_response(&ctx, "fl_new", "text/plain", b"hello"); + assert_eq!(flows::count(&ctx.db().unwrap()).unwrap(), 1); + } + + /// With the key lost from the keychain, nothing can be read, and resetting is the way out: + /// the traffic and the CA, which only the lost key could read, make way for new ones. + #[test] + fn resetting_while_locked_is_the_way_out_of_a_lost_key() { + let dir = tempfile::tempdir().unwrap(); + let pem = { + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + flow_with_response(&ctx, "fl_old", "text/plain", b"secret"); + ctx.load_ca().unwrap().pem().to_string() + }; + // Another keychain, without the master key that wrapped the data key. + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + assert!(ctx.locked().is_some()); + assert!(ctx.db().is_err()); + assert!(ctx.storage_size().unwrap() > 0, "the files, measured on disk"); + + ctx.reset_captured_data().unwrap(); + assert!(ctx.locked().is_none()); + assert_eq!(flows::count(&ctx.db().unwrap()).unwrap(), 0); + assert_ne!(ctx.load_ca().unwrap().pem(), pem, "a new CA"); + flow_with_response(&ctx, "fl_new", "text/plain", b"hello"); + assert_eq!(flows::count(&ctx.db().unwrap()).unwrap(), 1); + } + + /// A port that is free and unused now. + fn free_port() -> u16 { + std::net::TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port() + } + + /// The running engine moves to a new port at once; one it cannot bind leaves it, and the + /// setting, where they were. + #[test] + fn setting_the_port_restarts_the_engine_on_it() { + let dir = tempfile::tempdir().unwrap(); + let first = free_port(); + let settings = + ProxySettings { port: first, system_proxy_at_launch: false, ..Default::default() }; + settings.save(&dir.path().join("settings.json")).unwrap(); + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + assert!(ctx.set_port(0).is_err()); + ctx.start().unwrap(); + assert_eq!(ctx.port(), Some(first)); + + let second = free_port(); + assert_eq!(ctx.set_port(second).unwrap(), second); + assert_eq!((ctx.port(), ctx.settings().unwrap().port), (Some(second), second)); + assert!(is_listening(second)); + + let taken = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let taken_port = taken.local_addr().unwrap().port(); + assert!(ctx.set_port(taken_port).is_err()); + assert_eq!((ctx.port(), ctx.settings().unwrap().port), (Some(second), second)); + assert_eq!(ctx.start_error(), None); + ctx.stop().unwrap(); + + // Stopped, it is only saved, for the next start. + let third = free_port(); + assert_eq!(ctx.set_port(third).unwrap(), third); + assert_eq!(ctx.port(), None); + ctx.start().unwrap(); + assert_eq!(ctx.port(), Some(third)); + ctx.stop().unwrap(); + } + + /// The capture limit is saved and shows in the proxy's state, as the keep-traffic cap does. + #[tokio::test(flavor = "multi_thread")] + async fn the_state_has_the_capture_limit_and_the_traffic_cap() { + let dir = tempfile::tempdir().unwrap(); + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + let req = SetCaptureBodyLimitRequest { bytes: 4096 }; + assert_eq!(set_capture_body_limit(&ctx, req).unwrap(), 4096); + let req = SetKeepTrafficRequest { bytes: Some(1 << 30) }; + assert_eq!(set_keep_traffic(&ctx, req).unwrap(), Some(1 << 30)); + let state = get_proxy_state(ctx.clone(), GetProxyStateRequest {}).await.unwrap(); + assert_eq!((state.capture_body_limit, state.keep_traffic_bytes), (4096, Some(1 << 30))); + let size = get_storage_size(ctx.clone(), GetStorageSizeRequest {}).await.unwrap(); + assert!(size.bytes > 0); + ctx.set_keep_traffic(None).unwrap(); + } + /// A flow that fails to write is rolled back alone: the others in its batch are kept. #[test] fn one_bad_flow_loses_no_others_in_its_batch() { @@ -3581,14 +4475,7 @@ mod tests { Ok::<_, yaak_database::Error>(()) }) .unwrap(); - let sink = FlowSink { - db: ctx.db().unwrap(), - blobs: ctx.blobs().unwrap(), - events: ctx.events.clone(), - attribution: Attribution::new(0), - recording: ctx.recording.clone(), - finished: Default::default(), - }; + let sink = sink(&ctx); // One batch of three, the middle one refused. for url in ["before", "bad", "after"] { let flow = Flow { diff --git a/crates-proxy/yaak-proxy-lib/src/models.rs b/crates-proxy/yaak-proxy-lib/src/models.rs index ceedbb3be..a11671ef3 100644 --- a/crates-proxy/yaak-proxy-lib/src/models.rs +++ b/crates-proxy/yaak-proxy-lib/src/models.rs @@ -59,6 +59,18 @@ pub struct Flow { pub elapsed_headers_ms: Option, /// Milliseconds from `started_at` to the end of the response. pub elapsed_ms: Option, + /// Milliseconds from `started_at` to the TCP connection to the server, and to the end of + /// its TLS handshake. Only for intercepted HTTPS on a connection opened for this flow. + pub elapsed_connect_ms: Option, + pub elapsed_tls_ms: Option, + /// Milliseconds from `started_at` to the request body being sent whole. None without one. + pub elapsed_request_ms: Option, + /// The client's end of its connection: `127.0.0.1:54321`. + pub client_addr: Option, + /// The TLS negotiated with the server for intercepted HTTPS: `TLSv1.3`. + pub upstream_tls_version: Option, + /// `TLS13_AES_128_GCM_SHA256`. + pub upstream_tls_cipher: Option, pub error: Option, /// The client gave up before the response ended, by resetting its HTTP/2 stream or closing /// the connection. Not an error. @@ -244,6 +256,12 @@ impl UpsertModelInfo for Flow { (GrpcStatus, self.grpc_status.into()), (GrpcMessage, self.grpc_message.into()), (Tunnelled, self.tunnelled.into()), + (ElapsedConnectMs, self.elapsed_connect_ms.into()), + (ElapsedTlsMs, self.elapsed_tls_ms.into()), + (ElapsedRequestMs, self.elapsed_request_ms.into()), + (ClientAddr, self.client_addr.into()), + (UpstreamTlsVersion, self.upstream_tls_version.into()), + (UpstreamTlsCipher, self.upstream_tls_cipher.into()), ]) } @@ -281,6 +299,12 @@ impl UpsertModelInfo for Flow { FlowIden::GrpcStatus, FlowIden::GrpcMessage, FlowIden::Tunnelled, + FlowIden::ElapsedConnectMs, + FlowIden::ElapsedTlsMs, + FlowIden::ElapsedRequestMs, + FlowIden::ClientAddr, + FlowIden::UpstreamTlsVersion, + FlowIden::UpstreamTlsCipher, ] } @@ -311,6 +335,12 @@ impl UpsertModelInfo for Flow { response_truncated: r.get("response_truncated")?, elapsed_headers_ms: r.get("elapsed_headers_ms")?, elapsed_ms: r.get("elapsed_ms")?, + elapsed_connect_ms: added_column(r, "elapsed_connect_ms")?, + elapsed_tls_ms: added_column(r, "elapsed_tls_ms")?, + elapsed_request_ms: added_column(r, "elapsed_request_ms")?, + client_addr: added_column(r, "client_addr")?, + upstream_tls_version: added_column(r, "upstream_tls_version")?, + upstream_tls_cipher: added_column(r, "upstream_tls_cipher")?, error: r.get("error")?, cancelled: added_column(r, "cancelled")?.unwrap_or_default(), saved_request_id: added_column(r, "saved_request_id")?, diff --git a/crates-proxy/yaak-proxy-lib/src/settings.rs b/crates-proxy/yaak-proxy-lib/src/settings.rs index 28f7a1363..969d04523 100644 --- a/crates-proxy/yaak-proxy-lib/src/settings.rs +++ b/crates-proxy/yaak-proxy-lib/src/settings.rs @@ -15,6 +15,9 @@ pub struct ProxySettings { pub system_proxy_at_launch: bool, /// Bytes of each request and response body stored per flow. pub capture_body_limit: usize, + /// The most captured traffic kept, in bytes of the databases' used pages; the oldest flows + /// are deleted past it. None keeps everything. + pub keep_traffic_bytes: Option, /// The HTTP versions offered to clients in intercepted HTTPS. pub http_version: HttpVersion, /// The user's decrypt entries, in order: `host:api.example.com`, `app:Google Chrome`, `*`, @@ -31,6 +34,7 @@ impl Default for ProxySettings { port: 9090, system_proxy_at_launch: true, capture_body_limit: DEFAULT_CAPTURE_BODY_LIMIT, + keep_traffic_bytes: None, http_version: HttpVersion::Auto, decrypt: Vec::new(), decrypt_built_ins_removed: Vec::new(), @@ -178,6 +182,7 @@ mod tests { assert_eq!(settings.port, 9090); assert!(settings.system_proxy_at_launch); assert_eq!(settings.capture_body_limit, 10 * 1024 * 1024); + assert_eq!(settings.keep_traffic_bytes, None, "everything is kept"); } #[test] @@ -197,6 +202,7 @@ mod tests { port: 8080, system_proxy_at_launch: false, capture_body_limit: 1024, + keep_traffic_bytes: Some(1 << 30), http_version: HttpVersion::Http1Only, decrypt: vec!["host:api.example.com".into()], decrypt_built_ins_removed: vec!["!host:*.zoom.us".into()], @@ -212,6 +218,7 @@ mod tests { "port": 8080, "systemProxyAtLaunch": false, "captureBodyLimit": 1024, + "keepTrafficBytes": 1073741824, "httpVersion": "http1Only", "decrypt": ["host:api.example.com"], "decryptBuiltInsRemoved": ["!host:*.zoom.us"], diff --git a/crates-proxy/yaak-proxy-lib/src/terminal.rs b/crates-proxy/yaak-proxy-lib/src/terminal.rs new file mode 100644 index 000000000..1b153449e --- /dev/null +++ b/crates-proxy/yaak-proxy-lib/src/terminal.rs @@ -0,0 +1,86 @@ +//! Opens a terminal whose shell has the proxy's environment set, so what runs in it goes through +//! the proxy. macOS's Terminal only for now. + +/// The script Terminal runs: it deletes itself, exports `vars`, then becomes a fresh interactive +/// login shell, the user's own. Fails on a name a shell could not export. +#[cfg_attr(not(target_os = "macos"), allow(dead_code))] +fn script(vars: &[(String, String)]) -> Result { + let mut script = String::from("#!/bin/sh\nrm -f \"$0\"\n"); + for (name, value) in vars { + let valid = name.chars().next().is_some_and(|c| c.is_ascii_alphabetic() || c == '_') + && name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_'); + if !valid { + return Err(format!("{name:?} is not a name an environment variable can have")); + } + script.push_str(&format!("export {name}='{}'\n", value.replace('\'', r"'\''"))); + } + script.push_str("exec \"${SHELL:-/bin/zsh}\" -l\n"); + Ok(script) +} + +/// Opens a Terminal window whose shell starts with `vars` set, through a `.command` script in +/// the temporary directory that deletes itself as it runs. +#[cfg(target_os = "macos")] +pub fn open(vars: &[(String, String)]) -> Result<(), String> { + use std::os::unix::fs::PermissionsExt; + let script = script(vars)?; + let name = format!("yaak-proxy-{}.command", yaak_database::generate_prefixed_id("sh")); + let path = std::env::temp_dir().join(name); + let fail = |e: std::io::Error| format!("Failed to write {}: {e}", path.display()); + std::fs::write(&path, script).map_err(fail)?; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o700)).map_err(fail)?; + let output = std::process::Command::new("/usr/bin/open") + .args(["-a", "Terminal"]) + .arg(&path) + .output() + .map_err(|e| format!("Failed to run open: {e}"))?; + if output.status.success() { + return Ok(()); + } + let _ = std::fs::remove_file(&path); + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + Err(if stderr.is_empty() { format!("open exited with {}", output.status) } else { stderr }) +} + +#[cfg(not(target_os = "macos"))] +pub fn open(_vars: &[(String, String)]) -> Result<(), String> { + Err("Opening a terminal with the proxy set is only supported on macOS so far".to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_script_exports_each_variable_quoted_then_starts_a_login_shell() { + let vars = vec![ + ("HTTPS_PROXY".to_string(), "http://127.0.0.1:9090".to_string()), + ("NOTE".to_string(), "it's $HOME".to_string()), + ]; + assert_eq!( + script(&vars).unwrap(), + "#!/bin/sh\nrm -f \"$0\"\nexport HTTPS_PROXY='http://127.0.0.1:9090'\n\ + export NOTE='it'\\''s $HOME'\nexec \"${SHELL:-/bin/zsh}\" -l\n" + ); + for name in ["", "1A", "A B", "A;rm"] { + assert!(script(&[(name.to_string(), "x".to_string())]).is_err(), "{name:?}"); + } + } + + #[test] + fn the_script_runs_as_a_shell_sets_them() { + let vars = vec![("YAAK_TEST".to_string(), "a 'b' $c".to_string())]; + // Without deleting itself, as `$0` is no script here, or starting a shell. + let script = script(&vars) + .unwrap() + .replace("rm -f \"$0\"\n", "") + .replace("exec \"${SHELL:-/bin/zsh}\" -l\n", ""); + let output = std::process::Command::new("sh") + .args(["-c", &format!("{script}printf %s \"$YAAK_TEST\"")]) + .output(); + // Only where there is a POSIX shell. + if let Ok(output) = output { + assert_eq!(String::from_utf8_lossy(&output.stdout), "a 'b' $c"); + } + } +} diff --git a/crates-proxy/yaak-proxy-lib/tests/flow.rs b/crates-proxy/yaak-proxy-lib/tests/flow.rs index ceb04d904..612183589 100644 --- a/crates-proxy/yaak-proxy-lib/tests/flow.rs +++ b/crates-proxy/yaak-proxy-lib/tests/flow.rs @@ -336,6 +336,8 @@ async fn body_rpcs() { "recording": true, "systemProxyAtLaunch": false, "httpVersion": "auto", + "captureBodyLimit": CAPTURE_LIMIT, + "keepTrafficBytes": null, "startError": null, "encrypting": null, "locked": null, diff --git a/crates-tauri/yaak-app-proxy/capabilities/default.json b/crates-tauri/yaak-app-proxy/capabilities/default.json index f6b586f42..9e821f2ae 100644 --- a/crates-tauri/yaak-app-proxy/capabilities/default.json +++ b/crates-tauri/yaak-app-proxy/capabilities/default.json @@ -7,6 +7,7 @@ "dialog:allow-save", "opener:allow-default-urls", "opener:allow-open-url", + "opener:allow-reveal-item-in-dir", "os:allow-os-type", "core:window:allow-close", "core:window:allow-is-fullscreen", diff --git a/crates-tauri/yaak-app-proxy/src/lib.rs b/crates-tauri/yaak-app-proxy/src/lib.rs index b5e7d52df..dc5360ce1 100644 --- a/crates-tauri/yaak-app-proxy/src/lib.rs +++ b/crates-tauri/yaak-app-proxy/src/lib.rs @@ -177,4 +177,21 @@ mod tests { ); assert_eq!(target("../yaak-app-client/vendored/protoc/include"), Some(PROTOC_INCLUDE_DIR)); } + + /// The window may call each opener command the app's settings use: without its permission, + /// Tauri refuses the call and the button does nothing. + #[test] + fn the_window_may_open_links_and_reveal_folders() { + let capability: serde_json::Value = + serde_json::from_str(include_str!("../capabilities/default.json")).unwrap(); + let permissions: Vec<&str> = capability["permissions"] + .as_array() + .unwrap() + .iter() + .filter_map(|p| p.as_str()) + .collect(); + for needed in ["opener:allow-open-url", "opener:allow-reveal-item-in-dir"] { + assert!(permissions.contains(&needed), "{needed} missing from {permissions:?}"); + } + } } diff --git a/crates/yaak-proxy/src/body.rs b/crates/yaak-proxy/src/body.rs index 2630cf5d6..0c490fd43 100644 --- a/crates/yaak-proxy/src/body.rs +++ b/crates/yaak-proxy/src/body.rs @@ -56,6 +56,8 @@ pub(crate) struct CaptureBody { id: u64, start: Instant, engine: Arc, + /// The capture limit as the body started, so a change applies to bodies after it. + limit: usize, captured: usize, size: u64, finished: bool, @@ -77,6 +79,7 @@ impl CaptureBody { direction, id, start, + limit: engine.capture_body_limit(), engine, captured: 0, size: 0, @@ -122,16 +125,20 @@ impl CaptureBody { self.emit_messages(messages); } let truncated = self.size > self.captured as u64; + let elapsed_ms = self.start.elapsed().as_millis() as u64; let event = match self.direction { Direction::Request if self.size == 0 => return, - Direction::Request => { - ProxyEvent::RequestBodyComplete { id: self.id, size: self.size, truncated } - } + Direction::Request => ProxyEvent::RequestBodyComplete { + id: self.id, + size: self.size, + truncated, + elapsed_ms, + }, Direction::Response => ProxyEvent::ResponseBodyComplete { id: self.id, size: self.size, truncated, - elapsed_ms: self.start.elapsed().as_millis() as u64, + elapsed_ms, error, cancelled, }, @@ -153,7 +160,7 @@ impl Body for CaptureBody { Poll::Ready(Some(Ok(frame))) => { if let Some(frame_data) = frame.data_ref() { let len = frame_data.len(); - let room = self.engine.capture_body_limit.saturating_sub(self.captured); + let room = self.limit.saturating_sub(self.captured); let data = frame_data.slice(..len.min(room)); self.captured += data.len(); self.size += len as u64; diff --git a/crates/yaak-proxy/src/lib.rs b/crates/yaak-proxy/src/lib.rs index 54d7447b3..763f8391d 100644 --- a/crates/yaak-proxy/src/lib.rs +++ b/crates/yaak-proxy/src/lib.rs @@ -16,7 +16,7 @@ use std::collections::hash_map::RandomState; use std::collections::{HashMap, HashSet}; use std::hash::{BuildHasher, Hasher}; use std::net::SocketAddr; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::mpsc as std_mpsc; use std::sync::{Arc, Mutex, RwLock}; use std::time::SystemTime; @@ -139,6 +139,21 @@ pub enum ProxyEvent { size: u64, /// Whether `size` exceeded `capture_body_limit`. truncated: bool, + /// Milliseconds from `started_at` to the body's end. + elapsed_ms: u64, + }, + /// The connection to the server an intercepted HTTPS request goes on, before it is sent. + /// Plain HTTP shares pooled connections the engine does not see opened, so it has none. + UpstreamConnection { + id: u64, + /// Milliseconds from `started_at` to the TCP connection, and to the TLS handshake's + /// end. None when the connection was opened for an earlier request and reused. + elapsed_connect_ms: Option, + elapsed_tls_ms: Option, + /// `TLSv1.3`, `TLSv1.2`. + tls_version: Option, + /// `TLS13_AES_128_GCM_SHA256`. + tls_cipher: Option, }, /// Response headers arrived from upstream. ResponseStart { @@ -250,6 +265,11 @@ impl ProxyHandle { self.engine.http2.store(enabled, Ordering::Relaxed); } + /// Bytes of each body captured, for bodies that start from now on. + pub fn set_capture_body_limit(&self, limit: usize) { + self.engine.capture_body_limit.store(limit, Ordering::Relaxed); + } + /// Decides which HTTPS connections are decrypted from the next CONNECT on. Connections /// already open stay decrypted or tunnelled as they were. pub fn set_decrypt_policy(&self, policy: DecryptPolicy) { @@ -285,7 +305,7 @@ pub(crate) struct ClientConn { pub(crate) struct Engine { ca: Arc, - capture_body_limit: usize, + capture_body_limit: AtomicUsize, events: std_mpsc::Sender, next_id: AtomicU64, next_connection: AtomicU64, @@ -309,6 +329,11 @@ impl Engine { let _ = self.events.send(event); } + /// Bytes of each body captured, as it is when the body starts. + fn capture_body_limit(&self) -> usize { + self.capture_body_limit.load(Ordering::Relaxed) + } + fn next_flow_id(&self) -> u64 { self.next_id.fetch_add(1, Ordering::Relaxed) } @@ -408,7 +433,7 @@ pub fn start_proxy(config: ProxyConfig) -> Result { let (event_tx, event_rx) = std_mpsc::channel(); let engine = Arc::new(Engine { ca: config.ca, - capture_body_limit: config.capture_body_limit, + capture_body_limit: AtomicUsize::new(config.capture_body_limit), events: event_tx, next_id: AtomicU64::new(1), next_connection: AtomicU64::new(1), diff --git a/crates/yaak-proxy/src/request.rs b/crates/yaak-proxy/src/request.rs index f2883e2e1..a09f83c11 100644 --- a/crates/yaak-proxy/src/request.rs +++ b/crates/yaak-proxy/src/request.rs @@ -150,7 +150,7 @@ impl Flow { fn capture_request(&self, body: Incoming) -> BoxBody { let parser = self.grpc.as_ref().map(|call| { - let limit = self.engine.capture_body_limit; + let limit = self.engine.capture_body_limit(); let encoding = call.encoding.as_deref(); let parser = GrpcParser::new(MessageDirection::Client, encoding, call.text, limit) .with_content_encoding(call.content_encoding.as_deref()); @@ -165,7 +165,7 @@ impl Flow { /// How the response's body is read for messages, if it streams them, after recording that /// it does. fn response_parser(&self, resp: &Response) -> Option { - let limit = self.engine.capture_body_limit; + let limit = self.engine.capture_body_limit(); let headers = resp.headers(); if self.grpc.is_some() { // A call that fails at once answers with its status in the headers, and no body. @@ -189,6 +189,21 @@ impl Flow { Some(BodyParser::Sse(Box::new(SseParser::new(encoding, limit)))) } + /// Records the connection the request goes on: its timings when it was opened for this + /// request, and its TLS either way. + fn goes_on(&self, conn: &UpstreamConn, reused: bool) { + let since = |at: Instant| { + (!reused).then(|| at.saturating_duration_since(self.start).as_millis() as u64) + }; + self.engine.emit(ProxyEvent::UpstreamConnection { + id: self.id, + elapsed_connect_ms: since(conn.tls.connected), + elapsed_tls_ms: since(conn.tls.handshaken), + tls_version: conn.tls.version.clone(), + tls_cipher: conn.tls.cipher.clone(), + }); + } + fn respond_head(&mut self, resp: &Response, remote_addr: Option) { self.answered = true; self.engine.emit(ProxyEvent::ResponseStart { @@ -576,9 +591,36 @@ enum Upstream { struct UpstreamConn { sender: Upstream, remote_addr: Option, + tls: TlsInfo, +} + +/// When a connection to the server was made, and the TLS it negotiated. +#[derive(Debug, Clone)] +struct TlsInfo { + connected: Instant, + handshaken: Instant, + version: Option, + cipher: Option, +} + +impl TlsInfo { + fn of(conn: &rustls::ClientConnection, connected: Instant) -> Self { + let version = conn.protocol_version().map(|v| match v { + rustls::ProtocolVersion::TLSv1_3 => "TLSv1.3".to_string(), + rustls::ProtocolVersion::TLSv1_2 => "TLSv1.2".to_string(), + other => format!("{other:?}"), + }); + let cipher = conn.negotiated_cipher_suite().map(|s| format!("{:?}", s.suite())); + Self { connected, handshaken: Instant::now(), version, cipher } + } } impl UpstreamConn { + /// Shared by another request, as an HTTP/2 connection is. + fn share(&self, sender: Upstream) -> Self { + Self { sender, remote_addr: self.remote_addr, tls: self.tls.clone() } + } + fn is_h2(&self) -> bool { matches!(self.sender, Upstream::H2(_)) } @@ -636,10 +678,7 @@ impl Upstreams { conns.retain(|c| !c.is_closed()); if let Some(i) = conns.iter().position(UpstreamConn::is_ready) { let conn = match &conns[i].sender { - Upstream::H2(s) => UpstreamConn { - sender: Upstream::H2(s.clone()), - remote_addr: conns[i].remote_addr, - }, + Upstream::H2(s) => conns[i].share(Upstream::H2(s.clone())), Upstream::H1(_) => conns.swap_remove(i), }; return Ok((conn, true)); @@ -650,10 +689,7 @@ impl Upstreams { } let conn = connect_upstream(engine, &self.target, false).await?; match &conn.sender { - Upstream::H2(s) => conns.push(UpstreamConn { - sender: Upstream::H2(s.clone()), - remote_addr: conn.remote_addr, - }), + Upstream::H2(s) => conns.push(conn.share(Upstream::H2(s.clone()))), Upstream::H1(_) => self.h1.store(true, Ordering::Relaxed), } Ok((conn, false)) @@ -682,10 +718,12 @@ async fn connect_upstream( h1_only: bool, ) -> Result { let tcp = TcpStream::connect(&target.addr).await?; + let connected = Instant::now(); let remote_addr = tcp.peer_addr().ok(); let config = if h1_only { &engine.upstream_tls_h1 } else { &engine.upstream_tls }; let server_name = ServerName::try_from(target.server_name.clone())?; let tls = TlsConnector::from(config.clone()).connect(server_name, tcp).await?; + let tls_info = TlsInfo::of(tls.get_ref().1, connected); let is_h2 = tls.get_ref().1.alpn_protocol() == Some(b"h2"); let io = TokioIo::new(tls); @@ -711,7 +749,7 @@ async fn connect_upstream( }); Upstream::H1(sender) }; - Ok(UpstreamConn { sender: upstream, remote_addr }) + Ok(UpstreamConn { sender: upstream, remote_addr, tls: tls_info }) } /// Addresses a request to the server in the protocol of `conn`. It may be addressed for the @@ -790,6 +828,7 @@ async fn forward_https( Ok(taken) => taken, Err(e) => return Ok(flow.fail(error_chain(e.as_ref()))), }; + flow.goes_on(&conn, reused); let (mut parts, body) = req.into_parts(); if let Err(e) = address(&mut parts, &conn, target, &path, !body.is_end_stream()) { @@ -813,6 +852,7 @@ async fn forward_https( Err(e) => return Ok(flow.fail(error_chain(e.as_ref()))), }; reused = false; + flow.goes_on(&conn, reused); let (mut parts, body) = unsent.into_parts(); if let Err(e) = address(&mut parts, &conn, target, &path, !body.is_end_stream()) { return Ok(flow.fail(e)); diff --git a/crates/yaak-proxy/src/ws.rs b/crates/yaak-proxy/src/ws.rs index dc5753908..c9b7c327a 100644 --- a/crates/yaak-proxy/src/ws.rs +++ b/crates/yaak-proxy/src/ws.rs @@ -120,7 +120,7 @@ pub(crate) async fn relay( }; let (client_read, client_write) = tokio::io::split(TokioIo::new(client)); let (upstream_read, upstream_write) = tokio::io::split(TokioIo::new(upstream)); - let limit = flow.engine.capture_body_limit; + let limit = flow.engine.capture_body_limit(); let outbound = FrameParser::new( MessageDirection::Client, flow.clone(), diff --git a/crates/yaak-proxy/tests/proxy.rs b/crates/yaak-proxy/tests/proxy.rs index f3b32bc1d..ff6566a20 100644 --- a/crates/yaak-proxy/tests/proxy.rs +++ b/crates/yaak-proxy/tests/proxy.rs @@ -415,6 +415,7 @@ fn flow_id(event: &ProxyEvent) -> u64 { | ProxyEvent::RequestHeader { id, .. } | ProxyEvent::RequestBodyChunk { id, .. } | ProxyEvent::RequestBodyComplete { id, .. } + | ProxyEvent::UpstreamConnection { id, .. } | ProxyEvent::ResponseStart { id, .. } | ProxyEvent::ResponseHeader { id, .. } | ProxyEvent::ResponseBodyChunk { id, .. } @@ -1710,6 +1711,29 @@ async fn h2_client_to_an_h2_upstream() { (*size, error.clone(), *cancelled)); assert_eq!((size, error, cancelled), (text.len() as u64, None, false)); assert_eq!(body(&events, false).0, text); + + // The connection opened for it, timed from the flow's start, and its TLS. + let (connect, tls, version, cipher) = find!(events, + ProxyEvent::UpstreamConnection { elapsed_connect_ms, elapsed_tls_ms, tls_version, tls_cipher, .. } => + (*elapsed_connect_ms, *elapsed_tls_ms, tls_version.clone(), tls_cipher.clone())); + let headers_at = find!(events, ProxyEvent::ResponseStart { elapsed_ms, .. } => *elapsed_ms); + assert!(connect.unwrap() <= tls.unwrap() && tls.unwrap() <= headers_at); + assert_eq!(version.as_deref(), Some("TLSv1.3")); + assert!(cipher.unwrap().starts_with("TLS13_")); + + // The next request shares it: no connect or handshake of its own, the same TLS. + let resp = client.send_request(h2_request("POST", port, "/echo", full("again"))).await; + resp.unwrap().into_body().collect().await.unwrap(); + let events = h.flow_events(); + let (connect, tls, version) = find!(events, + ProxyEvent::UpstreamConnection { elapsed_connect_ms, elapsed_tls_ms, tls_version, .. } => + (*elapsed_connect_ms, *elapsed_tls_ms, tls_version.clone())); + assert_eq!((connect, tls, version.as_deref()), (None, None, Some("TLSv1.3"))); + let (sent_at, end_at) = ( + find!(events, ProxyEvent::RequestBodyComplete { elapsed_ms, .. } => *elapsed_ms), + find!(events, ProxyEvent::ResponseBodyComplete { elapsed_ms, .. } => *elapsed_ms), + ); + assert!(sent_at <= end_at, "{sent_at} {end_at}"); } #[tokio::test(flavor = "multi_thread")] diff --git a/packages/ui/src/components/events/AutoScroller.tsx b/packages/ui/src/components/events/AutoScroller.tsx index 7cf61d18e..8f335142d 100644 --- a/packages/ui/src/components/events/AutoScroller.tsx +++ b/packages/ui/src/components/events/AutoScroller.tsx @@ -116,7 +116,7 @@ export function AutoScroller({ className={classNames( "absolute bottom-3 left-1/2 -translate-x-1/2 z-10", "flex items-center gap-1.5 h-6 px-3 rounded-full", - "bg-surface border border-border shadow-lg text-text text-xs", + "bg-surface border border-border shadow-lg text-text text-xs tabular-nums", "hover:bg-surface-highlight", )} > diff --git a/packages/ui/src/components/http/HttpRequestPaneView.tsx b/packages/ui/src/components/http/HttpRequestPaneView.tsx index 3402eea16..56a6fd619 100644 --- a/packages/ui/src/components/http/HttpRequestPaneView.tsx +++ b/packages/ui/src/components/http/HttpRequestPaneView.tsx @@ -106,6 +106,18 @@ export interface HttpRequestPaneViewProps { /** Tabs only the app has, after the shared ones */ extraTabs?: ExtraPaneTab[]; + + /** Replaces the URL bar, as the proxy's card shows a captured request's method and URL */ + urlLine?: ReactNode; + /** Tabs with nothing in them read in the subtlest colour */ + dimEmptyTabs?: boolean; + /** + * The tab to open on for a request with no tab of its own; with it, a tab chosen for one + * request doesn't carry to the next + */ + defaultTab?: string; + /** A tab was chosen */ + onTabChange?: (tab: string) => void; } type HeadersEditorInherited = { name: string; value: string; enabled?: boolean; id?: string }[]; @@ -153,6 +165,10 @@ export function HttpRequestPaneView({ authTabs, authContent, extraTabs, + urlLine, + dimEmptyTabs = false, + defaultTab, + onTabChange, }: HttpRequestPaneViewProps) { const { ErrorBoundary } = useEditorHost(); const readOnly = !editable; @@ -168,27 +184,27 @@ export function HttpRequestPaneView({ : 0; } + const headerCount = request.headers.filter((h) => h.name && h.enabled !== false).length; const tabs = useMemo( () => [ bodyTab ?? { value: REQUEST_TAB_BODY, label: bodyTypeLabel(request.bodyType, request.contentType), + empty: dimEmptyTabs && request.bodyType == null, rightSlot: numParams > 0 ? : null, }, { value: REQUEST_TAB_PARAMS, rightSlot: , label: "Params", + empty: dimEmptyTabs && paramPairs.length === 0, }, ...(headersTabs ?? [ { value: REQUEST_TAB_HEADERS, label: "Headers", - rightSlot: ( - h.name && h.enabled !== false).length} - /> - ), + empty: dimEmptyTabs && headerCount === 0, + rightSlot: , }, ]), ...(authTabs ?? @@ -205,14 +221,15 @@ export function HttpRequestPaneView({ [ authTabs, bodyTab, + dimEmptyTabs, extraTabs, + headerCount, headersTabs, numParams, paramPairs.length, request.auth, request.bodyType, request.contentType, - request.headers, ], ); @@ -235,6 +252,7 @@ export function HttpRequestPaneView({ style={style} className={classNames(className, "h-full grid grid-rows-[auto_minmax(0,1fr)] grid-cols-1")} > + {urlLine ?? ( + )} {authContent ?? (request.auth != null && )} diff --git a/packages/ui/src/components/http/HttpResponsePaneView.tsx b/packages/ui/src/components/http/HttpResponsePaneView.tsx index 2549182df..7d0d10537 100644 --- a/packages/ui/src/components/http/HttpResponsePaneView.tsx +++ b/packages/ui/src/components/http/HttpResponsePaneView.tsx @@ -69,6 +69,22 @@ export interface HttpResponsePaneViewProps { infoRows?: ReactElement[]; /** The copy button for a body held back for its size */ renderCopyButton?: (getText: () => Promise) => ReactNode; + /** + * The request is shown apart, as the proxy's card shows it beside the response: no Request + * tab, and Headers and Cookies hold the response's alone + */ + requestShownApart?: boolean; + /** Drawn inside a card the app draws, without a card of its own */ + bare?: boolean; + /** Tabs with nothing in them read in the subtlest colour */ + dimEmptyTabs?: boolean; + /** + * The tab to open on for a key with no tab of its own; with it, a tab chosen under one key + * doesn't carry to the next + */ + defaultTab?: string; + /** A tab was chosen */ + onTabChange?: (tab: string) => void; } /** Yaak's response pane: a status line over Response, Request, Headers and Cookies tabs */ @@ -95,6 +111,11 @@ export function HttpResponsePaneView({ renderBodyViewer, infoRows, renderCopyButton, + requestShownApart = false, + bare = false, + dimEmptyTabs = false, + defaultTab, + onTabChange, }: HttpResponsePaneViewProps) { const { ErrorBoundary } = useEditorHost(); const mimeType = @@ -138,9 +159,15 @@ export function HttpResponsePaneView({ : []), ]; + const noBody = response == null || (response.contentLength ?? 0) === 0; + const headerCount = response?.headers.length ?? 0; + const cookieCount = requestShownApart + ? cookieCounts.received + : cookieCounts.sent + cookieCounts.received; return [ { value: RESPONSE_TAB_BODY, + empty: dimEmptyTabs && noBody, options: { value: viewMode, onChange: (v) => onViewModeChange(v === "raw" ? "raw" : "pretty"), @@ -153,40 +180,48 @@ export function HttpResponsePaneView({ itemsAfter: itemsAfter.length > 0 ? itemsAfter : undefined, }, }, - { - value: RESPONSE_TAB_REQUEST, - label: "Request", - rightSlot: (response?.requestContentLength ?? 0) > 0 ? : null, - }, + ...(requestShownApart + ? [] + : [ + { + value: RESPONSE_TAB_REQUEST, + label: "Request", + rightSlot: + (response?.requestContentLength ?? 0) > 0 ? : null, + }, + ]), { value: RESPONSE_TAB_HEADERS, label: "Headers", - rightSlot: ( - + empty: dimEmptyTabs && headerCount === 0, + rightSlot: requestShownApart ? ( + + ) : ( + ), }, { value: RESPONSE_TAB_COOKIES, label: "Cookies", - rightSlot: - cookieCounts.sent > 0 || cookieCounts.received > 0 ? ( - - ) : null, + empty: dimEmptyTabs && cookieCount === 0, + rightSlot: requestShownApart ? ( + + ) : cookieCounts.sent > 0 || cookieCounts.received > 0 ? ( + + ) : null, }, ...(extraTabs?.map((t) => t.tab) ?? []), ]; }, [ cookieCounts, + dimEmptyTabs, extraTabs, mimeType, onCopyAsCurl, onCopyBody, onSaveBody, onViewModeChange, + requestShownApart, response, viewMode, ]); @@ -197,9 +232,8 @@ export function HttpResponsePaneView({ className={classNames( className, "x-theme-responsePane", - "max-h-full h-full", - "bg-surface rounded-md border border-border-subtle overflow-hidden", - "relative", + "max-h-full h-full overflow-hidden relative", + !bare && "bg-surface rounded-md border border-border-subtle", )} > {response == null ? ( @@ -234,6 +268,9 @@ export function HttpResponsePaneView({ tabListClassName="mt-0.5 -mb-1.5" storageKey={tabsStorageKey} activeTabKey={activeTabKey} + defaultValue={defaultTab} + keyedDefault={defaultTab != null} + onChangeValue={onTabChange} > @@ -259,6 +296,7 @@ export function HttpResponsePaneView({ + {!requestShownApart && ( + )} - + - + {extraTabs?.map(({ tab, content }) => ( diff --git a/packages/ui/src/components/http/ResponseCookies.tsx b/packages/ui/src/components/http/ResponseCookies.tsx index 61649b941..29801ae9e 100644 --- a/packages/ui/src/components/http/ResponseCookies.tsx +++ b/packages/ui/src/components/http/ResponseCookies.tsx @@ -10,6 +10,8 @@ import type { HttpResponseView } from "./types"; interface Props { response: HttpResponseView; + /** Only the cookies sent or only those received, for a pane that shows one side */ + side?: "sent" | "received"; } interface ParsedCookie { @@ -102,7 +104,7 @@ export function parseCookieHeaders({ sent, received }: HttpResponseView["cookieH }; } -export function ResponseCookies({ response }: Props) { +export function ResponseCookies({ response, side }: Props) { const { useDetailsBannerStoredValue } = useEditorHost(); const { sentCookies, receivedCookies } = useMemo( () => parseCookieHeaders(response.cookieHeaders), @@ -111,6 +113,7 @@ export function ResponseCookies({ response }: Props) { return (
+ {side !== "received" && ( )} + )} + {side !== "sent" && ( )} + )}
); } diff --git a/packages/ui/src/components/http/ResponseHeaders.tsx b/packages/ui/src/components/http/ResponseHeaders.tsx index 3206152de..6f7c050c6 100644 --- a/packages/ui/src/components/http/ResponseHeaders.tsx +++ b/packages/ui/src/components/http/ResponseHeaders.tsx @@ -2,6 +2,7 @@ import { format, formatDistanceToNowStrict } from "date-fns"; import type { ReactElement } from "react"; import { DetailsBanner } from "../DetailsBanner"; import { useEditorHost } from "../EditorHost"; +import { EmptyStateText } from "../EmptyStateText"; import { HeadersTable } from "../HeadersTable"; import { IconButton } from "../IconButton"; import type { KeyValueRowProps } from "../KeyValueRow"; @@ -12,10 +13,30 @@ interface Props { response: HttpResponseView; /** More rows for the Info section, after the standard ones, each keyed */ infoRows?: ReactElement[]; + /** Only the response's headers, for a pane whose request and info are shown elsewhere */ + responseOnly?: boolean; } -export function ResponseHeaders({ response, infoRows }: Props) { +export function ResponseHeaders({ response, infoRows, responseOnly }: Props) { const { openUrl, useDetailsBannerStoredValue } = useEditorHost(); + if (responseOnly) { + return ( +
+ {response.headers.length === 0 ? ( + No Headers + ) : ( + + {response.headers.map((h, i) => ( + // oxlint-disable-next-line react/no-array-index-key + + {h.value} + + ))} + + )} +
+ ); + } return (
!view.hiddenInterfaceRows?.includes(row); return ( @@ -125,43 +131,55 @@ export function SettingsInterface({ - - callbacks.onChange({ editorKeymap })} - options={keymaps} - /> - callbacks.onChange({ editorSoftWrap })} - /> - callbacks.onChange({ coloredMethods })} - /> - + {(shown("editorKeymap") || shown("editorSoftWrap") || shown("coloredMethods")) && ( + + {shown("editorKeymap") && ( + callbacks.onChange({ editorKeymap })} + options={keymaps} + /> + )} + {shown("editorSoftWrap") && ( + callbacks.onChange({ editorSoftWrap })} + /> + )} + {shown("coloredMethods") && ( + callbacks.onChange({ coloredMethods })} + /> + )} + + )} - - - {view.windowControlsSetting && ( - callbacks.onChange({ hideWindowControls })} - /> - )} - + {(shown("nativeTitlebar") || view.windowControlsSetting) && ( + + {shown("nativeTitlebar") && ( + + )} + {view.windowControlsSetting && ( + callbacks.onChange({ hideWindowControls })} + /> + )} + + )} {trailingSections} diff --git a/packages/ui/src/components/settings/types.ts b/packages/ui/src/components/settings/types.ts index ea0355b39..72e71aa6a 100644 --- a/packages/ui/src/components/settings/types.ts +++ b/packages/ui/src/components/settings/types.ts @@ -106,8 +106,17 @@ export interface SettingsView { feedbackPrompts: boolean; /** Null when this build has no license */ license: SettingsLicenseView | null; + /** Interface rows this app doesn't act on yet, left out of the tab */ + hiddenInterfaceRows?: SettingsInterfaceRow[]; } +/** The Interface tab's rows an app may leave out */ +export type SettingsInterfaceRow = + | "editorKeymap" + | "editorSoftWrap" + | "coloredMethods" + | "nativeTitlebar"; + export interface SettingsCallbacks { onChange: (patch: SettingsPatch) => void | Promise; /** Save the native title bar choice and restart, since it only applies at launch */ diff --git a/packages/ui/src/components/tabs/Tabs.tsx b/packages/ui/src/components/tabs/Tabs.tsx index 8dd9a24bf..541117c8b 100644 --- a/packages/ui/src/components/tabs/Tabs.tsx +++ b/packages/ui/src/components/tabs/Tabs.tsx @@ -41,6 +41,8 @@ export type TabItem = value: string; label: string; hidden?: boolean; + /** Nothing in it: the label reads in the subtlest colour */ + empty?: boolean; leftSlot?: ReactNode; rightSlot?: ReactNode; } @@ -57,6 +59,8 @@ export type TabItem = export interface TabOptionsItem { value: string; options: Omit; + /** Nothing in it: the label reads in the subtlest colour */ + empty?: boolean; leftSlot?: ReactNode; rightSlot?: ReactNode; } @@ -89,6 +93,11 @@ export interface TabsProps { activeTabKey?: string; /** Where the order and active tab under `storageKey` are kept */ useStoredValue?: UseStoredValue; + /** + * A key with no stored tab opens on `defaultValue`, not on the tab last chosen under another + * key, and a stored tab no longer offered falls back the same way + */ + keyedDefault?: boolean; } export const Tabs = forwardRef(function Tabs( @@ -105,6 +114,7 @@ export const Tabs = forwardRef(function Tabs( storageKey, activeTabKey, useStoredValue: useStoredValueProp, + keyedDefault = false, }: TabsProps, forwardedRef: Ref, ) { @@ -131,8 +141,12 @@ export const Tabs = forwardRef(function Tabs( // Get the active tab value - prefer storage (if activeTabKey), then defaultValue, then first tab const storedActiveTab = activeTabKey ? storage?.activeTabs?.[activeTabKey] : undefined; - const [internalValue, setInternalValue] = useState(undefined); - const value = storedActiveTab ?? internalValue ?? defaultValue ?? originalTabs[0]?.value; + const [internal, setInternal] = useState<{ key?: string; value: string } | undefined>(undefined); + const internalValue = + keyedDefault && internal?.key !== activeTabKey ? undefined : internal?.value; + const offered = (v: string | undefined) => + v != null && (!keyedDefault || originalTabs.some((t) => t.value === v)); + const value = [storedActiveTab, internalValue, defaultValue].find(offered) ?? originalTabs[0]?.value; // Helper to normalize storage (handle migration from old format) const normalizeStorage = useCallback( @@ -144,7 +158,7 @@ export const Tabs = forwardRef(function Tabs( // Handle tab change - update internal state, storage if we have a key, and call prop callback const onChangeValue = useCallback( async (newValue: string) => { - setInternalValue(newValue); + setInternal({ key: activeTabKey, value: newValue }); if (storageKey && activeTabKey) { await setStorage((s) => { const normalized = normalizeStorage(s); @@ -501,7 +515,15 @@ function TabButton({ ) : "options" in tab ? ( ) : ( - ); @@ -574,6 +596,7 @@ function TabOptionsButton({
} {...buttonProps} + className={classNames(buttonProps.className, !isActive && tab.empty && "text-text-subtlest!")} > {option && "shortLabel" in option && option.shortLabel ? option.shortLabel