From 4538123d88243246a3e799ed09185d84d5c6f4c5 Mon Sep 17 00:00:00 2001 From: Gregory Schier Date: Fri, 6 Nov 2026 21:24:39 -0700 Subject: [PATCH] Make HTTPS decryption opt-in with a decrypt list (#822) Co-authored-by: Claude Opus 5.5 --- apps/yaak-client/components/Sidebar.tsx | 3 +- .../components/DecryptListEditor.tsx | 187 ++++++++ apps/yaak-proxy/components/FirstRun.tsx | 48 +- apps/yaak-proxy/components/FlowDetail.tsx | 3 + apps/yaak-proxy/components/FlowList.tsx | 20 +- apps/yaak-proxy/components/NotDecrypted.tsx | 85 ++++ apps/yaak-proxy/components/ProxyLayout.tsx | 2 + apps/yaak-proxy/components/ProxyStatus.tsx | 7 + apps/yaak-proxy/components/Sidebar.tsx | 77 ++- apps/yaak-proxy/harness/fixtures.ts | 1 + apps/yaak-proxy/harness/streamFixtures.ts | 1 + apps/yaak-proxy/hooks/useDecryptList.ts | 45 ++ apps/yaak-proxy/lib/decrypt.test.ts | 305 ++++++++++++ apps/yaak-proxy/lib/decrypt.ts | 394 ++++++++++++++++ apps/yaak-proxy/lib/firstRun.test.ts | 20 +- apps/yaak-proxy/lib/firstRun.ts | 21 + apps/yaak-proxy/lib/flowViews.test.ts | 1 + apps/yaak-proxy/lib/search.test.ts | 10 + apps/yaak-proxy/lib/search.ts | 9 +- apps/yaak-proxy/lib/sidebarTree.test.ts | 38 ++ apps/yaak-proxy/lib/sidebarTree.ts | 39 +- .../yaak-proxy-lib/bindings/gen_models.ts | 8 +- .../yaak-proxy-lib/bindings/gen_rpc.ts | 64 ++- .../migrations/00000009_flow_tunnelled.sql | 2 + crates-proxy/yaak-proxy-lib/src/client.rs | 38 +- crates-proxy/yaak-proxy-lib/src/control.rs | 20 +- crates-proxy/yaak-proxy-lib/src/db.rs | 125 ++++- crates-proxy/yaak-proxy-lib/src/flows.rs | 123 ++++- crates-proxy/yaak-proxy-lib/src/lib.rs | 179 ++++++- crates-proxy/yaak-proxy-lib/src/models.rs | 7 + crates-proxy/yaak-proxy-lib/src/settings.rs | 196 +++++++- crates-proxy/yaak-proxy-lib/tests/control.rs | 109 +++++ crates-proxy/yaak-proxy-lib/tests/flow.rs | 215 ++++++++- crates/yaak-proxy/src/decrypt.rs | 445 ++++++++++++++++++ crates/yaak-proxy/src/lib.rs | 75 ++- crates/yaak-proxy/src/request.rs | 54 ++- crates/yaak-proxy/src/tunnel.rs | 68 +++ crates/yaak-proxy/tests/proxy.rs | 175 ++++++- .../components/editor}/filter/extension.ts | 100 +++- .../components/editor}/filter/filter.grammar | 0 .../components/editor}/filter/filter.test.ts | 0 .../src/components/editor}/filter/filter.ts | 0 .../components/editor}/filter/highlight.ts | 0 packages/ui/src/editor.ts | 6 + 44 files changed, 3259 insertions(+), 66 deletions(-) create mode 100644 apps/yaak-proxy/components/DecryptListEditor.tsx create mode 100644 apps/yaak-proxy/components/NotDecrypted.tsx create mode 100644 apps/yaak-proxy/hooks/useDecryptList.ts create mode 100644 apps/yaak-proxy/lib/decrypt.test.ts create mode 100644 apps/yaak-proxy/lib/decrypt.ts create mode 100644 crates-proxy/yaak-proxy-lib/migrations/00000009_flow_tunnelled.sql create mode 100644 crates/yaak-proxy/src/decrypt.rs rename {apps/yaak-client/components/core/Editor => packages/ui/src/components/editor}/filter/extension.ts (66%) rename {apps/yaak-client/components/core/Editor => packages/ui/src/components/editor}/filter/filter.grammar (100%) rename {apps/yaak-client/components/core/Editor => packages/ui/src/components/editor}/filter/filter.test.ts (100%) rename {apps/yaak-client/components/core/Editor => packages/ui/src/components/editor}/filter/filter.ts (100%) rename {apps/yaak-client/components/core/Editor => packages/ui/src/components/editor}/filter/highlight.ts (100%) diff --git a/apps/yaak-client/components/Sidebar.tsx b/apps/yaak-client/components/Sidebar.tsx index b072b148a..80d474656 100644 --- a/apps/yaak-client/components/Sidebar.tsx +++ b/apps/yaak-client/components/Sidebar.tsx @@ -63,8 +63,7 @@ import { isSidebarFocused } from "../lib/scopes"; import { navigateToRequestOrFolderOrWorkspace } from "../lib/setWorkspaceSearchParams"; import type { ContextMenuProps, DropdownItem } from "./core/Dropdown"; import { ContextMenu, Dropdown } from "./core/Dropdown"; -import type { FieldDef } from "./core/Editor/filter/extension"; -import { filter } from "./core/Editor/filter/extension"; +import { type FieldDef, filter } from "@yaakapp-internal/ui/editor"; import type { Ast } from "./core/Editor/filter/query"; import { evaluate, parseQuery } from "./core/Editor/filter/query"; import { formatFieldFilter } from "./core/Editor/filter/format"; diff --git a/apps/yaak-proxy/components/DecryptListEditor.tsx b/apps/yaak-proxy/components/DecryptListEditor.tsx new file mode 100644 index 000000000..74f956d0a --- /dev/null +++ b/apps/yaak-proxy/components/DecryptListEditor.tsx @@ -0,0 +1,187 @@ +import { Prec } from "@codemirror/state"; +import { type EditorView, keymap } from "@codemirror/view"; +import type { DecryptListEntry, SeenTargets } from "@yaakapp-internal/proxy-lib"; +import { Dialog, Icon, IconButton, Input } from "@yaakapp-internal/ui"; +import { filterAutocomplete } from "@yaakapp-internal/ui/editor"; +import classNames from "classnames"; +import { atom, useAtom } from "jotai"; +import { useMemo, useRef, useState } from "react"; +import { useDecryptList, useEditDecryptList } from "../hooks/useDecryptList"; +import { useRpcQuery } from "../hooks/useRpcQuery"; +import { decryptFilterOptions, entryLabel, normalizeEntry } from "../lib/decrypt"; + +/** + * What HTTPS is decrypted: one row per entry, the user's in order and then the built-in + * exclusions, each removable, and an input that adds one, suggesting `host:`, `app:` and `!`, + * then the hosts and apps seen in the capture. Saves as it changes. Placed by Settings; until + * that exists, by the dialog the state pill's More… opens. + */ +export function DecryptListEditor({ className }: { className?: string }) { + const list = useDecryptList(); + const save = useEditDecryptList(); + // Read when the input opens its suggestions, so new traffic shows without rebuilding it. + const seen = useRpcQuery("list_seen", {}, { staleTime: 0, refetchOnMount: "always" }); + const seenRef = useRef(seen.data); + seenRef.current = seen.data; + + // Each change replaces the whole list, so one waits for the last to land rather than undo it. + const remove = (entry: string) => { + if (save.isPending) return; + save.mutate({ add: [], remove: [entry] }); + }; + + return ( +
+ save.mutate({ add: [entry], remove: [] })} + /> +
+ {list.length === 0 && ( +
Nothing decrypted
+ )} + {list.map((e) => ( + remove(e.entry)} + /> + ))} +
+
+ ); +} + +function EntryRow({ + entry, + disabled, + onRemove, +}: { + entry: DecryptListEntry; + disabled: boolean; + onRemove: () => void; +}) { + const negated = entry.entry.startsWith("!"); + return ( +
+ + + {entryLabel(entry.entry)} + + {entry.builtIn && ( + + built-in + + )} + +
+ ); +} + +/** The input that adds an entry on Enter, once the suggestions are closed or none is picked */ +function EntryInput({ + seenRef, + list, + disabled, + onAdd, +}: { + seenRef: React.RefObject; + list: DecryptListEntry[]; + /** A change is being saved; Enter waits for it, keeping what was typed */ + disabled: boolean; + onAdd: (entry: string) => void; +}) { + const [error, setError] = useState(null); + const onAddRef = useRef(onAdd); + onAddRef.current = onAdd; + const listRef = useRef(list); + listRef.current = list; + const disabledRef = useRef(disabled); + disabledRef.current = disabled; + + const extensions = useMemo(() => { + const options = decryptFilterOptions(null); + // The suggestions read the capture as they open, not as the input was made. + for (const field of options.fields ?? []) { + field.values = () => { + const fresh = decryptFilterOptions(seenRef.current).fields ?? []; + const values = fresh.find((f) => f.name === field.name)?.values; + return Array.isArray(values) ? values : []; + }; + } + const add = (view: EditorView) => { + const text = view.state.doc.toString(); + if (text.trim() === "") return false; + if (disabledRef.current) return true; + const entry = normalizeEntry(text); + if (entry == null) { + setError("Not a host or app"); + return true; + } + setError(null); + if (!listRef.current.some((e) => e.entry === entry)) onAddRef.current(entry); + view.dispatch({ changes: { from: 0, to: view.state.doc.length, insert: "" } }); + return true; + }; + // Low, so Enter picks a suggestion while they are open. + return [filterAutocomplete(options), Prec.low(keymap.of([{ key: "Enter", run: add }]))]; + }, [seenRef]); + + return ( +
+ setError(null)} + /> + {error != null &&
{error}
} +
+ ); +} + +/** Opens the decrypt list from the state pill's More…, until Settings holds it */ +export const decryptListDialogOpenAtom = atom(false); + +export function DecryptListDialog() { + const [open, setOpen] = useAtom(decryptListDialogOpenAtom); + return ( + setOpen(false)} + size="dynamic" + title="Decrypt List" + className="w-[36rem]" + > +
{open && }
+ + ); +} diff --git a/apps/yaak-proxy/components/FirstRun.tsx b/apps/yaak-proxy/components/FirstRun.tsx index 11c2c2185..a4b8faec4 100644 --- a/apps/yaak-proxy/components/FirstRun.tsx +++ b/apps/yaak-proxy/components/FirstRun.tsx @@ -2,7 +2,10 @@ import { Button, Icon, InlineCode, LoadingIcon } from "@yaakapp-internal/ui"; import classNames from "classnames"; import { useAtomValue } from "jotai"; import type { ReactNode } from "react"; +import { useDecryptList, useEditDecryptList } from "../hooks/useDecryptList"; import { useRpcMutation } from "../hooks/useRpcMutation"; +import { everythingEdit } from "../lib/decrypt"; +import { decryptStep } from "../lib/firstRun"; import { encryptingPercent } from "../lib/encrypting"; import { proxyStateAtom } from "../lib/store"; import { useCopyEnvironment, useProxySetup, useSystemProxySwitch } from "./ProxyStatus"; @@ -11,7 +14,7 @@ type StepState = "todo" | "next" | "wait" | "done" | "fail"; /** * The list's empty state until traffic arrives: trust the certificate, turn on the system proxy, - * then see traffic. Not dismissable and nothing remembered; the first flow replaces it. Copy + * choose what to decrypt, then see traffic. Not dismissable and nothing remembered; the first flow replaces it. Copy * environment beside it serves someone who wants one terminal proxied and nothing system-wide. */ export function FirstRun() { @@ -22,6 +25,8 @@ export function FirstRun() { const execute = useRpcMutation("execute_action"); const retry = useRpcMutation("take_over"); const copyEnvironment = useCopyEnvironment(); + const decryptList = useDecryptList(); + const saveDecryptList = useEditDecryptList(); const running = proxyState?.state === "running"; // Changed only by the process that owns the engine, so not from here while the CLI does. @@ -47,6 +52,9 @@ export function FirstRun() { : trusted ? "next" : "todo"; + const decrypt = decryptStep(decryptList, trusted); + const everything = decrypt.choice === "everything"; + const setEverything = (on: boolean) => saveDecryptList.mutate(everythingEdit(on)); const ready = trusted && (systemOn || !systemSupported); const encrypting = !running && !otherOwner ? encryptingPercent(proxyState) : null; const trafficState: StepState = @@ -57,8 +65,8 @@ export function FirstRun() {
Set up Yaak Proxy
- Three steps, once. Then every app on this Mac sends its traffic here, and HTTPS reads as - plain text. + {systemSupported ? "Four" : "Three"} steps, once. Then every app on this Mac sends its + traffic here.
Everything + ) : ( + Pick as they appear + ) + } + action={ + everything ? ( + + ) : ( + + ) + } + /> +
+ ) : flow.tunnelled ? ( + ) : ( )} diff --git a/apps/yaak-proxy/components/FlowList.tsx b/apps/yaak-proxy/components/FlowList.tsx index 37c12e924..36811a46c 100644 --- a/apps/yaak-proxy/components/FlowList.tsx +++ b/apps/yaak-proxy/components/FlowList.tsx @@ -457,14 +457,15 @@ const FlowRow = memo(function FlowRow({ }); /** - * One mark per row, by a fixed priority: failed, cancelled, cut, saved. Its tooltip lists every - * state the flow has. Streaming and agent reads join it once the engine records them. + * One mark per row, by a fixed priority: failed, cancelled, not decrypted, streaming, cut, saved. + * Its tooltip lists every state the flow has. Agent reads join it once the engine records them. */ function FlowMark({ flow }: { flow: Flow }) { const open = isOpenStream(flow); const states: string[] = []; if (flow.error != null) states.push(`Failed: ${flow.error}`); if (flow.cancelled) states.push("Cancelled by the app"); + if (flow.tunnelled) states.push("Not decrypted"); if (open) states.push("Streaming"); if (flow.responseTruncated) states.push("Body cut at the capture limit"); if (flow.savedRequestId != null) states.push("Saved to Yaak"); @@ -475,6 +476,8 @@ function FlowMark({ flow }: { flow: Flow }) { ) : flow.cancelled ? ( + ) : flow.tunnelled ? ( + ) : open ? ( ) : flow.responseTruncated ? ( @@ -488,6 +491,11 @@ function FlowMark({ flow }: { flow: Flow }) { const mono = "font-mono text-xs"; +/** An undecrypted tunnel still open: written as it opens, done once it closes */ +function isOpenTunnel(flow: Flow): boolean { + return flow.tunnelled && flow.elapsedMs == null && flow.error == null; +} + /** A WebSocket, event stream or gRPC call still open: written as it opens, done once it ends */ function isOpenStream(flow: Flow): boolean { return flow.streamKind != null && flow.elapsedMs == null && flow.error == null && !flow.cancelled; @@ -529,6 +537,8 @@ function FlowCell({ column, flow, span }: { column: ColumnId; flow: Flow; span: ); case "path": { + // A tunnel's only path is its host; what it carried is unread. + if (flow.tunnelled) return Not decrypted; const { path, query } = splitPath(flow.url); if (flow.streamKind == null) { return ( @@ -556,7 +566,7 @@ function FlowCell({ column, flow, span }: { column: ColumnId; flow: Flow; span: ); } case "status": - return flow.cancelled && flow.status == null ? ( + return (flow.cancelled || flow.tunnelled) && flow.status == null ? ( -- ) : ( @@ -589,8 +599,8 @@ function FlowCell({ column, flow, span }: { column: ColumnId; flow: Flow; span: ); case "time": - // An open stream shows how long it has been open, quietly, as its messages land. - if (isOpenStream(flow)) { + // An open stream or tunnel shows how long it has been open, quietly, as its messages land. + if (isOpenStream(flow) || isOpenTunnel(flow)) { const open = Date.now() - Date.parse(utcTimestamp(flow.startedAt)); return ( diff --git a/apps/yaak-proxy/components/NotDecrypted.tsx b/apps/yaak-proxy/components/NotDecrypted.tsx new file mode 100644 index 000000000..13b28ab3f --- /dev/null +++ b/apps/yaak-proxy/components/NotDecrypted.tsx @@ -0,0 +1,85 @@ +import { formatSize } from "@yaakapp-internal/lib/formatSize"; +import type { Flow } from "@yaakapp-internal/proxy-lib"; +import { Button, formatMillis, Icon } from "@yaakapp-internal/ui"; +import { useDecryptList, useEditDecryptList } from "../hooks/useDecryptList"; +import { type DecryptSubject, detailActions, listEdit, setDecrypted } from "../lib/decrypt"; +import { sidebarPath } from "../lib/search"; +import { EmptyState } from "./FlowBodyViewer"; + +/** + * The detail of HTTPS passed through undecrypted: its sizes and time, and buttons that add its + * host or its app to the decrypt list, from the next connection on. + */ +export function NotDecrypted({ flow }: { flow: Flow }) { + const list = useDecryptList(); + const save = useEditDecryptList(); + const host = sidebarPath(flow.url).split("/")[0] ?? flow.url; + const hostSubject: DecryptSubject = { kind: "host", host }; + const appSubject: DecryptSubject | null = + flow.clientName == null + ? null + : { kind: "app", name: flow.clientName, bundleId: flow.clientBundleId }; + const app = flow.clientName; + const actions = detailActions( + list, + host, + app == null ? null : { name: app, bundleId: flow.clientBundleId }, + ); + const open = flow.elapsedMs == null && flow.error == null; + + const decrypt = (subject: DecryptSubject) => + save.mutate(listEdit(list, setDecrypted(list, [subject], true))); + + const facts = [ + `${formatSize(Number(flow.requestSize))} sent`, + `${formatSize(Number(flow.responseSize))} received`, + open ? "open" : flow.elapsedMs == null ? null : formatMillis(flow.elapsedMs), + ].filter((f) => f != null); + + return ( + +
+
+ + Not decrypted +
+
{facts.join(" · ")}
+ {flow.error != null &&
{flow.error}
} + {actions.listed ? ( +
+ Decrypting new connections +
+ ) : actions.blocked != null ? ( +
+ Excluded by {actions.blocked} +
+ ) : ( +
+ {actions.host && ( + + )} + {actions.app && appSubject != null && ( + + )} +
+ )} +
+ + ); +} diff --git a/apps/yaak-proxy/components/ProxyLayout.tsx b/apps/yaak-proxy/components/ProxyLayout.tsx index 5c936cb9b..9c77af1c3 100644 --- a/apps/yaak-proxy/components/ProxyLayout.tsx +++ b/apps/yaak-proxy/components/ProxyLayout.tsx @@ -20,6 +20,7 @@ import { FlowDetail } from "./FlowDetail"; import { FlowList } from "./FlowList"; import { LockedState } from "./LockedState"; import { ProxyStatus, useProxySetup } from "./ProxyStatus"; +import { DecryptListDialog } from "./DecryptListEditor"; import { HelperDialog } from "./HelperDialog"; import { SaveDialog } from "./SaveDialog"; import { SearchBar } from "./SearchBar"; @@ -150,6 +151,7 @@ export function ProxyLayout() { +
); diff --git a/apps/yaak-proxy/components/ProxyStatus.tsx b/apps/yaak-proxy/components/ProxyStatus.tsx index 34e5761b4..185517c98 100644 --- a/apps/yaak-proxy/components/ProxyStatus.tsx +++ b/apps/yaak-proxy/components/ProxyStatus.tsx @@ -17,6 +17,7 @@ import { useRpcEvent } from "../hooks/useRpcEvent"; import { useRpcMutation } from "../hooks/useRpcMutation"; import { useRpcQuery } from "../hooks/useRpcQuery"; import { helperDialogOpenAtom, helperRegisteredAtAtom } from "../lib/atoms"; +import { decryptListDialogOpenAtom } from "./DecryptListEditor"; import { encryptingPercent } from "../lib/encrypting"; import { environmentExports } from "../lib/env"; import { proxyStateAtom, refreshProxyState, systemProxyEverOnAtom } from "../lib/store"; @@ -45,6 +46,7 @@ export function ProxyStatus() { const setHttpVersion = useSetHttpVersion(); const copyEnvironment = useCopyEnvironment(); const setHelperDialogOpen = useSetAtom(helperDialogOpenAtom); + const setDecryptListOpen = useSetAtom(decryptListDialogOpenAtom); const { caStatus, trust, systemProxy, setSystemProxy } = setup; const systemSwitch = useSystemProxySwitch(setup); @@ -165,6 +167,11 @@ export function ProxyStatus() { ), }, { type: "separator" }, + { + label: "Decrypt List…", + icon: "lock_open", + onSelect: () => setDecryptListOpen(true), + }, { label: "Copy Environment for a Terminal", icon: "square_terminal", diff --git a/apps/yaak-proxy/components/Sidebar.tsx b/apps/yaak-proxy/components/Sidebar.tsx index a7d1b1d18..d519b3ab7 100644 --- a/apps/yaak-proxy/components/Sidebar.tsx +++ b/apps/yaak-proxy/components/Sidebar.tsx @@ -1,12 +1,22 @@ -import { Icon, IconButton, Tree } from "@yaakapp-internal/ui"; +import { + ContextMenu, + type DropdownItem, + Icon, + IconButton, + Tree, + type TreeProps, +} from "@yaakapp-internal/ui"; import classNames from "classnames"; import { atom, useAtom, useAtomValue } from "jotai"; import { atomWithStorage } from "jotai/utils"; -import { useCallback } from "react"; +import { useCallback, useEffect } from "react"; +import { useDecryptList, useEditDecryptList } from "../hooks/useDecryptList"; import { SIDEBAR_TREE_ID } from "../lib/atoms"; +import { blockerOf, decryptMenuAction, isDecrypted, listEdit, setDecrypted } from "../lib/decrypt"; import { applyCollapsed, type CollapsedMap, collapsedMap, folderIds } from "../lib/sidebarCollapse"; import { type SidebarItem, SYSTEM_ID } from "../lib/sidebarTree"; import { hideSystemAtom, noteSidebarInput, sidebarTreeAtom } from "../lib/store"; +import { useShowToast } from "../lib/toast"; import { AppIcon } from "./AppIcon"; /** The apps, sites, hosts and paths the user expanded, by node id, kept across launches */ @@ -52,16 +62,33 @@ function ItemInner({ item }: { item: SidebarItem }) { return (
{item.kind === "all" && } + {item.kind === "tunnelled" && } {item.appPath != null && } {item.label} {item.sublabel != null && ( {item.sublabel} )} + {item.decrypt != null && }
); } +/** A row with nothing in its menu: the menu closes as it opens, so the row is not left marked */ +function CloseAtOnce({ onClose }: { onClose: () => void }) { + useEffect(() => onClose(), [onClose]); + return null; +} + +/** A small open lock on a host, site or app the decrypt list decrypts */ +function DecryptedMark({ item }: { item: SidebarItem }) { + const list = useDecryptList(); + if (item.decrypt == null || !isDecrypted(list, item.decrypt)) return null; + return ( + + ); +} + /** Beside the row's button rather than in it, since it is a button of its own */ function ItemRightSlot({ item }: { item: SidebarItem }) { return item.id === SYSTEM_ID ? : ; @@ -94,6 +121,50 @@ export function Sidebar() { (item: SidebarItem) => `${item.id}:${item.count}:${item.countLabel ?? ""}`, [], ); + const decryptList = useDecryptList(); + const setDecryptList = useEditDecryptList(); + const { mutate: saveDecryptList } = setDecryptList; + const showToast = useShowToast(); + + // Decrypt or Stop Decrypting the selected hosts, sites and apps, from new connections on. + const getContextMenu = useCallback( + (items: SidebarItem[]): DropdownItem[] => { + const subjects = items.flatMap((i) => (i.decrypt == null ? [] : [i.decrypt])); + const action = decryptMenuAction(decryptList, subjects); + if (action == null) return []; + return [ + { + label: action.label, + icon: action.on ? "lock_open" : "lock", + onSelect: () => { + const blocked = action.on ? blockerOf(decryptList, subjects) : null; + if (blocked != null) { + showToast({ + id: "decrypt-blocked", + color: "notice", + message: `Excluded by ${blocked}`, + }); + return; + } + // Only what this changes, applied to the list as it is then, so a second change + // made before this one lands keeps it. + const next = setDecrypted(decryptList, subjects, action.on); + saveDecryptList(listEdit(decryptList, next)); + }, + }, + ]; + }, + [decryptList, saveDecryptList, showToast], + ); + const renderContextMenu = useCallback["renderContextMenu"]>>( + ({ items, position, onClose }) => + items.length === 0 ? ( + + ) : ( + + ), + [], + ); return ( // The tree scrolls itself, in a row of bounded height, so its virtualizer mounts only the rows @@ -114,6 +185,8 @@ export function Sidebar() { getItemKey={getItemKey} ItemInner={ItemInner} ItemRightSlot={ItemRightSlot} + getContextMenu={getContextMenu} + renderContextMenu={renderContextMenu} /> ); diff --git a/apps/yaak-proxy/harness/fixtures.ts b/apps/yaak-proxy/harness/fixtures.ts index 4c12f1c24..b58b87f09 100644 --- a/apps/yaak-proxy/harness/fixtures.ts +++ b/apps/yaak-proxy/harness/fixtures.ts @@ -50,6 +50,7 @@ const base: Flow = { grpcMethod: null, grpcStatus: null, grpcMessage: null, + tunnelled: false, }; const jsonRequest = JSON.stringify({ amount: 4200, currency: "usd", capture: true }); diff --git a/apps/yaak-proxy/harness/streamFixtures.ts b/apps/yaak-proxy/harness/streamFixtures.ts index 020ac2aca..835b5d3ed 100644 --- a/apps/yaak-proxy/harness/streamFixtures.ts +++ b/apps/yaak-proxy/harness/streamFixtures.ts @@ -49,6 +49,7 @@ const flowBase: Flow = { grpcMethod: null, grpcStatus: null, grpcMessage: null, + tunnelled: false, }; /** Seconds past 21:41:12 as a stored timestamp */ diff --git a/apps/yaak-proxy/hooks/useDecryptList.ts b/apps/yaak-proxy/hooks/useDecryptList.ts new file mode 100644 index 000000000..c8d67bfbe --- /dev/null +++ b/apps/yaak-proxy/hooks/useDecryptList.ts @@ -0,0 +1,45 @@ +import { useQueryClient } from "@tanstack/react-query"; +import type { + DecryptList, + DecryptListChangedPayload, + DecryptListEntry, +} from "@yaakapp-internal/proxy-lib"; +import { useCallback } from "react"; +import { useShowToast } from "../lib/toast"; +import { useRpcEvent } from "./useRpcEvent"; +import { useRpcMutation } from "./useRpcMutation"; +import { useRpcQuery } from "./useRpcQuery"; + +const KEY = ["get_decrypt_list", {}]; +const EMPTY: DecryptListEntry[] = []; + +/** + * The decrypt list's entries, empty until read. Kept current as this process changes it, from + * the app or through its control API. + */ +export function useDecryptList(): DecryptListEntry[] { + const queryClient = useQueryClient(); + const onChanged = useCallback( + (payload: DecryptListChangedPayload) => + queryClient.setQueryData(KEY, payload.list), + [queryClient], + ); + useRpcEvent("decrypt_list_changed", onChanged); + return useRpcQuery("get_decrypt_list", {}).data?.entries ?? EMPTY; +} + +/** + * Adds and removes entries, `edit_decrypt_list` applying them to the list as it is when they + * land, so two changes made in quick succession both stick. Everything that reads the list + * then reads it again. A failure, such as an entry that does not parse, is a toast. + */ +export function useEditDecryptList() { + const queryClient = useQueryClient(); + const showToast = useShowToast(); + return useRpcMutation("edit_decrypt_list", { + // Read again rather than taken from the answer, which can land after a newer change's + // event and would put the older list back. + onSuccess: () => queryClient.invalidateQueries({ queryKey: KEY }), + onError: (e) => showToast({ id: "decrypt-list", color: "danger", message: e.message }), + }); +} diff --git a/apps/yaak-proxy/lib/decrypt.test.ts b/apps/yaak-proxy/lib/decrypt.test.ts new file mode 100644 index 000000000..b14d48ac5 --- /dev/null +++ b/apps/yaak-proxy/lib/decrypt.test.ts @@ -0,0 +1,305 @@ +import { CompletionContext } from "@codemirror/autocomplete"; +import { EditorState } from "@codemirror/state"; +import type { DecryptListEntry } from "@yaakapp-internal/proxy-lib"; +import { filterCompletionSource } from "@yaakapp-internal/ui/editor"; +import { describe, expect, test } from "vite-plus/test"; +import { + blockerOf, + blockingExclusion, + decryptFilterOptions, + detailActions, + decryptMenuAction, + decrypts, + decryptsEverything, + entryLabel, + everythingEdit, + isDecrypted, + listEdit, + normalizeEntry, + setDecrypted, + subjectQuery, + toggleDecrypt, +} from "./decrypt"; + +const user = (...entries: string[]): DecryptListEntry[] => + entries.map((entry) => ({ entry, builtIn: false })); +const builtIn = (...entries: string[]): DecryptListEntry[] => + entries.map((entry) => ({ entry, builtIn: true })); +const BUILT_INS = builtIn("!host:*.apple.com", "!host:*.icloud.com"); + +describe("normalizeEntry", () => { + test("a bare hostname is a host entry", () => { + expect(normalizeEntry("api.example.com")).toBe("host:api.example.com"); + expect(normalizeEntry(" API.Example.com. ")).toBe("host:api.example.com"); + expect(normalizeEntry("https://api.example.com:8443/v1")).toBe("host:api.example.com:8443"); + }); + + test("prefixes and exclusions", () => { + expect(normalizeEntry("HOST:*.Example.com")).toBe("host:*.example.com"); + expect(normalizeEntry("!x.com")).toBe("!host:x.com"); + expect(normalizeEntry("! host:x.com")).toBe("!host:x.com"); + expect(normalizeEntry('app:"Google Chrome"')).toBe("app:Google Chrome"); + expect(normalizeEntry("App: com.google.Chrome")).toBe("app:com.google.Chrome"); + expect(normalizeEntry("host:[::1]:8443")).toBe("host:[::1]:8443"); + expect(normalizeEntry(" * ")).toBe("*"); + }); + + test("what is not an entry", () => { + for (const bad of ["", "!", "app:", "host:a b", "host:a.com:http", "host:a.com:99999"]) { + expect(normalizeEntry(bad)).toBeNull(); + } + }); + + test("labels", () => { + expect(entryLabel("*")).toBe("Everything"); + expect(entryLabel("host:a.com")).toBe("host:a.com"); + }); +}); + +describe("decrypts", () => { + test("an empty list decrypts nothing, with or without the built-ins", () => { + expect(decrypts([], { host: "a.com" })).toBe(false); + expect(decrypts(BUILT_INS, { host: "a.com" })).toBe(false); + }); + + test("hosts, sites and ports", () => { + const list = user("host:*.example.com", "host:other.com:8443"); + expect(decrypts(list, { host: "api.example.com" })).toBe(true); + expect(decrypts(list, { host: "example.com" })).toBe(true); + expect(decrypts(list, { host: "notexample.com" })).toBe(false); + expect(decrypts(list, { host: "other.com", port: 8443 })).toBe(true); + expect(decrypts(list, { host: "other.com" })).toBe(false); + }); + + test("apps by name or bundle id", () => { + const chrome = { name: "Google Chrome", bundleId: "com.google.Chrome" }; + expect(decrypts(user("app:google chrome"), { host: "a.com", app: chrome })).toBe(true); + expect(decrypts(user("app:com.google.*"), { host: "a.com", app: chrome })).toBe(true); + expect(decrypts(user("app:Safari"), { host: "a.com", app: chrome })).toBe(false); + }); + + test("everything keeps the built-in exclusions, which a specific entry beats", () => { + const list = [...user("*"), ...BUILT_INS]; + expect(decrypts(list, { host: "api.example.com" })).toBe(true); + expect(decrypts(list, { host: "gateway.icloud.com" })).toBe(false); + const specific = [...user("*", "host:developer.apple.com"), ...BUILT_INS]; + expect(decrypts(specific, { host: "developer.apple.com" })).toBe(true); + expect(decrypts(specific, { host: "www.apple.com" })).toBe(false); + }); + + test("the user's exclusions win wherever they sit", () => { + for (const list of [user("!host:a.example.com", "*"), user("*", "!host:a.example.com")]) { + expect(decrypts(list, { host: "a.example.com" })).toBe(false); + expect(decrypts(list, { host: "b.example.com" })).toBe(true); + } + }); +}); + +describe("the sidebar's Decrypt and Stop Decrypting", () => { + test("Decrypt adds the host's entry before the built-ins", () => { + const list = BUILT_INS; + const subject = { kind: "host", host: "api.example.com" } as const; + expect(isDecrypted(list, subject)).toBe(false); + expect(toggleDecrypt(list, subject)).toEqual([ + "host:api.example.com", + "!host:*.apple.com", + "!host:*.icloud.com", + ]); + }); + + test("Stop Decrypting removes the entry that decrypts it", () => { + const list = [...user("host:api.example.com", "host:b.com"), ...BUILT_INS]; + const subject = { kind: "host", host: "api.example.com" } as const; + expect(isDecrypted(list, subject)).toBe(true); + expect(toggleDecrypt(list, subject)).toEqual([ + "host:b.com", + "!host:*.apple.com", + "!host:*.icloud.com", + ]); + }); + + test("Stop Decrypting under everything excludes the host", () => { + const list = user("*"); + const subject = { kind: "site", site: "example.com" } as const; + expect(isDecrypted(list, subject)).toBe(true); + expect(toggleDecrypt(list, subject)).toEqual(["*", "!host:*.example.com"]); + }); + + test("Decrypt adds nothing while a broader exclusion of the user's would still win", () => { + const list = user("*", "!host:*.example.com"); + const subject = { kind: "host", host: "api.example.com" } as const; + expect(blockingExclusion(list, subjectQuery(subject))).toBe("!host:*.example.com"); + expect(toggleDecrypt(list, subject)).toEqual(["*", "!host:*.example.com"]); + // The built-ins are not the user's to block with: a specific entry beats them. + expect(blockingExclusion(BUILT_INS, { host: "www.apple.com" })).toBeNull(); + }); + + test("Decrypt undoes the sidebar's own Stop Decrypting", () => { + const subject = { kind: "site", site: "example.com" } as const; + const stopped = toggleDecrypt(user("*"), subject); + expect(stopped).toEqual(["*", "!host:*.example.com"]); + const list = user(...stopped); + expect(blockerOf(list, [subject])).toBeNull(); + expect(toggleDecrypt(list, subject)).toEqual(["*"]); + const host = { kind: "host", host: "api.example.com" } as const; + expect(blockerOf(list, [host])).toBe("!host:*.example.com"); + }); + + test("an exact site entry beats the built-in exclusion of the same site", () => { + expect(decrypts([...user("host:apple.com"), ...BUILT_INS], { host: "apple.com" })).toBe(true); + }); + + test("Decrypt drops the user's own exclusion first", () => { + const list = user("*", "!host:*.example.com"); + const subject = { kind: "site", site: "example.com" } as const; + expect(toggleDecrypt(list, subject)).toEqual(["*"]); + }); + + test("apps by name, and a host with a port", () => { + const slack = { kind: "app", name: "Slack", bundleId: "com.tinyspeck.slackmacgap" } as const; + expect(toggleDecrypt([], slack)).toEqual(["app:Slack"]); + expect(isDecrypted(user("app:com.tinyspeck.slackmacgap"), slack)).toBe(true); + expect(isDecrypted(user("*"), slack)).toBe(true); + const port = { kind: "host", host: "localhost:8443" } as const; + expect(toggleDecrypt([], port)).toEqual(["host:localhost:8443"]); + expect(isDecrypted(user("host:localhost:8443"), port)).toBe(true); + }); +}); + +describe("the sidebar menu", () => { + const a = { kind: "host", host: "a.com" } as const; + const b = { kind: "host", host: "b.com" } as const; + + test("offers Decrypt unless every selected row is decrypted", () => { + expect(decryptMenuAction(user("host:a.com"), [a])).toEqual({ + label: "Stop Decrypting", + on: false, + }); + expect(decryptMenuAction(user("host:a.com"), [a, b])).toEqual({ label: "Decrypt", on: true }); + expect(decryptMenuAction([], [])).toBeNull(); + }); + + test("applies to every selected row, leaving those already so", () => { + const next = setDecrypted(user("host:a.com"), [a, b], true).map((e) => e.entry); + expect(next).toEqual(["host:a.com", "host:b.com"]); + expect(setDecrypted(user("host:a.com", "host:b.com"), [a, b], false)).toEqual([]); + }); +}); + +describe("the Not decrypted detail's buttons", () => { + const slack = { name: "Slack", bundleId: "com.tinyspeck.slackmacgap" }; + + test("each button is offered only when its own change would decrypt the connection", () => { + // `!app:Slack` still excludes Slack's connections whatever host is added. + const list = user("*", "!app:Slack"); + expect(detailActions(list, "edgeapi.slack.com", slack)).toEqual({ + listed: false, + host: false, + app: true, + blocked: null, + }); + // And `!host:` the other way round. + expect(detailActions(user("*", "!host:edgeapi.slack.com"), "edgeapi.slack.com", slack)).toEqual( + { listed: false, host: true, app: false, blocked: null }, + ); + }); + + test("names the exclusion when neither would", () => { + const list = user("*", "!host:*.slack.com", "!app:Slack"); + expect(detailActions(list, "edgeapi.slack.com", slack)).toEqual({ + listed: false, + host: false, + app: false, + blocked: "!host:*.slack.com", + }); + }); + + test("both on an empty list, and none once it is listed", () => { + expect(detailActions(BUILT_INS, "a.com", slack)).toEqual({ + listed: false, + host: true, + app: true, + blocked: null, + }); + expect(detailActions(user("host:a.com"), "a.com", null).listed).toBe(true); + }); +}); + +describe("the first-run step", () => { + test("Everything adds *, and Pick as they appear takes it out", () => { + expect(decryptsEverything(BUILT_INS)).toBe(false); + expect(decryptsEverything([...user("*"), ...BUILT_INS])).toBe(true); + expect(everythingEdit(true)).toEqual({ add: ["*"], remove: [] }); + expect(everythingEdit(false)).toEqual({ add: [], remove: ["*"] }); + }); +}); + +describe("edits sent as additions and removals", () => { + test("a toggle becomes what it adds and removes, not the whole list", () => { + const list = [...user("*", "!host:*.example.com"), ...BUILT_INS]; + const site = { kind: "site", site: "example.com" } as const; + expect(listEdit(list, setDecrypted(list, [site], true))).toEqual({ + add: [], + remove: ["!host:*.example.com"], + }); + const host = { kind: "host", host: "a.com" } as const; + expect(listEdit(BUILT_INS, setDecrypted(BUILT_INS, [host], true))).toEqual({ + add: ["host:a.com"], + remove: [], + }); + }); + + test("two edits from the same stale list both survive, applied in turn", () => { + // What the lib does with each edit: remove, then add after the user's own entries. + const apply = (entries: string[], e: { add: string[]; remove: string[] }) => [ + ...entries.filter((x) => !e.remove.includes(x)), + ...e.add, + ]; + const stale = BUILT_INS; + const first = listEdit(stale, setDecrypted(stale, [{ kind: "host", host: "a.com" }], true)); + const second = listEdit(stale, setDecrypted(stale, [{ kind: "host", host: "b.com" }], true)); + const saved = apply(apply([], first), second); + expect(saved).toEqual(["host:a.com", "host:b.com"]); + }); +}); + +describe("the list's autocomplete", () => { + const source = filterCompletionSource( + decryptFilterOptions({ + hosts: ["api.example.com", "www.example.org"], + apps: [ + { name: "Google Chrome", bundleId: "com.google.Chrome" }, + { name: "curl", bundleId: null }, + ], + }), + ); + + function complete(doc: string) { + const state = EditorState.create({ doc }); + const result = source(new CompletionContext(state, doc.length, true)); + if (result == null || result instanceof Promise) return null; + return { from: result.from, labels: result.options.map((o) => o.label) }; + } + + test("an empty input offers host:, app: and !", () => { + expect(complete("")).toEqual({ from: 0, labels: ["host", "app", "!"] }); + expect(complete("ho")).toEqual({ from: 0, labels: ["host", "app", "!"] }); + }); + + test("after ! it offers the fields negated", () => { + expect(complete("!")).toEqual({ from: 0, labels: ["!host", "!app"] }); + expect(complete("!ap")).toEqual({ from: 0, labels: ["!host", "!app"] }); + }); + + test("after host: the hosts seen, replacing only the value", () => { + expect(complete("host:")).toEqual({ from: 5, labels: ["api.example.com", "www.example.org"] }); + expect(complete("!host:api")?.from).toBe(6); + }); + + test("after app: the apps seen, by name and bundle id", () => { + expect(complete("app:")).toEqual({ + from: 4, + labels: ['"Google Chrome"', "com.google.Chrome", "curl"], + }); + }); +}); diff --git a/apps/yaak-proxy/lib/decrypt.ts b/apps/yaak-proxy/lib/decrypt.ts new file mode 100644 index 000000000..d8a554222 --- /dev/null +++ b/apps/yaak-proxy/lib/decrypt.ts @@ -0,0 +1,394 @@ +import type { DecryptListEntry, SeenTargets } from "@yaakapp-internal/proxy-lib"; +import type { FilterOptions } from "@yaakapp-internal/ui/editor"; + +/** + * The decrypt list as the engine reads it: `host:` and `app:` entries, `*` for everything, any + * of them after `!` to exclude. `DecryptEntry` and `DecryptPolicy` in `crates/yaak-proxy`, which + * decide each connection, follow the same rules; this copy marks the sidebar and the detail. + */ +export const EVERYTHING = "*"; + +export interface ParsedEntry { + negated: boolean; + kind: "everything" | "host" | "app"; + /** A host pattern, lowercased, or an app's name or bundle id as written */ + pattern: string; + /** The port a host entry is limited to */ + port: number | null; +} + +/** + * An entry as typed: with its prefix, or a bare hostname, which is a `host:` entry. A URL is + * taken as its host. Null when it is not one. + */ +export function parseEntry(input: string): ParsedEntry | null { + let rest = input.trim(); + const negated = rest.startsWith("!"); + if (negated) rest = rest.slice(1).trimStart(); + if (rest === "") return null; + if (rest === EVERYTHING) return { negated, kind: "everything", pattern: "", port: null }; + if (/^app:/i.test(rest)) { + const app = unquote(rest.slice(4).trim()); + return app === "" ? null : { negated, kind: "app", pattern: app, port: null }; + } + const host = unquote((/^host:/i.test(rest) ? rest.slice(5) : rest).trim()); + const parsed = parseHost(host); + return parsed == null ? null : { negated, kind: "host", ...parsed }; +} + +function unquote(s: string): string { + return s.length >= 2 && s.startsWith('"') && s.endsWith('"') ? s.slice(1, -1).trim() : s; +} + +function parseHost(input: string): { pattern: string; port: number | null } | null { + let host = input; + const scheme = host.indexOf("://"); + if (scheme !== -1) { + host = host.slice(scheme + 3).split(/[/?#]/)[0] ?? ""; + host = host.slice(host.lastIndexOf("@") + 1); + } + let pattern: string; + let port: number | null = null; + if (host.startsWith("[")) { + const end = host.indexOf("]"); + if (end === -1) return null; + pattern = host.slice(1, end); + const after = host.slice(end + 1); + if (after !== "") { + if (!after.startsWith(":")) return null; + port = parsePort(after.slice(1)); + if (port == null) return null; + } + } else { + const colon = host.lastIndexOf(":"); + const name = colon === -1 ? host : host.slice(0, colon); + // One colon is a port; more, a bare IPv6 address. + if (colon !== -1 && !name.includes(":")) { + port = parsePort(host.slice(colon + 1)); + if (port == null) return null; + pattern = name; + } else { + pattern = host; + } + } + pattern = pattern.replace(/\.+$/, "").toLowerCase(); + if (pattern === "" || !/^[a-z0-9\-._*:]+$/.test(pattern)) return null; + return { pattern, port }; +} + +function parsePort(s: string): number | null { + if (!/^\d+$/.test(s)) return null; + const n = Number(s); + return n <= 65535 ? n : null; +} + +/** The entry as stored: `host:api.example.com`, `!app:Slack`, `*` */ +export function formatEntry(e: ParsedEntry): string { + const not = e.negated ? "!" : ""; + if (e.kind === "everything") return `${not}${EVERYTHING}`; + if (e.kind === "app") return `${not}app:${e.pattern}`; + if (e.port == null) return `${not}host:${e.pattern}`; + const host = e.pattern.includes(":") ? `[${e.pattern}]` : e.pattern; + return `${not}host:${host}:${e.port}`; +} + +/** An entry as typed, normalized as the engine stores it, or null when it is not one */ +export function normalizeEntry(input: string): string | null { + const parsed = parseEntry(input); + return parsed == null ? null : formatEntry(parsed); +} + +/** How the list names an entry: `*` is Everything, anything else as written */ +export function entryLabel(entry: string): string { + if (entry === EVERYTHING) return "Everything"; + if (entry === `!${EVERYTHING}`) return "Nothing"; + return entry; +} + +/** What a connection is: where it goes and, when known, the app that sent it */ +export interface DecryptQuery { + host: string; + port?: number | null; + app?: { name: string; bundleId: string | null } | null; +} + +function glob(pattern: string, text: string): boolean { + let pi = 0; + let ti = 0; + let star: [number, number] | null = null; + while (ti < text.length) { + if (pi < pattern.length && pattern[pi] === "*") { + star = [pi, ti]; + pi++; + } else if (pi < pattern.length && pattern[pi] === text[ti]) { + pi++; + ti++; + } else if (star != null) { + pi = star[0] + 1; + ti = star[1] + 1; + star = [star[0], star[1] + 1]; + } else { + return false; + } + } + return pattern + .slice(pi) + .split("") + .every((c) => c === "*"); +} + +/** `*.example.com` matches `example.com` too, as the search's `@host` does */ +function hostMatches(pattern: string, host: string): boolean { + const h = host + .replace(/^\[|\]$/g, "") + .replace(/\.+$/, "") + .toLowerCase(); + return glob(pattern, h) || (pattern.startsWith("*.") && glob(pattern.slice(2), h)); +} + +function matches(e: ParsedEntry, q: DecryptQuery): boolean { + switch (e.kind) { + case "everything": + return true; + case "host": + return (e.port == null || e.port === (q.port ?? 443)) && hostMatches(e.pattern, q.host); + case "app": { + if (q.app == null) return false; + const p = e.pattern.toLowerCase(); + return ( + glob(p, q.app.name.toLowerCase()) || + (q.app.bundleId != null && glob(p, q.app.bundleId.toLowerCase())) + ); + } + } +} + +function specificity(e: ParsedEntry): number { + if (e.kind === "everything") return 0; + if (e.kind === "app") return Number.POSITIVE_INFINITY; + // A site's `*.` counts for nothing, as `*.apple.com` matches `apple.com` itself. + const pattern = e.pattern.startsWith("*.") ? e.pattern.slice(2) : e.pattern; + return pattern.replaceAll("*", "").length + (e.port == null ? 0 : 1); +} + +/** + * Whether the list decrypts `q`: a positive entry matches and no negative one does, wherever + * each sits. A built-in exclusion yields to a positive entry at least as specific as itself. + */ +export function decrypts(list: DecryptListEntry[], q: DecryptQuery): boolean { + const parsed = list.flatMap((e) => { + const p = parseEntry(e.entry); + return p == null ? [] : [{ ...p, builtIn: e.builtIn }]; + }); + let best = -1; + for (const e of parsed) { + if (!e.negated && matches(e, q)) best = Math.max(best, specificity(e)); + } + if (best < 0) return false; + return !parsed.some((e) => e.negated && matches(e, q) && (!e.builtIn || specificity(e) > best)); +} + +/** Some entry decrypts something: the list is not only exclusions */ +export function decryptsAnything(list: DecryptListEntry[]): boolean { + return list.some((e) => !e.entry.trim().startsWith("!")); +} + +/** What a sidebar row or the detail's buttons decrypt: a host, a site, or an app */ +export type DecryptSubject = + | { kind: "host"; host: string } + | { kind: "site"; site: string } + | { kind: "app"; name: string; bundleId: string | null }; + +/** The entry that names `subject` alone: `host:api.example.com`, `host:*.example.com`, `app:Slack` */ +export function subjectEntry(subject: DecryptSubject): string { + switch (subject.kind) { + case "host": + return normalizeEntry(`host:${subject.host}`) ?? `host:${subject.host}`; + case "site": + return `host:*.${subject.site.toLowerCase()}`; + case "app": + return `app:${subject.name}`; + } +} + +/** The connection a subject stands for, as the list judges it */ +export function subjectQuery(subject: DecryptSubject): DecryptQuery { + switch (subject.kind) { + case "host": { + const parsed = parseHost(subject.host); + return { host: parsed?.pattern ?? subject.host, port: parsed?.port ?? 443 }; + } + case "site": + return { host: subject.site }; + case "app": + // No host matches an empty one, so only `*` and app entries decide. + return { host: "", app: { name: subject.name, bundleId: subject.bundleId } }; + } +} + +/** + * An exclusion of the user's that matches `q`, which no entry added beats: Decrypt cannot help + * until it is edited. Null when there is none. + */ +export function blockingExclusion( + list: DecryptListEntry[], + q: DecryptQuery, + builtIns = false, +): string | null { + for (const e of list) { + if (e.builtIn && !builtIns) continue; + const parsed = parseEntry(e.entry); + if (parsed?.negated && matches(parsed, q)) return e.entry; + } + return null; +} + +/** + * What keeps Decrypt on `subject` from working: an exclusion of the user's that matches it, + * other than its own exact one, which Decrypt removes. Null when Decrypt works. + */ +export function blockerOf(list: DecryptListEntry[], subjects: DecryptSubject[]): string | null { + const own = new Set(subjects.map((s) => `!${subjectEntry(s)}`)); + const rest = list.filter((e) => !own.has(e.entry)); + for (const subject of subjects) { + const blocker = blockingExclusion(rest, subjectQuery(subject)); + if (blocker != null) return blocker; + } + return null; +} + +/** What the Not decrypted detail offers for one tunnelled connection */ +export interface DetailActions { + /** The connection is decrypted from the next one on already */ + listed: boolean; + /** Decrypt Host would decrypt it */ + host: boolean; + /** Decrypt App would decrypt it */ + app: boolean; + /** Neither would: the exclusion that keeps it out */ + blocked: string | null; +} + +/** + * Each button judged on its own: whether decrypting the host, or the app, would decrypt this + * connection, with its host, port and app together. A button whose change something else + * still excludes is not offered; with neither, the exclusion is named. + */ +export function detailActions( + list: DecryptListEntry[], + host: string, + app: { name: string; bundleId: string | null } | null, +): DetailActions { + const hostSubject: DecryptSubject = { kind: "host", host }; + const q = { ...subjectQuery(hostSubject), app }; + if (decrypts(list, q)) return { listed: true, host: false, app: false, blocked: null }; + const works = (subject: DecryptSubject) => decrypts(setDecrypted(list, [subject], true), q); + const hostWorks = works(hostSubject); + const appWorks = app != null && works({ kind: "app", ...app }); + const blocked = + hostWorks || appWorks ? null : (blockingExclusion(list, q) ?? blockingExclusion(list, q, true)); + return { listed: false, host: hostWorks, app: appWorks, blocked }; +} + +/** The list decrypts `subject`'s connections, as its sidebar row marks it */ +export function isDecrypted(list: DecryptListEntry[], subject: DecryptSubject): boolean { + return decrypts(list, subjectQuery(subject)); +} + +/** + * The entries after Decrypt or Stop Decrypting on `subject`, to save whole. Decrypt drops an + * exclusion of the subject the user added, then adds its entry if it is still not decrypted. + * Stop drops its entry, then excludes it if something broader, `*` say, still decrypts it. + */ +export function toggleDecrypt(list: DecryptListEntry[], subject: DecryptSubject): string[] { + return setDecrypted(list, [subject], !isDecrypted(list, subject)).map((e) => e.entry); +} + +/** + * The list with every one of `subjects` decrypted (`on`) or not, each as [`toggleDecrypt`] + * changes it, those already so left alone: several sidebar rows at once. + */ +export function setDecrypted( + list: DecryptListEntry[], + subjects: DecryptSubject[], + on: boolean, +): DecryptListEntry[] { + let next = list; + for (const subject of subjects) { + if (isDecrypted(next, subject) === on) continue; + const entry = subjectEntry(subject); + const drop = on ? `!${entry}` : entry; + next = next.filter((e) => e.builtIn || e.entry !== drop); + if (isDecrypted(next, subject) === on) continue; + // Something the user excluded more broadly still keeps it out; adding would change nothing. + if (on && blockingExclusion(next, subjectQuery(subject)) != null) continue; + const add = on ? entry : `!${entry}`; + next = [...next.filter((e) => !e.builtIn), { entry: add, builtIn: false }, ...builtIns(next)]; + } + return next; +} + +/** + * The sidebar menu's row for the selected rows that can be decrypted: Stop Decrypting when every + * one of them is, else Decrypt. Null when none can. + */ +export function decryptMenuAction( + list: DecryptListEntry[], + subjects: DecryptSubject[], +): { label: "Decrypt" | "Stop Decrypting"; on: boolean } | null { + if (subjects.length === 0) return null; + const on = !subjects.every((s) => isDecrypted(list, s)); + return { label: on ? "Decrypt" : "Stop Decrypting", on }; +} + +function builtIns(list: DecryptListEntry[]): DecryptListEntry[] { + return list.filter((e) => e.builtIn); +} + +/** + * What turns `before` into `after`, as `edit_decrypt_list` takes it: the entries to add and to + * remove, applied to the list as it is when they land, so edits made together each keep the + * other's. + */ +export function listEdit( + before: DecryptListEntry[], + after: DecryptListEntry[], +): { add: string[]; remove: string[] } { + const was = new Set(before.map((e) => e.entry)); + const now = new Set(after.map((e) => e.entry)); + return { + add: after.filter((e) => !was.has(e.entry)).map((e) => e.entry), + remove: before.filter((e) => !now.has(e.entry)).map((e) => e.entry), + }; +} + +/** The edit that adds `*`, or takes it out, as the first-run card's choice makes it */ +export function everythingEdit(on: boolean): { add: string[]; remove: string[] } { + return on ? { add: [EVERYTHING], remove: [] } : { add: [], remove: [EVERYTHING] }; +} + +/** Whether the list decrypts everything, as the first-run card's choice reads it */ +export function decryptsEverything(list: DecryptListEntry[]): boolean { + return list.some((e) => e.entry === EVERYTHING); +} + +/** + * What the list's input suggests: `host:`, `app:` and `!` to start an entry; after `host:` the + * hosts seen in the capture, after `app:` the apps seen, by name and bundle id. + */ +export function decryptFilterOptions(seen: SeenTargets | null | undefined): FilterOptions { + const apps = new Set(); + for (const app of seen?.apps ?? []) { + apps.add(app.name); + if (app.bundleId != null) apps.add(app.bundleId); + } + return { + fieldPrefix: "", + replaceValueOnly: true, + negation: { prefix: "!", info: "Exclude" }, + fields: [ + { name: "host", values: seen?.hosts ?? [], info: "Host or *.site" }, + { name: "app", values: [...apps], info: "App name or bundle id" }, + ], + }; +} diff --git a/apps/yaak-proxy/lib/firstRun.test.ts b/apps/yaak-proxy/lib/firstRun.test.ts index 096e5168d..8fb674103 100644 --- a/apps/yaak-proxy/lib/firstRun.test.ts +++ b/apps/yaak-proxy/lib/firstRun.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "vite-plus/test"; -import { setupIncomplete } from "./firstRun"; +import { decryptStep, setupIncomplete } from "./firstRun"; describe("setupIncomplete", () => { const done = { trusted: true, systemProxySupported: true, systemProxyEverOn: true }; @@ -25,3 +25,21 @@ describe("setupIncomplete", () => { expect(setupIncomplete({ ...done, systemProxySupported: null, trusted: false })).toBe(false); }); }); + +describe("decryptStep", () => { + const builtIns = [{ entry: "!host:*.apple.com", builtIn: true }]; + + test("picks hosts as they appear by default, after the certificate", () => { + expect(decryptStep(builtIns, false)).toEqual({ choice: "pick", state: "todo" }); + expect(decryptStep(builtIns, true)).toEqual({ choice: "pick", state: "next" }); + }); + + test("is done once everything, or a host, is on the list", () => { + const everything = [{ entry: "*", builtIn: false }, ...builtIns]; + expect(decryptStep(everything, true)).toEqual({ choice: "everything", state: "done" }); + const host = [{ entry: "host:api.example.com", builtIn: false }]; + expect(decryptStep(host, true)).toEqual({ choice: "pick", state: "done" }); + // An exclusion alone decrypts nothing. + expect(decryptStep([{ entry: "!host:a.com", builtIn: false }], true).state).toBe("next"); + }); +}); diff --git a/apps/yaak-proxy/lib/firstRun.ts b/apps/yaak-proxy/lib/firstRun.ts index b5f15ef2e..8c5dd28c9 100644 --- a/apps/yaak-proxy/lib/firstRun.ts +++ b/apps/yaak-proxy/lib/firstRun.ts @@ -1,3 +1,6 @@ +import type { DecryptListEntry } from "@yaakapp-internal/proxy-lib"; +import { decryptsEverything } from "./decrypt"; + /** What decides whether setup is done, as the app last read it */ export interface SetupState { /** The certificate is trusted, or null until read */ @@ -19,3 +22,21 @@ export function setupIncomplete(s: SetupState): boolean { if (!s.trusted) return true; return s.systemProxySupported && !s.systemProxyEverOn; } + +/** The first-run card's "Choose what to decrypt" step, as the decrypt list has it */ +export interface DecryptStep { + /** Everything is decrypted, `*` on the list; else hosts are picked as they appear */ + choice: "everything" | "pick"; + state: "todo" | "next" | "done"; +} + +/** + * Picking hosts as they appear is the default and needs nothing done, so the step waits on the + * certificate like the others and is done once a choice is on the list: everything, or any host + * or app picked already. + */ +export function decryptStep(list: DecryptListEntry[], trusted: boolean): DecryptStep { + const choice = decryptsEverything(list) ? "everything" : "pick"; + const picked = list.some((e) => !e.builtIn && !e.entry.startsWith("!")); + return { choice, state: picked ? "done" : trusted ? "next" : "todo" }; +} diff --git a/apps/yaak-proxy/lib/flowViews.test.ts b/apps/yaak-proxy/lib/flowViews.test.ts index afb557db5..7e8c35c65 100644 --- a/apps/yaak-proxy/lib/flowViews.test.ts +++ b/apps/yaak-proxy/lib/flowViews.test.ts @@ -58,6 +58,7 @@ const flow: Flow = { grpcMethod: null, grpcStatus: null, grpcMessage: null, + tunnelled: false, }; function bytes(text: string): FlowBodyData { diff --git a/apps/yaak-proxy/lib/search.test.ts b/apps/yaak-proxy/lib/search.test.ts index 54da3fc48..55c838f53 100644 --- a/apps/yaak-proxy/lib/search.test.ts +++ b/apps/yaak-proxy/lib/search.test.ts @@ -229,6 +229,16 @@ describe("parseSearch", () => { expect(parseSearch("@is:system", true)).toEqual({ system: true }); }); + test("keeps or drops HTTPS passed through undecrypted", () => { + expect(parseSearch("@is:tunnelled")).toEqual({ tunnelled: true }); + expect(parseSearch("-@is:tunnelled stripe")).toEqual({ q: "stripe", tunnelled: false }); + const tunnel = { url: "https://a.com", method: "CONNECT", tunnelled: true } as Flow; + const decrypted = { url: "https://a.com/x", method: "GET", tunnelled: false } as Flow; + expect(matchesQuery(tunnel, { tunnelled: true })).toBe(true); + expect(matchesQuery(decrypted, { tunnelled: true })).toBe(false); + expect(matchesQuery(decrypted, { tunnelled: false })).toBe(true); + }); + test("round-trips what the sidebar writes", () => { const text = [ formatFieldFilter("app", "Google Chrome"), diff --git a/apps/yaak-proxy/lib/search.ts b/apps/yaak-proxy/lib/search.ts index a172bc1a1..39207dde6 100644 --- a/apps/yaak-proxy/lib/search.ts +++ b/apps/yaak-proxy/lib/search.ts @@ -5,7 +5,7 @@ import { getDomain } from "tldts"; * A search over every flow in the database: the search box's terms, its field terms, and * whether System's flows are hidden, as `list_flows` takes them. */ -export type FlowQuery = Pick; +export type FlowQuery = Pick; /** A word of the search box: a free term, or `@field:value`, maybe negated with `-` */ export interface SearchToken { @@ -56,7 +56,8 @@ export function formatFieldFilter(field: string, value: string, negated = false) * not every pairing of their apps and hosts. Within a clause, `@app` terms match any of their * apps; `@host` terms any of their hosts, each with the `@path` terms after it; `@is:system` * keeps only macOS's daemons and `-@is:system` drops them, as hiding System does when the - * clause says neither. Other words match the URL, the method, the status or the type. + * clause says neither. `@is:tunnelled` keeps only HTTPS passed through undecrypted, and + * `-@is:tunnelled` drops it. Other words match the URL, the method, the status or the type. */ export function parseSearch(text: string, hideSystem = false): FlowQuery | null { const clauses = clausesOf(text); @@ -77,6 +78,7 @@ function parseClause(tokens: SearchToken[], hideSystem: boolean): FlowQuery | nu // Each host with the paths written after it. const hosts: { host: string; paths: string[] }[] = []; let system: boolean | undefined = hideSystem ? false : undefined; + let tunnelled: boolean | undefined; for (const t of tokens) { if (t.field === "app" && !t.negated && t.value !== "") apps.push(t.value); else if (t.field === "host" && !t.negated && t.value !== "") { @@ -91,6 +93,7 @@ function parseClause(tokens: SearchToken[], hideSystem: boolean): FlowQuery | nu host.paths.push(path); } } else if (t.field === "is" && t.value.toLowerCase() === "system") system = !t.negated; + else if (t.field === "is" && t.value.toLowerCase() === "tunnelled") tunnelled = !t.negated; else if (t.field == null && !t.negated) words.push(t.value); // Fields and exclusions the query language will add are ignored until it lands. } @@ -105,6 +108,7 @@ function parseClause(tokens: SearchToken[], hideSystem: boolean): FlowQuery | nu ...(under.length > 0 ? { under } : {}), ...(apps.length > 0 ? { app: apps } : {}), ...(system == null ? {} : { system }), + ...(tunnelled == null ? {} : { tunnelled }), }; return Object.keys(query).length === 0 ? null : query; } @@ -190,6 +194,7 @@ export function matchesQuery(flow: Flow, query: FlowQuery): boolean { if (query.under != null && !query.under.some((node) => isUnder(flow.url, node))) return false; if (query.app != null && !query.app.some((app) => matchesApp(flow, app))) return false; if (query.system != null && isSystemFlow(flow) !== query.system) return false; + if (query.tunnelled != null && flow.tunnelled !== query.tunnelled) return false; // One of the groups, each judged whole, as the database judges them. if (query.any != null && query.any.length > 0) { return query.any.some((group) => matchesQuery(flow, group)); diff --git a/apps/yaak-proxy/lib/sidebarTree.test.ts b/apps/yaak-proxy/lib/sidebarTree.test.ts index dd755b31e..986a089a6 100644 --- a/apps/yaak-proxy/lib/sidebarTree.test.ts +++ b/apps/yaak-proxy/lib/sidebarTree.test.ts @@ -103,6 +103,44 @@ describe("buildSidebarTree", () => { }); }); +describe("decrypting from the sidebar", () => { + const tunnel = flow("https://bank.example.com", { + ...slack, + method: "CONNECT", + tunnelled: true, + }); + const root = buildSidebarTree([...flows, tunnel], 5); + const items = itemsById(root); + + test("Not decrypted follows All traffic while there are tunnelled flows", () => { + expect((root.children ?? []).slice(0, 2).map((c) => shape(c)[0])).toEqual([ + "All traffic 5", + "Not decrypted 1", + ]); + expect(items.get("tunnelled")?.query).toBe("@is:tunnelled"); + expect(itemsById(buildSidebarTree(flows, 4)).has("tunnelled")).toBe(false); + }); + + test("hosts, sites and apps name what their menu's Decrypt covers", () => { + expect(items.get("host:bank.example.com")?.decrypt).toEqual({ + kind: "host", + host: "bank.example.com", + }); + expect(items.get("site:example.com")?.decrypt).toEqual({ kind: "site", site: "example.com" }); + expect(items.get("app:Slack|host:edgeapi.slack.com")?.decrypt).toEqual({ + kind: "host", + host: "edgeapi.slack.com", + }); + expect(items.get("app:Slack")?.decrypt).toEqual({ + kind: "app", + name: "Slack", + bundleId: "com.slack", + }); + expect(items.get("path:api.github.com/repos")?.decrypt).toBeUndefined(); + expect(items.get("domains")?.decrypt).toBeUndefined(); + }); +}); + describe("selection and the search text", () => { test("selects rows under different apps as groups of their own", () => { const items = itemsById(buildSidebarTree(flows, 4)); diff --git a/apps/yaak-proxy/lib/sidebarTree.ts b/apps/yaak-proxy/lib/sidebarTree.ts index 866ac65a6..c7a99798d 100644 --- a/apps/yaak-proxy/lib/sidebarTree.ts +++ b/apps/yaak-proxy/lib/sidebarTree.ts @@ -1,5 +1,6 @@ import type { Flow } from "@yaakapp-internal/proxy-lib"; import type { TreeNode } from "@yaakapp-internal/ui"; +import type { DecryptSubject } from "./decrypt"; import { appKeyOf, folderOf, @@ -10,7 +11,7 @@ import { tokenize, } from "./search"; -export type SidebarKind = "all" | "group" | "app" | "site" | "host" | "path"; +export type SidebarKind = "all" | "tunnelled" | "group" | "app" | "site" | "host" | "path"; /** A row of the sidebar: All traffic, a group, an app, or a site, host or path under one */ export interface SidebarItem { @@ -27,6 +28,8 @@ export interface SidebarItem { appPath?: string; /** What selecting the row writes into the search box; empty for all traffic */ query: string; + /** What Decrypt in the row's menu decrypts: its host, site or app */ + decrypt?: DecryptSubject; } /** The ids of All traffic and the groups */ @@ -34,6 +37,7 @@ export const ALL_ID = "all"; export const APPS_ID = "apps"; export const DOMAINS_ID = "domains"; export const SYSTEM_ID = "system"; +export const TUNNELLED_ID = "tunnelled"; /** A row a flow sits under, with what makes it, so a rebuild only looks it up */ interface PlaceRow { @@ -41,6 +45,7 @@ interface PlaceRow { kind: "site" | "host" | "path"; label: string; query: string; + decrypt?: DecryptSubject; } /** @@ -72,8 +77,15 @@ function placeOf(flow: Flow): Place { kind: "site", label: site, query: formatFieldFilter("host", `*.${site}`), + decrypt: { kind: "site", site }, + }, + { + id: `host:${host}`, + kind: "host", + label: host, + query: hostQuery, + decrypt: { kind: "host", host }, }, - { id: `host:${host}`, kind: "host", label: host, query: hostQuery }, ]; let p = ""; for (const segment of segments) { @@ -121,6 +133,7 @@ function child(parent: Trie, row: PlaceRow): Trie { item: { id: row.id, kind: row.kind, label: row.label, count: 0, query: row.query }, children: new Map(), }; + if (row.decrypt != null) node.item.decrypt = row.decrypt; parent.children.set(row.id, node); } node.item.count++; @@ -162,7 +175,8 @@ function group(id: string, label: string): Trie { /** * The sidebar: All traffic, then Apps, each app opening to its own sites, hosts and paths, then * Domains by site, host and path, then System, macOS's daemons, apart from the apps. A group with - * nothing in it is left out. `total` is All traffic's count: every flow it covers in the + * nothing in it is left out. Not decrypted, after All traffic, gathers the HTTPS passed through + * undecrypted, when there is any. `total` is All traffic's count: every flow it covers in the * database. */ export function buildSidebarTree(flows: Flow[], total: number): TreeNode { @@ -177,9 +191,11 @@ export function buildSidebarTree(flows: Flow[], total: number): TreeNode 0) { + root.children?.push({ + item: { + id: TUNNELLED_ID, + kind: "tunnelled", + label: "Not decrypted", + count: tunnelled, + query: formatFieldFilter("is", "tunnelled"), + }, + parent: root, + depth: 1, + draggable: false, + }); + } for (const g of [apps, domains, system]) { if (g.children.size === 0) continue; const node: TreeNode = { diff --git a/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts b/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts index d78e9bc97..70fbac00c 100644 --- a/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts +++ b/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts @@ -104,7 +104,13 @@ grpcStatus: number | null, /** * A gRPC call's `grpc-message`, decoded. */ -grpcMessage: string | null, }; +grpcMessage: string | null, +/** + * An HTTPS connection passed through undecrypted, as the decrypt list leaves it out or its + * client rejected the proxy's certificate: method `CONNECT`, its URL the host, its sizes + * the bytes each way, and no headers or bodies. Open until `elapsedMs` or `error` is set. + */ +tunnelled: boolean, }; export type FlowBody = "request" | "response"; diff --git a/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts b/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts index bd9125d90..302fa98d3 100644 --- a/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts +++ b/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts @@ -87,10 +87,40 @@ schema: GrpcSchema | null, */ note: string | null, }; +/** + * What HTTPS is decrypted: the user's entries in order, then the built-in exclusions not + * removed. A connection is decrypted when a positive entry matches it and no exclusion does. + */ +export type DecryptList = { entries: Array, }; + +/** + * The decrypt list changed, here or through the control API. + */ +export type DecryptListChangedPayload = { list: DecryptList, }; + +/** + * One entry of the decrypt list, as the app shows it. + */ +export type DecryptListEntry = { +/** + * As stored: `host:*.example.com`, `app:Google Chrome`, `!host:*.apple.com`, `*`. + */ +entry: string, +/** + * Shipped with the app: an exclusion of Apple's services or a pinned app's hosts. + */ +builtIn: boolean, }; + export type DropReason = "hop_by_hop" | "set_by_sending" | "body_decoded"; export type DroppedHeader = { name: string, value: string, reason: DropReason, }; +/** + * Entries to add after the user's own and to remove, applied to the list as it is when the + * change lands. Each is normalized first. + */ +export type EditDecryptListRequest = { add: Array, remove: Array, }; + export type ExportHarRequest = { /** * The file to write. @@ -145,6 +175,10 @@ app?: Array, * tells them by their executable. */ system?: boolean, +/** + * Only HTTPS connections tunnelled undecrypted (true), or none of them (false). + */ +tunnelled?: boolean, /** * Only flows one of these groups covers, each judged on its own: several sidebar rows * selected, `@app:Slack @host:a.com OR @app:Chrome @host:b.com`, are two groups and not every @@ -186,6 +220,8 @@ folderId: string | null, }; export type FormatBodyRequest = { text: string, }; +export type GetDecryptListRequest = Record; + export type GetFlowBodyRequest = { flowId: string, which: FlowBody, }; export type GetFlowBodyResponse = { @@ -319,6 +355,8 @@ messages: Array, */ total: number, }; +export type ListSeenRequest = Record; + export type ListWorkspacesRequest = Record; export type ListWorkspacesResponse = { @@ -374,9 +412,9 @@ host: string, }; export type RemoveGrpcSchemaRequest = { id: string, }; -export type RpcEventSchema = { flows_cleared: FlowsClearedPayload, message_write: MessagePayload, model_write: ModelPayload, proxy_state_changed: ProxyStatePayload, }; +export type RpcEventSchema = { decrypt_list_changed: DecryptListChangedPayload, flows_cleared: FlowsClearedPayload, message_write: MessagePayload, model_write: ModelPayload, proxy_state_changed: ProxyStatePayload, }; -export type RpcSchema = { execute_action: [ActionInvocation, boolean], format_body: [FormatBodyRequest, string], get_message_payload: [GetMessagePayloadRequest, GetMessagePayloadResponse], list_actions: [ListActionsRequest, ListActionsResponse], list_flows: [FlowFilter, ListFlowsResponse], list_grpc_schemas: [ListGrpcSchemasRequest, Array], list_messages: [ListMessagesRequest, ListMessagesResponse], remove_grpc_schema: [RemoveGrpcSchemaRequest, boolean], set_recording: [SetRecordingRequest, boolean], set_system_proxy_at_launch: [SetSystemProxyAtLaunchRequest, boolean], set_http_version: [SetHttpVersionRequest, HttpVersion], add_grpc_schema: [AddGrpcSchemaRequest, GrpcSchema], app_icon: [AppIconRequest, string | null], ca_status: [CaStatusRequest, CaStatus], ca_trust: [CaTrustRequest, CaStatus], clear_flows: [ClearFlowsRequest, number], decode_grpc_message: [DecodeGrpcMessageRequest, DecodedGrpcMessage], export_har: [ExportHarRequest, number], filter_body: [FilterBodyRequest, FilterBodyResponse], get_flow_body: [GetFlowBodyRequest, GetFlowBodyResponse], get_proxy_state: [GetProxyStateRequest, GetProxyStateResponse], helper_register: [HelperRegisterRequest, SystemProxyStatus], helper_unregister: [HelperUnregisterRequest, SystemProxyStatus], list_workspaces: [ListWorkspacesRequest, ListWorkspacesResponse], preview_save: [PreviewSaveRequest, SavePreview], reflect_grpc_host: [ReflectGrpcHostRequest, GrpcSchema], save_flow: [SaveFlowToYaakRequest, SavedFlow], save_flow_body: [SaveFlowBodyRequest, SaveFlowBodyResponse], system_proxy_set: [SystemProxySetRequest, SystemProxyStatus], system_proxy_status: [SystemProxyStatusRequest, SystemProxyStatus], take_over: [TakeOverRequest, boolean], }; +export type RpcSchema = { execute_action: [ActionInvocation, boolean], format_body: [FormatBodyRequest, string], get_decrypt_list: [GetDecryptListRequest, DecryptList], get_message_payload: [GetMessagePayloadRequest, GetMessagePayloadResponse], list_actions: [ListActionsRequest, ListActionsResponse], list_flows: [FlowFilter, ListFlowsResponse], list_grpc_schemas: [ListGrpcSchemasRequest, Array], list_messages: [ListMessagesRequest, ListMessagesResponse], remove_grpc_schema: [RemoveGrpcSchemaRequest, boolean], set_recording: [SetRecordingRequest, boolean], set_system_proxy_at_launch: [SetSystemProxyAtLaunchRequest, boolean], set_http_version: [SetHttpVersionRequest, HttpVersion], add_grpc_schema: [AddGrpcSchemaRequest, GrpcSchema], app_icon: [AppIconRequest, string | null], ca_status: [CaStatusRequest, CaStatus], ca_trust: [CaTrustRequest, CaStatus], clear_flows: [ClearFlowsRequest, number], decode_grpc_message: [DecodeGrpcMessageRequest, DecodedGrpcMessage], export_har: [ExportHarRequest, number], filter_body: [FilterBodyRequest, FilterBodyResponse], get_flow_body: [GetFlowBodyRequest, GetFlowBodyResponse], get_proxy_state: [GetProxyStateRequest, GetProxyStateResponse], helper_register: [HelperRegisterRequest, SystemProxyStatus], helper_unregister: [HelperUnregisterRequest, SystemProxyStatus], edit_decrypt_list: [EditDecryptListRequest, DecryptList], list_seen: [ListSeenRequest, SeenTargets], set_decrypt_list: [SetDecryptListRequest, DecryptList], list_workspaces: [ListWorkspacesRequest, ListWorkspacesResponse], preview_save: [PreviewSaveRequest, SavePreview], reflect_grpc_host: [ReflectGrpcHostRequest, GrpcSchema], save_flow: [SaveFlowToYaakRequest, SavedFlow], save_flow_body: [SaveFlowBodyRequest, SaveFlowBodyResponse], system_proxy_set: [SystemProxySetRequest, SystemProxyStatus], system_proxy_status: [SystemProxyStatusRequest, SystemProxyStatus], take_over: [TakeOverRequest, boolean], }; export type SaveFlowBodyRequest = { flowId: string, which: FlowBody, /** @@ -440,6 +478,21 @@ authorization: boolean, }; export type SavedFlow = { requestId: string, name: string, }; +export type SeenApp = { name: string, bundleId: string | null, }; + +/** + * The hosts and apps flows have gone to and come from, for the decrypt list's suggestions. + */ +export type SeenTargets = { +/** + * Hosts as the sidebar names them, with a port other than the scheme's: most recent first. + */ +hosts: Array, +/** + * Apps by name and bundle id, most recent first. + */ +apps: Array, }; + export type ServiceProxy = { /** * The network service, as System Settings names it: "Wi-Fi", "USB 10/100/1000 LAN". @@ -451,6 +504,13 @@ name: string, */ id?: string, http: ProxySetting, https: ProxySetting, }; +export type SetDecryptListRequest = { +/** + * Every entry, in order, as typed: a bare hostname is a `host:` entry. A built-in + * exclusion left out is removed. + */ +entries: Array, }; + export type SetHttpVersionRequest = { httpVersion: HttpVersion, }; export type SetRecordingRequest = { enabled: boolean, }; diff --git a/crates-proxy/yaak-proxy-lib/migrations/00000009_flow_tunnelled.sql b/crates-proxy/yaak-proxy-lib/migrations/00000009_flow_tunnelled.sql new file mode 100644 index 000000000..beea72933 --- /dev/null +++ b/crates-proxy/yaak-proxy-lib/migrations/00000009_flow_tunnelled.sql @@ -0,0 +1,2 @@ +-- An HTTPS connection passed through undecrypted, recorded with only its host and sizes. +ALTER TABLE flows ADD COLUMN tunnelled BOOLEAN NOT NULL DEFAULT FALSE; diff --git a/crates-proxy/yaak-proxy-lib/src/client.rs b/crates-proxy/yaak-proxy-lib/src/client.rs index f1601fb0f..c5458b82c 100644 --- a/crates-proxy/yaak-proxy-lib/src/client.rs +++ b/crates-proxy/yaak-proxy-lib/src/client.rs @@ -5,8 +5,8 @@ use crate::bodies::{self, DecodedBody}; use crate::ca::CaStatus; use crate::control::{ - self, ControlInfo, ControlStatus, ErrorBody, NOTE_HEADER, RecordingRequest, StartRequest, - SystemProxyRequest, TRUNCATED_HEADER, + self, ControlInfo, ControlStatus, DecryptRequest, ErrorBody, NOTE_HEADER, RecordingRequest, + StartRequest, SystemProxyRequest, TRUNCATED_HEADER, }; use crate::db::ProxyQueryManager; use crate::db::Stores; @@ -15,7 +15,7 @@ use crate::har::{self, Har}; use crate::macos::SystemProxyStatus; use crate::models::{Flow, FlowBody}; use crate::save::{SaveFlowRequest, SaveFlowResponse, WorkspaceSummary}; -use crate::settings::ProxySettings; +use crate::settings::{DecryptList, ProxySettings}; use serde::Serialize; use serde::de::DeserializeOwned; use std::path::Path; @@ -146,6 +146,38 @@ impl ControlClient { self.post("/system-proxy", &SystemProxyRequest { enabled }).await } + pub async fn decrypt_list(&self) -> Result { + self.get("/decrypt").await + } + + /// Adds and removes entries in the owner's decrypt list as it is there. + pub async fn edit_decrypt_list( + &self, + edit: &crate::EditDecryptListRequest, + ) -> Result { + let body = serde_json::to_vec(edit).map_err(|e| e.to_string())?; + let req = self + .http + .patch(self.url("/decrypt")) + .header("content-type", "application/json") + .body(body); + let resp = self.send(req).await?; + resp.json().await.map_err(|e| format!("Invalid response from the proxy: {e}")) + } + + /// Replaces the decrypt list in the owner, which saves it and applies it to its engine. + pub async fn set_decrypt_list(&self, entries: &[String]) -> Result { + let body = serde_json::to_vec(&DecryptRequest { entries: entries.to_vec() }) + .map_err(|e| e.to_string())?; + let req = self + .http + .put(self.url("/decrypt")) + .header("content-type", "application/json") + .body(body); + let resp = self.send(req).await?; + resp.json().await.map_err(|e| format!("Invalid response from the proxy: {e}")) + } + pub async fn workspaces(&self) -> Result, String> { self.get("/workspaces").await } diff --git a/crates-proxy/yaak-proxy-lib/src/control.rs b/crates-proxy/yaak-proxy-lib/src/control.rs index 1a2e67948..b6bdb2e23 100644 --- a/crates-proxy/yaak-proxy-lib/src/control.rs +++ b/crates-proxy/yaak-proxy-lib/src/control.rs @@ -12,8 +12,8 @@ use crate::messages::{self, FlowMessage, MessageKind}; use crate::models::{FlowBody, find_header}; use crate::save::SaveFlowRequest; use crate::{ - AddGrpcSchemaRequest, ListMessagesResponse, ProxyCtx, ProxyState, ReflectGrpcHostRequest, - bodies, + AddGrpcSchemaRequest, EditDecryptListRequest, ListMessagesResponse, ProxyCtx, ProxyState, + ReflectGrpcHostRequest, bodies, }; use bytes::Bytes; use chrono::{DateTime, Utc}; @@ -237,6 +237,13 @@ pub struct RecordingRequest { pub enabled: bool, } +/// `PUT /decrypt`: every entry of the decrypt list, in order. A built-in exclusion left out is +/// removed. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DecryptRequest { + pub entries: Vec, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ClearResponse { /// How many flows were deleted. @@ -527,6 +534,15 @@ async fn route(state: &ServerState, req: Request) -> Result json(&ctx.decrypt_list()?), + (&Method::PATCH, ["decrypt"]) => { + let body: EditDecryptListRequest = read_json(req).await?; + json(&ctx.edit_decrypt_list(&body.add, &body.remove).map_err(bad_request)?) + } + (&Method::PUT, ["decrypt"]) => { + let body: DecryptRequest = read_json(req).await?; + json(&ctx.set_decrypt_list(&body.entries).map_err(bad_request)?) + } (&Method::GET, ["flows"]) => { let filter = FlowFilter::from_query(&query).map_err(bad_request)?; json(&flows::list(&ctx.db()?, &filter)?) diff --git a/crates-proxy/yaak-proxy-lib/src/db.rs b/crates-proxy/yaak-proxy-lib/src/db.rs index 13b556277..5cf01fcb7 100644 --- a/crates-proxy/yaak-proxy-lib/src/db.rs +++ b/crates-proxy/yaak-proxy-lib/src/db.rs @@ -25,6 +25,23 @@ pub const BLOBS_DB: &str = "blobs.sqlite"; /// The key both are encrypted with, wrapped by the app's master key in the keychain. pub const DATA_KEY: &str = "data.key"; +/// A pre-release build added `flows.tunnelled` in migration 8. A database it made has the +/// column without migration 9 recorded, which would fail adding it again, so it is recorded +/// as applied first. +fn record_early_tunnelled_column(pool: &SqlitePool) -> Result<()> { + let conn = pool.get()?; + let has_column = conn.prepare("SELECT tunnelled FROM flows LIMIT 0").is_ok(); + let has_tracking = conn.prepare("SELECT version FROM _sqlx_migrations LIMIT 0").is_ok(); + if has_column && has_tracking { + conn.execute( + "INSERT OR IGNORE INTO _sqlx_migrations (version, description) \ + VALUES ('00000009', '00000009_flow_tunnelled.sql')", + [], + )?; + } + Ok(()) +} + /// Reads come from a pool; writes go through one connection, as in yaak-models' /// `QueryManager`. WAL lets the CLI read `proxy.db` while the app writes it. #[derive(Clone)] @@ -42,6 +59,7 @@ impl ProxyQueryManager { } encryption::encrypt_in_place(db_path, key, &|_| {})?; let writer = file_pool(db_path, 1, key)?; + record_early_tunnelled_column(&writer)?; run_migrations(&writer, &MIGRATIONS)?; Ok(Self { readers: file_pool(db_path, 5, key)?, writer }) } @@ -512,7 +530,7 @@ mod tests { .unwrap() .map(|v| v.unwrap()) .collect(); - assert_eq!(&applied[applied.len() - 2..], ["00000007", "00000008"]); + assert_eq!(&applied[applied.len() - 3..], ["00000007", "00000008", "00000009"]); let mark_columns = columns(&conn, "flow_clear_mark"); assert_eq!(mark_columns, ["id", "max_rowid", "cleaner", "cleaner_at"]); drop(conn); @@ -532,4 +550,109 @@ mod tests { ); assert_eq!(crate::messages::count(&db, "fl_ws").unwrap(), 0); } + + /// A database a pre-release build made, with `tunnelled` added by its migration 8, opens + /// without adding the column twice. + #[test] + fn opens_a_database_whose_migration_8_added_tunnelled() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("proxy.db"); + let conn = Connection::open(&path).unwrap(); + conn.execute_batch( + "CREATE TABLE _sqlx_migrations ( + version TEXT PRIMARY KEY, + description TEXT NOT NULL, + applied_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL + );", + ) + .unwrap(); + let mut files: Vec<_> = MIGRATIONS + .files() + .map(|f| (f.path().to_string_lossy().into_owned(), f.contents_utf8().unwrap())) + .filter(|(name, _)| name.as_str() < "00000009") + .collect(); + files.sort(); + for (name, sql) in files { + conn.execute_batch(sql).unwrap(); + conn.execute( + "INSERT INTO _sqlx_migrations (version, description) VALUES (?, ?)", + [&name[..8], &name], + ) + .unwrap(); + } + conn.execute_batch( + "ALTER TABLE flows ADD COLUMN tunnelled BOOLEAN NOT NULL DEFAULT FALSE;", + ) + .unwrap(); + drop(conn); + + let db = ProxyQueryManager::new(&path, &crate::db::test_key()).unwrap(); + let tunnel = crate::models::Flow { + id: "fl_tunnel".into(), + url: "https://a.com".into(), + tunnelled: true, + ..Default::default() + }; + db.with_tx(|c| c.upsert(&tunnel, &yaak_database::UpdateSource::Background).map(|_| ())) + .unwrap(); + assert!(crate::flows::find(&db, "fl_tunnel").unwrap().tunnelled); + drop(db); + // And again, with everything recorded. + drop(ProxyQueryManager::new(&path, &crate::db::test_key()).unwrap()); + } + + /// A database that ran every migration up to the clear cleaner's, as the build before + /// tunnelled flows left it, takes the `tunnelled` column and stores tunnelled flows. + #[test] + fn adds_tunnelled_to_a_database_at_migration_8() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("proxy.db"); + let conn = Connection::open(&path).unwrap(); + conn.execute_batch( + "CREATE TABLE _sqlx_migrations ( + version TEXT PRIMARY KEY, + description TEXT NOT NULL, + applied_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL + );", + ) + .unwrap(); + let mut files: Vec<_> = MIGRATIONS + .files() + .map(|f| (f.path().to_string_lossy().into_owned(), f.contents_utf8().unwrap())) + .filter(|(name, _)| name.as_str() < "00000009") + .collect(); + files.sort(); + assert_eq!(files.len(), 8); + for (name, sql) in files { + conn.execute_batch(sql).unwrap(); + conn.execute( + "INSERT INTO _sqlx_migrations (version, description) VALUES (?, ?)", + [&name[..8], &name], + ) + .unwrap(); + } + conn.execute( + "INSERT INTO flows (id, url, method) VALUES ('fl_old', 'https://a.com/', 'GET')", + [], + ) + .unwrap(); + assert!(!columns(&conn, "flows").contains(&"tunnelled".to_string())); + drop(conn); + + let db = ProxyQueryManager::new(&path, &crate::db::test_key()).unwrap(); + let tunnel = crate::models::Flow { + id: "fl_tunnel".into(), + method: "CONNECT".into(), + url: "https://bank.example.com".into(), + tunnelled: true, + ..Default::default() + }; + db.with_tx(|c| c.upsert(&tunnel, &yaak_database::UpdateSource::Background).map(|_| ())) + .unwrap(); + let filter = crate::flows::FlowFilter { tunnelled: Some(true), ..Default::default() }; + let listed = crate::flows::list(&db, &filter).unwrap(); + assert_eq!(listed.iter().map(|f| f.id.as_str()).collect::>(), ["fl_tunnel"]); + let old = crate::flows::find(&db, "fl_old").unwrap(); + assert!(!old.tunnelled); + } } diff --git a/crates-proxy/yaak-proxy-lib/src/flows.rs b/crates-proxy/yaak-proxy-lib/src/flows.rs index 565dad4aa..11bae5f43 100644 --- a/crates-proxy/yaak-proxy-lib/src/flows.rs +++ b/crates-proxy/yaak-proxy-lib/src/flows.rs @@ -35,6 +35,9 @@ pub struct FlowFilter { /// tells them by their executable. #[ts(optional)] pub system: Option, + /// Only HTTPS connections tunnelled undecrypted (true), or none of them (false). + #[ts(optional)] + pub tunnelled: Option, /// Only flows one of these groups covers, each judged on its own: several sidebar rows /// selected, `@app:Slack @host:a.com OR @app:Chrome @host:b.com`, are two groups and not every /// pairing of their apps and hosts. A group's `after`, `before` and `limit` do not apply. Not @@ -78,6 +81,11 @@ impl FlowFilter { .map(|v| v.parse().map_err(|_| format!("Invalid system {v:?}"))) .transpose()? } + "tunnelled" => { + filter.tunnelled = value + .map(|v| v.parse().map_err(|_| format!("Invalid tunnelled {v:?}"))) + .transpose()? + } "method" => filter.method = value, "status" => filter.status = value, "after" => filter.after = value, @@ -97,7 +105,13 @@ impl FlowFilter { let mut query = url::form_urlencoded::Serializer::new(String::new()); let limit = self.limit.map(|l| l.to_string()); let system = self.system.map(|s| s.to_string()); - for (key, value) in [("q", &self.q), ("host", &self.host), ("system", &system)] { + let tunnelled = self.tunnelled.map(|t| t.to_string()); + for (key, value) in [ + ("q", &self.q), + ("host", &self.host), + ("system", &system), + ("tunnelled", &tunnelled), + ] { if let Some(value) = value { query.append_pair(key, value); } @@ -131,6 +145,7 @@ impl FlowFilter { || self.under.is_some() || self.app.is_some() || self.system.is_some() + || self.tunnelled.is_some() || self.any.as_ref().is_some_and(|groups| groups.iter().any(FlowFilter::narrows)) } @@ -138,12 +153,17 @@ impl FlowFilter { /// for the database `conn` reads, which may be from before the folder or gRPC columns, when /// read without migrating. fn condition_in(&self, conn: &rusqlite::Connection) -> (String, Vec) { - self.condition(has_column(conn, "client_cwd"), has_column(conn, "grpc_status")) + let columns = Columns { + cwd: has_column(conn, "client_cwd"), + grpc: has_column(conn, "grpc_status"), + tunnelled: has_column(conn, "tunnelled"), + }; + self.condition(columns) } - /// [`Self::condition_in`], told which of the later columns the database has: `has_cwd`, - /// the folder a program ran in, and `grpc`, gRPC statuses. - fn condition(&self, has_cwd: bool, grpc: bool) -> (String, Vec) { + /// [`Self::condition_in`], told which of the later columns the database has. + fn condition(&self, columns: Columns) -> (String, Vec) { + let Columns { cwd: has_cwd, grpc, tunnelled } = columns; let mut sql = String::from("1"); let mut params: Vec = Vec::new(); for term in self.q.iter().flat_map(|q| q.split_whitespace()) { @@ -209,11 +229,18 @@ impl FlowFilter { Some(false) => sql.push_str(" AND NOT is_system(client_bundle_id, client_path)"), None => {} } + match (self.tunnelled, tunnelled) { + (Some(true), true) => sql.push_str(" AND tunnelled"), + (Some(false), true) => sql.push_str(" AND NOT tunnelled"), + // A database from before tunnelled flows has none. + (Some(true), false) => sql.push_str(" AND 0"), + _ => {} + } // Each group whole, then any of them. if let Some(groups) = self.any.as_ref().filter(|g| !g.is_empty()) { let mut any = Vec::new(); for group in groups { - let (condition, group_params) = group.condition(has_cwd, grpc); + let (condition, group_params) = group.condition(columns); any.push(format!("({condition})")); params.extend(group_params); } @@ -245,6 +272,17 @@ impl FlowFilter { } } +/// Which of the columns later migrations added a database has, as one opened read-only before +/// they ran may not. +#[derive(Clone, Copy)] +struct Columns { + /// The folder a program ran in. + cwd: bool, + /// gRPC statuses. + grpc: bool, + tunnelled: bool, +} + /// The host and port of `url`, in SQL: what follows `://` up to the first `/`, `?` or `#`. const HOST_SQL: &str = "(SELECT CASE WHEN instr(r, '/') > 0 THEN substr(r, 1, instr(r, '/') - 1) \ ELSE r END FROM (SELECT replace(replace(CASE WHEN instr(url, '://') > 0 \ @@ -647,6 +685,73 @@ pub fn get(db: &ProxyQueryManager, flow_id: &str) -> Result { find(db, flow_id).ok_or_else(|| format!("No flow {flow_id}")) } +/// The hosts and apps flows have gone to and come from, for the decrypt list's suggestions. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct SeenTargets { + /// Hosts as the sidebar names them, with a port other than the scheme's: most recent first. + pub hosts: Vec, + /// Apps by name and bundle id, most recent first. + pub apps: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct SeenApp { + pub name: String, + pub bundle_id: Option, +} + +/// Distinct hosts and apps listed by [`seen`] at most. +const SEEN_LIMIT: i64 = 1000; + +/// The distinct hosts and apps of the flows not cleared, most recent first, up to +/// [`SEEN_LIMIT`] of each. +pub fn seen(db: &ProxyQueryManager) -> Result { + db.with_conn(|c| { + let conn = c.conn().resolve(); + let mut params: Vec = Vec::new(); + let mut cleared = String::new(); + push_clear_mark(conn, &mut cleared, &mut params); + params.push(SEEN_LIMIT.into()); + // The newest flow of each host, whose URL names it as the sidebar does. + let sql = format!( + "SELECT url FROM flows WHERE rowid IN (SELECT max(rowid) FROM flows WHERE 1{cleared} \ + GROUP BY lower({HOST_SQL})) ORDER BY rowid DESC LIMIT ?" + ); + let mut stmt = conn.prepare(&sql).map_err(|e| e.to_string())?; + let urls = stmt + .query_map(rusqlite::params_from_iter(params.clone()), |r| r.get::<_, String>(0)) + .map_err(|e| e.to_string())? + .collect::, _>>() + .map_err(|e| e.to_string())?; + let mut hosts: Vec = Vec::new(); + for url in urls { + let path = sidebar_path(&url); + let host = path.split('/').next().unwrap_or_default().to_string(); + if !host.is_empty() && !hosts.contains(&host) { + hosts.push(host); + } + } + + let sql = format!( + "SELECT client_name, client_bundle_id FROM flows WHERE client_name IS NOT NULL{cleared} \ + GROUP BY client_name, client_bundle_id ORDER BY max(rowid) DESC LIMIT ?" + ); + let mut stmt = conn.prepare(&sql).map_err(|e| e.to_string())?; + let apps = stmt + .query_map(rusqlite::params_from_iter(params), |r| { + Ok(SeenApp { name: r.get(0)?, bundle_id: r.get(1)? }) + }) + .map_err(|e| e.to_string())? + .collect::, _>>() + .map_err(|e| e.to_string())?; + Ok(SeenTargets { hosts, apps }) + }) +} + pub fn count(db: &ProxyQueryManager) -> Result { db.with_conn(|c| count_in(c.conn().resolve())) } @@ -1326,6 +1431,12 @@ mod tests { assert_eq!(FlowFilter::from_query(&filter.to_query()).unwrap(), filter); assert_eq!(FlowFilter::from_query("").unwrap(), FlowFilter::default()); assert!(FlowFilter::from_query("limit=lots").is_err()); + + let tunnelled = FlowFilter { tunnelled: Some(true), ..Default::default() }; + assert_eq!(tunnelled.to_query(), "tunnelled=true"); + assert_eq!(FlowFilter::from_query("tunnelled=true").unwrap(), tunnelled); + assert!(tunnelled.narrows()); + assert!(FlowFilter::from_query("tunnelled=maybe").is_err()); } #[test] diff --git a/crates-proxy/yaak-proxy-lib/src/lib.rs b/crates-proxy/yaak-proxy-lib/src/lib.rs index f2fd75f3a..5411f408b 100644 --- a/crates-proxy/yaak-proxy-lib/src/lib.rs +++ b/crates-proxy/yaak-proxy-lib/src/lib.rs @@ -29,7 +29,7 @@ use crate::macos::{SystemProxyBackend, SystemProxyStatus}; use crate::messages::FlowMessage; use crate::models::{Flow, FlowBody, ModelPayload, ProxyHeader, StreamKind, find_header}; use crate::save::{ClientDb, SaveFlowRequest, SaveFlowResponse, WorkspaceSummary}; -use crate::settings::{HttpVersion, ProxySettings}; +use crate::settings::{DecryptList, HttpVersion, ProxySettings}; use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64; use chrono::{DateTime, Utc}; @@ -80,6 +80,9 @@ pub struct ProxyCtx { /// This context's name as the one deleting cleared flows, which the database leases to one /// process at a time. cleaner_id: String, + /// Held through each change to the decrypt list, so concurrent edits apply in turn, each to + /// the list the last one left. + decrypt_edits: Arc>, /// Where the master key that unwraps the data key is kept: the keychain, normally. keys: MasterKeySource, /// The data key, once read. Held while it is read and the databases converted and opened, @@ -127,6 +130,7 @@ impl ProxyCtx { start_error: Arc::new(Mutex::new(None)), deleting_cleared: Arc::new(Mutex::new(false)), cleaner_id: yaak_database::generate_prefixed_id("cln"), + decrypt_edits: Arc::new(Mutex::new(())), keys, unlocked: Arc::new(Mutex::new(None)), encrypting: Arc::new(Mutex::new(None)), @@ -370,9 +374,15 @@ impl ProxyCtx { let mut config = ProxyConfig::new(port.unwrap_or(settings.port), Arc::new(ca)); config.capture_body_limit = settings.capture_body_limit; config.http2 = settings.http_version == HttpVersion::Auto; + config.decrypt = settings.decrypt_policy(); // Filled once the engine has bound, since the lookup matches sockets by its port. let attribution_slot: Arc> = Arc::new(OnceLock::new()); let slot = attribution_slot.clone(); + config.app_of = Some(Arc::new(move |connection| { + let app = slot.get()?.get(connection, ATTRIBUTION_WAIT)?; + Some(yaak_proxy::DecryptApp { name: app.name, bundle_id: app.bundle_id }) + })); + let slot = attribution_slot.clone(); config.on_accept = Some(Arc::new(move |connection, client_addr| { // A client that is answered quickly may exit soon after, so it is found before // anything is read from it: a few milliseconds once per connection at most. @@ -430,11 +440,17 @@ impl ProxyCtx { return; }; let ended = db.with_tx(|c| { - c.conn().resolve().execute( + let conn = c.conn().resolve(); + conn.execute( "UPDATE flows SET error = ?1 WHERE stream_kind IS NOT NULL \ AND elapsed_ms IS NULL AND error IS NULL AND NOT cancelled", [STOPPED_MID_STREAM], )?; + conn.execute( + "UPDATE flows SET error = ?1 WHERE tunnelled \ + AND elapsed_ms IS NULL AND error IS NULL", + [STOPPED_MID_TUNNEL], + )?; Ok::<_, yaak_database::Error>(()) }); if let Err(e) = ended { @@ -687,6 +703,47 @@ impl ProxyCtx { Ok(()) } + /// What HTTPS is decrypted, as `settings.json` has it. + pub fn decrypt_list(&self) -> Result { + Ok(self.settings()?.decrypt_list()) + } + + /// Replaces the decrypt list, saved in `settings.json` and applied to the running engine's + /// next connections. Connections already open stay as they were. + pub fn set_decrypt_list(&self, entries: &[String]) -> Result { + self.change_decrypt_list(|settings| settings.set_decrypt_list(entries)) + } + + /// Adds and removes entries in the list as it is now, so edits made together each keep + /// the other's, unlike replacing the whole list from a copy read earlier. + pub fn edit_decrypt_list( + &self, + add: &[String], + remove: &[String], + ) -> Result { + self.change_decrypt_list(|settings| settings.edit_decrypt_list(add, remove)) + } + + /// Changes the list in `settings.json`, one change at a time, applies it to the running + /// engine, and tells the app. + fn change_decrypt_list( + &self, + change: impl FnOnce(&mut ProxySettings) -> Result<(), String>, + ) -> Result { + let _turn = self.decrypt_edits.lock().map_err(|_| "lock poisoned".to_string())?; + let mut settings = self.settings()?; + change(&mut settings)?; + settings.save(&self.settings_path())?; + if let Ok(handle) = self.handle.lock() + && let Some(handle) = handle.as_ref() + { + handle.set_decrypt_policy(settings.decrypt_policy()); + } + let list = settings.decrypt_list(); + self.events.emit("decrypt_list_changed", &DecryptListChangedPayload { list: list.clone() }); + Ok(list) + } + /// Intercepts again the hosts passed through after clients rejected the CA, as they no /// longer will once it is trusted. pub fn forget_passthrough(&self) { @@ -947,6 +1004,13 @@ pub enum ProxyState { Stopped, } +/// The decrypt list changed, here or through the control API. +#[derive(Serialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct DecryptListChangedPayload { + pub list: DecryptList, +} + #[derive(Serialize, TS)] #[ts(export, export_to = "gen_rpc.ts")] pub struct ProxyStatePayload { @@ -1090,6 +1154,32 @@ pub struct SetHttpVersionRequest { pub http_version: HttpVersion, } +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct GetDecryptListRequest {} + +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct SetDecryptListRequest { + /// Every entry, in order, as typed: a bare hostname is a `host:` entry. A built-in + /// exclusion left out is removed. + pub entries: Vec, +} + +/// Entries to add after the user's own and to remove, applied to the list as it is when the +/// change lands. Each is normalized first. +#[derive(Debug, Clone, Default, Serialize, Deserialize, TS)] +#[serde(default)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct EditDecryptListRequest { + pub add: Vec, + pub remove: Vec, +} + +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct ListSeenRequest {} + #[derive(Deserialize, TS)] #[ts(export, export_to = "gen_rpc.ts")] pub struct AppIconRequest { @@ -1498,6 +1588,42 @@ fn set_http_version(ctx: &ProxyCtx, req: SetHttpVersionRequest) -> Result Result { + ctx.decrypt_list().map_err(rpc_error) +} + +/// Saved and applied here, or, while another process such as the CLI daemon owns the engine, +/// there, as its engine is the one that reads it. +async fn set_decrypt_list( + ctx: ProxyCtx, + req: SetDecryptListRequest, +) -> Result { + if ctx.port().is_none() + && let Some(owner) = ctx.other_owner() + { + return ControlClient::new(owner).set_decrypt_list(&req.entries).await.map_err(rpc_error); + } + blocking(move || ctx.set_decrypt_list(&req.entries)).await +} + +/// [`set_decrypt_list`], adding and removing entries rather than replacing them all. +async fn edit_decrypt_list( + ctx: ProxyCtx, + req: EditDecryptListRequest, +) -> Result { + if ctx.port().is_none() + && let Some(owner) = ctx.other_owner() + { + return ControlClient::new(owner).edit_decrypt_list(&req).await.map_err(rpc_error); + } + blocking(move || ctx.edit_decrypt_list(&req.add, &req.remove)).await +} + +async fn list_seen(ctx: ProxyCtx, _req: ListSeenRequest) -> Result { + let db = ctx.db()?; + blocking(move || flows::seen(&db)).await +} + async fn take_over(ctx: ProxyCtx, _req: TakeOverRequest) -> Result { ctx.take_over().await.map_err(rpc_error)?; Ok(true) @@ -1802,6 +1928,9 @@ const DUE_CHECK_INTERVAL: Duration = Duration::from_millis(50); /// Why a stream still open when the engine stopped ended. const STOPPED_MID_STREAM: &str = "Yaak Proxy stopped before the stream ended"; +/// Why a tunnel still open when the engine stopped ended. +const STOPPED_MID_TUNNEL: &str = "Yaak Proxy stopped before the connection closed"; + /// A flow between its `RequestStart` and its end, with its bodies being written as they /// stream. struct InFlight { @@ -1956,7 +2085,8 @@ fn run_event_loop(rx: std::sync::mpsc::Receiver, sink: FlowSink) { // as they always have been. Every flow that ended is written. for (_, mut f) in in_flight.drain() { if f.persisted { - f.flow.error = Some(STOPPED_MID_STREAM.to_string()); + let why = if f.flow.tunnelled { STOPPED_MID_TUNNEL } else { STOPPED_MID_STREAM }; + f.flow.error = Some(why.to_string()); sink.finish_flow(f); } } @@ -2061,6 +2191,44 @@ impl FlowSink { }, ); } + ProxyEvent::TunnelStart { id, started_at, authority, connection, client_addr } => { + if !recording.lock().map(|r| r.was_on_at(started_at)).unwrap_or(true) { + return; + } + attribution.begin(connection, client_addr); + let flow = Flow { + id: Flow::generate_id(), + started_at: DateTime::::from(started_at).naive_utc(), + method: "CONNECT".to_string(), + url: format!("https://{authority}"), + client_http_version: "HTTP/1.1".to_string(), + tunnelled: true, + ..Default::default() + }; + let mut f = InFlight { + flow, + connection, + request_body: None, + response_body: None, + persisted: false, + cleared: false, + next_seq: 0, + dirty: false, + last_write: None, + }; + // Shown while it is open, as tunnels often stay open for minutes. + self.write_stream_flow(&mut f); + in_flight.insert(id, f); + } + ProxyEvent::TunnelEnd { id, sent, received, elapsed_ms, error } => { + if let Some(mut f) = in_flight.remove(&id) { + f.flow.request_size = sent as i64; + f.flow.response_size = received as i64; + f.flow.elapsed_ms = Some(clamp_ms(elapsed_ms)); + f.flow.error = error; + self.finish_flow(f); + } + } ProxyEvent::StreamStart { id, kind } => { if let Some(f) = in_flight.get_mut(&id) { let kind = StreamKind::from(kind); @@ -2479,6 +2647,7 @@ define_rpc! { commands { execute_action(ActionInvocation) -> bool, format_body(FormatBodyRequest) -> String, + get_decrypt_list(GetDecryptListRequest) -> DecryptList, get_message_payload(GetMessagePayloadRequest) -> GetMessagePayloadResponse, list_actions(ListActionsRequest) -> ListActionsResponse, list_flows(flows::FlowFilter) -> ListFlowsResponse, @@ -2502,6 +2671,9 @@ define_rpc! { get_proxy_state(GetProxyStateRequest) -> GetProxyStateResponse, helper_register(HelperRegisterRequest) -> SystemProxyStatus, helper_unregister(HelperUnregisterRequest) -> SystemProxyStatus, + edit_decrypt_list(EditDecryptListRequest) -> DecryptList, + list_seen(ListSeenRequest) -> flows::SeenTargets, + set_decrypt_list(SetDecryptListRequest) -> DecryptList, list_workspaces(ListWorkspacesRequest) -> ListWorkspacesResponse, preview_save(PreviewSaveRequest) -> SavePreview, reflect_grpc_host(ReflectGrpcHostRequest) -> GrpcSchema, @@ -2512,6 +2684,7 @@ define_rpc! { take_over(TakeOverRequest) -> bool, } events { + decrypt_list_changed(DecryptListChangedPayload), flows_cleared(FlowsClearedPayload), message_write(MessagePayload), model_write(ModelPayload), diff --git a/crates-proxy/yaak-proxy-lib/src/models.rs b/crates-proxy/yaak-proxy-lib/src/models.rs index 90a9c2d99..ceedbb3be 100644 --- a/crates-proxy/yaak-proxy-lib/src/models.rs +++ b/crates-proxy/yaak-proxy-lib/src/models.rs @@ -91,6 +91,10 @@ pub struct Flow { pub grpc_status: Option, /// A gRPC call's `grpc-message`, decoded. pub grpc_message: Option, + /// An HTTPS connection passed through undecrypted, as the decrypt list leaves it out or its + /// client rejected the proxy's certificate: method `CONNECT`, its URL the host, its sizes + /// the bytes each way, and no headers or bodies. Open until `elapsedMs` or `error` is set. + pub tunnelled: bool, } /// The streaming protocol a flow carries. @@ -239,6 +243,7 @@ impl UpsertModelInfo for Flow { (GrpcMethod, self.grpc_method.into()), (GrpcStatus, self.grpc_status.into()), (GrpcMessage, self.grpc_message.into()), + (Tunnelled, self.tunnelled.into()), ]) } @@ -275,6 +280,7 @@ impl UpsertModelInfo for Flow { FlowIden::GrpcMethod, FlowIden::GrpcStatus, FlowIden::GrpcMessage, + FlowIden::Tunnelled, ] } @@ -320,6 +326,7 @@ impl UpsertModelInfo for Flow { grpc_method: added_column(r, "grpc_method")?, grpc_status: added_column(r, "grpc_status")?, grpc_message: added_column(r, "grpc_message")?, + tunnelled: added_column(r, "tunnelled")?.unwrap_or_default(), }) } } diff --git a/crates-proxy/yaak-proxy-lib/src/settings.rs b/crates-proxy/yaak-proxy-lib/src/settings.rs index 59159ed3f..28f7a1363 100644 --- a/crates-proxy/yaak-proxy-lib/src/settings.rs +++ b/crates-proxy/yaak-proxy-lib/src/settings.rs @@ -1,7 +1,7 @@ use serde::{Deserialize, Serialize}; use std::path::Path; use ts_rs::TS; -use yaak_proxy::DEFAULT_CAPTURE_BODY_LIMIT; +use yaak_proxy::{BUILT_IN_EXCLUSIONS, DEFAULT_CAPTURE_BODY_LIMIT, DecryptEntry, DecryptPolicy}; /// `settings.json` in the proxy data directory. Missing keys take their defaults. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -17,6 +17,12 @@ pub struct ProxySettings { pub capture_body_limit: usize, /// The HTTP versions offered to clients in intercepted HTTPS. pub http_version: HttpVersion, + /// The user's decrypt entries, in order: `host:api.example.com`, `app:Google Chrome`, `*`, + /// `!host:*.bank.com`. Empty, so HTTPS is tunnelled, until hosts or apps are added. The + /// built-in exclusions are not kept here; see [`Self::decrypt_list`]. + pub decrypt: Vec, + /// The built-in exclusions the user removed. + pub decrypt_built_ins_removed: Vec, } impl Default for ProxySettings { @@ -26,10 +32,32 @@ impl Default for ProxySettings { system_proxy_at_launch: true, capture_body_limit: DEFAULT_CAPTURE_BODY_LIMIT, http_version: HttpVersion::Auto, + decrypt: Vec::new(), + decrypt_built_ins_removed: Vec::new(), } } } +/// One entry of the decrypt list, as the app shows it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct DecryptListEntry { + /// As stored: `host:*.example.com`, `app:Google Chrome`, `!host:*.apple.com`, `*`. + pub entry: String, + /// Shipped with the app: an exclusion of Apple's services or a pinned app's hosts. + pub built_in: bool, +} + +/// What HTTPS is decrypted: the user's entries in order, then the built-in exclusions not +/// removed. A connection is decrypted when a positive entry matches it and no exclusion does. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct DecryptList { + pub entries: Vec, +} + /// The HTTP versions offered to clients in intercepted HTTPS. Upstream, the engine speaks /// whichever the server picks either way. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, TS)] @@ -44,6 +72,84 @@ pub enum HttpVersion { } impl ProxySettings { + /// The user's entries, then the built-in exclusions they kept. + pub fn decrypt_list(&self) -> DecryptList { + let user = + self.decrypt.iter().map(|e| DecryptListEntry { entry: e.clone(), built_in: false }); + let built_in = BUILT_IN_EXCLUSIONS + .iter() + .filter(|e| !self.decrypt_built_ins_removed.iter().any(|r| r == *e)) + .map(|e| DecryptListEntry { entry: e.to_string(), built_in: true }); + DecryptList { entries: user.chain(built_in).collect() } + } + + /// Replaces the list with `entries`, as the user may type them: each is normalized, a bare + /// hostname becoming `host:`, and repeats are dropped. A built-in exclusion left out is + /// removed, and one written again is restored. Fails, changing nothing, on an entry that + /// does not parse. + pub fn set_decrypt_list>(&mut self, entries: &[S]) -> Result<(), String> { + let mut user: Vec = Vec::new(); + let mut kept_built_ins: Vec = Vec::new(); + for entry in entries { + let parsed = DecryptEntry::parse(entry.as_ref())?; + let normalized = parsed.to_string(); + let list = if parsed.built_in { &mut kept_built_ins } else { &mut user }; + if !list.contains(&normalized) { + list.push(normalized); + } + } + self.decrypt = user; + self.decrypt_built_ins_removed = BUILT_IN_EXCLUSIONS + .iter() + .filter(|e| !kept_built_ins.iter().any(|k| k == *e)) + .map(|e| e.to_string()) + .collect(); + Ok(()) + } + + /// Removes `remove` from the list as it is and adds `add` after the user's own entries, + /// each normalized, as [`Self::set_decrypt_list`] would take the result. A built-in removed + /// is remembered as removed, and one added is restored. Fails, changing nothing, on an + /// entry that does not parse. + pub fn edit_decrypt_list>( + &mut self, + add: &[S], + remove: &[S], + ) -> Result<(), String> { + let normalize = |e: &str| DecryptEntry::parse(e).ok().map(|p| p.to_string()); + let add = add + .iter() + .map(|e| DecryptEntry::parse(e.as_ref()).map(|p| p.to_string())) + .collect::, _>>()?; + // An entry goes as stored, so one edited by hand, odd case or malformed, can go too. + let removed = |stored: &str| { + remove.iter().any(|r| { + let r = r.as_ref(); + r == stored || normalize(r).is_some_and(|r| Some(r) == normalize(stored)) + }) + }; + let mut entries: Vec = self + .decrypt_list() + .entries + .into_iter() + .map(|e| e.entry) + .filter(|e| !removed(e)) + .collect(); + entries.extend(add); + self.set_decrypt_list(&entries) + } + + /// The engine's policy from the list. An entry saved by hand that does not parse is left + /// out, with a warning, rather than stopping the engine. + pub fn decrypt_policy(&self) -> DecryptPolicy { + let entries = self.decrypt_list().entries.into_iter().filter_map(|e| { + DecryptEntry::parse(&e.entry) + .inspect_err(|err| log::warn!("Ignoring decrypt entry: {err}")) + .ok() + }); + DecryptPolicy { entries: entries.collect() } + } + /// The settings at `path`, or the defaults when there is no file. pub fn load(path: &Path) -> Result { match std::fs::read_to_string(path) { @@ -92,6 +198,8 @@ mod tests { system_proxy_at_launch: false, capture_body_limit: 1024, http_version: HttpVersion::Http1Only, + decrypt: vec!["host:api.example.com".into()], + decrypt_built_ins_removed: vec!["!host:*.zoom.us".into()], }; settings.save(&path).unwrap(); assert_eq!(ProxySettings::load(&path).unwrap(), settings); @@ -105,6 +213,8 @@ mod tests { "systemProxyAtLaunch": false, "captureBodyLimit": 1024, "httpVersion": "http1Only", + "decrypt": ["host:api.example.com"], + "decryptBuiltInsRemoved": ["!host:*.zoom.us"], }) ); } @@ -137,6 +247,90 @@ mod tests { assert!(json.get("systemProxy").is_none()); } + #[test] + fn the_decrypt_list_starts_with_only_the_built_in_exclusions() { + let settings = ProxySettings::default(); + let list = settings.decrypt_list(); + assert!(!list.entries.is_empty()); + assert!(list.entries.iter().all(|e| e.built_in && e.entry.starts_with('!'))); + let policy = settings.decrypt_policy(); + assert!(!policy.decrypts_anything(), "nothing is decrypted by default"); + } + + #[test] + fn setting_the_decrypt_list() { + let mut settings = ProxySettings::default(); + let mut entries: Vec = vec![ + "api.example.com".into(), + "*".into(), + "host:API.example.com".into(), + "app:Google Chrome".into(), + ]; + // Every built-in exclusion but zoom.us, kept. + entries.extend( + BUILT_IN_EXCLUSIONS.iter().filter(|e| **e != "!host:*.zoom.us").map(|e| e.to_string()), + ); + settings.set_decrypt_list(&entries).unwrap(); + assert_eq!(settings.decrypt, ["host:api.example.com", "*", "app:Google Chrome"]); + assert_eq!(settings.decrypt_built_ins_removed, ["!host:*.zoom.us"]); + let list = settings.decrypt_list(); + assert_eq!( + list.entries[0], + DecryptListEntry { entry: "host:api.example.com".into(), built_in: false } + ); + assert!(list.entries[3].built_in); + assert!(!list.entries.iter().any(|e| e.entry == "!host:*.zoom.us")); + + // Written again, a built-in exclusion is restored. + let mut again: Vec = list.entries.iter().map(|e| e.entry.clone()).collect(); + again.push("!host:*.zoom.us".into()); + settings.set_decrypt_list(&again).unwrap(); + assert!(settings.decrypt_built_ins_removed.is_empty()); + + // An invalid entry changes nothing. + let before = settings.clone(); + assert!(settings.set_decrypt_list(&["host:a b"]).is_err()); + assert_eq!(settings, before); + } + + #[test] + fn editing_applies_to_the_list_as_it_is() { + let mut settings = + ProxySettings { decrypt: vec!["host:a.com".into()], ..Default::default() }; + // Two edits made from the same earlier copy both land. + settings.edit_decrypt_list(&["b.com"], &[]).unwrap(); + settings.edit_decrypt_list(&["app:Slack"], &["host:a.com"]).unwrap(); + assert_eq!(settings.decrypt, ["host:b.com", "app:Slack"]); + // A built-in removed and restored. + settings.edit_decrypt_list(&[], &["!host:*.zoom.us"]).unwrap(); + assert_eq!(settings.decrypt_built_ins_removed, ["!host:*.zoom.us"]); + settings.edit_decrypt_list(&["!host:*.zoom.us"], &[]).unwrap(); + assert!(settings.decrypt_built_ins_removed.is_empty()); + let before = settings.clone(); + assert!(settings.edit_decrypt_list(&["host:a b"], &[]).is_err()); + assert_eq!(settings, before); + } + + #[test] + fn entries_edited_by_hand_can_be_removed() { + let mut settings = ProxySettings { + decrypt: vec!["API.example.com".into(), "host:a b".into(), "b.com".into()], + ..Default::default() + }; + settings.edit_decrypt_list(&[], &["host:api.example.com", "host:a b"]).unwrap(); + assert_eq!(settings.decrypt, ["host:b.com"]); + } + + #[test] + fn a_hand_edited_entry_that_does_not_parse_is_skipped() { + let settings = ProxySettings { + decrypt: vec!["host:a b".into(), "x.com".into()], + ..Default::default() + }; + let policy = settings.decrypt_policy(); + assert_eq!(policy.entries.iter().filter(|e| !e.negated).count(), 1); + } + #[test] fn invalid_json_is_an_error() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates-proxy/yaak-proxy-lib/tests/control.rs b/crates-proxy/yaak-proxy-lib/tests/control.rs index ba22e7fca..65185eaff 100644 --- a/crates-proxy/yaak-proxy-lib/tests/control.rs +++ b/crates-proxy/yaak-proxy-lib/tests/control.rs @@ -772,3 +772,112 @@ async fn another_app_owning_the_engine_is_not_taken_over() { assert_eq!(app.port(), None); other.stop().unwrap(); } + +#[tokio::test(flavor = "multi_thread")] +async fn the_decrypt_list_is_read_and_replaced_over_the_control_api() { + let dir = tempfile::tempdir().unwrap(); + let client_dir = tempfile::tempdir().unwrap(); + let ctx = ctx(dir.path(), client_dir.path()); + let _server = ControlServer::start(ctx.clone()).unwrap(); + let info = control::read(dir.path()).unwrap(); + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let url = format!("{}/decrypt", info.base_url()); + + assert_eq!(http.get(&url).send().await.unwrap().status(), 401); + let list: Value = + http.get(&url).bearer_auth(&info.token).send().await.unwrap().json().await.unwrap(); + assert!(list["entries"].as_array().unwrap().iter().all(|e| e["builtIn"] == true)); + + let put = http + .put(&url) + .bearer_auth(&info.token) + .json(&json!({"entries": ["*", "!host:*.bank.example"]})) + .send() + .await + .unwrap(); + assert_eq!(put.status(), 200); + let list: Value = put.json().await.unwrap(); + assert_eq!( + list["entries"], + json!([ + {"entry": "*", "builtIn": false}, + {"entry": "!host:*.bank.example", "builtIn": false}, + ]), + "every built-in exclusion left out is removed" + ); + let read: Value = + http.get(&url).bearer_auth(&info.token).send().await.unwrap().json().await.unwrap(); + assert_eq!(read, list); + + let bad = http + .put(&url) + .bearer_auth(&info.token) + .json(&json!({"entries": ["app:"]})) + .send() + .await + .unwrap(); + assert_eq!(bad.status(), 400); +} + +/// While the CLI owns the engine, the app's change to the decrypt list goes to the CLI, whose +/// engine decrypts the next connection by it. +#[tokio::test(flavor = "multi_thread")] +async fn the_decrypt_list_reaches_the_cli_that_owns_the_engine() { + let dir = tempfile::tempdir().unwrap(); + let client_dir = tempfile::tempdir().unwrap(); + let cli = ctx(dir.path(), client_dir.path()).with_owner(Owner::Cli); + let _server = ControlServer::start(cli.clone()).unwrap(); + cli.start().unwrap(); + let app = ctx(dir.path(), client_dir.path()); + + let list = rpc(&app, "set_decrypt_list", json!({"entries": ["*"]})).await.unwrap(); + assert_eq!(list["entries"][0], json!({"entry": "*", "builtIn": false})); + + // Decrypted, the request is a flow of its own: a 502, as nothing listens there. + let ca = std::fs::read(dir.path().join("ca/ca.pem")).unwrap(); + let closed = TcpListener::bind("127.0.0.1:0").await.unwrap().local_addr().unwrap().port(); + let client = reqwest::Client::builder() + .proxy(reqwest::Proxy::all(format!("http://127.0.0.1:{}", cli.port().unwrap())).unwrap()) + .add_root_certificate(reqwest::Certificate::from_pem(&ca).unwrap()) + .build() + .unwrap(); + let url = format!("https://localhost:{closed}/forwarded"); + assert_eq!(client.get(&url).send().await.unwrap().status(), 502); + let deadline = Instant::now() + Duration::from_secs(5); + loop { + let flows = cli.db().unwrap().with_conn(|db| db.find_all::().unwrap()); + if let Some(flow) = flows.iter().find(|f| f.url == url) { + assert!(!flow.tunnelled); + break; + } + assert!(Instant::now() < deadline, "no decrypted flow for {url}"); + tokio::time::sleep(Duration::from_millis(20)).await; + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn the_decrypt_list_is_edited_over_the_control_api() { + let dir = tempfile::tempdir().unwrap(); + let client_dir = tempfile::tempdir().unwrap(); + let ctx = ctx(dir.path(), client_dir.path()); + let _server = ControlServer::start(ctx.clone()).unwrap(); + let info = control::read(dir.path()).unwrap(); + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let url = format!("{}/decrypt", info.base_url()); + + for add in ["a.com", "b.com"] { + let resp = + http.patch(&url).bearer_auth(&info.token).json(&json!({"add": [add]})).send().await; + assert_eq!(resp.unwrap().status(), 200); + } + let resp = http + .patch(&url) + .bearer_auth(&info.token) + .json(&json!({"remove": ["host:a.com"]})) + .send() + .await + .unwrap(); + let list: Value = resp.json().await.unwrap(); + assert_eq!(list["entries"][0], json!({"entry": "host:b.com", "builtIn": false})); + assert_eq!(list["entries"][1]["builtIn"], true); +} diff --git a/crates-proxy/yaak-proxy-lib/tests/flow.rs b/crates-proxy/yaak-proxy-lib/tests/flow.rs index 303c7904c..ceb04d904 100644 --- a/crates-proxy/yaak-proxy-lib/tests/flow.rs +++ b/crates-proxy/yaak-proxy-lib/tests/flow.rs @@ -267,7 +267,13 @@ async fn cancelled_flow_is_not_an_error() { #[tokio::test(flavor = "multi_thread")] async fn http_version_setting_applies_to_new_connections() { let dir = tempfile::tempdir().unwrap(); - let settings = ProxySettings { port: 0, system_proxy_at_launch: false, ..Default::default() }; + // Decrypted, as only intercepted HTTPS offers the client a version. + let settings = ProxySettings { + port: 0, + system_proxy_at_launch: false, + decrypt: vec!["*".into()], + ..Default::default() + }; settings.save(&dir.path().join("settings.json")).unwrap(); let (events, _event_rx) = RpcEventEmitter::new(); @@ -416,6 +422,213 @@ async fn body_rpcs() { assert_eq!(formatted, json!("{\n \"a\": [\n 1,\n 2\n ]\n}")); } +/// A TCP server that echoes what it reads, then closes once the client does. +async fn spawn_echo() -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + loop { + let (mut tcp, _) = listener.accept().await.unwrap(); + tokio::spawn(async move { + let (mut read, mut write) = tcp.split(); + let _ = tokio::io::copy(&mut read, &mut write).await; + }); + } + }); + addr +} + +/// Opens a CONNECT tunnel to `target` through the engine on `port`. +async fn connect_through(port: u16, target: &str) -> tokio::net::TcpStream { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let mut tcp = tokio::net::TcpStream::connect(("127.0.0.1", port)).await.unwrap(); + let connect = format!("CONNECT {target} HTTP/1.1\r\nHost: {target}\r\n\r\n"); + tcp.write_all(connect.as_bytes()).await.unwrap(); + let mut head = Vec::new(); + while !head.ends_with(b"\r\n\r\n") { + head.push(tcp.read_u8().await.unwrap()); + } + assert!(head.starts_with(b"HTTP/1.1 200"), "{}", String::from_utf8_lossy(&head)); + tcp +} + +async fn wait_for(ctx: &ProxyCtx, check: impl Fn(&Flow) -> bool) -> Flow { + let deadline = Instant::now() + Duration::from_secs(5); + loop { + let flows = ctx.db().unwrap().with_conn(|db| db.find_all::().unwrap()); + if let Some(flow) = flows.into_iter().find(|f| check(f)) { + return flow; + } + assert!(Instant::now() < deadline, "no such flow"); + tokio::time::sleep(Duration::from_millis(20)).await; + } +} + +/// With the default, empty decrypt list, a CONNECT is tunnelled untouched and recorded as a flow +/// with its host and the bytes each way, shown while open and finished once closed. +#[tokio::test(flavor = "multi_thread")] +async fn a_tunnelled_connect_is_recorded_with_its_sizes() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let dir = tempfile::tempdir().unwrap(); + let settings = ProxySettings { port: 0, system_proxy_at_launch: false, ..Default::default() }; + settings.save(&dir.path().join("settings.json")).unwrap(); + let (events, _event_rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + ctx.start().unwrap(); + let port = ctx.port().unwrap(); + let echo = spawn_echo().await; + let target = format!("localhost:{}", echo.port()); + let url = format!("https://{target}"); + + let mut tcp = connect_through(port, &target).await; + // TLS-looking bytes, which a decrypting engine would have taken for a ClientHello. + let sent = b"\x16\x03\x01 not decrypted"; + tcp.write_all(sent).await.unwrap(); + let mut echoed = vec![0u8; sent.len()]; + tcp.read_exact(&mut echoed).await.unwrap(); + assert_eq!(&echoed, sent, "bytes pass untouched"); + + let open = wait_for(&ctx, |f| f.url == url).await; + assert!(open.tunnelled); + assert_eq!(open.method, "CONNECT"); + assert_eq!(open.elapsed_ms, None, "open while the tunnel is"); + + drop(tcp); + let done = wait_for(&ctx, |f| f.url == url && f.elapsed_ms.is_some()).await; + assert_eq!(done.id, open.id); + assert_eq!((done.request_size, done.response_size), (sent.len() as i64, sent.len() as i64)); + assert_eq!(done.error, None); + assert!(done.request_headers.is_empty() && done.response_headers.is_empty()); + + let tunnelled = rpc(&ctx, "list_flows", json!({"tunnelled": true})).await.unwrap(); + assert_eq!(tunnelled["flows"].as_array().unwrap().len(), 1); + assert_eq!(tunnelled["flows"][0]["tunnelled"], true); + let decrypted = rpc(&ctx, "list_flows", json!({"tunnelled": false})).await.unwrap(); + assert_eq!(decrypted["flows"].as_array().unwrap().len(), 0); +} + +#[tokio::test(flavor = "multi_thread")] +async fn the_decrypt_list_rpcs_save_and_apply() { + let dir = tempfile::tempdir().unwrap(); + let settings = ProxySettings { port: 0, system_proxy_at_launch: false, ..Default::default() }; + settings.save(&dir.path().join("settings.json")).unwrap(); + let (events, _event_rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + + let list = rpc(&ctx, "get_decrypt_list", json!({})).await.unwrap(); + let entries = list["entries"].as_array().unwrap(); + assert!(entries.iter().all(|e| e["builtIn"] == true), "only the built-ins: {list}"); + assert!(entries.iter().any(|e| e["entry"] == "!host:*.apple.com")); + + let set = json!({"entries": ["API.example.com", "app:Google Chrome", "!host:*.apple.com"]}); + let list = rpc(&ctx, "set_decrypt_list", set).await.unwrap(); + assert_eq!( + list["entries"], + json!([ + {"entry": "host:api.example.com", "builtIn": false}, + {"entry": "app:Google Chrome", "builtIn": false}, + {"entry": "!host:*.apple.com", "builtIn": true}, + ]) + ); + let saved = ProxySettings::load(&dir.path().join("settings.json")).unwrap(); + assert_eq!(saved.decrypt, ["host:api.example.com", "app:Google Chrome"]); + assert!(saved.decrypt_built_ins_removed.contains(&"!host:*.icloud.com".to_string())); + + let invalid = rpc(&ctx, "set_decrypt_list", json!({"entries": ["host:a b"]})).await; + assert!(invalid.unwrap_err().contains("Invalid host")); + assert_eq!(rpc(&ctx, "get_decrypt_list", json!({})).await.unwrap(), list); + + // Applied to the running engine's next connection: decrypting everything intercepts a + // CONNECT, which a client that does not trust the CA then rejects. + ctx.start().unwrap(); + rpc(&ctx, "set_decrypt_list", json!({"entries": ["*"]})).await.unwrap(); + let echo = spawn_echo().await; + let client = reqwest::Client::builder() + .proxy(reqwest::Proxy::all(format!("http://127.0.0.1:{}", ctx.port().unwrap())).unwrap()) + .build() + .unwrap(); + let err = client.get(format!("https://localhost:{}/", echo.port())).send().await.unwrap_err(); + assert!(err.is_connect() || err.is_request(), "{err:?}"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn seen_lists_hosts_and_apps_most_recent_first() { + let dir = tempfile::tempdir().unwrap(); + let settings = ProxySettings { port: 0, system_proxy_at_launch: false, ..Default::default() }; + settings.save(&dir.path().join("settings.json")).unwrap(); + let (events, _event_rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + let flow = |url: &str, app: Option<(&str, Option<&str>)>, at: i64| Flow { + id: format!("fl_{at}"), + url: url.into(), + method: "GET".into(), + started_at: chrono::DateTime::from_timestamp(at, 0).unwrap().naive_utc(), + client_name: app.map(|(n, _)| n.to_string()), + client_bundle_id: app.and_then(|(_, b)| b.map(String::from)), + ..Default::default() + }; + let chrome = Some(("Google Chrome", Some("com.google.Chrome"))); + ctx.db() + .unwrap() + .with_tx(|c| { + for f in [ + flow("https://api.example.com/v1", chrome, 1), + flow("https://API.example.com:443/v2", None, 2), + flow("https://example.com:8443/", Some(("curl", None)), 3), + flow("http://localhost:80/x", chrome, 4), + ] { + c.upsert(&f, &yaak_database::UpdateSource::Background)?; + } + Ok::<_, yaak_database::Error>(()) + }) + .unwrap(); + + let seen = rpc(&ctx, "list_seen", json!({})).await.unwrap(); + assert_eq!(seen["hosts"], json!(["localhost", "example.com:8443", "api.example.com"])); + assert_eq!( + seen["apps"], + json!([ + {"name": "Google Chrome", "bundleId": "com.google.Chrome"}, + {"name": "curl", "bundleId": null}, + ]) + ); +} + +/// Edits made together each land on the list the other left, and every change is announced to +/// the app, so a list changed elsewhere shows at once. +#[tokio::test(flavor = "multi_thread")] +async fn decrypt_list_edits_apply_to_the_latest_list_and_are_announced() { + let dir = tempfile::tempdir().unwrap(); + let settings = ProxySettings { port: 0, system_proxy_at_launch: false, ..Default::default() }; + settings.save(&dir.path().join("settings.json")).unwrap(); + let (events, event_rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + + let (a, b) = tokio::join!( + rpc(&ctx, "edit_decrypt_list", json!({"add": ["a.com"]})), + rpc(&ctx, "edit_decrypt_list", json!({"add": ["app:Slack"]})), + ); + a.unwrap(); + b.unwrap(); + let list = + rpc(&ctx, "edit_decrypt_list", json!({"remove": ["!host:*.zoom.us"]})).await.unwrap(); + let user: Vec<&str> = list["entries"] + .as_array() + .unwrap() + .iter() + .filter(|e| e["builtIn"] == false) + .map(|e| e["entry"].as_str().unwrap()) + .collect(); + assert_eq!(user.len(), 2); + assert!(user.contains(&"host:a.com") && user.contains(&"app:Slack"), "{user:?}"); + assert!(!list["entries"].as_array().unwrap().iter().any(|e| e["entry"] == "!host:*.zoom.us")); + + let changes: Vec<_> = + event_rx.try_iter().filter(|e| e.event == "decrypt_list_changed").collect(); + assert_eq!(changes.len(), 3); + assert_eq!(changes[2].payload["list"], list); +} + /// Capture throughput through the engine, end to end: 3,000 requests, 16 at a time over /// kept-alive connections, each with a 4 KB response, timed until every flow is stored, and how /// long opening one then takes. Run in release: `cargo test --release -p yaak-proxy-lib --test diff --git a/crates/yaak-proxy/src/decrypt.rs b/crates/yaak-proxy/src/decrypt.rs new file mode 100644 index 000000000..2fbc6a14e --- /dev/null +++ b/crates/yaak-proxy/src/decrypt.rs @@ -0,0 +1,445 @@ +//! Which HTTPS connections the engine decrypts. The policy is one list of entries: +//! +//! - `host:api.example.com`, `host:*.example.com` (the host itself and every host below it), +//! `host:example.com:8443` (that port only), with `*` matching any run of characters; +//! - `app:Google Chrome` (by the app's name) or `app:com.google.Chrome` (by its bundle id); +//! - `*` alone, everything; +//! - any of these after `!`, which excludes what it matches. +//! +//! A connection is decrypted when a positive entry matches it and no negative one does, wherever +//! each sits in the list. Nothing matching tunnels it, so an empty list decrypts nothing. A +//! built-in exclusion (Apple's services, apps that pin their certificates) applies only to +//! positive entries broader than itself, such as `*` or `host:*.com`: an app entry, or a host +//! entry at least as specific as the exclusion, still decrypts what it names. + +use std::fmt; + +/// Hosts excluded by default: Apple's own services, which macOS daemons reach with pinned +/// certificates, and the sync hosts of common apps that pin theirs. Each is removable. +pub const BUILT_IN_EXCLUSIONS: &[&str] = &[ + "!host:*.apple.com", + "!host:*.icloud.com", + "!host:*.mzstatic.com", + "!host:*.apple-cloudkit.com", + "!host:*.icloud-content.com", + "!host:*.cdn-apple.com", + // Push and FaceTime, kept apart so they stay excluded without the rest of apple.com. + "!host:*.push.apple.com", + "!host:*.ess.apple.com", + "!host:*.1password.com", + "!host:*.1password.ca", + "!host:*.1password.eu", + "!host:*.signal.org", + "!host:*.whispersystems.org", + "!host:*.dropboxapi.com", + "!host:client*.dropbox.com", + "!host:bolt.dropbox.com", + "!host:*.zoom.us", +]; + +/// What an entry matches. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DecryptTarget { + /// `*`: every connection. + Everything, + /// A host pattern, lowercased, and the port it is limited to, if any. + Host { pattern: String, port: Option }, + /// An app's name or bundle id, as a pattern, matched in any case. + App { pattern: String }, +} + +/// One entry of the list. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DecryptEntry { + /// Written with `!`: excludes what it matches. + pub negated: bool, + pub target: DecryptTarget, + /// Shipped with the engine rather than added by the user. + pub built_in: bool, +} + +impl DecryptEntry { + /// Parses an entry as the user may type it: with its prefix, or as a bare hostname, which is + /// a `host:` entry. A URL is taken as its host. Prefixes are any case. + pub fn parse(input: &str) -> Result { + let trimmed = input.trim(); + let (negated, rest) = match trimmed.strip_prefix('!') { + Some(rest) => (true, rest.trim_start()), + None => (false, trimmed), + }; + if rest.is_empty() { + return Err(format!("Empty decrypt entry {input:?}")); + } + let target = if rest == "*" { + DecryptTarget::Everything + } else if let Some(app) = strip_prefix_ci(rest, "app:") { + let app = unquote(app.trim()); + if app.is_empty() { + return Err(format!("No app in decrypt entry {input:?}")); + } + DecryptTarget::App { pattern: app.to_string() } + } else { + let host = unquote(strip_prefix_ci(rest, "host:").unwrap_or(rest).trim()); + parse_host(host).ok_or_else(|| format!("Invalid host in decrypt entry {input:?}"))? + }; + let mut entry = Self { negated, target, built_in: false }; + entry.built_in = BUILT_IN_EXCLUSIONS.contains(&entry.to_string().as_str()); + Ok(entry) + } + + /// The literal characters of the pattern: how specific it is. A site's `*.` counts for + /// nothing, as `*.apple.com` matches `apple.com` itself, so `host:apple.com` ranks with it. + fn specificity(&self) -> usize { + match &self.target { + DecryptTarget::Everything => 0, + DecryptTarget::Host { pattern, port } => { + let pattern = pattern.strip_prefix("*.").unwrap_or(pattern); + pattern.chars().filter(|c| *c != '*').count() + usize::from(port.is_some()) + } + DecryptTarget::App { .. } => usize::MAX, + } + } + + fn matches(&self, conn: &DecryptQuery) -> bool { + match &self.target { + DecryptTarget::Everything => true, + DecryptTarget::Host { pattern, port } => { + port.is_none_or(|p| p == conn.port) && host_matches(pattern, &conn.host) + } + DecryptTarget::App { pattern } => conn.app.as_ref().is_some_and(|app| { + let pattern = pattern.to_lowercase(); + glob(&pattern, &app.name.to_lowercase()) + || app.bundle_id.as_ref().is_some_and(|id| glob(&pattern, &id.to_lowercase())) + }), + } + } +} + +/// The entry as stored and shown: `!host:*.apple.com`, `app:Google Chrome`, `*`. +impl fmt::Display for DecryptEntry { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.negated { + f.write_str("!")?; + } + match &self.target { + DecryptTarget::Everything => f.write_str("*"), + DecryptTarget::Host { pattern, port: None } => write!(f, "host:{pattern}"), + DecryptTarget::Host { pattern, port: Some(port) } if pattern.contains(':') => { + write!(f, "host:[{pattern}]:{port}") + } + DecryptTarget::Host { pattern, port: Some(port) } => write!(f, "host:{pattern}:{port}"), + DecryptTarget::App { pattern } => write!(f, "app:{pattern}"), + } + } +} + +/// A value as the search writes one with spaces in it: `"Google Chrome"`. +fn unquote(s: &str) -> &str { + match s.strip_prefix('"').and_then(|s| s.strip_suffix('"')) { + Some(inner) => inner.trim(), + None => s, + } +} + +fn strip_prefix_ci<'a>(s: &'a str, prefix: &str) -> Option<&'a str> { + let head = s.get(..prefix.len())?; + head.eq_ignore_ascii_case(prefix).then(|| &s[prefix.len()..]) +} + +/// A host pattern with an optional port: `api.example.com`, `*.example.com:8443`, +/// `[::1]:8443`, or a URL, whose host and port it takes. +fn parse_host(input: &str) -> Option { + let mut host = input; + if let Some((_, rest)) = host.split_once("://") { + host = rest.split(['/', '?', '#']).next().unwrap_or_default(); + host = host.rsplit_once('@').map_or(host, |(_, h)| h); + } + let (pattern, port) = if let Some(rest) = host.strip_prefix('[') { + let (addr, after) = rest.split_once(']')?; + let port = match after { + "" => None, + p => Some(p.strip_prefix(':')?.parse::().ok()?), + }; + (addr, port) + } else { + match host.rsplit_once(':') { + // One colon: a port. More: a bare IPv6 address. + Some((name, port)) if !name.contains(':') => (name, Some(port.parse::().ok()?)), + _ => (host, None), + } + }; + let pattern = pattern.trim_end_matches('.').to_lowercase(); + let valid = !pattern.is_empty() + && pattern.chars().all(|c| c.is_ascii_alphanumeric() || "-._*:".contains(c)); + valid.then_some(DecryptTarget::Host { pattern, port }) +} + +/// `*.example.com` matches `example.com` too, as cookie domains do and as the search's `@host` +/// does. +fn host_matches(pattern: &str, host: &str) -> bool { + let host = host.trim_start_matches('[').trim_end_matches(']').trim_end_matches('.'); + let host = host.to_lowercase(); + glob(pattern, &host) || pattern.strip_prefix("*.").is_some_and(|apex| glob(apex, &host)) +} + +/// `*` matches any run of characters, none included; everything else matches itself. +fn glob(pattern: &str, text: &str) -> bool { + let (p, t) = (pattern.as_bytes(), text.as_bytes()); + let (mut pi, mut ti) = (0, 0); + let mut star: Option<(usize, usize)> = None; + while ti < t.len() { + if pi < p.len() && p[pi] == b'*' { + star = Some((pi, ti)); + pi += 1; + } else if pi < p.len() && p[pi] == t[ti] { + pi += 1; + ti += 1; + } else if let Some((sp, st)) = star { + pi = sp + 1; + ti = st + 1; + star = Some((sp, st + 1)); + } else { + return false; + } + } + p[pi..].iter().all(|c| *c == b'*') +} + +/// The app that opened a connection, as the policy's `app:` entries match it. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct DecryptApp { + pub name: String, + pub bundle_id: Option, +} + +/// A CONNECT to judge: where it goes and, when known, the app that sent it. +#[derive(Debug, Clone, Default)] +pub struct DecryptQuery { + pub host: String, + pub port: u16, + pub app: Option, +} + +/// The list of entries, in order. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct DecryptPolicy { + pub entries: Vec, +} + +impl DecryptPolicy { + /// Decrypts everything, without exclusions: the engine's default, for callers that keep no + /// list. + pub fn everything() -> Self { + Self { entries: vec![DecryptEntry::parse("*").unwrap()] } + } + + /// The policy from entries as stored. Fails on the first one that does not parse. + pub fn parse>(entries: &[S]) -> Result { + let entries = entries.iter().map(|e| DecryptEntry::parse(e.as_ref())); + Ok(Self { entries: entries.collect::>()? }) + } + + /// Some entry names an app, so judging a connection needs the app that sent it. + pub fn needs_app(&self) -> bool { + self.entries.iter().any(|e| matches!(e.target, DecryptTarget::App { .. })) + } + + /// Some positive entry could match, so a connection may be decrypted at all. + pub fn decrypts_anything(&self) -> bool { + self.entries.iter().any(|e| !e.negated) + } + + /// Whether to decrypt `conn`: a positive entry matches and no negative one excludes it. A + /// built-in exclusion yields to a positive entry at least as specific as itself. + pub fn decrypts(&self, conn: &DecryptQuery) -> bool { + let Some(best) = self + .entries + .iter() + .filter(|e| !e.negated && e.matches(conn)) + .map(DecryptEntry::specificity) + .max() + else { + return false; + }; + !self + .entries + .iter() + .any(|e| e.negated && e.matches(conn) && (!e.built_in || e.specificity() > best)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn policy(entries: &[&str]) -> DecryptPolicy { + DecryptPolicy::parse(entries).unwrap() + } + + fn to(host: &str, port: u16) -> DecryptQuery { + DecryptQuery { host: host.into(), port, app: None } + } + + fn from_app(host: &str, name: &str, bundle_id: Option<&str>) -> DecryptQuery { + let app = DecryptApp { name: name.into(), bundle_id: bundle_id.map(String::from) }; + DecryptQuery { host: host.into(), port: 443, app: Some(app) } + } + + fn with_built_ins(entries: &[&str]) -> DecryptPolicy { + let mut all: Vec<&str> = entries.to_vec(); + all.extend(BUILT_IN_EXCLUSIONS); + policy(&all) + } + + #[test] + fn an_empty_list_tunnels_everything() { + let empty = policy(&[]); + assert!(!empty.decrypts(&to("api.example.com", 443))); + assert!(!empty.decrypts_anything()); + // The built-in exclusions alone decrypt nothing either. + let built_in = with_built_ins(&[]); + assert!(!built_in.decrypts(&to("api.example.com", 443))); + assert!(!built_in.decrypts_anything()); + } + + #[test] + fn host_patterns() { + let p = policy(&["host:api.example.com"]); + assert!(p.decrypts(&to("api.example.com", 443))); + assert!(p.decrypts(&to("API.Example.com", 8443)), "any case, any port"); + assert!(!p.decrypts(&to("www.example.com", 443))); + assert!(!p.decrypts(&to("example.com", 443))); + + let p = policy(&["host:*.example.com"]); + assert!(p.decrypts(&to("api.example.com", 443))); + assert!(p.decrypts(&to("a.b.example.com", 443))); + assert!(p.decrypts(&to("example.com", 443)), "the site itself too"); + assert!(!p.decrypts(&to("notexample.com", 443))); + assert!(!p.decrypts(&to("example.com.evil.net", 443))); + + let p = policy(&["host:api-*.example.com"]); + assert!(p.decrypts(&to("api-eu.example.com", 443))); + assert!(!p.decrypts(&to("web.example.com", 443))); + } + + #[test] + fn host_patterns_with_a_port() { + let p = policy(&["host:example.com:8443"]); + assert!(p.decrypts(&to("example.com", 8443))); + assert!(!p.decrypts(&to("example.com", 443))); + + let p = policy(&["host:[::1]:8443", "host:fe80::1"]); + assert!(p.decrypts(&to("[::1]", 8443))); + assert!(!p.decrypts(&to("::1", 443))); + assert!(p.decrypts(&to("fe80::1", 443))); + assert_eq!(p.entries[0].to_string(), "host:[::1]:8443"); + assert_eq!(p.entries[1].to_string(), "host:fe80::1"); + } + + #[test] + fn apps_by_name_and_bundle_id() { + let by_name = policy(&["app:Google Chrome"]); + assert!(by_name.decrypts(&from_app("a.com", "Google Chrome", Some("com.google.Chrome")))); + assert!(by_name.decrypts(&from_app("a.com", "google chrome", None))); + assert!(!by_name.decrypts(&from_app("a.com", "Safari", Some("com.apple.Safari")))); + assert!(!by_name.decrypts(&to("a.com", 443)), "an unknown app matches no app entry"); + assert!(by_name.needs_app()); + + let by_id = policy(&["app:com.google.Chrome"]); + assert!(by_id.decrypts(&from_app("a.com", "Google Chrome", Some("com.google.Chrome")))); + assert!(!by_id.decrypts(&from_app("a.com", "Chromium", Some("org.chromium.Chromium")))); + + let pattern = policy(&["app:com.google.*"]); + assert!(pattern.decrypts(&from_app("a.com", "Google Chrome", Some("com.google.Chrome")))); + assert!(!policy(&["host:a.com"]).needs_app()); + } + + #[test] + fn everything_keeps_the_built_in_exclusions() { + let p = with_built_ins(&["*"]); + assert!(p.decrypts(&to("api.example.com", 443))); + assert!(!p.decrypts(&to("gateway.icloud.com", 443))); + assert!(!p.decrypts(&to("apple.com", 443))); + assert!(!p.decrypts(&to("courier.push.apple.com", 5223))); + assert!(!p.decrypts(&to("my.1password.com", 443))); + assert!(!p.decrypts(&to("client-cf.dropbox.com", 443))); + assert!(p.decrypts(&to("www.dropbox.com", 443))); + // A broad wildcard yields to them too. + assert!(!with_built_ins(&["host:*.com"]).decrypts(&to("p01.icloud.com", 443))); + } + + #[test] + fn a_specific_entry_beats_a_built_in_exclusion() { + let p = with_built_ins(&["*", "host:developer.apple.com"]); + assert!(p.decrypts(&to("developer.apple.com", 443))); + assert!(!p.decrypts(&to("www.apple.com", 443))); + assert!(with_built_ins(&["host:*.apple.com"]).decrypts(&to("www.apple.com", 443))); + // The site itself, which the exclusion's `*.apple.com` matches too. + assert!(with_built_ins(&["host:apple.com"]).decrypts(&to("apple.com", 443))); + // A narrower exclusion still wins over a broader entry. + assert!( + !with_built_ins(&["host:*.apple.com"]).decrypts(&to("courier.push.apple.com", 443)) + ); + let safari = from_app("www.icloud.com", "Safari", Some("com.apple.Safari")); + assert!(with_built_ins(&["app:Safari"]).decrypts(&safari)); + } + + #[test] + fn a_removed_exclusion_no_longer_applies() { + let kept: Vec<&str> = + BUILT_IN_EXCLUSIONS.iter().copied().filter(|e| *e != "!host:*.zoom.us").collect(); + let mut entries = vec!["*"]; + entries.extend(kept); + let p = policy(&entries); + assert!(p.decrypts(&to("us02web.zoom.us", 443))); + assert!(!p.decrypts(&to("gateway.icloud.com", 443))); + } + + #[test] + fn negative_entries_win_wherever_they_sit() { + for entries in [ + ["!host:secret.example.com", "host:*.example.com"], + ["host:*.example.com", "!host:secret.example.com"], + ] { + let p = policy(&entries); + assert!(p.decrypts(&to("api.example.com", 443))); + assert!(!p.decrypts(&to("secret.example.com", 443))); + } + // The user's own exclusion beats even an exact entry. + let p = policy(&["host:a.example.com", "!host:*.example.com"]); + assert!(!p.decrypts(&to("a.example.com", 443))); + let p = policy(&["*", "!app:Slack"]); + assert!(!p.decrypts(&from_app("slack.com", "Slack", Some("com.tinyspeck.slackmacgap")))); + assert!(p.decrypts(&from_app("slack.com", "Safari", None))); + } + + #[test] + fn entries_are_normalized() { + let parse = |s: &str| DecryptEntry::parse(s).unwrap().to_string(); + assert_eq!(parse("api.example.com"), "host:api.example.com"); + assert_eq!(parse(" API.Example.com. "), "host:api.example.com"); + assert_eq!(parse("HOST:*.Example.com"), "host:*.example.com"); + assert_eq!(parse("! host:x.com"), "!host:x.com"); + assert_eq!(parse("!x.com"), "!host:x.com"); + assert_eq!(parse("https://api.example.com:8443/v1?q"), "host:api.example.com:8443"); + assert_eq!(parse("App: Google Chrome"), "app:Google Chrome"); + assert_eq!(parse(" * "), "*"); + assert_eq!(parse(r#"app:"Google Chrome""#), "app:Google Chrome"); + assert!(DecryptEntry::parse("").is_err()); + assert!(DecryptEntry::parse("!").is_err()); + assert!(DecryptEntry::parse("app:").is_err()); + assert!(DecryptEntry::parse("host:a b").is_err()); + assert!(DecryptEntry::parse("host:a.com:http").is_err()); + assert!(DecryptEntry::parse("!host:*.apple.com").unwrap().built_in); + assert!(!DecryptEntry::parse("host:*.apple.com").unwrap().built_in); + } + + #[test] + fn globs() { + assert!(glob("*", "")); + assert!(glob("a*c", "abbbc")); + assert!(glob("*b*", "abc")); + assert!(!glob("a*c", "abd")); + assert!(glob("client*.dropbox.com", "client-lb.dropbox.com")); + } +} diff --git a/crates/yaak-proxy/src/lib.rs b/crates/yaak-proxy/src/lib.rs index 3c973569d..54d7447b3 100644 --- a/crates/yaak-proxy/src/lib.rs +++ b/crates/yaak-proxy/src/lib.rs @@ -2,6 +2,7 @@ mod body; mod cert; mod connection; mod decode; +mod decrypt; mod grpc; mod messages; #[cfg(target_os = "macos")] @@ -17,7 +18,7 @@ use std::hash::{BuildHasher, Hasher}; use std::net::SocketAddr; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::mpsc as std_mpsc; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, RwLock}; use std::time::SystemTime; use bytes::Bytes; @@ -32,6 +33,9 @@ use tokio::net::TcpListener; pub use cert::{CaStore, KeyCipher}; use connection::handle_connection; +pub use decrypt::{ + BUILT_IN_EXCLUSIONS, DecryptApp, DecryptEntry, DecryptPolicy, DecryptQuery, DecryptTarget, +}; pub use grpc::{detect as detect_grpc, service_and_method}; pub use messages::{MessageDirection, MessageKind, RAW_LIMIT, StreamKind, StreamMessage}; use request::BoxBody; @@ -62,11 +66,21 @@ pub struct ProxyConfig { /// Record WebSocket frames as messages. Off, upgraded connections are spliced blind, as /// other upgrades are. pub websocket_capture: bool, + /// Which HTTPS connections are decrypted; the rest are tunnelled, recorded as a flow each + /// without their contents. Everything by default. See [`ProxyHandle::set_decrypt_policy`]. + pub decrypt: DecryptPolicy, + /// Finds the app that opened a client connection, by its number, for the policy's `app:` + /// entries. Runs on a blocking thread as each CONNECT is judged, only while the policy has + /// such an entry, and may wait briefly for a lookup in progress. + pub app_of: Option, } /// See [`ProxyConfig::on_accept`]. pub type AcceptHook = Arc; +/// See [`ProxyConfig::app_of`]. +pub type AppLookup = Arc Option + Send + Sync>; + impl ProxyConfig { pub fn new(port: u16, ca: Arc) -> Self { Self { @@ -77,6 +91,8 @@ impl ProxyConfig { on_accept: None, http2: true, websocket_capture: true, + decrypt: DecryptPolicy::everything(), + app_of: None, } } } @@ -85,8 +101,10 @@ impl ProxyConfig { /// `RequestStart` and ends with exactly one `ResponseBodyComplete`, `StreamEnd`, `Error` or /// `Cancelled`. Each HTTP/2 stream is a flow of its own. A streaming flow (WebSocket, /// server-sent events, gRPC) has a `StreamStart` and then a `Message` for each message as it -/// passes. Traffic the engine tunnels without decrypting (pinned clients, non-TLS CONNECT -/// payloads, upgrades to protocols other than WebSocket) produces no events. +/// passes. An HTTPS connection the engine tunnels without decrypting, because the decrypt policy +/// leaves it out or its client rejected the engine's certificate, is a flow of its own, from +/// `TunnelStart` to `TunnelEnd`. Other traffic tunnelled blind (non-TLS CONNECT payloads, +/// upgrades to protocols other than WebSocket) produces no events. #[derive(Debug, Clone)] pub enum ProxyEvent { /// Request headers arrived from the client. @@ -181,6 +199,26 @@ pub enum ProxyEvent { elapsed_ms: u64, error: Option, }, + /// A CONNECT is tunnelled without being decrypted: its bytes pass untouched, and only their + /// count is recorded. + TunnelStart { + id: u64, + started_at: SystemTime, + /// `host[:port]`, the port left out when it is 443. + authority: String, + connection: u64, + client_addr: SocketAddr, + }, + /// The tunnel closed, with `error` when connecting upstream or relaying failed. + TunnelEnd { + id: u64, + /// Bytes from the client to the server. + sent: u64, + /// Bytes from the server to the client. + received: u64, + elapsed_ms: u64, + error: Option, + }, } /// A running engine. Dropping it stops the listener and joins its thread. @@ -212,6 +250,12 @@ impl ProxyHandle { self.engine.http2.store(enabled, Ordering::Relaxed); } + /// Decides which HTTPS connections are decrypted from the next CONNECT on. Connections + /// already open stay decrypted or tunnelled as they were. + pub fn set_decrypt_policy(&self, policy: DecryptPolicy) { + *self.engine.decrypt.write().unwrap() = Arc::new(policy); + } + /// Handshake failures counted toward passing `addr` (`host:port`) through, and whether it /// already is. For tests that must wait for the engine to record a failure. #[doc(hidden)] @@ -253,6 +297,8 @@ pub(crate) struct Engine { /// Offer HTTP/2 to clients in intercepted TLS. http2: AtomicBool, websocket_capture: bool, + decrypt: RwLock>, + app_of: Option, /// `Via` value marking the plain-HTTP requests the engine forwards, unique to this run, so one /// that comes back to the engine is refused instead of forwarded again. via: HeaderValue, @@ -271,6 +317,27 @@ impl Engine { self.http2.load(Ordering::Relaxed) } + fn decrypt_policy(&self) -> Arc { + self.decrypt.read().unwrap().clone() + } + + /// Whether the policy decrypts a CONNECT to `host:port` on `connection`, looking up its app + /// only when an entry names one. + async fn decrypts(&self, host: &str, port: u16, connection: u64) -> bool { + let policy = self.decrypt_policy(); + if !policy.decrypts_anything() { + return false; + } + let app = match &self.app_of { + Some(app_of) if policy.needs_app() => { + let app_of = app_of.clone(); + tokio::task::spawn_blocking(move || app_of(connection)).await.ok().flatten() + } + _ => None, + }; + policy.decrypts(&DecryptQuery { host: host.to_string(), port, app }) + } + fn is_passthrough(&self, addr: &str) -> bool { self.passthrough.lock().unwrap().contains(addr) } @@ -352,6 +419,8 @@ pub fn start_proxy(config: ProxyConfig) -> Result { handshake_failures: Mutex::new(HashMap::new()), http2: AtomicBool::new(config.http2), websocket_capture: config.websocket_capture, + decrypt: RwLock::new(Arc::new(config.decrypt)), + app_of: config.app_of, via: via_marker(), }); diff --git a/crates/yaak-proxy/src/request.rs b/crates/yaak-proxy/src/request.rs index 5bf8db271..f2883e2e1 100644 --- a/crates/yaak-proxy/src/request.rs +++ b/crates/yaak-proxy/src/request.rs @@ -31,7 +31,7 @@ use crate::connection::serve_tls; use crate::grpc::{self, GrpcParser}; use crate::messages::{MessageDirection, StreamKind}; use crate::sse::SseParser; -use crate::tunnel::{Rewind, splice, tunnel}; +use crate::tunnel::{Counted, Rewind, is_own_connection, splice, tunnel}; use crate::ws::{self, WsFlow}; use crate::{ClientConn, Engine, ProxyEvent}; @@ -376,6 +376,7 @@ struct Target { authority: String, /// `host:port` to dial. addr: String, + port: u16, } fn handle_connect( @@ -401,6 +402,7 @@ fn handle_connect( server_name: host.trim_start_matches('[').trim_end_matches(']').to_string(), authority: if port == 443 { host.clone() } else { format!("{host}:{port}") }, addr: format!("{host}:{port}"), + port, }; if let Err(e) = intercept(upgraded, target, client, engine).await { debug!("Tunnel to {host}:{port} ended: {}", error_chain(e.as_ref())); @@ -419,8 +421,13 @@ async fn intercept( where S: AsyncRead + AsyncWrite + Unpin + Send + 'static, { + if !engine.decrypts(&target.server_name, target.port, conn.connection).await { + debug!("Tunneling {} undecrypted, as the decrypt list leaves it out", target.addr); + return Ok(tunnel_flow(client, &target, conn, &engine).await?); + } + // The fallback for clients that rejected the engine's certificate. if engine.is_passthrough(&target.addr) { - return Ok(tunnel(client, &target.addr, &[]).await?); + return Ok(tunnel_flow(client, &target, conn, &engine).await?); } let mut buf = vec![0u8; 4096]; @@ -480,6 +487,49 @@ where .await } +/// Tunnels a CONNECT undecrypted, recorded as a flow of its own with the bytes each way. Not +/// one of the engine's own connections coming back, which [`tunnel`] refuses. +async fn tunnel_flow( + client: S, + target: &Target, + conn: ClientConn, + engine: &Engine, +) -> std::io::Result<()> +where + S: AsyncRead + AsyncWrite + Unpin, +{ + if is_own_connection(&target.addr).await { + return tunnel(client, &target.addr, &[]).await; + } + let id = engine.next_flow_id(); + let start = Instant::now(); + engine.emit(ProxyEvent::TunnelStart { + id, + started_at: SystemTime::now(), + authority: target.authority.clone(), + connection: conn.connection, + client_addr: conn.addr, + }); + let client = Counted::new(client); + let (sent, received) = client.counters(); + let result = tunnel(client, &target.addr, &[]).await; + let (sent, received) = (sent.load(Ordering::Relaxed), received.load(Ordering::Relaxed)); + // A tunnel that carried the server's bytes and then broke ended as connections do; one that + // never did failed to reach the server. + let error = match &result { + Err(e) if received == 0 => Some(error_chain(e)), + _ => None, + }; + engine.emit(ProxyEvent::TunnelEnd { + id, + sent, + received, + elapsed_ms: start.elapsed().as_millis() as u64, + error, + }); + result +} + fn record_handshake_failure( engine: &Engine, addr: &str, diff --git a/crates/yaak-proxy/src/tunnel.rs b/crates/yaak-proxy/src/tunnel.rs index c27577ac4..ad77a2ff0 100644 --- a/crates/yaak-proxy/src/tunnel.rs +++ b/crates/yaak-proxy/src/tunnel.rs @@ -1,5 +1,7 @@ use std::io; use std::pin::Pin; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; use std::task::{Context, Poll}; use bytes::Bytes; @@ -23,6 +25,72 @@ where relay(client, TcpStream::connect(target).await?, prefix).await } +/// Whether a tunnel to `target` (`host:port`) would be one of the engine's own connections that +/// came back through the system proxy, which [`tunnel`] refuses. Only port 80 on macOS can be. +pub(crate) async fn is_own_connection(target: &str) -> bool { + #[cfg(target_os = "macos")] + if let Some((host, "80")) = target.rsplit_once(':') { + let host = host.trim_start_matches('[').trim_end_matches(']'); + return crate::nw::is_own_fallback(host, 80).await; + } + let _ = target; + false +} + +/// A stream that counts the bytes read from it and written to it, for a tunnel's sizes. +pub(crate) struct Counted { + inner: S, + read: Arc, + written: Arc, +} + +impl Counted { + pub(crate) fn new(inner: S) -> Self { + Self { inner, read: Arc::default(), written: Arc::default() } + } + + /// The counts of bytes read and written, which keep counting after the stream moves on. + pub(crate) fn counters(&self) -> (Arc, Arc) { + (self.read.clone(), self.written.clone()) + } +} + +impl AsyncRead for Counted { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + let before = buf.filled().len(); + let polled = Pin::new(&mut self.inner).poll_read(cx, buf); + let n = buf.filled().len() - before; + self.read.fetch_add(n as u64, Ordering::Relaxed); + polled + } +} + +impl AsyncWrite for Counted { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + let polled = Pin::new(&mut self.inner).poll_write(cx, buf); + if let Poll::Ready(Ok(n)) = &polled { + self.written.fetch_add(*n as u64, Ordering::Relaxed); + } + polled + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_shutdown(cx) + } +} + /// [`tunnel`] through Network.framework, which can skip the system proxy. When it can't, it falls /// back to the system proxy with a `CONNECT` that arrives here, and tunneling that out the same /// way would repeat it. diff --git a/crates/yaak-proxy/tests/proxy.rs b/crates/yaak-proxy/tests/proxy.rs index 7e045d70e..f3b32bc1d 100644 --- a/crates/yaak-proxy/tests/proxy.rs +++ b/crates/yaak-proxy/tests/proxy.rs @@ -33,8 +33,8 @@ use tokio_tungstenite::tungstenite::protocol::Role; use tokio_tungstenite::tungstenite::protocol::frame::CloseFrame; use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode; use yaak_proxy::{ - CaStore, MessageDirection, MessageKind, ProxyConfig, ProxyEvent, ProxyHandle, StreamKind, - StreamMessage, start_proxy, + CaStore, DecryptApp, DecryptPolicy, MessageDirection, MessageKind, ProxyConfig, ProxyEvent, + ProxyHandle, StreamKind, StreamMessage, start_proxy, }; const BIG_BODY: usize = 64 * 1024; @@ -405,6 +405,7 @@ fn is_terminal(event: &ProxyEvent) -> bool { | ProxyEvent::Error { .. } | ProxyEvent::Cancelled { .. } | ProxyEvent::StreamEnd { .. } + | ProxyEvent::TunnelEnd { .. } ) } @@ -422,7 +423,9 @@ fn flow_id(event: &ProxyEvent) -> u64 { | ProxyEvent::Cancelled { id, .. } | ProxyEvent::StreamStart { id, .. } | ProxyEvent::Message { id, .. } - | ProxyEvent::StreamEnd { id, .. } => *id, + | ProxyEvent::StreamEnd { id, .. } + | ProxyEvent::TunnelStart { id, .. } + | ProxyEvent::TunnelEnd { id, .. } => *id, } } @@ -1252,7 +1255,171 @@ async fn pinned_client_falls_back_to_passthrough() { tokio::time::sleep(Duration::from_millis(50)).await; } assert_eq!(body.as_deref(), Some("hello")); - assert!(h.events.try_recv().is_err(), "passthrough traffic must not produce flows"); + // Passed through, it is a tunnelled flow, recorded without its contents. + drop(pinned); + let events = h.flow_events(); + let authority = find!(events, ProxyEvent::TunnelStart { authority, .. } => authority.clone()); + assert_eq!(authority, format!("localhost:{}", upstream.port())); + let (sent, received) = + find!(events, ProxyEvent::TunnelEnd { sent, received, .. } => (*sent, *received)); + assert!(sent > 0 && received > 0, "sent {sent}, received {received}"); + assert!(!events.iter().any(|e| matches!(e, ProxyEvent::RequestStart { .. }))); +} + +/// TLS straight to the upstream server through a CONNECT tunnel, trusting only the server's own +/// CA, as a client that never trusted the proxy's does. +async fn upstream_tls( + h: &Harness, + port: u16, + alpn: &[&[u8]], +) -> tokio_rustls::client::TlsStream { + let mut roots = RootCertStore::empty(); + roots.add(h.pki.ca_der.clone()).unwrap(); + let mut config = ClientConfig::builder_with_provider(ring()) + .with_safe_default_protocol_versions() + .unwrap() + .with_root_certificates(roots) + .with_no_client_auth(); + config.alpn_protocols = alpn.iter().map(|p| p.to_vec()).collect(); + let name = ServerName::try_from("localhost").unwrap(); + let tcp = h.connect_tunnel(port).await; + TlsConnector::from(Arc::new(config)).connect(name, tcp).await.expect("TLS to the server") +} + +/// `GET /hello` over HTTP/1.1 on `tls`, closing the connection after. +async fn get_hello_and_close(mut tls: tokio_rustls::client::TlsStream) -> String { + tls.write_all(b"GET /hello HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n") + .await + .unwrap(); + let mut response = Vec::new(); + let _ = tls.read_to_end(&mut response).await; + String::from_utf8_lossy(&response).into_owned() +} + +fn with_policy(entries: &[&str]) -> impl FnOnce(&mut ProxyConfig) { + let policy = DecryptPolicy::parse(entries).unwrap(); + move |config| config.decrypt = policy +} + +#[tokio::test(flavor = "multi_thread")] +async fn an_empty_decrypt_list_tunnels_https_untouched() { + within(async { + let h = Harness::start_with(1024, with_policy(&[])); + let upstream = spawn_upstream(Some(h.pki.server.clone())).await; + + // The server's own certificate arrives, so no CA of the proxy's is involved. + let tls = upstream_tls(&h, upstream.port(), &[b"http/1.1"]).await; + let response = get_hello_and_close(tls).await; + assert!(response.starts_with("HTTP/1.1 200"), "{response}"); + assert!(response.ends_with("hello"), "{response}"); + + let events = h.flow_events(); + let (connection, authority) = find!(events, ProxyEvent::TunnelStart { connection, authority, .. } => (*connection, authority.clone())); + assert_eq!(connection, 1); + assert_eq!(authority, format!("localhost:{}", upstream.port())); + let (sent, received, error) = find!(events, ProxyEvent::TunnelEnd { sent, received, error, .. } => (*sent, *received, error.clone())); + assert_eq!(error, None); + // The ClientHello and request one way, the certificate and response the other. + assert!(sent > 100, "sent {sent}"); + assert!(received as usize > response.len(), "received {received}"); + assert!(!events.iter().any(|e| matches!(e, ProxyEvent::RequestStart { .. }))); + }) + .await +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_tunnelled_host_passes_http2_through() { + within(async { + let h = Harness::start_with(1024, with_policy(&["host:example.com"])); + let upstream = spawn_upstream(Some(h.pki.server.clone())).await; + let tls = upstream_tls(&h, upstream.port(), &[b"h2", b"http/1.1"]).await; + assert_eq!(tls.get_ref().1.alpn_protocol(), Some(&b"h2"[..]), "the server's ALPN"); + let (mut sender, conn) = + http2::handshake(TokioExecutor::new(), TokioIo::new(tls)).await.unwrap(); + let conn = tokio::spawn(conn); + let req = Request::get(format!("https://localhost:{}/hello", upstream.port())) + .body(full(Bytes::new())) + .unwrap(); + let resp = sender.send_request(req).await.unwrap(); + assert_eq!(resp.version(), Version::HTTP_2); + assert_eq!(resp.into_body().collect().await.unwrap().to_bytes(), "hello"); + drop(sender); + let _ = conn.await; + + let events = h.flow_events(); + find!(events, ProxyEvent::TunnelStart { .. } => ()); + let error = find!(events, ProxyEvent::TunnelEnd { error, .. } => error.clone()); + assert_eq!(error, None); + }) + .await +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_listed_host_is_decrypted_and_others_tunnelled() { + within(async { + let h = Harness::start_with(1024, with_policy(&["host:localhost"])); + let upstream = spawn_upstream(Some(h.pki.server.clone())).await; + let url = format!("https://localhost:{}/hello", upstream.port()); + assert_eq!(h.client().get(&url).send().await.unwrap().text().await.unwrap(), "hello"); + let events = h.flow_events(); + let flow_url = find!(events, ProxyEvent::RequestStart { url, .. } => url.clone()); + assert_eq!(flow_url, url); + + // Only from the next connection on. + h.proxy.set_decrypt_policy(DecryptPolicy::parse(&["host:*.example.com"]).unwrap()); + let tls = upstream_tls(&h, upstream.port(), &[b"http/1.1"]).await; + assert!(get_hello_and_close(tls).await.ends_with("hello")); + let events = h.flow_events(); + find!(events, ProxyEvent::TunnelEnd { .. } => ()); + }) + .await +} + +#[tokio::test(flavor = "multi_thread")] +async fn an_app_entry_decrypts_that_app_alone() { + within(async { + let h = Harness::start_with(1024, |config| { + config.decrypt = DecryptPolicy::parse(&["app:com.example.curl"]).unwrap(); + // The first connection is from the listed app, the rest from another. + config.app_of = Some(Arc::new(|connection| { + let bundle_id = if connection == 1 { "com.example.curl" } else { "com.other" }; + Some(DecryptApp { name: "curl".into(), bundle_id: Some(bundle_id.into()) }) + })); + }); + let upstream = spawn_upstream(Some(h.pki.server.clone())).await; + let url = format!("https://localhost:{}/hello", upstream.port()); + let client = h.client(); + assert_eq!(client.get(&url).send().await.unwrap().text().await.unwrap(), "hello"); + let events = h.flow_events(); + find!(events, ProxyEvent::RequestStart { .. } => ()); + drop(client); + + let tls = upstream_tls(&h, upstream.port(), &[b"http/1.1"]).await; + assert!(get_hello_and_close(tls).await.ends_with("hello")); + let events = h.flow_events(); + let connection = find!(events, ProxyEvent::TunnelStart { connection, .. } => *connection); + assert_eq!(connection, 2); + }) + .await +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_tunnel_to_nowhere_is_a_failed_flow() { + within(async { + let h = Harness::start_with(1024, with_policy(&[])); + // A port nothing listens on. + let closed = TcpListener::bind("127.0.0.1:0").await.unwrap().local_addr().unwrap().port(); + let mut tcp = h.connect_tunnel(closed).await; + let _ = tcp.write_all(b"\x16\x03\x01").await; + let mut rest = Vec::new(); + let _ = tcp.read_to_end(&mut rest).await; + let events = h.flow_events(); + let (received, error) = + find!(events, ProxyEvent::TunnelEnd { received, error, .. } => (*received, error.clone())); + assert_eq!(received, 0); + assert!(error.is_some(), "{events:#?}"); + }) + .await } #[tokio::test(flavor = "multi_thread")] diff --git a/apps/yaak-client/components/core/Editor/filter/extension.ts b/packages/ui/src/components/editor/filter/extension.ts similarity index 66% rename from apps/yaak-client/components/core/Editor/filter/extension.ts rename to packages/ui/src/components/editor/filter/extension.ts index f98c608c3..869a50853 100644 --- a/apps/yaak-client/components/core/Editor/filter/extension.ts +++ b/packages/ui/src/components/editor/filter/extension.ts @@ -13,6 +13,18 @@ export interface FieldDef { export interface FilterOptions { fields: FieldDef[] | null; // e.g., ['method','status','path'] or [{name:'tag', values:()=>cachedTags}] + /** Written before a field name, `@` by default: `@method:`. Empty for `host:` */ + fieldPrefix?: string; + /** + * A prefix that negates a term, such as `!`: offered where a field name is, and a term may + * start with it before its field name + */ + negation?: { prefix: string; info?: string }; + /** + * Value suggestions replace only what follows the field's colon, so `host:api` offers hosts + * matching `api`. Off, they replace the whole term + */ + replaceValueOnly?: boolean; } const FIELD_IDENT = /[A-Za-z0-9_/]+$/; @@ -44,19 +56,31 @@ function wordBefore( return { from, to: pos, text: m[0] }; } -function fieldCompletionFrom(doc: string, pos: number): { from: number; includeAt: boolean } | null { +function fieldCompletionFrom( + doc: string, + pos: number, + fieldPrefix = "@", + negation?: string, +): { from: number; includeAt: boolean; negated: boolean } | null { const w = wordBefore(doc, pos, FIELD_IDENT); - const from = w?.from ?? pos; - const beforeToken = doc[from - 1]; - - if (from === 0 || (beforeToken != null && /\s/.test(beforeToken))) { - return { from, includeAt: true }; + let from = w?.from ?? pos; + const atTermStart = (at: number) => { + const before = doc[at - 1]; + return at === 0 || (before != null && /\s/.test(before)); + }; + let negated = false; + if (negation != null && doc.slice(0, from).endsWith(negation)) { + from -= negation.length; + negated = true; } - if (beforeToken === "@") { - const beforeAt = doc[from - 2]; - if (from === 1 || (beforeAt != null && /\s/.test(beforeAt))) { - return { from, includeAt: false }; + if (atTermStart(from)) { + return { from, includeAt: true, negated }; + } + + if (fieldPrefix !== "" && doc.slice(0, from).endsWith(fieldPrefix)) { + if (atTermStart(from - fieldPrefix.length)) { + return { from, includeAt: false, negated }; } } @@ -117,13 +141,20 @@ function contextInfo(stateDoc: string, pos: number) { } /** Build a completion list for field names */ -function fieldNameCompletions(fieldNames: string[], includeAt: boolean): Completion[] { +function fieldNameCompletions( + fieldNames: string[], + includeAt: boolean, + fieldPrefix = "@", + fieldMap: Record = {}, + negated = "", +): Completion[] { return fieldNames.map((name) => ({ - label: name, + label: `${negated}${name}`, type: "property", + info: fieldMap[name]?.info, apply: (view, _completion, from, to) => { // Leave cursor right after the field filter colon. - const insert = `${includeAt ? "@" : ""}${name}:`; + const insert = `${negated}${includeAt ? fieldPrefix : ""}${name}:`; view.dispatch({ changes: { from, to, insert }, selection: { anchor: from + insert.length }, @@ -146,8 +177,24 @@ function fieldValueCompletions( })); } +/** Offers the negation prefix itself, reopening the list for a field name after it */ +function negationCompletion(negation: { prefix: string; info?: string }): Completion { + return { + label: negation.prefix, + type: "keyword", + info: negation.info, + apply: (view, _completion, from, to) => { + view.dispatch({ + changes: { from, to, insert: negation.prefix }, + selection: { anchor: from + negation.prefix.length }, + }); + startCompletion(view); + }, + }; +} + /** The main completion source */ -function makeCompletionSource(opts: FilterOptions) { +export function makeCompletionSource(opts: FilterOptions) { const { fieldNames, fieldMap } = normalizeFields(opts.fields ?? []); return (ctx: CompletionContext): CompletionResult | null => { const { state, pos } = ctx; @@ -157,12 +204,12 @@ function makeCompletionSource(opts: FilterOptions) { return null; } - const { inValue, fieldName, emptyAfterColon } = contextInfo(doc, pos); + const { inValue, fieldName, lastColon, emptyAfterColon } = contextInfo(doc, pos); // In field value position if (inValue && fieldName) { const w = wordBefore(doc, pos, VALUE_IDENT); - const from = w?.from ?? pos; + const from = opts.replaceValueOnly ? lastColon + 1 : (w?.from ?? pos); const to = pos; const valDefs = fieldMap[fieldName]; const vals = fieldValueCompletions(valDefs); @@ -186,11 +233,18 @@ function makeCompletionSource(opts: FilterOptions) { } // Not in a value: suggest field names (and maybe boolean ops) - const completion = fieldCompletionFrom(doc, pos); + const completion = fieldCompletionFrom(doc, pos, opts.fieldPrefix, opts.negation?.prefix); if (completion == null) return null; - const { from, includeAt } = completion; + const { from, includeAt, negated } = completion; const to = pos; - const options: Completion[] = fieldNameCompletions(fieldNames, includeAt); + const options: Completion[] = fieldNameCompletions( + fieldNames, + includeAt, + opts.fieldPrefix, + fieldMap, + negated ? (opts.negation?.prefix ?? "") : "", + ); + if (opts.negation != null && !negated) options.push(negationCompletion(opts.negation)); return { from, to, options, filter: true }; }; @@ -204,6 +258,14 @@ const language = LRLanguage.define({ }, }); +/** + * Field and value suggestions, as the sidebar's filter offers them, for a field of another + * language: a plain input of `host:` and `app:` entries, say + */ +export function filterAutocomplete(options: FilterOptions) { + return autocompletion({ override: [makeCompletionSource(options)] }); +} + /** Public extension */ export function filter(options: FilterOptions) { const source = makeCompletionSource(options); diff --git a/apps/yaak-client/components/core/Editor/filter/filter.grammar b/packages/ui/src/components/editor/filter/filter.grammar similarity index 100% rename from apps/yaak-client/components/core/Editor/filter/filter.grammar rename to packages/ui/src/components/editor/filter/filter.grammar diff --git a/apps/yaak-client/components/core/Editor/filter/filter.test.ts b/packages/ui/src/components/editor/filter/filter.test.ts similarity index 100% rename from apps/yaak-client/components/core/Editor/filter/filter.test.ts rename to packages/ui/src/components/editor/filter/filter.test.ts diff --git a/apps/yaak-client/components/core/Editor/filter/filter.ts b/packages/ui/src/components/editor/filter/filter.ts similarity index 100% rename from apps/yaak-client/components/core/Editor/filter/filter.ts rename to packages/ui/src/components/editor/filter/filter.ts diff --git a/apps/yaak-client/components/core/Editor/filter/highlight.ts b/packages/ui/src/components/editor/filter/highlight.ts similarity index 100% rename from apps/yaak-client/components/core/Editor/filter/highlight.ts rename to packages/ui/src/components/editor/filter/highlight.ts diff --git a/packages/ui/src/editor.ts b/packages/ui/src/editor.ts index d49dc84a9..347d2139b 100644 --- a/packages/ui/src/editor.ts +++ b/packages/ui/src/editor.ts @@ -1,6 +1,12 @@ export type { CodeEditorProps } from "./components/editor/CodeEditor"; export { CodeEditor } from "./components/editor/CodeEditor"; export type { LanguageSupports } from "./components/editor/extensions"; +export type { FieldDef, FilterOptions } from "./components/editor/filter/extension"; +export { + filter, + filterAutocomplete, + makeCompletionSource as filterCompletionSource, +} from "./components/editor/filter/extension"; export { closeBracketsExtensions, getLanguageExtension,