diff --git a/Cargo.lock b/Cargo.lock index 90c0664ee..73bdd8026 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11965,6 +11965,7 @@ dependencies = [ "yaak-models", "yaak-proxy", "yaak-proxy-helper", + "yaak-query", "yaak-rpc", "yaak-templates", ] diff --git a/apps/yaak-client/components/Sidebar.tsx b/apps/yaak-client/components/Sidebar.tsx index cb3044736..0a94f5426 100644 --- a/apps/yaak-client/components/Sidebar.tsx +++ b/apps/yaak-client/components/Sidebar.tsx @@ -65,7 +65,7 @@ import type { ContextMenuProps, DropdownItem } from "./core/Dropdown"; import { ContextMenu, Dropdown } from "./core/Dropdown"; import type { Node as QueryNode, Schema } from "@yaakapp-internal/query"; import { formatField, parseQuery, QueryMatcher } from "@yaakapp-internal/query"; -import { queryFilter } from "./core/Editor/filter/extension"; +import { queryFilter } from "@yaakapp-internal/ui/editor"; import { HttpMethodTag } from "./core/HttpMethodTag"; import { Icon, @@ -850,18 +850,27 @@ function setSidebarFilterText(text: string) { jotaiStore.set(sidebarFilterAtom, { text, key: `${Math.random()}` }); } -/** The fields `getItemFields` fills in. Bare words match `getItemText` by word prefix */ -const sidebarFilterColumns = [ - "method", - "url", - "name", - "type", - "folder", - "grpc_service", - "grpc_method", +/** + * The fields `getItemFields` fills in, each with what it is and a term using it, as completion + * shows them. Bare words match `getItemText` by word prefix + */ +const sidebarFilterFields: [name: string, info: string, example: string][] = [ + ["method", "GET, POST, …", "@method:post"], + ["url", "Request URL", "@url:*/users*"], + ["name", "Request name", '@name:"Get user"'], + ["type", "http, grpc or ws", "@type:grpc"], + ["folder", "Parent folder", "@folder:auth"], + ["grpc_service", "gRPC service", "@grpc_service:*Greeter"], + ["grpc_method", "gRPC method", "@grpc_method:SayHello"], ]; +const sidebarFilterColumns = sidebarFilterFields.map(([name]) => name); const sidebarFilterSchema: Schema = { - fields: sidebarFilterColumns.map((name) => ({ name, kind: { type: "text" } })), + fields: sidebarFilterFields.map(([name, info, example]) => ({ + name, + kind: { type: "text" }, + info, + example, + })), freeText: "fuzzy", }; @@ -1005,9 +1014,12 @@ const sidebarTreeAtom = atom< matcher.free(); markHidden(root); + // Values offered after `@field:`. URLs and names are left out: nearly every one is unique, + // so a list of them is noise rather than a shortcut. const fieldValues: Record = {}; for (const [name, values] of Object.entries(allFields)) { - fieldValues[name] = Array.from(values).filter((v) => v.length < 20); + if (name === "url" || name === "name") continue; + fieldValues[name] = Array.from(values); } const suggestions = Array.from(suggestionFields) .sort((a, b) => { diff --git a/apps/yaak-client/components/core/Editor/filter/extension.ts b/apps/yaak-client/components/core/Editor/filter/extension.ts deleted file mode 100644 index 003729fe5..000000000 --- a/apps/yaak-client/components/core/Editor/filter/extension.ts +++ /dev/null @@ -1,88 +0,0 @@ -import type { Completion, CompletionContext, CompletionResult } from "@codemirror/autocomplete"; -import { autocompletion, startCompletion } from "@codemirror/autocomplete"; -import { highlightingFor } from "@codemirror/language"; -import type { EditorState, Extension } from "@codemirror/state"; -import { RangeSetBuilder } from "@codemirror/state"; -import type { DecorationSet, ViewUpdate } from "@codemirror/view"; -import { Decoration, EditorView, ViewPlugin } from "@codemirror/view"; -import type { Tag } from "@lezer/highlight"; -import { tags as t } from "@lezer/highlight"; -import type { Schema, TokenKind } from "@yaakapp-internal/query"; -import { queryCompletions, queryTokens } from "@yaakapp-internal/query"; - -export interface QueryFilterOptions { - schema: Schema; - /** The values seen for each field path, offered after `@field:` */ - values?: Record; -} - -const tokenTags: Record = { - field: t.attributeName, - value: t.attributeValue, - operator: t.operatorKeyword, - quote: t.string, - paren: t.paren, - negation: t.operator, - text: null, -}; - -function buildDecorations(state: EditorState): DecorationSet { - const builder = new RangeSetBuilder(); - for (const token of queryTokens(state.doc.toString())) { - const tag = tokenTags[token.kind]; - const className = tag == null ? null : highlightingFor(state, [tag]); - if (className == null || token.span.start === token.span.end) continue; - builder.add(token.span.start, token.span.end, Decoration.mark({ class: className })); - } - return builder.finish(); -} - -const highlight = ViewPlugin.fromClass( - class { - decorations: DecorationSet; - - constructor(view: EditorView) { - this.decorations = buildDecorations(view.state); - } - - update(update: ViewUpdate) { - if (update.docChanged || update.viewportChanged) { - this.decorations = buildDecorations(update.state); - } - } - }, - { decorations: (v) => v.decorations }, -); - -function completionSource({ schema, values }: QueryFilterOptions) { - return (ctx: CompletionContext): CompletionResult | null => { - const result = queryCompletions(ctx.state.doc.toString(), ctx.pos, schema, values); - if (result == null) return null; - - const { start, end } = result.span; - // Field names pop up for something typed; on an empty spot, only when asked for - const onlyFields = result.items.every((item) => item.kind === "field"); - if (onlyFields && start === end && !ctx.explicit) return null; - - const options: Completion[] = result.items.map((item) => ({ - label: item.label, - type: item.kind === "field" ? "property" : "constant", - info: item.info ?? undefined, - apply: (view, _completion, from, to) => { - view.dispatch({ - changes: { from, to, insert: item.insert }, - selection: { anchor: from + item.insert.length }, - }); - if (item.reopen) startCompletion(view); - }, - })); - - // The engine has already filtered and ranked the options - return { from: start, to: end, options, filter: false }; - }; -} - -/** Highlighting and completion for a search box written in Yaak's query language */ -export function queryFilter(options: QueryFilterOptions): Extension { - return [highlight, autocompletion({ override: [completionSource(options)] })]; -} diff --git a/apps/yaak-proxy/components/FlowListEmpty.tsx b/apps/yaak-proxy/components/FlowListEmpty.tsx index 5c3b28a8e..6c51e596a 100644 --- a/apps/yaak-proxy/components/FlowListEmpty.tsx +++ b/apps/yaak-proxy/components/FlowListEmpty.tsx @@ -3,7 +3,7 @@ import classNames from "classnames"; import { useAtomValue, useSetAtom } from "jotai"; import { useEffect, useState } from "react"; import { rpc } from "../lib/rpc"; -import { parseSearch, widerQueries } from "../lib/search"; +import { activeQuery, widerQueries } from "../lib/query"; import { hideSystemAtom, pausedAtAtom, @@ -77,7 +77,7 @@ function NoResults({ text }: { text: string }) { const candidates = widerQueries(text); void Promise.all( candidates.map(async (candidate) => { - const query = parseSearch(candidate, hideSystem); + const query = activeQuery(candidate, hideSystem); const res = await rpc("list_flows", { ...query, limit: 0 }); return { text: candidate, count: query == null ? res.total : (res.matching ?? res.total) }; }), diff --git a/apps/yaak-proxy/components/ProxyLayout.tsx b/apps/yaak-proxy/components/ProxyLayout.tsx index 18601c197..95a4b181b 100644 --- a/apps/yaak-proxy/components/ProxyLayout.tsx +++ b/apps/yaak-proxy/components/ProxyLayout.tsx @@ -22,6 +22,7 @@ import { LockedState } from "./LockedState"; import { ProxyStatus, useProxySetup } from "./ProxyStatus"; import { HelperDialog } from "./HelperDialog"; import { SaveDialog } from "./SaveDialog"; +import { SavedSearchDialog } from "./SavedSearchDialog"; import { SearchBar } from "./SearchBar"; import { SettingsWindow, useOpenSettings } from "./Settings"; import { Sidebar } from "./Sidebar"; @@ -158,6 +159,7 @@ export function ProxyLayout() { + diff --git a/apps/yaak-proxy/components/SavedSearchDialog.tsx b/apps/yaak-proxy/components/SavedSearchDialog.tsx new file mode 100644 index 000000000..6fda7826f --- /dev/null +++ b/apps/yaak-proxy/components/SavedSearchDialog.tsx @@ -0,0 +1,82 @@ +import { Dialog, DialogFooter, PlainInput } from "@yaakapp-internal/ui"; +import { atom, useAtom } from "jotai"; +import { useState } from "react"; +import { useRpcMutation } from "../hooks/useRpcMutation"; +import { useShowToast } from "../lib/toast"; + +/** Saving the search box's text under a name, or renaming a saved search */ +export type SavedSearchDialog = + | { mode: "save"; query: string } + | { mode: "rename"; id: string; name: string }; + +export const savedSearchDialogAtom = atom(null); + +const FORM_ID = "saved-search"; + +/** Names a saved search, as `savedSearchDialogAtom` asks */ +export function SavedSearchDialog() { + const [dialog, setDialog] = useAtom(savedSearchDialogAtom); + if (dialog == null) return null; + // Keyed, so each opening starts from its own name. + return ( + setDialog(null)} + /> + ); +} + +function NameDialog({ dialog, onClose }: { dialog: SavedSearchDialog; onClose: () => void }) { + const [name, setName] = useState(dialog.mode === "rename" ? dialog.name : ""); + const showToast = useShowToast(); + const onError = (err: Error) => + showToast({ id: "saved-search", message: err.message, color: "danger" }); + const create = useRpcMutation("create_saved_query", { onSuccess: onClose, onError }); + const rename = useRpcMutation("rename_saved_query", { onSuccess: onClose, onError }); + const busy = create.isPending || rename.isPending; + + const submit = () => { + if (name.trim() === "" || busy) return; + if (dialog.mode === "save") create.mutate({ name, query: dialog.query }); + else rename.mutate({ id: dialog.id, name }); + }; + + return ( + +
{ + e.preventDefault(); + submit(); + }} + > + + + +
+ ); +} diff --git a/apps/yaak-proxy/components/SearchBar.tsx b/apps/yaak-proxy/components/SearchBar.tsx index 44d1cc045..cdcbf7271 100644 --- a/apps/yaak-proxy/components/SearchBar.tsx +++ b/apps/yaak-proxy/components/SearchBar.tsx @@ -1,34 +1,53 @@ -import { Icon, IconButton, PlainInput } from "@yaakapp-internal/ui"; +import { startCompletion } from "@codemirror/autocomplete"; +import type { Extension } from "@codemirror/state"; +import { Compartment } from "@codemirror/state"; +import type { EditorView } from "@codemirror/view"; +import { + Dropdown, + Icon, + IconButton, + Input, + type InputHandle, + LazyCodeEditor, +} from "@yaakapp-internal/ui"; +import { queryFilter } from "@yaakapp-internal/ui/editor"; import { useAtom, useAtomValue, useSetAtom, useStore } from "jotai"; -import { memo, type RefObject, useEffect, useRef } from "react"; +import { memo, useEffect, useMemo, useRef } from "react"; +import { useRpcQuery } from "../hooks/useRpcQuery"; +import { loadSeenValues, savedCompletions, searchProblems } from "../lib/searchBox"; import { allTrafficTotalAtom, lockedAtom, + savedQueriesAtom, searchTermsAtom, searchTextAtom, searchTextVersionAtom, setSearchTextAtom, } from "../lib/store"; +import { savedSearchDialogAtom } from "./SavedSearchDialog"; /** How long typing settles before the search runs */ const SEARCH_DEBOUNCE_MS = 200; /** - * The header's search over every flow in the database, by URL, method, status and content type, - * and the `@app`, `@host` and `@path` terms the sidebar writes. Its placeholder counts the flows. - * ⌘F focuses it and Escape clears it. + * The header's search over every flow in the database, in the query language: bare words match + * the URL, method, status and content type, and `@field:value` terms the rest, highlighted and + * completed as Yaak's sidebar filter is, with the values the proxy has seen. The backend runs + * it and marks what it cannot. Its placeholder counts the flows. ⌘F focuses it and Escape + * clears it. */ export const SearchBar = memo(function SearchBar() { const [text, setText] = useAtom(searchTextAtom); const setTerms = useSetAtom(searchTermsAtom); const setSearchText = useSetAtom(setSearchTextAtom); const version = useAtomValue(searchTextVersionAtom); + const total = useAtomValue(allTrafficTotalAtom); + const store = useStore(); + const setSavedDialog = useSetAtom(savedSearchDialogAtom); // Nothing to search while captured traffic is locked. const locked = useAtomValue(lockedAtom) != null; - // Read, not subscribed: `PlaceholderCount` keeps it current without re-rendering the field. - const total = useStore().get(allTrafficTotalAtom); - // PlainInput hands back its as the ref - const inputRef = useRef(null); + const schema = useRpcQuery("query_schema", {}, { staleTime: Infinity }).data?.schema; + const handleRef = useRef(null); useEffect(() => { const timer = setTimeout(() => setTerms(text), SEARCH_DEBOUNCE_MS); @@ -39,43 +58,69 @@ export const SearchBar = memo(function SearchBar() { const onKeyDown = (e: KeyboardEvent) => { if ((e.metaKey || e.ctrlKey) && !e.shiftKey && !e.altKey && e.key.toLowerCase() === "f") { e.preventDefault(); - inputRef.current?.focus(); - inputRef.current?.select(); + handleRef.current?.selectAll(); } }; window.addEventListener("keydown", onKeyDown); return () => window.removeEventListener("keydown", onKeyDown); }, []); + // The language goes in once the schema is read: into the editor there is, through a + // compartment, and into each one made after, as taking text from the sidebar makes one. + const compartment = useRef(new Compartment()); + const filter = useMemo( + () => + schema == null + ? [] + : queryFilter({ + schema, + loadValues: loadSeenValues, + // Read as completion opens, so saved searches are current without remaking it. + moreCompletions: (typed) => savedCompletions(store.get(savedQueriesAtom), typed), + problems: searchProblems, + }), + [schema, store], + ); + const extensions = useMemo(() => compartment.current.of(filter), [filter]); + useEffect(() => { + handleRef.current?.dispatch({ effects: compartment.current.reconfigure(filter) }); + }, [filter]); + // The field is replaced to take a value from outside, so focus moves to the new one. const clear = () => { setSearchText(""); - requestAnimationFrame(() => inputRef.current?.focus()); + requestAnimationFrame(() => handleRef.current?.focus()); }; return ( - ( + + )} + setRef={(h) => { + handleRef.current = h; + }} label="Search flows" hideLabel size="sm" className="font-mono text-xs" + stateKey={null} + language={null} + wrapLines={false} placeholder={locked ? "Traffic is locked" : placeholderFor(total)} disabled={locked} defaultValue={text} forceUpdateKey={String(version)} onChange={setText} - onKeyDownCapture={(e) => { - if (e.key === "Escape" && text !== "") { + extensions={extensions} + onKeyDown={(e) => { + if (e.key === "Escape" && handleRef.current?.value() !== "") { e.stopPropagation(); clear(); } }} leftSlot={ - <> - - - + } rightSlot={ text === "" ? ( @@ -83,14 +128,26 @@ export const SearchBar = memo(function SearchBar() { ⌘F ) : ( - +
+ , + onSelect: () => setSavedDialog({ mode: "save", query: text.trim() }), + }, + ]} + > + + + +
) } /> @@ -98,15 +155,14 @@ export const SearchBar = memo(function SearchBar() { }); /** - * Keeps the placeholder's count current as flows arrive, alone, so new traffic re-renders this - * and not the whole field + * Ctrl+Space lists what can be typed, fields and saved searches on an empty spot, as it does in + * Yaak's editors, whose shared editor leaves the key to each app */ -function PlaceholderCount({ inputRef }: { inputRef: RefObject }) { - const total = useAtomValue(allTrafficTotalAtom); - useEffect(() => { - if (inputRef.current != null) inputRef.current.placeholder = placeholderFor(total); - }, [inputRef, total]); - return null; +function startCompletionOnCtrlSpace(e: KeyboardEvent, view: EditorView): boolean { + if (!(e.ctrlKey && e.key === " " && !e.metaKey && !e.altKey && !e.shiftKey)) return false; + e.preventDefault(); + startCompletion(view); + return true; } /** "Search 4,102 flows", the title and count that sat in the header's middle */ diff --git a/apps/yaak-proxy/components/Sidebar.tsx b/apps/yaak-proxy/components/Sidebar.tsx index 5bea89e8a..23f3be64e 100644 --- a/apps/yaak-proxy/components/Sidebar.tsx +++ b/apps/yaak-proxy/components/Sidebar.tsx @@ -7,10 +7,11 @@ import { type TreeProps, } from "@yaakapp-internal/ui"; import classNames from "classnames"; -import { atom, useAtom, useAtomValue } from "jotai"; +import { atom, useAtom, useAtomValue, useSetAtom } from "jotai"; import { atomWithStorage } from "jotai/utils"; import { useCallback, useEffect } from "react"; import { useDecryptList, useEditDecryptList } from "../hooks/useDecryptList"; +import { useRpcMutation } from "../hooks/useRpcMutation"; import { SIDEBAR_TREE_ID } from "../lib/atoms"; import { blockerOf, decryptMenuAction, isDecrypted, listEdit, setDecrypted } from "../lib/decrypt"; import { applyCollapsed, type CollapsedMap, collapsedMap, folderIds } from "../lib/sidebarCollapse"; @@ -18,6 +19,7 @@ import { type SidebarItem, SYSTEM_ID } from "../lib/sidebarTree"; import { hideSystemAtom, noteSidebarInput, sidebarTreeAtom } from "../lib/store"; import { useShowToast } from "../lib/toast"; import { AppIcon } from "./AppIcon"; +import { savedSearchDialogAtom } from "./SavedSearchDialog"; /** The apps, sites, hosts and paths the user expanded, by node id, kept across launches */ const expandedAtom = atomWithStorage>("sidebar_expanded", {}, undefined, { @@ -63,6 +65,7 @@ function ItemInner({ item }: { item: SidebarItem }) {
{item.kind === "all" && } {item.kind === "tunnelled" && } + {item.kind === "saved" && } {item.kind === "folder" && } {item.appPath != null && } {item.label} @@ -119,17 +122,46 @@ function HideSystemToggle() { export function Sidebar() { const tree = useAtomValue(sidebarTreeAtom); const getItemKey = useCallback( - (item: SidebarItem) => `${item.id}:${item.count}:${item.countLabel ?? ""}`, + // The label too, as a saved search's changes when it is renamed + (item: SidebarItem) => `${item.id}:${item.count}:${item.countLabel ?? ""}:${item.label}`, [], ); const decryptList = useDecryptList(); const setDecryptList = useEditDecryptList(); const { mutate: saveDecryptList } = setDecryptList; const showToast = useShowToast(); + const setSavedDialog = useSetAtom(savedSearchDialogAtom); + const { mutate: deleteSaved } = useRpcMutation("delete_saved_query", { + onError: (err) => showToast({ id: "saved-search", message: err.message, color: "danger" }), + }); // Decrypt or Stop Decrypting the selected hosts, sites and apps, from new connections on. + // A saved search is renamed or deleted. const getContextMenu = useCallback( (items: SidebarItem[]): DropdownItem[] => { + const saved = items.flatMap((i) => (i.savedId == null ? [] : [{ ...i, id: i.savedId }])); + if (saved.length > 0) { + const [first] = saved; + return [ + { + label: "Rename", + leftSlot: , + hidden: saved.length > 1 || first == null, + onSelect: () => { + if (first != null) + setSavedDialog({ mode: "rename", id: first.id, name: first.label }); + }, + }, + { + label: "Delete", + color: "danger", + leftSlot: , + onSelect: () => { + for (const s of saved) deleteSaved({ id: s.id }); + }, + }, + ]; + } const subjects = items.flatMap((i) => (i.decrypt == null ? [] : [i.decrypt])); const action = decryptMenuAction(decryptList, subjects); if (action == null) return []; @@ -155,7 +187,7 @@ export function Sidebar() { }, ]; }, - [decryptList, saveDecryptList, showToast], + [decryptList, saveDecryptList, showToast, setSavedDialog, deleteSaved], ); const renderContextMenu = useCallback["renderContextMenu"]>>( ({ items, position, onClose }) => diff --git a/apps/yaak-proxy/harness/fixtures.ts b/apps/yaak-proxy/harness/fixtures.ts index 2f052dd74..cc90cf443 100644 --- a/apps/yaak-proxy/harness/fixtures.ts +++ b/apps/yaak-proxy/harness/fixtures.ts @@ -51,6 +51,7 @@ const base: Flow = { grpcStatus: null, grpcMessage: null, tunnelled: false, + agentRead: false, clientAddr: "127.0.0.1:54321", elapsedConnectMs: 18, elapsedTlsMs: 41, diff --git a/apps/yaak-proxy/harness/streamFixtures.ts b/apps/yaak-proxy/harness/streamFixtures.ts index 9fcbe3c9a..47fa878b7 100644 --- a/apps/yaak-proxy/harness/streamFixtures.ts +++ b/apps/yaak-proxy/harness/streamFixtures.ts @@ -50,6 +50,7 @@ const flowBase: Flow = { grpcStatus: null, grpcMessage: null, tunnelled: false, + agentRead: false, clientAddr: null, elapsedConnectMs: null, elapsedTlsMs: null, diff --git a/apps/yaak-proxy/index.html b/apps/yaak-proxy/index.html index 60d859012..cc49b8e4a 100644 --- a/apps/yaak-proxy/index.html +++ b/apps/yaak-proxy/index.html @@ -24,5 +24,8 @@ +
+
+
diff --git a/apps/yaak-proxy/lib/flowViews.test.ts b/apps/yaak-proxy/lib/flowViews.test.ts index 5c7daff3c..6ef8d323c 100644 --- a/apps/yaak-proxy/lib/flowViews.test.ts +++ b/apps/yaak-proxy/lib/flowViews.test.ts @@ -59,6 +59,7 @@ const flow: Flow = { grpcStatus: null, grpcMessage: null, tunnelled: false, + agentRead: false, clientAddr: null, elapsedConnectMs: null, elapsedTlsMs: null, diff --git a/apps/yaak-proxy/lib/query.test.ts b/apps/yaak-proxy/lib/query.test.ts new file mode 100644 index 000000000..dea324fbf --- /dev/null +++ b/apps/yaak-proxy/lib/query.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, test } from "vite-plus/test"; +import { + activeQuery, + anyCountsBackFromNow, + asOneTerm, + countsBackFromNow, + fieldTerm, + pathTerm, + termKeys, + widerQueries, +} from "./query"; +import { savedCompletions } from "./searchBox"; + +describe("terms", () => { + test("quote values that need it", () => { + expect(fieldTerm("app", "Google Chrome")).toBe('@app:"Google Chrome"'); + expect(fieldTerm("host", "api.x.com")).toBe("@host:api.x.com"); + expect(fieldTerm("is", "system", true)).toBe("-@is:system"); + }); + + test("select a path and what is under it", () => { + expect(pathTerm("/v1/charges")).toBe("@path:/v1/charges/*"); + expect(pathTerm("/a b")).toBe('@path:"/a b"'); + }); + + test("are read back as written, any case, with their exclusion", () => { + expect(termKeys('@app:"Google Chrome" -@is:System (a OR -b)')).toEqual([ + '@app:"google chrome"', + "-@is:system", + "a", + "-b", + ]); + expect(termKeys("")).toEqual([]); + }); +}); + +describe("activeQuery", () => { + test("is null for all traffic, and carries hiding System", () => { + expect(activeQuery(" ", false)).toBeNull(); + expect(activeQuery("", true)).toEqual({ query: "", hideSystem: true }); + expect(activeQuery(" stripe ", false)).toEqual({ query: "stripe", hideSystem: false }); + }); +}); + +describe("widerQueries", () => { + test("leaves out each term in turn", () => { + expect(widerQueries('@app:Slack @status:5xx "card declined"')).toEqual([ + '@status:5xx "card declined"', + '@app:Slack "card declined"', + "@app:Slack @status:5xx", + ]); + expect(widerQueries("@app:Slack")).toEqual([]); + }); + + test("widens one OR group at a time, keeping the groups", () => { + const text = "@app:Slack @host:absent.com OR @app:Chrome @host:absent.com"; + expect(widerQueries(text)).toEqual([ + "@host:absent.com OR @app:Chrome @host:absent.com", + "@app:Slack OR @app:Chrome @host:absent.com", + "@app:Slack @host:absent.com OR @host:absent.com", + "@app:Slack @host:absent.com OR @app:Chrome", + ]); + // A group of one term is not emptied, which would take in every flow. + expect(widerQueries("@app:Slack OR @app:Chrome @host:b.com")).toEqual([ + "@app:Slack OR @host:b.com", + "@app:Slack OR @app:Chrome", + ]); + }); + + test("keeps groups and exclusions whole", () => { + expect(widerQueries("(@status:4xx OR @status:5xx) -(a OR b) c")).toEqual([ + "-(a OR b) c", + "(@status:4xx OR @status:5xx) c", + "(@status:4xx OR @status:5xx) -(a OR b)", + ]); + }); +}); + +describe("countsBackFromNow", () => { + test("is a @time comparison or range, which ages", () => { + expect(countsBackFromNow("@time:<5m")).toBe(true); + expect(countsBackFromNow("x -(@time:[1h TO 2h] OR y)")).toBe(true); + expect(countsBackFromNow("@time:2026-11-07")).toBe(false); + expect(countsBackFromNow("@status:<500 time")).toBe(false); + }); + + test("of saved searches, tells when their counts change with time alone", () => { + expect(anyCountsBackFromNow(["@status:4xx", "-@time:>1h stripe"])).toBe(true); + expect(anyCountsBackFromNow(["@status:4xx", "@time:2026-11-07"])).toBe(false); + expect(anyCountsBackFromNow([])).toBe(false); + }); +}); + +describe("saved searches as completions", () => { + test("keep an OR at their top in one group, however it is spaced", () => { + expect(asOneTerm("@status:4xx OR @status:5xx")).toBe("(@status:4xx OR @status:5xx)"); + expect(asOneTerm("(@status:4xx)OR(@status:5xx)")).toBe("((@status:4xx)OR(@status:5xx))"); + expect(asOneTerm("(a OR b) c")).toBe("(a OR b) c"); + const saved = [ + { id: "1", name: "Errors", query: "(@status:4xx)OR(@status:5xx)", sortOrder: 0 }, + { id: "2", name: "Stripe", query: "@host:*.stripe.com", sortOrder: 1 }, + ]; + expect(savedCompletions(saved, "err").map((c) => c.apply)).toEqual([ + "((@status:4xx)OR(@status:5xx))", + ]); + }); +}); diff --git a/apps/yaak-proxy/lib/query.ts b/apps/yaak-proxy/lib/query.ts new file mode 100644 index 000000000..c0eb4021b --- /dev/null +++ b/apps/yaak-proxy/lib/query.ts @@ -0,0 +1,147 @@ +import type { Node } from "@yaakapp-internal/query"; +import { formatField, parseQuery } from "@yaakapp-internal/query"; + +/** + * A search over every flow in the database, as `list_flows` takes it: the search box's text, + * which the backend parses and runs, and whether System's flows are hidden. + */ +export interface FlowQuery { + query: string; + hideSystem: boolean; +} + +/** The search the list shows, or null for all traffic */ +export function activeQuery(text: string, hideSystem: boolean): FlowQuery | null { + if (text.trim() === "" && !hideSystem) return null; + return { query: text.trim(), hideSystem }; +} + +/** `@field:value`, or `-@field:value`, the value quoted when it needs to be */ +export function fieldTerm(field: string, value: string, negated = false): string { + return formatField(field, value, negated); +} + +/** A site, its domain and every host below it: `@host:*.github.com` */ +export function siteTerm(site: string): string { + const term = formatField("host", site); + return term.endsWith('"') ? term : `@host:*.${site}`; +} + +/** + * A path and everything under it, `@path:/v1/charges/*`, as a sidebar row selects it. A path + * that has to be quoted cannot carry the wildcard, so it matches as written. + */ +export function pathTerm(path: string): string { + const term = formatField("path", path); + if (term.endsWith('"')) return term; + return `${term.replace(/\/+$/, "")}/*`; +} + +/** `node` written back as query text, from its source, groups in parentheses where they need */ +export function nodeText(node: Node, source: string, nested = false): string { + switch (node.type) { + case "And": { + const text = node.children.map((c) => nodeText(c, source, true)).join(" "); + return nested ? `(${text})` : text; + } + case "Or": { + const text = node.children.map((c) => nodeText(c, source, true)).join(" OR "); + return nested ? `(${text})` : text; + } + case "Not": + return `-${nodeText(node.node, source, true)}`; + default: + return source.slice(node.span.start, node.span.end); + } +} + +/** The terms of a query, each as written and with a `-` when it is excluded, lowercased */ +export function termKeys(text: string): string[] { + const keys: string[] = []; + const walk = (node: Node, negated: boolean) => { + switch (node.type) { + case "And": + case "Or": + for (const c of node.children) walk(c, negated); + break; + case "Not": + walk(node.node, !negated); + break; + default: + keys.push( + `${negated ? "-" : ""}${text.slice(node.span.start, node.span.end).toLowerCase()}`, + ); + } + }; + const root = parseQuery(text); + if (root != null) walk(root, false); + return keys; +} + +/** + * Wider searches for one that matched nothing: the text with each of its terms left out in + * turn, without repeats. With groups `OR` separates, a term is left out of one group at a time + * and the groups kept, since each group alone is narrower still; a group left with nothing would + * take in every flow, so it is not offered. + */ +export function widerQueries(text: string): string[] { + const root = parseQuery(text); + if (root == null) return []; + const clauses = root.type === "Or" ? root.children : [root]; + const termsOf = (clause: Node) => (clause.type === "And" ? clause.children : [clause]); + const write = (terms: Node[]) => terms.map((t) => nodeText(t, text, true)).join(" "); + const out = new Set(); + clauses.forEach((clause, i) => { + const terms = termsOf(clause); + if (terms.length <= 1) return; + terms.forEach((_, j) => { + const wider = clauses.map((c, k) => + k === i ? write(terms.filter((_, l) => l !== j)) : write(termsOf(c)), + ); + out.add(wider.join(" OR ")); + }); + }); + return [...out]; +} + +/** + * The search has a `@time` term that counts back from now, `@time:<5m`, so which flows it covers + * changes as time passes, not only as flows are written + */ +export function countsBackFromNow(text: string): boolean { + let found = false; + const walk = (node: Node) => { + switch (node.type) { + case "And": + case "Or": + for (const c of node.children) walk(c); + break; + case "Not": + walk(node.node); + break; + case "Field": + if ( + node.path.length === 1 && + node.path[0]?.toLowerCase() === "time" && + (node.value.type === "compare" || node.value.type === "range") + ) { + found = true; + } + break; + default: + } + }; + const root = parseQuery(text); + if (root != null) walk(root); + return found; +} + +/** Any of the searches counts back from now, so its count changes as time passes */ +export function anyCountsBackFromNow(queries: string[]): boolean { + return queries.some(countsBackFromNow); +} + +/** `text` to sit beside other terms as one: in parentheses when it is an `OR` at its top */ +export function asOneTerm(text: string): string { + return parseQuery(text)?.type === "Or" ? `(${text})` : text; +} diff --git a/apps/yaak-proxy/lib/search.test.ts b/apps/yaak-proxy/lib/search.test.ts index 55c838f53..66f930494 100644 --- a/apps/yaak-proxy/lib/search.test.ts +++ b/apps/yaak-proxy/lib/search.test.ts @@ -1,144 +1,6 @@ import type { Flow } from "@yaakapp-internal/proxy-lib"; import { describe, expect, test } from "vite-plus/test"; -import { - appKeyOf, - formatFieldFilter, - isSystemFlow, - matchesQuery, - parseSearch, - sidebarPath, - siteOf, - tokenize, - widerQueries, -} from "./search"; - -function flow(url: string, extra: Partial = {}): Flow { - return { - url, - method: "GET", - status: 200, - contentType: null, - clientBundleId: null, - clientPath: null, - ...extra, - } as Flow; -} - -// The same flows and terms as `search_matches_url_method_status_and_content_type` in -// `flows.rs`, so the live matcher and the database agree. -const stripe = flow("https://api.stripe.com/v1/charges?customer=cus_9", { - method: "POST", - contentType: "application/json; charset=utf-8", -}); -const github = flow("https://api.github.com/repos/yaak", { - status: 404, - contentType: "application/json; charset=utf-8", -}); -const logo = flow("https://images.ctfassets.net/logo.png", { contentType: "image/png" }); -const health = flow("http://localhost:3000/health", { method: "HEAD", status: 503 }); -const all = [stripe, github, logo, health]; - -const search = (q: string) => all.filter((f) => matchesQuery(f, { q })); - -describe("matchesQuery", () => { - test("matches the host, the path and the query, any case", () => { - expect(search("STRIPE")).toEqual([stripe]); - expect(search("/repos/")).toEqual([github]); - expect(search("cus_9")).toEqual([stripe]); - }); - - test("matches the whole method", () => { - expect(search("head")).toEqual([health]); - expect(search("ead")).toEqual([]); - }); - - test("matches a status, its start, or its class", () => { - expect(search("404")).toEqual([github]); - expect(search("50")).toEqual([health]); - expect(search("2xx")).toEqual([stripe, logo]); - }); - - test("matches the content type", () => { - expect(search("png")).toEqual([logo]); - expect(search("json")).toEqual([stripe, github]); - }); - - test("needs every term to match", () => { - expect(search("json get")).toEqual([github]); - expect(search(" json 2xx ")).toEqual([stripe]); - expect(search("json 5xx")).toEqual([]); - expect(search("")).toEqual(all); - }); - - test("keeps flows under a host or path, as the sidebar places them", () => { - const under = (node: string, url: string) => matchesQuery(flow(url), { under: [node] }); - expect(under("api.stripe.com", "https://api.stripe.com/v1/charges")).toBe(true); - expect(under("api.stripe.com", "http://api.stripe.com")).toBe(true); - // Another port is a host node of its own, as in the sidebar. - expect(under("api.stripe.com", "https://api.stripe.com:8443/v1")).toBe(false); - // Empty segments and default ports are left out, as the sidebar leaves them. - expect(under("example.com/v1/users", "https://example.com/v1//users")).toBe(true); - expect(under("x.com/a", "http://x.com:80/a")).toBe(true); - expect(under("api.stripe.com", "https://api.stripe.community/")).toBe(false); - expect(under("api.stripe.com/v1/charges", "https://api.stripe.com/v1/charges/ch_1?x")).toBe( - true, - ); - expect(under("api.stripe.com/v1/charges", "https://api.stripe.com/v1/chargesx")).toBe(false); - expect(under("api.stripe.com:8443", "https://api.stripe.com/v1")).toBe(false); - // The sidebar lowercases hosts; paths keep their case. - expect(under("example.com/A", "http://EXAMPLE.com/A/b")).toBe(true); - expect(under("example.com/a", "http://example.com/A")).toBe(false); - }); - - test("keeps flows from an app, by bundle id or path", () => { - const chrome = flow("https://example.com/", { - clientBundleId: "com.google.Chrome", - clientPath: "/Applications/Google Chrome.app", - }); - const curl = flow("https://example.com/", { clientPath: "/usr/bin/curl" }); - const unknown = flow("https://example.com/"); - const from = (app: string[]) => [chrome, curl, unknown].filter((f) => matchesQuery(f, { app })); - expect(from(["com.google.Chrome"])).toEqual([chrome]); - expect(from(["/usr/bin/curl"])).toEqual([curl]); - expect(from(["/Applications/Google Chrome.app"])).toEqual([]); - }); - - test("keeps flows from an app by name, or a program by the folder it ran in", () => { - // The flows `filters_by_app_name_folder_site_and_system` in `flows.rs` checks. - const slack = flow("https://edgeapi.slack.com/cache", { - clientName: "Slack", - clientBundleId: "com.tinyspeck.slackmacgap", - clientPath: "/Applications/Slack.app", - }); - const node = flow("https://api.github.com:443/repos", { - clientName: "node", - clientPath: "/opt/homebrew/bin/node", - clientCwd: "/Users/me/src/fieldwork", - }); - const daemon = flow("https://github.com/login", { - clientName: "nsurlsessiond", - clientPath: "/usr/libexec/nsurlsessiond", - clientCwd: "/", - }); - const none = flow("http://notgithub.com/"); - const flows = [slack, node, daemon, none]; - const where = (query: Parameters[1]) => - flows.filter((f) => matchesQuery(f, query)); - expect(where({ app: ["slack"] })).toEqual([slack]); - expect(where({ app: ["Fieldwork"] })).toEqual([node]); - expect(where({ app: ["/opt/homebrew/bin/node"] })).toEqual([node]); - expect(where({ under: ["*.github.com"] })).toEqual([node, daemon]); - expect(where({ under: ["api.github.com"] })).toEqual([node]); - expect(where({ system: true })).toEqual([daemon]); - expect(where({ system: false })).toEqual([slack, node, none]); - expect(flows.map(appKeyOf)).toEqual(["Slack", "fieldwork", "nsurlsessiond", null]); - }); - - test("combines terms with the sidebar's selection", () => { - expect(matchesQuery(stripe, { q: "post", under: ["api.stripe.com"] })).toBe(true); - expect(matchesQuery(stripe, { q: "post", under: ["api.github.com"] })).toBe(false); - }); -}); +import { appKeyOf, isSystemFlow, sidebarPath, siteOf } from "./search"; describe("sidebarPath", () => { // The cases `sidebar_path_rules` in `flows.rs` checks, so the two agree. @@ -159,125 +21,6 @@ describe("sidebarPath", () => { }); }); -describe("parseSearch", () => { - test("is null for text that narrows nothing", () => { - expect(parseSearch("")).toBeNull(); - expect(parseSearch(" ")).toBeNull(); - expect(parseSearch("OR")).toBeNull(); - }); - - test("keeps free words as terms, trimmed and collapsed", () => { - expect(parseSearch(" stripe post ")).toEqual({ q: "stripe post" }); - }); - - test("reads apps, hosts and the paths below them", () => { - expect(parseSearch('@app:"Google Chrome" @app:Slack card')).toEqual({ - q: "card", - app: ["Google Chrome", "Slack"], - }); - expect(parseSearch("@app:fieldwork @host:api.stripe.com @path:/v1/payment_intents")).toEqual({ - app: ["fieldwork"], - under: ["api.stripe.com/v1/payment_intents"], - }); - expect(parseSearch("@host:*.github.com")).toEqual({ under: ["*.github.com"] }); - expect(parseSearch("@host:github.com @path:/mountain-loop/yaak/*")).toEqual({ - under: ["github.com/mountain-loop/yaak"], - }); - // A path with no host matches anywhere in the URL. - expect(parseSearch("@path:/v1")).toEqual({ q: "/v1" }); - // Each path stays with the host before it. - expect(parseSearch("@host:a.com @path:/x @host:b.com @path:/y")).toEqual({ - under: ["a.com/x", "b.com/y"], - }); - }); - - test("keeps each clause OR separates as a group of its own", () => { - // Two selected sidebar rows under different apps: Slack on a.com, or Chrome on b.com. - const text = "@app:Slack @host:a.com OR @app:Chrome @host:b.com"; - const query = parseSearch(text); - expect(query).toEqual({ - any: [ - { app: ["Slack"], under: ["a.com"] }, - { app: ["Chrome"], under: ["b.com"] }, - ], - }); - const on = (clientName: string, host: string) => - flow(`https://${host}/x`, { clientName, clientBundleId: `app.${clientName}` }); - const match = (f: Flow) => query != null && matchesQuery(f, query); - expect(match(on("Slack", "a.com"))).toBe(true); - expect(match(on("Chrome", "b.com"))).toBe(true); - // Not every pairing of the two rows' apps and hosts. - expect(match(on("Slack", "b.com"))).toBe(false); - expect(match(on("Chrome", "a.com"))).toBe(false); - - // Words bind to their clause, and hiding System applies to each group that does not ask. - expect(parseSearch("@host:a.com OR @is:system @app:trustd", true)).toEqual({ - any: [ - { under: ["a.com"], system: false }, - { app: ["trustd"], system: true }, - ], - }); - // A clause that narrows nothing takes in every flow. - expect(parseSearch("@app:Slack OR")).toEqual({ app: ["Slack"] }); - expect(parseSearch("@app:Slack OR -@foo:bar")).toBeNull(); - }); - - test("keeps or drops System, which hiding it drops unless the text asks", () => { - expect(parseSearch("@is:system")).toEqual({ system: true }); - expect(parseSearch("-@is:system stripe")).toEqual({ q: "stripe", system: false }); - expect(parseSearch("", true)).toEqual({ system: false }); - expect(parseSearch("@is:system", true)).toEqual({ system: true }); - }); - - test("keeps or drops HTTPS passed through undecrypted", () => { - expect(parseSearch("@is:tunnelled")).toEqual({ tunnelled: true }); - expect(parseSearch("-@is:tunnelled stripe")).toEqual({ q: "stripe", tunnelled: false }); - const tunnel = { url: "https://a.com", method: "CONNECT", tunnelled: true } as Flow; - const decrypted = { url: "https://a.com/x", method: "GET", tunnelled: false } as Flow; - expect(matchesQuery(tunnel, { tunnelled: true })).toBe(true); - expect(matchesQuery(decrypted, { tunnelled: true })).toBe(false); - expect(matchesQuery(decrypted, { tunnelled: false })).toBe(true); - }); - - test("round-trips what the sidebar writes", () => { - const text = [ - formatFieldFilter("app", "Google Chrome"), - formatFieldFilter("host", "api.x.com"), - ].join(" "); - expect(text).toBe('@app:"Google Chrome" @host:api.x.com'); - expect(tokenize(text)).toEqual([ - { field: "app", value: "Google Chrome", negated: false }, - { field: "host", value: "api.x.com", negated: false }, - ]); - }); -}); - -describe("widerQueries", () => { - test("leaves out each term in turn", () => { - expect(widerQueries('@app:Slack @status:5xx "card declined"')).toEqual([ - '@status:5xx "card declined"', - '@app:Slack "card declined"', - "@app:Slack @status:5xx", - ]); - expect(widerQueries("@app:Slack")).toEqual([]); - }); - - test("widens one OR group at a time, keeping the groups", () => { - const text = "@app:Slack @host:absent.com OR @app:Chrome @host:absent.com"; - expect(widerQueries(text)).toEqual([ - "@host:absent.com OR @app:Chrome @host:absent.com", - "@app:Slack OR @app:Chrome @host:absent.com", - "@app:Slack @host:absent.com OR @host:absent.com", - "@app:Slack @host:absent.com OR @app:Chrome", - ]); - // A group of one term is not emptied, which would take in every flow. - expect(widerQueries("@app:Slack OR @app:Chrome @host:b.com")).toEqual([ - "@app:Slack OR @host:b.com", - "@app:Slack OR @app:Chrome", - ]); - }); -}); - describe("isSystemFlow", () => { // The cases `tells_system_daemons_apart` in `flows.rs` checks, so the two agree. test("is a daemon outside an app, under the system's folders or by name", () => { @@ -307,3 +50,31 @@ describe("siteOf", () => { expect(siteOf("149.154.175.50")).toBe("149.154.175.50"); }); }); + +describe("appKeyOf", () => { + test("names an app, a program, or a runtime by the folder it ran in", () => { + const flow = (extra: Partial) => + ({ + clientName: null, + clientBundleId: null, + clientPath: null, + clientCwd: null, + ...extra, + }) as Flow; + const flows = [ + flow({ clientName: "Slack", clientBundleId: "com.tinyspeck.slackmacgap" }), + flow({ + clientName: "node", + clientPath: "/opt/homebrew/bin/node", + clientCwd: "/src/fieldwork", + }), + flow({ + clientName: "nsurlsessiond", + clientPath: "/usr/libexec/nsurlsessiond", + clientCwd: "/", + }), + flow({}), + ]; + expect(flows.map(appKeyOf)).toEqual(["Slack", "fieldwork", "nsurlsessiond", null]); + }); +}); diff --git a/apps/yaak-proxy/lib/search.ts b/apps/yaak-proxy/lib/search.ts index 8cd2f4046..7183554c0 100644 --- a/apps/yaak-proxy/lib/search.ts +++ b/apps/yaak-proxy/lib/search.ts @@ -1,147 +1,6 @@ -import type { Flow, FlowFilter } from "@yaakapp-internal/proxy-lib"; +import type { Flow } from "@yaakapp-internal/proxy-lib"; import { getDomain } from "tldts"; -/** - * A search over every flow in the database: the search box's terms, its field terms, and - * whether System's flows are hidden, as `list_flows` takes them. - */ -export type FlowQuery = Pick< - FlowFilter, - "q" | "under" | "app" | "cwd" | "system" | "tunnelled" | "any" ->; - -/** A word of the search box: a free term, or `@field:value`, maybe negated with `-` */ -export interface SearchToken { - field: string | null; - value: string; - negated: boolean; -} - -/** - * The search box's words. Values may be quoted, `@app:"Google Chrome"`; `OR` and `AND` between - * words are dropped here, and [`clausesOf`] splits on `OR`. - */ -export function tokenize(text: string): SearchToken[] { - return clausesOf(text).flat(); -} - -/** - * The search box's words in the clauses `OR` separates, each a group of terms that must all - * match: `@app:Slack @host:a.com OR @app:Chrome @host:b.com` is two. `AND` is dropped, since - * terms side by side must all match anyway. - */ -export function clausesOf(text: string): SearchToken[][] { - const clauses: SearchToken[][] = [[]]; - const re = /(-)?(?:@([a-z]+):)?(?:"([^"]*)"?|(\S+))/gi; - for (const m of text.matchAll(re)) { - const [, minus, field, quoted, bare] = m; - const value = quoted ?? bare ?? ""; - if (field == null && minus == null && quoted == null && value === "OR") { - clauses.push([]); - continue; - } - if (field == null && minus == null && quoted == null && value === "AND") continue; - if (field == null && value === "" && quoted == null) continue; - clauses.at(-1)?.push({ field: field?.toLowerCase() ?? null, value, negated: minus != null }); - } - return clauses.filter((c) => c.length > 0); -} - -/** `@field:value`, quoting a value with spaces or quotes in it */ -export function formatFieldFilter(field: string, value: string, negated = false): string { - const v = /[\s"]/.test(value) || value === "" ? `"${value.replaceAll('"', "")}"` : value; - return `${negated ? "-" : ""}@${field}:${v}`; -} - -/** - * The search box's text as a query, or null when it narrows nothing. Clauses `OR` separates are - * groups, each judged whole, so two sidebar rows under different apps match what each does and - * not every pairing of their apps and hosts. Within a clause, `@app` terms match any of their - * apps; `@host` terms any of their hosts, each with the `@path` terms after it; `@is:system` - * keeps only macOS's daemons and `-@is:system` drops them, as hiding System does when the - * clause says neither. `@is:tunnelled` keeps only HTTPS passed through undecrypted, and - * `-@is:tunnelled` drops it. Other words match the URL, the method, the status or the type. - */ -export function parseSearch(text: string, hideSystem = false): FlowQuery | null { - const clauses = clausesOf(text); - if (clauses.length <= 1) return parseClause(clauses[0] ?? [], hideSystem); - const any: FlowQuery[] = []; - for (const clause of clauses) { - const group = parseClause(clause, hideSystem); - // A clause that narrows nothing takes in every flow, so the whole does too. - if (group == null) return hideSystem ? { system: false } : null; - any.push(group); - } - return { any }; -} - -function parseClause(tokens: SearchToken[], hideSystem: boolean): FlowQuery | null { - const words: string[] = []; - const apps: string[] = []; - const cwds: string[] = []; - // Each host with the paths written after it. - const hosts: { host: string; paths: string[] }[] = []; - let system: boolean | undefined = hideSystem ? false : undefined; - let tunnelled: boolean | undefined; - for (const t of tokens) { - if (t.field === "app" && !t.negated && t.value !== "") apps.push(t.value); - else if (t.field === "cwd" && !t.negated && t.value !== "") cwds.push(t.value); - else if (t.field === "host" && !t.negated && t.value !== "") { - hosts.push({ host: t.value.replace(/\/+$/, ""), paths: [] }); - } else if (t.field === "path" && !t.negated && t.value !== "") { - const path = t.value.replace(/\/?\*$/, "").replace(/^\/+/, ""); - const host = hosts.at(-1); - // A path with no host to sit under matches anywhere in the URL. - if (host == null) { - if (path !== "") words.push(`/${path}`); - } else { - host.paths.push(path); - } - } else if (t.field === "is" && t.value.toLowerCase() === "system") system = !t.negated; - else if (t.field === "is" && t.value.toLowerCase() === "tunnelled") tunnelled = !t.negated; - else if (t.field == null && !t.negated) words.push(t.value); - // Fields and exclusions the query language will add are ignored until it lands. - } - const under: string[] = []; - for (const { host, paths } of hosts) { - if (paths.length === 0 || host.startsWith("*.")) under.push(host); - else for (const p of paths) under.push(p === "" ? host : `${host}/${p}`); - } - const q = words.join(" ").trim(); - const query: FlowQuery = { - ...(q === "" ? {} : { q }), - ...(under.length > 0 ? { under } : {}), - ...(apps.length > 0 ? { app: apps } : {}), - ...(cwds.length > 0 ? { cwd: cwds } : {}), - ...(system == null ? {} : { system }), - ...(tunnelled == null ? {} : { tunnelled }), - }; - return Object.keys(query).length === 0 ? null : query; -} - -/** - * Wider searches for one that matched nothing: the text with each of its terms left out in turn, - * without repeats. With groups `OR` separates, a term is left out of one group at a time and - * the groups kept, since each group alone is narrower still; a group left with nothing would - * take in every flow, so it is not offered. - */ -export function widerQueries(text: string): string[] { - const format = (t: SearchToken) => - t.field == null - ? `${t.negated ? "-" : ""}${/\s/.test(t.value) ? `"${t.value}"` : t.value}` - : formatFieldFilter(t.field, t.value, t.negated); - const clauses = clausesOf(text).map((c) => c.map(format)); - const out = new Set(); - clauses.forEach((clause, i) => { - if (clause.length <= 1) return; - clause.forEach((_, j) => { - const wider = clauses.map((c, k) => (k === i ? c.filter((_, l) => l !== j) : c)); - out.add(wider.map((c) => c.join(" ")).join(" OR ")); - }); - }); - return [...out]; -} - /** The folder a program outside an app ran in */ export function folderOf(flow: Pick): string | null { if (flow.clientBundleId != null) return null; @@ -202,8 +61,8 @@ const SYSTEM_PROGRAMS = new Set([ /** * A flow from one of macOS's own daemons: a process outside an app bundle whose executable is * under `/System`, `/usr/libexec` and the like, or one of a few daemons by name, but not WebKit's - * helpers, which send apps' traffic. `is_system` in `crates-proxy/yaak-proxy-lib`, which the - * database filters by, follows the same rules. + * helpers, which send apps' traffic. `is_system` in `crates-proxy/yaak-proxy-lib`, which + * `@is:system` searches by, follows the same rules. */ export function isSystemFlow(flow: Pick): boolean { const path = flow.clientPath; @@ -213,66 +72,6 @@ export function isSystemFlow(flow: Pick): return SYSTEM_DIRS.some((dir) => path.startsWith(dir)) || SYSTEM_PROGRAMS.has(name); } -/** - * Whether `flow` is one `query` covers, as `list_flows` decides it in SQL, so new traffic joins a - * search as it arrives. Every term must match the URL, the method, the status or the content type. - */ -export function matchesQuery(flow: Flow, query: FlowQuery): boolean { - for (const term of (query.q ?? "").toLowerCase().split(/\s+/)) { - if (term === "") continue; - const matched = - flow.url.toLowerCase().includes(term) || - flow.method.toLowerCase() === term || - (flow.contentType?.toLowerCase().includes(term) ?? false) || - matchesStatus(flow.status, term); - if (!matched) return false; - } - if (query.under != null && !query.under.some((node) => isUnder(flow.url, node))) return false; - if (query.app != null && !query.app.some((app) => matchesApp(flow, app))) return false; - if (query.cwd != null && !query.cwd.some((cwd) => matchesCwd(flow, cwd))) return false; - if (query.system != null && isSystemFlow(flow) !== query.system) return false; - if (query.tunnelled != null && flow.tunnelled !== query.tunnelled) return false; - // One of the groups, each judged whole, as the database judges them. - if (query.any != null && query.any.length > 0) { - return query.any.some((group) => matchesQuery(flow, group)); - } - return true; -} - -/** By bundle id or path exactly, or by name or folder in any case */ -function matchesApp(flow: Flow, app: string): boolean { - if ((flow.clientBundleId ?? flow.clientPath) === app) return true; - const lower = app.toLowerCase(); - return flow.clientName?.toLowerCase() === lower || folderOf(flow)?.toLowerCase() === lower; -} - -/** By the whole working directory exactly, or its folder's name in any case */ -function matchesCwd(flow: Flow, cwd: string): boolean { - if (flow.clientCwd == null) return false; - return flow.clientCwd === cwd || folderName(flow.clientCwd)?.toLowerCase() === cwd.toLowerCase(); -} - -/** `404` exactly, `40` as the start of one, or a class such as `4xx` */ -function matchesStatus(status: number | null, term: string): boolean { - if (status == null) return false; - const klass = /^([1-9])xx$/.exec(term); - if (klass != null) return Math.trunc(status / 100) === Number(klass[1]); - if (!/^\d{1,3}$/.test(term)) return false; - return term.length === 3 ? status === Number(term) : String(status).startsWith(term); -} - -/** The URL is the host or path `node` names, or something below it, by the sidebar's rules */ -function isUnder(url: string, node: string): boolean { - const path = sidebarPath(url); - if (node.startsWith("*.") && !node.includes("/")) { - const site = node.slice(2).toLowerCase(); - const host = hostnameOf(path.split("/")[0] ?? ""); - return host === site || host.endsWith(`.${site}`); - } - const want = normalizeNode(node); - return path === want || path.startsWith(`${want}/`); -} - /** A host without its port, keeping an IPv6 address's brackets */ export function hostnameOf(host: string): string { if (host.startsWith("[")) { @@ -304,7 +103,7 @@ export function siteOf(host: string): string { * Where the sidebar places a URL: its host, lowercased and without its scheme's default port, * then its path's non-empty segments, joined by `/`, as in `api.stripe.com/v1/charges`. A URL * without a scheme is a host of its own. `sidebar_path` in `crates-proxy/yaak-proxy-lib`, which - * the database filters by, follows the same rules. + * `@host` searches by, follows the same rules. */ export function sidebarPath(url: string): string { const schemeEnd = url.indexOf("://"); @@ -326,11 +125,3 @@ export function sidebarPath(url: string): string { const segments = path.split("/").filter((s) => s !== ""); return [host, ...segments].join("/"); } - -/** A sidebar node as `sidebarPath` writes it, its host lowercased */ -function normalizeNode(node: string): string { - const slash = node.indexOf("/"); - return slash === -1 - ? node.toLowerCase() - : `${node.slice(0, slash).toLowerCase()}${node.slice(slash)}`; -} diff --git a/apps/yaak-proxy/lib/searchBox.ts b/apps/yaak-proxy/lib/searchBox.ts new file mode 100644 index 000000000..ffac3bd24 --- /dev/null +++ b/apps/yaak-proxy/lib/searchBox.ts @@ -0,0 +1,79 @@ +import type { Completion } from "@codemirror/autocomplete"; +import type { QueryDiagnostic, SavedQuery } from "@yaakapp-internal/proxy-lib"; +import type { QueryProblem } from "@yaakapp-internal/ui/editor"; +import { asOneTerm } from "./query"; +import { rpc } from "./rpc"; + +/** How long values read for completion are offered before they are read again */ +const VALUES_MS = 10_000; + +const seenValues = new Map(); +const reading = new Map>(); + +/** + * The values the proxy has seen for field paths, for completing them: read from the database + * the first time each is asked for, and again once they are a few seconds old, so the list + * keeps up with the traffic without a read per keystroke. + */ +export async function loadSeenValues(paths: string[]): Promise> { + const now = Date.now(); + const entries = await Promise.all( + paths.map(async (path): Promise<[string, string[]]> => { + const cached = seenValues.get(path); + if (cached != null && now - cached.at < VALUES_MS) return [path, cached.values]; + let read = reading.get(path); + if (read == null) { + read = rpc("query_values", { path }) + .then((values) => { + seenValues.set(path, { values, at: Date.now() }); + return values; + }) + .catch(() => cached?.values ?? []) + .finally(() => reading.delete(path)); + reading.set(path, read); + } + return [path, await read]; + }), + ); + return Object.fromEntries(entries); +} + +const PROBLEM_LABELS: Record = { + unknownField: "Unknown field", + invalidValue: "Invalid value", + notYet: "Not yet", +}; + +/** The terms of a search the backend cannot run, as the box underlines them */ +export function problemsOf(diagnostics: QueryDiagnostic[]): QueryProblem[] { + return diagnostics.map((d) => ({ + from: d.start, + to: d.end, + label: PROBLEM_LABELS[d.kind], + quiet: d.kind === "notYet", + })); +} + +export async function searchProblems(text: string): Promise { + if (text.trim() === "") return []; + return problemsOf(await rpc("validate_query", { query: text })); +} + +/** + * Saved searches whose names hold the word being typed, each replacing it with its search. One + * with `OR` at its top goes in parentheses, so it stays one group beside the box's other terms. + */ +export function savedCompletions(saved: SavedQuery[], typed: string): Completion[] { + const needle = typed.toLowerCase(); + return saved + .filter((s) => s.name.toLowerCase().includes(needle)) + .map((s) => { + const insert = asOneTerm(s.query); + return { + label: s.name, + detail: s.query, + type: "text", + apply: insert, + }; + }); +} diff --git a/apps/yaak-proxy/lib/searchChanges.test.ts b/apps/yaak-proxy/lib/searchChanges.test.ts new file mode 100644 index 000000000..6801cff8a --- /dev/null +++ b/apps/yaak-proxy/lib/searchChanges.test.ts @@ -0,0 +1,96 @@ +import type { Flow, ModelPayload } from "@yaakapp-internal/proxy-lib"; +import { describe, expect, test } from "vite-plus/test"; +import { emptyWindow, type FlowWindow } from "./flowWindow"; +import { changesFor, FlowEvents } from "./searchChanges"; + +const none = () => {}; + +function write(id: string, matches: boolean, key = "k"): ModelPayload { + return { + model: { id, startedAt: "2026-11-07 12:00:00", updatedAt: "2026-11-07 12:00:00" } as Flow, + change: { type: "upsert", created: false }, + search: { key, matches }, + }; +} + +describe("changesFor", () => { + test("takes matching flows and drops those that stop matching", () => { + const win: FlowWindow = { ...emptyWindow, flows: [write("a", true).model] }; + const kept = changesFor( + "k", + win, + [write("a", false), write("b", true), write("c", false)], + none, + ); + expect(kept.map((c) => [c.model.id, c.change.type])).toEqual([ + ["a", "delete"], + ["b", "upsert"], + ]); + }); + + test("a flow that joins and stops matching in one batch leaves again", () => { + const kept = changesFor("k", emptyWindow, [write("a", true), write("a", false)], none); + expect(kept.map((c) => [c.model.id, c.change.type])).toEqual([ + ["a", "upsert"], + ["a", "delete"], + ]); + }); + + test("leaves flows matched against another search to be matched again", () => { + const later: string[] = []; + const kept = changesFor("k", emptyWindow, [write("a", true, "old")], (unknown) => + later.push(...unknown.map((c) => c.model.id)), + ); + expect([kept, later]).toEqual([[], ["a"]]); + }); +}); + +describe("FlowEvents", () => { + test("drops a late answer for a flow a newer event removed, though the window no longer holds it", () => { + const events = new FlowEvents(); + const old = write("a", true, "old"); + events.arrived([old]); + // Under the new search the flow stops matching and leaves the window... + events.arrived([write("a", false)]); + // ...and the re-check of the old event answers after + expect(events.current([old])).toEqual([]); + }); + + test("drops a late answer for a flow the sweep removed", () => { + const events = new FlowEvents(); + const old = write("a", true, "old"); + events.arrived([old]); + events.removed(["a"]); + expect(events.current([old])).toEqual([]); + }); + + test("keeps an answer no newer event has passed, and one for other flows", () => { + const events = new FlowEvents(); + const a = write("a", true, "old"); + const b = write("b", true, "old"); + events.arrived([a, b]); + events.arrived([write("c", true)]); + expect(events.current([a, b])).toEqual([a, b]); + // Not noted: nothing to judge it by + const unseen = write("d", true, "old"); + expect(events.current([unseen])).toEqual([unseen]); + }); + + test("past what it remembers, drops what it can no longer judge", () => { + const events = new FlowEvents(2); + const a = write("a", true, "old"); + events.arrived([a, write("b", true), write("c", true)]); + expect(events.current([a])).toEqual([]); + const c = write("c", true, "old"); + events.arrived([c]); + expect(events.current([c])).toEqual([c]); + }); + + test("drops answers from before a clear", () => { + const events = new FlowEvents(); + const a = write("a", true, "old"); + events.arrived([a]); + events.reset(); + expect(events.current([a])).toEqual([]); + }); +}); diff --git a/apps/yaak-proxy/lib/searchChanges.ts b/apps/yaak-proxy/lib/searchChanges.ts new file mode 100644 index 000000000..314590306 --- /dev/null +++ b/apps/yaak-proxy/lib/searchChanges.ts @@ -0,0 +1,105 @@ +import type { ModelPayload } from "@yaakapp-internal/proxy-lib"; +import type { FlowWindow } from "./flowWindow"; + +/** + * The changes a search's window `win` takes: flows the backend matched against the search, by + * its `key`, as it sent their events, and every deletion. A flow the window holds that no longer + * matches, such as a pending flow that completed under `@is:pending`, leaves it, even when it + * joined earlier in the same batch. Flows matched against another search, sent as the search + * changed or read from the database while another process owns the engine, go to `unknown` to + * be matched again. + */ +export function changesFor( + key: string, + win: FlowWindow, + changes: ModelPayload[], + unknown: (changes: ModelPayload[]) => void, +): ModelPayload[] { + const others: ModelPayload[] = []; + // What the window holds as each change applies + const held = new Set(win.flows.map((f) => f.id)); + const kept: ModelPayload[] = []; + for (const c of changes) { + if (c.change.type === "delete") { + held.delete(c.model.id); + kept.push(c); + } else if (c.search?.key !== key) { + others.push(c); + } else if (c.search.matches) { + held.add(c.model.id); + kept.push(c); + } else if (held.delete(c.model.id)) { + kept.push({ ...c, change: { type: "delete" } }); + } + } + if (others.length > 0) unknown(others); + return kept; +} + +/** Flows whose newest event `FlowEvents` remembers, at most */ +const REMEMBERED = 20_000; + +/** + * The order flow events arrived in, so an answer that comes later than newer events for the + * same flow can be told apart: a re-check started before the search changed, say, whose flow a + * newer event has since updated or removed, even from the window. + */ +export class FlowEvents { + private count = 0; + /** Each event's place, by the payload as it arrived */ + private seqs = new WeakMap(); + /** The newest event's place, by flow, removals included, the least recent first */ + private latest = new Map(); + /** Places at or below this may have newer events no longer remembered */ + private floor = 0; + + constructor(private readonly remembered = REMEMBERED) {} + + /** Notes events as they arrive */ + arrived(changes: ModelPayload[]) { + for (const c of changes) { + this.count++; + this.seqs.set(c, this.count); + this.note(c.model.id); + } + } + + /** Notes flows removed with no event of their own, as the keep-traffic sweep removes them */ + removed(ids: string[]) { + for (const id of ids) { + this.count++; + this.note(id); + } + } + + private note(id: string) { + // Moved to the end, so the least recent go first + this.latest.delete(id); + this.latest.set(id, this.count); + for (const [oldest, seq] of this.latest) { + if (this.latest.size <= this.remembered) break; + this.latest.delete(oldest); + this.floor = Math.max(this.floor, seq); + } + } + + /** + * Of the changes a late answer is for, those no newer event for their flow has passed. One + * from before the oldest event remembered is dropped, as it can't be judged; the next count + * and reload correct the list. One whose arrival was not noted is kept. + */ + current(changes: T[]): T[] { + return changes.filter((c) => { + const seq = this.seqs.get(c); + if (seq == null) return true; + const newest = this.latest.get(c.model.id); + return newest == null ? seq > this.floor : seq >= newest; + }); + } + + /** Forgets everything, as a clear does */ + reset() { + this.latest.clear(); + this.floor = this.count; + } +} diff --git a/apps/yaak-proxy/lib/sidebarTree.test.ts b/apps/yaak-proxy/lib/sidebarTree.test.ts index 002a7bb15..f1f44dac3 100644 --- a/apps/yaak-proxy/lib/sidebarTree.test.ts +++ b/apps/yaak-proxy/lib/sidebarTree.test.ts @@ -1,7 +1,7 @@ import type { Flow } from "@yaakapp-internal/proxy-lib"; import type { TreeNode } from "@yaakapp-internal/ui"; import { describe, expect, test } from "vite-plus/test"; -import { parseSearch } from "./search"; +import { parseQuery } from "@yaakapp-internal/query"; import { buildSidebarTree, itemsById, @@ -83,8 +83,9 @@ describe("buildSidebarTree", () => { const query = (id: string) => items.get(id)?.query; expect(query("all")).toBe(""); expect(query("app:fieldwork")).toBe("@app:fieldwork"); + // A path row takes in what is under the path. expect(query("app:fieldwork|path:api.github.com/repos")).toBe( - "@app:fieldwork @host:api.github.com @path:/repos", + "@app:fieldwork @host:api.github.com @path:/repos/*", ); expect(query("site:github.com")).toBe("@host:*.github.com"); expect(query("system")).toBe("@is:system"); @@ -93,12 +94,12 @@ describe("buildSidebarTree", () => { expect(query("app:trustd|host:ocsp.apple.com")).toBe( "@is:system @app:trustd @host:ocsp.apple.com", ); - expect(parseSearch(query("app:trustd") ?? "", true)?.system).toBe(true); - // What each writes, read back, covers the flows the row counts. + // What each writes reads back as the terms it wrote. for (const item of items.values()) { - const parsed = parseSearch(item.query); - if (parsed == null || item.kind === "group") continue; + if (item.kind === "group" || item.kind === "all") continue; + const parsed = parseQuery(item.query); expect(parsed, item.id).toBeTruthy(); + expect(parsed?.type, item.id).not.toBe("Or"); } }); }); @@ -164,7 +165,7 @@ describe("programs outside an app", () => { expect(items.get("app:claude|cwd:yaak|host:statsig.anthropic.com")?.query).toBe( "@app:claude @cwd:yaak @host:statsig.anthropic.com", ); - expect(parseSearch(query)).toEqual({ app: ["claude"], cwd: ["yaak"] }); + expect(parseQuery(query)?.type).toBe("And"); }); }); @@ -215,12 +216,10 @@ describe("selection and the search text", () => { pick("app:Slack|site:slack.com"), ]); expect(text).toBe("@app:fieldwork @host:api.github.com OR @app:Slack @host:*.slack.com"); - expect(parseSearch(text)).toEqual({ - any: [ - { app: ["fieldwork"], under: ["api.github.com"] }, - { app: ["Slack"], under: ["*.slack.com"] }, - ], - }); + // Two groups, each judged whole + const parsed = parseQuery(text); + expect(parsed?.type).toBe("Or"); + expect(parsed?.type === "Or" ? parsed.children.map((c) => c.type) : []).toEqual(["And", "And"]); }); test("writes the selected rows' queries, OR'd, or nothing for All traffic", () => { @@ -240,3 +239,29 @@ describe("selection and the search text", () => { expect(queryInText("", "card")).toBe(false); }); }); + +describe("saved searches", () => { + test("follow All traffic, each a row with its count that writes its query", () => { + const saved = [ + { id: "sq_1", name: "Errors", query: "@status:>=400", count: 12 }, + { id: "sq_2", name: "Stripe", query: "@host:*.stripe.com", count: 0 }, + ]; + const root = buildSidebarTree(flows, 4, saved); + expect((root.children ?? []).slice(0, 2).flatMap((c) => shape(c))).toEqual([ + "All traffic 4", + "Saved 0", + " Errors 12", + " Stripe 0", + ]); + const items = itemsById(root); + expect(items.get("saved:sq_1")).toMatchObject({ + kind: "saved", + query: "@status:>=400", + savedId: "sq_1", + }); + // Rows, not folders + const group = (root.children ?? []).find((c) => c.item.id === "saved"); + expect(group?.children?.every((c) => c.children == null)).toBe(true); + expect(itemsById(buildSidebarTree(flows, 4)).has("saved")).toBe(false); + }); +}); diff --git a/apps/yaak-proxy/lib/sidebarTree.ts b/apps/yaak-proxy/lib/sidebarTree.ts index 3e1066689..76eaa535c 100644 --- a/apps/yaak-proxy/lib/sidebarTree.ts +++ b/apps/yaak-proxy/lib/sidebarTree.ts @@ -1,20 +1,13 @@ import type { Flow } from "@yaakapp-internal/proxy-lib"; import type { TreeNode } from "@yaakapp-internal/ui"; import type { DecryptSubject } from "./decrypt"; -import { - appKeyOf, - folderOf, - formatFieldFilter, - isSystemFlow, - runtimeFolderOf, - sidebarPath, - siteOf, - tokenize, -} from "./search"; +import { fieldTerm, pathTerm, siteTerm, termKeys } from "./query"; +import { appKeyOf, folderOf, isSystemFlow, runtimeFolderOf, sidebarPath, siteOf } from "./search"; export type SidebarKind = | "all" | "tunnelled" + | "saved" | "group" | "app" | "folder" @@ -23,8 +16,8 @@ export type SidebarKind = | "path"; /** - * A row of the sidebar: All traffic, a group, an app, a program's working directory, or a site, - * host or path under one + * A row of the sidebar: All traffic, a saved search, a group, an app, a program's working + * directory, or a site, host or path under one */ export interface SidebarItem { id: string; @@ -42,10 +35,21 @@ export interface SidebarItem { query: string; /** What Decrypt in the row's menu decrypts: its host, site or app */ decrypt?: DecryptSubject; + /** The saved search a Saved row is, for renaming and deleting it */ + savedId?: string; +} + +/** A saved search as the sidebar lists it, with how many flows it covers */ +export interface SavedRow { + id: string; + name: string; + query: string; + count: number; } /** The ids of All traffic and the groups */ export const ALL_ID = "all"; +export const SAVED_ID = "saved"; export const APPS_ID = "apps"; export const DOMAINS_ID = "domains"; export const SYSTEM_ID = "system"; @@ -83,13 +87,13 @@ function placeOf(flow: Flow): Place { const [host = "", ...segments] = schemeless ? [path] : path.split("/"); const site = siteOf(host); const app = appKeyOf(flow); - const hostQuery = formatFieldFilter("host", host); + const hostQuery = fieldTerm("host", host); const rows: PlaceRow[] = [ { id: `site:${site}`, kind: "site", label: site, - query: formatFieldFilter("host", `*.${site}`), + query: siteTerm(site), decrypt: { kind: "site", site }, }, { @@ -107,7 +111,7 @@ function placeOf(flow: Flow): Place { id: `path:${host}${p}`, kind: "path", label: `/${segment}`, - query: `${hostQuery} ${formatFieldFilter("path", p)}`, + query: `${hostQuery} ${pathTerm(p)}`, }); } const system = isSystemFlow(flow); @@ -123,7 +127,7 @@ function placeOf(flow: Flow): Place { })); } else { const folderId = `app:${app}|cwd:${folder}`; - const folderQuery = `${appQuery} ${formatFieldFilter("cwd", folder)}`; + const folderQuery = `${appQuery} ${fieldTerm("cwd", folder)}`; appRows = [ { id: folderId, kind: "folder", label: folder, query: folderQuery }, ...rows.map((r) => ({ @@ -144,8 +148,8 @@ function placeOf(flow: Flow): Place { * while System is hidden from All traffic. */ function appQueryFor(app: string, system: boolean): string { - const query = formatFieldFilter("app", app); - return system ? `${formatFieldFilter("is", "system")} ${query}` : query; + const query = fieldTerm("app", app); + return system ? `${fieldTerm("is", "system")} ${query}` : query; } /** A node being built, its children by id */ @@ -209,10 +213,14 @@ function group(id: string, label: string): Trie { * The sidebar: All traffic, then Apps, each app opening to its own sites, hosts and paths, then * Domains by site, host and path, then System, macOS's daemons, apart from the apps. A group with * nothing in it is left out. Not decrypted, after All traffic, gathers the HTTPS passed through - * undecrypted, when there is any. `total` is All traffic's count: every flow it covers in the - * database. + * undecrypted, when there is any, and Saved lists the saved searches, each with its count from + * the database. `total` is All traffic's count: every flow it covers in the database. */ -export function buildSidebarTree(flows: Flow[], total: number): TreeNode { +export function buildSidebarTree( + flows: Flow[], + total: number, + saved: SavedRow[] = [], +): TreeNode { const root: TreeNode = { item: { id: "root", kind: "group", label: "", count: 0, query: "" }, parent: null, @@ -223,7 +231,7 @@ export function buildSidebarTree(flows: Flow[], total: number): TreeNode 0) { + const node: TreeNode = { + item: { id: SAVED_ID, kind: "group", label: "Saved", count: 0, query: "" }, + parent: root, + depth: 1, + children: [], + draggable: false, + }; + node.children = saved.map((s) => ({ + item: { + id: `saved:${s.id}`, + kind: "saved", + label: s.name, + count: s.count, + query: s.query, + savedId: s.id, + }, + parent: node, + depth: 2, + draggable: false, + })); + root.children?.push(node); + } for (const g of [apps, domains, system]) { if (g.children.size === 0) continue; const node: TreeNode = { @@ -318,12 +349,8 @@ export function selectionText(items: SidebarItem[]): string { * stays selected while it is, so typing after selecting keeps it. */ export function queryInText(query: string, text: string): boolean { - const have = new Set(tokenize(text).map(tokenKey)); - const want = tokenize(query); + const have = new Set(termKeys(text)); + const want = termKeys(query); if (want.length === 0) return have.size === 0; - return want.every((t) => have.has(tokenKey(t))); -} - -function tokenKey(t: { field: string | null; value: string; negated: boolean }): string { - return `${t.negated ? "-" : ""}${t.field ?? ""}:${t.value.toLowerCase()}`; + return want.every((t) => have.has(t)); } diff --git a/apps/yaak-proxy/lib/store.ts b/apps/yaak-proxy/lib/store.ts index 59f86eb9b..fc9655c4d 100644 --- a/apps/yaak-proxy/lib/store.ts +++ b/apps/yaak-proxy/lib/store.ts @@ -1,4 +1,9 @@ -import type { Flow, GetProxyStateResponse, ModelPayload } from "@yaakapp-internal/proxy-lib"; +import type { + Flow, + GetProxyStateResponse, + ModelPayload, + SavedQuery, +} from "@yaakapp-internal/proxy-lib"; import { selectedIdsFamily } from "@yaakapp-internal/ui"; import { atom, type createStore } from "jotai"; import { atomWithStorage } from "jotai/utils"; @@ -20,8 +25,9 @@ import { WINDOW_SIZE, windowFromPage, } from "./flowWindow"; +import { activeQuery, anyCountsBackFromNow, countsBackFromNow, type FlowQuery } from "./query"; import { listen, rpc } from "./rpc"; -import { type FlowQuery, matchesQuery, parseSearch } from "./search"; +import { changesFor, FlowEvents } from "./searchChanges"; import { ALL_ID, APPS_ID, @@ -29,6 +35,8 @@ import { DOMAINS_ID, itemsById, queryInText, + SAVED_ID, + type SavedRow, selectionText, } from "./sidebarTree"; @@ -40,6 +48,8 @@ const FLUSH_MS = 100; const RECOUNT_MS = 1000; /** How often the database is read for new flows while another process owns the engine */ const POLL_MS = 1000; +/** How often a search counting back from now is read again, as flows age in and out of it */ +const AGE_REFRESH_MS = 15_000; /** How often the engine's owner is checked while it is another process */ const OWNER_POLL_MS = 2000; @@ -90,9 +100,26 @@ export const hideSystemAtom = atomWithStorage("sidebar_hide_system", false, unde /** The search the list shows, from the search box's settled text, or null for all traffic */ export const activeQueryAtom = atom((get) => - parseSearch(get(searchTermsAtom), get(hideSystemAtom)), + activeQuery(get(searchTermsAtom), get(hideSystemAtom)), ); +/** The saved searches, in their order */ +export const savedQueriesAtom = atom([]); + +/** How many flows each saved search covers, by id, counted in the database */ +export const savedCountsAtom = atom>({}); + +/** The saved searches as the sidebar lists them */ +const savedRowsAtom = atom((get) => { + const counts = get(savedCountsAtom); + return get(savedQueriesAtom).map((s) => ({ + id: s.id, + name: s.name, + query: s.query, + count: counts[s.id] ?? 0, + })); +}); + /** Writes `text` into the search box at once, as the sidebar does, without waiting for typing */ export const setSearchTextAtom = atom(null, (_get, set, text: string) => { set(searchTextAtom, text); @@ -119,7 +146,7 @@ const SIDEBAR_MS = 1000; /** The sidebar's rows, from the flows loaded */ export const sidebarTreeAtom = atom((get) => - buildSidebarTree(get(sidebarFlowsAtom), get(allTrafficTotalAtom)), + buildSidebarTree(get(sidebarFlowsAtom), get(allTrafficTotalAtom), get(savedRowsAtom)), ); const storedColumnPrefsAtom = atomWithStorage( @@ -244,7 +271,7 @@ export const loadOlderFlowsAtom = atom(null, async (get, set) => { const merge = (win: FlowWindow) => { if (win.flows[0]?.id !== first.id) return win; const merged = prependOlder(win, withoutRemoved(res.flows), PAGE_SIZE); - const kept = search == null ? changes : changesFor(search.query, changes); + const kept = search == null ? changes : changesOf(search, merged, changes); // Counted when they first applied. return { ...applyChanges(merged, kept), total: merged.total }; }; @@ -261,9 +288,47 @@ export const loadOlderFlowsAtom = atom(null, async (get, set) => { } }); -/** The changes a search's window takes: flows it matches, and every deletion */ -function changesFor(query: FlowQuery, changes: ModelPayload[]): ModelPayload[] { - return changes.filter((c) => c.change.type === "delete" || matchesQuery(c.model, query)); +/** `changesFor`, with flows matched against another search sent to be matched again */ +function changesOf( + search: Pick, + win: FlowWindow, + changes: ModelPayload[], +): ModelPayload[] { + return changesFor(search.key, win, changes, (unknown) => matchLater(search, unknown)); +} + +/** The store `syncFlows` keeps, for answers that come after the changes they are for */ +let syncedStore: Store | null = null; + +/** The order flow events arrived in, for judging answers that come late */ +const flowEvents = new FlowEvents(); + +/** + * Matches flows against the search in the database, then has its window take them as it takes + * events matched as they were sent. A flow a newer event updated or removed meanwhile, in the + * window or not, keeps what that event left. + */ +function matchLater(search: Pick, changes: ModelPayload[]) { + const store = syncedStore; + if (store == null) return; + const ids = [...new Set(changes.map((c) => c.model.id))]; + rpc("match_flows", { filter: search.query, ids }) + .then((matched) => { + const covered = new Set(matched); + const marked = flowEvents.current(changes).map((c) => ({ + ...c, + search: { key: search.key, matches: covered.has(c.model.id) }, + })); + if (marked.length === 0) return; + const following = store.get(followingAtom); + store.set(searchWindowAtom, (s) => { + if (s?.key !== search.key) return s; + if (s.loading) return { ...s, buffered: [...s.buffered, ...marked] }; + const next = applyChanges(s.win, changesOf(s, s.win, marked)); + return { ...s, win: following ? trimWindow(next) : next }; + }); + }) + .catch((err) => console.warn("Failed to match flows", err)); } /** @@ -274,6 +339,7 @@ function changesFor(query: FlowQuery, changes: ModelPayload[]): ModelPayload[] { * way. */ export function syncFlows(store: Store) { + syncedStore = store; let pending: ModelPayload[] = []; let timer: ReturnType | null = null; let loaded = false; @@ -297,7 +363,7 @@ export function syncFlows(store: Store) { if (s == null) return s; // A matching flow written after the page's snapshot would otherwise be missed. if (s.loading) return { ...s, buffered: [...s.buffered, ...changes] }; - const next = applyChanges(s.win, changesFor(s.query, changes)); + const next = applyChanges(s.win, changesOf(s, s.win, changes)); return { ...s, win: following ? trimWindow(next) : next }; }); const selected = store.get(selectedFlowAtom); @@ -309,6 +375,7 @@ export function syncFlows(store: Store) { } function receive(changes: ModelPayload[]) { + flowEvents.arrived(changes); pending.push(...changes); if (timer == null) timer = setTimeout(flush, FLUSH_MS); } @@ -326,10 +393,11 @@ export function syncFlows(store: Store) { ); }); if (store.get(hideSystemAtom)) { - void rpc("list_flows", { system: false, limit: 0 }).then((res) => { + void rpc("list_flows", { hideSystem: true, limit: 0 }).then((res) => { if (thisLoad === loads) store.set(nonSystemTotalAtom, res.matching ?? res.total); }); } + countSaved(); const search = store.get(searchWindowAtom); if (search == null || search.loading) return; void rpc("list_flows", { ...search.query, limit: 0 }).then((res) => { @@ -340,21 +408,60 @@ export function syncFlows(store: Store) { }); } + // Saved searches, and their counts, which change with the traffic. + function loadSaved() { + rpc("list_saved_queries", {}) + .then((saved) => { + store.set(savedQueriesAtom, saved); + countSaved(); + }) + .catch((err) => console.warn("Failed to list saved searches", err)); + } + let counting = false; + let countAgain = false; + function countSaved() { + if (store.get(savedQueriesAtom).length === 0) return; + // One count at a time; one asked for meanwhile runs after it, with the latest traffic. + if (counting) { + countAgain = true; + return; + } + counting = true; + rpc("count_saved_queries", { hideSystem: store.get(hideSystemAtom) }) + .then((counts) => + store.set(savedCountsAtom, Object.fromEntries(counts.map((c) => [c.id, c.count]))), + ) + .catch((err) => console.warn("Failed to count saved searches", err)) + .finally(() => { + counting = false; + if (countAgain) { + countAgain = false; + countSaved(); + } + }); + } + listen("saved_queries_changed", loadSaved); + loadSaved(); + listen("model_write", (payload) => { seenId = payload.model.id; receive([payload]); }); // Loads again rather than only emptying, for a flow written between the clear and the event. + // Saved searches too, which could not be read while the database was locked. listen("flows_cleared", () => { + flowEvents.reset(); store.set(emptyFlowsAtom); pending = []; load(); + loadSaved(); }); // The oldest flows, swept to keep the capture under its size limit: dropped wherever they // are held, and the totals counted again from the database. listen("flows_removed", ({ ids }) => { + flowEvents.removed(ids); const removed = new Set(ids); const drop = (win: FlowWindow) => win.flows.some((f) => removed.has(f.id)) @@ -399,21 +506,31 @@ export function syncFlows(store: Store) { } reloadFlows = load; - /** Loads the first page of `query`'s flows into a window of its own, or drops it for null */ - function loadSearch(query: FlowQuery | null) { + /** + * Loads the first page of `query`'s flows into a window of its own, or drops it for null. The + * backend matches the flows written from now on against it, and their events say so. + */ + function loadSearch(query: FlowQuery | null, keepRows = false) { if (query == null) { store.set(searchWindowAtom, null); + void rpc("watch_query", { key: "" }).catch(() => {}); return; } // Unique to this load, so a slower answer to an earlier one with the same query, such as // one read before a clear, is dropped. const key = `${++searchLoads}:${JSON.stringify(query)}`; - store.set(searchWindowAtom, { key, query, win: emptyWindow, loading: true, buffered: [] }); + // Read again, it keeps showing its rows until the new ones are in. + const win = (keepRows ? store.get(searchWindowAtom)?.win : null) ?? emptyWindow; + store.set(searchWindowAtom, { key, query, win, loading: true, buffered: [] }); + void rpc("watch_query", { key, query: query.query, hideSystem: query.hideSystem }).catch( + (err) => console.warn("Failed to watch the search", err), + ); const settle = (page: FlowWindow | null) => store.set(searchWindowAtom, (s) => { if (s?.key !== key) return s; // Applied again over the page: a flow already in it is replaced, not added. - const win = applyChanges(page ?? s.win, changesFor(query, s.buffered)); + const base = page ?? s.win; + const win = applyChanges(base, changesOf(s, base, s.buffered)); return { ...s, win: page == null ? win : { ...win, total: page.total }, @@ -451,11 +568,21 @@ export function syncFlows(store: Store) { } }); - // All traffic's count leaves System out while it is hidden. + // All traffic's count, and the saved searches', leave System out while it is hidden. store.sub(hideSystemAtom, () => { if (loaded) recount(); }); + // A search counting back from now, `@time:<5m`, is read again while it is shown and the list + // follows new traffic, so flows leave it and join it as they age. Saved ones are counted + // again, as their counts change with no traffic at all. + setInterval(() => { + if (anyCountsBackFromNow(store.get(savedQueriesAtom).map((q) => q.query))) countSaved(); + const s = store.get(searchWindowAtom); + if (s == null || s.loading || !store.get(followingAtom)) return; + if (countsBackFromNow(s.query.query)) loadSearch(s.query, true); + }, AGE_REFRESH_MS); + let activeKey = JSON.stringify(store.get(activeQueryAtom)); store.sub(activeQueryAtom, () => { const query = store.get(activeQueryAtom); @@ -576,7 +703,7 @@ export function syncSidebarSelection(store: Store) { lastRows = key; if (syncing || Date.now() - lastSidebarInput > SIDEBAR_INPUT_MS) return; // Group headings are not filters. - const rows = ids.filter((id) => id !== APPS_ID && id !== DOMAINS_ID); + const rows = ids.filter((id) => id !== APPS_ID && id !== DOMAINS_ID && id !== SAVED_ID); if (rows.length === 0) return; const items = itemsById(store.get(sidebarTreeAtom)); const selected = rows.flatMap((id) => { diff --git a/apps/yaak-proxy/package.json b/apps/yaak-proxy/package.json index 0da1b2bc0..7e160c0b9 100644 --- a/apps/yaak-proxy/package.json +++ b/apps/yaak-proxy/package.json @@ -35,6 +35,7 @@ "babel-plugin-react-compiler": "^1.0.0", "typescript": "^5.8.3", "vite": "npm:@voidzero-dev/vite-plus-core@^1.4.0", + "vite-plugin-wasm": "^3.5.0", "vite-plus": "^1.4.0" } } diff --git a/apps/yaak-proxy/vite.config.ts b/apps/yaak-proxy/vite.config.ts index 3d56934f6..73206609c 100644 --- a/apps/yaak-proxy/vite.config.ts +++ b/apps/yaak-proxy/vite.config.ts @@ -1,9 +1,13 @@ import react from "@vitejs/plugin-react"; import { defineConfig } from "vite"; +import wasm from "vite-plugin-wasm"; export default defineConfig({ - plugins: [react()], + // The search box's query language is a WebAssembly package + plugins: [wasm(), react()], build: { + // Top-level await, which the WebAssembly package loads with + target: "esnext", outDir: "../../dist/apps/yaak-proxy", emptyOutDir: true, }, diff --git a/crates-proxy/yaak-proxy-lib/Cargo.toml b/crates-proxy/yaak-proxy-lib/Cargo.toml index bf52d7d8c..b0112ecb5 100644 --- a/crates-proxy/yaak-proxy-lib/Cargo.toml +++ b/crates-proxy/yaak-proxy-lib/Cargo.toml @@ -36,6 +36,8 @@ yaak-lifecycle = { workspace = true } yaak-models = { workspace = true, features = ["sqlcipher"] } yaak-proxy = { workspace = true } yaak-proxy-helper = { workspace = true, features = ["ts"] } +# The search query language: parsing, the evaluator for live flows, completion. +yaak-query = { workspace = true } yaak-rpc = { workspace = true } yaak-templates = { workspace = true } diff --git a/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts b/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts index 7e5b70058..98ca43b83 100644 --- a/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts +++ b/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts @@ -131,7 +131,12 @@ grpcMessage: string | null, * client rejected the proxy's certificate: method `CONNECT`, its URL the host, its sizes * the bytes each way, and no headers or bodies. Open until `elapsedMs` or `error` is set. */ -tunnelled: boolean, }; +tunnelled: boolean, +/** + * An agent read the flow through the control API: its detail, a body or its messages. + * Never written by an upsert, which keeps what is stored. + */ +agentRead: boolean, }; export type FlowBody = "request" | "response"; @@ -239,10 +244,34 @@ grpcStatus?: number, */ grpcMessage?: string, }; -export type ModelPayload = { model: Flow, change: ModelChangeEvent, }; +export type ModelPayload = { model: Flow, change: ModelChangeEvent, +/** + * Whether the flow belongs in the list of the search the app shows, decided as the event + * is sent. None while no search is shown. + */ +search?: SearchMatch, }; export type ProxyHeader = { name: string, value: string, }; +export type SavedQuery = { id: string, name: string, +/** + * The search, as the search box takes it. + */ +query: string, +/** + * Where it sits in the list, lowest first. + */ +sortOrder: number, }; + +/** + * A written flow, matched against the search the app shows. + */ +export type SearchMatch = { +/** + * The search's key, as the app gave it with the search. + */ +key: string, matches: boolean, }; + /** * The streaming protocol a flow carries. */ diff --git a/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts b/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts index 2e70e0589..d94a860b5 100644 --- a/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts +++ b/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts @@ -1,5 +1,5 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { Flow, FlowBody, FlowMessage, GrpcSchema, ModelPayload } from "./gen_models"; +import type { Flow, FlowBody, FlowMessage, GrpcSchema, ModelPayload, SavedQuery } from "./gen_models"; export type ActionInvocation = { "scope": "global", action: GlobalAction, }; @@ -58,6 +58,10 @@ path: string, */ cwd: string | null, }; +export type CountSavedQueriesRequest = { hideSystem?: boolean, }; + +export type CreateSavedQueryRequest = { name: string, query: string, }; + export type DecodeGrpcMessageRequest = { flowId: string, seq: number, /** * Decode without a schema even when one is found. @@ -111,6 +115,8 @@ entry: string, */ builtIn: boolean, }; +export type DeleteSavedQueryRequest = { id: string, }; + export type DropReason = "hop_by_hop" | "set_by_sending" | "body_decoded"; export type DroppedHeader = { name: string, value: string, reason: DropReason, }; @@ -146,60 +152,19 @@ export type FilterBodyResponse = { content: string, error: string | null, }; export type FilterLanguage = "jsonpath" | "xpath"; /** - * Which flows a listing or a HAR export covers. Every field narrows it; none is all of them. + * Which flows a listing or a HAR export covers: a search, and a page of it. */ export type FlowFilter = { /** - * Search terms, any case, each of which must match the URL (host, path or query), the - * method, the status (`404`, `40`, `4xx`) or the content type: `stripe post 4xx`. + * A search in the query language (`crate::query`): `stripe @status:4xx`, + * `@host:*.github.com -@is:system`. None or blank covers every flow. */ -q?: string, +query?: string, /** - * Part of the host, any case: `stripe` matches `api.stripe.com`. + * Leaves out macOS's own daemons, as hiding System does, unless the query says + * `@is:system` itself. */ -host?: string, -/** - * Only flows under one of these hosts or paths, as the app's sidebar names them: - * `api.stripe.com` or `api.stripe.com/v1/charges`, with any port after the host. A host - * written `*.github.com` is a site: `github.com` and every host below it. - */ -under?: Array, -/** - * Only flows from one of these apps: by bundle id, by path for a process outside an app - * bundle, by name in any case (`slack`), or, for a process outside an app, by the folder - * it ran in (`fieldwork`). - */ -app?: Array, -/** - * Only flows from a process that ran in one of these working directories: the whole path, - * or its folder's name in any case (`fieldwork`). With `app`, a program in a folder. - */ -cwd?: Array, -/** - * Only flows from macOS's own daemons (true), or none of them (false), as [`is_system`] - * tells them by their executable. - */ -system?: boolean, -/** - * Only HTTPS connections tunnelled undecrypted (true), or none of them (false). - */ -tunnelled?: boolean, -/** - * Only flows one of these groups covers, each judged on its own: several sidebar rows - * selected, `@app:Slack @host:a.com OR @app:Chrome @host:b.com`, are two groups and not every - * pairing of their apps and hosts. A group's `after`, `before` and `limit` do not apply. Not - * carried by a query string. - */ -any?: Array, -/** - * The method, any case. - */ -method?: string, -/** - * A status (`404`), a class of them (`4xx`), or `error` for flows that failed, gRPC calls - * that ended with a status other than OK among them. - */ -status?: string, +hideSystem?: boolean, /** * At most this many flows, the newest. */ @@ -382,6 +347,8 @@ messages: Array, */ total: number, }; +export type ListSavedQueriesRequest = Record; + export type ListSeenRequest = Record; export type ListWorkspacesRequest = Record; @@ -392,6 +359,11 @@ export type ListWorkspacesResponse = { */ yaakFound: boolean, workspaces: Array, }; +/** + * Which of some flows a search covers, for flows whose events were matched against another. + */ +export type MatchFlowsRequest = { filter: FlowFilter, ids: Array, }; + /** * A message of a streaming flow was recorded. */ @@ -436,6 +408,30 @@ export type ProxyState = "running" | "stopped"; export type ProxyStatePayload = { state: ProxyState, }; +/** + * A term the query cannot run, which then matches nothing. `start` and `end` count UTF-16 + * code units, as JavaScript does. + */ +export type QueryDiagnostic = { kind: QueryDiagnosticKind, path: string, start: number, end: number, }; + +/** + * What is wrong with a term, for the search box to mark. + */ +export type QueryDiagnosticKind = "unknownField" | "invalidValue" | "notYet"; + +export type QuerySchemaRequest = Record; + +/** + * The search's fields, for the search box's highlighting and completion. + */ +export type QuerySchemaResponse = { schema: import("@yaakapp-internal/query").Schema, }; + +export type QueryValuesRequest = { +/** + * A field's path: `host`, `header` for header names, `header.accept` for a header's values. + */ +path: string, }; + export type ReflectGrpcHostRequest = { /** * `host` or `host:port`, over TLS on 443 unless a port is given, or either after @@ -445,11 +441,13 @@ host: string, }; export type RemoveGrpcSchemaRequest = { id: string, }; +export type RenameSavedQueryRequest = { id: string, name: string, }; + export type ResetCapturedDataRequest = Record; -export type RpcEventSchema = { decrypt_list_changed: DecryptListChangedPayload, flows_cleared: FlowsClearedPayload, flows_removed: FlowsRemovedPayload, message_write: MessagePayload, model_write: ModelPayload, proxy_state_changed: ProxyStatePayload, }; +export type RpcEventSchema = { decrypt_list_changed: DecryptListChangedPayload, flows_cleared: FlowsClearedPayload, flows_removed: FlowsRemovedPayload, message_write: MessagePayload, model_write: ModelPayload, proxy_state_changed: ProxyStatePayload, saved_queries_changed: SavedQueriesChangedPayload, }; -export type RpcSchema = { execute_action: [ActionInvocation, boolean], format_body: [FormatBodyRequest, string], get_decrypt_list: [GetDecryptListRequest, DecryptList], get_message_payload: [GetMessagePayloadRequest, GetMessagePayloadResponse], list_actions: [ListActionsRequest, ListActionsResponse], list_flows: [FlowFilter, ListFlowsResponse], list_grpc_schemas: [ListGrpcSchemasRequest, Array], list_messages: [ListMessagesRequest, ListMessagesResponse], remove_grpc_schema: [RemoveGrpcSchemaRequest, boolean], set_recording: [SetRecordingRequest, boolean], set_system_proxy_at_launch: [SetSystemProxyAtLaunchRequest, boolean], set_http_version: [SetHttpVersionRequest, HttpVersion], set_capture_body_limit: [SetCaptureBodyLimitRequest, number], set_keep_traffic: [SetKeepTrafficRequest, number | null], add_grpc_schema: [AddGrpcSchemaRequest, GrpcSchema], app_icon: [AppIconRequest, string | null], ca_status: [CaStatusRequest, CaStatus], ca_trust: [CaTrustRequest, CaStatus], clear_flows: [ClearFlowsRequest, number], decode_grpc_message: [DecodeGrpcMessageRequest, DecodedGrpcMessage], export_har: [ExportHarRequest, number], filter_body: [FilterBodyRequest, FilterBodyResponse], get_flow_body: [GetFlowBodyRequest, GetFlowBodyResponse], get_proxy_state: [GetProxyStateRequest, GetProxyStateResponse], helper_register: [HelperRegisterRequest, SystemProxyStatus], helper_unregister: [HelperUnregisterRequest, SystemProxyStatus], edit_decrypt_list: [EditDecryptListRequest, DecryptList], list_seen: [ListSeenRequest, SeenTargets], set_decrypt_list: [SetDecryptListRequest, DecryptList], list_workspaces: [ListWorkspacesRequest, ListWorkspacesResponse], preview_save: [PreviewSaveRequest, SavePreview], reflect_grpc_host: [ReflectGrpcHostRequest, GrpcSchema], save_flow: [SaveFlowToYaakRequest, SavedFlow], save_flow_body: [SaveFlowBodyRequest, SaveFlowBodyResponse], system_proxy_set: [SystemProxySetRequest, SystemProxyStatus], system_proxy_status: [SystemProxyStatusRequest, SystemProxyStatus], take_over: [TakeOverRequest, boolean], set_port: [SetPortRequest, number], get_storage_size: [GetStorageSizeRequest, GetStorageSizeResponse], reset_captured_data: [ResetCapturedDataRequest, boolean], open_terminal: [OpenTerminalRequest, boolean], }; +export type RpcSchema = { execute_action: [ActionInvocation, boolean], format_body: [FormatBodyRequest, string], get_decrypt_list: [GetDecryptListRequest, DecryptList], get_message_payload: [GetMessagePayloadRequest, GetMessagePayloadResponse], list_actions: [ListActionsRequest, ListActionsResponse], list_flows: [FlowFilter, ListFlowsResponse], query_schema: [QuerySchemaRequest, QuerySchemaResponse], validate_query: [ValidateQueryRequest, Array], watch_query: [WatchQueryRequest, boolean], list_saved_queries: [ListSavedQueriesRequest, Array], create_saved_query: [CreateSavedQueryRequest, SavedQuery], rename_saved_query: [RenameSavedQueryRequest, SavedQuery], delete_saved_query: [DeleteSavedQueryRequest, boolean], list_grpc_schemas: [ListGrpcSchemasRequest, Array], list_messages: [ListMessagesRequest, ListMessagesResponse], remove_grpc_schema: [RemoveGrpcSchemaRequest, boolean], set_recording: [SetRecordingRequest, boolean], set_system_proxy_at_launch: [SetSystemProxyAtLaunchRequest, boolean], set_http_version: [SetHttpVersionRequest, HttpVersion], set_capture_body_limit: [SetCaptureBodyLimitRequest, number], set_keep_traffic: [SetKeepTrafficRequest, number | null], add_grpc_schema: [AddGrpcSchemaRequest, GrpcSchema], app_icon: [AppIconRequest, string | null], ca_status: [CaStatusRequest, CaStatus], ca_trust: [CaTrustRequest, CaStatus], clear_flows: [ClearFlowsRequest, number], decode_grpc_message: [DecodeGrpcMessageRequest, DecodedGrpcMessage], export_har: [ExportHarRequest, number], filter_body: [FilterBodyRequest, FilterBodyResponse], get_flow_body: [GetFlowBodyRequest, GetFlowBodyResponse], get_proxy_state: [GetProxyStateRequest, GetProxyStateResponse], helper_register: [HelperRegisterRequest, SystemProxyStatus], helper_unregister: [HelperUnregisterRequest, SystemProxyStatus], edit_decrypt_list: [EditDecryptListRequest, DecryptList], list_seen: [ListSeenRequest, SeenTargets], set_decrypt_list: [SetDecryptListRequest, DecryptList], list_workspaces: [ListWorkspacesRequest, ListWorkspacesResponse], preview_save: [PreviewSaveRequest, SavePreview], reflect_grpc_host: [ReflectGrpcHostRequest, GrpcSchema], save_flow: [SaveFlowToYaakRequest, SavedFlow], save_flow_body: [SaveFlowBodyRequest, SaveFlowBodyResponse], system_proxy_set: [SystemProxySetRequest, SystemProxyStatus], system_proxy_status: [SystemProxyStatusRequest, SystemProxyStatus], take_over: [TakeOverRequest, boolean], set_port: [SetPortRequest, number], get_storage_size: [GetStorageSizeRequest, GetStorageSizeResponse], reset_captured_data: [ResetCapturedDataRequest, boolean], open_terminal: [OpenTerminalRequest, boolean], query_values: [QueryValuesRequest, Array], match_flows: [MatchFlowsRequest, Array], count_saved_queries: [CountSavedQueriesRequest, Array], }; export type SaveFlowBodyRequest = { flowId: string, which: FlowBody, /** @@ -513,6 +511,13 @@ authorization: boolean, }; export type SavedFlow = { requestId: string, name: string, }; +export type SavedQueriesChangedPayload = Record; + +/** + * How many flows a saved query covers. + */ +export type SavedQueryCount = { id: string, count: number, }; + export type SeenApp = { name: string, bundleId: string | null, }; /** @@ -591,4 +596,12 @@ export type SystemProxyStatusRequest = Record; export type TakeOverRequest = Record; +export type ValidateQueryRequest = { query: string, }; + +/** + * The search the app shows, which flows written from now on are matched against, their events + * naming `key`. No query stops it. + */ +export type WatchQueryRequest = { key: string, query?: string, hideSystem?: boolean, }; + export type WorkspaceSummary = { id: string, name: string, folders: Array, }; diff --git a/crates-proxy/yaak-proxy-lib/migrations/00000012_search.sql b/crates-proxy/yaak-proxy-lib/migrations/00000012_search.sql new file mode 100644 index 000000000..939b111ae --- /dev/null +++ b/crates-proxy/yaak-proxy-lib/migrations/00000012_search.sql @@ -0,0 +1,13 @@ +-- An agent read the flow through the control API: its detail, a body or its messages. +ALTER TABLE flows ADD COLUMN agent_read BOOLEAN NOT NULL DEFAULT FALSE; + +-- Searches saved by name, listed in the sidebar by `sort_order`. +CREATE TABLE saved_queries +( + id TEXT NOT NULL PRIMARY KEY, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL, + name TEXT NOT NULL, + query TEXT NOT NULL, + sort_order INTEGER NOT NULL DEFAULT 0 +); diff --git a/crates-proxy/yaak-proxy-lib/src/control.rs b/crates-proxy/yaak-proxy-lib/src/control.rs index b6bdb2e23..61b262eb2 100644 --- a/crates-proxy/yaak-proxy-lib/src/control.rs +++ b/crates-proxy/yaak-proxy-lib/src/control.rs @@ -554,7 +554,9 @@ async fn route(state: &ServerState, req: Request) -> Result { found(ctx, id)?; let limit = body_limit(&query)?; - json(&flows::detail(&ctx.db()?, &ctx.blobs()?, id, limit).await?) + let detail = flows::detail(&ctx.db()?, &ctx.blobs()?, id, limit).await?; + ctx.note_agent_read(id); + json(&detail) } (&Method::GET, ["flows", id, "request-body"]) => raw_body(ctx, id, FlowBody::Request).await, (&Method::GET, ["flows", id, "response-body"]) => { @@ -565,6 +567,7 @@ async fn route(state: &ServerState, req: Request) -> Result { @@ -584,6 +587,7 @@ async fn route(state: &ServerState, req: Request) -> Result json(&grpc::list(&ctx.db()?)?), @@ -687,6 +691,7 @@ async fn raw_body(ctx: &ProxyCtx, id: &str, which: FlowBody) -> Result bool { reason.starts_with(KEY_UNAVAILABLE) } -/// The functions filters use: `sidebar_path(url)`, the app sidebar's place for a URL, and -/// `under_site(url, site)`, for filtering by its nodes; `folder_name(dir)`, for naming a program -/// by its folder; and `is_system(bundle_id, path)`, for macOS's own daemons. +/// The functions searches use: `folder_name(dir)`, for naming a program by its folder; +/// `is_system(bundle_id, path)`, for macOS's own daemons; and those the query compiler reads +/// flows' values and tests them with ([`crate::query::add_functions`]). fn add_functions(conn: &mut rusqlite::Connection) -> rusqlite::Result<()> { let flags = FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC; - conn.create_scalar_function("sidebar_path", 1, flags, |ctx| { - Ok(crate::flows::sidebar_path(&ctx.get::(0)?)) - })?; - conn.create_scalar_function("under_site", 2, flags, |ctx| { - Ok(crate::flows::under_site(&ctx.get::(0)?, &ctx.get::(1)?)) - })?; conn.create_scalar_function("folder_name", 1, flags, |ctx| { let dir = ctx.get::>(0)?; Ok(crate::flows::folder_name(dir.as_deref()).map(str::to_string)) @@ -339,7 +333,8 @@ fn add_functions(conn: &mut rusqlite::Connection) -> rusqlite::Result<()> { let bundle_id = ctx.get::>(0)?; let path = ctx.get::>(1)?; Ok(crate::flows::is_system(bundle_id.as_deref(), path.as_deref())) - }) + })?; + crate::query::add_functions(conn) } fn read_only_pool(path: &Path, key: Option<&DatabaseKey>) -> Result { @@ -447,8 +442,7 @@ mod tests { assert!(!flows[0].cancelled); assert_eq!((flows[0].stream_kind, flows[0].message_count), (None, 0)); // Failed flows are found without the gRPC status column too. - let failed = - crate::flows::FlowFilter { status: Some("error".into()), ..Default::default() }; + let failed = crate::flows::FlowFilter::search("@is:error"); assert_eq!(crate::flows::list(&db, &failed).unwrap().len(), 0); // Filters by app work without the folder column, and count without the clear mark. @@ -465,8 +459,7 @@ mod tests { .unwrap(); drop(conn); let db = ProxyQueryManager::open_read_only(&path, Some(&test_key())).unwrap(); - let filter = - crate::flows::FlowFilter { app: Some(vec!["curl".into()]), ..Default::default() }; + let filter = crate::flows::FlowFilter::search("@app:curl"); let (page, total, matching) = crate::flows::page(&db, &filter).unwrap(); assert_eq!((page.len(), total, matching), (1, 1, Some(1))); assert_eq!(page[0].number, 1); @@ -513,7 +506,8 @@ mod tests { "flow_messages", "flow_sweep_pending", "flows", - "grpc_schemas" + "grpc_schemas", + "saved_queries" ] ); @@ -532,6 +526,7 @@ mod tests { "grpc_method", "grpc_status", "grpc_message", + "agent_read", ] { assert!(flow_columns.contains(&column.to_string()), "{column} in {flow_columns:?}"); } @@ -599,7 +594,12 @@ mod tests { .unwrap() .map(|v| v.unwrap()) .collect(); - assert_eq!(&applied[6..], ["00000007", "00000008", "00000009", "00000010", "00000011"]); + assert_eq!( + &applied[6..], + [ + "00000007", "00000008", "00000009", "00000010", "00000011", "00000012" + ] + ); let mark_columns = columns(&conn, "flow_clear_mark"); assert_eq!(mark_columns, ["id", "max_rowid", "cleaner", "cleaner_at"]); drop(conn); @@ -718,7 +718,7 @@ mod tests { }; db.with_tx(|c| c.upsert(&tunnel, &yaak_database::UpdateSource::Background).map(|_| ())) .unwrap(); - let filter = crate::flows::FlowFilter { tunnelled: Some(true), ..Default::default() }; + let filter = crate::flows::FlowFilter::search("@is:tunnelled"); let listed = crate::flows::list(&db, &filter).unwrap(); assert_eq!(listed.iter().map(|f| f.id.as_str()).collect::>(), ["fl_tunnel"]); let old = crate::flows::find(&db, "fl_old").unwrap(); diff --git a/crates-proxy/yaak-proxy-lib/src/flows.rs b/crates-proxy/yaak-proxy-lib/src/flows.rs index 619fddb66..27876ee6c 100644 --- a/crates-proxy/yaak-proxy-lib/src/flows.rs +++ b/crates-proxy/yaak-proxy-lib/src/flows.rs @@ -1,6 +1,7 @@ use crate::bodies; use crate::db::{ProxyQueryManager, Stores}; use crate::models::{Flow, FlowBody}; +use crate::query::{FlowColumns, FlowQuery}; use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64; use rusqlite::types::Value; @@ -9,52 +10,19 @@ use ts_rs::TS; use yaak_database::UpsertModelInfo; use yaak_models::blob_manager::BlobManager; -/// Which flows a listing or a HAR export covers. Every field narrows it; none is all of them. +/// Which flows a listing or a HAR export covers: a search, and a page of it. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, TS)] #[serde(default, rename_all = "camelCase")] #[ts(export, export_to = "gen_rpc.ts")] pub struct FlowFilter { - /// Search terms, any case, each of which must match the URL (host, path or query), the - /// method, the status (`404`, `40`, `4xx`) or the content type: `stripe post 4xx`. + /// A search in the query language (`crate::query`): `stripe @status:4xx`, + /// `@host:*.github.com -@is:system`. None or blank covers every flow. #[ts(optional)] - pub q: Option, - /// Part of the host, any case: `stripe` matches `api.stripe.com`. - #[ts(optional)] - pub host: Option, - /// Only flows under one of these hosts or paths, as the app's sidebar names them: - /// `api.stripe.com` or `api.stripe.com/v1/charges`, with any port after the host. A host - /// written `*.github.com` is a site: `github.com` and every host below it. - #[ts(optional)] - pub under: Option>, - /// Only flows from one of these apps: by bundle id, by path for a process outside an app - /// bundle, by name in any case (`slack`), or, for a process outside an app, by the folder - /// it ran in (`fieldwork`). - #[ts(optional)] - pub app: Option>, - /// Only flows from a process that ran in one of these working directories: the whole path, - /// or its folder's name in any case (`fieldwork`). With `app`, a program in a folder. - #[ts(optional)] - pub cwd: Option>, - /// Only flows from macOS's own daemons (true), or none of them (false), as [`is_system`] - /// tells them by their executable. - #[ts(optional)] - pub system: Option, - /// Only HTTPS connections tunnelled undecrypted (true), or none of them (false). - #[ts(optional)] - pub tunnelled: Option, - /// Only flows one of these groups covers, each judged on its own: several sidebar rows - /// selected, `@app:Slack @host:a.com OR @app:Chrome @host:b.com`, are two groups and not every - /// pairing of their apps and hosts. A group's `after`, `before` and `limit` do not apply. Not - /// carried by a query string. - #[ts(optional)] - pub any: Option>, - /// The method, any case. - #[ts(optional)] - pub method: Option, - /// A status (`404`), a class of them (`4xx`), or `error` for flows that failed, gRPC calls - /// that ended with a status other than OK among them. - #[ts(optional)] - pub status: Option, + pub query: Option, + /// Leaves out macOS's own daemons, as hiding System does, unless the query says + /// `@is:system` itself. + #[ts(optional, as = "Option")] + pub hide_system: bool, /// At most this many flows, the newest. #[ts(optional, type = "number")] pub limit: Option, @@ -69,30 +37,24 @@ pub struct FlowFilter { } impl FlowFilter { - /// The filter from a query string's `q`, `host`, `under`, `app`, `cwd`, `system`, `method`, - /// `status`, `limit`, `after` and `before`. `under`, `app` and `cwd` may repeat. + /// A filter for a search alone. + pub fn search(query: impl Into) -> Self { + Self { query: Some(query.into()), ..Self::default() } + } + + /// The filter from a query string's `query`, `hide_system`, `limit`, `after` and `before`. pub fn from_query(query: &str) -> Result { let mut filter = Self::default(); for (key, value) in url::form_urlencoded::parse(query.as_bytes()) { let value = Some(value.into_owned()).filter(|v| !v.is_empty()); match key.as_ref() { - "q" => filter.q = value, - "host" => filter.host = value, - "under" => filter.under.get_or_insert_default().extend(value), - "app" => filter.app.get_or_insert_default().extend(value), - "cwd" => filter.cwd.get_or_insert_default().extend(value), - "system" => { - filter.system = value - .map(|v| v.parse().map_err(|_| format!("Invalid system {v:?}"))) + "query" => filter.query = value, + "hide_system" => { + filter.hide_system = value + .map(|v| v.parse().map_err(|_| format!("Invalid hide_system {v:?}"))) .transpose()? + .unwrap_or_default() } - "tunnelled" => { - filter.tunnelled = value - .map(|v| v.parse().map_err(|_| format!("Invalid tunnelled {v:?}"))) - .transpose()? - } - "method" => filter.method = value, - "status" => filter.status = value, "after" => filter.after = value, "before" => filter.before = value, "limit" => { @@ -109,30 +71,10 @@ impl FlowFilter { pub fn to_query(&self) -> String { let mut query = url::form_urlencoded::Serializer::new(String::new()); let limit = self.limit.map(|l| l.to_string()); - let system = self.system.map(|s| s.to_string()); - let tunnelled = self.tunnelled.map(|t| t.to_string()); + let hide_system = self.hide_system.then(|| "true".to_string()); for (key, value) in [ - ("q", &self.q), - ("host", &self.host), - ("system", &system), - ("tunnelled", &tunnelled), - ] { - if let Some(value) = value { - query.append_pair(key, value); - } - } - for (key, values) in [ - ("under", &self.under), - ("app", &self.app), - ("cwd", &self.cwd), - ] { - for value in values.iter().flatten() { - query.append_pair(key, value); - } - } - for (key, value) in [ - ("method", &self.method), - ("status", &self.status), + ("query", &self.query), + ("hide_system", &hide_system), ("limit", &limit), ("after", &self.after), ("before", &self.before), @@ -144,166 +86,21 @@ impl FlowFilter { query.finish() } + /// The search, prepared to run now. + pub fn flow_query(&self) -> FlowQuery { + FlowQuery::new(self.query.as_deref().unwrap_or_default(), self.hide_system) + } + /// Narrows more than `after`, `before` and `limit` do, so it covers fewer than all flows. pub fn narrows(&self) -> bool { - let given = |v: &Option| v.as_deref().is_some_and(|v| !v.trim().is_empty()); - given(&self.q) - || given(&self.host) - || given(&self.method) - || given(&self.status) - || self.under.is_some() - || self.app.is_some() - || self.cwd.is_some() - || self.system.is_some() - || self.tunnelled.is_some() - || self.any.as_ref().is_some_and(|groups| groups.iter().any(FlowFilter::narrows)) + !self.flow_query().is_empty() } - /// The SQL condition for every field but `after`, `before` and `limit`, and its parameters, - /// for the database `conn` reads, which may be from before the folder or gRPC columns, when - /// read without migrating. + /// The SQL condition for the search, and its parameters, for the database `conn` reads, + /// which may be from before later columns, when read without migrating. fn condition_in(&self, conn: &rusqlite::Connection) -> (String, Vec) { - let columns = Columns { - cwd: has_column(conn, "client_cwd"), - grpc: has_column(conn, "grpc_status"), - tunnelled: has_column(conn, "tunnelled"), - }; - self.condition(columns) + self.flow_query().sql(&FlowColumns::of(conn)) } - - /// [`Self::condition_in`], told which of the later columns the database has. - fn condition(&self, columns: Columns) -> (String, Vec) { - let Columns { cwd: has_cwd, grpc, tunnelled } = columns; - let mut sql = String::from("1"); - let mut params: Vec = Vec::new(); - for term in self.q.iter().flat_map(|q| q.split_whitespace()) { - let term = term.to_lowercase(); - sql.push_str( - " AND (instr(lower(url), ?) > 0 OR lower(method) = ? \ - OR instr(lower(content_type), ?) > 0", - ); - params.extend([ - term.clone().into(), - term.clone().into(), - term.clone().into(), - ]); - if let Some(status) = status_condition(&term, &mut params) { - sql.push_str(" OR "); - sql.push_str(&status); - } - sql.push(')'); - } - if let Some(host) = self.host.as_deref().filter(|h| !h.is_empty()) { - sql.push_str(&format!(" AND instr(lower({HOST_SQL}), ?) > 0")); - params.push(host.to_lowercase().into()); - } - if let Some(under) = &self.under { - // The node itself, or anything below it, by the sidebar's rules. - let mut any = Vec::new(); - for node in under { - if let Some(site) = site_of_node(node) { - any.push("under_site(url, ?)"); - params.push(site.into()); - continue; - } - let node = normalize_node(node); - any.push("(sidebar_path(url) = ? OR substr(sidebar_path(url), 1, ?) = ?)"); - let below = format!("{node}/"); - params.extend([ - node.into(), - (below.chars().count() as i64).into(), - below.into(), - ]); - } - let any = if any.is_empty() { "0".to_string() } else { any.join(" OR ") }; - sql.push_str(&format!(" AND ({any})")); - } - if let Some(apps) = &self.app { - let marks = vec!["?"; apps.len()].join(", "); - sql.push_str(&format!( - " AND (coalesce(client_bundle_id, client_path) IN ({marks}) \ - OR lower(client_name) IN ({marks})" - )); - params.extend(apps.iter().map(|a| a.clone().into())); - params.extend(apps.iter().map(|a| a.to_lowercase().into())); - if has_cwd { - sql.push_str(&format!( - " OR (client_bundle_id IS NULL AND lower(folder_name(client_cwd)) IN ({marks}))" - )); - params.extend(apps.iter().map(|a| a.to_lowercase().into())); - } - sql.push(')'); - } - if let Some(cwds) = &self.cwd { - if has_cwd && !cwds.is_empty() { - let marks = vec!["?"; cwds.len()].join(", "); - sql.push_str(&format!( - " AND (client_cwd IN ({marks}) OR lower(folder_name(client_cwd)) IN ({marks}))" - )); - params.extend(cwds.iter().map(|c| c.clone().into())); - params.extend(cwds.iter().map(|c| c.to_lowercase().into())); - } else { - // None, or a database from before the column, which has no folders. - sql.push_str(" AND 0"); - } - } - match self.system { - Some(true) => sql.push_str(" AND is_system(client_bundle_id, client_path)"), - Some(false) => sql.push_str(" AND NOT is_system(client_bundle_id, client_path)"), - None => {} - } - match (self.tunnelled, tunnelled) { - (Some(true), true) => sql.push_str(" AND tunnelled"), - (Some(false), true) => sql.push_str(" AND NOT tunnelled"), - // A database from before tunnelled flows has none. - (Some(true), false) => sql.push_str(" AND 0"), - _ => {} - } - // Each group whole, then any of them. - if let Some(groups) = self.any.as_ref().filter(|g| !g.is_empty()) { - let mut any = Vec::new(); - for group in groups { - let (condition, group_params) = group.condition(columns); - any.push(format!("({condition})")); - params.extend(group_params); - } - sql.push_str(&format!(" AND ({})", any.join(" OR "))); - } - if let Some(method) = self.method.as_deref().filter(|m| !m.is_empty()) { - sql.push_str(" AND lower(method) = ?"); - params.push(method.to_lowercase().into()); - } - match self.status.as_deref().map(str::trim) { - None | Some("") => {} - // A gRPC call that ended with a status other than OK failed too. - Some(s) if s.eq_ignore_ascii_case("error") && grpc => { - sql.push_str(" AND (error IS NOT NULL OR coalesce(grpc_status, 0) != 0)") - } - Some(s) if s.eq_ignore_ascii_case("error") => sql.push_str(" AND error IS NOT NULL"), - // A whole status or a class, not the start of one as a search term may be. - Some(s) if s.len() == 3 => match status_condition(&s.to_ascii_lowercase(), &mut params) - { - Some(status) => { - sql.push_str(" AND "); - sql.push_str(&status); - } - None => sql.push_str(" AND 0"), - }, - Some(_) => sql.push_str(" AND 0"), - } - (sql, params) - } -} - -/// Which of the columns later migrations added a database has, as one opened read-only before -/// they ran may not. -#[derive(Clone, Copy)] -struct Columns { - /// The folder a program ran in. - cwd: bool, - /// gRPC statuses. - grpc: bool, - tunnelled: bool, } /// The host and port of `url`, in SQL: what follows `://` up to the first `/`, `?` or `#`. @@ -341,38 +138,6 @@ pub fn sidebar_path(url: &str) -> String { key } -/// A sidebar node as `sidebar_path` writes it, its host lowercased. -fn normalize_node(node: &str) -> String { - match node.split_once('/') { - Some((host, path)) => format!("{}/{path}", host.to_ascii_lowercase()), - None => node.to_ascii_lowercase(), - } -} - -/// The site a sidebar node names, `github.com` for `*.github.com`, lowercased. None for a host or -/// a path. -fn site_of_node(node: &str) -> Option { - let site = node.strip_prefix("*.")?; - (!site.is_empty() && !site.contains('/')).then(|| site.to_ascii_lowercase()) -} - -/// `url`'s host, without its port, is `site` or a host below it: `api.github.com` is under -/// `github.com`. -pub fn under_site(url: &str, site: &str) -> bool { - let path = sidebar_path(url); - let host = path.split('/').next().unwrap_or_default(); - let host = strip_port(host); - host == site || host.strip_suffix(site).is_some_and(|rest| rest.ends_with('.')) -} - -/// `host` without a `:port` after it, keeping an IPv6 address's brackets. -fn strip_port(host: &str) -> &str { - if host.starts_with('[') { - return host.find(']').map_or(host, |end| &host[..=end]); - } - host.rsplit_once(':').map_or(host, |(h, _)| h) -} - /// The last part of a directory's path, `fieldwork` for `/Users/me/src/fieldwork`. None for the /// root, where daemons run, or no path. pub fn folder_name(path: Option<&str>) -> Option<&str> { @@ -418,26 +183,6 @@ pub fn is_system(bundle_id: Option<&str>, path: Option<&str>) -> bool { SYSTEM_DIRS.iter().any(|dir| path.starts_with(dir)) || SYSTEM_PROGRAMS.contains(&name) } -/// The SQL condition for a status term: `404` exactly, `40` as the start of one, or a class -/// such as `4xx`. None when the term is not one. -fn status_condition(term: &str, params: &mut Vec) -> Option { - if let Some(class) = term.strip_suffix("xx") - && let Ok(class @ 1..=9) = class.parse::() - { - params.push(class.into()); - return Some("status / 100 = ?".into()); - } - if term.is_empty() || term.len() > 3 || !term.chars().all(|c| c.is_ascii_digit()) { - return None; - } - if term.len() == 3 { - params.push(term.parse::().ok()?.into()); - return Some("status = ?".into()); - } - params.push(format!("{term}%").into()); - Some("CAST(status AS TEXT) LIKE ?".into()) -} - /// The flows `filter` covers, newest first: by start, then by id. Reads the table in that order /// through its index and stops at the limit, so a page of a common match costs the same however /// many flows there are. @@ -462,13 +207,6 @@ pub fn page( }) } -/// The database has the gRPC status column, added by a later migration. -/// The flows table has `column`: a database opened read-only before its migrations have run may -/// not. -fn has_column(conn: &rusqlite::Connection, column: &str) -> bool { - conn.prepare(&format!("SELECT {column} FROM flows LIMIT 0")).is_ok() -} - fn list_in(conn: &rusqlite::Connection, filter: &FlowFilter) -> Result, String> { let (condition, mut params) = filter.condition_in(conn); // The rowid, for the flow's number. @@ -511,6 +249,40 @@ fn count_matching(conn: &rusqlite::Connection, filter: &FlowFilter) -> Result Result, String> { + db.with_conn(|c| { + let tx = c.conn().resolve().unchecked_transaction().map_err(|e| e.to_string())?; + let counts = filters.iter().map(|f| count_matching(&tx, f)).collect::>()?; + tx.commit().map_err(|e| e.to_string())?; + Ok(counts) + }) +} + +/// The flows of `ids` that `filter`'s search covers, in any order. Cleared ones are left out. +pub fn matching_ids( + db: &ProxyQueryManager, + filter: &FlowFilter, + ids: &[String], +) -> Result, String> { + if ids.is_empty() { + return Ok(Vec::new()); + } + db.with_conn(|c| { + let conn = c.conn().resolve(); + let (condition, mut params) = filter.condition_in(conn); + let marks = vec!["?"; ids.len()].join(", "); + let mut sql = format!("SELECT id FROM flows WHERE ({condition}) AND id IN ({marks})"); + params.extend(ids.iter().map(|id| Value::from(id.clone()))); + push_clear_mark(conn, &mut sql, &mut params); + let mut stmt = conn.prepare(&sql).map_err(|e| e.to_string())?; + let rows = stmt + .query_map(rusqlite::params_from_iter(params), |r| r.get(0)) + .map_err(|e| e.to_string())?; + rows.collect::, _>>().map_err(|e| e.to_string()) + }) +} + /// Leaves out cleared flows, when a clear is pending; otherwise adds nothing, so SQLite keeps its /// cheapest plan. fn push_clear_mark(conn: &rusqlite::Connection, sql: &mut String, params: &mut Vec) { @@ -1031,6 +803,13 @@ mod tests { list(db, &filter).unwrap().into_iter().map(|f| f.url).collect() } + /// The urls a search covers, sorted. + fn found(db: &ProxyQueryManager, query: &str) -> Vec { + let mut found = urls(db, FlowFilter::search(query)); + found.sort(); + found + } + #[test] fn filters() { let (_dir, db) = db_of(vec![ @@ -1038,18 +817,20 @@ mod tests { flow("http://example.com/missing", "GET", Some(404), None), flow("https://down.example.com/", "GET", None, Some("refused")), ]); - let by = |filter: FlowFilter| urls(&db, filter).len(); - assert_eq!(by(FlowFilter::default()), 3); - assert_eq!(by(FlowFilter { host: Some("STRIPE".into()), ..Default::default() }), 1); - assert_eq!(by(FlowFilter { host: Some("example.com".into()), ..Default::default() }), 2); - assert_eq!(by(FlowFilter { host: Some("charges".into()), ..Default::default() }), 0); - assert_eq!(by(FlowFilter { host: Some("8443".into()), ..Default::default() }), 1); - assert_eq!(by(FlowFilter { method: Some("get".into()), ..Default::default() }), 2); - assert_eq!(by(FlowFilter { status: Some("404".into()), ..Default::default() }), 1); - assert_eq!(by(FlowFilter { status: Some("2xx".into()), ..Default::default() }), 1); - assert_eq!(by(FlowFilter { status: Some("5XX".into()), ..Default::default() }), 0); - assert_eq!(by(FlowFilter { status: Some("40".into()), ..Default::default() }), 0); - assert_eq!(by(FlowFilter { status: Some("error".into()), ..Default::default() }), 1); + let by = |query: &str| urls(&db, FlowFilter::search(query)).len(); + assert_eq!(urls(&db, FlowFilter::default()).len(), 3); + // A host whole, with or without its port; a part of one with wildcards. + assert_eq!(by("@host:API.STRIPE.COM"), 1); + assert_eq!(by("@host:api.stripe.com:8443"), 1); + assert_eq!(by("@host:stripe"), 0); + assert_eq!(by("@host:*stripe*"), 1); + assert_eq!(by("@host:*.example.com"), 2); + assert_eq!(by("@method:get"), 2); + assert_eq!(by("@status:404"), 1); + assert_eq!(by("@status:2xx"), 1); + assert_eq!(by("@status:5XX"), 0); + assert_eq!(by("@status:40"), 0); + assert_eq!(by("@is:error"), 1); } #[test] @@ -1067,12 +848,7 @@ mod tests { }, flow("http://localhost:3000/health", "HEAD", Some(503), None), ]); - let search = |q: &str| { - let filter = FlowFilter { q: Some(q.into()), ..Default::default() }; - let mut found = urls(&db, filter); - found.sort(); - found - }; + let search = |q: &str| found(&db, q); let stripe = "https://api.stripe.com/v1/charges?customer=cus_9"; let github = "https://api.github.com/repos/yaak"; let logo = "https://images.ctfassets.net/logo.png"; @@ -1081,28 +857,28 @@ mod tests { assert_eq!(search("STRIPE"), vec![stripe]); assert_eq!(search("/repos/"), vec![github]); assert_eq!(search("cus_9"), vec![stripe]); - // The method, whole. + // The method and the status, any part of them. assert_eq!(search("head"), vec![health]); - assert_eq!(search("ead"), Vec::<&str>::new()); - // A status, its start, or its class. + assert_eq!(search("ead"), vec![health]); assert_eq!(search("404"), vec![github]); assert_eq!(search("50"), vec![health]); - assert_eq!(search("2xx"), vec![stripe, logo]); // The content type. assert_eq!(search("png"), vec![logo]); assert_eq!(search("json"), vec![github, stripe]); // Every term must match, each anywhere. assert_eq!(search("json get"), vec![github]); - assert_eq!(search(" json 2xx "), vec![stripe]); - assert_eq!(search("json 5xx"), Vec::<&str>::new()); + assert_eq!(search(" json @status:2xx "), vec![stripe]); + assert_eq!(search("json @status:5xx"), Vec::<&str>::new()); // `%` and `_` are plain characters, not patterns. assert_eq!(search("%"), Vec::<&str>::new()); assert_eq!(search("cus_"), vec![stripe]); assert_eq!(search(""), vec![health, github, stripe, logo]); } + /// The terms a sidebar row writes cover what is under it: a host, with any port, and a + /// path with what is below it. #[test] - fn under_matches_a_host_or_a_path_as_the_sidebar_does() { + fn hosts_and_paths_as_the_sidebar_writes_them() { let (_dir, db) = db_of(vec![ flow("https://api.stripe.com/v1/charges", "GET", Some(200), None), flow("https://api.stripe.com/v1/charges/ch_1?expand=x", "GET", Some(200), None), @@ -1110,29 +886,38 @@ mod tests { flow("https://api.stripe.com:8443/v1/charges", "GET", Some(200), None), flow("http://api.stripe.com", "GET", Some(200), None), flow("https://api.stripe.community/", "GET", Some(200), None), - flow("https://example.com/v1//users", "GET", Some(200), None), flow("http://x.com:80/a", "GET", Some(200), None), flow("https://x.com:443/a", "GET", Some(200), None), + flow("https://example.com/v1//users", "GET", Some(200), None), ]); - let under = |nodes: &[&str]| { - let nodes = nodes.iter().map(|n| n.to_string()).collect(); - urls(&db, FlowFilter { under: Some(nodes), ..Default::default() }).len() - }; - // A host on another port is a node of its own, as in the sidebar. - assert_eq!(under(&["api.stripe.com"]), 4); - assert_eq!(under(&["api.stripe.com:8443"]), 1); - assert_eq!(under(&["api.stripe.com/v1/charges"]), 2); - // Empty segments and default ports are left out, as the sidebar leaves them. - assert_eq!(under(&["example.com/v1/users"]), 1); - assert_eq!(under(&["x.com/a"]), 2); - assert_eq!(under(&["api.stripe.com/v1/charges", "example.com"]), 3); - assert_eq!(under(&[]), 0); - // The sidebar lowercases hosts; paths keep their case. - assert_eq!(under(&["API.stripe.com/v1/charges"]), 2); - assert_eq!(under(&["api.stripe.com/V1/charges"]), 0); + let count = |query: &str| found(&db, query).len(); + assert_eq!(count("@host:api.stripe.com"), 5); + assert_eq!(count("@host:api.stripe.com:8443"), 1); + assert_eq!(count("@host:api.stripe.com @path:/v1/charges/*"), 3); + assert_eq!(count("@host:x.com @path:/a"), 2); + assert_eq!(count("@host:api.stripe.com @path:/v1/charges/* OR @host:x.com"), 5); + assert_eq!(count("@host:API.stripe.com @path:/V1/charges/*"), 3); + // A path with empty segments is found by the sidebar's row for it. + assert_eq!(count("@host:example.com @path:/v1/users/*"), 1); } /// The cases `lib/search.test.ts` checks `sidebarPath` against, so the two agree. + /// A search with `OR` at its top still pages and leaves out cleared flows. + #[test] + fn a_top_level_or_keeps_to_the_page_and_the_clear() { + let (_dir, db) = db_of(vec![ + flow("https://a.com/1", "GET", Some(200), None), + flow("https://a.com/2", "GET", Some(404), None), + flow("https://a.com/3", "GET", Some(200), None), + ]); + let query = "@status:200 OR @status:404"; + let before = FlowFilter { before: Some("fl_001".into()), ..FlowFilter::search(query) }; + assert_eq!(urls(&db, before), vec!["https://a.com/1"]); + clear(&db).unwrap(); + assert!(urls(&db, FlowFilter::search(query)).is_empty()); + assert_eq!(page(&db, &FlowFilter::search(query)).unwrap().2, Some(0)); + } + #[test] fn sidebar_path_rules() { for (url, path) in [ @@ -1163,18 +948,15 @@ mod tests { from(None, "/usr/bin/curl"), flow("https://example.com/", "GET", Some(200), None), ]); - let app = |apps: &[&str]| { - let apps = apps.iter().map(|a| a.to_string()).collect(); - urls(&db, FlowFilter { app: Some(apps), ..Default::default() }).len() - }; - assert_eq!(app(&["com.google.Chrome"]), 2); - assert_eq!(app(&["/usr/bin/curl"]), 1); - assert_eq!(app(&["/Applications/Google Chrome.app"]), 0); - assert_eq!(app(&["com.google.Chrome", "/usr/bin/curl"]), 3); + let app = |query: &str| found(&db, query).len(); + assert_eq!(app("@app:com.google.Chrome"), 2); + assert_eq!(app("@app:/usr/bin/curl"), 1); + assert_eq!(app("@app:\"/Applications/Google Chrome.app\""), 0); + assert_eq!(app("@app:com.google.Chrome OR @app:/usr/bin/curl"), 3); } - /// A program in a folder: its working directory by the whole path, or its folder's name in - /// any case, along with whatever else the filter asks. + /// A program in a folder: its working directory by the whole path, or its folder's name, + /// any case, along with whatever else the search asks. #[test] fn cwd_matches_a_path_or_a_folder_name() { let node_in = |cwd: Option<&str>| Flow { @@ -1191,22 +973,14 @@ mod tests { Flow { client_cwd: Some("/tmp/fieldwork".into()), ..node_in(None) }, Flow { client_path: Some("/usr/bin/curl".into()), ..node_in(Some("/tmp/ledger")) }, ]); - let count = |app: Option<&str>, cwds: &[&str]| { - let cwd = Some(cwds.iter().map(|c| c.to_string()).collect()); - let app = app.map(|a| vec![a.to_string()]); - urls(&db, FlowFilter { app, cwd, ..Default::default() }).len() - }; - assert_eq!(count(None, &["/Users/ada/Workspace/fieldwork"]), 2, "the whole path"); - assert_eq!(count(None, &["/users/ada/workspace/fieldwork"]), 0, "a path exactly"); - assert_eq!(count(None, &["FieldWork"]), 3, "a folder name, any case"); - assert_eq!(count(None, &["fieldwork", "ledger"]), 5, "any of them"); - assert_eq!(count(Some("/usr/local/bin/node"), &["ledger"]), 1, "and the program"); - assert_eq!(count(Some("/usr/bin/curl"), &["ledger"]), 1); - assert_eq!(count(None, &[]), 0); - let query = FlowFilter { cwd: Some(vec!["a".into(), "/b".into()]), ..Default::default() }; - assert_eq!(query.to_query(), "cwd=a&cwd=%2Fb"); - assert_eq!(FlowFilter::from_query(&query.to_query()).unwrap(), query); - assert!(query.narrows()); + let count = |query: &str| found(&db, query).len(); + assert_eq!(count("@cwd:/Users/ada/Workspace/fieldwork"), 2, "the whole path"); + assert_eq!(count("@cwd:/users/ada/workspace/fieldwork"), 2, "any case"); + assert_eq!(count("@cwd:FieldWork"), 3, "a folder name"); + assert_eq!(count("@cwd:fieldwork OR @cwd:ledger"), 5, "any of them"); + assert_eq!(count("@app:/usr/local/bin/node @cwd:ledger"), 1, "and the program"); + assert_eq!(count("@app:/usr/bin/curl @cwd:ledger"), 1); + assert_eq!(count("@cwd:work"), 0, "whole"); } #[test] @@ -1221,7 +995,7 @@ mod tests { .collect(), ); let search = |before: Option| FlowFilter { - q: Some("post".into()), + query: Some("post".into()), before, limit: Some(4), ..Default::default() @@ -1245,8 +1019,7 @@ mod tests { let (_dir, db) = db_of(vec![]); let plan: Vec = db.with_conn(|c| { let conn = c.conn().resolve(); - let (condition, params) = - FlowFilter { q: Some("stripe".into()), ..Default::default() }.condition_in(conn); + let (condition, params) = FlowFilter::search("stripe @status:>=400").condition_in(conn); let sql = format!( "EXPLAIN QUERY PLAN SELECT * FROM flows WHERE {condition} \ ORDER BY started_at DESC, id DESC LIMIT 500" @@ -1322,7 +1095,7 @@ mod tests { // The limit counts flows the filter keeps. let ok = FlowFilter { before: Some("fl_e".into()), - status: Some("2xx".into()), + query: Some("@status:2xx".into()), limit: Some(2), ..Default::default() }; @@ -1380,7 +1153,7 @@ mod tests { write_flow("fl_d"); assert_eq!(ids(&FlowFilter::default()), vec!["fl_d"]); assert_eq!(find(&db, "fl_d").map(|f| f.number), Some(4)); - let narrowed = FlowFilter { q: Some("example".into()), ..Default::default() }; + let narrowed = FlowFilter::search("example"); assert_eq!(page(&db, &narrowed).unwrap().1, 1); // Bodies that fail to delete keep their flows, to delete again. @@ -1540,49 +1313,39 @@ mod tests { ids.sort(); ids }; - let app = |a: &str| FlowFilter { app: Some(vec![a.into()]), ..Default::default() }; - assert_eq!(ids(app("slack")), vec!["fl_slack"]); - assert_eq!(ids(app("com.tinyspeck.slackmacgap")), vec!["fl_slack"]); - assert_eq!(ids(app("Fieldwork")), vec!["fl_node"]); - assert_eq!(ids(app("/opt/homebrew/bin/node")), vec!["fl_node"]); - let under = |n: &str| FlowFilter { under: Some(vec![n.into()]), ..Default::default() }; - assert_eq!(ids(under("*.github.com")), vec!["fl_daemon", "fl_node"]); - assert_eq!(ids(under("api.github.com")), vec!["fl_node"]); - let system = |s| FlowFilter { system: Some(s), ..Default::default() }; - assert_eq!(ids(system(true)), vec!["fl_daemon"]); - assert_eq!(ids(system(false)), vec!["fl_node", "fl_none", "fl_slack"]); + let ids = |query: &str| ids(FlowFilter::search(query)); + assert_eq!(ids("@app:slack"), vec!["fl_slack"]); + assert_eq!(ids("@app:com.tinyspeck.slackmacgap"), vec!["fl_slack"]); + assert_eq!(ids("@app:Fieldwork"), vec!["fl_node"]); + assert_eq!(ids("@app:/opt/homebrew/bin/node"), vec!["fl_node"]); + assert_eq!(ids("@host:*.github.com"), vec!["fl_daemon", "fl_node"]); + assert_eq!(ids("@host:api.github.com"), vec!["fl_node"]); + assert_eq!(ids("@is:system"), vec!["fl_daemon"]); + assert_eq!(ids("-@is:system"), vec!["fl_node", "fl_none", "fl_slack"]); // Groups, as two sidebar rows under different apps write them, each judged whole: // flattened, the apps and hosts would pair across rows and match both flows. - let group = |a: &str, n: &str| FlowFilter { - app: Some(vec![a.into()]), - under: Some(vec![n.into()]), - ..Default::default() - }; - let crossed = FlowFilter { - any: Some(vec![ - group("fieldwork", "github.com"), - group("nsurlsessiond", "api.github.com"), - ]), - ..Default::default() - }; - assert_eq!(ids(crossed.clone()), Vec::::new()); - assert_eq!(page(&db, &crossed).unwrap().2, Some(0)); - let flattened = FlowFilter { - app: Some(vec!["fieldwork".into(), "nsurlsessiond".into()]), - under: Some(vec!["github.com".into(), "api.github.com".into()]), - ..Default::default() - }; + let crossed = "@app:fieldwork @host:github.com OR @app:nsurlsessiond @host:api.github.com"; + assert_eq!(ids(crossed), Vec::::new()); + assert_eq!(page(&db, &FlowFilter::search(crossed)).unwrap().2, Some(0)); + let flattened = "(@app:fieldwork OR @app:nsurlsessiond) \ + (@host:github.com OR @host:api.github.com)"; assert_eq!(ids(flattened), vec!["fl_daemon", "fl_node"]); - let either = FlowFilter { - any: Some(vec![ - group("fieldwork", "api.github.com"), - FlowFilter { app: Some(vec!["slack".into()]), ..Default::default() }, - ]), - q: Some("https".into()), - ..Default::default() - }; + let either = "(@app:fieldwork @host:api.github.com OR @app:slack) https"; assert_eq!(ids(either), vec!["fl_node", "fl_slack"]); + + // Hiding System leaves out its daemons, unless the search names them. + let hidden = |query: &str| { + let filter = FlowFilter { hide_system: true, ..FlowFilter::search(query) }; + let mut ids: Vec = + list(&db, &filter).unwrap().into_iter().map(|f| f.id).collect(); + ids.sort(); + ids + }; + assert_eq!(hidden(""), vec!["fl_node", "fl_none", "fl_slack"]); + assert_eq!(hidden("@host:*.github.com"), vec!["fl_node"]); + assert_eq!(hidden("@is:system"), vec!["fl_daemon"]); + assert_eq!(hidden("@host:*.github.com OR @is:SYSTEM"), vec!["fl_daemon", "fl_node"]); } #[test] @@ -1603,35 +1366,24 @@ mod tests { assert!(!is_system(None, None)); assert_eq!(folder_name(Some("/Users/me/src/fieldwork/")), Some("fieldwork")); assert_eq!(folder_name(Some("/")), None); - assert!(under_site("https://api.github.com:8443/x", "github.com")); - assert!(!under_site("https://notgithub.com/", "github.com")); } #[test] fn query_round_trip() { let filter = FlowFilter { - q: Some("post json".into()), - host: Some("api.stripe.com".into()), - under: Some(vec!["api.stripe.com/v1".into(), "example.com".into()]), - app: Some(vec!["com.google.Chrome".into()]), - status: Some("4xx".into()), + query: Some("post @status:4xx".into()), + hide_system: true, limit: Some(20), ..Default::default() }; - assert_eq!( - filter.to_query(), - "q=post+json&host=api.stripe.com&under=api.stripe.com%2Fv1&under=example.com\ - &app=com.google.Chrome&status=4xx&limit=20" - ); + assert_eq!(filter.to_query(), "query=post+%40status%3A4xx&hide_system=true&limit=20"); assert_eq!(FlowFilter::from_query(&filter.to_query()).unwrap(), filter); assert_eq!(FlowFilter::from_query("").unwrap(), FlowFilter::default()); assert!(FlowFilter::from_query("limit=lots").is_err()); - - let tunnelled = FlowFilter { tunnelled: Some(true), ..Default::default() }; - assert_eq!(tunnelled.to_query(), "tunnelled=true"); - assert_eq!(FlowFilter::from_query("tunnelled=true").unwrap(), tunnelled); - assert!(tunnelled.narrows()); - assert!(FlowFilter::from_query("tunnelled=maybe").is_err()); + assert!(FlowFilter::from_query("hide_system=maybe").is_err()); + assert!(filter.narrows()); + assert!(FlowFilter { hide_system: true, ..Default::default() }.narrows()); + assert!(!FlowFilter::search(" ").narrows()); } #[test] diff --git a/crates-proxy/yaak-proxy-lib/src/lib.rs b/crates-proxy/yaak-proxy-lib/src/lib.rs index 80e16fb68..fc935c80d 100644 --- a/crates-proxy/yaak-proxy-lib/src/lib.rs +++ b/crates-proxy/yaak-proxy-lib/src/lib.rs @@ -14,7 +14,9 @@ pub mod helper; pub mod macos; pub mod messages; pub mod models; +pub mod query; pub mod save; +pub mod saved_queries; pub mod settings; pub mod terminal; @@ -47,7 +49,7 @@ use std::time::{Duration, SystemTime}; use ts_rs::TS; pub use yaak_crypto::MasterKeySource; use yaak_crypto::data_key::DataKey; -use yaak_database::{ModelChangeEvent, UpdateSource, UpsertModelInfo}; +use yaak_database::{UpdateSource, UpsertModelInfo}; pub use yaak_grpc::GrpcConfig; use yaak_models::blob_manager::{BlobManager, BodyChunkWriter}; use yaak_proxy::{CaStore, KeyCipher, ProxyConfig, ProxyEvent, ProxyHandle}; @@ -107,6 +109,8 @@ pub struct ProxyCtx { pub events: RpcEventEmitter, /// gRPC schemas as decoded, and the protoc that compiles attached protos. pub grpc: GrpcState, + /// The search the app shows, which written flows' events are matched against. + live_search: Arc, } impl ProxyCtx { @@ -148,6 +152,7 @@ impl ProxyCtx { sweeper: Default::default(), events, grpc: GrpcState::default(), + live_search: Default::default(), }; if db::needs_encrypting(data_dir) { let unlocking = ctx.clone(); @@ -418,6 +423,7 @@ impl ProxyCtx { db: self.db()?, blobs: self.blobs()?, events: self.events.clone(), + live_search: self.live_search.clone(), attribution, recording: self.recording.clone(), finished: Default::default(), @@ -1118,16 +1124,38 @@ impl ProxyCtx { } let (flow, created) = tx.upsert(&flow, &UpdateSource::Background)?; // Sent before the commit, so it cannot reach the app after a clear that follows. - self.events.emit( - "model_write", - &ModelPayload { model: flow, change: ModelChangeEvent::Upsert { created } }, - ); + self.events.emit("model_write", &self.live_search.payload(flow, created)); Ok::<_, yaak_database::Error>(()) }) .map_err(|e| format!("Failed to record the saved request on the flow: {e}"))?; Ok(saved) } + /// Records that an agent read the flow through the control API, for `@is:read`, and tells + /// the app. Only the first read writes anything. + pub fn note_agent_read(&self, flow_id: &str) { + let noted = self.db().map_err(String::from).and_then(|db| { + db.with_tx(|tx| { + let conn = tx.conn().resolve(); + let changed = conn.execute( + "UPDATE flows SET agent_read = TRUE WHERE id = ? AND NOT agent_read", + [flow_id], + )?; + // Sent before the commit, as every flow's event is. + if changed > 0 + && let Some(flow) = flows::find_in(conn, flow_id) + { + self.events.emit("model_write", &self.live_search.payload(flow, false)); + } + Ok::<_, yaak_database::Error>(()) + }) + .map_err(|e| e.to_string()) + }); + if let Err(e) = noted { + warn!("Failed to note that an agent read flow {flow_id}: {e}"); + } + } + /// Clears every flow at once and tells the app so its list empties, then deletes them and /// their bodies in the background. Returns how many there were. pub fn clear_flows(&self) -> Result { @@ -1385,6 +1413,91 @@ pub struct ListFlowsResponse { pub matching: Option, } +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct QuerySchemaRequest {} + +/// The search's fields, for the search box's highlighting and completion. +#[derive(Serialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct QuerySchemaResponse { + #[ts(type = "import(\"@yaakapp-internal/query\").Schema")] + pub schema: serde_json::Value, +} + +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct ValidateQueryRequest { + pub query: String, +} + +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct QueryValuesRequest { + /// A field's path: `host`, `header` for header names, `header.accept` for a header's values. + pub path: String, +} + +/// The search the app shows, which flows written from now on are matched against, their events +/// naming `key`. No query stops it. +#[derive(Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct WatchQueryRequest { + pub key: String, + #[ts(optional)] + pub query: Option, + #[serde(default)] + #[ts(optional, as = "Option")] + pub hide_system: bool, +} + +/// Which of some flows a search covers, for flows whose events were matched against another. +#[derive(Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct MatchFlowsRequest { + pub filter: flows::FlowFilter, + pub ids: Vec, +} + +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct ListSavedQueriesRequest {} + +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct CreateSavedQueryRequest { + pub name: String, + pub query: String, +} + +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct RenameSavedQueryRequest { + pub id: String, + pub name: String, +} + +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct DeleteSavedQueryRequest { + pub id: String, +} + +#[derive(Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct CountSavedQueriesRequest { + #[serde(default)] + #[ts(optional, as = "Option")] + pub hide_system: bool, +} + +#[derive(Serialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct SavedQueriesChangedPayload {} + #[derive(Deserialize, TS)] #[ts(export, export_to = "gen_rpc.ts")] pub struct ClearFlowsRequest {} @@ -2326,6 +2439,73 @@ async fn clear_flows(ctx: ProxyCtx, _req: ClearFlowsRequest) -> Result Result { + let schema = serde_json::to_value(query::schema()).map_err(rpc_error)?; + Ok(QuerySchemaResponse { schema }) +} + +fn validate_query( + _ctx: &ProxyCtx, + req: ValidateQueryRequest, +) -> Result, RpcError> { + Ok(query::diagnostics(&req.query)) +} + +async fn query_values(ctx: ProxyCtx, req: QueryValuesRequest) -> Result, RpcError> { + blocking(move || query::seen_values(&ctx.db()?, &req.path)).await +} + +fn watch_query(ctx: &ProxyCtx, req: WatchQueryRequest) -> Result { + let watched = req.query.filter(|q| !q.trim().is_empty() || req.hide_system); + ctx.live_search.watch(watched.map(|q| (req.key, q, req.hide_system))); + Ok(true) +} + +async fn match_flows(ctx: ProxyCtx, req: MatchFlowsRequest) -> Result, RpcError> { + blocking(move || flows::matching_ids(&ctx.db()?, &req.filter, &req.ids)).await +} + +fn list_saved_queries( + ctx: &ProxyCtx, + _req: ListSavedQueriesRequest, +) -> Result, RpcError> { + saved_queries::list(&ctx.db()?).map_err(rpc_error) +} + +fn create_saved_query( + ctx: &ProxyCtx, + req: CreateSavedQueryRequest, +) -> Result { + let saved = saved_queries::create(&ctx.db()?, &req.name, &req.query).map_err(rpc_error)?; + ctx.events.emit("saved_queries_changed", &SavedQueriesChangedPayload {}); + Ok(saved) +} + +fn rename_saved_query( + ctx: &ProxyCtx, + req: RenameSavedQueryRequest, +) -> Result { + let saved = saved_queries::rename(&ctx.db()?, &req.id, &req.name).map_err(rpc_error)?; + ctx.events.emit("saved_queries_changed", &SavedQueriesChangedPayload {}); + Ok(saved) +} + +fn delete_saved_query(ctx: &ProxyCtx, req: DeleteSavedQueryRequest) -> Result { + let deleted = saved_queries::delete(&ctx.db()?, &req.id).map_err(rpc_error)?; + ctx.events.emit("saved_queries_changed", &SavedQueriesChangedPayload {}); + Ok(deleted) +} + +async fn count_saved_queries( + ctx: ProxyCtx, + req: CountSavedQueriesRequest, +) -> Result, RpcError> { + blocking(move || saved_queries::counts(&ctx.db()?, req.hide_system)).await +} + // -- Event loop -- /// Why the engine cannot be taken over from another app. @@ -2429,6 +2609,8 @@ struct FlowSink { db: ProxyQueryManager, blobs: BlobManager, events: RpcEventEmitter, + /// The search the app shows, which each flow written is matched against. + live_search: Arc, attribution: Attribution, recording: Arc>, /// Flows that have ended, waiting to be written together. @@ -3076,10 +3258,7 @@ impl FlowSink { let (saved, created) = ctx.upsert(&flow, &UpdateSource::Background)?; // Sent while this holds the writer, before the commit, so a clear's event, sent after // its own commit, always comes after it. - self.events.emit( - "model_write", - &ModelPayload { model: saved, change: ModelChangeEvent::Upsert { created } }, - ); + self.events.emit("model_write", &self.live_search.payload(saved, created)); Ok(true) } @@ -3142,6 +3321,13 @@ define_rpc! { get_message_payload(GetMessagePayloadRequest) -> GetMessagePayloadResponse, list_actions(ListActionsRequest) -> ListActionsResponse, list_flows(flows::FlowFilter) -> ListFlowsResponse, + query_schema(QuerySchemaRequest) -> QuerySchemaResponse, + validate_query(ValidateQueryRequest) -> Vec, + watch_query(WatchQueryRequest) -> bool, + list_saved_queries(ListSavedQueriesRequest) -> Vec, + create_saved_query(CreateSavedQueryRequest) -> saved_queries::SavedQuery, + rename_saved_query(RenameSavedQueryRequest) -> saved_queries::SavedQuery, + delete_saved_query(DeleteSavedQueryRequest) -> bool, list_grpc_schemas(ListGrpcSchemasRequest) -> Vec, list_messages(ListMessagesRequest) -> ListMessagesResponse, remove_grpc_schema(RemoveGrpcSchemaRequest) -> bool, @@ -3179,6 +3365,9 @@ define_rpc! { get_storage_size(GetStorageSizeRequest) -> GetStorageSizeResponse, reset_captured_data(ResetCapturedDataRequest) -> bool, open_terminal(OpenTerminalRequest) -> bool, + query_values(QueryValuesRequest) -> Vec, + match_flows(MatchFlowsRequest) -> Vec, + count_saved_queries(CountSavedQueriesRequest) -> Vec, } events { decrypt_list_changed(DecryptListChangedPayload), @@ -3187,6 +3376,7 @@ define_rpc! { message_write(MessagePayload), model_write(ModelPayload), proxy_state_changed(ProxyStatePayload), + saved_queries_changed(SavedQueriesChangedPayload), } } @@ -3256,6 +3446,7 @@ mod tests { db: ctx.db().unwrap(), blobs: ctx.blobs().unwrap(), events: ctx.events.clone(), + live_search: ctx.live_search.clone(), attribution: Attribution::new(0), recording: ctx.recording.clone(), finished: Default::default(), diff --git a/crates-proxy/yaak-proxy-lib/src/models.rs b/crates-proxy/yaak-proxy-lib/src/models.rs index a11671ef3..2d922fd07 100644 --- a/crates-proxy/yaak-proxy-lib/src/models.rs +++ b/crates-proxy/yaak-proxy-lib/src/models.rs @@ -107,6 +107,9 @@ pub struct Flow { /// client rejected the proxy's certificate: method `CONNECT`, its URL the host, its sizes /// the bytes each way, and no headers or bodies. Open until `elapsedMs` or `error` is set. pub tunnelled: bool, + /// An agent read the flow through the control API: its detail, a body or its messages. + /// Never written by an upsert, which keeps what is stored. + pub agent_read: bool, } /// The streaming protocol a flow carries. @@ -194,6 +197,21 @@ pub fn find_header<'a>(headers: &'a [ProxyHeader], name: &str) -> Option<&'a str pub struct ModelPayload { pub model: Flow, pub change: ModelChangeEvent, + /// Whether the flow belongs in the list of the search the app shows, decided as the event + /// is sent. None while no search is shown. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub search: Option, +} + +/// A written flow, matched against the search the app shows. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_models.ts")] +pub struct SearchMatch { + /// The search's key, as the app gave it with the search. + pub key: String, + pub matches: bool, } impl UpsertModelInfo for Flow { @@ -357,6 +375,7 @@ impl UpsertModelInfo for Flow { grpc_status: added_column(r, "grpc_status")?, grpc_message: added_column(r, "grpc_message")?, tunnelled: added_column(r, "tunnelled")?.unwrap_or_default(), + agent_read: added_column(r, "agent_read")?.unwrap_or_default(), }) } } diff --git a/crates-proxy/yaak-proxy-lib/src/query.rs b/crates-proxy/yaak-proxy-lib/src/query.rs new file mode 100644 index 000000000..a91185092 --- /dev/null +++ b/crates-proxy/yaak-proxy-lib/src/query.rs @@ -0,0 +1,1242 @@ +//! Yaak Proxy's search: the query language of `yaak-query` over flows. +//! +//! A query is parsed by the crate, then prepared here: the shorthands the proxy keeps +//! (`@status:4xx`, `@host:*.github.com` for a site, `@path:/v1/*` for a folder and what is under +//! it, `@time:<5m`) become plain terms, and the hidden System flows are left out. The prepared +//! tree then runs two ways that must agree: compiled to a SQLite condition for listing and +//! counting ([`FlowQuery::sql`]), and in memory over one flow ([`FlowQuery::matches`]), for a +//! flow written while a search is shown. Both resolve each field term to the crate's +//! [`Condition`], and the values they test are made by the same functions here, which the +//! database calls as SQL functions ([`add_functions`]). + +use crate::flows::{folder_name, is_system, sidebar_path}; +use crate::models::{Flow, ModelPayload, ProxyHeader, SearchMatch, find_header}; +use chrono::{DateTime, Local, NaiveDate, NaiveDateTime, NaiveTime, TimeZone, Utc}; +use rusqlite::functions::FunctionFlags; +use rusqlite::types::{Value, ValueRef}; +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; +use std::sync::{OnceLock, PoisonError, RwLock}; +use ts_rs::TS; +use yaak_database::ModelChangeEvent; +use yaak_query::{ + CompareOp, Condition, DocValue, DocValues, Document, FieldDef, FieldKind, FieldValue, FreeText, + Matcher, Node, NumberLiteral, Schema, Span, Unit, Utf16Offsets, contains_ignore_case, + format_number, read_number, wildcard_match, +}; + +/// The flags `@is:` takes, in the order completion offers them. +pub const FLAGS: &[&str] = &[ + "error", + "pending", + "system", + "tunnelled", + "saved", + "read", + "loop", + "ws", + "sse", + "grpc", +]; + +/// Flags `@is:` will take once flows have what they need. Valid, matching nothing. +const FLAGS_NOT_YET: &[&str] = &["marked"]; + +/// Fields declared so completion shows them, matching nothing until they are built. +const FIELDS_NOT_YET: &[&str] = &["body", "req.body", "res.body"]; + +/// The values `@proto:` takes. +pub const PROTOS: &[&str] = &["h1", "h2", "ws", "sse", "grpc"]; + +/// The proxy's fields. +pub fn schema() -> &'static Schema { + static SCHEMA: OnceLock = OnceLock::new(); + SCHEMA.get_or_init(|| { + let units = |units: &[(&str, f64)]| FieldKind::Number { + units: units + .iter() + .map(|(name, factor)| Unit { name: (*name).into(), factor: *factor }) + .collect(), + }; + // As the app shows sizes: a kilobyte is 1000 bytes. + let bytes = || units(&[("b", 1.0), ("kb", 1e3), ("mb", 1e6), ("gb", 1e9)]); + // Each with what it is and a term using it, as completion shows them + let field = |name: &str, kind: FieldKind, info: &str, example: &str| FieldDef { + name: name.into(), + kind, + map: false, + info: Some(info.into()), + example: (!example.is_empty()).then(|| example.into()), + }; + let map = |name: &str, info: &str, example: &str| FieldDef { + map: true, + ..field(name, FieldKind::Text, info, example) + }; + let text = || FieldKind::Text; + let number = || FieldKind::Number { units: vec![] }; + let ms = || units(&[("ms", 1.0), ("s", 1e3), ("m", 6e4)]); + Schema::new( + vec![ + field("host", text(), "Host, wildcards allowed", "@host:*.github.com"), + field("path", text(), "URL path, /* for below", "@path:/v1/*"), + field("method", text(), "GET, POST, …", "@method:post"), + field("status", number(), "Response status", "@status:4xx @status:>=400"), + field("type", text(), "Content type", "@type:json"), + field("app", text(), "App or program", "@app:Slack"), + field("is", text(), "error, pending, system, …", "@is:error -@is:system"), + map("header", "A header on either side", "@header.content-type:json"), + map("req.header", "A request header", "@req.header.accept:*"), + map("res.header", "A response header", "@res.header.x-cache:hit"), + field("size", bytes(), "Response body size", "@size:>100kb"), + field("req.size", bytes(), "Request body size", "@req.size:>1kb"), + field("res.size", bytes(), "Response body size", "@res.size:<1mb"), + field("duration", ms(), "Total time", "@duration:>500ms"), + field("time", units(TIME_UNITS), "When the flow started", "@time:<5m"), + field("url", text(), "The whole URL", "@url:*token=*"), + map("query", "A query parameter", "@query.page:2"), + field("scheme", text(), "http or https", "@scheme:https"), + field("port", number(), "Port", "@port:8080"), + field("proto", text(), "h1, h2, ws, sse, grpc", "@proto:h2"), + field("id", number(), "Flow number", "@id:>1200"), + field("cwd", text(), "Working folder", "@cwd:myapp"), + field("pid", number(), "Process id", "@pid:4121"), + field("body", text(), "Not yet", ""), + field("req.body", text(), "Not yet", ""), + field("res.body", text(), "Not yet", ""), + ], + FreeText::Substring, + ) + }) +} + +/// How long ago `@time` counts in, in milliseconds. +const TIME_UNITS: &[(&str, f64)] = &[ + ("ms", 1.0), + ("s", 1e3), + ("m", 6e4), + ("h", 3.6e6), + ("d", 8.64e7), + ("w", 6.048e8), +]; + +// -- Preparing -- + +/// A query prepared for running: its shorthands written out, System left out when hidden. +#[derive(Debug, Clone, PartialEq)] +pub struct FlowQuery { + root: Option, +} + +impl FlowQuery { + /// `text` as of now. With `hide_system`, macOS's own daemons are left out, unless the text + /// says `@is:system` itself. + pub fn new(text: &str, hide_system: bool) -> Self { + Self::at(text, hide_system, Utc::now()) + } + + /// [`Self::new`], with `now` for the times `@time` counts back from. + pub fn at(text: &str, hide_system: bool, now: DateTime) -> Self { + let parsed = yaak_query::parse(text); + let mentions_system = parsed.as_ref().is_some_and(mentions_system); + let mut root = parsed.map(|root| prepare(&root, now, &mut Vec::new())); + if hide_system && !mentions_system { + let span = Span::default(); + let not_system = Node::Not { node: Box::new(is_flag("system", span)), span }; + root = Some(match root { + None => not_system, + Some(root) => Node::And { children: vec![root, not_system], span }, + }); + } + Self { root } + } + + /// Covers every flow. + pub fn is_empty(&self) -> bool { + self.root.is_none() + } + + /// Whether `flow` is one the query covers, as [`Self::sql`] decides it in the database. + pub fn matches(&self, flow: &Flow) -> bool { + self.matcher().matches(&document(flow)) + } + + /// The query resolved for matching many flows in memory. + pub fn matcher(&self) -> Matcher { + Matcher::new(self.root.as_ref(), schema()) + } + + /// The SQL condition on `flows` and its parameters, for a database with `columns`. In + /// parentheses, so a caller may add to it with `AND`. + pub fn sql(&self, columns: &FlowColumns) -> (String, Vec) { + let mut sql = Sql { columns, params: Vec::new() }; + let condition = match &self.root { + None => "1".to_string(), + Some(root) => sql.node(root), + }; + (format!("({condition})"), sql.params) + } +} + +/// `@is:`, as a prepared term. +fn is_flag(flag: &str, span: Span) -> Node { + Node::Field { + path: vec!["is".into()], + value: FieldValue::Wildcard { pattern: flag.into() }, + span, + path_span: span, + value_span: span, + } +} + +/// The query names `@is:system`, either way, so hiding System leaves it be. +fn mentions_system(root: &Node) -> bool { + let mut found = false; + root.walk(&mut |node| { + if let Node::Field { path, value, .. } = node + && path.len() == 1 + && path[0].eq_ignore_ascii_case("is") + && let FieldValue::Text { text, .. } | FieldValue::Wildcard { pattern: text } = value + && text.eq_ignore_ascii_case("system") + { + found = true; + } + }); + found +} + +/// What the schema alone does not catch, found while preparing. +#[derive(Debug, Clone, PartialEq)] +struct Issue { + kind: QueryDiagnosticKind, + path: String, + span: Span, +} + +/// A term that matches nothing, in place of one that cannot run. +fn never(span: Span) -> Node { + Node::Phrase { text: String::new(), span } +} + +fn number(value: f64) -> NumberLiteral { + NumberLiteral { value, unit: None, raw: format_number(value) } +} + +/// `node` with the proxy's shorthands written out as plain terms, keeping their spans. +fn prepare(node: &Node, now: DateTime, issues: &mut Vec) -> Node { + match node { + Node::And { children, span } => Node::And { + children: children.iter().map(|c| prepare(c, now, issues)).collect(), + span: *span, + }, + Node::Or { children, span } => Node::Or { + children: children.iter().map(|c| prepare(c, now, issues)).collect(), + span: *span, + }, + Node::Not { node: inner, span } => { + Node::Not { node: Box::new(prepare(inner, now, issues)), span: *span } + } + Node::Term { .. } | Node::Phrase { .. } => node.clone(), + Node::Field { path, value, span, path_span, value_span } => { + let Some(field) = schema().resolve(path) else { + return node.clone(); + }; + let with = |value: FieldValue| Node::Field { + path: path.clone(), + value, + span: *span, + path_span: *path_span, + value_span: *value_span, + }; + let mut issue = |kind| { + let marked = if kind == QueryDiagnosticKind::NotYet { *span } else { *value_span }; + issues.push(Issue { kind, path: path.join("."), span: marked }); + never(*span) + }; + match (field.def.name.as_str(), value) { + // A class of statuses: `4xx` + ("status", FieldValue::Text { text, quoted: false }) => match status_class(text) { + Some(class) => with(FieldValue::Range { + low: number(class * 100.0), + high: number(class * 100.0 + 99.0), + }), + None => node.clone(), + }, + // A site: `*.github.com` is `github.com` and every host below it + ("host", FieldValue::Wildcard { pattern }) if pattern.starts_with("*.") => { + Node::Or { + children: vec![ + with(FieldValue::Wildcard { pattern: pattern[2..].to_string() }), + node.clone(), + ], + span: *span, + } + } + // A folder: `/v1/*` is `/v1` and everything under it + ("path", FieldValue::Wildcard { pattern }) if pattern.ends_with("/*") => Node::Or { + children: vec![ + with(FieldValue::Wildcard { pattern: pattern[..pattern.len() - 2].into() }), + node.clone(), + ], + span: *span, + }, + // Named whole, as the sidebar names them: a part needs wildcards + ("host" | "scheme" | "app" | "cwd", FieldValue::Text { text, .. }) + if !text.is_empty() => + { + with(FieldValue::Wildcard { pattern: text.clone() }) + } + ("is", FieldValue::Text { text, .. }) if !text.is_empty() => { + let flag = text.to_lowercase(); + if FLAGS.contains(&flag.as_str()) { + with(FieldValue::Wildcard { pattern: flag }) + } else if FLAGS_NOT_YET.contains(&flag.as_str()) { + issue(QueryDiagnosticKind::NotYet) + } else { + issue(QueryDiagnosticKind::InvalidValue) + } + } + // Not written yet + ("time", FieldValue::Text { text, .. }) if text.is_empty() => node.clone(), + ("time", value) => match time_node(value, now, &with) { + Some(node) => node, + None => issue(QueryDiagnosticKind::InvalidValue), + }, + (name, _) if FIELDS_NOT_YET.contains(&name) => issue(QueryDiagnosticKind::NotYet), + _ => node.clone(), + } + } + } +} + +/// The class `4xx` names, 4, or None when `text` is not one. +fn status_class(text: &str) -> Option { + let class = text.strip_suffix("xx").or_else(|| text.strip_suffix("XX"))?; + match class.as_bytes() { + [d @ b'1'..=b'9'] => Some(f64::from(d - b'0')), + _ => None, + } +} + +/// `@time`'s value as plain comparisons of when flows started, in Unix milliseconds. A +/// comparison or range of durations is how long ago (`<5m`: in the last five minutes); a date, +/// `2026-11-07`, a time today, `14:30`, or both, `2026-11-07T14:30`, all local, or an ISO time +/// with its offset, is the span it names, compared whole. None when the value is none of these. +fn time_node( + value: &FieldValue, + now: DateTime, + with: &impl Fn(FieldValue) -> Node, +) -> Option { + let now_ms = now.timestamp_millis() as f64; + let compare = |op, ms: f64| with(FieldValue::Compare { op, number: number(ms) }); + match value { + FieldValue::Exists => Some(with(FieldValue::Exists)), + FieldValue::Compare { op, number: age } => { + // Less long ago is later + let op = match op { + CompareOp::Gt => CompareOp::Lt, + CompareOp::Gte => CompareOp::Lte, + CompareOp::Lt => CompareOp::Gt, + CompareOp::Lte => CompareOp::Gte, + }; + Some(compare(op, now_ms - age_ms(age)?)) + } + FieldValue::Range { low, high } => { + let (a, b) = (age_ms(low)?, age_ms(high)?); + Some(with(FieldValue::Range { + low: number(now_ms - a.max(b)), + high: number(now_ms - a.min(b)), + })) + } + FieldValue::Text { text, .. } => { + let (op, point) = split_op(text); + let (start, end) = time_span(point, now)?; + Some(match op { + None => { + let span = Span::default(); + Node::And { + children: vec![compare(CompareOp::Gte, start), compare(CompareOp::Lt, end)], + span, + } + } + Some(CompareOp::Gt) => compare(CompareOp::Gte, end), + Some(CompareOp::Gte) => compare(CompareOp::Gte, start), + Some(CompareOp::Lt) => compare(CompareOp::Lt, start), + Some(CompareOp::Lte) => compare(CompareOp::Lt, end), + }) + } + FieldValue::Wildcard { .. } => None, + } +} + +/// A duration in milliseconds, which must have a unit. +fn age_ms(number: &NumberLiteral) -> Option { + let unit = number.unit.as_deref()?; + let (_, factor) = TIME_UNITS.iter().find(|(name, _)| name.eq_ignore_ascii_case(unit))?; + Some(number.value * factor) +} + +fn split_op(text: &str) -> (Option, &str) { + for (prefix, op) in [ + (">=", CompareOp::Gte), + ("<=", CompareOp::Lte), + (">", CompareOp::Gt), + ("<", CompareOp::Lt), + ] { + if let Some(rest) = text.strip_prefix(prefix) { + return (Some(op), rest); + } + } + (None, text) +} + +/// The span of time `text` names, as Unix milliseconds from its start to just past its end. +fn time_span(text: &str, now: DateTime) -> Option<(f64, f64)> { + let local = |dt: NaiveDateTime| Local.from_local_datetime(&dt).earliest(); + let ms = |dt: DateTime| dt.timestamp_millis() as f64; + if let Ok(dt) = DateTime::parse_from_rfc3339(text) { + let start = dt.timestamp_millis() as f64; + return Some((start, start + 1e3)); + } + if let Ok(date) = NaiveDate::parse_from_str(text, "%Y-%m-%d") { + let start = local(date.and_time(NaiveTime::MIN))?; + let end = local(date.succ_opt()?.and_time(NaiveTime::MIN))?; + return Some((ms(start), ms(end))); + } + let today = now.with_timezone(&Local).date_naive(); + for (format, length) in [ + ("%Y-%m-%dT%H:%M:%S", 1e3), + ("%Y-%m-%dT%H:%M", 6e4), + ("%H:%M:%S", 1e3), + ("%H:%M", 6e4), + ] { + let parsed = NaiveDateTime::parse_from_str(text, format) + .ok() + .or_else(|| NaiveTime::parse_from_str(text, format).ok().map(|t| today.and_time(t))); + if let Some(dt) = parsed { + let start = ms(local(dt)?); + return Some((start, start + length)); + } + } + None +} + +// -- Live flows -- + +/// The search the app shows, while it shows one. Each flow written meanwhile is matched +/// against it as its event is sent, so the event says whether it belongs in the search's list +/// and the app needs no matching of its own. +#[derive(Debug, Default)] +pub struct LiveSearch(RwLock>); + +#[derive(Debug)] +struct Watched { + key: String, + text: String, + hide_system: bool, +} + +impl LiveSearch { + /// Matches flows against `text` from now on, their events naming `key`; None stops. + pub fn watch(&self, watched: Option<(String, String, bool)>) { + let watched = watched.map(|(key, text, hide_system)| Watched { key, text, hide_system }); + *self.0.write().unwrap_or_else(PoisonError::into_inner) = watched; + } + + /// `flow` matched against the search shown, if any. + pub fn check(&self, flow: &Flow) -> Option { + let watched = self.0.read().unwrap_or_else(PoisonError::into_inner); + let Watched { key, text, hide_system } = watched.as_ref()?; + // Prepared again for each flow, so `@time:<5m` counts back from now. + let matches = FlowQuery::new(text, *hide_system).matches(flow); + Some(SearchMatch { key: key.clone(), matches }) + } + + /// The event for `flow`, written: created, or updated. + pub fn payload(&self, flow: Flow, created: bool) -> ModelPayload { + let search = self.check(&flow); + ModelPayload { model: flow, change: ModelChangeEvent::Upsert { created }, search } + } +} + +// -- Seen values, for completion -- + +/// The newest flows completion looks through for values. +const SEEN_FLOWS: i64 = 2000; + +/// Values offered per field at most. +const SEEN_VALUES: i64 = 200; + +/// A header's values are offered only when it has had at most this many: more, and they are +/// near-unique (dates, ids), which is noise. +const HEADER_VALUES: i64 = 20; + +/// The values seen for a field path, for completing it, most recent first: hosts, methods, +/// statuses, content types, protocols, schemes, apps, working folders, `@is` flags, and header +/// names, or a header's values when it has had few. Nothing for fields whose values are +/// near-unique (URLs, paths, ids, sizes, times, query parameters). +pub fn seen_values(db: &crate::db::ProxyQueryManager, path: &str) -> Result, String> { + let path = path.to_lowercase(); + let column = match path.as_str() { + "proto" => return Ok(PROTOS.iter().map(|p| p.to_string()).collect()), + "is" => return Ok(FLAGS.iter().map(|f| f.to_string()).collect()), + "host" => "url_host(url)", + "method" => "method", + "status" => "CAST(status AS TEXT)", + "type" => "coalesce(content_type, header_value(request_headers, 'content-type'))", + "scheme" => "url_scheme(url)", + "app" => "client_name", + "cwd" => "folder_name(client_cwd)", + "header" | "req.header" | "res.header" => { + return seen_headers(db, &path, None); + } + _ => { + for map in ["req.header.", "res.header.", "header."] { + if let Some(name) = path.strip_prefix(map) { + return seen_headers(db, &map[..map.len() - 1], Some(name)); + } + } + return Ok(Vec::new()); + } + }; + let sql = format!( + "SELECT v FROM (SELECT {column} AS v, rid FROM ({})) WHERE v IS NOT NULL AND v != '' \ + GROUP BY v ORDER BY max(rid) DESC LIMIT {SEEN_VALUES}", + newest_flows() + ); + let mut values = select_strings(db, &sql, None)?; + if path == "type" { + // The media type alone, as a search names it + let mut seen = HashSet::new(); + values = values + .into_iter() + .filter_map(|t| t.split(';').next().map(|m| m.trim().to_lowercase())) + .filter(|m| !m.is_empty() && seen.insert(m.clone())) + .collect(); + } + Ok(values) +} + +/// The newest flows not cleared, with their rowid as `rid`. +fn newest_flows() -> String { + format!( + "SELECT rowid AS rid, * FROM flows WHERE rowid > (SELECT coalesce(max(max_rowid), 0) \ + FROM flow_clear_mark) ORDER BY rowid DESC LIMIT {SEEN_FLOWS}" + ) +} + +/// Header names under a map, or one header's values when it has had few. +fn seen_headers( + db: &crate::db::ProxyQueryManager, + map: &str, + name: Option<&str>, +) -> Result, String> { + let sides = match map { + "req.header" => vec!["request_headers"], + "res.header" => vec!["response_headers"], + _ => vec!["request_headers", "response_headers"], + }; + let pairs = sides + .iter() + .map(|side| { + format!( + "SELECT q_lower(json_extract(kv.value, '$.name')) AS name, \ + json_extract(kv.value, '$.value') AS value, f.rid AS rid \ + FROM ({}) AS f, json_each(f.{side}) AS kv", + newest_flows() + ) + }) + .collect::>() + .join(" UNION ALL "); + match name { + None => { + let sql = format!( + "SELECT name FROM ({pairs}) WHERE name IS NOT NULL AND name != '' \ + GROUP BY name ORDER BY max(rid) DESC LIMIT {SEEN_VALUES}" + ); + select_strings(db, &sql, None) + } + Some(name) => { + let sql = format!( + "SELECT value FROM ({pairs}) WHERE name = ? AND value IS NOT NULL AND value != '' \ + GROUP BY value ORDER BY max(rid) DESC LIMIT {}", + HEADER_VALUES + 1 + ); + let values = select_strings(db, &sql, Some(name.to_lowercase()))?; + Ok(if values.len() as i64 > HEADER_VALUES { Vec::new() } else { values }) + } + } +} + +fn select_strings( + db: &crate::db::ProxyQueryManager, + sql: &str, + param: Option, +) -> Result, String> { + db.with_conn(|c| { + let conn = c.conn().resolve(); + let mut stmt = conn.prepare(sql).map_err(|e| e.to_string())?; + let params: Vec = param.into_iter().map(Value::from).collect(); + let rows = stmt + .query_map(rusqlite::params_from_iter(params), |r| r.get::<_, String>(0)) + .map_err(|e| e.to_string())?; + rows.collect::, _>>().map_err(|e| e.to_string()) + }) +} + +// -- Diagnostics -- + +/// What is wrong with a term, for the search box to mark. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_rpc.ts")] +pub enum QueryDiagnosticKind { + /// No field has this path. + UnknownField, + /// The value cannot apply to the field: a unit it does not take, a flag it does not have. + InvalidValue, + /// Searchable later; matches nothing for now. + NotYet, +} + +/// A term the query cannot run, which then matches nothing. `start` and `end` count UTF-16 +/// code units, as JavaScript does. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct QueryDiagnostic { + pub kind: QueryDiagnosticKind, + pub path: String, + pub start: u32, + pub end: u32, +} + +/// The terms of `text` that cannot run, in order. +pub fn diagnostics(text: &str) -> Vec { + let Some(root) = yaak_query::parse(text) else { + return Vec::new(); + }; + let mut issues = Vec::new(); + let prepared = prepare(&root, Utc::now(), &mut issues); + for diagnostic in schema().diagnostics(&prepared) { + issues.push(match diagnostic { + yaak_query::Diagnostic::UnknownField { path, span } => { + Issue { kind: QueryDiagnosticKind::UnknownField, path, span } + } + yaak_query::Diagnostic::InvalidValue { path, span } => { + Issue { kind: QueryDiagnosticKind::InvalidValue, path, span } + } + }); + } + issues.sort_by_key(|i| (i.span.start, i.span.end)); + let offsets = Utf16Offsets::new(text); + issues + .into_iter() + .map(|Issue { kind, path, span }| { + let span = offsets.span(span); + QueryDiagnostic { kind, path, start: span.start, end: span.end } + }) + .collect() +} + +// -- The values a flow is matched by -- + +/// The parts of a URL the fields read. A URL without a scheme, as an undecrypted connection's +/// may be, is all host. +struct UrlParts<'a> { + scheme: Option<&'a str>, + authority: &'a str, + path: Option<&'a str>, + query: Option<&'a str>, +} + +fn url_parts(url: &str) -> UrlParts<'_> { + let Some((scheme, rest)) = url.split_once("://") else { + return UrlParts { scheme: None, authority: url, path: None, query: None }; + }; + let end = rest.find(['/', '?', '#']).unwrap_or(rest.len()); + let (authority, rest) = rest.split_at(end); + let rest = rest.split('#').next().unwrap_or_default(); + let (path, query) = match rest.split_once('?') { + Some((path, query)) => (path, Some(query)), + None => (rest, None), + }; + UrlParts { scheme: Some(scheme), authority, path: Some(path), query } +} + +/// The authority less any user info. +fn host_port(authority: &str) -> &str { + authority.rsplit_once('@').map_or(authority, |(_, host)| host) +} + +/// The host, lowercased and without its port. +pub fn url_hostname(url: &str) -> String { + let host = host_port(url_parts(url).authority); + let host = if host.starts_with('[') { + host.find(']').map_or(host, |end| &host[..=end]) + } else { + host.rsplit_once(':').map_or(host, |(h, _)| h) + }; + host.to_lowercase() +} + +/// The host as the sidebar names it: with its port when that is not the scheme's. +pub fn url_host(url: &str) -> String { + sidebar_path(url).split('/').next().unwrap_or_default().to_string() +} + +/// The port the URL names, or its scheme's. +pub fn url_port(url: &str) -> Option { + let parts = url_parts(url); + let host = host_port(parts.authority); + let after_host = if host.starts_with('[') { + host.find(']').map(|end| &host[end + 1..]).unwrap_or_default() + } else { + host + }; + if let Some((_, port)) = after_host.rsplit_once(':') + && let Ok(port) = port.parse() + { + return Some(port); + } + match parts.scheme?.to_ascii_lowercase().as_str() { + "http" | "ws" => Some(80), + "https" | "wss" => Some(443), + _ => None, + } +} + +pub fn url_scheme(url: &str) -> Option { + url_parts(url).scheme.map(str::to_lowercase) +} + +/// The path, without the query or fragment. +pub fn url_path(url: &str) -> Option<&str> { + url_parts(url).path +} + +/// The path as the sidebar names it, its empty segments left out (`/v1//users` is +/// `/v1/users`), when that differs from the path, so a sidebar row's term finds it. +pub fn url_sidebar_path(url: &str) -> Option { + let path = url_path(url)?; + let segments: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect(); + let normalized = format!("/{}", segments.join("/")); + (normalized != path && !path.is_empty()).then_some(normalized) +} + +/// The query's parameters, decoded, their names lowercased. +pub fn url_query(url: &str) -> Vec<(String, String)> { + let Some(query) = url_parts(url).query else { + return Vec::new(); + }; + url::form_urlencoded::parse(query.as_bytes()) + .filter(|(name, _)| !name.is_empty()) + .map(|(name, value)| (name.to_lowercase(), value.into_owned())) + .collect() +} + +/// What bare words search: the URL, the method, the status and the content type. +pub fn flow_text( + url: &str, + method: &str, + status: Option, + content_type: Option<&str>, +) -> String { + let status = status.map(|s| s.to_string()).unwrap_or_default(); + format!("{url}\n{method}\n{status}\n{}", content_type.unwrap_or_default()) +} + +/// `h1` or `h2`, from the version the client spoke. +pub fn http_proto(version: &str) -> Option<&'static str> { + match version { + "HTTP/2" | "HTTP/2.0" => Some("h2"), + "HTTP/3" | "HTTP/3.0" => Some("h3"), + v if v.starts_with("HTTP/1") || v.starts_with("HTTP/0") => Some("h1"), + _ => None, + } +} + +/// `ws`, `sse` or `grpc`, from a stream's kind as stored. +pub fn stream_proto(kind: &str) -> Option<&'static str> { + match kind { + "websocket" => Some("ws"), + "sse" => Some("sse"), + "grpc" | "grpc_web" => Some("grpc"), + _ => None, + } +} + +/// The flags `@is:` finds on `flow`. +fn flags(flow: &Flow) -> Vec<&'static str> { + let stream = flow.stream_kind.map(|k| k.as_str()); + let set = |flag: &str| match flag { + "error" => flow.error.is_some() || flow.grpc_status.unwrap_or(0) != 0, + "pending" => { + flow.status.is_none() + && flow.error.is_none() + && !flow.cancelled + && flow.elapsed_ms.is_none() + } + "system" => is_system(flow.client_bundle_id.as_deref(), flow.client_path.as_deref()), + "tunnelled" => flow.tunnelled, + "saved" => flow.saved_request_id.is_some(), + "read" => flow.agent_read, + "loop" => flow.status == Some(508), + "ws" => stream == Some("websocket"), + "sse" => stream == Some("sse"), + "grpc" => matches!(stream, Some("grpc" | "grpc_web")), + _ => false, + }; + FLAGS.iter().copied().filter(|f| set(f)).collect() +} + +/// The response's content type, or the request's when it has none. +fn content_type(flow: &Flow) -> Option<&str> { + flow.content_type.as_deref().or_else(|| find_header(&flow.request_headers, "content-type")) +} + +/// `flow` as the evaluator sees it, with the values [`FlowQuery::sql`] reads in the database. +pub fn document(flow: &Flow) -> Document { + let mut fields: HashMap> = HashMap::new(); + let mut add = |path: &str, value: DocValue| { + fields.entry(path.to_lowercase()).or_default().push(value); + }; + let text = |s: &str| DocValue::Text(s.to_string()); + let num = |n: f64| DocValue::Number(n); + + add("host", DocValue::Text(url_hostname(&flow.url))); + add("host", DocValue::Text(url_host(&flow.url))); + if let Some(port) = url_port(&flow.url) { + add("port", num(port as f64)); + } + if let Some(scheme) = url_scheme(&flow.url) { + add("scheme", DocValue::Text(scheme)); + } + if let Some(path) = url_path(&flow.url) { + add("path", text(path)); + } + if let Some(path) = url_sidebar_path(&flow.url) { + add("path", DocValue::Text(path)); + } + add("url", text(&flow.url)); + for (name, value) in url_query(&flow.url) { + add(&format!("query.{name}"), DocValue::Text(value)); + } + add("method", text(&flow.method)); + if let Some(t) = content_type(flow) { + add("type", text(t)); + } + if let Some(p) = http_proto(&flow.client_http_version) { + add("proto", text(p)); + } + if let Some(p) = flow.stream_kind.and_then(|k| stream_proto(k.as_str())) { + add("proto", text(p)); + } + add("id", num(flow.number as f64)); + if let Some(status) = flow.status { + add("status", num(f64::from(status))); + } + add("size", num(flow.response_size as f64)); + add("res.size", num(flow.response_size as f64)); + add("req.size", num(flow.request_size as f64)); + if let Some(ms) = flow.elapsed_ms { + add("duration", num(f64::from(ms))); + } + add("time", num(started_ms(&flow.started_at))); + if let Some(app) = flow.client_bundle_id.as_deref().or(flow.client_path.as_deref()) { + add("app", text(app)); + } + if let Some(name) = &flow.client_name { + add("app", text(name)); + } + if flow.client_bundle_id.is_none() + && let Some(folder) = folder_name(flow.client_cwd.as_deref()) + { + add("app", text(folder)); + } + if let Some(cwd) = &flow.client_cwd { + add("cwd", text(cwd)); + if let Some(folder) = folder_name(Some(cwd)) { + add("cwd", text(folder)); + } + } + if let Some(pid) = flow.client_pid { + add("pid", num(f64::from(pid))); + } + for (side, headers) in [ + ("req", &flow.request_headers), + ("res", &flow.response_headers), + ] { + for ProxyHeader { name, value } in headers.iter().filter(|h| !h.name.is_empty()) { + add(&format!("header.{name}"), text(value)); + add(&format!("{side}.header.{name}"), text(value)); + } + } + for flag in flags(flow) { + add("is", text(flag)); + } + + Document { + text: flow_text( + &flow.url, + &flow.method, + flow.status.map(i64::from), + flow.content_type.as_deref(), + ), + fields: fields.into_iter().map(|(path, values)| (path, DocValues::Many(values))).collect(), + } +} + +/// When a flow started, in Unix milliseconds, fractions kept, as `@time` compares it. +fn started_ms(started_at: &NaiveDateTime) -> f64 { + let utc = started_at.and_utc(); + utc.timestamp() as f64 * 1e3 + f64::from(utc.timestamp_subsec_nanos()) / 1e6 +} + +/// Unix milliseconds as `started_at` is stored, so the database compares them as text in its +/// index's order: equal times are equal text, and later ones sort after. +fn stored_time(ms: f64) -> String { + let nanos = (ms * 1e6).round() as i64; + let dt = DateTime::::from_timestamp_nanos(nanos).naive_utc(); + dt.format("%Y-%m-%d %H:%M:%S%.f").to_string() +} + +// -- Compiling to SQL -- + +/// The columns of a database's flows table: one opened read-only before its migrations ran may +/// lack later ones, which then read as the model reads them, empty. +#[derive(Debug, Clone)] +pub struct FlowColumns(HashSet); + +impl FlowColumns { + pub fn of(conn: &rusqlite::Connection) -> Self { + let names = conn + .prepare("SELECT name FROM pragma_table_info('flows')") + .and_then(|mut stmt| { + stmt.query_map([], |r| r.get::<_, String>(0))?.collect::>() + }) + .unwrap_or_default(); + Self(names) + } + + /// `name`, or what the model reads in its place when the table lacks it. + fn col(&self, name: &str) -> String { + if self.0.contains(name) { + return name.to_string(); + } + match name { + "cancelled" | "tunnelled" | "agent_read" => "0".into(), + _ => "NULL".into(), + } + } +} + +struct Sql<'a> { + columns: &'a FlowColumns, + params: Vec, +} + +impl Sql<'_> { + fn param(&mut self, value: impl Into) -> &'static str { + self.params.push(value.into()); + "?" + } + + fn node(&mut self, node: &Node) -> String { + match node { + Node::And { children, .. } => self.join(children, " AND ", "1"), + Node::Or { children, .. } => self.join(children, " OR ", "0"), + Node::Not { node, .. } => format!("NOT ({})", self.node(node)), + Node::Term { text, .. } => self.free_text(text.trim().to_lowercase()), + Node::Phrase { text, .. } => self.free_text(text.to_lowercase()), + Node::Field { path, value, .. } => { + let Some(field) = schema().resolve(path) else { + return "0".into(); + }; + let condition = Condition::new(&field.def.kind, value); + if condition == Condition::Never { + return "0".into(); + } + self.field(&field.def.name, field.key.as_deref(), &condition) + } + } + } + + fn join(&mut self, children: &[Node], op: &str, empty: &str) -> String { + if children.is_empty() { + return empty.into(); + } + let parts: Vec = children.iter().map(|c| format!("({})", self.node(c))).collect(); + parts.join(op) + } + + fn free_text(&mut self, needle: String) -> String { + if needle.is_empty() { + return "0".into(); + } + let p = self.param(needle); + format!("coalesce(q_contains(flow_text(url, method, status, content_type), {p}), 0)") + } + + fn field(&mut self, name: &str, key: Option<&str>, condition: &Condition) -> String { + let c = |n: &str| self.columns.col(n); + let (values, numeric): (Vec, bool) = match name { + "host" => (vec!["url_hostname(url)".into(), "url_host(url)".into()], false), + "port" => (vec!["url_port(url)".into()], true), + "scheme" => (vec!["url_scheme(url)".into()], false), + "path" => (vec!["url_path(url)".into(), "url_sidebar_path(url)".into()], false), + "url" => (vec!["url".into()], false), + "method" => (vec!["method".into()], false), + "type" => ( + vec![ + "coalesce(content_type, header_value(request_headers, 'content-type'))".into(), + ], + false, + ), + "proto" => ( + vec![ + "http_proto(client_http_version)".into(), + format!("stream_proto({})", c("stream_kind")), + ], + false, + ), + "id" => (vec!["flows.rowid".into()], true), + "status" => (vec!["status".into()], true), + "size" | "res.size" => (vec!["response_size".into()], true), + "req.size" => (vec!["request_size".into()], true), + "duration" => (vec!["elapsed_ms".into()], true), + "pid" => (vec![c("client_pid")], true), + "app" => { + let (bundle, path, name) = + (c("client_bundle_id"), c("client_path"), c("client_name")); + let folder = + format!("CASE WHEN {bundle} IS NULL THEN folder_name({}) END", c("client_cwd")); + (vec![format!("coalesce({bundle}, {path})"), name, folder], false) + } + "cwd" => { + let cwd = c("client_cwd"); + (vec![cwd.clone(), format!("folder_name({cwd})")], false) + } + "time" => return self.time(condition), + "is" => return self.flags(condition), + "header" => { + let both = [ + self.pairs("request_headers", key, condition), + self.pairs("response_headers", key, condition), + ]; + return format!("({} OR {})", both[0], both[1]); + } + "req.header" => return self.pairs("request_headers", key, condition), + "res.header" => return self.pairs("response_headers", key, condition), + "query" => return self.pairs("url_query_json(url)", key, condition), + _ => return "0".into(), + }; + self.any_value(&values, numeric, condition) + } + + /// Any of the values passes. + fn any_value(&mut self, values: &[String], numeric: bool, condition: &Condition) -> String { + let parts: Vec = values.iter().map(|v| self.test(v, numeric, condition)).collect(); + if parts.len() == 1 { + parts.into_iter().next().unwrap_or_default() + } else { + format!("({})", parts.join(" OR ")) + } + } + + /// `condition` on one value, never NULL. `numeric` for a value stored as a number, compared + /// without reading it as one first. + fn test(&mut self, value: &str, numeric: bool, condition: &Condition) -> String { + let as_number = if numeric { value.to_string() } else { format!("q_number({value})") }; + let test = match condition { + Condition::Contains(needle) => { + format!("q_contains({value}, {})", self.param(needle.clone())) + } + Condition::Wildcard(pattern) => { + format!("q_wildcard({value}, {})", self.param(pattern.clone())) + } + Condition::Present => format!("q_present({value})"), + Condition::IsNumber => format!("{as_number} IS NOT NULL"), + Condition::Equals(n) => format!("{as_number} = {}", self.param(*n)), + Condition::Compare(op, n) => format!("{as_number} {} {}", op.as_str(), self.param(*n)), + Condition::Range(low, high) => { + format!("{as_number} BETWEEN {} AND {}", self.param(*low), self.param(*high)) + } + // The proxy has no flag fields; `@is` is text. + Condition::Flag(_) | Condition::Never => return "0".into(), + }; + format!("coalesce({test}, 0)") + } + + /// `@time`, prepared into comparisons of Unix milliseconds, compared with `started_at` as + /// stored, through its index. + fn time(&mut self, condition: &Condition) -> String { + match condition { + Condition::Present | Condition::IsNumber => "1".into(), + Condition::Equals(ms) => format!("started_at = {}", self.param(stored_time(*ms))), + Condition::Compare(op, ms) => { + format!("started_at {} {}", op.as_str(), self.param(stored_time(*ms))) + } + Condition::Range(low, high) => format!( + "started_at BETWEEN {} AND {}", + self.param(stored_time(*low)), + self.param(stored_time(*high)) + ), + // Preparing leaves `@time` nothing else. + _ => "0".into(), + } + } + + /// A flag's condition on the flow's columns. + fn flag(&self, flag: &str) -> String { + let c = |n: &str| self.columns.col(n); + match flag { + "error" => format!("(error IS NOT NULL OR coalesce({}, 0) != 0)", c("grpc_status")), + "pending" => format!( + "(status IS NULL AND error IS NULL AND NOT {} AND elapsed_ms IS NULL)", + c("cancelled") + ), + "system" => format!("is_system({}, {})", c("client_bundle_id"), c("client_path")), + "tunnelled" => c("tunnelled"), + "saved" => format!("{} IS NOT NULL", c("saved_request_id")), + "read" => c("agent_read"), + "loop" => "status = 508".into(), + "ws" => format!("{} = 'websocket'", c("stream_kind")), + "sse" => format!("{} = 'sse'", c("stream_kind")), + "grpc" => format!("{} IN ('grpc', 'grpc_web')", c("stream_kind")), + _ => "0".into(), + } + } + + /// `@is`: each flag set is a value, its name. A flag named whole is its column condition. + fn flags(&mut self, condition: &Condition) -> String { + if let Condition::Wildcard(pattern) = condition + && !pattern.contains('*') + { + return match FLAGS.contains(&pattern.as_str()) { + true => format!("coalesce({}, 0)", self.flag(pattern)), + false => "0".into(), + }; + } + let values: Vec = FLAGS + .iter() + .map(|f| format!("CASE WHEN coalesce({}, 0) THEN '{f}' END", self.flag(f))) + .collect(); + self.any_value(&values, false, condition) + } + + /// A map's values in a JSON array of `{ name, value }`: under `key` when given, any + /// otherwise. + fn pairs(&mut self, json: &str, key: Option<&str>, condition: &Condition) -> String { + let name = match key { + Some(key) => { + format!( + "q_lower(json_extract(kv.value, '$.name')) = {} AND ", + self.param(key.to_lowercase()) + ) + } + None => String::new(), + }; + let test = self.test("json_extract(kv.value, '$.value')", false, condition); + format!("EXISTS (SELECT 1 FROM json_each({json}) AS kv WHERE {name}{test})") + } +} + +// -- SQL functions -- + +/// A value as the evaluator reads text from it: text as it is, a number as `200`. +fn value_text(value: ValueRef<'_>) -> Option> { + match value { + ValueRef::Text(t) => Some(String::from_utf8_lossy(t)), + ValueRef::Integer(i) => Some(format_number(i as f64).into()), + ValueRef::Real(f) => Some(format_number(f).into()), + ValueRef::Null | ValueRef::Blob(_) => None, + } +} + +/// Adds the functions [`FlowQuery::sql`] compiles to: the value tests of the crate's +/// conditions, and the values fields read from a flow's columns. +pub(crate) fn add_functions(conn: &rusqlite::Connection) -> rusqlite::Result<()> { + let flags = FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC; + let text_arg = |ctx: &rusqlite::functions::Context, i| ctx.get::>(i); + + conn.create_scalar_function("q_contains", 2, flags, |ctx| { + let needle = ctx.get::(1)?; + Ok(match ctx.get_raw(0) { + ValueRef::Text(t) => contains_ignore_case(&String::from_utf8_lossy(t), &needle), + v => value_text(v).is_some_and(|t| t.contains(needle.as_str())), + }) + })?; + conn.create_scalar_function("q_wildcard", 2, flags, |ctx| { + let pattern = ctx.get::(1)?; + Ok(match ctx.get_raw(0) { + ValueRef::Text(t) => { + wildcard_match(&pattern, &String::from_utf8_lossy(t).to_lowercase()) + } + v => value_text(v).is_some_and(|t| wildcard_match(&pattern, &t)), + }) + })?; + conn.create_scalar_function("q_present", 1, flags, |ctx| { + Ok(match ctx.get_raw(0) { + ValueRef::Text(t) => !t.is_empty(), + ValueRef::Integer(_) | ValueRef::Real(_) => true, + ValueRef::Null | ValueRef::Blob(_) => false, + }) + })?; + conn.create_scalar_function("q_number", 1, flags, |ctx| { + Ok(match ctx.get_raw(0) { + ValueRef::Integer(i) => Some(i as f64), + ValueRef::Real(f) => Some(f), + ValueRef::Text(t) => read_number(&String::from_utf8_lossy(t)), + ValueRef::Null | ValueRef::Blob(_) => None, + }) + })?; + conn.create_scalar_function("q_lower", 1, flags, move |ctx| { + Ok(text_arg(ctx, 0)?.map(|t| t.to_lowercase())) + })?; + conn.create_scalar_function("flow_text", 4, flags, move |ctx| { + let url = text_arg(ctx, 0)?.unwrap_or_default(); + let method = text_arg(ctx, 1)?.unwrap_or_default(); + let content_type = text_arg(ctx, 3)?; + Ok(flow_text(&url, &method, ctx.get::>(2)?, content_type.as_deref())) + })?; + conn.create_scalar_function("url_hostname", 1, flags, move |ctx| { + Ok(url_hostname(&text_arg(ctx, 0)?.unwrap_or_default())) + })?; + conn.create_scalar_function("url_host", 1, flags, move |ctx| { + Ok(url_host(&text_arg(ctx, 0)?.unwrap_or_default())) + })?; + conn.create_scalar_function("url_port", 1, flags, move |ctx| { + Ok(url_port(&text_arg(ctx, 0)?.unwrap_or_default())) + })?; + conn.create_scalar_function("url_scheme", 1, flags, move |ctx| { + Ok(url_scheme(&text_arg(ctx, 0)?.unwrap_or_default())) + })?; + conn.create_scalar_function("url_path", 1, flags, move |ctx| { + Ok(url_path(&text_arg(ctx, 0)?.unwrap_or_default()).map(str::to_string)) + })?; + conn.create_scalar_function("url_sidebar_path", 1, flags, move |ctx| { + Ok(url_sidebar_path(&text_arg(ctx, 0)?.unwrap_or_default())) + })?; + conn.create_scalar_function("url_query_json", 1, flags, move |ctx| { + let pairs: Vec = url_query(&text_arg(ctx, 0)?.unwrap_or_default()) + .into_iter() + .map(|(name, value)| ProxyHeader { name, value }) + .collect(); + serde_json::to_string(&pairs).map_err(|e| rusqlite::Error::UserFunctionError(e.into())) + })?; + conn.create_scalar_function("header_value", 2, flags, move |ctx| { + let headers: Vec = + serde_json::from_str(&text_arg(ctx, 0)?.unwrap_or_default()).unwrap_or_default(); + let name = ctx.get::(1)?; + Ok(find_header(&headers, &name).map(str::to_string)) + })?; + conn.create_scalar_function("http_proto", 1, flags, move |ctx| { + Ok(text_arg(ctx, 0)?.and_then(|v| http_proto(&v))) + })?; + conn.create_scalar_function("stream_proto", 1, flags, move |ctx| { + Ok(text_arg(ctx, 0)?.and_then(|v| stream_proto(&v))) + }) +} + +#[cfg(test)] +mod tests; diff --git a/crates-proxy/yaak-proxy-lib/src/query/tests.rs b/crates-proxy/yaak-proxy-lib/src/query/tests.rs new file mode 100644 index 000000000..499ecf52a --- /dev/null +++ b/crates-proxy/yaak-proxy-lib/src/query/tests.rs @@ -0,0 +1,1001 @@ +//! The compiler's tests. Each term kind against flows made for it, then the drift guard: for a +//! sample of varied flows and a battery of queries, the database's answer and the evaluator's +//! must be the same flows. + +use super::*; +use crate::db::ProxyQueryManager; +use crate::models::StreamKind; +use chrono::Duration; +use yaak_database::UpdateSource; + +/// When the tests run: every flow's start and every `@time` counts from here. +fn now() -> DateTime { + DateTime::parse_from_rfc3339("2026-11-07T12:00:00Z").unwrap().with_timezone(&Utc) +} + +fn header(name: &str, value: &str) -> ProxyHeader { + ProxyHeader { name: name.into(), value: value.into() } +} + +/// A database holding `flows`, numbered in order, with what they read back as. +fn db_of(flows: Vec) -> (tempfile::TempDir, ProxyQueryManager, Vec) { + let dir = tempfile::tempdir().unwrap(); + let db = ProxyQueryManager::new(&dir.path().join("proxy.db"), &crate::db::test_key()).unwrap(); + let mut stored = Vec::new(); + for flow in flows { + let read = flow.agent_read; + let (mut saved, _) = db.with_tx(|tx| tx.upsert(&flow, &UpdateSource::Background)).unwrap(); + if read { + db.with_tx(|tx| { + tx.conn() + .resolve() + .execute("UPDATE flows SET agent_read = TRUE WHERE id = ?", [&saved.id]) + .map_err(yaak_database::Error::from) + }) + .unwrap(); + saved.agent_read = true; + } + stored.push(saved); + } + (dir, db, stored) +} + +/// The ids the database finds for `query`, sorted. +fn sql_ids(db: &ProxyQueryManager, query: &FlowQuery) -> Vec { + db.with_conn(|c| { + let conn = c.conn().resolve(); + let (condition, params) = query.sql(&FlowColumns::of(conn)); + let sql = format!("SELECT id FROM flows WHERE {condition} ORDER BY id"); + let mut stmt = conn.prepare(&sql).unwrap_or_else(|e| panic!("{e}: {sql}")); + stmt.query_map(rusqlite::params_from_iter(params), |r| r.get(0)) + .unwrap() + .collect::>>() + .unwrap() + }) +} + +/// The ids the evaluator finds for `query`, sorted. +fn memory_ids(flows: &[Flow], query: &FlowQuery) -> Vec { + let matcher = query.matcher(); + let mut ids: Vec = + flows.iter().filter(|f| matcher.matches(&document(f))).map(|f| f.id.clone()).collect(); + ids.sort(); + ids +} + +/// Flows made for the tests below, each named by what sets it apart. +fn fixture() -> Vec { + let at = |minutes_ago: i64| (now() - Duration::minutes(minutes_ago)).naive_utc(); + let base = |id: &str, url: &str| Flow { + id: id.into(), + url: url.into(), + method: "GET".into(), + client_http_version: "HTTP/1.1".into(), + started_at: at(1), + ..Default::default() + }; + vec![ + Flow { + status: Some(200), + content_type: Some("application/json; charset=utf-8".into()), + request_headers: vec![header("Accept", "application/json")], + response_headers: vec![ + header("X-Cache", "HIT"), + header("Content-Type", "application/json"), + ], + response_size: 250_000, + request_size: 120, + elapsed_ms: Some(820), + client_http_version: "HTTP/2".into(), + client_name: Some("Google Chrome".into()), + client_bundle_id: Some("com.google.Chrome".into()), + client_path: Some("/Applications/Google Chrome.app".into()), + client_pid: Some(501), + saved_request_id: Some("rq_1".into()), + started_at: at(2), + ..base("fl_github", "https://api.github.com/repos/yaak?page=2&sort=Stars") + }, + Flow { + method: "POST".into(), + status: Some(404), + content_type: Some("text/html".into()), + request_headers: vec![header("Content-Type", "application/x-www-form-urlencoded")], + response_headers: vec![header("x-cache", "MISS")], + response_size: 900, + elapsed_ms: Some(40), + client_name: Some("curl".into()), + client_path: Some("/usr/bin/curl".into()), + client_cwd: Some("/Users/ada/src/fieldwork".into()), + started_at: at(30), + ..base("fl_stripe", "http://api.stripe.com:8080/v1/charges/ch_1") + }, + Flow { + status: Some(503), + error: None, + request_headers: vec![header("content-type", "application/json")], + response_size: 2_000_000, + elapsed_ms: Some(3_000), + client_name: Some("nsurlsessiond".into()), + client_path: Some("/usr/libexec/nsurlsessiond".into()), + client_cwd: Some("/".into()), + started_at: at(90), + ..base("fl_daemon", "https://github.com/login") + }, + Flow { + method: "CONNECT".into(), + tunnelled: true, + response_size: 5_000, + elapsed_ms: Some(10_000), + started_at: at(60 * 25), + ..base("fl_tunnel", "https://bank.example.com") + }, + Flow { + error: Some("connection refused".into()), + started_at: at(3), + ..base("fl_error", "http://localhost:3000/health") + }, + Flow { + started_at: at(0), + agent_read: true, + ..base("fl_pending", "https://example.com/slow") + }, + Flow { + status: Some(101), + stream_kind: Some(StreamKind::Websocket), + elapsed_ms: Some(50), + ..base("fl_ws", "wss://ws.example.com/socket") + }, + Flow { + method: "POST".into(), + status: Some(200), + stream_kind: Some(StreamKind::Grpc), + grpc_status: Some(5), + client_http_version: "HTTP/2".into(), + elapsed_ms: Some(12), + ..base("fl_grpc", "https://grpc.example.com/pkg.Service/Method") + }, + Flow { status: Some(508), elapsed_ms: Some(1), ..base("fl_loop", "http://neverssl.com/") }, + ] +} + +/// The fixture's ids `query` finds, after checking the database and the evaluator agree. +fn finds(query: &str) -> Vec { + finds_with(query, false) +} + +fn finds_with(query: &str, hide_system: bool) -> Vec { + let (_dir, db, flows) = db_of(fixture()); + let prepared = FlowQuery::at(query, hide_system, now()); + let sql = sql_ids(&db, &prepared); + assert_eq!(sql, memory_ids(&flows, &prepared), "the two disagree on {query:?}"); + sql +} + +fn ids(ids: &[&str]) -> Vec { + let mut ids: Vec = ids.iter().map(|id| id.to_string()).collect(); + ids.sort(); + ids +} + +#[test] +fn bare_words_search_url_method_status_and_type() { + assert_eq!(finds("GITHUB"), ids(&["fl_daemon", "fl_github"])); + assert_eq!(finds("post"), ids(&["fl_grpc", "fl_stripe"])); + assert_eq!(finds("404"), ids(&["fl_stripe"])); + assert_eq!(finds("html"), ids(&["fl_stripe"])); + assert_eq!(finds("\"api.github.com/repos\""), ids(&["fl_github"])); + assert_eq!(finds("github repos"), ids(&["fl_github"])); + // Headers are not free text + assert_eq!(finds("HIT"), ids(&[])); +} + +#[test] +fn url_fields() { + assert_eq!(finds("@host:api.github.com"), ids(&["fl_github"])); + assert_eq!(finds("@host:github"), ids(&[])); + assert_eq!(finds("@host:*github*"), ids(&["fl_daemon", "fl_github"])); + // A site is its domain and every host below it + assert_eq!(finds("@host:*.github.com"), ids(&["fl_daemon", "fl_github"])); + assert_eq!(finds("@host:*.example.com"), ids(&["fl_grpc", "fl_pending", "fl_tunnel", "fl_ws"])); + // A host on another port, with or without it + assert_eq!(finds("@host:api.stripe.com"), ids(&["fl_stripe"])); + assert_eq!(finds("@host:api.stripe.com:8080"), ids(&["fl_stripe"])); + assert_eq!(finds("@port:8080"), ids(&["fl_stripe"])); + assert_eq!(finds("@port:443 @scheme:wss"), ids(&["fl_ws"])); + assert_eq!(finds("@port:<1000 @scheme:http"), ids(&["fl_loop"])); + assert_eq!(finds("@scheme:http"), ids(&["fl_error", "fl_loop", "fl_stripe"])); + assert_eq!(finds("@path:/v1"), ids(&["fl_stripe"])); + assert_eq!(finds("@path:/v1/*"), ids(&["fl_stripe"])); + assert_eq!(finds("@path:/v1/charges/*"), ids(&["fl_stripe"])); + assert_eq!(finds("@path:/v1/charges/ch_1/*"), ids(&["fl_stripe"])); + assert_eq!(finds("@path:/v1/charge/*"), ids(&[])); + assert_eq!( + finds("@path:/"), + ids(&[ + "fl_daemon", + "fl_error", + "fl_github", + "fl_grpc", + "fl_loop", + "fl_pending", + "fl_stripe", + "fl_ws" + ]) + ); + assert_eq!(finds("@url:*page=2*"), ids(&["fl_github"])); + assert_eq!(finds("@query.page:2"), ids(&["fl_github"])); + assert_eq!(finds("@query.SORT:stars"), ids(&["fl_github"])); + assert_eq!(finds("@query.page:*"), ids(&["fl_github"])); + assert_eq!(finds("@query.page:>1"), ids(&["fl_github"])); + assert_eq!(finds("@query:stars"), ids(&["fl_github"])); + assert_eq!( + finds("-@query.page:*"), + ids(&[ + "fl_daemon", + "fl_error", + "fl_grpc", + "fl_loop", + "fl_pending", + "fl_stripe", + "fl_tunnel", + "fl_ws" + ]) + ); +} + +#[test] +fn request_and_response_fields() { + assert_eq!(finds("@method:post"), ids(&["fl_grpc", "fl_stripe"])); + assert_eq!(finds("@method:CONN*"), ids(&["fl_tunnel"])); + assert_eq!(finds("@type:json"), ids(&["fl_daemon", "fl_github"])); + // The request's only when the response has none + assert_eq!(finds("@type:form"), ids(&[])); + assert_eq!(finds("@proto:h2"), ids(&["fl_github", "fl_grpc"])); + assert_eq!(finds("@proto:ws"), ids(&["fl_ws"])); + assert_eq!(finds("@proto:grpc"), ids(&["fl_grpc"])); + assert_eq!(finds("@proto:h1 @method:post"), ids(&["fl_stripe"])); + assert_eq!(finds("@id:2"), ids(&["fl_stripe"])); + assert_eq!(finds("@id:>=8"), ids(&["fl_grpc", "fl_loop"])); +} + +#[test] +fn statuses() { + assert_eq!(finds("@status:404"), ids(&["fl_stripe"])); + assert_eq!(finds("@status:4xx"), ids(&["fl_stripe"])); + assert_eq!(finds("@status:5XX"), ids(&["fl_daemon", "fl_loop"])); + assert_eq!(finds("@status:>=400"), ids(&["fl_daemon", "fl_loop", "fl_stripe"])); + assert_eq!(finds("@status:[200 TO 299]"), ids(&["fl_github", "fl_grpc"])); + assert_eq!(finds("@status:50*"), ids(&["fl_daemon", "fl_loop"])); + assert_eq!( + finds("@status:*"), + ids(&[ + "fl_daemon", + "fl_github", + "fl_grpc", + "fl_loop", + "fl_stripe", + "fl_ws" + ]) + ); + assert_eq!(finds("-@status:*"), ids(&["fl_error", "fl_pending", "fl_tunnel"])); + assert_eq!(finds("@status:40"), ids(&[])); +} + +#[test] +fn sizes_and_durations_with_units() { + assert_eq!(finds("@size:>100kb"), ids(&["fl_daemon", "fl_github"])); + assert_eq!(finds("@size:>1mb"), ids(&["fl_daemon"])); + assert_eq!(finds("@res.size:[1kb TO 10kb]"), ids(&["fl_tunnel"])); + assert_eq!(finds("@req.size:120b"), ids(&["fl_github"])); + assert_eq!(finds("@size:>1.5gb"), ids(&[])); + assert_eq!(finds("@duration:>500ms"), ids(&["fl_daemon", "fl_github", "fl_tunnel"])); + assert_eq!(finds("@duration:>=3s"), ids(&["fl_daemon", "fl_tunnel"])); + assert_eq!(finds("@duration:<1m @duration:>1s"), ids(&["fl_daemon", "fl_tunnel"])); + // A unit the field does not take matches nothing + assert_eq!(finds("@duration:>1h"), ids(&[])); + assert_eq!(finds("-@duration:*"), ids(&["fl_error", "fl_pending"])); +} + +#[test] +fn times_ago_and_absolute() { + assert_eq!( + finds("@time:<5m"), + ids(&[ + "fl_error", + "fl_github", + "fl_grpc", + "fl_loop", + "fl_pending", + "fl_ws" + ]) + ); + assert_eq!(finds("@time:<=30m @time:>5m"), ids(&["fl_stripe"])); + assert_eq!(finds("@time:>1h"), ids(&["fl_daemon", "fl_tunnel"])); + assert_eq!(finds("@time:>1d"), ids(&["fl_tunnel"])); + assert_eq!(finds("@time:[1h TO 2h]"), ids(&["fl_daemon"])); + assert_eq!(finds("@time:<1w"), finds("@time:*")); + // An instant with its offset + assert_eq!(finds("@time:<2026-11-07T11:00:00Z"), ids(&["fl_daemon", "fl_tunnel"])); + assert_eq!( + finds("@time:>=2026-11-07T11:30:00Z @time:<2026-11-07T11:58:00Z"), + ids(&["fl_error", "fl_stripe"]) + ); + // Local: the day the fixture's now falls on holds all but the tunnel + let today = now().with_timezone(&Local).format("%Y-%m-%d").to_string(); + let yesterday_ish = finds(&format!("@time:<{today}")); + let on_day = finds(&format!("@time:{today}")); + assert_eq!(yesterday_ish.len() + on_day.len(), fixture().len(), "{yesterday_ish:?} {on_day:?}"); + assert!(yesterday_ish.contains(&"fl_tunnel".to_string())); + // A time today, local: the minute two minutes before now holds the GitHub flow + let minute = (now() - Duration::minutes(2)).with_timezone(&Local).format("%H:%M").to_string(); + let by_minute = finds(&format!("@time:{minute}")); + if (now() - Duration::minutes(2)).with_timezone(&Local).date_naive() + == now().with_timezone(&Local).date_naive() + { + assert_eq!(by_minute, ids(&["fl_github"])); + } + // A time that is none of these matches nothing + assert_eq!(finds("@time:soon"), ids(&[])); + assert_eq!(finds("@time:5m"), ids(&[])); +} + +#[test] +fn sources() { + assert_eq!(finds("@app:\"google chrome\""), ids(&["fl_github"])); + assert_eq!(finds("@app:com.google.Chrome"), ids(&["fl_github"])); + assert_eq!(finds("@app:chrome"), ids(&[])); + assert_eq!(finds("@app:*chrome*"), ids(&["fl_github"])); + // A program outside an app, by its name, its path or its folder + assert_eq!(finds("@app:curl"), ids(&["fl_stripe"])); + assert_eq!(finds("@app:/usr/bin/curl"), ids(&["fl_stripe"])); + assert_eq!(finds("@app:FIELDWORK"), ids(&["fl_stripe"])); + assert_eq!(finds("@cwd:fieldwork"), ids(&["fl_stripe"])); + assert_eq!(finds("@cwd:/Users/ada/src/fieldwork"), ids(&["fl_stripe"])); + assert_eq!(finds("@cwd:*"), ids(&["fl_daemon", "fl_stripe"])); + assert_eq!(finds("@pid:501"), ids(&["fl_github"])); +} + +#[test] +fn headers_on_either_side_or_one() { + assert_eq!(finds("@header.x-cache:hit"), ids(&["fl_github"])); + assert_eq!(finds("@header.X-CACHE:*"), ids(&["fl_github", "fl_stripe"])); + assert_eq!(finds("@res.header.x-cache:*"), ids(&["fl_github", "fl_stripe"])); + assert_eq!(finds("@req.header.x-cache:*"), ids(&[])); + assert_eq!(finds("@header.content-type:json"), ids(&["fl_daemon", "fl_github"])); + assert_eq!(finds("@req.header.content-type:json"), ids(&["fl_daemon"])); + assert_eq!(finds("@res.header.content-type:json"), ids(&["fl_github"])); + assert_eq!(finds("@header.accept:application/*"), ids(&["fl_github"])); + assert_eq!(finds("@header:miss"), ids(&["fl_stripe"])); + assert_eq!( + finds("-@header.x-cache:hit"), + ids(&[ + "fl_daemon", + "fl_error", + "fl_grpc", + "fl_loop", + "fl_pending", + "fl_stripe", + "fl_tunnel", + "fl_ws" + ]) + ); +} + +#[test] +fn flags() { + assert_eq!(finds("@is:system"), ids(&["fl_daemon"])); + assert_eq!(finds("@is:tunnelled"), ids(&["fl_tunnel"])); + assert_eq!(finds("@is:error"), ids(&["fl_error", "fl_grpc"])); + assert_eq!(finds("@is:pending"), ids(&["fl_pending"])); + assert_eq!(finds("@is:saved"), ids(&["fl_github"])); + assert_eq!(finds("@is:read"), ids(&["fl_pending"])); + assert_eq!(finds("@is:loop"), ids(&["fl_loop"])); + assert_eq!(finds("@is:ws"), ids(&["fl_ws"])); + assert_eq!(finds("@is:sse"), ids(&[])); + assert_eq!(finds("@is:GRPC"), ids(&["fl_grpc"])); + assert_eq!( + finds("@is:*"), + ids(&[ + "fl_daemon", + "fl_error", + "fl_github", + "fl_grpc", + "fl_loop", + "fl_pending", + "fl_tunnel", + "fl_ws" + ]) + ); + assert_eq!(finds("@is:s*"), ids(&["fl_daemon", "fl_github"])); + // Not yet, and not a flag + assert_eq!(finds("@is:marked"), ids(&[])); + assert_eq!(finds("@is:nope"), ids(&[])); + assert_eq!(finds("-@is:nope").len(), fixture().len()); +} + +#[test] +fn groups_and_negation() { + assert_eq!(finds("@status:4xx OR @is:tunnelled"), ids(&["fl_stripe", "fl_tunnel"])); + assert_eq!(finds("(@status:4xx OR @is:tunnelled) -@method:connect"), ids(&["fl_stripe"])); + assert_eq!(finds("-(@status:* OR @is:pending)"), ids(&["fl_error", "fl_tunnel"])); + assert_eq!(finds("github -@host:github.com"), ids(&["fl_github"])); + assert_eq!( + finds("@host:*.example.com AND -@is:ws"), + ids(&["fl_grpc", "fl_pending", "fl_tunnel"]) + ); +} + +#[test] +fn bodies_and_unknown_fields_match_nothing() { + assert_eq!(finds("@body:x"), ids(&[])); + assert_eq!(finds("@res.body:*"), ids(&[])); + assert_eq!(finds("@nope:x"), ids(&[])); + assert_eq!(finds("-@body:x").len(), fixture().len()); +} + +#[test] +fn hiding_system() { + let all = fixture().len(); + assert_eq!(finds_with("", true).len(), all - 1); + assert!(!finds_with("", true).contains(&"fl_daemon".to_string())); + assert_eq!(finds_with("github", true), ids(&["fl_github"])); + // The search names System itself, either way + assert_eq!(finds_with("@is:system", true), ids(&["fl_daemon"])); + assert_eq!(finds_with("github OR @is:System", true), ids(&["fl_daemon", "fl_github"])); + assert_eq!(finds_with("-@is:system", true).len(), all - 1); + assert!(FlowQuery::new("", true).matcher().matches(&document(&fixture()[0]))); + assert!(!FlowQuery::new("", true).is_empty()); + assert!(FlowQuery::new(" ", false).is_empty()); +} + +#[test] +fn diagnostics_mark_what_cannot_run() { + let kinds = + |q: &str| diagnostics(q).into_iter().map(|d| (d.kind, d.start, d.end)).collect::>(); + use QueryDiagnosticKind::*; + assert_eq!(kinds("@status:4xx @time:<5m @host:*.a.com"), vec![]); + assert_eq!(kinds("@nope:x"), vec![(UnknownField, 0, 6)]); + assert_eq!(kinds("@size:>5parsecs"), vec![(InvalidValue, 6, 15)]); + assert_eq!(kinds("@time:soon"), vec![(InvalidValue, 6, 10)]); + assert_eq!(kinds("@is:nope"), vec![(InvalidValue, 4, 8)]); + assert_eq!(kinds("@is:marked"), vec![(NotYet, 0, 10)]); + assert_eq!(kinds("x @body:token"), vec![(NotYet, 2, 13)]); + // Spans count UTF-16 units + assert_eq!(kinds("é😀 @nope:x"), vec![(UnknownField, 4, 10)]); + // Typing a field is not yet a mistake + assert_eq!(kinds("@is:"), vec![]); + assert_eq!(kinds("@time:"), vec![]); +} + +#[test] +fn time_is_compared_as_stored() { + // The database compares `started_at` as text: what this writes must be how it is stored. + let (_dir, db, flows) = db_of(fixture()); + let stored: Vec = db.with_conn(|c| { + let conn = c.conn().resolve(); + let mut stmt = conn.prepare("SELECT started_at FROM flows ORDER BY rowid").unwrap(); + stmt.query_map([], |r| r.get(0)).unwrap().map(|r| r.unwrap()).collect() + }); + for (flow, stored) in flows.iter().zip(stored) { + assert_eq!(stored_time(started_ms(&flow.started_at)), stored); + } + let precise = + NaiveDateTime::parse_from_str("2026-11-07 10:00:00.123456", "%Y-%m-%d %H:%M:%S%.f") + .unwrap(); + assert_eq!(stored_time(started_ms(&precise)), "2026-11-07 10:00:00.123456"); +} + +#[test] +fn reads_url_parts() { + assert_eq!(url_hostname("https://user:pw@API.Example.com:8443/x"), "api.example.com"); + assert_eq!(url_host("https://API.Example.com:8443/x"), "api.example.com:8443"); + assert_eq!(url_hostname("http://[::1]:8080/"), "[::1]"); + assert_eq!(url_port("http://[::1]:8080/"), Some(8080)); + assert_eq!(url_port("https://a.com/"), Some(443)); + assert_eq!(url_port("bank.example.com:443"), Some(443)); + assert_eq!(url_scheme("bank.example.com:443"), None); + assert_eq!(url_path("https://a.com/x/y?q=1#f"), Some("/x/y")); + assert_eq!(url_path("https://a.com?q=1"), Some("")); + assert_eq!(url_sidebar_path("https://a.com/v1//users/"), Some("/v1/users".into())); + assert_eq!(url_sidebar_path("https://a.com/v1/users"), None); + assert_eq!(url_sidebar_path("https://a.com"), None); + assert_eq!( + url_query("https://a.com/?A=1&b=%20two&a=3#f"), + vec![ + ("a".to_string(), "1".to_string()), + ("b".to_string(), " two".to_string()), + ("a".to_string(), "3".to_string()), + ] + ); +} + +// -- The drift guard -- + +/// A small deterministic generator, so a failure names a flow that comes back every run. +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + self.0 >> 33 + } + + fn pick<'a, T>(&mut self, items: &'a [T]) -> &'a T { + &items[self.next() as usize % items.len()] + } + + fn chance(&mut self, percent: u64) -> bool { + self.next() % 100 < percent + } +} + +/// Flows as varied as the fields can tell apart: hosts with and without ports and schemes, +/// queries and their encodings, any case, text outside ASCII, every flag, and the empty cases. +fn sample(count: usize) -> Vec { + let mut rng = Rng(7); + let hosts = [ + "https://api.github.com", + "https://github.com", + "http://API.Stripe.com:8080", + "https://api.stripe.community", + "http://localhost:3000", + "https://[::1]:8443", + "wss://ws.example.com", + "https://café.example.org", + "http://neverssl.com:80", + "https://user:pw@intranet.local", + "bank.example.com:443", + ]; + let paths = [ + "", + "/", + "/v1/charges", + "/v1/charges/ch_1", + "/v1//users", + "/repos/yaak", + "/ÜBER/x", + "/a%20b", + ]; + let queries = [ + "", + "?page=2", + "?page=10&sort=Stars", + "?q=%C3%A9t%C3%A9", + "?a=1&a=2", + "?flag", + "?page=two", + ]; + let methods = ["GET", "POST", "put", "DELETE", "CONNECT", "HEAD"]; + let statuses = [ + None, + Some(101), + Some(200), + Some(204), + Some(301), + Some(404), + Some(500), + Some(503), + Some(508), + ]; + let types = [ + None, + Some("application/json"), + Some("Application/JSON; charset=UTF-8"), + Some("text/html"), + Some("image/png"), + Some(""), + ]; + let header_names = [ + "Content-Type", + "accept", + "X-Cache", + "x-trace", + "Set-Cookie", + "ÉTAG", + ]; + let header_values = [ + "application/json", + "HIT", + "miss", + "", + "a=1; Path=/", + "200", + "Wörld", + "text/*", + ]; + // Name, bundle id, path and working directory + type App<'a> = (Option<&'a str>, Option<&'a str>, Option<&'a str>, Option<&'a str>); + let apps: [App; 7] = [ + (None, None, None, None), + ( + Some("Google Chrome"), + Some("com.google.Chrome"), + Some("/Applications/Google Chrome.app"), + None, + ), + (Some("curl"), None, Some("/usr/bin/curl"), Some("/Users/ada/src/fieldwork")), + (Some("node"), None, Some("/opt/homebrew/bin/node"), Some("/Users/ada/src/Ledger/")), + (Some("nsurlsessiond"), None, Some("/usr/libexec/nsurlsessiond"), Some("/")), + (Some("trustd"), None, Some("/somewhere/trustd"), None), + ( + Some("Slack"), + Some("com.tinyspeck.slackmacgap"), + Some("/Applications/Slack.app"), + Some("/tmp"), + ), + ]; + let versions = ["HTTP/1.1", "HTTP/2", "HTTP/1.0", ""]; + let streams = [ + None, + None, + None, + Some(StreamKind::Websocket), + Some(StreamKind::Sse), + Some(StreamKind::Grpc), + Some(StreamKind::GrpcWeb), + ]; + let sizes = [0, 1, 999, 1000, 1001, 250_000, 1_000_000, 2_500_000_000]; + let elapsed = [ + None, + Some(0), + Some(12), + Some(500), + Some(501), + Some(1500), + Some(60_000), + Some(61_000), + ]; + let ages_s = [ + 0, 30, 59, 60, 61, 299, 300, 301, 3599, 3600, 7200, 86_399, 86_400, 90_000, 700_000, + ]; + + (0..count) + .map(|i| { + let tunnelled = rng.chance(8); + let url = if tunnelled { + rng.pick(&["bank.example.com:443", "https://bank.example.com"]).to_string() + } else { + format!("{}{}{}", rng.pick(&hosts), rng.pick(&paths), rng.pick(&queries)) + }; + let headers = |rng: &mut Rng| { + (0..rng.next() % 4) + .map(|_| { + header(rng.pick::<&str>(&header_names), rng.pick::<&str>(&header_values)) + }) + .collect::>() + }; + let (name, bundle, path, cwd) = *rng.pick(&apps); + let age = *rng.pick(&ages_s) as i64; + let started_at = (now() + - Duration::seconds(age) + - Duration::microseconds((rng.next() % 3) as i64 * 250)) + .naive_utc(); + Flow { + id: format!("fl_{i:03}"), + url, + method: rng.pick(&methods).to_string(), + status: *rng.pick(&statuses), + content_type: rng.pick(&types).map(String::from), + request_headers: headers(&mut rng), + response_headers: headers(&mut rng), + request_size: *rng.pick(&sizes), + response_size: *rng.pick(&sizes), + elapsed_ms: *rng.pick(&elapsed), + started_at, + client_http_version: rng.pick(&versions).to_string(), + client_name: name.map(String::from), + client_bundle_id: bundle.map(String::from), + client_path: path.map(String::from), + client_cwd: cwd.map(String::from), + client_pid: rng.chance(50).then(|| (rng.next() % 1000) as i32), + stream_kind: *rng.pick(&streams), + grpc_status: *rng.pick(&[None, Some(0), Some(5)]), + error: rng.chance(15).then(|| "connection reset".to_string()), + cancelled: rng.chance(10), + saved_request_id: rng.chance(15).then(|| "rq_x".to_string()), + tunnelled, + agent_read: rng.chance(20), + ..Default::default() + } + }) + .collect() +} + +/// Queries over every field kind and value shape, and the ways they combine. +const BATTERY: &[&str] = &[ + "", + "github", + "GITHUB stripe", + "json", + "post", + "50", + "é", + "café", + "\"api.github.com/repos\"", + "\"GET\"", + "%", + "_", + "@host:api.github.com", + "@host:API.STRIPE.COM", + "@host:api.stripe.com:8080", + "@host:*.github.com", + "@host:*.example.com", + "@host:*stripe*", + "@host:café.example.org", + "@host:[::1]", + "@host:*", + "-@host:*.github.com", + "@port:443", + "@port:>=8000", + "@port:[80 TO 3000]", + "@port:*", + "-@port:*", + "@scheme:https", + "@scheme:ws*", + "-@scheme:*", + "@path:/v1", + "@path:/v1/*", + "@path:/v1/charges/*", + "@path:\"/v1/charges\"", + "@path:*users*", + "@path:/über/*", + "@path:/", + "@path:*", + "@url:*page=*", + "@url:\"%20\"", + "@query.page:2", + "@query.page:>3", + "@query.page:[1 TO 5]", + "@query.page:*", + "@query.sort:STARS", + "@query.q:été", + "@query.a:2", + "@query.flag:*", + "@query:2", + "-@query:*", + "@method:get", + "@method:P*", + "@method:*", + "@type:json", + "@type:application/json", + "@type:*json*", + "@type:*", + "-@type:*", + "@proto:h1", + "@proto:h2", + "@proto:ws", + "@proto:grpc", + "@proto:sse", + "-@proto:*", + "@id:7", + "@id:>20", + "@id:[5 TO 9]", + "@status:200", + "@status:2xx", + "@status:5XX", + "@status:1xx", + "@status:>=400", + "@status:<300", + "@status:[300 TO 499]", + "@status:50*", + "@status:*", + "-@status:*", + "@status:\"50\"", + "@status:abc", + "@status:4xy", + "@size:>100kb", + "@size:1000", + "@size:1kb", + "@size:>=1mb", + "@size:[1b TO 1kb]", + "@size:>2gb", + "@req.size:<1kb", + "@res.size:*", + "@size:>5parsecs", + "@duration:>500ms", + "@duration:500", + "@duration:>=1.5s", + "@duration:[1s TO 1m]", + "@duration:*", + "-@duration:*", + "@duration:>1h", + "@time:<5m", + "@time:<=1m", + "@time:>1h", + "@time:>=1d", + "@time:[1m TO 1h]", + "@time:[1h TO 1m]", + "@time:<500ms", + "@time:*", + "@time:<2026-11-07T11:00:00Z", + "@time:>2026-11-07T11:59:00Z", + "@time:2026-11-07T11:59:59Z", + "@time:2026-11-07", + "@time:<2026-11-07", + "@time:>=2026-11-06", + "@time:11:59", + "@time:>11:00", + "@time:<=11:59:30", + "@time:soon", + "@app:\"google chrome\"", + "@app:com.google.Chrome", + "@app:curl", + "@app:fieldwork", + "@app:ledger", + "@app:/usr/bin/curl", + "@app:*", + "@app:*o*", + "-@app:*", + "@cwd:fieldwork", + "@cwd:/tmp", + "@cwd:*", + "@cwd:/", + "@pid:>500", + "@pid:*", + "@header.content-type:json", + "@header.Content-Type:*", + "@header.x-cache:hit", + "@header.x-cache:*", + "-@header.x-cache:*", + "@header.accept:text/*", + "@header.étag:wörld", + "@header.set-cookie:\"Path=/\"", + "@header:miss", + "@header:*", + "@header:200", + "@header.x-trace:>100", + "@req.header.content-type:json", + "@res.header.x-cache:*", + "@res.header:hit", + "-@req.header:*", + "@is:system", + "-@is:system", + "@is:tunnelled", + "@is:error", + "@is:pending", + "@is:saved", + "@is:read", + "@is:loop", + "@is:ws", + "@is:sse", + "@is:grpc", + "@is:*", + "-@is:*", + "@is:s*", + "@is:marked", + "@is:nope", + "@body:x", + "-@res.body:*", + "@nope:x", + "-@nope:x", + "@status:4xx OR @is:tunnelled", + "(@status:4xx OR @is:error) -@method:post", + "-(@status:* OR @is:pending)", + "github -@host:github.com", + "@host:*.example.com AND -@is:ws", + "(@app:curl OR @app:node) (@host:*.github.com OR @path:/v1/*)", + "@app:Slack @host:a.com OR @app:Chrome @host:api.github.com", + "-(-@is:system)", + "json @size:>1kb OR @duration:<100ms @proto:h2", +]; + +#[test] +fn the_database_and_the_evaluator_find_the_same_flows() { + let (_dir, db, flows) = db_of(sample(400)); + let mut checked = 0; + for query in BATTERY { + for hide_system in [false, true] { + let prepared = FlowQuery::at(query, hide_system, now()); + let sql = sql_ids(&db, &prepared); + let memory = memory_ids(&flows, &prepared); + if sql != memory { + let only_sql: Vec<_> = sql.iter().filter(|id| !memory.contains(id)).collect(); + let only_memory: Vec<_> = memory.iter().filter(|id| !sql.contains(id)).collect(); + let example = only_sql + .first() + .or(only_memory.first()) + .map(|id| flows.iter().find(|f| &&f.id == id).cloned()); + panic!( + "{query:?} (hide_system {hide_system}): only in SQL {only_sql:?}, \ + only in memory {only_memory:?}; e.g. {example:#?}" + ); + } + checked += 1; + } + } + assert_eq!(checked, BATTERY.len() * 2); + // The battery means something only if the sample spreads its answers. + let spread = BATTERY + .iter() + .map(|q| sql_ids(&db, &FlowQuery::at(q, false, now())).len()) + .filter(|n| *n > 0 && *n < flows.len()) + .count(); + assert!(spread > BATTERY.len() * 2 / 3, "only {spread} queries split the sample"); +} + +/// Live flows are matched as the list is: a flow's event and a listing agree. +#[test] +fn live_search_matches_as_the_list_does() { + let live = LiveSearch::default(); + let flow = fixture().remove(0); + assert_eq!(live.check(&flow), None); + live.watch(Some(("k1".into(), "@status:2xx @host:*.github.com".into(), false))); + assert_eq!(live.check(&flow), Some(SearchMatch { key: "k1".into(), matches: true })); + live.watch(Some(("k2".into(), "@status:4xx".into(), false))); + let payload = live.payload(flow, true); + assert_eq!(payload.search, Some(SearchMatch { key: "k2".into(), matches: false })); + live.watch(None); + assert_eq!(live.check(&payload.model), None); +} + +#[test] +fn seen_values_for_completion() { + let (_dir, db, _) = db_of(fixture()); + let values = |path: &str| seen_values(&db, path).unwrap(); + let mut hosts = values("host"); + hosts.sort(); + assert!(hosts.contains(&"api.stripe.com:8080".to_string()), "{hosts:?}"); + assert_eq!(values("method").len(), 3); + assert!(values("status").contains(&"404".to_string())); + let types = values("type"); + assert!(types.contains(&"application/json".to_string()), "{types:?}"); + assert!(types.contains(&"text/html".to_string()), "{types:?}"); + assert_eq!(values("cwd"), vec!["fieldwork".to_string()]); + assert!(values("app").contains(&"Google Chrome".to_string())); + assert_eq!(values("is"), FLAGS.iter().map(|f| f.to_string()).collect::>()); + let mut names = values("header"); + names.sort(); + assert_eq!(names, vec!["accept", "content-type", "x-cache"]); + let mut response_names = values("res.header"); + response_names.sort(); + assert_eq!(response_names, vec!["content-type", "x-cache"]); + let mut cache = values("header.X-Cache"); + cache.sort(); + assert_eq!(cache, vec!["HIT", "MISS"]); + // Near-unique fields offer nothing + for path in [ + "url", + "path", + "id", + "pid", + "size", + "duration", + "time", + "query", + "query.page", + ] { + assert!(values(path).is_empty(), "{path}"); + } +} + +#[test] +fn a_header_with_many_values_offers_none() { + let flows = (0..25) + .map(|i| Flow { + id: format!("fl_{i}"), + url: "https://a.com/".into(), + response_headers: vec![ + header("Date", &format!("day {i}")), + header("Server", "nginx"), + ], + ..Default::default() + }) + .collect(); + let (_dir, db, _) = db_of(flows); + assert!(seen_values(&db, "header.date").unwrap().is_empty()); + assert_eq!(seen_values(&db, "res.header.server").unwrap(), vec!["nginx"]); +} + +/// Completion shows each field with what it is and a term using it, and every such term runs. +#[test] +fn fields_have_descriptions_and_examples_that_run() { + for field in &schema().fields { + assert!(field.info.as_deref().is_some_and(|i| !i.is_empty()), "{}", field.name); + if FIELDS_NOT_YET.contains(&field.name.as_str()) { + continue; + } + let example = field.example.as_deref().unwrap_or_else(|| panic!("{}", field.name)); + assert!(example.contains(&format!("@{}", field.name)), "{example}"); + assert_eq!(diagnostics(example), vec![], "{example}"); + } +} diff --git a/crates-proxy/yaak-proxy-lib/src/saved_queries.rs b/crates-proxy/yaak-proxy-lib/src/saved_queries.rs new file mode 100644 index 000000000..7524e86be --- /dev/null +++ b/crates-proxy/yaak-proxy-lib/src/saved_queries.rs @@ -0,0 +1,181 @@ +//! Searches saved by name, which the sidebar lists with a count each. + +use crate::db::ProxyQueryManager; +use crate::flows::{self, FlowFilter}; +use serde::{Deserialize, Serialize}; +use ts_rs::TS; +use yaak_database::generate_prefixed_id; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_models.ts")] +pub struct SavedQuery { + pub id: String, + pub name: String, + /// The search, as the search box takes it. + pub query: String, + /// Where it sits in the list, lowest first. + #[ts(type = "number")] + pub sort_order: i64, +} + +/// How many flows a saved query covers. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct SavedQueryCount { + pub id: String, + #[ts(type = "number")] + pub count: usize, +} + +fn failed(what: &str, e: impl std::fmt::Display) -> String { + format!("Failed to {what} the saved search: {e}") +} + +fn read(row: &rusqlite::Row) -> rusqlite::Result { + Ok(SavedQuery { + id: row.get("id")?, + name: row.get("name")?, + query: row.get("query")?, + sort_order: row.get("sort_order")?, + }) +} + +/// Every saved query, in order. None for a database from before them, read without migrating. +pub fn list(db: &ProxyQueryManager) -> Result, String> { + db.with_conn(|c| { + let conn = c.conn().resolve(); + let Ok(mut stmt) = + conn.prepare("SELECT * FROM saved_queries ORDER BY sort_order, created_at, id") + else { + return Ok(Vec::new()); + }; + stmt.query_map([], read) + .and_then(|rows| rows.collect::>>()) + .map_err(|e| failed("list", e)) + }) +} + +fn name_of(name: &str) -> Result { + let name = name.trim(); + if name.is_empty() { + return Err("A saved search needs a name".into()); + } + Ok(name.to_string()) +} + +/// Saves `query` as `name`, after the others. +pub fn create(db: &ProxyQueryManager, name: &str, query: &str) -> Result { + let name = name_of(name)?; + let query = query.trim(); + if query.is_empty() { + return Err("Nothing to save".into()); + } + let id = generate_prefixed_id("sq"); + db.with_tx(|c| { + let conn = c.conn().resolve(); + conn.query_row( + "INSERT INTO saved_queries (id, name, query, sort_order) \ + VALUES (?1, ?2, ?3, (SELECT coalesce(max(sort_order), -1) + 1 FROM saved_queries)) \ + RETURNING *", + rusqlite::params![id, name, query], + read, + ) + .map_err(yaak_database::Error::from) + }) + .map_err(|e| failed("save", e)) +} + +pub fn rename(db: &ProxyQueryManager, id: &str, name: &str) -> Result { + let name = name_of(name)?; + db.with_tx(|c| { + c.conn() + .resolve() + .query_row( + "UPDATE saved_queries SET name = ?2, updated_at = CURRENT_TIMESTAMP WHERE id = ?1 \ + RETURNING *", + rusqlite::params![id, name], + read, + ) + .map_err(yaak_database::Error::from) + }) + .map_err(|e| failed("rename", e)) +} + +/// Deletes one. False when there was none. +pub fn delete(db: &ProxyQueryManager, id: &str) -> Result { + db.with_tx(|c| { + c.conn() + .resolve() + .execute("DELETE FROM saved_queries WHERE id = ?", [id]) + .map(|n| n > 0) + .map_err(yaak_database::Error::from) + }) + .map_err(|e| failed("delete", e)) +} + +/// Each saved query's count of flows, with System's left out when `hide_system`, as the list +/// would show it, read from one snapshot. +pub fn counts(db: &ProxyQueryManager, hide_system: bool) -> Result, String> { + let saved = list(db)?; + let filters: Vec = saved + .iter() + .map(|s| FlowFilter { hide_system, ..FlowFilter::search(s.query.clone()) }) + .collect(); + let counts = flows::count_each(db, &filters)?; + Ok(saved + .into_iter() + .zip(counts) + .map(|(s, count)| SavedQueryCount { id: s.id, count }) + .collect()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn saves_renames_counts_and_deletes() { + use crate::models::Flow; + use yaak_database::UpdateSource; + let dir = tempfile::tempdir().unwrap(); + let db = + ProxyQueryManager::new(&dir.path().join("proxy.db"), &crate::db::test_key()).unwrap(); + for (i, status) in [200, 404, 500].into_iter().enumerate() { + let flow = Flow { + id: format!("fl_{i}"), + url: format!("https://example.com/{i}"), + status: Some(status), + ..Default::default() + }; + db.with_tx(|tx| tx.upsert(&flow, &UpdateSource::Background)).unwrap(); + } + + let errors = create(&db, " Errors ", "@status:>=400").unwrap(); + let ok = create(&db, "OK", "@status:2xx").unwrap(); + assert_eq!((errors.name.as_str(), errors.sort_order, ok.sort_order), ("Errors", 0, 1)); + assert!(create(&db, " ", "x").is_err()); + assert!(create(&db, "x", " ").is_err()); + + let renamed = rename(&db, &ok.id, "Fine").unwrap(); + assert_eq!(renamed.name, "Fine"); + assert_eq!( + list(&db).unwrap().iter().map(|s| s.name.as_str()).collect::>(), + ["Errors", "Fine"] + ); + + let counts = counts(&db, false).unwrap(); + assert_eq!( + counts, + vec![ + SavedQueryCount { id: errors.id.clone(), count: 2 }, + SavedQueryCount { id: ok.id.clone(), count: 1 }, + ] + ); + + assert!(delete(&db, &errors.id).unwrap()); + assert!(!delete(&db, &errors.id).unwrap()); + assert_eq!(list(&db).unwrap(), vec![renamed]); + } +} diff --git a/crates-proxy/yaak-proxy-lib/tests/at_rest.rs b/crates-proxy/yaak-proxy-lib/tests/at_rest.rs index 3c5dd487c..65cb8f72a 100644 --- a/crates-proxy/yaak-proxy-lib/tests/at_rest.rs +++ b/crates-proxy/yaak-proxy-lib/tests/at_rest.rs @@ -267,17 +267,13 @@ fn search_reaches_inside_the_encrypted_database() { let db = ctx.db().unwrap(); // Query strings are searchable, being inside the encrypted file. - let q = |q: &str| FlowFilter { q: Some(q.into()), limit: Some(50), ..Default::default() }; + let q = |q: &str| FlowFilter { limit: Some(50), ..FlowFilter::search(q) }; let (page, total, matching) = flows::page(&db, &q(QUERY_SECRET)).unwrap(); assert_eq!((page.len(), total, matching), (50, FLOWS, Some(FLOWS))); let (page, _, matching) = flows::page(&db, &q("items/2999")).unwrap(); assert_eq!((page[0].id.as_str(), matching), ("fl_02999", Some(1))); - let posts_not_found = FlowFilter { - method: Some("post".into()), - status: Some("4xx".into()), - limit: Some(1), - ..Default::default() - }; + let posts_not_found = + FlowFilter { limit: Some(1), ..FlowFilter::search("@method:post @status:4xx") }; let (_, _, matching) = flows::page(&db, &posts_not_found).unwrap(); assert_eq!(matching, Some((0..FLOWS).filter(|i| i % 3 == 0 && i % 10 == 0).count())); @@ -295,11 +291,11 @@ fn search_reaches_inside_the_encrypted_database() { } assert_eq!(seen, FLOWS); - // The sidebar's site and path filters, which run the database's own functions. - let under = |u: &str| FlowFilter { under: Some(vec![u.into()]), ..Default::default() }; - assert_eq!(flows::list(&db, &under("api.example.com")).unwrap().len(), FLOWS); - assert_eq!(flows::list(&db, &under("*.example.com")).unwrap().len(), FLOWS); - assert_eq!(flows::list(&db, &under("api.example.com/v1/items/42")).unwrap().len(), 1); + // The sidebar's host, site and path terms, which run the database's own functions. + let found = |q: &str| flows::list(&db, &FlowFilter::search(q)).unwrap().len(); + assert_eq!(found("@host:api.example.com"), FLOWS); + assert_eq!(found("@host:*.example.com"), FLOWS); + assert_eq!(found("@host:api.example.com @path:/v1/items/42/*"), 1); // A full-text table, as body search will want, made and searched in the encrypted file. let hits: i64 = db diff --git a/crates-proxy/yaak-proxy-lib/tests/control.rs b/crates-proxy/yaak-proxy-lib/tests/control.rs index 65185eaff..7290e1ef1 100644 --- a/crates-proxy/yaak-proxy-lib/tests/control.rs +++ b/crates-proxy/yaak-proxy-lib/tests/control.rs @@ -144,14 +144,18 @@ async fn round_trip() { assert_eq!(flows.iter().map(|f| &f.id).collect::>(), vec![&flow.id]); let following = FlowFilter { after: Some(flow.id.clone()), ..Default::default() }; assert!(client.flows(&following).await.unwrap().is_empty()); - let filter = FlowFilter { status: Some("4xx".into()), ..Default::default() }; + let filter = FlowFilter::search("@status:4xx"); assert!(client.flows(&filter).await.unwrap().is_empty()); - let search = FlowFilter { q: Some("CHARGES post 2xx".into()), ..Default::default() }; + let search = FlowFilter::search("CHARGES post @status:2xx"); assert_eq!(client.flows(&search).await.unwrap().len(), 1); - let search = FlowFilter { q: Some("charges get".into()), ..Default::default() }; + let search = FlowFilter::search("charges @method:get"); assert!(client.flows(&search).await.unwrap().is_empty()); + // Listing is not reading; an agent reading the flow marks it. + let read = FlowFilter::search("@is:read"); + assert!(client.flows(&read).await.unwrap().is_empty()); let detail = client.flow(&flow.id, 8).await.unwrap(); + assert_eq!(client.flows(&read).await.unwrap().len(), 1); assert_eq!(detail.flow.method, "POST"); let request_body = detail.request_body.unwrap(); assert_eq!((request_body.data.as_str(), request_body.truncated), (r#"{"amount"#, true)); @@ -283,7 +287,7 @@ async fn offline_reads_proxy_db() { assert_eq!(detail.response_body.unwrap().data, r#"{"id":"ch_1","amount":42}"#); let body = connection.body(&flow.id, FlowBody::Request).await.unwrap(); assert_eq!(body.data, br#"{"amount":42}"#); - let posts = FlowFilter { method: Some("post".into()), ..Default::default() }; + let posts = FlowFilter::search("@method:post"); assert_eq!(connection.har(&posts).await.unwrap().log.entries.len(), 1); let empty = tempfile::tempdir().unwrap(); @@ -602,7 +606,7 @@ async fn export_har_writes_the_flows_asked_for() { assert_eq!(read(&selected)["log"]["entries"].as_array().unwrap().len(), 1); let searched = dir.path().join("searched.har"); - let filter = json!({ "q": "charges", "limit": 1 }); + let filter = json!({ "query": "charges", "limit": 1 }); let written = rpc(&ctx, "export_har", json!({ "path": searched, "filter": filter })).await.unwrap(); // The whole search, not the page the list had loaded. @@ -613,7 +617,7 @@ async fn export_har_writes_the_flows_asked_for() { let none = dir.path().join("none.har"); let written = - rpc(&ctx, "export_har", json!({ "path": none, "filter": { "q": "nothing-matches" } })) + rpc(&ctx, "export_har", json!({ "path": none, "filter": { "query": "nothing-matches" } })) .await .unwrap(); assert_eq!(written, 0); @@ -657,7 +661,7 @@ async fn flows_name_the_app_that_sent_them() { assert_eq!(flow.client_path.as_deref(), Some("/usr/bin/curl")); // The search and the sidebar's Apps group find it by its path. - let filter = FlowFilter { app: Some(vec!["/usr/bin/curl".into()]), ..Default::default() }; + let filter = FlowFilter::search("@app:/usr/bin/curl"); assert_eq!(yaak_proxy_lib::flows::list(&ctx.db().unwrap(), &filter).unwrap().len(), 1); } diff --git a/crates-proxy/yaak-proxy-lib/tests/flow.rs b/crates-proxy/yaak-proxy-lib/tests/flow.rs index 612183589..b08109fc1 100644 --- a/crates-proxy/yaak-proxy-lib/tests/flow.rs +++ b/crates-proxy/yaak-proxy-lib/tests/flow.rs @@ -502,10 +502,10 @@ async fn a_tunnelled_connect_is_recorded_with_its_sizes() { assert_eq!(done.error, None); assert!(done.request_headers.is_empty() && done.response_headers.is_empty()); - let tunnelled = rpc(&ctx, "list_flows", json!({"tunnelled": true})).await.unwrap(); + let tunnelled = rpc(&ctx, "list_flows", json!({"query": "@is:tunnelled"})).await.unwrap(); assert_eq!(tunnelled["flows"].as_array().unwrap().len(), 1); assert_eq!(tunnelled["flows"][0]["tunnelled"], true); - let decrypted = rpc(&ctx, "list_flows", json!({"tunnelled": false})).await.unwrap(); + let decrypted = rpc(&ctx, "list_flows", json!({"query": "-@is:tunnelled"})).await.unwrap(); assert_eq!(decrypted["flows"].as_array().unwrap().len(), 0); } diff --git a/crates-proxy/yaak-proxy-lib/tests/streams.rs b/crates-proxy/yaak-proxy-lib/tests/streams.rs index b34184fa0..d2a25040c 100644 --- a/crates-proxy/yaak-proxy-lib/tests/streams.rs +++ b/crates-proxy/yaak-proxy-lib/tests/streams.rs @@ -370,7 +370,7 @@ async fn grpc_web_calls_are_stored_and_decoded() { (failed.grpc_status, failed.grpc_message.as_deref()), (Some(12), Some("not here")) ); - let errors = rpc(&ctx, "list_flows", json!({ "status": "error" })).await.unwrap(); + let errors = rpc(&ctx, "list_flows", json!({ "query": "@is:error" })).await.unwrap(); let ids: Vec<_> = errors["flows"].as_array().unwrap().iter().map(|f| f["id"].clone()).collect(); assert_eq!(ids, vec![json!(failed.id)], "a non-zero status is a failed call"); diff --git a/crates/yaak-query/bindings/query.ts b/crates/yaak-query/bindings/query.ts index a9efad919..b64cf5d0d 100644 --- a/crates/yaak-query/bindings/query.ts +++ b/crates/yaak-query/bindings/query.ts @@ -10,7 +10,15 @@ label: string, /** * What replaces the span: `@method:`, `@header.`, or a value quoted when it needs to be. */ -insert: string, kind: CompletionKind, info: string | null, +insert: string, kind: CompletionKind, +/** + * What a field is, from its definition. + */ +info: string | null, +/** + * A term using the field, from its definition. + */ +example: string | null, /** * The term is not finished after inserting (a field still needs its value, a map field its * key), so completion should open again. @@ -57,9 +65,13 @@ name: string, kind: FieldKind, */ map?: boolean, /** - * A short note shown with the field in completions. + * What the field is, in a few words, shown with it in completions: `GET, POST, …`. */ -info?: string, }; +info?: string, +/** + * A term using it, shown with it in completions: `@method:post`. + */ +example?: string, }; export type FieldKind = { "type": "text" } | { "type": "number", units: Array, } | { "type": "flag" }; diff --git a/crates/yaak-query/pkg/yaak_query_bg.wasm b/crates/yaak-query/pkg/yaak_query_bg.wasm index c71e7d627..2ad25df49 100644 Binary files a/crates/yaak-query/pkg/yaak_query_bg.wasm and b/crates/yaak-query/pkg/yaak_query_bg.wasm differ diff --git a/crates/yaak-query/src/complete.rs b/crates/yaak-query/src/complete.rs index 9849c50c2..4c68b45e5 100644 --- a/crates/yaak-query/src/complete.rs +++ b/crates/yaak-query/src/complete.rs @@ -25,7 +25,10 @@ pub struct Completion { /// What replaces the span: `@method:`, `@header.`, or a value quoted when it needs to be. pub insert: String, pub kind: CompletionKind, + /// What a field is, from its definition. pub info: Option, + /// A term using the field, from its definition. + pub example: Option, /// The term is not finished after inserting (a field still needs its value, a map field its /// key), so completion should open again. pub reopen: bool, @@ -115,6 +118,7 @@ fn missing_at(word: &str, typed: &str, schema: &Schema) -> Option { insert: term, kind: CompletionKind::Field, info: None, + example: None, // With no value yet, go on to the field's values reopen: value.is_empty(), }) @@ -143,7 +147,8 @@ fn field_completions( insert: format!("@{path}:"), label: path, kind: CompletionKind::Field, - info: None, + info: map.info.clone(), + example: map.example.clone(), reopen: true, }); } @@ -157,6 +162,7 @@ fn field_completions( insert, kind: CompletionKind::Field, info: field.info.clone(), + example: field.example.clone(), reopen: true, }); } @@ -191,6 +197,7 @@ fn value_completions( insert: format_value(value), kind: CompletionKind::Value, info: None, + example: None, reopen: false, }) .collect(); @@ -236,14 +243,27 @@ mod tests { kind: FieldKind::Text, map: false, info: None, + example: None, }; Schema::new( vec![ text("method"), text("url"), text("name"), - FieldDef { name: "header".into(), kind: FieldKind::Text, map: true, info: None }, - FieldDef { name: "error".into(), kind: FieldKind::Flag, map: false, info: None }, + FieldDef { + name: "header".into(), + kind: FieldKind::Text, + map: true, + info: None, + example: None, + }, + FieldDef { + name: "error".into(), + kind: FieldKind::Flag, + map: false, + info: None, + example: None, + }, ], FreeText::Fuzzy, ) @@ -369,6 +389,57 @@ mod tests { assert_eq!(at("a|b"), Some(("a".into(), inserts(&["@name:", "@header."])))); } + /// Field names that start with what is typed come first, then those holding it, in the + /// schema's order; nothing matches loosely. + #[test] + fn prefix_matches_first() { + let field = |name: &str| FieldDef { + name: name.into(), + kind: FieldKind::Text, + map: false, + info: None, + example: None, + }; + let s = Schema::new( + vec![ + field("time"), + field("scheme"), + field("method"), + field("host"), + ], + FreeText::Substring, + ); + let labels = |input: &str| { + let c = completions(input, input.len(), &s, &HashMap::new()).unwrap(); + c.items.into_iter().map(|i| i.label).collect::>() + }; + assert_eq!(labels("m"), ["method", "time", "scheme"]); + assert_eq!(labels("@h"), ["host", "scheme", "method"]); + assert!(completions("mt", 2, &s, &HashMap::new()).is_none(), "no fuzzy match"); + } + + /// A field's description and example go with it, and with the keys under a map. + #[test] + fn descriptions_and_examples() { + let s = Schema::new( + vec![FieldDef { + name: "header".into(), + kind: FieldKind::Text, + map: true, + info: Some("A header on either side".into()), + example: Some("@header.content-type:json".into()), + }], + FreeText::Substring, + ); + let values = HashMap::from([("header".to_string(), vec!["accept".to_string()])]); + let c = completions("@he", 3, &s, &values).unwrap(); + assert_eq!(c.items[0].info.as_deref(), Some("A header on either side")); + assert_eq!(c.items[0].example.as_deref(), Some("@header.content-type:json")); + let c = completions("@header.", 8, &s, &values).unwrap(); + assert_eq!(c.items[0].label, "header.accept"); + assert_eq!(c.items[0].example.as_deref(), Some("@header.content-type:json")); + } + #[test] fn cursor_is_clamped() { let c = completions("é", 1, &schema(), &values()); diff --git a/crates/yaak-query/src/condition.rs b/crates/yaak-query/src/condition.rs new file mode 100644 index 000000000..657931217 --- /dev/null +++ b/crates/yaak-query/src/condition.rs @@ -0,0 +1,241 @@ +//! What a field term asks of a field's values, resolved once from the term's value and the +//! field's kind. +//! +//! The evaluator matches with these, and a consumer that compiles queries elsewhere (to SQL, +//! say) translates them rather than deriving the rules again, so the two agree. The helpers +//! below are the value tests the conditions are made of, for a compiler that has to run them +//! where the values are. + +use crate::ast::{CompareOp, FieldValue}; +use crate::lexer::parse_number; +use crate::schema::{FieldKind, Unit, parse_flag, to_base}; + +/// A field term's test, applied to each of the field's values: the term matches when any value +/// passes, except [`Condition::Flag`] `false`, which matches when none is set. +#[derive(Clone, Debug, PartialEq)] +pub enum Condition { + /// The value, as lowercase text, contains this lowercase text, which is never empty. + Contains(String), + /// The whole value, as lowercase text, matches this lowercase pattern, where `*` matches + /// any run of characters. A pattern without `*` must equal the value. + Wildcard(String), + /// The value is there: text that is not empty, any number, or a flag that is set. + Present, + /// The value reads as a number. + IsNumber, + /// The value, read as a number, equals this one, in the field's base unit. + Equals(f64), + /// The value, read as a number, compares so with this one, in the field's base unit. + Compare(CompareOp, f64), + /// The value, read as a number, is in this inclusive range, in the field's base unit. + Range(f64, f64), + /// With `true`, some value is a set flag; with `false`, none is. + Flag(bool), + /// Nothing matches: a value the field cannot take, such as a unit it does not have. + Never, +} + +impl Condition { + /// The condition `value` sets for a field of `kind`. + pub fn new(kind: &FieldKind, value: &FieldValue) -> Self { + match kind { + FieldKind::Text => text_condition(value), + FieldKind::Number { units } => number_condition(value, units), + FieldKind::Flag => flag_condition(value), + } + } + + /// Whether a text value passes. + pub fn matches_text(&self, text: &str) -> bool { + match self { + Condition::Contains(needle) => contains_ignore_case(text, needle), + Condition::Wildcard(pattern) => wildcard_match(pattern, &text.to_lowercase()), + Condition::Present => !text.is_empty(), + Condition::Flag(_) => parse_flag(text) == Some(true), + _ => self.matches_parsed_number(read_number(text)), + } + } + + /// Whether a number value passes. + pub fn matches_number(&self, n: f64) -> bool { + match self { + Condition::Contains(needle) => format_number(n).contains(needle.as_str()), + Condition::Wildcard(pattern) => wildcard_match(pattern, &format_number(n)), + Condition::Present => true, + Condition::Flag(_) => n != 0.0, + _ => self.matches_parsed_number(Some(n)), + } + } + + /// Whether a flag value passes. + pub fn matches_flag(&self, b: bool) -> bool { + let text = if b { "true" } else { "false" }; + match self { + Condition::Contains(needle) => text.contains(needle.as_str()), + Condition::Wildcard(pattern) => wildcard_match(pattern, text), + Condition::Present | Condition::Flag(_) => b, + _ => false, + } + } + + fn matches_parsed_number(&self, n: Option) -> bool { + match self { + Condition::IsNumber => n.is_some(), + Condition::Equals(target) => n == Some(*target), + Condition::Compare(op, target) => n.is_some_and(|n| op.test(n, *target)), + Condition::Range(low, high) => n.is_some_and(|n| n >= *low && n <= *high), + _ => false, + } + } +} + +fn text_condition(value: &FieldValue) -> Condition { + match value { + FieldValue::Text { text, .. } if text.is_empty() => Condition::Never, + FieldValue::Text { text, .. } => Condition::Contains(text.to_lowercase()), + FieldValue::Wildcard { pattern } => Condition::Wildcard(pattern.to_lowercase()), + FieldValue::Exists => Condition::Present, + // A text field holding numbers, such as a status shown as text + FieldValue::Compare { op, number } if number.unit.is_none() => { + Condition::Compare(*op, number.value) + } + FieldValue::Range { low, high } if low.unit.is_none() && high.unit.is_none() => { + Condition::Range(low.value, high.value) + } + FieldValue::Compare { .. } | FieldValue::Range { .. } => Condition::Never, + } +} + +fn number_condition(value: &FieldValue, units: &[Unit]) -> Condition { + match value { + FieldValue::Text { text, .. } if text.is_empty() => Condition::Never, + // `@status:404` is equality; anything else matches the number as text + FieldValue::Text { text, quoted: false } => match parse_number(text) { + Some(number) => to_base(&number, units).map_or(Condition::Never, Condition::Equals), + None => text_condition(value), + }, + FieldValue::Text { quoted: true, .. } | FieldValue::Wildcard { .. } => { + text_condition(value) + } + FieldValue::Exists => Condition::IsNumber, + FieldValue::Compare { op, number } => to_base(number, units) + .map_or(Condition::Never, |target| Condition::Compare(*op, target)), + FieldValue::Range { low, high } => match (to_base(low, units), to_base(high, units)) { + (Some(low), Some(high)) => Condition::Range(low, high), + _ => Condition::Never, + }, + } +} + +fn flag_condition(value: &FieldValue) -> Condition { + match value { + FieldValue::Exists => Condition::Flag(true), + FieldValue::Text { text, .. } => parse_flag(text).map_or(Condition::Never, Condition::Flag), + _ => Condition::Never, + } +} + +/// Whether `hay`, lowercased, contains `needle`, which is lowercase already. Copies nothing +/// when both are ASCII. +pub fn contains_ignore_case(hay: &str, needle: &str) -> bool { + if !(hay.is_ascii() && needle.is_ascii()) { + return hay.to_lowercase().contains(needle); + } + let (hay, needle) = (hay.as_bytes(), needle.as_bytes()); + needle.is_empty() || hay.windows(needle.len()).any(|w| w.eq_ignore_ascii_case(needle)) +} + +/// Whether `pattern`, where `*` matches any run of characters, matches the whole of `text`. +/// Case-sensitive: both are lowercase already when matching a query. +pub fn wildcard_match(pattern: &str, text: &str) -> bool { + let mut parts = pattern.split('*'); + let first = parts.next().unwrap_or_default(); + let Some(mut rest) = text.strip_prefix(first) else { + return false; + }; + let parts: Vec<&str> = parts.collect(); + let Some((last, middle)) = parts.split_last() else { + // No `*` at all: the whole text must equal the pattern + return rest.is_empty(); + }; + for part in middle { + match rest.find(part) { + Some(i) => rest = &rest[i + part.len()..], + None => return false, + } + } + rest.len() >= last.len() && rest.ends_with(last) +} + +/// A text value read as a number, as comparisons read it: the whole text, spaces around it +/// aside. +pub fn read_number(text: &str) -> Option { + text.trim().parse().ok() +} + +/// A number as text, as a field's text matching sees it: `200`, not `200.0`. +pub fn format_number(n: f64) -> String { + if n.fract() == 0.0 && n.abs() < 1e15 { format!("{}", n as i64) } else { n.to_string() } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ast::Node; + use crate::parse; + + fn condition(kind: FieldKind, input: &str) -> Condition { + let Some(Node::Field { value, .. }) = parse(input) else { + panic!("not a field") + }; + Condition::new(&kind, &value) + } + + fn ms() -> FieldKind { + FieldKind::Number { + units: vec![ + Unit { name: "ms".into(), factor: 1.0 }, + Unit { name: "s".into(), factor: 1000.0 }, + ], + } + } + + #[test] + fn resolves_text_values() { + assert_eq!(condition(FieldKind::Text, "@a:Foo"), Condition::Contains("foo".into())); + assert_eq!(condition(FieldKind::Text, "@a:"), Condition::Never); + assert_eq!(condition(FieldKind::Text, "@a:F*o"), Condition::Wildcard("f*o".into())); + assert_eq!(condition(FieldKind::Text, "@a:*"), Condition::Present); + assert_eq!(condition(FieldKind::Text, "@a:>5"), Condition::Compare(CompareOp::Gt, 5.0)); + assert_eq!(condition(FieldKind::Text, "@a:>5s"), Condition::Never); + } + + #[test] + fn resolves_number_values_in_base_units() { + assert_eq!(condition(ms(), "@d:1.5s"), Condition::Equals(1500.0)); + assert_eq!(condition(ms(), "@d:1m"), Condition::Never); + assert_eq!(condition(ms(), "@d:<2s"), Condition::Compare(CompareOp::Lt, 2000.0)); + assert_eq!(condition(ms(), "@d:[1s TO 2s]"), Condition::Range(1000.0, 2000.0)); + assert_eq!(condition(ms(), "@d:*"), Condition::IsNumber); + assert_eq!(condition(ms(), "@d:\"15\""), Condition::Contains("15".into())); + assert_eq!(condition(ms(), "@d:abc"), Condition::Contains("abc".into())); + } + + #[test] + fn resolves_flags() { + assert_eq!(condition(FieldKind::Flag, "@f:*"), Condition::Flag(true)); + assert_eq!(condition(FieldKind::Flag, "@f:no"), Condition::Flag(false)); + assert_eq!(condition(FieldKind::Flag, "@f:maybe"), Condition::Never); + } + + #[test] + fn tests_values() { + let c = Condition::Wildcard("40*".into()); + assert!(c.matches_number(404.0)); + assert!(!c.matches_text("504")); + assert!(Condition::Equals(200.0).matches_text(" 200 ")); + assert!(Condition::Contains("é".into()).matches_text("CAFÉ")); + assert!(!Condition::Present.matches_text("")); + assert!(Condition::Flag(true).matches_flag(true)); + } +} diff --git a/crates/yaak-query/src/eval.rs b/crates/yaak-query/src/eval.rs index 85dd998e8..f494678c6 100644 --- a/crates/yaak-query/src/eval.rs +++ b/crates/yaak-query/src/eval.rs @@ -1,8 +1,11 @@ //! Runs a parsed query in memory, against one document or a whole table of them. -use crate::ast::{FieldValue, Node}; +use crate::ast::Node; +use crate::condition::Condition; +#[cfg(test)] +use crate::condition::wildcard_match; use crate::fuzzy::{FuzzyScratch, fuzzy_match_in}; -use crate::schema::{FieldKind, FreeText, Schema, parse_flag, to_base}; +use crate::schema::{FreeText, Schema}; use serde::{Deserialize, Serialize}; use std::borrow::Cow; use std::collections::HashMap; @@ -208,55 +211,13 @@ enum Val<'a> { } impl Val<'_> { - /// The value as lowercase text. - fn lower(&self) -> Cow<'_, str> { + fn passes(&self, condition: &Condition) -> bool { match self { - Val::Text(text) => Cow::Owned(text.to_lowercase()), - Val::Number(n) => Cow::Owned(format_number(*n)), - Val::Flag(b) => Cow::Borrowed(if *b { "true" } else { "false" }), + Val::Text(text) => condition.matches_text(text), + Val::Number(n) => condition.matches_number(*n), + Val::Flag(b) => condition.matches_flag(*b), } } - - /// Whether the value, lowercased, contains `needle` (lowercase). - fn contains(&self, needle: &str) -> bool { - match self { - Val::Text(text) => contains_ignore_case(text, needle), - _ => self.lower().contains(needle), - } - } - - fn as_number(&self) -> Option { - match self { - Val::Number(n) => Some(*n), - Val::Text(text) => text.trim().parse().ok(), - Val::Flag(_) => None, - } - } - - fn as_flag(&self) -> Option { - match self { - Val::Flag(b) => Some(*b), - Val::Text(text) => parse_flag(text), - Val::Number(n) => Some(*n != 0.0), - } - } - - fn is_present(&self) -> bool { - match self { - Val::Text(text) => !text.is_empty(), - Val::Number(_) => true, - Val::Flag(b) => *b, - } - } -} - -/// Whether `hay`, lowercased, contains `needle` (lowercase), without copying ASCII text. -fn contains_ignore_case(hay: &str, needle: &str) -> bool { - if !(hay.is_ascii() && needle.is_ascii()) { - return hay.to_lowercase().contains(needle); - } - let (hay, needle) = (hay.as_bytes(), needle.as_bytes()); - needle.is_empty() || hay.windows(needle.len()).any(|w| w.eq_ignore_ascii_case(needle)) } /// `text` lowercased into `out`, without a fresh allocation per call. @@ -270,11 +231,6 @@ fn lowercase_into(text: &str, out: &mut String) { } } -/// `200`, not `200.0`, so a number field's text matches what the user sees. -fn format_number(n: f64) -> String { - if n.fract() == 0.0 && n.abs() < 1e15 { format!("{}", n as i64) } else { n.to_string() } -} - /// Where field values come from: a [`Document`], or a row of a [`Table`]. trait FieldSource { /// Whether any value under `path` (lowercase) passes `test`. With `any_key`, `path` is a @@ -351,9 +307,8 @@ enum Plan { path: String, /// A map field with no key: every key's values. any_key: bool, - kind: FieldKind, - /// With its text and wildcard pattern lowercased. - value: FieldValue, + /// Never [`Condition::Never`], which plans as [`Plan::Never`]. + condition: Condition, }, /// A term that can match nothing: an unknown field, an empty word or phrase. Never, @@ -418,14 +373,10 @@ impl Matcher { Plan::Not(node) => !self.eval(node, text, source, fuzzy), Plan::Term(term) if self.fuzzy => fuzzy_match_in(text, term, fuzzy), Plan::Term(term) | Plan::Phrase(term) => text.contains(term.as_str()), - Plan::Field { path, any_key, kind, value } => { - let mut any = - |test: &mut dyn FnMut(Val) -> bool| source.any_value(path, *any_key, test); - match kind { - FieldKind::Text => match_text(&mut any, value), - FieldKind::Number { units } => match_number(&mut any, value, units), - FieldKind::Flag => match_flag(&mut any, value), - } + Plan::Field { path, any_key, condition } => { + let any = source.any_value(path, *any_key, &mut |v| v.passes(condition)); + // `@flag:false`: no value is set + if *condition == Condition::Flag(false) { !any } else { any } } Plan::Never => false, } @@ -458,117 +409,27 @@ fn plan_node(node: &Node, schema: &Schema, uses_text: &mut bool) -> Plan { let Some(field) = schema.resolve(path) else { return Plan::Never; }; - let value = match value { - FieldValue::Text { text, quoted } => { - FieldValue::Text { text: text.to_lowercase(), quoted: *quoted } - } - FieldValue::Wildcard { pattern } => { - FieldValue::Wildcard { pattern: pattern.to_lowercase() } - } - other => other.clone(), - }; + let condition = Condition::new(&field.def.kind, value); + if condition == Condition::Never { + return Plan::Never; + } Plan::Field { path: field.document_path(), any_key: field.def.map && field.key.is_none(), - kind: field.def.kind.clone(), - value, + condition, } } } } -/// Asks whether any of a field's values passes a test. -type AnyValue<'s> = dyn FnMut(&mut dyn FnMut(Val) -> bool) -> bool + 's; - -/// `value`'s text and wildcard pattern are already lowercase (see [`plan_node`]). -fn match_text(any: &mut AnyValue, value: &FieldValue) -> bool { - match value { - FieldValue::Text { text: needle, .. } => { - !needle.is_empty() && any(&mut |v| v.contains(needle)) - } - FieldValue::Wildcard { pattern } => any(&mut |v| wildcard_match(pattern, &v.lower())), - FieldValue::Exists => any(&mut |v| v.is_present()), - // A text field holding numbers, such as a status shown as text - FieldValue::Compare { op, number } => { - number.unit.is_none() - && any(&mut |v| v.as_number().is_some_and(|n| op.test(n, number.value))) - } - FieldValue::Range { low, high } => { - low.unit.is_none() - && high.unit.is_none() - && any(&mut |v| v.as_number().is_some_and(|n| n >= low.value && n <= high.value)) - } - } -} - -fn match_number(any: &mut AnyValue, value: &FieldValue, units: &[crate::Unit]) -> bool { - match value { - FieldValue::Text { text, quoted } => { - if text.is_empty() { - return false; - } - // `@status:404` is equality; anything else matches the number as text - match (!quoted).then(|| crate::lexer::parse_number(text)).flatten() { - Some(number) => to_base(&number, units) - .is_some_and(|target| any(&mut |v| v.as_number() == Some(target))), - None => match_text(any, value), - } - } - FieldValue::Wildcard { .. } => match_text(any, value), - FieldValue::Exists => any(&mut |v| v.as_number().is_some()), - FieldValue::Compare { op, number } => to_base(number, units) - .is_some_and(|target| any(&mut |v| v.as_number().is_some_and(|n| op.test(n, target)))), - FieldValue::Range { low, high } => match (to_base(low, units), to_base(high, units)) { - (Some(low), Some(high)) => { - any(&mut |v| v.as_number().is_some_and(|n| n >= low && n <= high)) - } - _ => false, - }, - } -} - -fn match_flag(any: &mut AnyValue, value: &FieldValue) -> bool { - let mut is_set = || any(&mut |v| v.as_flag() == Some(true)); - match value { - FieldValue::Exists => is_set(), - FieldValue::Text { text, .. } => match parse_flag(text) { - Some(true) => is_set(), - Some(false) => !is_set(), - None => false, - }, - _ => false, - } -} - -/// Whether `pattern`, where `*` matches any run of characters, matches the whole of `text`. -pub(crate) fn wildcard_match(pattern: &str, text: &str) -> bool { - let mut parts = pattern.split('*'); - let first = parts.next().unwrap_or_default(); - let Some(mut rest) = text.strip_prefix(first) else { - return false; - }; - let parts: Vec<&str> = parts.collect(); - let Some((last, middle)) = parts.split_last() else { - // No `*` at all: the whole text must equal the pattern - return rest.is_empty(); - }; - for part in middle { - match rest.find(part) { - Some(i) => rest = &rest[i + part.len()..], - None => return false, - } - } - rest.len() >= last.len() && rest.ends_with(last) -} - #[cfg(test)] mod tests { use super::*; - use crate::schema::{FieldDef, Unit}; + use crate::schema::{FieldDef, FieldKind, Unit}; use crate::{format_field, parse}; fn text_field(name: &str) -> FieldDef { - FieldDef { name: name.into(), kind: FieldKind::Text, map: false, info: None } + FieldDef { name: name.into(), kind: FieldKind::Text, map: false, info: None, example: None } } /// The fields and free-text mode of Yaak's sidebar. @@ -594,12 +455,19 @@ mod tests { Schema::new( vec![ text_field("url"), - FieldDef { name: "header".into(), kind: FieldKind::Text, map: true, info: None }, + FieldDef { + name: "header".into(), + kind: FieldKind::Text, + map: true, + info: None, + example: None, + }, FieldDef { name: "status".into(), kind: FieldKind::Number { units: vec![] }, map: false, info: None, + example: None, }, FieldDef { name: "duration".into(), @@ -611,8 +479,15 @@ mod tests { }, map: false, info: None, + example: None, + }, + FieldDef { + name: "error".into(), + kind: FieldKind::Flag, + map: false, + info: None, + example: None, }, - FieldDef { name: "error".into(), kind: FieldKind::Flag, map: false, info: None }, ], FreeText::Substring, ) diff --git a/crates/yaak-query/src/lib.rs b/crates/yaak-query/src/lib.rs index 14cc87e66..e0fc05bd3 100644 --- a/crates/yaak-query/src/lib.rs +++ b/crates/yaak-query/src/lib.rs @@ -9,7 +9,9 @@ //! //! [`parse`] gives the tree, which a consumer may walk to compile it elsewhere (to SQL, say). //! [`evaluate`] runs it in memory against a [`Document`], with a [`Schema`] saying what the -//! fields are. [`tokens`], [`completions`] and [`format_field`] serve the search box. +//! fields are. [`tokens`], [`completions`] and [`format_field`] serve the search box. A compiler +//! resolves each field term to a [`Condition`] and translates that, so it matches as +//! [`evaluate`] does. // The wasm-bindgen exports expand to unsafe glue; nothing else in the crate may use unsafe #![cfg_attr( @@ -20,10 +22,12 @@ pub mod ast; mod complete; +mod condition; mod eval; mod format; pub mod fuzzy; mod lexer; +mod offsets; mod parser; mod schema; mod tokens; @@ -33,8 +37,10 @@ mod wasm; pub use ast::{CompareOp, FieldValue, Node, NumberLiteral, Span}; pub use complete::{Completion, CompletionKind, Completions, completions}; +pub use condition::{Condition, contains_ignore_case, format_number, read_number, wildcard_match}; pub use eval::{DocValue, DocValues, Document, Matcher, Table, evaluate}; pub use format::{format_field, format_value, is_valid_path, quote}; +pub use offsets::Utf16Offsets; pub use parser::parse; pub use schema::{Diagnostic, FieldDef, FieldKind, FreeText, ResolvedField, Schema, Unit}; pub use tokens::{Token, TokenKind, tokens}; diff --git a/crates/yaak-query/src/offsets.rs b/crates/yaak-query/src/offsets.rs new file mode 100644 index 000000000..999c339ac --- /dev/null +++ b/crates/yaak-query/src/offsets.rs @@ -0,0 +1,52 @@ +//! Byte offsets to UTF-16 offsets and back, for spans handed to JavaScript, which counts string +//! positions in UTF-16 code units. + +use crate::ast::Span; + +/// Byte offset to UTF-16 offset, and back, for one input. +pub struct Utf16Offsets { + /// `utf16[i]` is the UTF-16 offset of byte `i`, for every char boundary and the end. + utf16: Vec, +} + +impl Utf16Offsets { + pub fn new(input: &str) -> Self { + let mut utf16 = vec![0; input.len() + 1]; + let mut units = 0u32; + for (i, c) in input.char_indices() { + for b in 0..c.len_utf8() { + utf16[i + b] = units; + } + units += c.len_utf16() as u32; + } + utf16[input.len()] = units; + Self { utf16 } + } + + /// A byte span as a UTF-16 one. + pub fn span(&self, span: Span) -> Span { + Span { start: self.utf16[span.start()], end: self.utf16[span.end()] } + } + + /// The byte offset of UTF-16 offset `pos`, or of the char it falls inside. + pub fn byte(&self, pos: u32) -> usize { + self.utf16.partition_point(|&u| u < pos).min(self.utf16.len() - 1) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn converts_both_ways() { + let input = "é@a:😀x"; + let offsets = Utf16Offsets::new(input); + // `é` is two bytes and one unit; `😀` is four bytes and two units + assert_eq!(offsets.span(Span::new(2, 5)), Span { start: 1, end: 4 }); + assert_eq!(offsets.span(Span::new(5, 10)), Span { start: 4, end: 7 }); + assert_eq!(offsets.byte(4), 5); + assert_eq!(offsets.byte(6), 9); + assert_eq!(offsets.byte(99), input.len()); + } +} diff --git a/crates/yaak-query/src/schema.rs b/crates/yaak-query/src/schema.rs index 17a1c02fb..b37abc7a5 100644 --- a/crates/yaak-query/src/schema.rs +++ b/crates/yaak-query/src/schema.rs @@ -45,10 +45,14 @@ pub struct FieldDef { #[serde(default)] #[cfg_attr(test, ts(optional, as = "Option"))] pub map: bool, - /// A short note shown with the field in completions. + /// What the field is, in a few words, shown with it in completions: `GET, POST, …`. #[serde(default)] #[cfg_attr(test, ts(optional))] pub info: Option, + /// A term using it, shown with it in completions: `@method:post`. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[cfg_attr(test, ts(optional))] + pub example: Option, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -207,7 +211,7 @@ mod tests { use crate::parse; fn field(name: &str, kind: FieldKind, map: bool) -> FieldDef { - FieldDef { name: name.to_string(), kind, map, info: None } + FieldDef { name: name.to_string(), kind, map, info: None, example: None } } fn schema() -> Schema { diff --git a/crates/yaak-query/src/wasm.rs b/crates/yaak-query/src/wasm.rs index fbf0089b3..85019be6a 100644 --- a/crates/yaak-query/src/wasm.rs +++ b/crates/yaak-query/src/wasm.rs @@ -3,43 +3,13 @@ //! JavaScript counts string positions in UTF-16 code units, so every span going out is //! converted from bytes, and the cursor coming in is converted to bytes. -use crate::{Document, Matcher, Schema, Span, Table}; +use crate::{Document, Matcher, Schema, Table, Utf16Offsets as Offsets}; use serde::Serialize; use serde::de::DeserializeOwned; use std::collections::HashMap; use wasm_bindgen::JsValue; use wasm_bindgen::prelude::wasm_bindgen; -/// Byte offset to UTF-16 offset, and back, for one input. -struct Offsets { - /// `utf16[i]` is the UTF-16 offset of byte `i`, for every char boundary and the end. - utf16: Vec, -} - -impl Offsets { - fn new(input: &str) -> Self { - let mut utf16 = vec![0; input.len() + 1]; - let mut units = 0u32; - for (i, c) in input.char_indices() { - for b in 0..c.len_utf8() { - utf16[i + b] = units; - } - units += c.len_utf16() as u32; - } - utf16[input.len()] = units; - Self { utf16 } - } - - fn span(&self, span: Span) -> Span { - Span { start: self.utf16[span.start()], end: self.utf16[span.end()] } - } - - /// The byte offset of UTF-16 offset `pos`, or of the char it falls inside. - fn byte(&self, pos: u32) -> usize { - self.utf16.partition_point(|&u| u < pos).min(self.utf16.len() - 1) - } -} - fn to_js(value: &T) -> Result { value .serialize(&serde_wasm_bindgen::Serializer::json_compatible()) diff --git a/package-lock.json b/package-lock.json index 9a34d7914..1238d6bc1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -272,6 +272,7 @@ "babel-plugin-react-compiler": "^1.0.0", "typescript": "^5.8.3", "vite": "npm:@voidzero-dev/vite-plus-core@^1.4.0", + "vite-plugin-wasm": "^3.5.0", "vite-plus": "^1.4.0" } }, diff --git a/packages/ui/src/components/editor/filter/extension.ts b/packages/ui/src/components/editor/filter/extension.ts index 869a50853..a522e3fe6 100644 --- a/packages/ui/src/components/editor/filter/extension.ts +++ b/packages/ui/src/components/editor/filter/extension.ts @@ -1,8 +1,5 @@ import type { Completion, CompletionContext, CompletionResult } from "@codemirror/autocomplete"; import { autocompletion, startCompletion } from "@codemirror/autocomplete"; -import { LanguageSupport, LRLanguage, syntaxTree } from "@codemirror/language"; -import type { SyntaxNode } from "@lezer/common"; -import { parser } from "./filter"; export interface FieldDef { name: string; @@ -87,17 +84,7 @@ function fieldCompletionFrom( return null; } -function inPhrase(ctx: CompletionContext): boolean { - // Lezer node names from your grammar: Phrase is the quoted token - let n: SyntaxNode | null = syntaxTree(ctx.state).resolveInner(ctx.pos, -1); - while (n) { - if (n.name === "Phrase") return true; - n = n.parent; - } - return false; -} - -// While typing an incomplete quote, there's no Phrase token yet. +/** Inside a quote, opened and not yet closed */ function inUnclosedQuote(doc: string, pos: number): boolean { let quotes = 0; for (let i = 0; i < pos; i++) { @@ -200,7 +187,7 @@ export function makeCompletionSource(opts: FilterOptions) { const { state, pos } = ctx; const doc = state.doc.toString(); - if (inPhrase(ctx) || inUnclosedQuote(doc, pos)) { + if (inUnclosedQuote(doc, pos)) { return null; } @@ -250,14 +237,6 @@ export function makeCompletionSource(opts: FilterOptions) { }; } -const language = LRLanguage.define({ - name: "filter", - parser, - languageData: { - autocompletion: {}, - }, -}); - /** * Field and value suggestions, as the sidebar's filter offers them, for a field of another * language: a plain input of `host:` and `app:` entries, say @@ -265,9 +244,3 @@ const language = LRLanguage.define({ export function filterAutocomplete(options: FilterOptions) { return autocompletion({ override: [makeCompletionSource(options)] }); } - -/** Public extension */ -export function filter(options: FilterOptions) { - const source = makeCompletionSource(options); - return new LanguageSupport(language, [autocompletion({ override: [source] })]); -} diff --git a/packages/ui/src/components/editor/filter/filter.grammar b/packages/ui/src/components/editor/filter/filter.grammar deleted file mode 100644 index dcf8b81ec..000000000 --- a/packages/ui/src/components/editor/filter/filter.grammar +++ /dev/null @@ -1,78 +0,0 @@ -@top Query { Expr } - -@skip { space+ } -@tokens { - space { $[ \t\r\n]+ } - - LParen { "(" } - RParen { ")" } - At { "@" } - Colon { ":" } - Not { "-" | "NOT" } - - // Keywords (case-insensitive) - And { "AND" } - Or { "OR" } - - // "quoted phrase" with simple escapes: \" and \\ - Phrase { '"' (!["\\] | "\\" _)* '"' } - - // Bare words run until filter syntax or whitespace. Leading '-' remains unary - // negation, but '-' may appear after the first character. - Word { ![ \t\r\n():"@-] ![ \t\r\n():"@]* } - FieldValueWord { ![ \t\r\n"] ![ \t\r\n]* } - - @precedence { Not, And, Or, Word } -} - -@detectDelim - -// Precedence: NOT (highest) > AND > OR (lowest) -// We also allow implicit AND in your parser/evaluator, but for highlighting, -// this grammar parses explicit AND/OR/NOT + adjacency as a sequence (Seq). -Expr { - OrExpr -} - -OrExpr { - AndExpr (Or AndExpr)* -} - -AndExpr { - Unary (And Unary | Unary)* // allow implicit AND by adjacency: Unary Unary -} - -Unary { - Not Unary -| Primary -} - -Primary { - Group -| Field -| Phrase -| Term -} - -Group { - LParen Expr RParen -} - -Field { - FieldName Colon FieldValue -} - -FieldName { - At? Word -} - -FieldValue { - Phrase -| FieldValueWord -} - -Term { - Word -} - -@external propSource highlight from "./highlight" diff --git a/packages/ui/src/components/editor/filter/filter.test.ts b/packages/ui/src/components/editor/filter/filter.test.ts deleted file mode 100644 index 6002175f6..000000000 --- a/packages/ui/src/components/editor/filter/filter.test.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { describe, expect, test } from "vite-plus/test"; -import { parser } from "./filter"; - -function getNodeNames(input: string): string[] { - const tree = parser.parse(input); - const nodes: string[] = []; - const cursor = tree.cursor(); - do { - if (cursor.name !== "Query") { - nodes.push(cursor.name); - } - } while (cursor.next()); - return nodes; -} - -describe("filter grammar", () => { - test("parses URL-like field values as one value", () => { - const nodes = getNodeNames("@url:yaak.app/foo-bar"); - - expect(nodes).not.toContain("⚠"); - expect(nodes).toContain("FieldValue"); - expect(nodes).toContain("FieldValueWord"); - }); - - test("parses punctuation-heavy field values as one value", () => { - const nodes = getNodeNames("@url:yaa$&#*@tsrna(*)"); - - expect(nodes).not.toContain("⚠"); - expect(nodes).toContain("FieldValue"); - expect(nodes).toContain("FieldValueWord"); - }); - - test("parses operator-looking field values as one value", () => { - const negativeValueNodes = getNodeNames("@url:-foo"); - const operatorWordNodes = getNodeNames("@url:AND"); - - expect(negativeValueNodes).not.toContain("⚠"); - expect(negativeValueNodes).toContain("FieldValueWord"); - expect(operatorWordNodes).not.toContain("⚠"); - expect(operatorWordNodes).toContain("FieldValueWord"); - }); -}); diff --git a/packages/ui/src/components/editor/filter/filter.ts b/packages/ui/src/components/editor/filter/filter.ts deleted file mode 100644 index 717a6daa4..000000000 --- a/packages/ui/src/components/editor/filter/filter.ts +++ /dev/null @@ -1,22 +0,0 @@ -// This file was generated by lezer-generator. You probably shouldn't edit it. -import {LRParser} from "@lezer/lr" -import {highlight} from "./highlight" -export const parser = LRParser.deserialize({ - version: 14, - states: "%WOVQPOOPhOPOOOVQPO'#CfOmQPO'#ChO!_QPO'#ChO!dQPO'#CgOOQO'#Cc'#CcOVQPO'#CaOOQO'#Ca'#CaO!iQPO'#C`O!yQPO'#C_OOQO'#C^'#C^QOQPOOPOOO'#Cr'#CrP#UOPO)C>lO#]QPO,59QOOQO,59S,59SO#bQQO,59ROOQO,58{,58{OVQPO'#CsOOQO'#Cs'#CsO#jQPO,58zOVQPO'#CtO#zQPO,58yPOOO-E6p-E6pOOQO1G.l1G.lOOQO'#Cl'#ClOOQO1G.m1G.mOOQO,59_,59_OOQO-E6q-E6qOOQO,59`,59`OOQO-E6r-E6r", - stateData: "$]~OkPQ~OUVOXQO]SO^ROaUO~Ok]O~OUcXXcX]cX^cX_[XacXdcXecXicXWcX~O^`O~O_aO~OdcOeSXiSXWSX~PVOefOiRXWRX~Ok]O~Qj]WiO~OajObjO~OdcOeSaiSaWSa~PVOefOiRaWRa~OUde^e~", - goto: "#^iPPjpt{P![PP!e!e!nPPP!wPP!ePP!z#Q#WQ[OR_QTZOQSYOQRnfUXOQfQbVSdXeRlc_WOQVXcef_UOQVXcef_TOQVXcefRkaQ^PRh^QeXRmeQgYRog", - nodeNames: "⚠ Query Expr OrExpr AndExpr Unary Not Primary RParen LParen Group Field FieldName At Word Colon FieldValue Phrase FieldValueWord Term And Or", - maxTerm: 27, - nodeProps: [ - ["openedBy", 8,"LParen"], - ["closedBy", 9,"RParen"] - ], - propSources: [highlight], - skippedNodes: [0,22], - repeatNodeCount: 3, - tokenData: "2h~RiOX!pXY$hYZ$hZ]!p]^$h^p!ppq$hqr!prs$ysx!pxy&gyz'Qz}!p}!O'k!O![!p![!](U!]!b!p!b!c(o!c!d)Y!d!p!p!p!q,q!q!r0Y!r;'S!p;'S;=`$b<%lO!pR!w^bQ^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c;'S!p;'S;=`$b<%lO!pQ#xUbQOX#sZ]#s^p#sq;'S#s;'S;=`$[<%lO#sQ$_P;=`<%l#sR$eP;=`<%l!p~$mSk~XY$hYZ$h]^$hpq$h~$|VOr$yrs%cs#O$y#O#P%h#P;'S$y;'S;=`&a<%lO$y~%hOa~~%kRO;'S$y;'S;=`%t;=`O$y~%wWOr$yrs%cs#O$y#O#P%h#P;'S$y;'S;=`&a;=`<%l$y<%lO$y~&dP;=`<%l$yR&nUbQXPOX#sZ]#s^p#sq;'S#s;'S;=`$[<%lO#sR'XUbQWPOX#sZ]#s^p#sq;'S#s;'S;=`$[<%lO#sR'rUbQUPOX#sZ]#s^p#sq;'S#s;'S;=`$[<%lO#sR(]U_PbQOX#sZ]#s^p#sq;'S#s;'S;=`$[<%lO#sR(vU]PbQOX#sZ]#s^p#sq;'S#s;'S;=`$[<%lO#sR)a`bQ^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c!p!p!p!q*c!q;'S!p;'S;=`$b<%lO!pR*j`bQ^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c!f!p!f!g+l!g;'S!p;'S;=`$b<%lO!pR+u^bQdP^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c;'S!p;'S;=`$b<%lO!pR,x`bQ^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c!q!p!q!r-z!r;'S!p;'S;=`$b<%lO!pR.R`bQ^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c!v!p!v!w/T!w;'S!p;'S;=`$b<%lO!pR/^^bQUP^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c;'S!p;'S;=`$b<%lO!pR0a`bQ^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c!t!p!t!u1c!u;'S!p;'S;=`$b<%lO!pR1l^bQeP^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c;'S!p;'S;=`$b<%lO!p", - tokenizers: [0, 1], - topRules: {"Query":[0,1]}, - tokenPrec: 145 -}) diff --git a/packages/ui/src/components/editor/filter/highlight.ts b/packages/ui/src/components/editor/filter/highlight.ts deleted file mode 100644 index 9c05d7267..000000000 --- a/packages/ui/src/components/editor/filter/highlight.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { styleTags, tags as t } from "@lezer/highlight"; - -export const highlight = styleTags({ - // Boolean operators - And: t.operatorKeyword, - Or: t.operatorKeyword, - Not: t.operatorKeyword, - - // Structural punctuation - LParen: t.paren, - RParen: t.paren, - Colon: t.punctuation, - Minus: t.operator, - - // Literals - Phrase: t.string, // "quoted string" - - // Fields - "FieldName/At": t.attributeName, - "FieldName/Word": t.attributeName, - "FieldValue/FieldValueWord": t.attributeValue, -}); diff --git a/packages/ui/src/components/editor/query/extension.test.ts b/packages/ui/src/components/editor/query/extension.test.ts new file mode 100644 index 000000000..16a4be1d1 --- /dev/null +++ b/packages/ui/src/components/editor/query/extension.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, test } from "vite-plus/test"; +import { pathsAt } from "./extension"; + +describe("pathsAt", () => { + const at = (text: string) => pathsAt(text, text.length); + + test("is the field whose value is being typed, quoted values with spaces too", () => { + expect(at("@host:api")).toEqual(["host"]); + expect(at('x @app:"Google C')).toEqual(["app"]); + expect(at('@app:"Google Chrome" ')).toEqual([]); + expect(at("@header.content-type:")).toEqual(["header.content-type"]); + }); + + test("is a map's name while its key is typed", () => { + expect(at("@header.x-")).toEqual(["header"]); + expect(at("@res.header.")).toEqual(["res.header"]); + }); + + test("is nothing outside a field", () => { + expect(at("stripe ")).toEqual([]); + expect(at("")).toEqual([]); + }); +}); diff --git a/packages/ui/src/components/editor/query/extension.ts b/packages/ui/src/components/editor/query/extension.ts new file mode 100644 index 000000000..aed6e0f52 --- /dev/null +++ b/packages/ui/src/components/editor/query/extension.ts @@ -0,0 +1,264 @@ +import type { Completion, CompletionContext, CompletionResult } from "@codemirror/autocomplete"; +import { autocompletion, startCompletion } from "@codemirror/autocomplete"; +import { highlightingFor } from "@codemirror/language"; +import type { EditorState, Extension } from "@codemirror/state"; +import { RangeSetBuilder, StateEffect, StateField } from "@codemirror/state"; +import type { DecorationSet, ViewUpdate } from "@codemirror/view"; +import { Decoration, EditorView, ViewPlugin } from "@codemirror/view"; +import type { Tag } from "@lezer/highlight"; +import { tags as t } from "@lezer/highlight"; +import type { Schema, TokenKind } from "@yaakapp-internal/query"; +import { queryCompletions, queryTokens } from "@yaakapp-internal/query"; + +/** A term the search cannot run, to underline: `from` and `to` count UTF-16 units */ +export interface QueryProblem { + from: number; + to: number; + /** Shown on hover, a few words */ + label: string; + /** Searchable later rather than wrong: marked more quietly */ + quiet?: boolean; +} + +export interface QueryFilterOptions { + schema: Schema; + /** The values seen for each field path, offered after `@field:` */ + values?: Record; + /** + * Values for field paths `values` lacks, read as completion opens: `host` for `@host:`, and a + * map's name, `header`, for the keys after `@header.`. Null for a path with none to offer. + */ + loadValues?: (paths: string[]) => Promise>; + /** + * More completions for a bare word or an empty spot, after the engine's, such as saved + * searches. `typed` is the word, and the completions replace it. + */ + moreCompletions?: (typed: string) => Completion[]; + /** The terms of a query the search cannot run, underlined as typing settles */ + problems?: (text: string) => Promise; +} + +const tokenTags: Record = { + field: t.attributeName, + value: t.attributeValue, + operator: t.operatorKeyword, + quote: t.string, + paren: t.paren, + negation: t.operator, + text: null, +}; + +function buildDecorations(state: EditorState): DecorationSet { + const builder = new RangeSetBuilder(); + for (const token of queryTokens(state.doc.toString())) { + const tag = tokenTags[token.kind]; + const className = tag == null ? null : highlightingFor(state, [tag]); + if (className == null || token.span.start === token.span.end) continue; + builder.add(token.span.start, token.span.end, Decoration.mark({ class: className })); + } + return builder.finish(); +} + +const highlight = ViewPlugin.fromClass( + class { + decorations: DecorationSet; + + constructor(view: EditorView) { + this.decorations = buildDecorations(view.state); + } + + update(update: ViewUpdate) { + if (update.docChanged || update.viewportChanged) { + this.decorations = buildDecorations(update.state); + } + } + }, + { decorations: (v) => v.decorations }, +); + +/** + * The field paths whose values the term at `pos` would offer: its field's, a quoted value with + * spaces included, or its map's keys + */ +export function pathsAt(text: string, pos: number): string[] { + const before = text.slice(0, pos); + const value = /@([A-Za-z0-9_.-]+):(?:"(?:[^"\\]|\\.)*|[^\s()"]*)$/.exec(before); + if (value?.[1] != null) return [value[1].toLowerCase()]; + const key = /@([A-Za-z0-9_.-]+)\.[A-Za-z0-9_-]*$/.exec(before); + if (key?.[1] != null) return [key[1].toLowerCase()]; + return []; +} + +/** The bare word being typed at `pos`, or empty at the start of a term */ +function wordAt(text: string, pos: number): { word: string; from: number } | null { + const before = text.slice(0, pos); + const m = /(^|[\s(])([^\s()"@:-][^\s()":]*)?$/.exec(before); + if (m == null) return null; + const word = m[2] ?? ""; + return { word, from: pos - word.length }; +} + +/** The panel beside a completion: what the field is, and a term using it */ +function infoOf(info: string | null, example: string | null): Completion["info"] { + if (example == null) return undefined; + return () => { + const el = document.createElement("div"); + if (info != null) { + const what = document.createElement("div"); + what.textContent = info; + el.append(what); + } + const term = document.createElement("code"); + term.className = "cm-query-example"; + term.textContent = example; + el.append(term); + return el; + }; +} + +function completionSource({ schema, values, loadValues, moreCompletions }: QueryFilterOptions) { + return async (ctx: CompletionContext): Promise => { + const text = ctx.state.doc.toString(); + let known = values ?? {}; + const wanted = pathsAt(text, ctx.pos).filter((p) => known[p] == null); + if (loadValues != null && wanted.length > 0) { + known = { ...known, ...(await loadValues(wanted)) }; + if (ctx.aborted) return null; + } + const result = queryCompletions(text, ctx.pos, schema, known); + + const options: Completion[] = (result?.items ?? []).map((item) => ({ + label: item.label, + // Not a CodeMirror type: it names the kind for `onlyFields` below, and draws no icon + type: item.kind === "field" ? "field" : "value", + detail: item.info ?? undefined, + info: infoOf(item.info, item.example), + apply: (view, _completion, from, to) => { + view.dispatch({ + changes: { from, to, insert: item.insert }, + selection: { anchor: from + item.insert.length }, + }); + if (item.reopen) startCompletion(view); + }, + })); + + let from = result?.span.start ?? ctx.pos; + let to = result?.span.end ?? ctx.pos; + const word = moreCompletions == null ? null : wordAt(text, ctx.pos); + // On an empty spot, only when asked for + if (moreCompletions != null && word != null && (word.word !== "" || ctx.explicit)) { + const more = moreCompletions(word.word); + if (more.length > 0) { + // Both replace the word being typed + if (options.length === 0) [from, to] = [word.from, ctx.pos]; + if (from === word.from) options.push(...more); + } + } + if (options.length === 0) return null; + + // Field names pop up for something typed; on an empty spot, only when asked for + const onlyFields = options.every((o) => o.type === "field"); + if (onlyFields && from === to && !ctx.explicit) return null; + + // The engine has already filtered and ranked the options + return { from, to, options, filter: false }; + }; +} + +const setProblems = StateEffect.define(); + +/** Underlines what the search cannot run, as `problems` last answered for the text */ +const problemField = StateField.define({ + create: () => Decoration.none, + update(decorations, tr) { + let next = decorations.map(tr.changes); + for (const effect of tr.effects) { + if (!effect.is(setProblems)) continue; + const builder = new RangeSetBuilder(); + const length = tr.state.doc.length; + for (const p of [...effect.value].sort((a, b) => a.from - b.from)) { + const from = Math.min(p.from, length); + // An empty value, `@size:`, marks the colon before it + const to = Math.min(Math.max(p.to, p.from + 1), length); + const at = p.to === p.from ? Math.max(0, from - 1) : from; + if (at >= to) continue; + const className = p.quiet ? "cm-query-quiet" : "cm-query-problem"; + builder.add(at, to, Decoration.mark({ class: className, attributes: { title: p.label } })); + } + next = builder.finish(); + } + return next; + }, + provide: (f) => EditorView.decorations.from(f), +}); + +/** How long typing settles before the search's problems are asked for */ +const PROBLEMS_MS = 150; + +function problemChecker(problems: (text: string) => Promise) { + return ViewPlugin.fromClass( + class { + timer: ReturnType | null = null; + + constructor(readonly view: EditorView) { + this.check(); + } + + update(update: ViewUpdate) { + if (!update.docChanged) return; + if (this.timer != null) clearTimeout(this.timer); + this.timer = setTimeout(() => this.check(), PROBLEMS_MS); + } + + check() { + this.timer = null; + const text = this.view.state.doc.toString(); + problems(text) + .then((found) => { + // Answered for text since changed: the next check covers it + if (this.view.state.doc.toString() !== text) return; + this.view.dispatch({ effects: setProblems.of(found) }); + }) + .catch(() => {}); + } + + destroy() { + if (this.timer != null) clearTimeout(this.timer); + } + }, + ); +} + +const problemTheme = EditorView.baseTheme({ + ".cm-query-problem": { + textDecoration: "underline wavy var(--color-danger)", + textUnderlineOffset: "3px", + }, + ".cm-query-quiet": { + textDecoration: "underline dotted var(--color-text-subtlest, currentColor)", + textUnderlineOffset: "3px", + }, +}); + +const completionTheme = EditorView.baseTheme({ + ".cm-query-example": { + display: "block", + marginTop: "0.25rem", + color: "var(--color-text)", + whiteSpace: "pre", + }, +}); + +/** Highlighting and completion for a search box written in Yaak's query language */ +export function queryFilter(options: QueryFilterOptions): Extension { + const extensions: Extension[] = [ + highlight, + // No type icons: the fields and values of a search have no kind the editor's icons name + autocompletion({ override: [completionSource(options)], icons: false }), + completionTheme, + ]; + if (options.problems != null) { + extensions.push(problemField, problemChecker(options.problems), problemTheme); + } + return extensions; +} diff --git a/packages/ui/src/editor.ts b/packages/ui/src/editor.ts index 347d2139b..df2ae1b56 100644 --- a/packages/ui/src/editor.ts +++ b/packages/ui/src/editor.ts @@ -3,10 +3,11 @@ export { CodeEditor } from "./components/editor/CodeEditor"; export type { LanguageSupports } from "./components/editor/extensions"; export type { FieldDef, FilterOptions } from "./components/editor/filter/extension"; export { - filter, filterAutocomplete, makeCompletionSource as filterCompletionSource, } from "./components/editor/filter/extension"; +export type { QueryFilterOptions, QueryProblem } from "./components/editor/query/extension"; +export { queryFilter } from "./components/editor/query/extension"; export { closeBracketsExtensions, getLanguageExtension,