starred/yaak-mountain-loop
1
0 Fork 0
mirror of https://github.com/mountain-loop/yaak.git synced 2026-11-13 12:13:42 +01:00
Code Issues 8 Packages Projects Releases 10 Wiki Activity

Put Yaak Proxy's search on the shared query engine (#826)

Browse Source 
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Gregory SchierandClaude Fable 5.1 authored and GitHub committed 2026-11-08 09:07:04 -07:00
1 parent 7453869ac6
commit 77226dbedb
58 files changed
+4754 -1594
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Cargo.lock
Generated
+1
View File
@@ -11965,6 +11965,7 @@ dependencies = [
"yaak-models",
"yaak-proxy",
"yaak-proxy-helper",
"yaak-query",
"yaak-rpc",
"yaak-templates",
]
apps/yaak-client/components/Sidebar.tsx
+24 -12
View File
@@ -65,7 +65,7 @@ import type { ContextMenuProps, DropdownItem } from "./core/Dropdown";
import { ContextMenu, Dropdown } from "./core/Dropdown";
import type { Node as QueryNode, Schema } from "@yaakapp-internal/query";
import { formatField, parseQuery, QueryMatcher } from "@yaakapp-internal/query";
import { queryFilter } from "./core/Editor/filter/extension";
import { queryFilter } from "@yaakapp-internal/ui/editor";
import { HttpMethodTag } from "./core/HttpMethodTag";
import {
Icon,
@@ -850,18 +850,27 @@ function setSidebarFilterText(text: string) {
jotaiStore.set(sidebarFilterAtom, { text, key: `${Math.random()}` });
}
/** The fields `getItemFields` fills in. Bare words match `getItemText` by word prefix */
const sidebarFilterColumns = [
"method",
"url",
"name",
"type",
"folder",
"grpc_service",
"grpc_method",
/**
* The fields `getItemFields` fills in, each with what it is and a term using it, as completion
* shows them. Bare words match `getItemText` by word prefix
*/
const sidebarFilterFields: [name: string, info: string, example: string][] = [
["method", "GET, POST, …", "@method:post"],
["url", "Request URL", "@url:*/users*"],
["name", "Request name", '@name:"Get user"'],
["type", "http, grpc or ws", "@type:grpc"],
["folder", "Parent folder", "@folder:auth"],
["grpc_service", "gRPC service", "@grpc_service:*Greeter"],
["grpc_method", "gRPC method", "@grpc_method:SayHello"],
];
const sidebarFilterColumns = sidebarFilterFields.map(([name]) => name);
const sidebarFilterSchema: Schema = {
fields: sidebarFilterColumns.map((name) => ({ name, kind: { type: "text" } })),
fields: sidebarFilterFields.map(([name, info, example]) => ({
name,
kind: { type: "text" },
info,
example,
})),
freeText: "fuzzy",
};
@@ -1005,9 +1014,12 @@ const sidebarTreeAtom = atom<
matcher.free();
markHidden(root);
// Values offered after `@field:`. URLs and names are left out: nearly every one is unique,
// so a list of them is noise rather than a shortcut.
const fieldValues: Record<string, string[]> = {};
for (const [name, values] of Object.entries(allFields)) {
fieldValues[name] = Array.from(values).filter((v) => v.length < 20);
if (name === "url" || name === "name") continue;
fieldValues[name] = Array.from(values);
}
const suggestions = Array.from(suggestionFields)
.sort((a, b) => {
apps/yaak-client/components/core/Editor/filter/extension.ts
-88
View File
@@ -1,88 +0,0 @@
import type { Completion, CompletionContext, CompletionResult } from "@codemirror/autocomplete";
import { autocompletion, startCompletion } from "@codemirror/autocomplete";
import { highlightingFor } from "@codemirror/language";
import type { EditorState, Extension } from "@codemirror/state";
import { RangeSetBuilder } from "@codemirror/state";
import type { DecorationSet, ViewUpdate } from "@codemirror/view";
import { Decoration, EditorView, ViewPlugin } from "@codemirror/view";
import type { Tag } from "@lezer/highlight";
import { tags as t } from "@lezer/highlight";
import type { Schema, TokenKind } from "@yaakapp-internal/query";
import { queryCompletions, queryTokens } from "@yaakapp-internal/query";
export interface QueryFilterOptions {
schema: Schema;
/** The values seen for each field path, offered after `@field:` */
values?: Record<string, string[]>;
}
const tokenTags: Record<TokenKind, Tag | null> = {
field: t.attributeName,
value: t.attributeValue,
operator: t.operatorKeyword,
quote: t.string,
paren: t.paren,
negation: t.operator,
text: null,
};
function buildDecorations(state: EditorState): DecorationSet {
const builder = new RangeSetBuilder<Decoration>();
for (const token of queryTokens(state.doc.toString())) {
const tag = tokenTags[token.kind];
const className = tag == null ? null : highlightingFor(state, [tag]);
if (className == null || token.span.start === token.span.end) continue;
builder.add(token.span.start, token.span.end, Decoration.mark({ class: className }));
}
return builder.finish();
}
const highlight = ViewPlugin.fromClass(
class {
decorations: DecorationSet;
constructor(view: EditorView) {
this.decorations = buildDecorations(view.state);
}
update(update: ViewUpdate) {
if (update.docChanged || update.viewportChanged) {
this.decorations = buildDecorations(update.state);
}
}
},
{ decorations: (v) => v.decorations },
);
function completionSource({ schema, values }: QueryFilterOptions) {
return (ctx: CompletionContext): CompletionResult | null => {
const result = queryCompletions(ctx.state.doc.toString(), ctx.pos, schema, values);
if (result == null) return null;
const { start, end } = result.span;
// Field names pop up for something typed; on an empty spot, only when asked for
const onlyFields = result.items.every((item) => item.kind === "field");
if (onlyFields && start === end && !ctx.explicit) return null;
const options: Completion[] = result.items.map((item) => ({
label: item.label,
type: item.kind === "field" ? "property" : "constant",
info: item.info ?? undefined,
apply: (view, _completion, from, to) => {
view.dispatch({
changes: { from, to, insert: item.insert },
selection: { anchor: from + item.insert.length },
});
if (item.reopen) startCompletion(view);
},
}));
// The engine has already filtered and ranked the options
return { from: start, to: end, options, filter: false };
};
}
/** Highlighting and completion for a search box written in Yaak's query language */
export function queryFilter(options: QueryFilterOptions): Extension {
return [highlight, autocompletion({ override: [completionSource(options)] })];
}
apps/yaak-proxy/components/FlowListEmpty.tsx
+2 -2
View File
@@ -3,7 +3,7 @@ import classNames from "classnames";
import { useAtomValue, useSetAtom } from "jotai";
import { useEffect, useState } from "react";
import { rpc } from "../lib/rpc";
import { parseSearch, widerQueries } from "../lib/search";
import { activeQuery, widerQueries } from "../lib/query";
import {
hideSystemAtom,
pausedAtAtom,
@@ -77,7 +77,7 @@ function NoResults({ text }: { text: string }) {
const candidates = widerQueries(text);
void Promise.all(
candidates.map(async (candidate) => {
const query = parseSearch(candidate, hideSystem);
const query = activeQuery(candidate, hideSystem);
const res = await rpc("list_flows", { ...query, limit: 0 });
return { text: candidate, count: query == null ? res.total : (res.matching ?? res.total) };
}),
apps/yaak-proxy/components/ProxyLayout.tsx
+2
View File
@@ -22,6 +22,7 @@ import { LockedState } from "./LockedState";
import { ProxyStatus, useProxySetup } from "./ProxyStatus";
import { HelperDialog } from "./HelperDialog";
import { SaveDialog } from "./SaveDialog";
import { SavedSearchDialog } from "./SavedSearchDialog";
import { SearchBar } from "./SearchBar";
import { SettingsWindow, useOpenSettings } from "./Settings";
import { Sidebar } from "./Sidebar";
@@ -158,6 +159,7 @@ export function ProxyLayout() {
</div>
</SidebarLayout>
<SaveDialog />
<SavedSearchDialog />
<HelperDialog />
<SettingsWindow />
<Toasts />
apps/yaak-proxy/components/SavedSearchDialog.tsx
+82
View File
@@ -0,0 +1,82 @@
import { Dialog, DialogFooter, PlainInput } from "@yaakapp-internal/ui";
import { atom, useAtom } from "jotai";
import { useState } from "react";
import { useRpcMutation } from "../hooks/useRpcMutation";
import { useShowToast } from "../lib/toast";
/** Saving the search box's text under a name, or renaming a saved search */
export type SavedSearchDialog =
| { mode: "save"; query: string }
| { mode: "rename"; id: string; name: string };
export const savedSearchDialogAtom = atom<SavedSearchDialog | null>(null);
const FORM_ID = "saved-search";
/** Names a saved search, as `savedSearchDialogAtom` asks */
export function SavedSearchDialog() {
const [dialog, setDialog] = useAtom(savedSearchDialogAtom);
if (dialog == null) return null;
// Keyed, so each opening starts from its own name.
return (
<NameDialog
key={dialog.mode === "save" ? `save:${dialog.query}` : `rename:${dialog.id}`}
dialog={dialog}
onClose={() => setDialog(null)}
/>
);
}
function NameDialog({ dialog, onClose }: { dialog: SavedSearchDialog; onClose: () => void }) {
const [name, setName] = useState(dialog.mode === "rename" ? dialog.name : "");
const showToast = useShowToast();
const onError = (err: Error) =>
showToast({ id: "saved-search", message: err.message, color: "danger" });
const create = useRpcMutation("create_saved_query", { onSuccess: onClose, onError });
const rename = useRpcMutation("rename_saved_query", { onSuccess: onClose, onError });
const busy = create.isPending || rename.isPending;
const submit = () => {
if (name.trim() === "" || busy) return;
if (dialog.mode === "save") create.mutate({ name, query: dialog.query });
else rename.mutate({ id: dialog.id, name });
};
return (
<Dialog
open
onClose={onClose}
size="dynamic"
title={dialog.mode === "save" ? "Save search" : "Rename search"}
className="w-[26rem]"
>
<form
id={FORM_ID}
onSubmit={(e) => {
e.preventDefault();
submit();
}}
>
<PlainInput
label="Name"
autoFocus
defaultValue={name}
onChange={setName}
placeholder={dialog.mode === "save" ? dialog.query : undefined}
/>
</form>
<DialogFooter
actions={[
{ label: "Cancel", variant: "border", onClick: onClose },
{
label: dialog.mode === "save" ? "Save" : "Rename",
color: "primary",
form: FORM_ID,
disabled: name.trim() === "",
isLoading: busy,
},
]}
/>
</Dialog>
);
}
apps/yaak-proxy/components/SearchBar.tsx
+92 -36
View File
@@ -1,34 +1,53 @@
import { Icon, IconButton, PlainInput } from "@yaakapp-internal/ui";
import { startCompletion } from "@codemirror/autocomplete";
import type { Extension } from "@codemirror/state";
import { Compartment } from "@codemirror/state";
import type { EditorView } from "@codemirror/view";
import {
Dropdown,
Icon,
IconButton,
Input,
type InputHandle,
LazyCodeEditor,
} from "@yaakapp-internal/ui";
import { queryFilter } from "@yaakapp-internal/ui/editor";
import { useAtom, useAtomValue, useSetAtom, useStore } from "jotai";
import { memo, type RefObject, useEffect, useRef } from "react";
import { memo, useEffect, useMemo, useRef } from "react";
import { useRpcQuery } from "../hooks/useRpcQuery";
import { loadSeenValues, savedCompletions, searchProblems } from "../lib/searchBox";
import {
allTrafficTotalAtom,
lockedAtom,
savedQueriesAtom,
searchTermsAtom,
searchTextAtom,
searchTextVersionAtom,
setSearchTextAtom,
} from "../lib/store";
import { savedSearchDialogAtom } from "./SavedSearchDialog";
/** How long typing settles before the search runs */
const SEARCH_DEBOUNCE_MS = 200;
/**
* The header's search over every flow in the database, by URL, method, status and content type,
* and the `@app`, `@host` and `@path` terms the sidebar writes. Its placeholder counts the flows.
* ⌘F focuses it and Escape clears it.
* The header's search over every flow in the database, in the query language: bare words match
* the URL, method, status and content type, and `@field:value` terms the rest, highlighted and
* completed as Yaak's sidebar filter is, with the values the proxy has seen. The backend runs
* it and marks what it cannot. Its placeholder counts the flows. ⌘F focuses it and Escape
* clears it.
*/
export const SearchBar = memo(function SearchBar() {
const [text, setText] = useAtom(searchTextAtom);
const setTerms = useSetAtom(searchTermsAtom);
const setSearchText = useSetAtom(setSearchTextAtom);
const version = useAtomValue(searchTextVersionAtom);
const total = useAtomValue(allTrafficTotalAtom);
const store = useStore();
const setSavedDialog = useSetAtom(savedSearchDialogAtom);
// Nothing to search while captured traffic is locked.
const locked = useAtomValue(lockedAtom) != null;
// Read, not subscribed: `PlaceholderCount` keeps it current without re-rendering the field.
const total = useStore().get(allTrafficTotalAtom);
// PlainInput hands back its <input> as the ref
const inputRef = useRef<HTMLInputElement>(null);
const schema = useRpcQuery("query_schema", {}, { staleTime: Infinity }).data?.schema;
const handleRef = useRef<InputHandle | null>(null);
useEffect(() => {
const timer = setTimeout(() => setTerms(text), SEARCH_DEBOUNCE_MS);
@@ -39,43 +58,69 @@ export const SearchBar = memo(function SearchBar() {
const onKeyDown = (e: KeyboardEvent) => {
if ((e.metaKey || e.ctrlKey) && !e.shiftKey && !e.altKey && e.key.toLowerCase() === "f") {
e.preventDefault();
inputRef.current?.focus();
inputRef.current?.select();
handleRef.current?.selectAll();
}
};
window.addEventListener("keydown", onKeyDown);
return () => window.removeEventListener("keydown", onKeyDown);
}, []);
// The language goes in once the schema is read: into the editor there is, through a
// compartment, and into each one made after, as taking text from the sidebar makes one.
const compartment = useRef(new Compartment());
const filter = useMemo<Extension>(
() =>
schema == null
? []
: queryFilter({
schema,
loadValues: loadSeenValues,
// Read as completion opens, so saved searches are current without remaking it.
moreCompletions: (typed) => savedCompletions(store.get(savedQueriesAtom), typed),
problems: searchProblems,
}),
[schema, store],
);
const extensions = useMemo(() => compartment.current.of(filter), [filter]);
useEffect(() => {
handleRef.current?.dispatch({ effects: compartment.current.reconfigure(filter) });
}, [filter]);
// The field is replaced to take a value from outside, so focus moves to the new one.
const clear = () => {
setSearchText("");
requestAnimationFrame(() => inputRef.current?.focus());
requestAnimationFrame(() => handleRef.current?.focus());
};
return (
<PlainInput
ref={inputRef}
<Input
renderEditor={(props) => (
<LazyCodeEditor {...props} keyDownHandler={startCompletionOnCtrlSpace} />
)}
setRef={(h) => {
handleRef.current = h;
}}
label="Search flows"
hideLabel
size="sm"
className="font-mono text-xs"
stateKey={null}
language={null}
wrapLines={false}
placeholder={locked ? "Traffic is locked" : placeholderFor(total)}
disabled={locked}
defaultValue={text}
forceUpdateKey={String(version)}
onChange={setText}
onKeyDownCapture={(e) => {
if (e.key === "Escape" && text !== "") {
extensions={extensions}
onKeyDown={(e) => {
if (e.key === "Escape" && handleRef.current?.value() !== "") {
e.stopPropagation();
clear();
}
}}
leftSlot={
<>
<Icon icon="search" size="sm" className="ml-2 mr-2 self-center text-text-subtlest" />
<PlaceholderCount inputRef={inputRef} />
</>
<Icon icon="search" size="sm" className="ml-2 mr-1 self-center text-text-subtlest" />
}
rightSlot={
text === "" ? (
@@ -83,14 +128,26 @@ export const SearchBar = memo(function SearchBar() {
⌘F
</kbd>
) : (
<IconButton
size="xs"
icon="x"
title="Clear search"
iconColor="secondary"
className="self-center ml-2 mr-0.5"
onClick={clear}
/>
<div className="flex items-center self-center ml-1 mr-0.5">
<Dropdown
items={[
{
label: "Save search",
leftSlot: <Icon icon="bookmark" />,
onSelect: () => setSavedDialog({ mode: "save", query: text.trim() }),
},
]}
>
<IconButton size="xs" icon="ellipsis" title="Search actions" iconColor="secondary" />
</Dropdown>
<IconButton
size="xs"
icon="x"
title="Clear search"
iconColor="secondary"
onClick={clear}
/>
</div>
)
}
/>
@@ -98,15 +155,14 @@ export const SearchBar = memo(function SearchBar() {
});
/**
* Keeps the placeholder's count current as flows arrive, alone, so new traffic re-renders this
* and not the whole field
* Ctrl+Space lists what can be typed, fields and saved searches on an empty spot, as it does in
* Yaak's editors, whose shared editor leaves the key to each app
*/
function PlaceholderCount({ inputRef }: { inputRef: RefObject<HTMLInputElement | null> }) {
const total = useAtomValue(allTrafficTotalAtom);
useEffect(() => {
if (inputRef.current != null) inputRef.current.placeholder = placeholderFor(total);
}, [inputRef, total]);
return null;
function startCompletionOnCtrlSpace(e: KeyboardEvent, view: EditorView): boolean {
if (!(e.ctrlKey && e.key === " " && !e.metaKey && !e.altKey && !e.shiftKey)) return false;
e.preventDefault();
startCompletion(view);
return true;
}
/** "Search 4,102 flows", the title and count that sat in the header's middle */
apps/yaak-proxy/components/Sidebar.tsx
+35 -3
View File
@@ -7,10 +7,11 @@ import {
type TreeProps,
} from "@yaakapp-internal/ui";
import classNames from "classnames";
import { atom, useAtom, useAtomValue } from "jotai";
import { atom, useAtom, useAtomValue, useSetAtom } from "jotai";
import { atomWithStorage } from "jotai/utils";
import { useCallback, useEffect } from "react";
import { useDecryptList, useEditDecryptList } from "../hooks/useDecryptList";
import { useRpcMutation } from "../hooks/useRpcMutation";
import { SIDEBAR_TREE_ID } from "../lib/atoms";
import { blockerOf, decryptMenuAction, isDecrypted, listEdit, setDecrypted } from "../lib/decrypt";
import { applyCollapsed, type CollapsedMap, collapsedMap, folderIds } from "../lib/sidebarCollapse";
@@ -18,6 +19,7 @@ import { type SidebarItem, SYSTEM_ID } from "../lib/sidebarTree";
import { hideSystemAtom, noteSidebarInput, sidebarTreeAtom } from "../lib/store";
import { useShowToast } from "../lib/toast";
import { AppIcon } from "./AppIcon";
import { savedSearchDialogAtom } from "./SavedSearchDialog";
/** The apps, sites, hosts and paths the user expanded, by node id, kept across launches */
const expandedAtom = atomWithStorage<Record<string, boolean>>("sidebar_expanded", {}, undefined, {
@@ -63,6 +65,7 @@ function ItemInner({ item }: { item: SidebarItem }) {
<div className="flex items-center gap-2 w-full min-w-0">
{item.kind === "all" && <Icon icon="layers" size="sm" className="text-text-subtle" />}
{item.kind === "tunnelled" && <Icon icon="lock" size="sm" className="text-text-subtle" />}
{item.kind === "saved" && <Icon icon="bookmark" size="sm" className="text-text-subtle" />}
{item.kind === "folder" && <Icon icon="folder" size="sm" className="text-text-subtle" />}
{item.appPath != null && <AppIcon path={item.appPath} />}
<span className="truncate">{item.label}</span>
@@ -119,17 +122,46 @@ function HideSystemToggle() {
export function Sidebar() {
const tree = useAtomValue(sidebarTreeAtom);
const getItemKey = useCallback(
(item: SidebarItem) => `${item.id}:${item.count}:${item.countLabel ?? ""}`,
// The label too, as a saved search's changes when it is renamed
(item: SidebarItem) => `${item.id}:${item.count}:${item.countLabel ?? ""}:${item.label}`,
[],
);
const decryptList = useDecryptList();
const setDecryptList = useEditDecryptList();
const { mutate: saveDecryptList } = setDecryptList;
const showToast = useShowToast();
const setSavedDialog = useSetAtom(savedSearchDialogAtom);
const { mutate: deleteSaved } = useRpcMutation("delete_saved_query", {
onError: (err) => showToast({ id: "saved-search", message: err.message, color: "danger" }),
});
// Decrypt or Stop Decrypting the selected hosts, sites and apps, from new connections on.
// A saved search is renamed or deleted.
const getContextMenu = useCallback(
(items: SidebarItem[]): DropdownItem[] => {
const saved = items.flatMap((i) => (i.savedId == null ? [] : [{ ...i, id: i.savedId }]));
if (saved.length > 0) {
const [first] = saved;
return [
{
label: "Rename",
leftSlot: <Icon icon="pencil" />,
hidden: saved.length > 1 || first == null,
onSelect: () => {
if (first != null)
setSavedDialog({ mode: "rename", id: first.id, name: first.label });
},
},
{
label: "Delete",
color: "danger",
leftSlot: <Icon icon="trash" />,
onSelect: () => {
for (const s of saved) deleteSaved({ id: s.id });
},
},
];
}
const subjects = items.flatMap((i) => (i.decrypt == null ? [] : [i.decrypt]));
const action = decryptMenuAction(decryptList, subjects);
if (action == null) return [];
@@ -155,7 +187,7 @@ export function Sidebar() {
},
];
},
[decryptList, saveDecryptList, showToast],
[decryptList, saveDecryptList, showToast, setSavedDialog, deleteSaved],
);
const renderContextMenu = useCallback<NonNullable<TreeProps<SidebarItem>["renderContextMenu"]>>(
({ items, position, onClose }) =>
apps/yaak-proxy/harness/fixtures.ts
+1
View File
@@ -51,6 +51,7 @@ const base: Flow = {
grpcStatus: null,
grpcMessage: null,
tunnelled: false,
agentRead: false,
clientAddr: "127.0.0.1:54321",
elapsedConnectMs: 18,
elapsedTlsMs: 41,
apps/yaak-proxy/harness/streamFixtures.ts
+1
View File
@@ -50,6 +50,7 @@ const flowBase: Flow = {
grpcStatus: null,
grpcMessage: null,
tunnelled: false,
agentRead: false,
clientAddr: null,
elapsedConnectMs: null,
elapsedTlsMs: null,
apps/yaak-proxy/index.html
+3
View File
@@ -24,5 +24,8 @@
<script type="module" src="/font-size.ts"></script>
<script type="module" src="/lib/theme.ts"></script>
<script type="module" src="/main.tsx"></script>
<div id="cm-portal" class="cm-portal"></div>
<div id="react-portal"></div>
<div id="radix-portal" class="cm-portal"></div>
</body>
</html>
apps/yaak-proxy/lib/flowViews.test.ts
+1
View File
@@ -59,6 +59,7 @@ const flow: Flow = {
grpcStatus: null,
grpcMessage: null,
tunnelled: false,
agentRead: false,
clientAddr: null,
elapsedConnectMs: null,
elapsedTlsMs: null,
apps/yaak-proxy/lib/query.test.ts
+107
View File
@@ -0,0 +1,107 @@
import { describe, expect, test } from "vite-plus/test";
import {
activeQuery,
anyCountsBackFromNow,
asOneTerm,
countsBackFromNow,
fieldTerm,
pathTerm,
termKeys,
widerQueries,
} from "./query";
import { savedCompletions } from "./searchBox";
describe("terms", () => {
test("quote values that need it", () => {
expect(fieldTerm("app", "Google Chrome")).toBe('@app:"Google Chrome"');
expect(fieldTerm("host", "api.x.com")).toBe("@host:api.x.com");
expect(fieldTerm("is", "system", true)).toBe("-@is:system");
});
test("select a path and what is under it", () => {
expect(pathTerm("/v1/charges")).toBe("@path:/v1/charges/*");
expect(pathTerm("/a b")).toBe('@path:"/a b"');
});
test("are read back as written, any case, with their exclusion", () => {
expect(termKeys('@app:"Google Chrome" -@is:System (a OR -b)')).toEqual([
'@app:"google chrome"',
"-@is:system",
"a",
"-b",
]);
expect(termKeys("")).toEqual([]);
});
});
describe("activeQuery", () => {
test("is null for all traffic, and carries hiding System", () => {
expect(activeQuery(" ", false)).toBeNull();
expect(activeQuery("", true)).toEqual({ query: "", hideSystem: true });
expect(activeQuery(" stripe ", false)).toEqual({ query: "stripe", hideSystem: false });
});
});
describe("widerQueries", () => {
test("leaves out each term in turn", () => {
expect(widerQueries('@app:Slack @status:5xx "card declined"')).toEqual([
'@status:5xx "card declined"',
'@app:Slack "card declined"',
"@app:Slack @status:5xx",
]);
expect(widerQueries("@app:Slack")).toEqual([]);
});
test("widens one OR group at a time, keeping the groups", () => {
const text = "@app:Slack @host:absent.com OR @app:Chrome @host:absent.com";
expect(widerQueries(text)).toEqual([
"@host:absent.com OR @app:Chrome @host:absent.com",
"@app:Slack OR @app:Chrome @host:absent.com",
"@app:Slack @host:absent.com OR @host:absent.com",
"@app:Slack @host:absent.com OR @app:Chrome",
]);
// A group of one term is not emptied, which would take in every flow.
expect(widerQueries("@app:Slack OR @app:Chrome @host:b.com")).toEqual([
"@app:Slack OR @host:b.com",
"@app:Slack OR @app:Chrome",
]);
});
test("keeps groups and exclusions whole", () => {
expect(widerQueries("(@status:4xx OR @status:5xx) -(a OR b) c")).toEqual([
"-(a OR b) c",
"(@status:4xx OR @status:5xx) c",
"(@status:4xx OR @status:5xx) -(a OR b)",
]);
});
});
describe("countsBackFromNow", () => {
test("is a @time comparison or range, which ages", () => {
expect(countsBackFromNow("@time:<5m")).toBe(true);
expect(countsBackFromNow("x -(@time:[1h TO 2h] OR y)")).toBe(true);
expect(countsBackFromNow("@time:2026-11-07")).toBe(false);
expect(countsBackFromNow("@status:<500 time")).toBe(false);
});
test("of saved searches, tells when their counts change with time alone", () => {
expect(anyCountsBackFromNow(["@status:4xx", "-@time:>1h stripe"])).toBe(true);
expect(anyCountsBackFromNow(["@status:4xx", "@time:2026-11-07"])).toBe(false);
expect(anyCountsBackFromNow([])).toBe(false);
});
});
describe("saved searches as completions", () => {
test("keep an OR at their top in one group, however it is spaced", () => {
expect(asOneTerm("@status:4xx OR @status:5xx")).toBe("(@status:4xx OR @status:5xx)");
expect(asOneTerm("(@status:4xx)OR(@status:5xx)")).toBe("((@status:4xx)OR(@status:5xx))");
expect(asOneTerm("(a OR b) c")).toBe("(a OR b) c");
const saved = [
{ id: "1", name: "Errors", query: "(@status:4xx)OR(@status:5xx)", sortOrder: 0 },
{ id: "2", name: "Stripe", query: "@host:*.stripe.com", sortOrder: 1 },
];
expect(savedCompletions(saved, "err").map((c) => c.apply)).toEqual([
"((@status:4xx)OR(@status:5xx))",
]);
});
});
apps/yaak-proxy/lib/query.ts
+147
View File
@@ -0,0 +1,147 @@
import type { Node } from "@yaakapp-internal/query";
import { formatField, parseQuery } from "@yaakapp-internal/query";
/**
* A search over every flow in the database, as `list_flows` takes it: the search box's text,
* which the backend parses and runs, and whether System's flows are hidden.
*/
export interface FlowQuery {
query: string;
hideSystem: boolean;
}
/** The search the list shows, or null for all traffic */
export function activeQuery(text: string, hideSystem: boolean): FlowQuery | null {
if (text.trim() === "" && !hideSystem) return null;
return { query: text.trim(), hideSystem };
}
/** `@field:value`, or `-@field:value`, the value quoted when it needs to be */
export function fieldTerm(field: string, value: string, negated = false): string {
return formatField(field, value, negated);
}
/** A site, its domain and every host below it: `@host:*.github.com` */
export function siteTerm(site: string): string {
const term = formatField("host", site);
return term.endsWith('"') ? term : `@host:*.${site}`;
}
/**
* A path and everything under it, `@path:/v1/charges/*`, as a sidebar row selects it. A path
* that has to be quoted cannot carry the wildcard, so it matches as written.
*/
export function pathTerm(path: string): string {
const term = formatField("path", path);
if (term.endsWith('"')) return term;
return `${term.replace(/\/+$/, "")}/*`;
}
/** `node` written back as query text, from its source, groups in parentheses where they need */
export function nodeText(node: Node, source: string, nested = false): string {
switch (node.type) {
case "And": {
const text = node.children.map((c) => nodeText(c, source, true)).join(" ");
return nested ? `(${text})` : text;
}
case "Or": {
const text = node.children.map((c) => nodeText(c, source, true)).join(" OR ");
return nested ? `(${text})` : text;
}
case "Not":
return `-${nodeText(node.node, source, true)}`;
default:
return source.slice(node.span.start, node.span.end);
}
}
/** The terms of a query, each as written and with a `-` when it is excluded, lowercased */
export function termKeys(text: string): string[] {
const keys: string[] = [];
const walk = (node: Node, negated: boolean) => {
switch (node.type) {
case "And":
case "Or":
for (const c of node.children) walk(c, negated);
break;
case "Not":
walk(node.node, !negated);
break;
default:
keys.push(
`${negated ? "-" : ""}${text.slice(node.span.start, node.span.end).toLowerCase()}`,
);
}
};
const root = parseQuery(text);
if (root != null) walk(root, false);
return keys;
}
/**
* Wider searches for one that matched nothing: the text with each of its terms left out in
* turn, without repeats. With groups `OR` separates, a term is left out of one group at a time
* and the groups kept, since each group alone is narrower still; a group left with nothing would
* take in every flow, so it is not offered.
*/
export function widerQueries(text: string): string[] {
const root = parseQuery(text);
if (root == null) return [];
const clauses = root.type === "Or" ? root.children : [root];
const termsOf = (clause: Node) => (clause.type === "And" ? clause.children : [clause]);
const write = (terms: Node[]) => terms.map((t) => nodeText(t, text, true)).join(" ");
const out = new Set<string>();
clauses.forEach((clause, i) => {
const terms = termsOf(clause);
if (terms.length <= 1) return;
terms.forEach((_, j) => {
const wider = clauses.map((c, k) =>
k === i ? write(terms.filter((_, l) => l !== j)) : write(termsOf(c)),
);
out.add(wider.join(" OR "));
});
});
return [...out];
}
/**
* The search has a `@time` term that counts back from now, `@time:<5m`, so which flows it covers
* changes as time passes, not only as flows are written
*/
export function countsBackFromNow(text: string): boolean {
let found = false;
const walk = (node: Node) => {
switch (node.type) {
case "And":
case "Or":
for (const c of node.children) walk(c);
break;
case "Not":
walk(node.node);
break;
case "Field":
if (
node.path.length === 1 &&
node.path[0]?.toLowerCase() === "time" &&
(node.value.type === "compare" || node.value.type === "range")
) {
found = true;
}
break;
default:
}
};
const root = parseQuery(text);
if (root != null) walk(root);
return found;
}
/** Any of the searches counts back from now, so its count changes as time passes */
export function anyCountsBackFromNow(queries: string[]): boolean {
return queries.some(countsBackFromNow);
}
/** `text` to sit beside other terms as one: in parentheses when it is an `OR` at its top */
export function asOneTerm(text: string): string {
return parseQuery(text)?.type === "Or" ? `(${text})` : text;
}
apps/yaak-proxy/lib/search.test.ts
+29 -258
View File
@@ -1,144 +1,6 @@
import type { Flow } from "@yaakapp-internal/proxy-lib";
import { describe, expect, test } from "vite-plus/test";
import {
appKeyOf,
formatFieldFilter,
isSystemFlow,
matchesQuery,
parseSearch,
sidebarPath,
siteOf,
tokenize,
widerQueries,
} from "./search";
function flow(url: string, extra: Partial<Flow> = {}): Flow {
return {
url,
method: "GET",
status: 200,
contentType: null,
clientBundleId: null,
clientPath: null,
...extra,
} as Flow;
}
// The same flows and terms as `search_matches_url_method_status_and_content_type` in
// `flows.rs`, so the live matcher and the database agree.
const stripe = flow("https://api.stripe.com/v1/charges?customer=cus_9", {
method: "POST",
contentType: "application/json; charset=utf-8",
});
const github = flow("https://api.github.com/repos/yaak", {
status: 404,
contentType: "application/json; charset=utf-8",
});
const logo = flow("https://images.ctfassets.net/logo.png", { contentType: "image/png" });
const health = flow("http://localhost:3000/health", { method: "HEAD", status: 503 });
const all = [stripe, github, logo, health];
const search = (q: string) => all.filter((f) => matchesQuery(f, { q }));
describe("matchesQuery", () => {
test("matches the host, the path and the query, any case", () => {
expect(search("STRIPE")).toEqual([stripe]);
expect(search("/repos/")).toEqual([github]);
expect(search("cus_9")).toEqual([stripe]);
});
test("matches the whole method", () => {
expect(search("head")).toEqual([health]);
expect(search("ead")).toEqual([]);
});
test("matches a status, its start, or its class", () => {
expect(search("404")).toEqual([github]);
expect(search("50")).toEqual([health]);
expect(search("2xx")).toEqual([stripe, logo]);
});
test("matches the content type", () => {
expect(search("png")).toEqual([logo]);
expect(search("json")).toEqual([stripe, github]);
});
test("needs every term to match", () => {
expect(search("json get")).toEqual([github]);
expect(search(" json 2xx ")).toEqual([stripe]);
expect(search("json 5xx")).toEqual([]);
expect(search("")).toEqual(all);
});
test("keeps flows under a host or path, as the sidebar places them", () => {
const under = (node: string, url: string) => matchesQuery(flow(url), { under: [node] });
expect(under("api.stripe.com", "https://api.stripe.com/v1/charges")).toBe(true);
expect(under("api.stripe.com", "http://api.stripe.com")).toBe(true);
// Another port is a host node of its own, as in the sidebar.
expect(under("api.stripe.com", "https://api.stripe.com:8443/v1")).toBe(false);
// Empty segments and default ports are left out, as the sidebar leaves them.
expect(under("example.com/v1/users", "https://example.com/v1//users")).toBe(true);
expect(under("x.com/a", "http://x.com:80/a")).toBe(true);
expect(under("api.stripe.com", "https://api.stripe.community/")).toBe(false);
expect(under("api.stripe.com/v1/charges", "https://api.stripe.com/v1/charges/ch_1?x")).toBe(
true,
);
expect(under("api.stripe.com/v1/charges", "https://api.stripe.com/v1/chargesx")).toBe(false);
expect(under("api.stripe.com:8443", "https://api.stripe.com/v1")).toBe(false);
// The sidebar lowercases hosts; paths keep their case.
expect(under("example.com/A", "http://EXAMPLE.com/A/b")).toBe(true);
expect(under("example.com/a", "http://example.com/A")).toBe(false);
});
test("keeps flows from an app, by bundle id or path", () => {
const chrome = flow("https://example.com/", {
clientBundleId: "com.google.Chrome",
clientPath: "/Applications/Google Chrome.app",
});
const curl = flow("https://example.com/", { clientPath: "/usr/bin/curl" });
const unknown = flow("https://example.com/");
const from = (app: string[]) => [chrome, curl, unknown].filter((f) => matchesQuery(f, { app }));
expect(from(["com.google.Chrome"])).toEqual([chrome]);
expect(from(["/usr/bin/curl"])).toEqual([curl]);
expect(from(["/Applications/Google Chrome.app"])).toEqual([]);
});
test("keeps flows from an app by name, or a program by the folder it ran in", () => {
// The flows `filters_by_app_name_folder_site_and_system` in `flows.rs` checks.
const slack = flow("https://edgeapi.slack.com/cache", {
clientName: "Slack",
clientBundleId: "com.tinyspeck.slackmacgap",
clientPath: "/Applications/Slack.app",
});
const node = flow("https://api.github.com:443/repos", {
clientName: "node",
clientPath: "/opt/homebrew/bin/node",
clientCwd: "/Users/me/src/fieldwork",
});
const daemon = flow("https://github.com/login", {
clientName: "nsurlsessiond",
clientPath: "/usr/libexec/nsurlsessiond",
clientCwd: "/",
});
const none = flow("http://notgithub.com/");
const flows = [slack, node, daemon, none];
const where = (query: Parameters<typeof matchesQuery>[1]) =>
flows.filter((f) => matchesQuery(f, query));
expect(where({ app: ["slack"] })).toEqual([slack]);
expect(where({ app: ["Fieldwork"] })).toEqual([node]);
expect(where({ app: ["/opt/homebrew/bin/node"] })).toEqual([node]);
expect(where({ under: ["*.github.com"] })).toEqual([node, daemon]);
expect(where({ under: ["api.github.com"] })).toEqual([node]);
expect(where({ system: true })).toEqual([daemon]);
expect(where({ system: false })).toEqual([slack, node, none]);
expect(flows.map(appKeyOf)).toEqual(["Slack", "fieldwork", "nsurlsessiond", null]);
});
test("combines terms with the sidebar's selection", () => {
expect(matchesQuery(stripe, { q: "post", under: ["api.stripe.com"] })).toBe(true);
expect(matchesQuery(stripe, { q: "post", under: ["api.github.com"] })).toBe(false);
});
});
import { appKeyOf, isSystemFlow, sidebarPath, siteOf } from "./search";
describe("sidebarPath", () => {
// The cases `sidebar_path_rules` in `flows.rs` checks, so the two agree.
@@ -159,125 +21,6 @@ describe("sidebarPath", () => {
});
});
describe("parseSearch", () => {
test("is null for text that narrows nothing", () => {
expect(parseSearch("")).toBeNull();
expect(parseSearch(" ")).toBeNull();
expect(parseSearch("OR")).toBeNull();
});
test("keeps free words as terms, trimmed and collapsed", () => {
expect(parseSearch(" stripe post ")).toEqual({ q: "stripe post" });
});
test("reads apps, hosts and the paths below them", () => {
expect(parseSearch('@app:"Google Chrome" @app:Slack card')).toEqual({
q: "card",
app: ["Google Chrome", "Slack"],
});
expect(parseSearch("@app:fieldwork @host:api.stripe.com @path:/v1/payment_intents")).toEqual({
app: ["fieldwork"],
under: ["api.stripe.com/v1/payment_intents"],
});
expect(parseSearch("@host:*.github.com")).toEqual({ under: ["*.github.com"] });
expect(parseSearch("@host:github.com @path:/mountain-loop/yaak/*")).toEqual({
under: ["github.com/mountain-loop/yaak"],
});
// A path with no host matches anywhere in the URL.
expect(parseSearch("@path:/v1")).toEqual({ q: "/v1" });
// Each path stays with the host before it.
expect(parseSearch("@host:a.com @path:/x @host:b.com @path:/y")).toEqual({
under: ["a.com/x", "b.com/y"],
});
});
test("keeps each clause OR separates as a group of its own", () => {
// Two selected sidebar rows under different apps: Slack on a.com, or Chrome on b.com.
const text = "@app:Slack @host:a.com OR @app:Chrome @host:b.com";
const query = parseSearch(text);
expect(query).toEqual({
any: [
{ app: ["Slack"], under: ["a.com"] },
{ app: ["Chrome"], under: ["b.com"] },
],
});
const on = (clientName: string, host: string) =>
flow(`https://${host}/x`, { clientName, clientBundleId: `app.${clientName}` });
const match = (f: Flow) => query != null && matchesQuery(f, query);
expect(match(on("Slack", "a.com"))).toBe(true);
expect(match(on("Chrome", "b.com"))).toBe(true);
// Not every pairing of the two rows' apps and hosts.
expect(match(on("Slack", "b.com"))).toBe(false);
expect(match(on("Chrome", "a.com"))).toBe(false);
// Words bind to their clause, and hiding System applies to each group that does not ask.
expect(parseSearch("@host:a.com OR @is:system @app:trustd", true)).toEqual({
any: [
{ under: ["a.com"], system: false },
{ app: ["trustd"], system: true },
],
});
// A clause that narrows nothing takes in every flow.
expect(parseSearch("@app:Slack OR")).toEqual({ app: ["Slack"] });
expect(parseSearch("@app:Slack OR -@foo:bar")).toBeNull();
});
test("keeps or drops System, which hiding it drops unless the text asks", () => {
expect(parseSearch("@is:system")).toEqual({ system: true });
expect(parseSearch("-@is:system stripe")).toEqual({ q: "stripe", system: false });
expect(parseSearch("", true)).toEqual({ system: false });
expect(parseSearch("@is:system", true)).toEqual({ system: true });
});
test("keeps or drops HTTPS passed through undecrypted", () => {
expect(parseSearch("@is:tunnelled")).toEqual({ tunnelled: true });
expect(parseSearch("-@is:tunnelled stripe")).toEqual({ q: "stripe", tunnelled: false });
const tunnel = { url: "https://a.com", method: "CONNECT", tunnelled: true } as Flow;
const decrypted = { url: "https://a.com/x", method: "GET", tunnelled: false } as Flow;
expect(matchesQuery(tunnel, { tunnelled: true })).toBe(true);
expect(matchesQuery(decrypted, { tunnelled: true })).toBe(false);
expect(matchesQuery(decrypted, { tunnelled: false })).toBe(true);
});
test("round-trips what the sidebar writes", () => {
const text = [
formatFieldFilter("app", "Google Chrome"),
formatFieldFilter("host", "api.x.com"),
].join(" ");
expect(text).toBe('@app:"Google Chrome" @host:api.x.com');
expect(tokenize(text)).toEqual([
{ field: "app", value: "Google Chrome", negated: false },
{ field: "host", value: "api.x.com", negated: false },
]);
});
});
describe("widerQueries", () => {
test("leaves out each term in turn", () => {
expect(widerQueries('@app:Slack @status:5xx "card declined"')).toEqual([
'@status:5xx "card declined"',
'@app:Slack "card declined"',
"@app:Slack @status:5xx",
]);
expect(widerQueries("@app:Slack")).toEqual([]);
});
test("widens one OR group at a time, keeping the groups", () => {
const text = "@app:Slack @host:absent.com OR @app:Chrome @host:absent.com";
expect(widerQueries(text)).toEqual([
"@host:absent.com OR @app:Chrome @host:absent.com",
"@app:Slack OR @app:Chrome @host:absent.com",
"@app:Slack @host:absent.com OR @host:absent.com",
"@app:Slack @host:absent.com OR @app:Chrome",
]);
// A group of one term is not emptied, which would take in every flow.
expect(widerQueries("@app:Slack OR @app:Chrome @host:b.com")).toEqual([
"@app:Slack OR @host:b.com",
"@app:Slack OR @app:Chrome",
]);
});
});
describe("isSystemFlow", () => {
// The cases `tells_system_daemons_apart` in `flows.rs` checks, so the two agree.
test("is a daemon outside an app, under the system's folders or by name", () => {
@@ -307,3 +50,31 @@ describe("siteOf", () => {
expect(siteOf("149.154.175.50")).toBe("149.154.175.50");
});
});
describe("appKeyOf", () => {
test("names an app, a program, or a runtime by the folder it ran in", () => {
const flow = (extra: Partial<Flow>) =>
({
clientName: null,
clientBundleId: null,
clientPath: null,
clientCwd: null,
...extra,
}) as Flow;
const flows = [
flow({ clientName: "Slack", clientBundleId: "com.tinyspeck.slackmacgap" }),
flow({
clientName: "node",
clientPath: "/opt/homebrew/bin/node",
clientCwd: "/src/fieldwork",
}),
flow({
clientName: "nsurlsessiond",
clientPath: "/usr/libexec/nsurlsessiond",
clientCwd: "/",
}),
flow({}),
];
expect(flows.map(appKeyOf)).toEqual(["Slack", "fieldwork", "nsurlsessiond", null]);
});
});
apps/yaak-proxy/lib/search.ts
+4 -213
View File
@@ -1,147 +1,6 @@
import type { Flow, FlowFilter } from "@yaakapp-internal/proxy-lib";
import type { Flow } from "@yaakapp-internal/proxy-lib";
import { getDomain } from "tldts";
/**
* A search over every flow in the database: the search box's terms, its field terms, and
* whether System's flows are hidden, as `list_flows` takes them.
*/
export type FlowQuery = Pick<
FlowFilter,
"q" | "under" | "app" | "cwd" | "system" | "tunnelled" | "any"
>;
/** A word of the search box: a free term, or `@field:value`, maybe negated with `-` */
export interface SearchToken {
field: string | null;
value: string;
negated: boolean;
}
/**
* The search box's words. Values may be quoted, `@app:"Google Chrome"`; `OR` and `AND` between
* words are dropped here, and [`clausesOf`] splits on `OR`.
*/
export function tokenize(text: string): SearchToken[] {
return clausesOf(text).flat();
}
/**
* The search box's words in the clauses `OR` separates, each a group of terms that must all
* match: `@app:Slack @host:a.com OR @app:Chrome @host:b.com` is two. `AND` is dropped, since
* terms side by side must all match anyway.
*/
export function clausesOf(text: string): SearchToken[][] {
const clauses: SearchToken[][] = [[]];
const re = /(-)?(?:@([a-z]+):)?(?:"([^"]*)"?|(\S+))/gi;
for (const m of text.matchAll(re)) {
const [, minus, field, quoted, bare] = m;
const value = quoted ?? bare ?? "";
if (field == null && minus == null && quoted == null && value === "OR") {
clauses.push([]);
continue;
}
if (field == null && minus == null && quoted == null && value === "AND") continue;
if (field == null && value === "" && quoted == null) continue;
clauses.at(-1)?.push({ field: field?.toLowerCase() ?? null, value, negated: minus != null });
}
return clauses.filter((c) => c.length > 0);
}
/** `@field:value`, quoting a value with spaces or quotes in it */
export function formatFieldFilter(field: string, value: string, negated = false): string {
const v = /[\s"]/.test(value) || value === "" ? `"${value.replaceAll('"', "")}"` : value;
return `${negated ? "-" : ""}@${field}:${v}`;
}
/**
* The search box's text as a query, or null when it narrows nothing. Clauses `OR` separates are
* groups, each judged whole, so two sidebar rows under different apps match what each does and
* not every pairing of their apps and hosts. Within a clause, `@app` terms match any of their
* apps; `@host` terms any of their hosts, each with the `@path` terms after it; `@is:system`
* keeps only macOS's daemons and `-@is:system` drops them, as hiding System does when the
* clause says neither. `@is:tunnelled` keeps only HTTPS passed through undecrypted, and
* `-@is:tunnelled` drops it. Other words match the URL, the method, the status or the type.
*/
export function parseSearch(text: string, hideSystem = false): FlowQuery | null {
const clauses = clausesOf(text);
if (clauses.length <= 1) return parseClause(clauses[0] ?? [], hideSystem);
const any: FlowQuery[] = [];
for (const clause of clauses) {
const group = parseClause(clause, hideSystem);
// A clause that narrows nothing takes in every flow, so the whole does too.
if (group == null) return hideSystem ? { system: false } : null;
any.push(group);
}
return { any };
}
function parseClause(tokens: SearchToken[], hideSystem: boolean): FlowQuery | null {
const words: string[] = [];
const apps: string[] = [];
const cwds: string[] = [];
// Each host with the paths written after it.
const hosts: { host: string; paths: string[] }[] = [];
let system: boolean | undefined = hideSystem ? false : undefined;
let tunnelled: boolean | undefined;
for (const t of tokens) {
if (t.field === "app" && !t.negated && t.value !== "") apps.push(t.value);
else if (t.field === "cwd" && !t.negated && t.value !== "") cwds.push(t.value);
else if (t.field === "host" && !t.negated && t.value !== "") {
hosts.push({ host: t.value.replace(/\/+$/, ""), paths: [] });
} else if (t.field === "path" && !t.negated && t.value !== "") {
const path = t.value.replace(/\/?\*$/, "").replace(/^\/+/, "");
const host = hosts.at(-1);
// A path with no host to sit under matches anywhere in the URL.
if (host == null) {
if (path !== "") words.push(`/${path}`);
} else {
host.paths.push(path);
}
} else if (t.field === "is" && t.value.toLowerCase() === "system") system = !t.negated;
else if (t.field === "is" && t.value.toLowerCase() === "tunnelled") tunnelled = !t.negated;
else if (t.field == null && !t.negated) words.push(t.value);
// Fields and exclusions the query language will add are ignored until it lands.
}
const under: string[] = [];
for (const { host, paths } of hosts) {
if (paths.length === 0 || host.startsWith("*.")) under.push(host);
else for (const p of paths) under.push(p === "" ? host : `${host}/${p}`);
}
const q = words.join(" ").trim();
const query: FlowQuery = {
...(q === "" ? {} : { q }),
...(under.length > 0 ? { under } : {}),
...(apps.length > 0 ? { app: apps } : {}),
...(cwds.length > 0 ? { cwd: cwds } : {}),
...(system == null ? {} : { system }),
...(tunnelled == null ? {} : { tunnelled }),
};
return Object.keys(query).length === 0 ? null : query;
}
/**
* Wider searches for one that matched nothing: the text with each of its terms left out in turn,
* without repeats. With groups `OR` separates, a term is left out of one group at a time and
* the groups kept, since each group alone is narrower still; a group left with nothing would
* take in every flow, so it is not offered.
*/
export function widerQueries(text: string): string[] {
const format = (t: SearchToken) =>
t.field == null
? `${t.negated ? "-" : ""}${/\s/.test(t.value) ? `"${t.value}"` : t.value}`
: formatFieldFilter(t.field, t.value, t.negated);
const clauses = clausesOf(text).map((c) => c.map(format));
const out = new Set<string>();
clauses.forEach((clause, i) => {
if (clause.length <= 1) return;
clause.forEach((_, j) => {
const wider = clauses.map((c, k) => (k === i ? c.filter((_, l) => l !== j) : c));
out.add(wider.map((c) => c.join(" ")).join(" OR "));
});
});
return [...out];
}
/** The folder a program outside an app ran in */
export function folderOf(flow: Pick<Flow, "clientBundleId" | "clientCwd">): string | null {
if (flow.clientBundleId != null) return null;
@@ -202,8 +61,8 @@ const SYSTEM_PROGRAMS = new Set([
/**
* A flow from one of macOS's own daemons: a process outside an app bundle whose executable is
* under `/System`, `/usr/libexec` and the like, or one of a few daemons by name, but not WebKit's
* helpers, which send apps' traffic. `is_system` in `crates-proxy/yaak-proxy-lib`, which the
* database filters by, follows the same rules.
* helpers, which send apps' traffic. `is_system` in `crates-proxy/yaak-proxy-lib`, which
* `@is:system` searches by, follows the same rules.
*/
export function isSystemFlow(flow: Pick<Flow, "clientBundleId" | "clientPath">): boolean {
const path = flow.clientPath;
@@ -213,66 +72,6 @@ export function isSystemFlow(flow: Pick<Flow, "clientBundleId" | "clientPath">):
return SYSTEM_DIRS.some((dir) => path.startsWith(dir)) || SYSTEM_PROGRAMS.has(name);
}
/**
* Whether `flow` is one `query` covers, as `list_flows` decides it in SQL, so new traffic joins a
* search as it arrives. Every term must match the URL, the method, the status or the content type.
*/
export function matchesQuery(flow: Flow, query: FlowQuery): boolean {
for (const term of (query.q ?? "").toLowerCase().split(/\s+/)) {
if (term === "") continue;
const matched =
flow.url.toLowerCase().includes(term) ||
flow.method.toLowerCase() === term ||
(flow.contentType?.toLowerCase().includes(term) ?? false) ||
matchesStatus(flow.status, term);
if (!matched) return false;
}
if (query.under != null && !query.under.some((node) => isUnder(flow.url, node))) return false;
if (query.app != null && !query.app.some((app) => matchesApp(flow, app))) return false;
if (query.cwd != null && !query.cwd.some((cwd) => matchesCwd(flow, cwd))) return false;
if (query.system != null && isSystemFlow(flow) !== query.system) return false;
if (query.tunnelled != null && flow.tunnelled !== query.tunnelled) return false;
// One of the groups, each judged whole, as the database judges them.
if (query.any != null && query.any.length > 0) {
return query.any.some((group) => matchesQuery(flow, group));
}
return true;
}
/** By bundle id or path exactly, or by name or folder in any case */
function matchesApp(flow: Flow, app: string): boolean {
if ((flow.clientBundleId ?? flow.clientPath) === app) return true;
const lower = app.toLowerCase();
return flow.clientName?.toLowerCase() === lower || folderOf(flow)?.toLowerCase() === lower;
}
/** By the whole working directory exactly, or its folder's name in any case */
function matchesCwd(flow: Flow, cwd: string): boolean {
if (flow.clientCwd == null) return false;
return flow.clientCwd === cwd || folderName(flow.clientCwd)?.toLowerCase() === cwd.toLowerCase();
}
/** `404` exactly, `40` as the start of one, or a class such as `4xx` */
function matchesStatus(status: number | null, term: string): boolean {
if (status == null) return false;
const klass = /^([1-9])xx$/.exec(term);
if (klass != null) return Math.trunc(status / 100) === Number(klass[1]);
if (!/^\d{1,3}$/.test(term)) return false;
return term.length === 3 ? status === Number(term) : String(status).startsWith(term);
}
/** The URL is the host or path `node` names, or something below it, by the sidebar's rules */
function isUnder(url: string, node: string): boolean {
const path = sidebarPath(url);
if (node.startsWith("*.") && !node.includes("/")) {
const site = node.slice(2).toLowerCase();
const host = hostnameOf(path.split("/")[0] ?? "");
return host === site || host.endsWith(`.${site}`);
}
const want = normalizeNode(node);
return path === want || path.startsWith(`${want}/`);
}
/** A host without its port, keeping an IPv6 address's brackets */
export function hostnameOf(host: string): string {
if (host.startsWith("[")) {
@@ -304,7 +103,7 @@ export function siteOf(host: string): string {
* Where the sidebar places a URL: its host, lowercased and without its scheme's default port,
* then its path's non-empty segments, joined by `/`, as in `api.stripe.com/v1/charges`. A URL
* without a scheme is a host of its own. `sidebar_path` in `crates-proxy/yaak-proxy-lib`, which
* the database filters by, follows the same rules.
* `@host` searches by, follows the same rules.
*/
export function sidebarPath(url: string): string {
const schemeEnd = url.indexOf("://");
@@ -326,11 +125,3 @@ export function sidebarPath(url: string): string {
const segments = path.split("/").filter((s) => s !== "");
return [host, ...segments].join("/");
}
/** A sidebar node as `sidebarPath` writes it, its host lowercased */
function normalizeNode(node: string): string {
const slash = node.indexOf("/");
return slash === -1
? node.toLowerCase()
: `${node.slice(0, slash).toLowerCase()}${node.slice(slash)}`;
}
apps/yaak-proxy/lib/searchBox.ts
+79
View File
@@ -0,0 +1,79 @@
import type { Completion } from "@codemirror/autocomplete";
import type { QueryDiagnostic, SavedQuery } from "@yaakapp-internal/proxy-lib";
import type { QueryProblem } from "@yaakapp-internal/ui/editor";
import { asOneTerm } from "./query";
import { rpc } from "./rpc";
/** How long values read for completion are offered before they are read again */
const VALUES_MS = 10_000;
const seenValues = new Map<string, { values: string[]; at: number }>();
const reading = new Map<string, Promise<string[]>>();
/**
* The values the proxy has seen for field paths, for completing them: read from the database
* the first time each is asked for, and again once they are a few seconds old, so the list
* keeps up with the traffic without a read per keystroke.
*/
export async function loadSeenValues(paths: string[]): Promise<Record<string, string[]>> {
const now = Date.now();
const entries = await Promise.all(
paths.map(async (path): Promise<[string, string[]]> => {
const cached = seenValues.get(path);
if (cached != null && now - cached.at < VALUES_MS) return [path, cached.values];
let read = reading.get(path);
if (read == null) {
read = rpc("query_values", { path })
.then((values) => {
seenValues.set(path, { values, at: Date.now() });
return values;
})
.catch(() => cached?.values ?? [])
.finally(() => reading.delete(path));
reading.set(path, read);
}
return [path, await read];
}),
);
return Object.fromEntries(entries);
}
const PROBLEM_LABELS: Record<QueryDiagnostic["kind"], string> = {
unknownField: "Unknown field",
invalidValue: "Invalid value",
notYet: "Not yet",
};
/** The terms of a search the backend cannot run, as the box underlines them */
export function problemsOf(diagnostics: QueryDiagnostic[]): QueryProblem[] {
return diagnostics.map((d) => ({
from: d.start,
to: d.end,
label: PROBLEM_LABELS[d.kind],
quiet: d.kind === "notYet",
}));
}
export async function searchProblems(text: string): Promise<QueryProblem[]> {
if (text.trim() === "") return [];
return problemsOf(await rpc("validate_query", { query: text }));
}
/**
* Saved searches whose names hold the word being typed, each replacing it with its search. One
* with `OR` at its top goes in parentheses, so it stays one group beside the box's other terms.
*/
export function savedCompletions(saved: SavedQuery[], typed: string): Completion[] {
const needle = typed.toLowerCase();
return saved
.filter((s) => s.name.toLowerCase().includes(needle))
.map((s) => {
const insert = asOneTerm(s.query);
return {
label: s.name,
detail: s.query,
type: "text",
apply: insert,
};
});
}
apps/yaak-proxy/lib/searchChanges.test.ts
+96
View File
@@ -0,0 +1,96 @@
import type { Flow, ModelPayload } from "@yaakapp-internal/proxy-lib";
import { describe, expect, test } from "vite-plus/test";
import { emptyWindow, type FlowWindow } from "./flowWindow";
import { changesFor, FlowEvents } from "./searchChanges";
const none = () => {};
function write(id: string, matches: boolean, key = "k"): ModelPayload {
return {
model: { id, startedAt: "2026-11-07 12:00:00", updatedAt: "2026-11-07 12:00:00" } as Flow,
change: { type: "upsert", created: false },
search: { key, matches },
};
}
describe("changesFor", () => {
test("takes matching flows and drops those that stop matching", () => {
const win: FlowWindow = { ...emptyWindow, flows: [write("a", true).model] };
const kept = changesFor(
"k",
win,
[write("a", false), write("b", true), write("c", false)],
none,
);
expect(kept.map((c) => [c.model.id, c.change.type])).toEqual([
["a", "delete"],
["b", "upsert"],
]);
});
test("a flow that joins and stops matching in one batch leaves again", () => {
const kept = changesFor("k", emptyWindow, [write("a", true), write("a", false)], none);
expect(kept.map((c) => [c.model.id, c.change.type])).toEqual([
["a", "upsert"],
["a", "delete"],
]);
});
test("leaves flows matched against another search to be matched again", () => {
const later: string[] = [];
const kept = changesFor("k", emptyWindow, [write("a", true, "old")], (unknown) =>
later.push(...unknown.map((c) => c.model.id)),
);
expect([kept, later]).toEqual([[], ["a"]]);
});
});
describe("FlowEvents", () => {
test("drops a late answer for a flow a newer event removed, though the window no longer holds it", () => {
const events = new FlowEvents();
const old = write("a", true, "old");
events.arrived([old]);
// Under the new search the flow stops matching and leaves the window...
events.arrived([write("a", false)]);
// ...and the re-check of the old event answers after
expect(events.current([old])).toEqual([]);
});
test("drops a late answer for a flow the sweep removed", () => {
const events = new FlowEvents();
const old = write("a", true, "old");
events.arrived([old]);
events.removed(["a"]);
expect(events.current([old])).toEqual([]);
});
test("keeps an answer no newer event has passed, and one for other flows", () => {
const events = new FlowEvents();
const a = write("a", true, "old");
const b = write("b", true, "old");
events.arrived([a, b]);
events.arrived([write("c", true)]);
expect(events.current([a, b])).toEqual([a, b]);
// Not noted: nothing to judge it by
const unseen = write("d", true, "old");
expect(events.current([unseen])).toEqual([unseen]);
});
test("past what it remembers, drops what it can no longer judge", () => {
const events = new FlowEvents(2);
const a = write("a", true, "old");
events.arrived([a, write("b", true), write("c", true)]);
expect(events.current([a])).toEqual([]);
const c = write("c", true, "old");
events.arrived([c]);
expect(events.current([c])).toEqual([c]);
});
test("drops answers from before a clear", () => {
const events = new FlowEvents();
const a = write("a", true, "old");
events.arrived([a]);
events.reset();
expect(events.current([a])).toEqual([]);
});
});
apps/yaak-proxy/lib/searchChanges.ts
+105
View File
@@ -0,0 +1,105 @@
import type { ModelPayload } from "@yaakapp-internal/proxy-lib";
import type { FlowWindow } from "./flowWindow";
/**
* The changes a search's window `win` takes: flows the backend matched against the search, by
* its `key`, as it sent their events, and every deletion. A flow the window holds that no longer
* matches, such as a pending flow that completed under `@is:pending`, leaves it, even when it
* joined earlier in the same batch. Flows matched against another search, sent as the search
* changed or read from the database while another process owns the engine, go to `unknown` to
* be matched again.
*/
export function changesFor(
key: string,
win: FlowWindow,
changes: ModelPayload[],
unknown: (changes: ModelPayload[]) => void,
): ModelPayload[] {
const others: ModelPayload[] = [];
// What the window holds as each change applies
const held = new Set(win.flows.map((f) => f.id));
const kept: ModelPayload[] = [];
for (const c of changes) {
if (c.change.type === "delete") {
held.delete(c.model.id);
kept.push(c);
} else if (c.search?.key !== key) {
others.push(c);
} else if (c.search.matches) {
held.add(c.model.id);
kept.push(c);
} else if (held.delete(c.model.id)) {
kept.push({ ...c, change: { type: "delete" } });
}
}
if (others.length > 0) unknown(others);
return kept;
}
/** Flows whose newest event `FlowEvents` remembers, at most */
const REMEMBERED = 20_000;
/**
* The order flow events arrived in, so an answer that comes later than newer events for the
* same flow can be told apart: a re-check started before the search changed, say, whose flow a
* newer event has since updated or removed, even from the window.
*/
export class FlowEvents {
private count = 0;
/** Each event's place, by the payload as it arrived */
private seqs = new WeakMap<ModelPayload, number>();
/** The newest event's place, by flow, removals included, the least recent first */
private latest = new Map<string, number>();
/** Places at or below this may have newer events no longer remembered */
private floor = 0;
constructor(private readonly remembered = REMEMBERED) {}
/** Notes events as they arrive */
arrived(changes: ModelPayload[]) {
for (const c of changes) {
this.count++;
this.seqs.set(c, this.count);
this.note(c.model.id);
}
}
/** Notes flows removed with no event of their own, as the keep-traffic sweep removes them */
removed(ids: string[]) {
for (const id of ids) {
this.count++;
this.note(id);
}
}
private note(id: string) {
// Moved to the end, so the least recent go first
this.latest.delete(id);
this.latest.set(id, this.count);
for (const [oldest, seq] of this.latest) {
if (this.latest.size <= this.remembered) break;
this.latest.delete(oldest);
this.floor = Math.max(this.floor, seq);
}
}
/**
* Of the changes a late answer is for, those no newer event for their flow has passed. One
* from before the oldest event remembered is dropped, as it can't be judged; the next count
* and reload correct the list. One whose arrival was not noted is kept.
*/
current<T extends ModelPayload>(changes: T[]): T[] {
return changes.filter((c) => {
const seq = this.seqs.get(c);
if (seq == null) return true;
const newest = this.latest.get(c.model.id);
return newest == null ? seq > this.floor : seq >= newest;
});
}
/** Forgets everything, as a clear does */
reset() {
this.latest.clear();
this.floor = this.count;
}
}
apps/yaak-proxy/lib/sidebarTree.test.ts
+38 -13
View File
@@ -1,7 +1,7 @@
import type { Flow } from "@yaakapp-internal/proxy-lib";
import type { TreeNode } from "@yaakapp-internal/ui";
import { describe, expect, test } from "vite-plus/test";
import { parseSearch } from "./search";
import { parseQuery } from "@yaakapp-internal/query";
import {
buildSidebarTree,
itemsById,
@@ -83,8 +83,9 @@ describe("buildSidebarTree", () => {
const query = (id: string) => items.get(id)?.query;
expect(query("all")).toBe("");
expect(query("app:fieldwork")).toBe("@app:fieldwork");
// A path row takes in what is under the path.
expect(query("app:fieldwork|path:api.github.com/repos")).toBe(
"@app:fieldwork @host:api.github.com @path:/repos",
"@app:fieldwork @host:api.github.com @path:/repos/*",
);
expect(query("site:github.com")).toBe("@host:*.github.com");
expect(query("system")).toBe("@is:system");
@@ -93,12 +94,12 @@ describe("buildSidebarTree", () => {
expect(query("app:trustd|host:ocsp.apple.com")).toBe(
"@is:system @app:trustd @host:ocsp.apple.com",
);
expect(parseSearch(query("app:trustd") ?? "", true)?.system).toBe(true);
// What each writes, read back, covers the flows the row counts.
// What each writes reads back as the terms it wrote.
for (const item of items.values()) {
const parsed = parseSearch(item.query);
if (parsed == null || item.kind === "group") continue;
if (item.kind === "group" || item.kind === "all") continue;
const parsed = parseQuery(item.query);
expect(parsed, item.id).toBeTruthy();
expect(parsed?.type, item.id).not.toBe("Or");
}
});
});
@@ -164,7 +165,7 @@ describe("programs outside an app", () => {
expect(items.get("app:claude|cwd:yaak|host:statsig.anthropic.com")?.query).toBe(
"@app:claude @cwd:yaak @host:statsig.anthropic.com",
);
expect(parseSearch(query)).toEqual({ app: ["claude"], cwd: ["yaak"] });
expect(parseQuery(query)?.type).toBe("And");
});
});
@@ -215,12 +216,10 @@ describe("selection and the search text", () => {
pick("app:Slack|site:slack.com"),
]);
expect(text).toBe("@app:fieldwork @host:api.github.com OR @app:Slack @host:*.slack.com");
expect(parseSearch(text)).toEqual({
any: [
{ app: ["fieldwork"], under: ["api.github.com"] },
{ app: ["Slack"], under: ["*.slack.com"] },
],
});
// Two groups, each judged whole
const parsed = parseQuery(text);
expect(parsed?.type).toBe("Or");
expect(parsed?.type === "Or" ? parsed.children.map((c) => c.type) : []).toEqual(["And", "And"]);
});
test("writes the selected rows' queries, OR'd, or nothing for All traffic", () => {
@@ -240,3 +239,29 @@ describe("selection and the search text", () => {
expect(queryInText("", "card")).toBe(false);
});
});
describe("saved searches", () => {
test("follow All traffic, each a row with its count that writes its query", () => {
const saved = [
{ id: "sq_1", name: "Errors", query: "@status:>=400", count: 12 },
{ id: "sq_2", name: "Stripe", query: "@host:*.stripe.com", count: 0 },
];
const root = buildSidebarTree(flows, 4, saved);
expect((root.children ?? []).slice(0, 2).flatMap((c) => shape(c))).toEqual([
"All traffic 4",
"Saved 0",
" Errors 12",
" Stripe 0",
]);
const items = itemsById(root);
expect(items.get("saved:sq_1")).toMatchObject({
kind: "saved",
query: "@status:>=400",
savedId: "sq_1",
});
// Rows, not folders
const group = (root.children ?? []).find((c) => c.item.id === "saved");
expect(group?.children?.every((c) => c.children == null)).toBe(true);
expect(itemsById(buildSidebarTree(flows, 4)).has("saved")).toBe(false);
});
});
apps/yaak-proxy/lib/sidebarTree.ts
+57 -30
View File
@@ -1,20 +1,13 @@
import type { Flow } from "@yaakapp-internal/proxy-lib";
import type { TreeNode } from "@yaakapp-internal/ui";
import type { DecryptSubject } from "./decrypt";
import {
appKeyOf,
folderOf,
formatFieldFilter,
isSystemFlow,
runtimeFolderOf,
sidebarPath,
siteOf,
tokenize,
} from "./search";
import { fieldTerm, pathTerm, siteTerm, termKeys } from "./query";
import { appKeyOf, folderOf, isSystemFlow, runtimeFolderOf, sidebarPath, siteOf } from "./search";
export type SidebarKind =
| "all"
| "tunnelled"
| "saved"
| "group"
| "app"
| "folder"
@@ -23,8 +16,8 @@ export type SidebarKind =
| "path";
/**
* A row of the sidebar: All traffic, a group, an app, a program's working directory, or a site,
* host or path under one
* A row of the sidebar: All traffic, a saved search, a group, an app, a program's working
* directory, or a site, host or path under one
*/
export interface SidebarItem {
id: string;
@@ -42,10 +35,21 @@ export interface SidebarItem {
query: string;
/** What Decrypt in the row's menu decrypts: its host, site or app */
decrypt?: DecryptSubject;
/** The saved search a Saved row is, for renaming and deleting it */
savedId?: string;
}
/** A saved search as the sidebar lists it, with how many flows it covers */
export interface SavedRow {
id: string;
name: string;
query: string;
count: number;
}
/** The ids of All traffic and the groups */
export const ALL_ID = "all";
export const SAVED_ID = "saved";
export const APPS_ID = "apps";
export const DOMAINS_ID = "domains";
export const SYSTEM_ID = "system";
@@ -83,13 +87,13 @@ function placeOf(flow: Flow): Place {
const [host = "", ...segments] = schemeless ? [path] : path.split("/");
const site = siteOf(host);
const app = appKeyOf(flow);
const hostQuery = formatFieldFilter("host", host);
const hostQuery = fieldTerm("host", host);
const rows: PlaceRow[] = [
{
id: `site:${site}`,
kind: "site",
label: site,
query: formatFieldFilter("host", `*.${site}`),
query: siteTerm(site),
decrypt: { kind: "site", site },
},
{
@@ -107,7 +111,7 @@ function placeOf(flow: Flow): Place {
id: `path:${host}${p}`,
kind: "path",
label: `/${segment}`,
query: `${hostQuery} ${formatFieldFilter("path", p)}`,
query: `${hostQuery} ${pathTerm(p)}`,
});
}
const system = isSystemFlow(flow);
@@ -123,7 +127,7 @@ function placeOf(flow: Flow): Place {
}));
} else {
const folderId = `app:${app}|cwd:${folder}`;
const folderQuery = `${appQuery} ${formatFieldFilter("cwd", folder)}`;
const folderQuery = `${appQuery} ${fieldTerm("cwd", folder)}`;
appRows = [
{ id: folderId, kind: "folder", label: folder, query: folderQuery },
...rows.map((r) => ({
@@ -144,8 +148,8 @@ function placeOf(flow: Flow): Place {
* while System is hidden from All traffic.
*/
function appQueryFor(app: string, system: boolean): string {
const query = formatFieldFilter("app", app);
return system ? `${formatFieldFilter("is", "system")} ${query}` : query;
const query = fieldTerm("app", app);
return system ? `${fieldTerm("is", "system")} ${query}` : query;
}
/** A node being built, its children by id */
@@ -209,10 +213,14 @@ function group(id: string, label: string): Trie {
* The sidebar: All traffic, then Apps, each app opening to its own sites, hosts and paths, then
* Domains by site, host and path, then System, macOS's daemons, apart from the apps. A group with
* nothing in it is left out. Not decrypted, after All traffic, gathers the HTTPS passed through
* undecrypted, when there is any. `total` is All traffic's count: every flow it covers in the
* database.
* undecrypted, when there is any, and Saved lists the saved searches, each with its count from
* the database. `total` is All traffic's count: every flow it covers in the database.
*/
export function buildSidebarTree(flows: Flow[], total: number): TreeNode<SidebarItem> {
export function buildSidebarTree(
flows: Flow[],
total: number,
saved: SavedRow[] = [],
): TreeNode<SidebarItem> {
const root: TreeNode<SidebarItem> = {
item: { id: "root", kind: "group", label: "", count: 0, query: "" },
parent: null,
@@ -223,7 +231,7 @@ export function buildSidebarTree(flows: Flow[], total: number): TreeNode<Sidebar
const apps = group(APPS_ID, "Apps");
const domains = group(DOMAINS_ID, "Domains");
const system = { ...group(SYSTEM_ID, "System") };
system.item.query = formatFieldFilter("is", "system");
system.item.query = fieldTerm("is", "system");
let tunnelled = 0;
for (const flow of flows) {
@@ -273,13 +281,36 @@ export function buildSidebarTree(flows: Flow[], total: number): TreeNode<Sidebar
kind: "tunnelled",
label: "Not decrypted",
count: tunnelled,
query: formatFieldFilter("is", "tunnelled"),
query: fieldTerm("is", "tunnelled"),
},
parent: root,
depth: 1,
draggable: false,
});
}
if (saved.length > 0) {
const node: TreeNode<SidebarItem> = {
item: { id: SAVED_ID, kind: "group", label: "Saved", count: 0, query: "" },
parent: root,
depth: 1,
children: [],
draggable: false,
};
node.children = saved.map((s) => ({
item: {
id: `saved:${s.id}`,
kind: "saved",
label: s.name,
count: s.count,
query: s.query,
savedId: s.id,
},
parent: node,
depth: 2,
draggable: false,
}));
root.children?.push(node);
}
for (const g of [apps, domains, system]) {
if (g.children.size === 0) continue;
const node: TreeNode<SidebarItem> = {
@@ -318,12 +349,8 @@ export function selectionText(items: SidebarItem[]): string {
* stays selected while it is, so typing after selecting keeps it.
*/
export function queryInText(query: string, text: string): boolean {
const have = new Set(tokenize(text).map(tokenKey));
const want = tokenize(query);
const have = new Set(termKeys(text));
const want = termKeys(query);
if (want.length === 0) return have.size === 0;
return want.every((t) => have.has(tokenKey(t)));
}
function tokenKey(t: { field: string | null; value: string; negated: boolean }): string {
return `${t.negated ? "-" : ""}${t.field ?? ""}:${t.value.toLowerCase()}`;
return want.every((t) => have.has(t));
}
apps/yaak-proxy/lib/store.ts
+143 -16
View File
@@ -1,4 +1,9 @@
import type { Flow, GetProxyStateResponse, ModelPayload } from "@yaakapp-internal/proxy-lib";
import type {
Flow,
GetProxyStateResponse,
ModelPayload,
SavedQuery,
} from "@yaakapp-internal/proxy-lib";
import { selectedIdsFamily } from "@yaakapp-internal/ui";
import { atom, type createStore } from "jotai";
import { atomWithStorage } from "jotai/utils";
@@ -20,8 +25,9 @@ import {
WINDOW_SIZE,
windowFromPage,
} from "./flowWindow";
import { activeQuery, anyCountsBackFromNow, countsBackFromNow, type FlowQuery } from "./query";
import { listen, rpc } from "./rpc";
import { type FlowQuery, matchesQuery, parseSearch } from "./search";
import { changesFor, FlowEvents } from "./searchChanges";
import {
ALL_ID,
APPS_ID,
@@ -29,6 +35,8 @@ import {
DOMAINS_ID,
itemsById,
queryInText,
SAVED_ID,
type SavedRow,
selectionText,
} from "./sidebarTree";
@@ -40,6 +48,8 @@ const FLUSH_MS = 100;
const RECOUNT_MS = 1000;
/** How often the database is read for new flows while another process owns the engine */
const POLL_MS = 1000;
/** How often a search counting back from now is read again, as flows age in and out of it */
const AGE_REFRESH_MS = 15_000;
/** How often the engine's owner is checked while it is another process */
const OWNER_POLL_MS = 2000;
@@ -90,9 +100,26 @@ export const hideSystemAtom = atomWithStorage("sidebar_hide_system", false, unde
/** The search the list shows, from the search box's settled text, or null for all traffic */
export const activeQueryAtom = atom<FlowQuery | null>((get) =>
parseSearch(get(searchTermsAtom), get(hideSystemAtom)),
activeQuery(get(searchTermsAtom), get(hideSystemAtom)),
);
/** The saved searches, in their order */
export const savedQueriesAtom = atom<SavedQuery[]>([]);
/** How many flows each saved search covers, by id, counted in the database */
export const savedCountsAtom = atom<Record<string, number>>({});
/** The saved searches as the sidebar lists them */
const savedRowsAtom = atom<SavedRow[]>((get) => {
const counts = get(savedCountsAtom);
return get(savedQueriesAtom).map((s) => ({
id: s.id,
name: s.name,
query: s.query,
count: counts[s.id] ?? 0,
}));
});
/** Writes `text` into the search box at once, as the sidebar does, without waiting for typing */
export const setSearchTextAtom = atom(null, (_get, set, text: string) => {
set(searchTextAtom, text);
@@ -119,7 +146,7 @@ const SIDEBAR_MS = 1000;
/** The sidebar's rows, from the flows loaded */
export const sidebarTreeAtom = atom((get) =>
buildSidebarTree(get(sidebarFlowsAtom), get(allTrafficTotalAtom)),
buildSidebarTree(get(sidebarFlowsAtom), get(allTrafficTotalAtom), get(savedRowsAtom)),
);
const storedColumnPrefsAtom = atomWithStorage<unknown>(
@@ -244,7 +271,7 @@ export const loadOlderFlowsAtom = atom(null, async (get, set) => {
const merge = (win: FlowWindow) => {
if (win.flows[0]?.id !== first.id) return win;
const merged = prependOlder(win, withoutRemoved(res.flows), PAGE_SIZE);
const kept = search == null ? changes : changesFor(search.query, changes);
const kept = search == null ? changes : changesOf(search, merged, changes);
// Counted when they first applied.
return { ...applyChanges(merged, kept), total: merged.total };
};
@@ -261,9 +288,47 @@ export const loadOlderFlowsAtom = atom(null, async (get, set) => {
}
});
/** The changes a search's window takes: flows it matches, and every deletion */
function changesFor(query: FlowQuery, changes: ModelPayload[]): ModelPayload[] {
return changes.filter((c) => c.change.type === "delete" || matchesQuery(c.model, query));
/** `changesFor`, with flows matched against another search sent to be matched again */
function changesOf(
search: Pick<SearchWindow, "key" | "query">,
win: FlowWindow,
changes: ModelPayload[],
): ModelPayload[] {
return changesFor(search.key, win, changes, (unknown) => matchLater(search, unknown));
}
/** The store `syncFlows` keeps, for answers that come after the changes they are for */
let syncedStore: Store | null = null;
/** The order flow events arrived in, for judging answers that come late */
const flowEvents = new FlowEvents();
/**
* Matches flows against the search in the database, then has its window take them as it takes
* events matched as they were sent. A flow a newer event updated or removed meanwhile, in the
* window or not, keeps what that event left.
*/
function matchLater(search: Pick<SearchWindow, "key" | "query">, changes: ModelPayload[]) {
const store = syncedStore;
if (store == null) return;
const ids = [...new Set(changes.map((c) => c.model.id))];
rpc("match_flows", { filter: search.query, ids })
.then((matched) => {
const covered = new Set(matched);
const marked = flowEvents.current(changes).map((c) => ({
...c,
search: { key: search.key, matches: covered.has(c.model.id) },
}));
if (marked.length === 0) return;
const following = store.get(followingAtom);
store.set(searchWindowAtom, (s) => {
if (s?.key !== search.key) return s;
if (s.loading) return { ...s, buffered: [...s.buffered, ...marked] };
const next = applyChanges(s.win, changesOf(s, s.win, marked));
return { ...s, win: following ? trimWindow(next) : next };
});
})
.catch((err) => console.warn("Failed to match flows", err));
}
/**
@@ -274,6 +339,7 @@ function changesFor(query: FlowQuery, changes: ModelPayload[]): ModelPayload[] {
* way.
*/
export function syncFlows(store: Store) {
syncedStore = store;
let pending: ModelPayload[] = [];
let timer: ReturnType<typeof setTimeout> | null = null;
let loaded = false;
@@ -297,7 +363,7 @@ export function syncFlows(store: Store) {
if (s == null) return s;
// A matching flow written after the page's snapshot would otherwise be missed.
if (s.loading) return { ...s, buffered: [...s.buffered, ...changes] };
const next = applyChanges(s.win, changesFor(s.query, changes));
const next = applyChanges(s.win, changesOf(s, s.win, changes));
return { ...s, win: following ? trimWindow(next) : next };
});
const selected = store.get(selectedFlowAtom);
@@ -309,6 +375,7 @@ export function syncFlows(store: Store) {
}
function receive(changes: ModelPayload[]) {
flowEvents.arrived(changes);
pending.push(...changes);
if (timer == null) timer = setTimeout(flush, FLUSH_MS);
}
@@ -326,10 +393,11 @@ export function syncFlows(store: Store) {
);
});
if (store.get(hideSystemAtom)) {
void rpc("list_flows", { system: false, limit: 0 }).then((res) => {
void rpc("list_flows", { hideSystem: true, limit: 0 }).then((res) => {
if (thisLoad === loads) store.set(nonSystemTotalAtom, res.matching ?? res.total);
});
}
countSaved();
const search = store.get(searchWindowAtom);
if (search == null || search.loading) return;
void rpc("list_flows", { ...search.query, limit: 0 }).then((res) => {
@@ -340,21 +408,60 @@ export function syncFlows(store: Store) {
});
}
// Saved searches, and their counts, which change with the traffic.
function loadSaved() {
rpc("list_saved_queries", {})
.then((saved) => {
store.set(savedQueriesAtom, saved);
countSaved();
})
.catch((err) => console.warn("Failed to list saved searches", err));
}
let counting = false;
let countAgain = false;
function countSaved() {
if (store.get(savedQueriesAtom).length === 0) return;
// One count at a time; one asked for meanwhile runs after it, with the latest traffic.
if (counting) {
countAgain = true;
return;
}
counting = true;
rpc("count_saved_queries", { hideSystem: store.get(hideSystemAtom) })
.then((counts) =>
store.set(savedCountsAtom, Object.fromEntries(counts.map((c) => [c.id, c.count]))),
)
.catch((err) => console.warn("Failed to count saved searches", err))
.finally(() => {
counting = false;
if (countAgain) {
countAgain = false;
countSaved();
}
});
}
listen("saved_queries_changed", loadSaved);
loadSaved();
listen("model_write", (payload) => {
seenId = payload.model.id;
receive([payload]);
});
// Loads again rather than only emptying, for a flow written between the clear and the event.
// Saved searches too, which could not be read while the database was locked.
listen("flows_cleared", () => {
flowEvents.reset();
store.set(emptyFlowsAtom);
pending = [];
load();
loadSaved();
});
// The oldest flows, swept to keep the capture under its size limit: dropped wherever they
// are held, and the totals counted again from the database.
listen("flows_removed", ({ ids }) => {
flowEvents.removed(ids);
const removed = new Set(ids);
const drop = (win: FlowWindow) =>
win.flows.some((f) => removed.has(f.id))
@@ -399,21 +506,31 @@ export function syncFlows(store: Store) {
}
reloadFlows = load;
/** Loads the first page of `query`'s flows into a window of its own, or drops it for null */
function loadSearch(query: FlowQuery | null) {
/**
* Loads the first page of `query`'s flows into a window of its own, or drops it for null. The
* backend matches the flows written from now on against it, and their events say so.
*/
function loadSearch(query: FlowQuery | null, keepRows = false) {
if (query == null) {
store.set(searchWindowAtom, null);
void rpc("watch_query", { key: "" }).catch(() => {});
return;
}
// Unique to this load, so a slower answer to an earlier one with the same query, such as
// one read before a clear, is dropped.
const key = `${++searchLoads}:${JSON.stringify(query)}`;
store.set(searchWindowAtom, { key, query, win: emptyWindow, loading: true, buffered: [] });
// Read again, it keeps showing its rows until the new ones are in.
const win = (keepRows ? store.get(searchWindowAtom)?.win : null) ?? emptyWindow;
store.set(searchWindowAtom, { key, query, win, loading: true, buffered: [] });
void rpc("watch_query", { key, query: query.query, hideSystem: query.hideSystem }).catch(
(err) => console.warn("Failed to watch the search", err),
);
const settle = (page: FlowWindow | null) =>
store.set(searchWindowAtom, (s) => {
if (s?.key !== key) return s;
// Applied again over the page: a flow already in it is replaced, not added.
const win = applyChanges(page ?? s.win, changesFor(query, s.buffered));
const base = page ?? s.win;
const win = applyChanges(base, changesOf(s, base, s.buffered));
return {
...s,
win: page == null ? win : { ...win, total: page.total },
@@ -451,11 +568,21 @@ export function syncFlows(store: Store) {
}
});
// All traffic's count leaves System out while it is hidden.
// All traffic's count, and the saved searches', leave System out while it is hidden.
store.sub(hideSystemAtom, () => {
if (loaded) recount();
});
// A search counting back from now, `@time:<5m`, is read again while it is shown and the list
// follows new traffic, so flows leave it and join it as they age. Saved ones are counted
// again, as their counts change with no traffic at all.
setInterval(() => {
if (anyCountsBackFromNow(store.get(savedQueriesAtom).map((q) => q.query))) countSaved();
const s = store.get(searchWindowAtom);
if (s == null || s.loading || !store.get(followingAtom)) return;
if (countsBackFromNow(s.query.query)) loadSearch(s.query, true);
}, AGE_REFRESH_MS);
let activeKey = JSON.stringify(store.get(activeQueryAtom));
store.sub(activeQueryAtom, () => {
const query = store.get(activeQueryAtom);
@@ -576,7 +703,7 @@ export function syncSidebarSelection(store: Store) {
lastRows = key;
if (syncing || Date.now() - lastSidebarInput > SIDEBAR_INPUT_MS) return;
// Group headings are not filters.
const rows = ids.filter((id) => id !== APPS_ID && id !== DOMAINS_ID);
const rows = ids.filter((id) => id !== APPS_ID && id !== DOMAINS_ID && id !== SAVED_ID);
if (rows.length === 0) return;
const items = itemsById(store.get(sidebarTreeAtom));
const selected = rows.flatMap((id) => {
apps/yaak-proxy/package.json
+1
View File
@@ -35,6 +35,7 @@
"babel-plugin-react-compiler": "^1.0.0",
"typescript": "^5.8.3",
"vite": "npm:@voidzero-dev/vite-plus-core@^1.4.0",
"vite-plugin-wasm": "^3.5.0",
"vite-plus": "^1.4.0"
}
}
apps/yaak-proxy/vite.config.ts
+5 -1
View File
@@ -1,9 +1,13 @@
import react from "@vitejs/plugin-react";
import { defineConfig } from "vite";
import wasm from "vite-plugin-wasm";
export default defineConfig({
plugins: [react()],
// The search box's query language is a WebAssembly package
plugins: [wasm(), react()],
build: {
// Top-level await, which the WebAssembly package loads with
target: "esnext",
outDir: "../../dist/apps/yaak-proxy",
emptyOutDir: true,
},
crates-proxy/yaak-proxy-lib/Cargo.toml
+2
View File
@@ -36,6 +36,8 @@ yaak-lifecycle = { workspace = true }
yaak-models = { workspace = true, features = ["sqlcipher"] }
yaak-proxy = { workspace = true }
yaak-proxy-helper = { workspace = true, features = ["ts"] }
# The search query language: parsing, the evaluator for live flows, completion.
yaak-query = { workspace = true }
yaak-rpc = { workspace = true }
yaak-templates = { workspace = true }
crates-proxy/yaak-proxy-lib/bindings/gen_models.ts
Generated
+31 -2
View File
@@ -131,7 +131,12 @@ grpcMessage: string | null,
* client rejected the proxy's certificate: method `CONNECT`, its URL the host, its sizes
* the bytes each way, and no headers or bodies. Open until `elapsedMs` or `error` is set.
*/
tunnelled: boolean, };
tunnelled: boolean,
/**
* An agent read the flow through the control API: its detail, a body or its messages.
* Never written by an upsert, which keeps what is stored.
*/
agentRead: boolean, };
export type FlowBody = "request" | "response";
@@ -239,10 +244,34 @@ grpcStatus?: number,
*/
grpcMessage?: string, };
export type ModelPayload = { model: Flow, change: ModelChangeEvent, };
export type ModelPayload = { model: Flow, change: ModelChangeEvent,
/**
* Whether the flow belongs in the list of the search the app shows, decided as the event
* is sent. None while no search is shown.
*/
search?: SearchMatch, };
export type ProxyHeader = { name: string, value: string, };
export type SavedQuery = { id: string, name: string,
/**
* The search, as the search box takes it.
*/
query: string,
/**
* Where it sits in the list, lowest first.
*/
sortOrder: number, };
/**
* A written flow, matched against the search the app shows.
*/
export type SearchMatch = {
/**
* The search's key, as the app gave it with the search.
*/
key: string, matches: boolean, };
/**
* The streaming protocol a flow carries.
*/
crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts
Generated
+64 -51
View File
@@ -1,5 +1,5 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { Flow, FlowBody, FlowMessage, GrpcSchema, ModelPayload } from "./gen_models";
import type { Flow, FlowBody, FlowMessage, GrpcSchema, ModelPayload, SavedQuery } from "./gen_models";
export type ActionInvocation = { "scope": "global", action: GlobalAction, };
@@ -58,6 +58,10 @@ path: string,
*/
cwd: string | null, };
export type CountSavedQueriesRequest = { hideSystem?: boolean, };
export type CreateSavedQueryRequest = { name: string, query: string, };
export type DecodeGrpcMessageRequest = { flowId: string, seq: number,
/**
* Decode without a schema even when one is found.
@@ -111,6 +115,8 @@ entry: string,
*/
builtIn: boolean, };
export type DeleteSavedQueryRequest = { id: string, };
export type DropReason = "hop_by_hop" | "set_by_sending" | "body_decoded";
export type DroppedHeader = { name: string, value: string, reason: DropReason, };
@@ -146,60 +152,19 @@ export type FilterBodyResponse = { content: string, error: string | null, };
export type FilterLanguage = "jsonpath" | "xpath";
/**
* Which flows a listing or a HAR export covers. Every field narrows it; none is all of them.
* Which flows a listing or a HAR export covers: a search, and a page of it.
*/
export type FlowFilter = {
/**
* Search terms, any case, each of which must match the URL (host, path or query), the
* method, the status (`404`, `40`, `4xx`) or the content type: `stripe post 4xx`.
* A search in the query language (`crate::query`): `stripe @status:4xx`,
* `@host:*.github.com -@is:system`. None or blank covers every flow.
*/
q?: string,
query?: string,
/**
* Part of the host, any case: `stripe` matches `api.stripe.com`.
* Leaves out macOS's own daemons, as hiding System does, unless the query says
* `@is:system` itself.
*/
host?: string,
/**
* Only flows under one of these hosts or paths, as the app's sidebar names them:
* `api.stripe.com` or `api.stripe.com/v1/charges`, with any port after the host. A host
* written `*.github.com` is a site: `github.com` and every host below it.
*/
under?: Array<string>,
/**
* Only flows from one of these apps: by bundle id, by path for a process outside an app
* bundle, by name in any case (`slack`), or, for a process outside an app, by the folder
* it ran in (`fieldwork`).
*/
app?: Array<string>,
/**
* Only flows from a process that ran in one of these working directories: the whole path,
* or its folder's name in any case (`fieldwork`). With `app`, a program in a folder.
*/
cwd?: Array<string>,
/**
* Only flows from macOS's own daemons (true), or none of them (false), as [`is_system`]
* tells them by their executable.
*/
system?: boolean,
/**
* Only HTTPS connections tunnelled undecrypted (true), or none of them (false).
*/
tunnelled?: boolean,
/**
* Only flows one of these groups covers, each judged on its own: several sidebar rows
* selected, `@app:Slack @host:a.com OR @app:Chrome @host:b.com`, are two groups and not every
* pairing of their apps and hosts. A group's `after`, `before` and `limit` do not apply. Not
* carried by a query string.
*/
any?: Array<FlowFilter>,
/**
* The method, any case.
*/
method?: string,
/**
* A status (`404`), a class of them (`4xx`), or `error` for flows that failed, gRPC calls
* that ended with a status other than OK among them.
*/
status?: string,
hideSystem?: boolean,
/**
* At most this many flows, the newest.
*/
@@ -382,6 +347,8 @@ messages: Array<FlowMessage>,
*/
total: number, };
export type ListSavedQueriesRequest = Record<string, never>;
export type ListSeenRequest = Record<string, never>;
export type ListWorkspacesRequest = Record<string, never>;
@@ -392,6 +359,11 @@ export type ListWorkspacesResponse = {
*/
yaakFound: boolean, workspaces: Array<WorkspaceSummary>, };
/**
* Which of some flows a search covers, for flows whose events were matched against another.
*/
export type MatchFlowsRequest = { filter: FlowFilter, ids: Array<string>, };
/**
* A message of a streaming flow was recorded.
*/
@@ -436,6 +408,30 @@ export type ProxyState = "running" | "stopped";
export type ProxyStatePayload = { state: ProxyState, };
/**
* A term the query cannot run, which then matches nothing. `start` and `end` count UTF-16
* code units, as JavaScript does.
*/
export type QueryDiagnostic = { kind: QueryDiagnosticKind, path: string, start: number, end: number, };
/**
* What is wrong with a term, for the search box to mark.
*/
export type QueryDiagnosticKind = "unknownField" | "invalidValue" | "notYet";
export type QuerySchemaRequest = Record<string, never>;
/**
* The search's fields, for the search box's highlighting and completion.
*/
export type QuerySchemaResponse = { schema: import("@yaakapp-internal/query").Schema, };
export type QueryValuesRequest = {
/**
* A field's path: `host`, `header` for header names, `header.accept` for a header's values.
*/
path: string, };
export type ReflectGrpcHostRequest = {
/**
* `host` or `host:port`, over TLS on 443 unless a port is given, or either after
@@ -445,11 +441,13 @@ host: string, };
export type RemoveGrpcSchemaRequest = { id: string, };
export type RenameSavedQueryRequest = { id: string, name: string, };
export type ResetCapturedDataRequest = Record<string, never>;
export type RpcEventSchema = { decrypt_list_changed: DecryptListChangedPayload, flows_cleared: FlowsClearedPayload, flows_removed: FlowsRemovedPayload, message_write: MessagePayload, model_write: ModelPayload, proxy_state_changed: ProxyStatePayload, };
export type RpcEventSchema = { decrypt_list_changed: DecryptListChangedPayload, flows_cleared: FlowsClearedPayload, flows_removed: FlowsRemovedPayload, message_write: MessagePayload, model_write: ModelPayload, proxy_state_changed: ProxyStatePayload, saved_queries_changed: SavedQueriesChangedPayload, };
export type RpcSchema = { execute_action: [ActionInvocation, boolean], format_body: [FormatBodyRequest, string], get_decrypt_list: [GetDecryptListRequest, DecryptList], get_message_payload: [GetMessagePayloadRequest, GetMessagePayloadResponse], list_actions: [ListActionsRequest, ListActionsResponse], list_flows: [FlowFilter, ListFlowsResponse], list_grpc_schemas: [ListGrpcSchemasRequest, Array<GrpcSchema>], list_messages: [ListMessagesRequest, ListMessagesResponse], remove_grpc_schema: [RemoveGrpcSchemaRequest, boolean], set_recording: [SetRecordingRequest, boolean], set_system_proxy_at_launch: [SetSystemProxyAtLaunchRequest, boolean], set_http_version: [SetHttpVersionRequest, HttpVersion], set_capture_body_limit: [SetCaptureBodyLimitRequest, number], set_keep_traffic: [SetKeepTrafficRequest, number | null], add_grpc_schema: [AddGrpcSchemaRequest, GrpcSchema], app_icon: [AppIconRequest, string | null], ca_status: [CaStatusRequest, CaStatus], ca_trust: [CaTrustRequest, CaStatus], clear_flows: [ClearFlowsRequest, number], decode_grpc_message: [DecodeGrpcMessageRequest, DecodedGrpcMessage], export_har: [ExportHarRequest, number], filter_body: [FilterBodyRequest, FilterBodyResponse], get_flow_body: [GetFlowBodyRequest, GetFlowBodyResponse], get_proxy_state: [GetProxyStateRequest, GetProxyStateResponse], helper_register: [HelperRegisterRequest, SystemProxyStatus], helper_unregister: [HelperUnregisterRequest, SystemProxyStatus], edit_decrypt_list: [EditDecryptListRequest, DecryptList], list_seen: [ListSeenRequest, SeenTargets], set_decrypt_list: [SetDecryptListRequest, DecryptList], list_workspaces: [ListWorkspacesRequest, ListWorkspacesResponse], preview_save: [PreviewSaveRequest, SavePreview], reflect_grpc_host: [ReflectGrpcHostRequest, GrpcSchema], save_flow: [SaveFlowToYaakRequest, SavedFlow], save_flow_body: [SaveFlowBodyRequest, SaveFlowBodyResponse], system_proxy_set: [SystemProxySetRequest, SystemProxyStatus], system_proxy_status: [SystemProxyStatusRequest, SystemProxyStatus], take_over: [TakeOverRequest, boolean], set_port: [SetPortRequest, number], get_storage_size: [GetStorageSizeRequest, GetStorageSizeResponse], reset_captured_data: [ResetCapturedDataRequest, boolean], open_terminal: [OpenTerminalRequest, boolean], };
export type RpcSchema = { execute_action: [ActionInvocation, boolean], format_body: [FormatBodyRequest, string], get_decrypt_list: [GetDecryptListRequest, DecryptList], get_message_payload: [GetMessagePayloadRequest, GetMessagePayloadResponse], list_actions: [ListActionsRequest, ListActionsResponse], list_flows: [FlowFilter, ListFlowsResponse], query_schema: [QuerySchemaRequest, QuerySchemaResponse], validate_query: [ValidateQueryRequest, Array<QueryDiagnostic>], watch_query: [WatchQueryRequest, boolean], list_saved_queries: [ListSavedQueriesRequest, Array<SavedQuery>], create_saved_query: [CreateSavedQueryRequest, SavedQuery], rename_saved_query: [RenameSavedQueryRequest, SavedQuery], delete_saved_query: [DeleteSavedQueryRequest, boolean], list_grpc_schemas: [ListGrpcSchemasRequest, Array<GrpcSchema>], list_messages: [ListMessagesRequest, ListMessagesResponse], remove_grpc_schema: [RemoveGrpcSchemaRequest, boolean], set_recording: [SetRecordingRequest, boolean], set_system_proxy_at_launch: [SetSystemProxyAtLaunchRequest, boolean], set_http_version: [SetHttpVersionRequest, HttpVersion], set_capture_body_limit: [SetCaptureBodyLimitRequest, number], set_keep_traffic: [SetKeepTrafficRequest, number | null], add_grpc_schema: [AddGrpcSchemaRequest, GrpcSchema], app_icon: [AppIconRequest, string | null], ca_status: [CaStatusRequest, CaStatus], ca_trust: [CaTrustRequest, CaStatus], clear_flows: [ClearFlowsRequest, number], decode_grpc_message: [DecodeGrpcMessageRequest, DecodedGrpcMessage], export_har: [ExportHarRequest, number], filter_body: [FilterBodyRequest, FilterBodyResponse], get_flow_body: [GetFlowBodyRequest, GetFlowBodyResponse], get_proxy_state: [GetProxyStateRequest, GetProxyStateResponse], helper_register: [HelperRegisterRequest, SystemProxyStatus], helper_unregister: [HelperUnregisterRequest, SystemProxyStatus], edit_decrypt_list: [EditDecryptListRequest, DecryptList], list_seen: [ListSeenRequest, SeenTargets], set_decrypt_list: [SetDecryptListRequest, DecryptList], list_workspaces: [ListWorkspacesRequest, ListWorkspacesResponse], preview_save: [PreviewSaveRequest, SavePreview], reflect_grpc_host: [ReflectGrpcHostRequest, GrpcSchema], save_flow: [SaveFlowToYaakRequest, SavedFlow], save_flow_body: [SaveFlowBodyRequest, SaveFlowBodyResponse], system_proxy_set: [SystemProxySetRequest, SystemProxyStatus], system_proxy_status: [SystemProxyStatusRequest, SystemProxyStatus], take_over: [TakeOverRequest, boolean], set_port: [SetPortRequest, number], get_storage_size: [GetStorageSizeRequest, GetStorageSizeResponse], reset_captured_data: [ResetCapturedDataRequest, boolean], open_terminal: [OpenTerminalRequest, boolean], query_values: [QueryValuesRequest, Array<string>], match_flows: [MatchFlowsRequest, Array<string>], count_saved_queries: [CountSavedQueriesRequest, Array<SavedQueryCount>], };
export type SaveFlowBodyRequest = { flowId: string, which: FlowBody,
/**
@@ -513,6 +511,13 @@ authorization: boolean, };
export type SavedFlow = { requestId: string, name: string, };
export type SavedQueriesChangedPayload = Record<string, never>;
/**
* How many flows a saved query covers.
*/
export type SavedQueryCount = { id: string, count: number, };
export type SeenApp = { name: string, bundleId: string | null, };
/**
@@ -591,4 +596,12 @@ export type SystemProxyStatusRequest = Record<string, never>;
export type TakeOverRequest = Record<string, never>;
export type ValidateQueryRequest = { query: string, };
/**
* The search the app shows, which flows written from now on are matched against, their events
* naming `key`. No query stops it.
*/
export type WatchQueryRequest = { key: string, query?: string, hideSystem?: boolean, };
export type WorkspaceSummary = { id: string, name: string, folders: Array<FolderSummary>, };
crates-proxy/yaak-proxy-lib/migrations/00000012_search.sql
+13
View File
@@ -0,0 +1,13 @@
-- An agent read the flow through the control API: its detail, a body or its messages.
ALTER TABLE flows ADD COLUMN agent_read BOOLEAN NOT NULL DEFAULT FALSE;
-- Searches saved by name, listed in the sidebar by `sort_order`.
CREATE TABLE saved_queries
(
id TEXT NOT NULL PRIMARY KEY,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL,
name TEXT NOT NULL,
query TEXT NOT NULL,
sort_order INTEGER NOT NULL DEFAULT 0
);
crates-proxy/yaak-proxy-lib/src/control.rs
+6 -1
View File
@@ -554,7 +554,9 @@ async fn route(state: &ServerState, req: Request<Incoming>) -> Result<ControlRes
(&Method::GET, ["flows", id]) => {
found(ctx, id)?;
let limit = body_limit(&query)?;
json(&flows::detail(&ctx.db()?, &ctx.blobs()?, id, limit).await?)
let detail = flows::detail(&ctx.db()?, &ctx.blobs()?, id, limit).await?;
ctx.note_agent_read(id);
json(&detail)
}
(&Method::GET, ["flows", id, "request-body"]) => raw_body(ctx, id, FlowBody::Request).await,
(&Method::GET, ["flows", id, "response-body"]) => {
@@ -565,6 +567,7 @@ async fn route(state: &ServerState, req: Request<Incoming>) -> Result<ControlRes
let (after_seq, limit) = message_page(&query)?;
let messages = messages::list(&ctx.db()?, id, after_seq, limit)?;
let total = messages::count(&ctx.db()?, id)?;
ctx.note_agent_read(id);
json(&ListMessagesResponse { messages, total })
}
(&Method::GET, ["flows", id, "messages", seq]) => {
@@ -584,6 +587,7 @@ async fn route(state: &ServerState, req: Request<Incoming>) -> Result<ControlRes
(None, None)
};
let payload = BodyPreview::new(&payload, message.truncated);
ctx.note_agent_read(id);
json(&MessageDetail { message, payload, decoded, decode_error })
}
(&Method::GET, ["grpc-schemas"]) => json(&grpc::list(&ctx.db()?)?),
@@ -687,6 +691,7 @@ async fn raw_body(ctx: &ProxyCtx, id: &str, which: FlowBody) -> Result<ControlRe
let body =
bodies::read_body(&ctx.db()?, &ctx.blobs()?, &flow, which, ctx.capture_body_limit()?)
.await?;
ctx.note_agent_read(id);
let content_type =
find_header(flow.headers(which), "content-type").unwrap_or("application/octet-stream");
let mut resp = response(StatusCode::OK, content_type, body.data);
crates-proxy/yaak-proxy-lib/src/db.rs
+17 -17
View File
@@ -320,17 +320,11 @@ pub fn is_key_unavailable(reason: &str) -> bool {
reason.starts_with(KEY_UNAVAILABLE)
}
/// The functions filters use: `sidebar_path(url)`, the app sidebar's place for a URL, and
/// `under_site(url, site)`, for filtering by its nodes; `folder_name(dir)`, for naming a program
/// by its folder; and `is_system(bundle_id, path)`, for macOS's own daemons.
/// The functions searches use: `folder_name(dir)`, for naming a program by its folder;
/// `is_system(bundle_id, path)`, for macOS's own daemons; and those the query compiler reads
/// flows' values and tests them with ([`crate::query::add_functions`]).
fn add_functions(conn: &mut rusqlite::Connection) -> rusqlite::Result<()> {
let flags = FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC;
conn.create_scalar_function("sidebar_path", 1, flags, |ctx| {
Ok(crate::flows::sidebar_path(&ctx.get::<String>(0)?))
})?;
conn.create_scalar_function("under_site", 2, flags, |ctx| {
Ok(crate::flows::under_site(&ctx.get::<String>(0)?, &ctx.get::<String>(1)?))
})?;
conn.create_scalar_function("folder_name", 1, flags, |ctx| {
let dir = ctx.get::<Option<String>>(0)?;
Ok(crate::flows::folder_name(dir.as_deref()).map(str::to_string))
@@ -339,7 +333,8 @@ fn add_functions(conn: &mut rusqlite::Connection) -> rusqlite::Result<()> {
let bundle_id = ctx.get::<Option<String>>(0)?;
let path = ctx.get::<Option<String>>(1)?;
Ok(crate::flows::is_system(bundle_id.as_deref(), path.as_deref()))
})
})?;
crate::query::add_functions(conn)
}
fn read_only_pool(path: &Path, key: Option<&DatabaseKey>) -> Result<SqlitePool> {
@@ -447,8 +442,7 @@ mod tests {
assert!(!flows[0].cancelled);
assert_eq!((flows[0].stream_kind, flows[0].message_count), (None, 0));
// Failed flows are found without the gRPC status column too.
let failed =
crate::flows::FlowFilter { status: Some("error".into()), ..Default::default() };
let failed = crate::flows::FlowFilter::search("@is:error");
assert_eq!(crate::flows::list(&db, &failed).unwrap().len(), 0);
// Filters by app work without the folder column, and count without the clear mark.
@@ -465,8 +459,7 @@ mod tests {
.unwrap();
drop(conn);
let db = ProxyQueryManager::open_read_only(&path, Some(&test_key())).unwrap();
let filter =
crate::flows::FlowFilter { app: Some(vec!["curl".into()]), ..Default::default() };
let filter = crate::flows::FlowFilter::search("@app:curl");
let (page, total, matching) = crate::flows::page(&db, &filter).unwrap();
assert_eq!((page.len(), total, matching), (1, 1, Some(1)));
assert_eq!(page[0].number, 1);
@@ -513,7 +506,8 @@ mod tests {
"flow_messages",
"flow_sweep_pending",
"flows",
"grpc_schemas"
"grpc_schemas",
"saved_queries"
]
);
@@ -532,6 +526,7 @@ mod tests {
"grpc_method",
"grpc_status",
"grpc_message",
"agent_read",
] {
assert!(flow_columns.contains(&column.to_string()), "{column} in {flow_columns:?}");
}
@@ -599,7 +594,12 @@ mod tests {
.unwrap()
.map(|v| v.unwrap())
.collect();
assert_eq!(&applied[6..], ["00000007", "00000008", "00000009", "00000010", "00000011"]);
assert_eq!(
&applied[6..],
[
"00000007", "00000008", "00000009", "00000010", "00000011", "00000012"
]
);
let mark_columns = columns(&conn, "flow_clear_mark");
assert_eq!(mark_columns, ["id", "max_rowid", "cleaner", "cleaner_at"]);
drop(conn);
@@ -718,7 +718,7 @@ mod tests {
};
db.with_tx(|c| c.upsert(&tunnel, &yaak_database::UpdateSource::Background).map(|_| ()))
.unwrap();
let filter = crate::flows::FlowFilter { tunnelled: Some(true), ..Default::default() };
let filter = crate::flows::FlowFilter::search("@is:tunnelled");
let listed = crate::flows::list(&db, &filter).unwrap();
assert_eq!(listed.iter().map(|f| f.id.as_str()).collect::<Vec<_>>(), ["fl_tunnel"]);
let old = crate::flows::find(&db, "fl_old").unwrap();
crates-proxy/yaak-proxy-lib/src/flows.rs
+175 -423
View File
@@ -1,6 +1,7 @@
use crate::bodies;
use crate::db::{ProxyQueryManager, Stores};
use crate::models::{Flow, FlowBody};
use crate::query::{FlowColumns, FlowQuery};
use base64::Engine;
use base64::engine::general_purpose::STANDARD as BASE64;
use rusqlite::types::Value;
@@ -9,52 +10,19 @@ use ts_rs::TS;
use yaak_database::UpsertModelInfo;
use yaak_models::blob_manager::BlobManager;
/// Which flows a listing or a HAR export covers. Every field narrows it; none is all of them.
/// Which flows a listing or a HAR export covers: a search, and a page of it.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, TS)]
#[serde(default, rename_all = "camelCase")]
#[ts(export, export_to = "gen_rpc.ts")]
pub struct FlowFilter {
/// Search terms, any case, each of which must match the URL (host, path or query), the
/// method, the status (`404`, `40`, `4xx`) or the content type: `stripe post 4xx`.
/// A search in the query language (`crate::query`): `stripe @status:4xx`,
/// `@host:*.github.com -@is:system`. None or blank covers every flow.
#[ts(optional)]
pub q: Option<String>,
/// Part of the host, any case: `stripe` matches `api.stripe.com`.
#[ts(optional)]
pub host: Option<String>,
/// Only flows under one of these hosts or paths, as the app's sidebar names them:
/// `api.stripe.com` or `api.stripe.com/v1/charges`, with any port after the host. A host
/// written `*.github.com` is a site: `github.com` and every host below it.
#[ts(optional)]
pub under: Option<Vec<String>>,
/// Only flows from one of these apps: by bundle id, by path for a process outside an app
/// bundle, by name in any case (`slack`), or, for a process outside an app, by the folder
/// it ran in (`fieldwork`).
#[ts(optional)]
pub app: Option<Vec<String>>,
/// Only flows from a process that ran in one of these working directories: the whole path,
/// or its folder's name in any case (`fieldwork`). With `app`, a program in a folder.
#[ts(optional)]
pub cwd: Option<Vec<String>>,
/// Only flows from macOS's own daemons (true), or none of them (false), as [`is_system`]
/// tells them by their executable.
#[ts(optional)]
pub system: Option<bool>,
/// Only HTTPS connections tunnelled undecrypted (true), or none of them (false).
#[ts(optional)]
pub tunnelled: Option<bool>,
/// Only flows one of these groups covers, each judged on its own: several sidebar rows
/// selected, `@app:Slack @host:a.com OR @app:Chrome @host:b.com`, are two groups and not every
/// pairing of their apps and hosts. A group's `after`, `before` and `limit` do not apply. Not
/// carried by a query string.
#[ts(optional)]
pub any: Option<Vec<FlowFilter>>,
/// The method, any case.
#[ts(optional)]
pub method: Option<String>,
/// A status (`404`), a class of them (`4xx`), or `error` for flows that failed, gRPC calls
/// that ended with a status other than OK among them.
#[ts(optional)]
pub status: Option<String>,
pub query: Option<String>,
/// Leaves out macOS's own daemons, as hiding System does, unless the query says
/// `@is:system` itself.
#[ts(optional, as = "Option<bool>")]
pub hide_system: bool,
/// At most this many flows, the newest.
#[ts(optional, type = "number")]
pub limit: Option<usize>,
@@ -69,30 +37,24 @@ pub struct FlowFilter {
}
impl FlowFilter {
/// The filter from a query string's `q`, `host`, `under`, `app`, `cwd`, `system`, `method`,
/// `status`, `limit`, `after` and `before`. `under`, `app` and `cwd` may repeat.
/// A filter for a search alone.
pub fn search(query: impl Into<String>) -> Self {
Self { query: Some(query.into()), ..Self::default() }
}
/// The filter from a query string's `query`, `hide_system`, `limit`, `after` and `before`.
pub fn from_query(query: &str) -> Result<Self, String> {
let mut filter = Self::default();
for (key, value) in url::form_urlencoded::parse(query.as_bytes()) {
let value = Some(value.into_owned()).filter(|v| !v.is_empty());
match key.as_ref() {
"q" => filter.q = value,
"host" => filter.host = value,
"under" => filter.under.get_or_insert_default().extend(value),
"app" => filter.app.get_or_insert_default().extend(value),
"cwd" => filter.cwd.get_or_insert_default().extend(value),
"system" => {
filter.system = value
.map(|v| v.parse().map_err(|_| format!("Invalid system {v:?}")))
"query" => filter.query = value,
"hide_system" => {
filter.hide_system = value
.map(|v| v.parse().map_err(|_| format!("Invalid hide_system {v:?}")))
.transpose()?
.unwrap_or_default()
}
"tunnelled" => {
filter.tunnelled = value
.map(|v| v.parse().map_err(|_| format!("Invalid tunnelled {v:?}")))
.transpose()?
}
"method" => filter.method = value,
"status" => filter.status = value,
"after" => filter.after = value,
"before" => filter.before = value,
"limit" => {
@@ -109,30 +71,10 @@ impl FlowFilter {
pub fn to_query(&self) -> String {
let mut query = url::form_urlencoded::Serializer::new(String::new());
let limit = self.limit.map(|l| l.to_string());
let system = self.system.map(|s| s.to_string());
let tunnelled = self.tunnelled.map(|t| t.to_string());
let hide_system = self.hide_system.then(|| "true".to_string());
for (key, value) in [
("q", &self.q),
("host", &self.host),
("system", &system),
("tunnelled", &tunnelled),
] {
if let Some(value) = value {
query.append_pair(key, value);
}
}
for (key, values) in [
("under", &self.under),
("app", &self.app),
("cwd", &self.cwd),
] {
for value in values.iter().flatten() {
query.append_pair(key, value);
}
}
for (key, value) in [
("method", &self.method),
("status", &self.status),
("query", &self.query),
("hide_system", &hide_system),
("limit", &limit),
("after", &self.after),
("before", &self.before),
@@ -144,166 +86,21 @@ impl FlowFilter {
query.finish()
}
/// The search, prepared to run now.
pub fn flow_query(&self) -> FlowQuery {
FlowQuery::new(self.query.as_deref().unwrap_or_default(), self.hide_system)
}
/// Narrows more than `after`, `before` and `limit` do, so it covers fewer than all flows.
pub fn narrows(&self) -> bool {
let given = |v: &Option<String>| v.as_deref().is_some_and(|v| !v.trim().is_empty());
given(&self.q)
|| given(&self.host)
|| given(&self.method)
|| given(&self.status)
|| self.under.is_some()
|| self.app.is_some()
|| self.cwd.is_some()
|| self.system.is_some()
|| self.tunnelled.is_some()
|| self.any.as_ref().is_some_and(|groups| groups.iter().any(FlowFilter::narrows))
!self.flow_query().is_empty()
}
/// The SQL condition for every field but `after`, `before` and `limit`, and its parameters,
/// for the database `conn` reads, which may be from before the folder or gRPC columns, when
/// read without migrating.
/// The SQL condition for the search, and its parameters, for the database `conn` reads,
/// which may be from before later columns, when read without migrating.
fn condition_in(&self, conn: &rusqlite::Connection) -> (String, Vec<Value>) {
let columns = Columns {
cwd: has_column(conn, "client_cwd"),
grpc: has_column(conn, "grpc_status"),
tunnelled: has_column(conn, "tunnelled"),
};
self.condition(columns)
self.flow_query().sql(&FlowColumns::of(conn))
}
/// [`Self::condition_in`], told which of the later columns the database has.
fn condition(&self, columns: Columns) -> (String, Vec<Value>) {
let Columns { cwd: has_cwd, grpc, tunnelled } = columns;
let mut sql = String::from("1");
let mut params: Vec<Value> = Vec::new();
for term in self.q.iter().flat_map(|q| q.split_whitespace()) {
let term = term.to_lowercase();
sql.push_str(
" AND (instr(lower(url), ?) > 0 OR lower(method) = ? \
OR instr(lower(content_type), ?) > 0",
);
params.extend([
term.clone().into(),
term.clone().into(),
term.clone().into(),
]);
if let Some(status) = status_condition(&term, &mut params) {
sql.push_str(" OR ");
sql.push_str(&status);
}
sql.push(')');
}
if let Some(host) = self.host.as_deref().filter(|h| !h.is_empty()) {
sql.push_str(&format!(" AND instr(lower({HOST_SQL}), ?) > 0"));
params.push(host.to_lowercase().into());
}
if let Some(under) = &self.under {
// The node itself, or anything below it, by the sidebar's rules.
let mut any = Vec::new();
for node in under {
if let Some(site) = site_of_node(node) {
any.push("under_site(url, ?)");
params.push(site.into());
continue;
}
let node = normalize_node(node);
any.push("(sidebar_path(url) = ? OR substr(sidebar_path(url), 1, ?) = ?)");
let below = format!("{node}/");
params.extend([
node.into(),
(below.chars().count() as i64).into(),
below.into(),
]);
}
let any = if any.is_empty() { "0".to_string() } else { any.join(" OR ") };
sql.push_str(&format!(" AND ({any})"));
}
if let Some(apps) = &self.app {
let marks = vec!["?"; apps.len()].join(", ");
sql.push_str(&format!(
" AND (coalesce(client_bundle_id, client_path) IN ({marks}) \
OR lower(client_name) IN ({marks})"
));
params.extend(apps.iter().map(|a| a.clone().into()));
params.extend(apps.iter().map(|a| a.to_lowercase().into()));
if has_cwd {
sql.push_str(&format!(
" OR (client_bundle_id IS NULL AND lower(folder_name(client_cwd)) IN ({marks}))"
));
params.extend(apps.iter().map(|a| a.to_lowercase().into()));
}
sql.push(')');
}
if let Some(cwds) = &self.cwd {
if has_cwd && !cwds.is_empty() {
let marks = vec!["?"; cwds.len()].join(", ");
sql.push_str(&format!(
" AND (client_cwd IN ({marks}) OR lower(folder_name(client_cwd)) IN ({marks}))"
));
params.extend(cwds.iter().map(|c| c.clone().into()));
params.extend(cwds.iter().map(|c| c.to_lowercase().into()));
} else {
// None, or a database from before the column, which has no folders.
sql.push_str(" AND 0");
}
}
match self.system {
Some(true) => sql.push_str(" AND is_system(client_bundle_id, client_path)"),
Some(false) => sql.push_str(" AND NOT is_system(client_bundle_id, client_path)"),
None => {}
}
match (self.tunnelled, tunnelled) {
(Some(true), true) => sql.push_str(" AND tunnelled"),
(Some(false), true) => sql.push_str(" AND NOT tunnelled"),
// A database from before tunnelled flows has none.
(Some(true), false) => sql.push_str(" AND 0"),
_ => {}
}
// Each group whole, then any of them.
if let Some(groups) = self.any.as_ref().filter(|g| !g.is_empty()) {
let mut any = Vec::new();
for group in groups {
let (condition, group_params) = group.condition(columns);
any.push(format!("({condition})"));
params.extend(group_params);
}
sql.push_str(&format!(" AND ({})", any.join(" OR ")));
}
if let Some(method) = self.method.as_deref().filter(|m| !m.is_empty()) {
sql.push_str(" AND lower(method) = ?");
params.push(method.to_lowercase().into());
}
match self.status.as_deref().map(str::trim) {
None | Some("") => {}
// A gRPC call that ended with a status other than OK failed too.
Some(s) if s.eq_ignore_ascii_case("error") && grpc => {
sql.push_str(" AND (error IS NOT NULL OR coalesce(grpc_status, 0) != 0)")
}
Some(s) if s.eq_ignore_ascii_case("error") => sql.push_str(" AND error IS NOT NULL"),
// A whole status or a class, not the start of one as a search term may be.
Some(s) if s.len() == 3 => match status_condition(&s.to_ascii_lowercase(), &mut params)
{
Some(status) => {
sql.push_str(" AND ");
sql.push_str(&status);
}
None => sql.push_str(" AND 0"),
},
Some(_) => sql.push_str(" AND 0"),
}
(sql, params)
}
}
/// Which of the columns later migrations added a database has, as one opened read-only before
/// they ran may not.
#[derive(Clone, Copy)]
struct Columns {
/// The folder a program ran in.
cwd: bool,
/// gRPC statuses.
grpc: bool,
tunnelled: bool,
}
/// The host and port of `url`, in SQL: what follows `://` up to the first `/`, `?` or `#`.
@@ -341,38 +138,6 @@ pub fn sidebar_path(url: &str) -> String {
key
}
/// A sidebar node as `sidebar_path` writes it, its host lowercased.
fn normalize_node(node: &str) -> String {
match node.split_once('/') {
Some((host, path)) => format!("{}/{path}", host.to_ascii_lowercase()),
None => node.to_ascii_lowercase(),
}
}
/// The site a sidebar node names, `github.com` for `*.github.com`, lowercased. None for a host or
/// a path.
fn site_of_node(node: &str) -> Option<String> {
let site = node.strip_prefix("*.")?;
(!site.is_empty() && !site.contains('/')).then(|| site.to_ascii_lowercase())
}
/// `url`'s host, without its port, is `site` or a host below it: `api.github.com` is under
/// `github.com`.
pub fn under_site(url: &str, site: &str) -> bool {
let path = sidebar_path(url);
let host = path.split('/').next().unwrap_or_default();
let host = strip_port(host);
host == site || host.strip_suffix(site).is_some_and(|rest| rest.ends_with('.'))
}
/// `host` without a `:port` after it, keeping an IPv6 address's brackets.
fn strip_port(host: &str) -> &str {
if host.starts_with('[') {
return host.find(']').map_or(host, |end| &host[..=end]);
}
host.rsplit_once(':').map_or(host, |(h, _)| h)
}
/// The last part of a directory's path, `fieldwork` for `/Users/me/src/fieldwork`. None for the
/// root, where daemons run, or no path.
pub fn folder_name(path: Option<&str>) -> Option<&str> {
@@ -418,26 +183,6 @@ pub fn is_system(bundle_id: Option<&str>, path: Option<&str>) -> bool {
SYSTEM_DIRS.iter().any(|dir| path.starts_with(dir)) || SYSTEM_PROGRAMS.contains(&name)
}
/// The SQL condition for a status term: `404` exactly, `40` as the start of one, or a class
/// such as `4xx`. None when the term is not one.
fn status_condition(term: &str, params: &mut Vec<Value>) -> Option<String> {
if let Some(class) = term.strip_suffix("xx")
&& let Ok(class @ 1..=9) = class.parse::<i64>()
{
params.push(class.into());
return Some("status / 100 = ?".into());
}
if term.is_empty() || term.len() > 3 || !term.chars().all(|c| c.is_ascii_digit()) {
return None;
}
if term.len() == 3 {
params.push(term.parse::<i64>().ok()?.into());
return Some("status = ?".into());
}
params.push(format!("{term}%").into());
Some("CAST(status AS TEXT) LIKE ?".into())
}
/// The flows `filter` covers, newest first: by start, then by id. Reads the table in that order
/// through its index and stops at the limit, so a page of a common match costs the same however
/// many flows there are.
@@ -462,13 +207,6 @@ pub fn page(
})
}
/// The database has the gRPC status column, added by a later migration.
/// The flows table has `column`: a database opened read-only before its migrations have run may
/// not.
fn has_column(conn: &rusqlite::Connection, column: &str) -> bool {
conn.prepare(&format!("SELECT {column} FROM flows LIMIT 0")).is_ok()
}
fn list_in(conn: &rusqlite::Connection, filter: &FlowFilter) -> Result<Vec<Flow>, String> {
let (condition, mut params) = filter.condition_in(conn);
// The rowid, for the flow's number.
@@ -511,6 +249,40 @@ fn count_matching(conn: &rusqlite::Connection, filter: &FlowFilter) -> Result<us
.map_err(|e| e.to_string())
}
/// How many flows each filter's search covers, whatever its limit, read from one snapshot.
pub fn count_each(db: &ProxyQueryManager, filters: &[FlowFilter]) -> Result<Vec<usize>, String> {
db.with_conn(|c| {
let tx = c.conn().resolve().unchecked_transaction().map_err(|e| e.to_string())?;
let counts = filters.iter().map(|f| count_matching(&tx, f)).collect::<Result<_, _>>()?;
tx.commit().map_err(|e| e.to_string())?;
Ok(counts)
})
}
/// The flows of `ids` that `filter`'s search covers, in any order. Cleared ones are left out.
pub fn matching_ids(
db: &ProxyQueryManager,
filter: &FlowFilter,
ids: &[String],
) -> Result<Vec<String>, String> {
if ids.is_empty() {
return Ok(Vec::new());
}
db.with_conn(|c| {
let conn = c.conn().resolve();
let (condition, mut params) = filter.condition_in(conn);
let marks = vec!["?"; ids.len()].join(", ");
let mut sql = format!("SELECT id FROM flows WHERE ({condition}) AND id IN ({marks})");
params.extend(ids.iter().map(|id| Value::from(id.clone())));
push_clear_mark(conn, &mut sql, &mut params);
let mut stmt = conn.prepare(&sql).map_err(|e| e.to_string())?;
let rows = stmt
.query_map(rusqlite::params_from_iter(params), |r| r.get(0))
.map_err(|e| e.to_string())?;
rows.collect::<Result<Vec<String>, _>>().map_err(|e| e.to_string())
})
}
/// Leaves out cleared flows, when a clear is pending; otherwise adds nothing, so SQLite keeps its
/// cheapest plan.
fn push_clear_mark(conn: &rusqlite::Connection, sql: &mut String, params: &mut Vec<Value>) {
@@ -1031,6 +803,13 @@ mod tests {
list(db, &filter).unwrap().into_iter().map(|f| f.url).collect()
}
/// The urls a search covers, sorted.
fn found(db: &ProxyQueryManager, query: &str) -> Vec<String> {
let mut found = urls(db, FlowFilter::search(query));
found.sort();
found
}
#[test]
fn filters() {
let (_dir, db) = db_of(vec![
@@ -1038,18 +817,20 @@ mod tests {
flow("http://example.com/missing", "GET", Some(404), None),
flow("https://down.example.com/", "GET", None, Some("refused")),
]);
let by = |filter: FlowFilter| urls(&db, filter).len();
assert_eq!(by(FlowFilter::default()), 3);
assert_eq!(by(FlowFilter { host: Some("STRIPE".into()), ..Default::default() }), 1);
assert_eq!(by(FlowFilter { host: Some("example.com".into()), ..Default::default() }), 2);
assert_eq!(by(FlowFilter { host: Some("charges".into()), ..Default::default() }), 0);
assert_eq!(by(FlowFilter { host: Some("8443".into()), ..Default::default() }), 1);
assert_eq!(by(FlowFilter { method: Some("get".into()), ..Default::default() }), 2);
assert_eq!(by(FlowFilter { status: Some("404".into()), ..Default::default() }), 1);
assert_eq!(by(FlowFilter { status: Some("2xx".into()), ..Default::default() }), 1);
assert_eq!(by(FlowFilter { status: Some("5XX".into()), ..Default::default() }), 0);
assert_eq!(by(FlowFilter { status: Some("40".into()), ..Default::default() }), 0);
assert_eq!(by(FlowFilter { status: Some("error".into()), ..Default::default() }), 1);
let by = |query: &str| urls(&db, FlowFilter::search(query)).len();
assert_eq!(urls(&db, FlowFilter::default()).len(), 3);
// A host whole, with or without its port; a part of one with wildcards.
assert_eq!(by("@host:API.STRIPE.COM"), 1);
assert_eq!(by("@host:api.stripe.com:8443"), 1);
assert_eq!(by("@host:stripe"), 0);
assert_eq!(by("@host:*stripe*"), 1);
assert_eq!(by("@host:*.example.com"), 2);
assert_eq!(by("@method:get"), 2);
assert_eq!(by("@status:404"), 1);
assert_eq!(by("@status:2xx"), 1);
assert_eq!(by("@status:5XX"), 0);
assert_eq!(by("@status:40"), 0);
assert_eq!(by("@is:error"), 1);
}
#[test]
@@ -1067,12 +848,7 @@ mod tests {
},
flow("http://localhost:3000/health", "HEAD", Some(503), None),
]);
let search = |q: &str| {
let filter = FlowFilter { q: Some(q.into()), ..Default::default() };
let mut found = urls(&db, filter);
found.sort();
found
};
let search = |q: &str| found(&db, q);
let stripe = "https://api.stripe.com/v1/charges?customer=cus_9";
let github = "https://api.github.com/repos/yaak";
let logo = "https://images.ctfassets.net/logo.png";
@@ -1081,28 +857,28 @@ mod tests {
assert_eq!(search("STRIPE"), vec![stripe]);
assert_eq!(search("/repos/"), vec![github]);
assert_eq!(search("cus_9"), vec![stripe]);
// The method, whole.
// The method and the status, any part of them.
assert_eq!(search("head"), vec![health]);
assert_eq!(search("ead"), Vec::<&str>::new());
// A status, its start, or its class.
assert_eq!(search("ead"), vec![health]);
assert_eq!(search("404"), vec![github]);
assert_eq!(search("50"), vec![health]);
assert_eq!(search("2xx"), vec![stripe, logo]);
// The content type.
assert_eq!(search("png"), vec![logo]);
assert_eq!(search("json"), vec![github, stripe]);
// Every term must match, each anywhere.
assert_eq!(search("json get"), vec![github]);
assert_eq!(search(" json 2xx "), vec![stripe]);
assert_eq!(search("json 5xx"), Vec::<&str>::new());
assert_eq!(search(" json @status:2xx "), vec![stripe]);
assert_eq!(search("json @status:5xx"), Vec::<&str>::new());
// `%` and `_` are plain characters, not patterns.
assert_eq!(search("%"), Vec::<&str>::new());
assert_eq!(search("cus_"), vec![stripe]);
assert_eq!(search(""), vec![health, github, stripe, logo]);
}
/// The terms a sidebar row writes cover what is under it: a host, with any port, and a
/// path with what is below it.
#[test]
fn under_matches_a_host_or_a_path_as_the_sidebar_does() {
fn hosts_and_paths_as_the_sidebar_writes_them() {
let (_dir, db) = db_of(vec![
flow("https://api.stripe.com/v1/charges", "GET", Some(200), None),
flow("https://api.stripe.com/v1/charges/ch_1?expand=x", "GET", Some(200), None),
@@ -1110,29 +886,38 @@ mod tests {
flow("https://api.stripe.com:8443/v1/charges", "GET", Some(200), None),
flow("http://api.stripe.com", "GET", Some(200), None),
flow("https://api.stripe.community/", "GET", Some(200), None),
flow("https://example.com/v1//users", "GET", Some(200), None),
flow("http://x.com:80/a", "GET", Some(200), None),
flow("https://x.com:443/a", "GET", Some(200), None),
flow("https://example.com/v1//users", "GET", Some(200), None),
]);
let under = |nodes: &[&str]| {
let nodes = nodes.iter().map(|n| n.to_string()).collect();
urls(&db, FlowFilter { under: Some(nodes), ..Default::default() }).len()
};
// A host on another port is a node of its own, as in the sidebar.
assert_eq!(under(&["api.stripe.com"]), 4);
assert_eq!(under(&["api.stripe.com:8443"]), 1);
assert_eq!(under(&["api.stripe.com/v1/charges"]), 2);
// Empty segments and default ports are left out, as the sidebar leaves them.
assert_eq!(under(&["example.com/v1/users"]), 1);
assert_eq!(under(&["x.com/a"]), 2);
assert_eq!(under(&["api.stripe.com/v1/charges", "example.com"]), 3);
assert_eq!(under(&[]), 0);
// The sidebar lowercases hosts; paths keep their case.
assert_eq!(under(&["API.stripe.com/v1/charges"]), 2);
assert_eq!(under(&["api.stripe.com/V1/charges"]), 0);
let count = |query: &str| found(&db, query).len();
assert_eq!(count("@host:api.stripe.com"), 5);
assert_eq!(count("@host:api.stripe.com:8443"), 1);
assert_eq!(count("@host:api.stripe.com @path:/v1/charges/*"), 3);
assert_eq!(count("@host:x.com @path:/a"), 2);
assert_eq!(count("@host:api.stripe.com @path:/v1/charges/* OR @host:x.com"), 5);
assert_eq!(count("@host:API.stripe.com @path:/V1/charges/*"), 3);
// A path with empty segments is found by the sidebar's row for it.
assert_eq!(count("@host:example.com @path:/v1/users/*"), 1);
}
/// The cases `lib/search.test.ts` checks `sidebarPath` against, so the two agree.
/// A search with `OR` at its top still pages and leaves out cleared flows.
#[test]
fn a_top_level_or_keeps_to_the_page_and_the_clear() {
let (_dir, db) = db_of(vec![
flow("https://a.com/1", "GET", Some(200), None),
flow("https://a.com/2", "GET", Some(404), None),
flow("https://a.com/3", "GET", Some(200), None),
]);
let query = "@status:200 OR @status:404";
let before = FlowFilter { before: Some("fl_001".into()), ..FlowFilter::search(query) };
assert_eq!(urls(&db, before), vec!["https://a.com/1"]);
clear(&db).unwrap();
assert!(urls(&db, FlowFilter::search(query)).is_empty());
assert_eq!(page(&db, &FlowFilter::search(query)).unwrap().2, Some(0));
}
#[test]
fn sidebar_path_rules() {
for (url, path) in [
@@ -1163,18 +948,15 @@ mod tests {
from(None, "/usr/bin/curl"),
flow("https://example.com/", "GET", Some(200), None),
]);
let app = |apps: &[&str]| {
let apps = apps.iter().map(|a| a.to_string()).collect();
urls(&db, FlowFilter { app: Some(apps), ..Default::default() }).len()
};
assert_eq!(app(&["com.google.Chrome"]), 2);
assert_eq!(app(&["/usr/bin/curl"]), 1);
assert_eq!(app(&["/Applications/Google Chrome.app"]), 0);
assert_eq!(app(&["com.google.Chrome", "/usr/bin/curl"]), 3);
let app = |query: &str| found(&db, query).len();
assert_eq!(app("@app:com.google.Chrome"), 2);
assert_eq!(app("@app:/usr/bin/curl"), 1);
assert_eq!(app("@app:\"/Applications/Google Chrome.app\""), 0);
assert_eq!(app("@app:com.google.Chrome OR @app:/usr/bin/curl"), 3);
}
/// A program in a folder: its working directory by the whole path, or its folder's name in
/// any case, along with whatever else the filter asks.
/// A program in a folder: its working directory by the whole path, or its folder's name,
/// any case, along with whatever else the search asks.
#[test]
fn cwd_matches_a_path_or_a_folder_name() {
let node_in = |cwd: Option<&str>| Flow {
@@ -1191,22 +973,14 @@ mod tests {
Flow { client_cwd: Some("/tmp/fieldwork".into()), ..node_in(None) },
Flow { client_path: Some("/usr/bin/curl".into()), ..node_in(Some("/tmp/ledger")) },
]);
let count = |app: Option<&str>, cwds: &[&str]| {
let cwd = Some(cwds.iter().map(|c| c.to_string()).collect());
let app = app.map(|a| vec![a.to_string()]);
urls(&db, FlowFilter { app, cwd, ..Default::default() }).len()
};
assert_eq!(count(None, &["/Users/ada/Workspace/fieldwork"]), 2, "the whole path");
assert_eq!(count(None, &["/users/ada/workspace/fieldwork"]), 0, "a path exactly");
assert_eq!(count(None, &["FieldWork"]), 3, "a folder name, any case");
assert_eq!(count(None, &["fieldwork", "ledger"]), 5, "any of them");
assert_eq!(count(Some("/usr/local/bin/node"), &["ledger"]), 1, "and the program");
assert_eq!(count(Some("/usr/bin/curl"), &["ledger"]), 1);
assert_eq!(count(None, &[]), 0);
let query = FlowFilter { cwd: Some(vec!["a".into(), "/b".into()]), ..Default::default() };
assert_eq!(query.to_query(), "cwd=a&cwd=%2Fb");
assert_eq!(FlowFilter::from_query(&query.to_query()).unwrap(), query);
assert!(query.narrows());
let count = |query: &str| found(&db, query).len();
assert_eq!(count("@cwd:/Users/ada/Workspace/fieldwork"), 2, "the whole path");
assert_eq!(count("@cwd:/users/ada/workspace/fieldwork"), 2, "any case");
assert_eq!(count("@cwd:FieldWork"), 3, "a folder name");
assert_eq!(count("@cwd:fieldwork OR @cwd:ledger"), 5, "any of them");
assert_eq!(count("@app:/usr/local/bin/node @cwd:ledger"), 1, "and the program");
assert_eq!(count("@app:/usr/bin/curl @cwd:ledger"), 1);
assert_eq!(count("@cwd:work"), 0, "whole");
}
#[test]
@@ -1221,7 +995,7 @@ mod tests {
.collect(),
);
let search = |before: Option<String>| FlowFilter {
q: Some("post".into()),
query: Some("post".into()),
before,
limit: Some(4),
..Default::default()
@@ -1245,8 +1019,7 @@ mod tests {
let (_dir, db) = db_of(vec![]);
let plan: Vec<String> = db.with_conn(|c| {
let conn = c.conn().resolve();
let (condition, params) =
FlowFilter { q: Some("stripe".into()), ..Default::default() }.condition_in(conn);
let (condition, params) = FlowFilter::search("stripe @status:>=400").condition_in(conn);
let sql = format!(
"EXPLAIN QUERY PLAN SELECT * FROM flows WHERE {condition} \
ORDER BY started_at DESC, id DESC LIMIT 500"
@@ -1322,7 +1095,7 @@ mod tests {
// The limit counts flows the filter keeps.
let ok = FlowFilter {
before: Some("fl_e".into()),
status: Some("2xx".into()),
query: Some("@status:2xx".into()),
limit: Some(2),
..Default::default()
};
@@ -1380,7 +1153,7 @@ mod tests {
write_flow("fl_d");
assert_eq!(ids(&FlowFilter::default()), vec!["fl_d"]);
assert_eq!(find(&db, "fl_d").map(|f| f.number), Some(4));
let narrowed = FlowFilter { q: Some("example".into()), ..Default::default() };
let narrowed = FlowFilter::search("example");
assert_eq!(page(&db, &narrowed).unwrap().1, 1);
// Bodies that fail to delete keep their flows, to delete again.
@@ -1540,49 +1313,39 @@ mod tests {
ids.sort();
ids
};
let app = |a: &str| FlowFilter { app: Some(vec![a.into()]), ..Default::default() };
assert_eq!(ids(app("slack")), vec!["fl_slack"]);
assert_eq!(ids(app("com.tinyspeck.slackmacgap")), vec!["fl_slack"]);
assert_eq!(ids(app("Fieldwork")), vec!["fl_node"]);
assert_eq!(ids(app("/opt/homebrew/bin/node")), vec!["fl_node"]);
let under = |n: &str| FlowFilter { under: Some(vec![n.into()]), ..Default::default() };
assert_eq!(ids(under("*.github.com")), vec!["fl_daemon", "fl_node"]);
assert_eq!(ids(under("api.github.com")), vec!["fl_node"]);
let system = |s| FlowFilter { system: Some(s), ..Default::default() };
assert_eq!(ids(system(true)), vec!["fl_daemon"]);
assert_eq!(ids(system(false)), vec!["fl_node", "fl_none", "fl_slack"]);
let ids = |query: &str| ids(FlowFilter::search(query));
assert_eq!(ids("@app:slack"), vec!["fl_slack"]);
assert_eq!(ids("@app:com.tinyspeck.slackmacgap"), vec!["fl_slack"]);
assert_eq!(ids("@app:Fieldwork"), vec!["fl_node"]);
assert_eq!(ids("@app:/opt/homebrew/bin/node"), vec!["fl_node"]);
assert_eq!(ids("@host:*.github.com"), vec!["fl_daemon", "fl_node"]);
assert_eq!(ids("@host:api.github.com"), vec!["fl_node"]);
assert_eq!(ids("@is:system"), vec!["fl_daemon"]);
assert_eq!(ids("-@is:system"), vec!["fl_node", "fl_none", "fl_slack"]);
// Groups, as two sidebar rows under different apps write them, each judged whole:
// flattened, the apps and hosts would pair across rows and match both flows.
let group = |a: &str, n: &str| FlowFilter {
app: Some(vec![a.into()]),
under: Some(vec![n.into()]),
..Default::default()
};
let crossed = FlowFilter {
any: Some(vec![
group("fieldwork", "github.com"),
group("nsurlsessiond", "api.github.com"),
]),
..Default::default()
};
assert_eq!(ids(crossed.clone()), Vec::<String>::new());
assert_eq!(page(&db, &crossed).unwrap().2, Some(0));
let flattened = FlowFilter {
app: Some(vec!["fieldwork".into(), "nsurlsessiond".into()]),
under: Some(vec!["github.com".into(), "api.github.com".into()]),
..Default::default()
};
let crossed = "@app:fieldwork @host:github.com OR @app:nsurlsessiond @host:api.github.com";
assert_eq!(ids(crossed), Vec::<String>::new());
assert_eq!(page(&db, &FlowFilter::search(crossed)).unwrap().2, Some(0));
let flattened = "(@app:fieldwork OR @app:nsurlsessiond) \
(@host:github.com OR @host:api.github.com)";
assert_eq!(ids(flattened), vec!["fl_daemon", "fl_node"]);
let either = FlowFilter {
any: Some(vec![
group("fieldwork", "api.github.com"),
FlowFilter { app: Some(vec!["slack".into()]), ..Default::default() },
]),
q: Some("https".into()),
..Default::default()
};
let either = "(@app:fieldwork @host:api.github.com OR @app:slack) https";
assert_eq!(ids(either), vec!["fl_node", "fl_slack"]);
// Hiding System leaves out its daemons, unless the search names them.
let hidden = |query: &str| {
let filter = FlowFilter { hide_system: true, ..FlowFilter::search(query) };
let mut ids: Vec<String> =
list(&db, &filter).unwrap().into_iter().map(|f| f.id).collect();
ids.sort();
ids
};
assert_eq!(hidden(""), vec!["fl_node", "fl_none", "fl_slack"]);
assert_eq!(hidden("@host:*.github.com"), vec!["fl_node"]);
assert_eq!(hidden("@is:system"), vec!["fl_daemon"]);
assert_eq!(hidden("@host:*.github.com OR @is:SYSTEM"), vec!["fl_daemon", "fl_node"]);
}
#[test]
@@ -1603,35 +1366,24 @@ mod tests {
assert!(!is_system(None, None));
assert_eq!(folder_name(Some("/Users/me/src/fieldwork/")), Some("fieldwork"));
assert_eq!(folder_name(Some("/")), None);
assert!(under_site("https://api.github.com:8443/x", "github.com"));
assert!(!under_site("https://notgithub.com/", "github.com"));
}
#[test]
fn query_round_trip() {
let filter = FlowFilter {
q: Some("post json".into()),
host: Some("api.stripe.com".into()),
under: Some(vec!["api.stripe.com/v1".into(), "example.com".into()]),
app: Some(vec!["com.google.Chrome".into()]),
status: Some("4xx".into()),
query: Some("post @status:4xx".into()),
hide_system: true,
limit: Some(20),
..Default::default()
};
assert_eq!(
filter.to_query(),
"q=post+json&host=api.stripe.com&under=api.stripe.com%2Fv1&under=example.com\
&app=com.google.Chrome&status=4xx&limit=20"
);
assert_eq!(filter.to_query(), "query=post+%40status%3A4xx&hide_system=true&limit=20");
assert_eq!(FlowFilter::from_query(&filter.to_query()).unwrap(), filter);
assert_eq!(FlowFilter::from_query("").unwrap(), FlowFilter::default());
assert!(FlowFilter::from_query("limit=lots").is_err());
let tunnelled = FlowFilter { tunnelled: Some(true), ..Default::default() };
assert_eq!(tunnelled.to_query(), "tunnelled=true");
assert_eq!(FlowFilter::from_query("tunnelled=true").unwrap(), tunnelled);
assert!(tunnelled.narrows());
assert!(FlowFilter::from_query("tunnelled=maybe").is_err());
assert!(FlowFilter::from_query("hide_system=maybe").is_err());
assert!(filter.narrows());
assert!(FlowFilter { hide_system: true, ..Default::default() }.narrows());
assert!(!FlowFilter::search(" ").narrows());
}
#[test]
crates-proxy/yaak-proxy-lib/src/lib.rs
+200 -9
View File
@@ -14,7 +14,9 @@ pub mod helper;
pub mod macos;
pub mod messages;
pub mod models;
pub mod query;
pub mod save;
pub mod saved_queries;
pub mod settings;
pub mod terminal;
@@ -47,7 +49,7 @@ use std::time::{Duration, SystemTime};
use ts_rs::TS;
pub use yaak_crypto::MasterKeySource;
use yaak_crypto::data_key::DataKey;
use yaak_database::{ModelChangeEvent, UpdateSource, UpsertModelInfo};
use yaak_database::{UpdateSource, UpsertModelInfo};
pub use yaak_grpc::GrpcConfig;
use yaak_models::blob_manager::{BlobManager, BodyChunkWriter};
use yaak_proxy::{CaStore, KeyCipher, ProxyConfig, ProxyEvent, ProxyHandle};
@@ -107,6 +109,8 @@ pub struct ProxyCtx {
pub events: RpcEventEmitter,
/// gRPC schemas as decoded, and the protoc that compiles attached protos.
pub grpc: GrpcState,
/// The search the app shows, which written flows' events are matched against.
live_search: Arc<query::LiveSearch>,
}
impl ProxyCtx {
@@ -148,6 +152,7 @@ impl ProxyCtx {
sweeper: Default::default(),
events,
grpc: GrpcState::default(),
live_search: Default::default(),
};
if db::needs_encrypting(data_dir) {
let unlocking = ctx.clone();
@@ -418,6 +423,7 @@ impl ProxyCtx {
db: self.db()?,
blobs: self.blobs()?,
events: self.events.clone(),
live_search: self.live_search.clone(),
attribution,
recording: self.recording.clone(),
finished: Default::default(),
@@ -1118,16 +1124,38 @@ impl ProxyCtx {
}
let (flow, created) = tx.upsert(&flow, &UpdateSource::Background)?;
// Sent before the commit, so it cannot reach the app after a clear that follows.
self.events.emit(
"model_write",
&ModelPayload { model: flow, change: ModelChangeEvent::Upsert { created } },
);
self.events.emit("model_write", &self.live_search.payload(flow, created));
Ok::<_, yaak_database::Error>(())
})
.map_err(|e| format!("Failed to record the saved request on the flow: {e}"))?;
Ok(saved)
}
/// Records that an agent read the flow through the control API, for `@is:read`, and tells
/// the app. Only the first read writes anything.
pub fn note_agent_read(&self, flow_id: &str) {
let noted = self.db().map_err(String::from).and_then(|db| {
db.with_tx(|tx| {
let conn = tx.conn().resolve();
let changed = conn.execute(
"UPDATE flows SET agent_read = TRUE WHERE id = ? AND NOT agent_read",
[flow_id],
)?;
// Sent before the commit, as every flow's event is.
if changed > 0
&& let Some(flow) = flows::find_in(conn, flow_id)
{
self.events.emit("model_write", &self.live_search.payload(flow, false));
}
Ok::<_, yaak_database::Error>(())
})
.map_err(|e| e.to_string())
});
if let Err(e) = noted {
warn!("Failed to note that an agent read flow {flow_id}: {e}");
}
}
/// Clears every flow at once and tells the app so its list empties, then deletes them and
/// their bodies in the background. Returns how many there were.
pub fn clear_flows(&self) -> Result<usize, String> {
@@ -1385,6 +1413,91 @@ pub struct ListFlowsResponse {
pub matching: Option<usize>,
}
#[derive(Deserialize, TS)]
#[ts(export, export_to = "gen_rpc.ts")]
pub struct QuerySchemaRequest {}
/// The search's fields, for the search box's highlighting and completion.
#[derive(Serialize, TS)]
#[ts(export, export_to = "gen_rpc.ts")]
pub struct QuerySchemaResponse {
#[ts(type = "import(\"@yaakapp-internal/query\").Schema")]
pub schema: serde_json::Value,
}
#[derive(Deserialize, TS)]
#[ts(export, export_to = "gen_rpc.ts")]
pub struct ValidateQueryRequest {
pub query: String,
}
#[derive(Deserialize, TS)]
#[ts(export, export_to = "gen_rpc.ts")]
pub struct QueryValuesRequest {
/// A field's path: `host`, `header` for header names, `header.accept` for a header's values.
pub path: String,
}
/// The search the app shows, which flows written from now on are matched against, their events
/// naming `key`. No query stops it.
#[derive(Deserialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export, export_to = "gen_rpc.ts")]
pub struct WatchQueryRequest {
pub key: String,
#[ts(optional)]
pub query: Option<String>,
#[serde(default)]
#[ts(optional, as = "Option<bool>")]
pub hide_system: bool,
}
/// Which of some flows a search covers, for flows whose events were matched against another.
#[derive(Deserialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export, export_to = "gen_rpc.ts")]
pub struct MatchFlowsRequest {
pub filter: flows::FlowFilter,
pub ids: Vec<String>,
}
#[derive(Deserialize, TS)]
#[ts(export, export_to = "gen_rpc.ts")]
pub struct ListSavedQueriesRequest {}
#[derive(Deserialize, TS)]
#[ts(export, export_to = "gen_rpc.ts")]
pub struct CreateSavedQueryRequest {
pub name: String,
pub query: String,
}
#[derive(Deserialize, TS)]
#[ts(export, export_to = "gen_rpc.ts")]
pub struct RenameSavedQueryRequest {
pub id: String,
pub name: String,
}
#[derive(Deserialize, TS)]
#[ts(export, export_to = "gen_rpc.ts")]
pub struct DeleteSavedQueryRequest {
pub id: String,
}
#[derive(Deserialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export, export_to = "gen_rpc.ts")]
pub struct CountSavedQueriesRequest {
#[serde(default)]
#[ts(optional, as = "Option<bool>")]
pub hide_system: bool,
}
#[derive(Serialize, TS)]
#[ts(export, export_to = "gen_rpc.ts")]
pub struct SavedQueriesChangedPayload {}
#[derive(Deserialize, TS)]
#[ts(export, export_to = "gen_rpc.ts")]
pub struct ClearFlowsRequest {}
@@ -2326,6 +2439,73 @@ async fn clear_flows(ctx: ProxyCtx, _req: ClearFlowsRequest) -> Result<usize, Rp
blocking(move || ctx.clear_flows()).await
}
fn query_schema(
_ctx: &ProxyCtx,
_req: QuerySchemaRequest,
) -> Result<QuerySchemaResponse, RpcError> {
let schema = serde_json::to_value(query::schema()).map_err(rpc_error)?;
Ok(QuerySchemaResponse { schema })
}
fn validate_query(
_ctx: &ProxyCtx,
req: ValidateQueryRequest,
) -> Result<Vec<query::QueryDiagnostic>, RpcError> {
Ok(query::diagnostics(&req.query))
}
async fn query_values(ctx: ProxyCtx, req: QueryValuesRequest) -> Result<Vec<String>, RpcError> {
blocking(move || query::seen_values(&ctx.db()?, &req.path)).await
}
fn watch_query(ctx: &ProxyCtx, req: WatchQueryRequest) -> Result<bool, RpcError> {
let watched = req.query.filter(|q| !q.trim().is_empty() || req.hide_system);
ctx.live_search.watch(watched.map(|q| (req.key, q, req.hide_system)));
Ok(true)
}
async fn match_flows(ctx: ProxyCtx, req: MatchFlowsRequest) -> Result<Vec<String>, RpcError> {
blocking(move || flows::matching_ids(&ctx.db()?, &req.filter, &req.ids)).await
}
fn list_saved_queries(
ctx: &ProxyCtx,
_req: ListSavedQueriesRequest,
) -> Result<Vec<saved_queries::SavedQuery>, RpcError> {
saved_queries::list(&ctx.db()?).map_err(rpc_error)
}
fn create_saved_query(
ctx: &ProxyCtx,
req: CreateSavedQueryRequest,
) -> Result<saved_queries::SavedQuery, RpcError> {
let saved = saved_queries::create(&ctx.db()?, &req.name, &req.query).map_err(rpc_error)?;
ctx.events.emit("saved_queries_changed", &SavedQueriesChangedPayload {});
Ok(saved)
}
fn rename_saved_query(
ctx: &ProxyCtx,
req: RenameSavedQueryRequest,
) -> Result<saved_queries::SavedQuery, RpcError> {
let saved = saved_queries::rename(&ctx.db()?, &req.id, &req.name).map_err(rpc_error)?;
ctx.events.emit("saved_queries_changed", &SavedQueriesChangedPayload {});
Ok(saved)
}
fn delete_saved_query(ctx: &ProxyCtx, req: DeleteSavedQueryRequest) -> Result<bool, RpcError> {
let deleted = saved_queries::delete(&ctx.db()?, &req.id).map_err(rpc_error)?;
ctx.events.emit("saved_queries_changed", &SavedQueriesChangedPayload {});
Ok(deleted)
}
async fn count_saved_queries(
ctx: ProxyCtx,
req: CountSavedQueriesRequest,
) -> Result<Vec<saved_queries::SavedQueryCount>, RpcError> {
blocking(move || saved_queries::counts(&ctx.db()?, req.hide_system)).await
}
// -- Event loop --
/// Why the engine cannot be taken over from another app.
@@ -2429,6 +2609,8 @@ struct FlowSink {
db: ProxyQueryManager,
blobs: BlobManager,
events: RpcEventEmitter,
/// The search the app shows, which each flow written is matched against.
live_search: Arc<query::LiveSearch>,
attribution: Attribution,
recording: Arc<Mutex<RecordingLog>>,
/// Flows that have ended, waiting to be written together.
@@ -3076,10 +3258,7 @@ impl FlowSink {
let (saved, created) = ctx.upsert(&flow, &UpdateSource::Background)?;
// Sent while this holds the writer, before the commit, so a clear's event, sent after
// its own commit, always comes after it.
self.events.emit(
"model_write",
&ModelPayload { model: saved, change: ModelChangeEvent::Upsert { created } },
);
self.events.emit("model_write", &self.live_search.payload(saved, created));
Ok(true)
}
@@ -3142,6 +3321,13 @@ define_rpc! {
get_message_payload(GetMessagePayloadRequest) -> GetMessagePayloadResponse,
list_actions(ListActionsRequest) -> ListActionsResponse,
list_flows(flows::FlowFilter) -> ListFlowsResponse,
query_schema(QuerySchemaRequest) -> QuerySchemaResponse,
validate_query(ValidateQueryRequest) -> Vec<query::QueryDiagnostic>,
watch_query(WatchQueryRequest) -> bool,
list_saved_queries(ListSavedQueriesRequest) -> Vec<saved_queries::SavedQuery>,
create_saved_query(CreateSavedQueryRequest) -> saved_queries::SavedQuery,
rename_saved_query(RenameSavedQueryRequest) -> saved_queries::SavedQuery,
delete_saved_query(DeleteSavedQueryRequest) -> bool,
list_grpc_schemas(ListGrpcSchemasRequest) -> Vec<GrpcSchema>,
list_messages(ListMessagesRequest) -> ListMessagesResponse,
remove_grpc_schema(RemoveGrpcSchemaRequest) -> bool,
@@ -3179,6 +3365,9 @@ define_rpc! {
get_storage_size(GetStorageSizeRequest) -> GetStorageSizeResponse,
reset_captured_data(ResetCapturedDataRequest) -> bool,
open_terminal(OpenTerminalRequest) -> bool,
query_values(QueryValuesRequest) -> Vec<String>,
match_flows(MatchFlowsRequest) -> Vec<String>,
count_saved_queries(CountSavedQueriesRequest) -> Vec<saved_queries::SavedQueryCount>,
}
events {
decrypt_list_changed(DecryptListChangedPayload),
@@ -3187,6 +3376,7 @@ define_rpc! {
message_write(MessagePayload),
model_write(ModelPayload),
proxy_state_changed(ProxyStatePayload),
saved_queries_changed(SavedQueriesChangedPayload),
}
}
@@ -3256,6 +3446,7 @@ mod tests {
db: ctx.db().unwrap(),
blobs: ctx.blobs().unwrap(),
events: ctx.events.clone(),
live_search: ctx.live_search.clone(),
attribution: Attribution::new(0),
recording: ctx.recording.clone(),
finished: Default::default(),
crates-proxy/yaak-proxy-lib/src/models.rs
+19
View File
@@ -107,6 +107,9 @@ pub struct Flow {
/// client rejected the proxy's certificate: method `CONNECT`, its URL the host, its sizes
/// the bytes each way, and no headers or bodies. Open until `elapsedMs` or `error` is set.
pub tunnelled: bool,
/// An agent read the flow through the control API: its detail, a body or its messages.
/// Never written by an upsert, which keeps what is stored.
pub agent_read: bool,
}
/// The streaming protocol a flow carries.
@@ -194,6 +197,21 @@ pub fn find_header<'a>(headers: &'a [ProxyHeader], name: &str) -> Option<&'a str
pub struct ModelPayload {
pub model: Flow,
pub change: ModelChangeEvent,
/// Whether the flow belongs in the list of the search the app shows, decided as the event
/// is sent. None while no search is shown.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[ts(optional)]
pub search: Option<SearchMatch>,
}
/// A written flow, matched against the search the app shows.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export, export_to = "gen_models.ts")]
pub struct SearchMatch {
/// The search's key, as the app gave it with the search.
pub key: String,
pub matches: bool,
}
impl UpsertModelInfo for Flow {
@@ -357,6 +375,7 @@ impl UpsertModelInfo for Flow {
grpc_status: added_column(r, "grpc_status")?,
grpc_message: added_column(r, "grpc_message")?,
tunnelled: added_column(r, "tunnelled")?.unwrap_or_default(),
agent_read: added_column(r, "agent_read")?.unwrap_or_default(),
})
}
}
crates-proxy/yaak-proxy-lib/src/query.rs
+1242
View File
File diff suppressed because it is too large. Load diff
crates-proxy/yaak-proxy-lib/src/query/tests.rs
+1001
View File
File diff suppressed because it is too large. Load diff
crates-proxy/yaak-proxy-lib/src/saved_queries.rs
+181
View File
@@ -0,0 +1,181 @@
//! Searches saved by name, which the sidebar lists with a count each.
use crate::db::ProxyQueryManager;
use crate::flows::{self, FlowFilter};
use serde::{Deserialize, Serialize};
use ts_rs::TS;
use yaak_database::generate_prefixed_id;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export, export_to = "gen_models.ts")]
pub struct SavedQuery {
pub id: String,
pub name: String,
/// The search, as the search box takes it.
pub query: String,
/// Where it sits in the list, lowest first.
#[ts(type = "number")]
pub sort_order: i64,
}
/// How many flows a saved query covers.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export, export_to = "gen_rpc.ts")]
pub struct SavedQueryCount {
pub id: String,
#[ts(type = "number")]
pub count: usize,
}
fn failed(what: &str, e: impl std::fmt::Display) -> String {
format!("Failed to {what} the saved search: {e}")
}
fn read(row: &rusqlite::Row) -> rusqlite::Result<SavedQuery> {
Ok(SavedQuery {
id: row.get("id")?,
name: row.get("name")?,
query: row.get("query")?,
sort_order: row.get("sort_order")?,
})
}
/// Every saved query, in order. None for a database from before them, read without migrating.
pub fn list(db: &ProxyQueryManager) -> Result<Vec<SavedQuery>, String> {
db.with_conn(|c| {
let conn = c.conn().resolve();
let Ok(mut stmt) =
conn.prepare("SELECT * FROM saved_queries ORDER BY sort_order, created_at, id")
else {
return Ok(Vec::new());
};
stmt.query_map([], read)
.and_then(|rows| rows.collect::<rusqlite::Result<Vec<_>>>())
.map_err(|e| failed("list", e))
})
}
fn name_of(name: &str) -> Result<String, String> {
let name = name.trim();
if name.is_empty() {
return Err("A saved search needs a name".into());
}
Ok(name.to_string())
}
/// Saves `query` as `name`, after the others.
pub fn create(db: &ProxyQueryManager, name: &str, query: &str) -> Result<SavedQuery, String> {
let name = name_of(name)?;
let query = query.trim();
if query.is_empty() {
return Err("Nothing to save".into());
}
let id = generate_prefixed_id("sq");
db.with_tx(|c| {
let conn = c.conn().resolve();
conn.query_row(
"INSERT INTO saved_queries (id, name, query, sort_order) \
VALUES (?1, ?2, ?3, (SELECT coalesce(max(sort_order), -1) + 1 FROM saved_queries)) \
RETURNING *",
rusqlite::params![id, name, query],
read,
)
.map_err(yaak_database::Error::from)
})
.map_err(|e| failed("save", e))
}
pub fn rename(db: &ProxyQueryManager, id: &str, name: &str) -> Result<SavedQuery, String> {
let name = name_of(name)?;
db.with_tx(|c| {
c.conn()
.resolve()
.query_row(
"UPDATE saved_queries SET name = ?2, updated_at = CURRENT_TIMESTAMP WHERE id = ?1 \
RETURNING *",
rusqlite::params![id, name],
read,
)
.map_err(yaak_database::Error::from)
})
.map_err(|e| failed("rename", e))
}
/// Deletes one. False when there was none.
pub fn delete(db: &ProxyQueryManager, id: &str) -> Result<bool, String> {
db.with_tx(|c| {
c.conn()
.resolve()
.execute("DELETE FROM saved_queries WHERE id = ?", [id])
.map(|n| n > 0)
.map_err(yaak_database::Error::from)
})
.map_err(|e| failed("delete", e))
}
/// Each saved query's count of flows, with System's left out when `hide_system`, as the list
/// would show it, read from one snapshot.
pub fn counts(db: &ProxyQueryManager, hide_system: bool) -> Result<Vec<SavedQueryCount>, String> {
let saved = list(db)?;
let filters: Vec<FlowFilter> = saved
.iter()
.map(|s| FlowFilter { hide_system, ..FlowFilter::search(s.query.clone()) })
.collect();
let counts = flows::count_each(db, &filters)?;
Ok(saved
.into_iter()
.zip(counts)
.map(|(s, count)| SavedQueryCount { id: s.id, count })
.collect())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn saves_renames_counts_and_deletes() {
use crate::models::Flow;
use yaak_database::UpdateSource;
let dir = tempfile::tempdir().unwrap();
let db =
ProxyQueryManager::new(&dir.path().join("proxy.db"), &crate::db::test_key()).unwrap();
for (i, status) in [200, 404, 500].into_iter().enumerate() {
let flow = Flow {
id: format!("fl_{i}"),
url: format!("https://example.com/{i}"),
status: Some(status),
..Default::default()
};
db.with_tx(|tx| tx.upsert(&flow, &UpdateSource::Background)).unwrap();
}
let errors = create(&db, " Errors ", "@status:>=400").unwrap();
let ok = create(&db, "OK", "@status:2xx").unwrap();
assert_eq!((errors.name.as_str(), errors.sort_order, ok.sort_order), ("Errors", 0, 1));
assert!(create(&db, " ", "x").is_err());
assert!(create(&db, "x", " ").is_err());
let renamed = rename(&db, &ok.id, "Fine").unwrap();
assert_eq!(renamed.name, "Fine");
assert_eq!(
list(&db).unwrap().iter().map(|s| s.name.as_str()).collect::<Vec<_>>(),
["Errors", "Fine"]
);
let counts = counts(&db, false).unwrap();
assert_eq!(
counts,
vec![
SavedQueryCount { id: errors.id.clone(), count: 2 },
SavedQueryCount { id: ok.id.clone(), count: 1 },
]
);
assert!(delete(&db, &errors.id).unwrap());
assert!(!delete(&db, &errors.id).unwrap());
assert_eq!(list(&db).unwrap(), vec![renamed]);
}
}
crates-proxy/yaak-proxy-lib/tests/at_rest.rs
+8 -12
View File
@@ -267,17 +267,13 @@ fn search_reaches_inside_the_encrypted_database() {
let db = ctx.db().unwrap();
// Query strings are searchable, being inside the encrypted file.
let q = |q: &str| FlowFilter { q: Some(q.into()), limit: Some(50), ..Default::default() };
let q = |q: &str| FlowFilter { limit: Some(50), ..FlowFilter::search(q) };
let (page, total, matching) = flows::page(&db, &q(QUERY_SECRET)).unwrap();
assert_eq!((page.len(), total, matching), (50, FLOWS, Some(FLOWS)));
let (page, _, matching) = flows::page(&db, &q("items/2999")).unwrap();
assert_eq!((page[0].id.as_str(), matching), ("fl_02999", Some(1)));
let posts_not_found = FlowFilter {
method: Some("post".into()),
status: Some("4xx".into()),
limit: Some(1),
..Default::default()
};
let posts_not_found =
FlowFilter { limit: Some(1), ..FlowFilter::search("@method:post @status:4xx") };
let (_, _, matching) = flows::page(&db, &posts_not_found).unwrap();
assert_eq!(matching, Some((0..FLOWS).filter(|i| i % 3 == 0 && i % 10 == 0).count()));
@@ -295,11 +291,11 @@ fn search_reaches_inside_the_encrypted_database() {
}
assert_eq!(seen, FLOWS);
// The sidebar's site and path filters, which run the database's own functions.
let under = |u: &str| FlowFilter { under: Some(vec![u.into()]), ..Default::default() };
assert_eq!(flows::list(&db, &under("api.example.com")).unwrap().len(), FLOWS);
assert_eq!(flows::list(&db, &under("*.example.com")).unwrap().len(), FLOWS);
assert_eq!(flows::list(&db, &under("api.example.com/v1/items/42")).unwrap().len(), 1);
// The sidebar's host, site and path terms, which run the database's own functions.
let found = |q: &str| flows::list(&db, &FlowFilter::search(q)).unwrap().len();
assert_eq!(found("@host:api.example.com"), FLOWS);
assert_eq!(found("@host:*.example.com"), FLOWS);
assert_eq!(found("@host:api.example.com @path:/v1/items/42/*"), 1);
// A full-text table, as body search will want, made and searched in the encrypted file.
let hits: i64 = db
crates-proxy/yaak-proxy-lib/tests/control.rs
+11 -7
View File
@@ -144,14 +144,18 @@ async fn round_trip() {
assert_eq!(flows.iter().map(|f| &f.id).collect::<Vec<_>>(), vec![&flow.id]);
let following = FlowFilter { after: Some(flow.id.clone()), ..Default::default() };
assert!(client.flows(&following).await.unwrap().is_empty());
let filter = FlowFilter { status: Some("4xx".into()), ..Default::default() };
let filter = FlowFilter::search("@status:4xx");
assert!(client.flows(&filter).await.unwrap().is_empty());
let search = FlowFilter { q: Some("CHARGES post 2xx".into()), ..Default::default() };
let search = FlowFilter::search("CHARGES post @status:2xx");
assert_eq!(client.flows(&search).await.unwrap().len(), 1);
let search = FlowFilter { q: Some("charges get".into()), ..Default::default() };
let search = FlowFilter::search("charges @method:get");
assert!(client.flows(&search).await.unwrap().is_empty());
// Listing is not reading; an agent reading the flow marks it.
let read = FlowFilter::search("@is:read");
assert!(client.flows(&read).await.unwrap().is_empty());
let detail = client.flow(&flow.id, 8).await.unwrap();
assert_eq!(client.flows(&read).await.unwrap().len(), 1);
assert_eq!(detail.flow.method, "POST");
let request_body = detail.request_body.unwrap();
assert_eq!((request_body.data.as_str(), request_body.truncated), (r#"{"amount"#, true));
@@ -283,7 +287,7 @@ async fn offline_reads_proxy_db() {
assert_eq!(detail.response_body.unwrap().data, r#"{"id":"ch_1","amount":42}"#);
let body = connection.body(&flow.id, FlowBody::Request).await.unwrap();
assert_eq!(body.data, br#"{"amount":42}"#);
let posts = FlowFilter { method: Some("post".into()), ..Default::default() };
let posts = FlowFilter::search("@method:post");
assert_eq!(connection.har(&posts).await.unwrap().log.entries.len(), 1);
let empty = tempfile::tempdir().unwrap();
@@ -602,7 +606,7 @@ async fn export_har_writes_the_flows_asked_for() {
assert_eq!(read(&selected)["log"]["entries"].as_array().unwrap().len(), 1);
let searched = dir.path().join("searched.har");
let filter = json!({ "q": "charges", "limit": 1 });
let filter = json!({ "query": "charges", "limit": 1 });
let written =
rpc(&ctx, "export_har", json!({ "path": searched, "filter": filter })).await.unwrap();
// The whole search, not the page the list had loaded.
@@ -613,7 +617,7 @@ async fn export_har_writes_the_flows_asked_for() {
let none = dir.path().join("none.har");
let written =
rpc(&ctx, "export_har", json!({ "path": none, "filter": { "q": "nothing-matches" } }))
rpc(&ctx, "export_har", json!({ "path": none, "filter": { "query": "nothing-matches" } }))
.await
.unwrap();
assert_eq!(written, 0);
@@ -657,7 +661,7 @@ async fn flows_name_the_app_that_sent_them() {
assert_eq!(flow.client_path.as_deref(), Some("/usr/bin/curl"));
// The search and the sidebar's Apps group find it by its path.
let filter = FlowFilter { app: Some(vec!["/usr/bin/curl".into()]), ..Default::default() };
let filter = FlowFilter::search("@app:/usr/bin/curl");
assert_eq!(yaak_proxy_lib::flows::list(&ctx.db().unwrap(), &filter).unwrap().len(), 1);
}
crates-proxy/yaak-proxy-lib/tests/flow.rs
+2 -2
View File
@@ -502,10 +502,10 @@ async fn a_tunnelled_connect_is_recorded_with_its_sizes() {
assert_eq!(done.error, None);
assert!(done.request_headers.is_empty() && done.response_headers.is_empty());
let tunnelled = rpc(&ctx, "list_flows", json!({"tunnelled": true})).await.unwrap();
let tunnelled = rpc(&ctx, "list_flows", json!({"query": "@is:tunnelled"})).await.unwrap();
assert_eq!(tunnelled["flows"].as_array().unwrap().len(), 1);
assert_eq!(tunnelled["flows"][0]["tunnelled"], true);
let decrypted = rpc(&ctx, "list_flows", json!({"tunnelled": false})).await.unwrap();
let decrypted = rpc(&ctx, "list_flows", json!({"query": "-@is:tunnelled"})).await.unwrap();
assert_eq!(decrypted["flows"].as_array().unwrap().len(), 0);
}
crates-proxy/yaak-proxy-lib/tests/streams.rs
+1 -1
View File
@@ -370,7 +370,7 @@ async fn grpc_web_calls_are_stored_and_decoded() {
(failed.grpc_status, failed.grpc_message.as_deref()),
(Some(12), Some("not here"))
);
let errors = rpc(&ctx, "list_flows", json!({ "status": "error" })).await.unwrap();
let errors = rpc(&ctx, "list_flows", json!({ "query": "@is:error" })).await.unwrap();
let ids: Vec<_> =
errors["flows"].as_array().unwrap().iter().map(|f| f["id"].clone()).collect();
assert_eq!(ids, vec![json!(failed.id)], "a non-zero status is a failed call");
crates/yaak-query/bindings/query.ts
Generated
+15 -3
View File
@@ -10,7 +10,15 @@ label: string,
/**
* What replaces the span: `@method:`, `@header.`, or a value quoted when it needs to be.
*/
insert: string, kind: CompletionKind, info: string | null,
insert: string, kind: CompletionKind,
/**
* What a field is, from its definition.
*/
info: string | null,
/**
* A term using the field, from its definition.
*/
example: string | null,
/**
* The term is not finished after inserting (a field still needs its value, a map field its
* key), so completion should open again.
@@ -57,9 +65,13 @@ name: string, kind: FieldKind,
*/
map?: boolean,
/**
* A short note shown with the field in completions.
* What the field is, in a few words, shown with it in completions: `GET, POST, …`.
*/
info?: string, };
info?: string,
/**
* A term using it, shown with it in completions: `@method:post`.
*/
example?: string, };
export type FieldKind = { "type": "text" } | { "type": "number", units: Array<Unit>, } | { "type": "flag" };
crates/yaak-query/pkg/yaak_query_bg.wasm
BIN
View File
Binary file not shown.
crates/yaak-query/src/complete.rs
+74 -3
View File
@@ -25,7 +25,10 @@ pub struct Completion {
/// What replaces the span: `@method:`, `@header.`, or a value quoted when it needs to be.
pub insert: String,
pub kind: CompletionKind,
/// What a field is, from its definition.
pub info: Option<String>,
/// A term using the field, from its definition.
pub example: Option<String>,
/// The term is not finished after inserting (a field still needs its value, a map field its
/// key), so completion should open again.
pub reopen: bool,
@@ -115,6 +118,7 @@ fn missing_at(word: &str, typed: &str, schema: &Schema) -> Option<Completion> {
insert: term,
kind: CompletionKind::Field,
info: None,
example: None,
// With no value yet, go on to the field's values
reopen: value.is_empty(),
})
@@ -143,7 +147,8 @@ fn field_completions(
insert: format!("@{path}:"),
label: path,
kind: CompletionKind::Field,
info: None,
info: map.info.clone(),
example: map.example.clone(),
reopen: true,
});
}
@@ -157,6 +162,7 @@ fn field_completions(
insert,
kind: CompletionKind::Field,
info: field.info.clone(),
example: field.example.clone(),
reopen: true,
});
}
@@ -191,6 +197,7 @@ fn value_completions(
insert: format_value(value),
kind: CompletionKind::Value,
info: None,
example: None,
reopen: false,
})
.collect();
@@ -236,14 +243,27 @@ mod tests {
kind: FieldKind::Text,
map: false,
info: None,
example: None,
};
Schema::new(
vec![
text("method"),
text("url"),
text("name"),
FieldDef { name: "header".into(), kind: FieldKind::Text, map: true, info: None },
FieldDef { name: "error".into(), kind: FieldKind::Flag, map: false, info: None },
FieldDef {
name: "header".into(),
kind: FieldKind::Text,
map: true,
info: None,
example: None,
},
FieldDef {
name: "error".into(),
kind: FieldKind::Flag,
map: false,
info: None,
example: None,
},
],
FreeText::Fuzzy,
)
@@ -369,6 +389,57 @@ mod tests {
assert_eq!(at("a|b"), Some(("a".into(), inserts(&["@name:", "@header."]))));
}
/// Field names that start with what is typed come first, then those holding it, in the
/// schema's order; nothing matches loosely.
#[test]
fn prefix_matches_first() {
let field = |name: &str| FieldDef {
name: name.into(),
kind: FieldKind::Text,
map: false,
info: None,
example: None,
};
let s = Schema::new(
vec![
field("time"),
field("scheme"),
field("method"),
field("host"),
],
FreeText::Substring,
);
let labels = |input: &str| {
let c = completions(input, input.len(), &s, &HashMap::new()).unwrap();
c.items.into_iter().map(|i| i.label).collect::<Vec<_>>()
};
assert_eq!(labels("m"), ["method", "time", "scheme"]);
assert_eq!(labels("@h"), ["host", "scheme", "method"]);
assert!(completions("mt", 2, &s, &HashMap::new()).is_none(), "no fuzzy match");
}
/// A field's description and example go with it, and with the keys under a map.
#[test]
fn descriptions_and_examples() {
let s = Schema::new(
vec![FieldDef {
name: "header".into(),
kind: FieldKind::Text,
map: true,
info: Some("A header on either side".into()),
example: Some("@header.content-type:json".into()),
}],
FreeText::Substring,
);
let values = HashMap::from([("header".to_string(), vec!["accept".to_string()])]);
let c = completions("@he", 3, &s, &values).unwrap();
assert_eq!(c.items[0].info.as_deref(), Some("A header on either side"));
assert_eq!(c.items[0].example.as_deref(), Some("@header.content-type:json"));
let c = completions("@header.", 8, &s, &values).unwrap();
assert_eq!(c.items[0].label, "header.accept");
assert_eq!(c.items[0].example.as_deref(), Some("@header.content-type:json"));
}
#[test]
fn cursor_is_clamped() {
let c = completions("é", 1, &schema(), &values());
crates/yaak-query/src/condition.rs
+241
View File
@@ -0,0 +1,241 @@
//! What a field term asks of a field's values, resolved once from the term's value and the
//! field's kind.
//!
//! The evaluator matches with these, and a consumer that compiles queries elsewhere (to SQL,
//! say) translates them rather than deriving the rules again, so the two agree. The helpers
//! below are the value tests the conditions are made of, for a compiler that has to run them
//! where the values are.
use crate::ast::{CompareOp, FieldValue};
use crate::lexer::parse_number;
use crate::schema::{FieldKind, Unit, parse_flag, to_base};
/// A field term's test, applied to each of the field's values: the term matches when any value
/// passes, except [`Condition::Flag`] `false`, which matches when none is set.
#[derive(Clone, Debug, PartialEq)]
pub enum Condition {
/// The value, as lowercase text, contains this lowercase text, which is never empty.
Contains(String),
/// The whole value, as lowercase text, matches this lowercase pattern, where `*` matches
/// any run of characters. A pattern without `*` must equal the value.
Wildcard(String),
/// The value is there: text that is not empty, any number, or a flag that is set.
Present,
/// The value reads as a number.
IsNumber,
/// The value, read as a number, equals this one, in the field's base unit.
Equals(f64),
/// The value, read as a number, compares so with this one, in the field's base unit.
Compare(CompareOp, f64),
/// The value, read as a number, is in this inclusive range, in the field's base unit.
Range(f64, f64),
/// With `true`, some value is a set flag; with `false`, none is.
Flag(bool),
/// Nothing matches: a value the field cannot take, such as a unit it does not have.
Never,
}
impl Condition {
/// The condition `value` sets for a field of `kind`.
pub fn new(kind: &FieldKind, value: &FieldValue) -> Self {
match kind {
FieldKind::Text => text_condition(value),
FieldKind::Number { units } => number_condition(value, units),
FieldKind::Flag => flag_condition(value),
}
}
/// Whether a text value passes.
pub fn matches_text(&self, text: &str) -> bool {
match self {
Condition::Contains(needle) => contains_ignore_case(text, needle),
Condition::Wildcard(pattern) => wildcard_match(pattern, &text.to_lowercase()),
Condition::Present => !text.is_empty(),
Condition::Flag(_) => parse_flag(text) == Some(true),
_ => self.matches_parsed_number(read_number(text)),
}
}
/// Whether a number value passes.
pub fn matches_number(&self, n: f64) -> bool {
match self {
Condition::Contains(needle) => format_number(n).contains(needle.as_str()),
Condition::Wildcard(pattern) => wildcard_match(pattern, &format_number(n)),
Condition::Present => true,
Condition::Flag(_) => n != 0.0,
_ => self.matches_parsed_number(Some(n)),
}
}
/// Whether a flag value passes.
pub fn matches_flag(&self, b: bool) -> bool {
let text = if b { "true" } else { "false" };
match self {
Condition::Contains(needle) => text.contains(needle.as_str()),
Condition::Wildcard(pattern) => wildcard_match(pattern, text),
Condition::Present | Condition::Flag(_) => b,
_ => false,
}
}
fn matches_parsed_number(&self, n: Option<f64>) -> bool {
match self {
Condition::IsNumber => n.is_some(),
Condition::Equals(target) => n == Some(*target),
Condition::Compare(op, target) => n.is_some_and(|n| op.test(n, *target)),
Condition::Range(low, high) => n.is_some_and(|n| n >= *low && n <= *high),
_ => false,
}
}
}
fn text_condition(value: &FieldValue) -> Condition {
match value {
FieldValue::Text { text, .. } if text.is_empty() => Condition::Never,
FieldValue::Text { text, .. } => Condition::Contains(text.to_lowercase()),
FieldValue::Wildcard { pattern } => Condition::Wildcard(pattern.to_lowercase()),
FieldValue::Exists => Condition::Present,
// A text field holding numbers, such as a status shown as text
FieldValue::Compare { op, number } if number.unit.is_none() => {
Condition::Compare(*op, number.value)
}
FieldValue::Range { low, high } if low.unit.is_none() && high.unit.is_none() => {
Condition::Range(low.value, high.value)
}
FieldValue::Compare { .. } | FieldValue::Range { .. } => Condition::Never,
}
}
fn number_condition(value: &FieldValue, units: &[Unit]) -> Condition {
match value {
FieldValue::Text { text, .. } if text.is_empty() => Condition::Never,
// `@status:404` is equality; anything else matches the number as text
FieldValue::Text { text, quoted: false } => match parse_number(text) {
Some(number) => to_base(&number, units).map_or(Condition::Never, Condition::Equals),
None => text_condition(value),
},
FieldValue::Text { quoted: true, .. } | FieldValue::Wildcard { .. } => {
text_condition(value)
}
FieldValue::Exists => Condition::IsNumber,
FieldValue::Compare { op, number } => to_base(number, units)
.map_or(Condition::Never, |target| Condition::Compare(*op, target)),
FieldValue::Range { low, high } => match (to_base(low, units), to_base(high, units)) {
(Some(low), Some(high)) => Condition::Range(low, high),
_ => Condition::Never,
},
}
}
fn flag_condition(value: &FieldValue) -> Condition {
match value {
FieldValue::Exists => Condition::Flag(true),
FieldValue::Text { text, .. } => parse_flag(text).map_or(Condition::Never, Condition::Flag),
_ => Condition::Never,
}
}
/// Whether `hay`, lowercased, contains `needle`, which is lowercase already. Copies nothing
/// when both are ASCII.
pub fn contains_ignore_case(hay: &str, needle: &str) -> bool {
if !(hay.is_ascii() && needle.is_ascii()) {
return hay.to_lowercase().contains(needle);
}
let (hay, needle) = (hay.as_bytes(), needle.as_bytes());
needle.is_empty() || hay.windows(needle.len()).any(|w| w.eq_ignore_ascii_case(needle))
}
/// Whether `pattern`, where `*` matches any run of characters, matches the whole of `text`.
/// Case-sensitive: both are lowercase already when matching a query.
pub fn wildcard_match(pattern: &str, text: &str) -> bool {
let mut parts = pattern.split('*');
let first = parts.next().unwrap_or_default();
let Some(mut rest) = text.strip_prefix(first) else {
return false;
};
let parts: Vec<&str> = parts.collect();
let Some((last, middle)) = parts.split_last() else {
// No `*` at all: the whole text must equal the pattern
return rest.is_empty();
};
for part in middle {
match rest.find(part) {
Some(i) => rest = &rest[i + part.len()..],
None => return false,
}
}
rest.len() >= last.len() && rest.ends_with(last)
}
/// A text value read as a number, as comparisons read it: the whole text, spaces around it
/// aside.
pub fn read_number(text: &str) -> Option<f64> {
text.trim().parse().ok()
}
/// A number as text, as a field's text matching sees it: `200`, not `200.0`.
pub fn format_number(n: f64) -> String {
if n.fract() == 0.0 && n.abs() < 1e15 { format!("{}", n as i64) } else { n.to_string() }
}
#[cfg(test)]
mod tests {
use super::*;
use crate::ast::Node;
use crate::parse;
fn condition(kind: FieldKind, input: &str) -> Condition {
let Some(Node::Field { value, .. }) = parse(input) else {
panic!("not a field")
};
Condition::new(&kind, &value)
}
fn ms() -> FieldKind {
FieldKind::Number {
units: vec![
Unit { name: "ms".into(), factor: 1.0 },
Unit { name: "s".into(), factor: 1000.0 },
],
}
}
#[test]
fn resolves_text_values() {
assert_eq!(condition(FieldKind::Text, "@a:Foo"), Condition::Contains("foo".into()));
assert_eq!(condition(FieldKind::Text, "@a:"), Condition::Never);
assert_eq!(condition(FieldKind::Text, "@a:F*o"), Condition::Wildcard("f*o".into()));
assert_eq!(condition(FieldKind::Text, "@a:*"), Condition::Present);
assert_eq!(condition(FieldKind::Text, "@a:>5"), Condition::Compare(CompareOp::Gt, 5.0));
assert_eq!(condition(FieldKind::Text, "@a:>5s"), Condition::Never);
}
#[test]
fn resolves_number_values_in_base_units() {
assert_eq!(condition(ms(), "@d:1.5s"), Condition::Equals(1500.0));
assert_eq!(condition(ms(), "@d:1m"), Condition::Never);
assert_eq!(condition(ms(), "@d:<2s"), Condition::Compare(CompareOp::Lt, 2000.0));
assert_eq!(condition(ms(), "@d:[1s TO 2s]"), Condition::Range(1000.0, 2000.0));
assert_eq!(condition(ms(), "@d:*"), Condition::IsNumber);
assert_eq!(condition(ms(), "@d:\"15\""), Condition::Contains("15".into()));
assert_eq!(condition(ms(), "@d:abc"), Condition::Contains("abc".into()));
}
#[test]
fn resolves_flags() {
assert_eq!(condition(FieldKind::Flag, "@f:*"), Condition::Flag(true));
assert_eq!(condition(FieldKind::Flag, "@f:no"), Condition::Flag(false));
assert_eq!(condition(FieldKind::Flag, "@f:maybe"), Condition::Never);
}
#[test]
fn tests_values() {
let c = Condition::Wildcard("40*".into());
assert!(c.matches_number(404.0));
assert!(!c.matches_text("504"));
assert!(Condition::Equals(200.0).matches_text(" 200 "));
assert!(Condition::Contains("é".into()).matches_text("CAFÉ"));
assert!(!Condition::Present.matches_text(""));
assert!(Condition::Flag(true).matches_flag(true));
}
}
crates/yaak-query/src/eval.rs
+38 -163
View File
@@ -1,8 +1,11 @@
//! Runs a parsed query in memory, against one document or a whole table of them.
use crate::ast::{FieldValue, Node};
use crate::ast::Node;
use crate::condition::Condition;
#[cfg(test)]
use crate::condition::wildcard_match;
use crate::fuzzy::{FuzzyScratch, fuzzy_match_in};
use crate::schema::{FieldKind, FreeText, Schema, parse_flag, to_base};
use crate::schema::{FreeText, Schema};
use serde::{Deserialize, Serialize};
use std::borrow::Cow;
use std::collections::HashMap;
@@ -208,55 +211,13 @@ enum Val<'a> {
}
impl Val<'_> {
/// The value as lowercase text.
fn lower(&self) -> Cow<'_, str> {
fn passes(&self, condition: &Condition) -> bool {
match self {
Val::Text(text) => Cow::Owned(text.to_lowercase()),
Val::Number(n) => Cow::Owned(format_number(*n)),
Val::Flag(b) => Cow::Borrowed(if *b { "true" } else { "false" }),
Val::Text(text) => condition.matches_text(text),
Val::Number(n) => condition.matches_number(*n),
Val::Flag(b) => condition.matches_flag(*b),
}
}
/// Whether the value, lowercased, contains `needle` (lowercase).
fn contains(&self, needle: &str) -> bool {
match self {
Val::Text(text) => contains_ignore_case(text, needle),
_ => self.lower().contains(needle),
}
}
fn as_number(&self) -> Option<f64> {
match self {
Val::Number(n) => Some(*n),
Val::Text(text) => text.trim().parse().ok(),
Val::Flag(_) => None,
}
}
fn as_flag(&self) -> Option<bool> {
match self {
Val::Flag(b) => Some(*b),
Val::Text(text) => parse_flag(text),
Val::Number(n) => Some(*n != 0.0),
}
}
fn is_present(&self) -> bool {
match self {
Val::Text(text) => !text.is_empty(),
Val::Number(_) => true,
Val::Flag(b) => *b,
}
}
}
/// Whether `hay`, lowercased, contains `needle` (lowercase), without copying ASCII text.
fn contains_ignore_case(hay: &str, needle: &str) -> bool {
if !(hay.is_ascii() && needle.is_ascii()) {
return hay.to_lowercase().contains(needle);
}
let (hay, needle) = (hay.as_bytes(), needle.as_bytes());
needle.is_empty() || hay.windows(needle.len()).any(|w| w.eq_ignore_ascii_case(needle))
}
/// `text` lowercased into `out`, without a fresh allocation per call.
@@ -270,11 +231,6 @@ fn lowercase_into(text: &str, out: &mut String) {
}
}
/// `200`, not `200.0`, so a number field's text matches what the user sees.
fn format_number(n: f64) -> String {
if n.fract() == 0.0 && n.abs() < 1e15 { format!("{}", n as i64) } else { n.to_string() }
}
/// Where field values come from: a [`Document`], or a row of a [`Table`].
trait FieldSource {
/// Whether any value under `path` (lowercase) passes `test`. With `any_key`, `path` is a
@@ -351,9 +307,8 @@ enum Plan {
path: String,
/// A map field with no key: every key's values.
any_key: bool,
kind: FieldKind,
/// With its text and wildcard pattern lowercased.
value: FieldValue,
/// Never [`Condition::Never`], which plans as [`Plan::Never`].
condition: Condition,
},
/// A term that can match nothing: an unknown field, an empty word or phrase.
Never,
@@ -418,14 +373,10 @@ impl Matcher {
Plan::Not(node) => !self.eval(node, text, source, fuzzy),
Plan::Term(term) if self.fuzzy => fuzzy_match_in(text, term, fuzzy),
Plan::Term(term) | Plan::Phrase(term) => text.contains(term.as_str()),
Plan::Field { path, any_key, kind, value } => {
let mut any =
|test: &mut dyn FnMut(Val) -> bool| source.any_value(path, *any_key, test);
match kind {
FieldKind::Text => match_text(&mut any, value),
FieldKind::Number { units } => match_number(&mut any, value, units),
FieldKind::Flag => match_flag(&mut any, value),
}
Plan::Field { path, any_key, condition } => {
let any = source.any_value(path, *any_key, &mut |v| v.passes(condition));
// `@flag:false`: no value is set
if *condition == Condition::Flag(false) { !any } else { any }
}
Plan::Never => false,
}
@@ -458,117 +409,27 @@ fn plan_node(node: &Node, schema: &Schema, uses_text: &mut bool) -> Plan {
let Some(field) = schema.resolve(path) else {
return Plan::Never;
};
let value = match value {
FieldValue::Text { text, quoted } => {
FieldValue::Text { text: text.to_lowercase(), quoted: *quoted }
}
FieldValue::Wildcard { pattern } => {
FieldValue::Wildcard { pattern: pattern.to_lowercase() }
}
other => other.clone(),
};
let condition = Condition::new(&field.def.kind, value);
if condition == Condition::Never {
return Plan::Never;
}
Plan::Field {
path: field.document_path(),
any_key: field.def.map && field.key.is_none(),
kind: field.def.kind.clone(),
value,
condition,
}
}
}
}
/// Asks whether any of a field's values passes a test.
type AnyValue<'s> = dyn FnMut(&mut dyn FnMut(Val) -> bool) -> bool + 's;
/// `value`'s text and wildcard pattern are already lowercase (see [`plan_node`]).
fn match_text(any: &mut AnyValue, value: &FieldValue) -> bool {
match value {
FieldValue::Text { text: needle, .. } => {
!needle.is_empty() && any(&mut |v| v.contains(needle))
}
FieldValue::Wildcard { pattern } => any(&mut |v| wildcard_match(pattern, &v.lower())),
FieldValue::Exists => any(&mut |v| v.is_present()),
// A text field holding numbers, such as a status shown as text
FieldValue::Compare { op, number } => {
number.unit.is_none()
&& any(&mut |v| v.as_number().is_some_and(|n| op.test(n, number.value)))
}
FieldValue::Range { low, high } => {
low.unit.is_none()
&& high.unit.is_none()
&& any(&mut |v| v.as_number().is_some_and(|n| n >= low.value && n <= high.value))
}
}
}
fn match_number(any: &mut AnyValue, value: &FieldValue, units: &[crate::Unit]) -> bool {
match value {
FieldValue::Text { text, quoted } => {
if text.is_empty() {
return false;
}
// `@status:404` is equality; anything else matches the number as text
match (!quoted).then(|| crate::lexer::parse_number(text)).flatten() {
Some(number) => to_base(&number, units)
.is_some_and(|target| any(&mut |v| v.as_number() == Some(target))),
None => match_text(any, value),
}
}
FieldValue::Wildcard { .. } => match_text(any, value),
FieldValue::Exists => any(&mut |v| v.as_number().is_some()),
FieldValue::Compare { op, number } => to_base(number, units)
.is_some_and(|target| any(&mut |v| v.as_number().is_some_and(|n| op.test(n, target)))),
FieldValue::Range { low, high } => match (to_base(low, units), to_base(high, units)) {
(Some(low), Some(high)) => {
any(&mut |v| v.as_number().is_some_and(|n| n >= low && n <= high))
}
_ => false,
},
}
}
fn match_flag(any: &mut AnyValue, value: &FieldValue) -> bool {
let mut is_set = || any(&mut |v| v.as_flag() == Some(true));
match value {
FieldValue::Exists => is_set(),
FieldValue::Text { text, .. } => match parse_flag(text) {
Some(true) => is_set(),
Some(false) => !is_set(),
None => false,
},
_ => false,
}
}
/// Whether `pattern`, where `*` matches any run of characters, matches the whole of `text`.
pub(crate) fn wildcard_match(pattern: &str, text: &str) -> bool {
let mut parts = pattern.split('*');
let first = parts.next().unwrap_or_default();
let Some(mut rest) = text.strip_prefix(first) else {
return false;
};
let parts: Vec<&str> = parts.collect();
let Some((last, middle)) = parts.split_last() else {
// No `*` at all: the whole text must equal the pattern
return rest.is_empty();
};
for part in middle {
match rest.find(part) {
Some(i) => rest = &rest[i + part.len()..],
None => return false,
}
}
rest.len() >= last.len() && rest.ends_with(last)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::schema::{FieldDef, Unit};
use crate::schema::{FieldDef, FieldKind, Unit};
use crate::{format_field, parse};
fn text_field(name: &str) -> FieldDef {
FieldDef { name: name.into(), kind: FieldKind::Text, map: false, info: None }
FieldDef { name: name.into(), kind: FieldKind::Text, map: false, info: None, example: None }
}
/// The fields and free-text mode of Yaak's sidebar.
@@ -594,12 +455,19 @@ mod tests {
Schema::new(
vec![
text_field("url"),
FieldDef { name: "header".into(), kind: FieldKind::Text, map: true, info: None },
FieldDef {
name: "header".into(),
kind: FieldKind::Text,
map: true,
info: None,
example: None,
},
FieldDef {
name: "status".into(),
kind: FieldKind::Number { units: vec![] },
map: false,
info: None,
example: None,
},
FieldDef {
name: "duration".into(),
@@ -611,8 +479,15 @@ mod tests {
},
map: false,
info: None,
example: None,
},
FieldDef {
name: "error".into(),
kind: FieldKind::Flag,
map: false,
info: None,
example: None,
},
FieldDef { name: "error".into(), kind: FieldKind::Flag, map: false, info: None },
],
FreeText::Substring,
)
crates/yaak-query/src/lib.rs
+7 -1
View File
@@ -9,7 +9,9 @@
//!
//! [`parse`] gives the tree, which a consumer may walk to compile it elsewhere (to SQL, say).
//! [`evaluate`] runs it in memory against a [`Document`], with a [`Schema`] saying what the
//! fields are. [`tokens`], [`completions`] and [`format_field`] serve the search box.
//! fields are. [`tokens`], [`completions`] and [`format_field`] serve the search box. A compiler
//! resolves each field term to a [`Condition`] and translates that, so it matches as
//! [`evaluate`] does.
// The wasm-bindgen exports expand to unsafe glue; nothing else in the crate may use unsafe
#![cfg_attr(
@@ -20,10 +22,12 @@
pub mod ast;
mod complete;
mod condition;
mod eval;
mod format;
pub mod fuzzy;
mod lexer;
mod offsets;
mod parser;
mod schema;
mod tokens;
@@ -33,8 +37,10 @@ mod wasm;
pub use ast::{CompareOp, FieldValue, Node, NumberLiteral, Span};
pub use complete::{Completion, CompletionKind, Completions, completions};
pub use condition::{Condition, contains_ignore_case, format_number, read_number, wildcard_match};
pub use eval::{DocValue, DocValues, Document, Matcher, Table, evaluate};
pub use format::{format_field, format_value, is_valid_path, quote};
pub use offsets::Utf16Offsets;
pub use parser::parse;
pub use schema::{Diagnostic, FieldDef, FieldKind, FreeText, ResolvedField, Schema, Unit};
pub use tokens::{Token, TokenKind, tokens};
crates/yaak-query/src/offsets.rs
+52
View File
@@ -0,0 +1,52 @@
//! Byte offsets to UTF-16 offsets and back, for spans handed to JavaScript, which counts string
//! positions in UTF-16 code units.
use crate::ast::Span;
/// Byte offset to UTF-16 offset, and back, for one input.
pub struct Utf16Offsets {
/// `utf16[i]` is the UTF-16 offset of byte `i`, for every char boundary and the end.
utf16: Vec<u32>,
}
impl Utf16Offsets {
pub fn new(input: &str) -> Self {
let mut utf16 = vec![0; input.len() + 1];
let mut units = 0u32;
for (i, c) in input.char_indices() {
for b in 0..c.len_utf8() {
utf16[i + b] = units;
}
units += c.len_utf16() as u32;
}
utf16[input.len()] = units;
Self { utf16 }
}
/// A byte span as a UTF-16 one.
pub fn span(&self, span: Span) -> Span {
Span { start: self.utf16[span.start()], end: self.utf16[span.end()] }
}
/// The byte offset of UTF-16 offset `pos`, or of the char it falls inside.
pub fn byte(&self, pos: u32) -> usize {
self.utf16.partition_point(|&u| u < pos).min(self.utf16.len() - 1)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn converts_both_ways() {
let input = "é@a:😀x";
let offsets = Utf16Offsets::new(input);
// `é` is two bytes and one unit; `😀` is four bytes and two units
assert_eq!(offsets.span(Span::new(2, 5)), Span { start: 1, end: 4 });
assert_eq!(offsets.span(Span::new(5, 10)), Span { start: 4, end: 7 });
assert_eq!(offsets.byte(4), 5);
assert_eq!(offsets.byte(6), 9);
assert_eq!(offsets.byte(99), input.len());
}
}
crates/yaak-query/src/schema.rs
+6 -2
View File
@@ -45,10 +45,14 @@ pub struct FieldDef {
#[serde(default)]
#[cfg_attr(test, ts(optional, as = "Option<bool>"))]
pub map: bool,
/// A short note shown with the field in completions.
/// What the field is, in a few words, shown with it in completions: `GET, POST, …`.
#[serde(default)]
#[cfg_attr(test, ts(optional))]
pub info: Option<String>,
/// A term using it, shown with it in completions: `@method:post`.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[cfg_attr(test, ts(optional))]
pub example: Option<String>,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@@ -207,7 +211,7 @@ mod tests {
use crate::parse;
fn field(name: &str, kind: FieldKind, map: bool) -> FieldDef {
FieldDef { name: name.to_string(), kind, map, info: None }
FieldDef { name: name.to_string(), kind, map, info: None, example: None }
}
fn schema() -> Schema {
crates/yaak-query/src/wasm.rs
+1 -31
View File
@@ -3,43 +3,13 @@
//! JavaScript counts string positions in UTF-16 code units, so every span going out is
//! converted from bytes, and the cursor coming in is converted to bytes.
use crate::{Document, Matcher, Schema, Span, Table};
use crate::{Document, Matcher, Schema, Table, Utf16Offsets as Offsets};
use serde::Serialize;
use serde::de::DeserializeOwned;
use std::collections::HashMap;
use wasm_bindgen::JsValue;
use wasm_bindgen::prelude::wasm_bindgen;
/// Byte offset to UTF-16 offset, and back, for one input.
struct Offsets {
/// `utf16[i]` is the UTF-16 offset of byte `i`, for every char boundary and the end.
utf16: Vec<u32>,
}
impl Offsets {
fn new(input: &str) -> Self {
let mut utf16 = vec![0; input.len() + 1];
let mut units = 0u32;
for (i, c) in input.char_indices() {
for b in 0..c.len_utf8() {
utf16[i + b] = units;
}
units += c.len_utf16() as u32;
}
utf16[input.len()] = units;
Self { utf16 }
}
fn span(&self, span: Span) -> Span {
Span { start: self.utf16[span.start()], end: self.utf16[span.end()] }
}
/// The byte offset of UTF-16 offset `pos`, or of the char it falls inside.
fn byte(&self, pos: u32) -> usize {
self.utf16.partition_point(|&u| u < pos).min(self.utf16.len() - 1)
}
}
fn to_js<T: Serialize>(value: &T) -> Result<JsValue, JsValue> {
value
.serialize(&serde_wasm_bindgen::Serializer::json_compatible())
package-lock.json
Generated
+1
View File
@@ -272,6 +272,7 @@
"babel-plugin-react-compiler": "^1.0.0",
"typescript": "^5.8.3",
"vite": "npm:@voidzero-dev/vite-plus-core@^1.4.0",
"vite-plugin-wasm": "^3.5.0",
"vite-plus": "^1.4.0"
}
},
packages/ui/src/components/editor/filter/extension.ts
+2 -29
View File
@@ -1,8 +1,5 @@
import type { Completion, CompletionContext, CompletionResult } from "@codemirror/autocomplete";
import { autocompletion, startCompletion } from "@codemirror/autocomplete";
import { LanguageSupport, LRLanguage, syntaxTree } from "@codemirror/language";
import type { SyntaxNode } from "@lezer/common";
import { parser } from "./filter";
export interface FieldDef {
name: string;
@@ -87,17 +84,7 @@ function fieldCompletionFrom(
return null;
}
function inPhrase(ctx: CompletionContext): boolean {
// Lezer node names from your grammar: Phrase is the quoted token
let n: SyntaxNode | null = syntaxTree(ctx.state).resolveInner(ctx.pos, -1);
while (n) {
if (n.name === "Phrase") return true;
n = n.parent;
}
return false;
}
// While typing an incomplete quote, there's no Phrase token yet.
/** Inside a quote, opened and not yet closed */
function inUnclosedQuote(doc: string, pos: number): boolean {
let quotes = 0;
for (let i = 0; i < pos; i++) {
@@ -200,7 +187,7 @@ export function makeCompletionSource(opts: FilterOptions) {
const { state, pos } = ctx;
const doc = state.doc.toString();
if (inPhrase(ctx) || inUnclosedQuote(doc, pos)) {
if (inUnclosedQuote(doc, pos)) {
return null;
}
@@ -250,14 +237,6 @@ export function makeCompletionSource(opts: FilterOptions) {
};
}
const language = LRLanguage.define({
name: "filter",
parser,
languageData: {
autocompletion: {},
},
});
/**
* Field and value suggestions, as the sidebar's filter offers them, for a field of another
* language: a plain input of `host:` and `app:` entries, say
@@ -265,9 +244,3 @@ const language = LRLanguage.define({
export function filterAutocomplete(options: FilterOptions) {
return autocompletion({ override: [makeCompletionSource(options)] });
}
/** Public extension */
export function filter(options: FilterOptions) {
const source = makeCompletionSource(options);
return new LanguageSupport(language, [autocompletion({ override: [source] })]);
}
packages/ui/src/components/editor/filter/filter.grammar
-78
View File
@@ -1,78 +0,0 @@
@top Query { Expr }
@skip { space+ }
@tokens {
space { $[ \t\r\n]+ }
LParen { "(" }
RParen { ")" }
At { "@" }
Colon { ":" }
Not { "-" | "NOT" }
// Keywords (case-insensitive)
And { "AND" }
Or { "OR" }
// "quoted phrase" with simple escapes: \" and \\
Phrase { '"' (!["\\] | "\\" _)* '"' }
// Bare words run until filter syntax or whitespace. Leading '-' remains unary
// negation, but '-' may appear after the first character.
Word { ![ \t\r\n():"@-] ![ \t\r\n():"@]* }
FieldValueWord { ![ \t\r\n"] ![ \t\r\n]* }
@precedence { Not, And, Or, Word }
}
@detectDelim
// Precedence: NOT (highest) > AND > OR (lowest)
// We also allow implicit AND in your parser/evaluator, but for highlighting,
// this grammar parses explicit AND/OR/NOT + adjacency as a sequence (Seq).
Expr {
OrExpr
}
OrExpr {
AndExpr (Or AndExpr)*
}
AndExpr {
Unary (And Unary | Unary)* // allow implicit AND by adjacency: Unary Unary
}
Unary {
Not Unary
| Primary
}
Primary {
Group
| Field
| Phrase
| Term
}
Group {
LParen Expr RParen
}
Field {
FieldName Colon FieldValue
}
FieldName {
At? Word
}
FieldValue {
Phrase
| FieldValueWord
}
Term {
Word
}
@external propSource highlight from "./highlight"
packages/ui/src/components/editor/filter/filter.test.ts
-42
View File
@@ -1,42 +0,0 @@
import { describe, expect, test } from "vite-plus/test";
import { parser } from "./filter";
function getNodeNames(input: string): string[] {
const tree = parser.parse(input);
const nodes: string[] = [];
const cursor = tree.cursor();
do {
if (cursor.name !== "Query") {
nodes.push(cursor.name);
}
} while (cursor.next());
return nodes;
}
describe("filter grammar", () => {
test("parses URL-like field values as one value", () => {
const nodes = getNodeNames("@url:yaak.app/foo-bar");
expect(nodes).not.toContain("⚠");
expect(nodes).toContain("FieldValue");
expect(nodes).toContain("FieldValueWord");
});
test("parses punctuation-heavy field values as one value", () => {
const nodes = getNodeNames("@url:yaa$&#*@tsrna(*)");
expect(nodes).not.toContain("⚠");
expect(nodes).toContain("FieldValue");
expect(nodes).toContain("FieldValueWord");
});
test("parses operator-looking field values as one value", () => {
const negativeValueNodes = getNodeNames("@url:-foo");
const operatorWordNodes = getNodeNames("@url:AND");
expect(negativeValueNodes).not.toContain("⚠");
expect(negativeValueNodes).toContain("FieldValueWord");
expect(operatorWordNodes).not.toContain("⚠");
expect(operatorWordNodes).toContain("FieldValueWord");
});
});
packages/ui/src/components/editor/filter/filter.ts
-22
View File
@@ -1,22 +0,0 @@
// This file was generated by lezer-generator. You probably shouldn't edit it.
import {LRParser} from "@lezer/lr"
import {highlight} from "./highlight"
export const parser = LRParser.deserialize({
version: 14,
states: "%WOVQPOOPhOPOOOVQPO'#CfOmQPO'#ChO!_QPO'#ChO!dQPO'#CgOOQO'#Cc'#CcOVQPO'#CaOOQO'#Ca'#CaO!iQPO'#C`O!yQPO'#C_OOQO'#C^'#C^QOQPOOPOOO'#Cr'#CrP#UOPO)C>lO#]QPO,59QOOQO,59S,59SO#bQQO,59ROOQO,58{,58{OVQPO'#CsOOQO'#Cs'#CsO#jQPO,58zOVQPO'#CtO#zQPO,58yPOOO-E6p-E6pOOQO1G.l1G.lOOQO'#Cl'#ClOOQO1G.m1G.mOOQO,59_,59_OOQO-E6q-E6qOOQO,59`,59`OOQO-E6r-E6r",
stateData: "$]~OkPQ~OUVOXQO]SO^ROaUO~Ok]O~OUcXXcX]cX^cX_[XacXdcXecXicXWcX~O^`O~O_aO~OdcOeSXiSXWSX~PVOefOiRXWRX~Ok]O~Qj]WiO~OajObjO~OdcOeSaiSaWSa~PVOefOiRaWRa~OUde^e~",
goto: "#^iPPjpt{P![PP!e!e!nPPP!wPP!ePP!z#Q#WQ[OR_QTZOQSYOQRnfUXOQfQbVSdXeRlc_WOQVXcef_UOQVXcef_TOQVXcefRkaQ^PRh^QeXRmeQgYRog",
nodeNames: "⚠ Query Expr OrExpr AndExpr Unary Not Primary RParen LParen Group Field FieldName At Word Colon FieldValue Phrase FieldValueWord Term And Or",
maxTerm: 27,
nodeProps: [
["openedBy", 8,"LParen"],
["closedBy", 9,"RParen"]
],
propSources: [highlight],
skippedNodes: [0,22],
repeatNodeCount: 3,
tokenData: "2h~RiOX!pXY$hYZ$hZ]!p]^$h^p!ppq$hqr!prs$ysx!pxy&gyz'Qz}!p}!O'k!O![!p![!](U!]!b!p!b!c(o!c!d)Y!d!p!p!p!q,q!q!r0Y!r;'S!p;'S;=`$b<%lO!pR!w^bQ^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c;'S!p;'S;=`$b<%lO!pQ#xUbQOX#sZ]#s^p#sq;'S#s;'S;=`$[<%lO#sQ$_P;=`<%l#sR$eP;=`<%l!p~$mSk~XY$hYZ$h]^$hpq$h~$|VOr$yrs%cs#O$y#O#P%h#P;'S$y;'S;=`&a<%lO$y~%hOa~~%kRO;'S$y;'S;=`%t;=`O$y~%wWOr$yrs%cs#O$y#O#P%h#P;'S$y;'S;=`&a;=`<%l$y<%lO$y~&dP;=`<%l$yR&nUbQXPOX#sZ]#s^p#sq;'S#s;'S;=`$[<%lO#sR'XUbQWPOX#sZ]#s^p#sq;'S#s;'S;=`$[<%lO#sR'rUbQUPOX#sZ]#s^p#sq;'S#s;'S;=`$[<%lO#sR(]U_PbQOX#sZ]#s^p#sq;'S#s;'S;=`$[<%lO#sR(vU]PbQOX#sZ]#s^p#sq;'S#s;'S;=`$[<%lO#sR)a`bQ^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c!p!p!p!q*c!q;'S!p;'S;=`$b<%lO!pR*j`bQ^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c!f!p!f!g+l!g;'S!p;'S;=`$b<%lO!pR+u^bQdP^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c;'S!p;'S;=`$b<%lO!pR,x`bQ^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c!q!p!q!r-z!r;'S!p;'S;=`$b<%lO!pR.R`bQ^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c!v!p!v!w/T!w;'S!p;'S;=`$b<%lO!pR/^^bQUP^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c;'S!p;'S;=`$b<%lO!pR0a`bQ^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c!t!p!t!u1c!u;'S!p;'S;=`$b<%lO!pR1l^bQeP^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c;'S!p;'S;=`$b<%lO!p",
tokenizers: [0, 1],
topRules: {"Query":[0,1]},
tokenPrec: 145
})
packages/ui/src/components/editor/filter/highlight.ts
-22
View File
@@ -1,22 +0,0 @@
import { styleTags, tags as t } from "@lezer/highlight";
export const highlight = styleTags({
// Boolean operators
And: t.operatorKeyword,
Or: t.operatorKeyword,
Not: t.operatorKeyword,
// Structural punctuation
LParen: t.paren,
RParen: t.paren,
Colon: t.punctuation,
Minus: t.operator,
// Literals
Phrase: t.string, // "quoted string"
// Fields
"FieldName/At": t.attributeName,
"FieldName/Word": t.attributeName,
"FieldValue/FieldValueWord": t.attributeValue,
});
packages/ui/src/components/editor/query/extension.test.ts
+23
View File
@@ -0,0 +1,23 @@
import { describe, expect, test } from "vite-plus/test";
import { pathsAt } from "./extension";
describe("pathsAt", () => {
const at = (text: string) => pathsAt(text, text.length);
test("is the field whose value is being typed, quoted values with spaces too", () => {
expect(at("@host:api")).toEqual(["host"]);
expect(at('x @app:"Google C')).toEqual(["app"]);
expect(at('@app:"Google Chrome" ')).toEqual([]);
expect(at("@header.content-type:")).toEqual(["header.content-type"]);
});
test("is a map's name while its key is typed", () => {
expect(at("@header.x-")).toEqual(["header"]);
expect(at("@res.header.")).toEqual(["res.header"]);
});
test("is nothing outside a field", () => {
expect(at("stripe ")).toEqual([]);
expect(at("")).toEqual([]);
});
});
packages/ui/src/components/editor/query/extension.ts
+264
View File
@@ -0,0 +1,264 @@
import type { Completion, CompletionContext, CompletionResult } from "@codemirror/autocomplete";
import { autocompletion, startCompletion } from "@codemirror/autocomplete";
import { highlightingFor } from "@codemirror/language";
import type { EditorState, Extension } from "@codemirror/state";
import { RangeSetBuilder, StateEffect, StateField } from "@codemirror/state";
import type { DecorationSet, ViewUpdate } from "@codemirror/view";
import { Decoration, EditorView, ViewPlugin } from "@codemirror/view";
import type { Tag } from "@lezer/highlight";
import { tags as t } from "@lezer/highlight";
import type { Schema, TokenKind } from "@yaakapp-internal/query";
import { queryCompletions, queryTokens } from "@yaakapp-internal/query";
/** A term the search cannot run, to underline: `from` and `to` count UTF-16 units */
export interface QueryProblem {
from: number;
to: number;
/** Shown on hover, a few words */
label: string;
/** Searchable later rather than wrong: marked more quietly */
quiet?: boolean;
}
export interface QueryFilterOptions {
schema: Schema;
/** The values seen for each field path, offered after `@field:` */
values?: Record<string, string[]>;
/**
* Values for field paths `values` lacks, read as completion opens: `host` for `@host:`, and a
* map's name, `header`, for the keys after `@header.`. Null for a path with none to offer.
*/
loadValues?: (paths: string[]) => Promise<Record<string, string[]>>;
/**
* More completions for a bare word or an empty spot, after the engine's, such as saved
* searches. `typed` is the word, and the completions replace it.
*/
moreCompletions?: (typed: string) => Completion[];
/** The terms of a query the search cannot run, underlined as typing settles */
problems?: (text: string) => Promise<QueryProblem[]>;
}
const tokenTags: Record<TokenKind, Tag | null> = {
field: t.attributeName,
value: t.attributeValue,
operator: t.operatorKeyword,
quote: t.string,
paren: t.paren,
negation: t.operator,
text: null,
};
function buildDecorations(state: EditorState): DecorationSet {
const builder = new RangeSetBuilder<Decoration>();
for (const token of queryTokens(state.doc.toString())) {
const tag = tokenTags[token.kind];
const className = tag == null ? null : highlightingFor(state, [tag]);
if (className == null || token.span.start === token.span.end) continue;
builder.add(token.span.start, token.span.end, Decoration.mark({ class: className }));
}
return builder.finish();
}
const highlight = ViewPlugin.fromClass(
class {
decorations: DecorationSet;
constructor(view: EditorView) {
this.decorations = buildDecorations(view.state);
}
update(update: ViewUpdate) {
if (update.docChanged || update.viewportChanged) {
this.decorations = buildDecorations(update.state);
}
}
},
{ decorations: (v) => v.decorations },
);
/**
* The field paths whose values the term at `pos` would offer: its field's, a quoted value with
* spaces included, or its map's keys
*/
export function pathsAt(text: string, pos: number): string[] {
const before = text.slice(0, pos);
const value = /@([A-Za-z0-9_.-]+):(?:"(?:[^"\\]|\\.)*|[^\s()"]*)$/.exec(before);
if (value?.[1] != null) return [value[1].toLowerCase()];
const key = /@([A-Za-z0-9_.-]+)\.[A-Za-z0-9_-]*$/.exec(before);
if (key?.[1] != null) return [key[1].toLowerCase()];
return [];
}
/** The bare word being typed at `pos`, or empty at the start of a term */
function wordAt(text: string, pos: number): { word: string; from: number } | null {
const before = text.slice(0, pos);
const m = /(^|[\s(])([^\s()"@:-][^\s()":]*)?$/.exec(before);
if (m == null) return null;
const word = m[2] ?? "";
return { word, from: pos - word.length };
}
/** The panel beside a completion: what the field is, and a term using it */
function infoOf(info: string | null, example: string | null): Completion["info"] {
if (example == null) return undefined;
return () => {
const el = document.createElement("div");
if (info != null) {
const what = document.createElement("div");
what.textContent = info;
el.append(what);
}
const term = document.createElement("code");
term.className = "cm-query-example";
term.textContent = example;
el.append(term);
return el;
};
}
function completionSource({ schema, values, loadValues, moreCompletions }: QueryFilterOptions) {
return async (ctx: CompletionContext): Promise<CompletionResult | null> => {
const text = ctx.state.doc.toString();
let known = values ?? {};
const wanted = pathsAt(text, ctx.pos).filter((p) => known[p] == null);
if (loadValues != null && wanted.length > 0) {
known = { ...known, ...(await loadValues(wanted)) };
if (ctx.aborted) return null;
}
const result = queryCompletions(text, ctx.pos, schema, known);
const options: Completion[] = (result?.items ?? []).map((item) => ({
label: item.label,
// Not a CodeMirror type: it names the kind for `onlyFields` below, and draws no icon
type: item.kind === "field" ? "field" : "value",
detail: item.info ?? undefined,
info: infoOf(item.info, item.example),
apply: (view, _completion, from, to) => {
view.dispatch({
changes: { from, to, insert: item.insert },
selection: { anchor: from + item.insert.length },
});
if (item.reopen) startCompletion(view);
},
}));
let from = result?.span.start ?? ctx.pos;
let to = result?.span.end ?? ctx.pos;
const word = moreCompletions == null ? null : wordAt(text, ctx.pos);
// On an empty spot, only when asked for
if (moreCompletions != null && word != null && (word.word !== "" || ctx.explicit)) {
const more = moreCompletions(word.word);
if (more.length > 0) {
// Both replace the word being typed
if (options.length === 0) [from, to] = [word.from, ctx.pos];
if (from === word.from) options.push(...more);
}
}
if (options.length === 0) return null;
// Field names pop up for something typed; on an empty spot, only when asked for
const onlyFields = options.every((o) => o.type === "field");
if (onlyFields && from === to && !ctx.explicit) return null;
// The engine has already filtered and ranked the options
return { from, to, options, filter: false };
};
}
const setProblems = StateEffect.define<QueryProblem[]>();
/** Underlines what the search cannot run, as `problems` last answered for the text */
const problemField = StateField.define<DecorationSet>({
create: () => Decoration.none,
update(decorations, tr) {
let next = decorations.map(tr.changes);
for (const effect of tr.effects) {
if (!effect.is(setProblems)) continue;
const builder = new RangeSetBuilder<Decoration>();
const length = tr.state.doc.length;
for (const p of [...effect.value].sort((a, b) => a.from - b.from)) {
const from = Math.min(p.from, length);
// An empty value, `@size:`, marks the colon before it
const to = Math.min(Math.max(p.to, p.from + 1), length);
const at = p.to === p.from ? Math.max(0, from - 1) : from;
if (at >= to) continue;
const className = p.quiet ? "cm-query-quiet" : "cm-query-problem";
builder.add(at, to, Decoration.mark({ class: className, attributes: { title: p.label } }));
}
next = builder.finish();
}
return next;
},
provide: (f) => EditorView.decorations.from(f),
});
/** How long typing settles before the search's problems are asked for */
const PROBLEMS_MS = 150;
function problemChecker(problems: (text: string) => Promise<QueryProblem[]>) {
return ViewPlugin.fromClass(
class {
timer: ReturnType<typeof setTimeout> | null = null;
constructor(readonly view: EditorView) {
this.check();
}
update(update: ViewUpdate) {
if (!update.docChanged) return;
if (this.timer != null) clearTimeout(this.timer);
this.timer = setTimeout(() => this.check(), PROBLEMS_MS);
}
check() {
this.timer = null;
const text = this.view.state.doc.toString();
problems(text)
.then((found) => {
// Answered for text since changed: the next check covers it
if (this.view.state.doc.toString() !== text) return;
this.view.dispatch({ effects: setProblems.of(found) });
})
.catch(() => {});
}
destroy() {
if (this.timer != null) clearTimeout(this.timer);
}
},
);
}
const problemTheme = EditorView.baseTheme({
".cm-query-problem": {
textDecoration: "underline wavy var(--color-danger)",
textUnderlineOffset: "3px",
},
".cm-query-quiet": {
textDecoration: "underline dotted var(--color-text-subtlest, currentColor)",
textUnderlineOffset: "3px",
},
});
const completionTheme = EditorView.baseTheme({
".cm-query-example": {
display: "block",
marginTop: "0.25rem",
color: "var(--color-text)",
whiteSpace: "pre",
},
});
/** Highlighting and completion for a search box written in Yaak's query language */
export function queryFilter(options: QueryFilterOptions): Extension {
const extensions: Extension[] = [
highlight,
// No type icons: the fields and values of a search have no kind the editor's icons name
autocompletion({ override: [completionSource(options)], icons: false }),
completionTheme,
];
if (options.problems != null) {
extensions.push(problemField, problemChecker(options.problems), problemTheme);
}
return extensions;
}
packages/ui/src/editor.ts
+2 -1
View File
@@ -3,10 +3,11 @@ export { CodeEditor } from "./components/editor/CodeEditor";
export type { LanguageSupports } from "./components/editor/extensions";
export type { FieldDef, FilterOptions } from "./components/editor/filter/extension";
export {
filter,
filterAutocomplete,
makeCompletionSource as filterCompletionSource,
} from "./components/editor/filter/extension";
export type { QueryFilterOptions, QueryProblem } from "./components/editor/query/extension";
export { queryFilter } from "./components/editor/query/extension";
export {
closeBracketsExtensions,
getLanguageExtension,