diff --git a/Cargo.lock b/Cargo.lock index d18d8bebb..90c0664ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11969,6 +11969,17 @@ dependencies = [ "yaak-templates", ] +[[package]] +name = "yaak-query" +version = "0.1.0" +dependencies = [ + "serde", + "serde-wasm-bindgen", + "serde_json", + "ts-rs", + "wasm-bindgen", +] + [[package]] name = "yaak-rpc" version = "0.0.0" diff --git a/Cargo.toml b/Cargo.toml index 40557a8ea..5e29d0da6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ members = [ "crates/yaak-mcp", "crates/yaak-models", "crates/yaak-plugins", + "crates/yaak-query", "crates/yaak-sse", "crates/yaak-sync", "crates/yaak-templates", @@ -96,6 +97,7 @@ yaak-lifecycle = { path = "crates/yaak-lifecycle" } yaak-mcp = { path = "crates/yaak-mcp" } yaak-models = { path = "crates/yaak-models" } yaak-plugins = { path = "crates/yaak-plugins" } +yaak-query = { path = "crates/yaak-query" } yaak-sse = { path = "crates/yaak-sse" } yaak-sync = { path = "crates/yaak-sync" } yaak-templates = { path = "crates/yaak-templates" } diff --git a/apps/yaak-client/components/Sidebar.tsx b/apps/yaak-client/components/Sidebar.tsx index 80d474656..cb3044736 100644 --- a/apps/yaak-client/components/Sidebar.tsx +++ b/apps/yaak-client/components/Sidebar.tsx @@ -63,10 +63,9 @@ import { isSidebarFocused } from "../lib/scopes"; import { navigateToRequestOrFolderOrWorkspace } from "../lib/setWorkspaceSearchParams"; import type { ContextMenuProps, DropdownItem } from "./core/Dropdown"; import { ContextMenu, Dropdown } from "./core/Dropdown"; -import { type FieldDef, filter } from "@yaakapp-internal/ui/editor"; -import type { Ast } from "./core/Editor/filter/query"; -import { evaluate, parseQuery } from "./core/Editor/filter/query"; -import { formatFieldFilter } from "./core/Editor/filter/format"; +import type { Node as QueryNode, Schema } from "@yaakapp-internal/query"; +import { formatField, parseQuery, QueryMatcher } from "@yaakapp-internal/query"; +import { queryFilter } from "./core/Editor/filter/extension"; import { HttpMethodTag } from "./core/HttpMethodTag"; import { Icon, @@ -111,7 +110,7 @@ function Sidebar({ className }: { className?: string }) { const activeWorkspaceId = useAtomValue(activeWorkspaceAtom)?.id; const treeId = `tree.${activeWorkspaceId ?? "unknown"}`; const filterText = useAtomValue(sidebarFilterAtom); - const [tree, allFields, emptyFilterSuggestions] = useAtomValue(sidebarTreeAtom) ?? []; + const [tree, fieldValues, emptyFilterSuggestions] = useAtomValue(sidebarTreeAtom) ?? []; const wrapperRef = useRef(null); const treeRef = useRef(null); @@ -552,18 +551,18 @@ function Sidebar({ className }: { className?: string }) { const filterCompartmentMountExtRef = useRef(null); if (filterCompartmentMountExtRef.current == null) { filterCompartmentMountExtRef.current = filterLanguageCompartmentRef.current.of( - filter({ fields: allFields ?? [] }), + queryFilter({ schema: sidebarFilterSchema, values: fieldValues }), ); } useEffect(() => { const view = filterRef.current; if (!view) return; - const ext = filter({ fields: allFields ?? [] }); + const ext = queryFilter({ schema: sidebarFilterSchema, values: fieldValues }); view.dispatch({ effects: filterLanguageCompartmentRef.current.reconfigure(ext), }); - }, [allFields]); + }, [fieldValues]); if (tree == null || hidden) { return null; @@ -851,16 +850,35 @@ function setSidebarFilterText(text: string) { jotaiStore.set(sidebarFilterAtom, { text, key: `${Math.random()}` }); } -function getSidebarSuggestionValue(ast: Ast | null) { +/** The fields `getItemFields` fills in. Bare words match `getItemText` by word prefix */ +const sidebarFilterColumns = [ + "method", + "url", + "name", + "type", + "folder", + "grpc_service", + "grpc_method", +]; +const sidebarFilterSchema: Schema = { + fields: sidebarFilterColumns.map((name) => ({ name, kind: { type: "text" } })), + freeText: "fuzzy", +}; + +function getSidebarSuggestionValue(ast: QueryNode | null, query: string) { if (ast == null) return null; if (ast.type === "Term" || ast.type === "Phrase") { - const value = ast.value.trim(); + const value = ast.text.trim(); return value.length > 0 ? value : null; } if (ast.type === "Field") { - const value = ast.value.trim(); + const raw = + ast.value.type === "text" + ? ast.value.text + : query.slice(ast.valueSpan.start, ast.valueSpan.end); + const value = raw.trim(); return value.length > 0 ? value : null; } @@ -882,7 +900,7 @@ const sidebarSuggestionFieldOrder = [ ]; const sidebarTreeAtom = atom< - [TreeNode, FieldDef[], SidebarFilterSuggestion[]] | null + [TreeNode, Record, SidebarFilterSuggestion[]] | null >((get) => { const allModels = get(memoAllPotentialChildrenAtom); const activeWorkspace = get(activeWorkspaceAtom); @@ -903,15 +921,18 @@ const sidebarTreeAtom = atom< return null; } - const queryAst = parseQuery(filter.text); - const suggestionValue = getSidebarSuggestionValue(queryAst); + const suggestionValue = getSidebarSuggestionValue(parseQuery(filter.text), filter.text); + const matcher = new QueryMatcher(filter.text, sidebarFilterSchema); + + // Every request goes into one flat table (its text, then a cell per column), matched in a + // single call after the tree is built + const leaves: TreeNode[] = []; + const cells: (string | null)[] = []; - // returns true if this node OR any child matches the filter const allFields: Record> = {}; const suggestionFields = new Set(); - const build = (node: TreeNode, depth: number): boolean => { + const build = (node: TreeNode, depth: number) => { const childItems = childrenMap[node.item.id] ?? []; - let matchesSelf = true; const fields = getItemFields(node); const model = node.item.model; const isLeafNode = !(model === "folder" || model === "workspace"); @@ -929,12 +950,12 @@ const sidebarTreeAtom = atom< } } - if (queryAst != null) { - matchesSelf = isLeafNode && evaluate(queryAst, { text: getItemText(node.item), fields }); + if (isLeafNode && !matcher.isEmpty) { + leaves.push(node); + cells.push(getItemText(node.item)); + for (const column of sidebarFilterColumns) cells.push(fields[column] ?? null); } - let matchesChild = false; - // Recurse to children node.children = !isLeafNode ? [] : undefined; @@ -952,19 +973,21 @@ const sidebarTreeAtom = atom< for (const item of childItems) { const childNode = { item, parent: node, depth }; - const childMatches = build(childNode, depth + 1); - if (childMatches) { - matchesChild = true; - } + build(childNode, depth + 1); node.children.push(childNode); } } + }; - // hide node IFF nothing in its subtree matches - const anyMatch = matchesSelf || matchesChild; - node.hidden = !anyMatch; - - return anyMatch; + // Hide a node IFF nothing in its subtree matches; with no query, nothing is hidden + let matched: Set> | null = null; + const markHidden = (node: TreeNode): boolean => { + let visible = matched == null || matched.has(node); + for (const child of node.children ?? []) { + if (markHidden(child)) visible = true; + } + node.hidden = !visible; + return visible; }; const root: TreeNode = { @@ -974,15 +997,17 @@ const sidebarTreeAtom = atom< depth: 0, }; - // Build tree and mark visibility in one pass build(root, 1); + if (!matcher.isEmpty) { + const rows = matcher.matchingRows(sidebarFilterColumns, cells); + matched = new Set(Array.from(rows, (i) => leaves[i]!)); + } + matcher.free(); + markHidden(root); - const fields: FieldDef[] = []; + const fieldValues: Record = {}; for (const [name, values] of Object.entries(allFields)) { - fields.push({ - name, - values: Array.from(values).filter((v) => v.length < 20), - }); + fieldValues[name] = Array.from(values).filter((v) => v.length < 20); } const suggestions = Array.from(suggestionFields) .sort((a, b) => { @@ -993,9 +1018,9 @@ const sidebarTreeAtom = atom< }) .map((field) => ({ field, - filterText: formatFieldFilter(field, suggestionValue ?? ""), + filterText: formatField(field, suggestionValue ?? ""), })); - return [root, fields, suggestions] as const; + return [root, fieldValues, suggestions] as const; }); const sidebarGitStatusByModelIdAtom = atom>((get) => { diff --git a/apps/yaak-client/components/core/Editor/filter/extension.ts b/apps/yaak-client/components/core/Editor/filter/extension.ts new file mode 100644 index 000000000..003729fe5 --- /dev/null +++ b/apps/yaak-client/components/core/Editor/filter/extension.ts @@ -0,0 +1,88 @@ +import type { Completion, CompletionContext, CompletionResult } from "@codemirror/autocomplete"; +import { autocompletion, startCompletion } from "@codemirror/autocomplete"; +import { highlightingFor } from "@codemirror/language"; +import type { EditorState, Extension } from "@codemirror/state"; +import { RangeSetBuilder } from "@codemirror/state"; +import type { DecorationSet, ViewUpdate } from "@codemirror/view"; +import { Decoration, EditorView, ViewPlugin } from "@codemirror/view"; +import type { Tag } from "@lezer/highlight"; +import { tags as t } from "@lezer/highlight"; +import type { Schema, TokenKind } from "@yaakapp-internal/query"; +import { queryCompletions, queryTokens } from "@yaakapp-internal/query"; + +export interface QueryFilterOptions { + schema: Schema; + /** The values seen for each field path, offered after `@field:` */ + values?: Record; +} + +const tokenTags: Record = { + field: t.attributeName, + value: t.attributeValue, + operator: t.operatorKeyword, + quote: t.string, + paren: t.paren, + negation: t.operator, + text: null, +}; + +function buildDecorations(state: EditorState): DecorationSet { + const builder = new RangeSetBuilder(); + for (const token of queryTokens(state.doc.toString())) { + const tag = tokenTags[token.kind]; + const className = tag == null ? null : highlightingFor(state, [tag]); + if (className == null || token.span.start === token.span.end) continue; + builder.add(token.span.start, token.span.end, Decoration.mark({ class: className })); + } + return builder.finish(); +} + +const highlight = ViewPlugin.fromClass( + class { + decorations: DecorationSet; + + constructor(view: EditorView) { + this.decorations = buildDecorations(view.state); + } + + update(update: ViewUpdate) { + if (update.docChanged || update.viewportChanged) { + this.decorations = buildDecorations(update.state); + } + } + }, + { decorations: (v) => v.decorations }, +); + +function completionSource({ schema, values }: QueryFilterOptions) { + return (ctx: CompletionContext): CompletionResult | null => { + const result = queryCompletions(ctx.state.doc.toString(), ctx.pos, schema, values); + if (result == null) return null; + + const { start, end } = result.span; + // Field names pop up for something typed; on an empty spot, only when asked for + const onlyFields = result.items.every((item) => item.kind === "field"); + if (onlyFields && start === end && !ctx.explicit) return null; + + const options: Completion[] = result.items.map((item) => ({ + label: item.label, + type: item.kind === "field" ? "property" : "constant", + info: item.info ?? undefined, + apply: (view, _completion, from, to) => { + view.dispatch({ + changes: { from, to, insert: item.insert }, + selection: { anchor: from + item.insert.length }, + }); + if (item.reopen) startCompletion(view); + }, + })); + + // The engine has already filtered and ranked the options + return { from: start, to: end, options, filter: false }; + }; +} + +/** Highlighting and completion for a search box written in Yaak's query language */ +export function queryFilter(options: QueryFilterOptions): Extension { + return [highlight, autocompletion({ override: [completionSource(options)] })]; +} diff --git a/apps/yaak-client/components/core/Editor/filter/format.test.ts b/apps/yaak-client/components/core/Editor/filter/format.test.ts deleted file mode 100644 index 07ba7afe6..000000000 --- a/apps/yaak-client/components/core/Editor/filter/format.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { describe, expect, test } from "vite-plus/test"; -import { formatFieldFilter } from "./format"; -import { evaluate, parseQuery } from "./query"; - -function matchesFormattedUrl(value: string) { - return evaluate(parseQuery(formatFieldFilter("url", value)), { - fields: { url: value }, - }); -} - -describe("formatFieldFilter", () => { - test("keeps URL-like values bare", () => { - expect(formatFieldFilter("url", "yaak.app/foo-bar")).toBe("@url:yaak.app/foo-bar"); - expect(matchesFormattedUrl("yaak.app/foo-bar")).toBe(true); - }); - - test("keeps non-syntax punctuation bare", () => { - expect(formatFieldFilter("url", "yaa$&#*@tsrna(*)")).toBe("@url:yaa$&#*@tsrna(*)"); - expect(matchesFormattedUrl("yaa$&#*@tsrna(*)")).toBe(true); - }); - - test("keeps values that start with an operator token bare", () => { - expect(formatFieldFilter("url", "-foo")).toBe("@url:-foo"); - expect(matchesFormattedUrl("-foo")).toBe(true); - }); - - test("keeps boolean operator words bare", () => { - expect(formatFieldFilter("url", "AND")).toBe("@url:AND"); - expect(formatFieldFilter("url", "or")).toBe("@url:or"); - expect(formatFieldFilter("url", "Not")).toBe("@url:Not"); - expect(matchesFormattedUrl("AND")).toBe(true); - }); - - test("escapes quoted values", () => { - expect(formatFieldFilter("url", 'say "hi"')).toBe('@url:"say \\"hi\\""'); - expect(matchesFormattedUrl('say "hi"')).toBe(true); - }); - - test("quotes values that start with a quote", () => { - expect(formatFieldFilter("url", '"hi"')).toBe('@url:"\\"hi\\""'); - expect(matchesFormattedUrl('"hi"')).toBe(true); - }); -}); diff --git a/apps/yaak-client/components/core/Editor/filter/format.ts b/apps/yaak-client/components/core/Editor/filter/format.ts deleted file mode 100644 index f1fc7215e..000000000 --- a/apps/yaak-client/components/core/Editor/filter/format.ts +++ /dev/null @@ -1,7 +0,0 @@ -const bareFieldValue = /^[^\s"]\S*$/; - -export function formatFieldFilter(field: string, value: string) { - const escapedValue = value.replace(/\\/g, "\\\\").replace(/"/g, '\\"'); - const filterValue = bareFieldValue.test(value) ? value : `"${escapedValue}"`; - return `@${field}:${filterValue}`; -} diff --git a/apps/yaak-client/components/core/Editor/filter/query.ts b/apps/yaak-client/components/core/Editor/filter/query.ts deleted file mode 100644 index 12bf4a4bb..000000000 --- a/apps/yaak-client/components/core/Editor/filter/query.ts +++ /dev/null @@ -1,308 +0,0 @@ -// query.ts -// A tiny query language parser with NOT/AND/OR, parentheses, phrases, negation, and field:value. - -import { fuzzyMatch } from "fuzzbunny"; -///////////////////////// -// AST -///////////////////////// - -export type Ast = - | { type: "Term"; value: string } // foo - | { type: "Phrase"; value: string } // "hi there" - | { type: "Field"; field: string; value: string } // method:POST or title:"exact phrase" - | { type: "Not"; node: Ast } // -foo or NOT foo - | { type: "And"; left: Ast; right: Ast } // a AND b - | { type: "Or"; left: Ast; right: Ast }; // a OR b - -///////////////////////// -// Tokenizer -///////////////////////// -type Tok = - | { kind: "LPAREN" } - | { kind: "RPAREN" } - | { kind: "AND" } - | { kind: "OR" } - | { kind: "NOT" } // explicit NOT - | { kind: "MINUS" } // unary minus before term/phrase/paren group - | { kind: "COLON" } - | { kind: "WORD"; text: string } // bareword (unquoted) - | { kind: "PHRASE"; text: string } // "quoted phrase" - | { kind: "EOF" }; - -const isSpace = (c: string) => /\s/.test(c); -const isWordStart = (c: string) => c !== "" && !isSpace(c) && !/[():"@-]/.test(c); -const isWordChar = (c: string) => c !== "" && !isSpace(c) && !/[():"@]/.test(c); - -export function tokenize(input: string): Tok[] { - const toks: Tok[] = []; - let i = 0; - const n = input.length; - - const peek = () => input[i] ?? ""; - const advance = () => input[i++]; - - const readWord = () => { - let s = ""; - while (i < n && isWordChar(peek())) s += advance(); - return s; - }; - - const readFieldValue = () => { - let s = ""; - while (i < n && !isSpace(peek())) s += advance(); - return s; - }; - - const readPhrase = () => { - // assumes current char is opening quote - advance(); // consume opening " - let s = ""; - while (i < n) { - const c = advance(); - if (c === `"`) break; - if (c === "\\" && i < n) { - // escape \" and \\ (simple) - const next = advance(); - s += next; - } else { - s += c; - } - } - return s; - }; - - while (i < n) { - const c = peek(); - - if (isSpace(c)) { - i++; - continue; - } - - if (c === "(") { - toks.push({ kind: "LPAREN" }); - i++; - continue; - } - if (c === ")") { - toks.push({ kind: "RPAREN" }); - i++; - continue; - } - if (c === ":") { - toks.push({ kind: "COLON" }); - i++; - if (peek() && !isSpace(peek()) && peek() !== `"`) { - toks.push({ kind: "WORD", text: readFieldValue() }); - } - continue; - } - if (c === `"`) { - const text = readPhrase(); - toks.push({ kind: "PHRASE", text }); - continue; - } - if (c === "-") { - toks.push({ kind: "MINUS" }); - i++; - continue; - } - - // WORD / AND / OR / NOT - if (isWordStart(c)) { - const w = readWord(); - const upper = w.toUpperCase(); - if (upper === "AND") toks.push({ kind: "AND" }); - else if (upper === "OR") toks.push({ kind: "OR" }); - else if (upper === "NOT") toks.push({ kind: "NOT" }); - else toks.push({ kind: "WORD", text: w }); - continue; - } - - // Unknown char—skip to be forgiving - i++; - } - - toks.push({ kind: "EOF" }); - return toks; -} - -class Parser { - private i = 0; - constructor(private toks: Tok[]) {} - - private peek(): Tok { - return this.toks[this.i] ?? { kind: "EOF" }; - } - private advance(): Tok { - return this.toks[this.i++] ?? { kind: "EOF" }; - } - private at(kind: Tok["kind"]) { - return this.peek().kind === kind; - } - - // Top-level: parse OR-precedence chain, allowing implicit AND. - parse(): Ast | null { - if (this.at("EOF")) return null; - const expr = this.parseOr(); - if (!this.at("EOF")) { - // Optionally, consume remaining tokens or throw - } - return expr; - } - - // Precedence: NOT (highest), AND, OR (lowest) - private parseOr(): Ast { - let node = this.parseAnd(); - while (this.at("OR")) { - this.advance(); - const rhs = this.parseAnd(); - node = { type: "Or", left: node, right: rhs }; - } - return node; - } - - private parseAnd(): Ast { - let node = this.parseUnary(); - // Implicit AND: if next token starts a primary, treat as AND. - while (this.at("AND") || this.startsPrimary()) { - if (this.at("AND")) this.advance(); - const rhs = this.parseUnary(); - node = { type: "And", left: node, right: rhs }; - } - return node; - } - - private parseUnary(): Ast { - if (this.at("NOT") || this.at("MINUS")) { - this.advance(); - const node = this.parseUnary(); - return { type: "Not", node }; - } - return this.parsePrimaryOrField(); - } - - private startsPrimary(): boolean { - const k = this.peek().kind; - return k === "WORD" || k === "PHRASE" || k === "LPAREN" || k === "MINUS" || k === "NOT"; - } - - private parsePrimaryOrField(): Ast { - // Parenthesized group - if (this.at("LPAREN")) { - this.advance(); - const inside = this.parseOr(); - // if (!this.at('RPAREN')) throw new Error("Missing closing ')'"); - this.advance(); - return inside; - } - - // Phrase - if (this.at("PHRASE")) { - const t = this.advance() as Extract; - return { type: "Phrase", value: t.text }; - } - - // Field or bare word - if (this.at("WORD")) { - const wordTok = this.advance() as Extract; - - if (this.at("COLON")) { - // field:value or field:"phrase" - this.advance(); // : - let value: string; - if (this.at("PHRASE")) { - const p = this.advance() as Extract; - value = p.text; - } else if (this.at("WORD")) { - const w = this.advance() as Extract; - value = w.text; - } else { - // Anything else after colon is treated literally as a single Term token. - const t = this.advance(); - value = tokText(t); - } - return { type: "Field", field: wordTok.text, value }; - } - - // plain term - return { type: "Term", value: wordTok.text }; - } - - const w = this.advance() as Extract; - return { type: "Phrase", value: "text" in w ? w.text : "" }; - } -} - -function tokText(t: Tok): string { - if ("text" in t) return t.text; - - switch (t.kind) { - case "COLON": - return ":"; - case "LPAREN": - return "("; - case "RPAREN": - return ")"; - default: - return ""; - } -} - -export function parseQuery(q: string): Ast | null { - if (q.trim() === "") return null; - const toks = tokenize(q); - const parser = new Parser(toks); - return parser.parse(); -} - -export type Doc = { - text?: string; - fields?: Record; -}; - -type Technique = "substring" | "fuzzy" | "strict"; - -function includes(hay: string | undefined, needle: string, technique: Technique): boolean { - if (!hay || !needle) return false; - if (technique === "strict") return hay === needle; - if (technique === "fuzzy") return !!fuzzyMatch(hay, needle); - return hay.indexOf(needle) !== -1; -} - -export function evaluate(ast: Ast | null, doc: Doc): boolean { - if (!ast) return true; // Match everything if no query is provided - - const text = (doc.text ?? "").toLowerCase(); - const fieldsNorm: Record = {}; - - for (const [k, v] of Object.entries(doc.fields ?? {})) { - if (!(typeof v === "string" || Array.isArray(v))) continue; - fieldsNorm[k.toLowerCase()] = Array.isArray(v) - ? v.filter((v) => typeof v === "string").map((s) => s.toLowerCase()) - : [String(v ?? "").toLowerCase()]; - } - - const evalNode = (node: Ast): boolean => { - switch (node.type) { - case "Term": - return includes(text, node.value.toLowerCase(), "fuzzy"); - case "Phrase": - // Quoted phrases match exactly - return includes(text, node.value.toLowerCase(), "substring"); - case "Field": { - const vals = fieldsNorm[node.field.toLowerCase()] ?? []; - if (vals.length === 0) return false; - return vals.some((v) => includes(v, node.value.toLowerCase(), "substring")); - } - case "Not": - return !evalNode(node.node); - case "And": - return evalNode(node.left) && evalNode(node.right); - case "Or": - return evalNode(node.left) || evalNode(node.right); - } - }; - - return evalNode(ast); -} diff --git a/crates/yaak-query/Cargo.toml b/crates/yaak-query/Cargo.toml new file mode 100644 index 000000000..cd1813563 --- /dev/null +++ b/crates/yaak-query/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "yaak-query" +version = "0.1.0" +edition = "2024" +publish = false + +# The search query language shared by Yaak's sidebar and Yaak Proxy: grammar, +# parser, schema, evaluator, completion and highlighting. Pure Rust, with a +# WebAssembly build (`@yaakapp-internal/query`) for the frontends. + +[package.metadata.wasm-pack.profile.release] +wasm-opt = false # Matches yaak-templates; wasm-opt has caused errors in CI + +[lib] +crate-type = ["cdylib", "rlib"] + +[features] +default = ["wasm"] +# The `#[wasm_bindgen]` exports that make up the @yaakapp-internal/query +# package. They only exist when compiling for wasm32, so native builds and +# `cargo test` never see them. Off for crates that link this one into their own +# wasm module and do not want these re-exported from theirs. +wasm = [] + +[dependencies] +serde = { workspace = true, features = ["derive"] } + +# Everything that only exists for the WebAssembly package is target-scoped, so +# a native consumer (the proxy, the CLI) links none of it. +[target.'cfg(target_arch = "wasm32")'.dependencies] +serde-wasm-bindgen = "0.6.5" +serde_json = { workspace = true } +wasm-bindgen = { version = "0.2.100", features = ["serde-serialize"] } + +[dev-dependencies] +serde_json = { workspace = true } +ts-rs = { workspace = true } diff --git a/crates/yaak-query/bindings/query.ts b/crates/yaak-query/bindings/query.ts new file mode 100644 index 000000000..a9efad919 --- /dev/null +++ b/crates/yaak-query/bindings/query.ts @@ -0,0 +1,128 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type CompareOp = ">" | ">=" | "<" | "<="; + +export type Completion = { +/** + * What to show: a field's path, or a value as it is. + */ +label: string, +/** + * What replaces the span: `@method:`, `@header.`, or a value quoted when it needs to be. + */ +insert: string, kind: CompletionKind, info: string | null, +/** + * The term is not finished after inserting (a field still needs its value, a map field its + * key), so completion should open again. + */ +reopen: boolean, }; + +export type CompletionKind = "field" | "value"; + +/** + * Completions for the term at the cursor, all replacing `span`. + */ +export type Completions = { span: Span, items: Array, }; + +/** + * Something in a query the schema does not accept. The query still parses and runs; the + * offending term matches nothing. + */ +export type Diagnostic = { "type": "unknownField", path: string, span: Span, } | { "type": "invalidValue", path: string, span: Span, }; + +export type DocValue = boolean | number | string; + +/** + * One value or several. + */ +export type DocValues = DocValue | Array; + +/** + * What a query is matched against: free text for bare words and phrases, and field values by + * lowercase dotted path. A map field's values sit under `name.key` (`header.x-cache`). + */ +export type Document = { text?: string, fields?: { [key in string]?: DocValues }, }; + +/** + * One field: `@name:value`. + */ +export type FieldDef = { +/** + * The field's path, dotted when nested: `method`, `req.header`. + */ +name: string, kind: FieldKind, +/** + * A map of named values, such as headers: `@header.:value` is valid for any key, and + * the key is matched against the document's keys (case-insensitively). + */ +map?: boolean, +/** + * A short note shown with the field in completions. + */ +info?: string, }; + +export type FieldKind = { "type": "text" } | { "type": "number", units: Array, } | { "type": "flag" }; + +/** + * What a field term's value asks for. The shape is decided by the parser alone; the schema + * decides what it means for a given field. + */ +export type FieldValue = { "type": "text", text: string, quoted: boolean, } | { "type": "wildcard", pattern: string, } | { "type": "exists" } | { "type": "compare", op: CompareOp, number: NumberLiteral, } | { "type": "range", low: NumberLiteral, high: NumberLiteral, }; + +/** + * How a bare word matches the document's free text. A quoted phrase is always a substring. + */ +export type FreeText = "fuzzy" | "substring"; + +/** + * A node of the parsed query. + */ +export type Node = { "type": "And", children: Array, span: Span, } | { "type": "Or", children: Array, span: Span, } | { "type": "Not", node: Node, span: Span, } | { "type": "Term", text: string, span: Span, } | { "type": "Phrase", text: string, span: Span, } | { "type": "Field", +/** + * The dotted path, one entry per identifier: `@req.header.accept` is + * `["req", "header", "accept"]`. + */ +path: Array, value: FieldValue, span: Span, +/** + * `@path:`, from the `@` through the colon. + */ +pathSpan: Span, +/** + * The value as written, quotes included. Empty (and at the end of the field) when + * nothing follows the colon. + */ +valueSpan: Span, }; + +/** + * A number as written in a comparison or range, with its unit when it has one (`500ms`, + * `2.5mb`). The schema says which units a field accepts and what they are worth. + */ +export type NumberLiteral = { value: number, unit: string | null, +/** + * The number as it appears in the source, unit included. + */ +raw: string, }; + +/** + * A consumer's searchable fields. + */ +export type Schema = { fields: Array, +/** + * Substring when left out. + */ +freeText?: FreeText, }; + +/** + * A half-open byte range `start..end` in the query source. + */ +export type Span = { start: number, end: number, }; + +export type Token = { kind: TokenKind, span: Span, }; + +export type TokenKind = "field" | "value" | "operator" | "quote" | "paren" | "negation" | "text"; + +/** + * A unit a number may be written in, and how many of the field's base unit it is worth: + * `{ name: "s", factor: 1000 }` for a field counted in milliseconds. + */ +export type Unit = { name: string, factor: number, }; diff --git a/crates/yaak-query/build-wasm.cjs b/crates/yaak-query/build-wasm.cjs new file mode 100644 index 000000000..22a6b65b3 --- /dev/null +++ b/crates/yaak-query/build-wasm.cjs @@ -0,0 +1,40 @@ +const { execSync } = require("node:child_process"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); + +if (process.env.SKIP_WASM_BUILD === "1") { + console.log("Skipping wasm-pack build (SKIP_WASM_BUILD=1)"); + return; +} + +// Remap machine-specific paths that rustc embeds into the binary (panic +// location strings), so builds are reproducible across machines +const sysroot = execSync("rustc --print sysroot").toString().trim(); +const cargoHome = process.env.CARGO_HOME ?? path.join(os.homedir(), ".cargo"); + +execSync("wasm-pack build --target bundler", { + stdio: "inherit", + env: { + ...process.env, + CARGO_BUILD_TARGET: "wasm32-unknown-unknown", + RUSTFLAGS: `--remap-path-prefix=${cargoHome}=/cargo --remap-path-prefix=${sysroot}=/rustc`, + }, +}); + +// Rewrite the generated entry to use Vite's ?init import style instead of +// the ES Module Integration style that wasm-pack generates, which Vite/rolldown +// does not support in production builds. +const entry = path.join(__dirname, "pkg", "yaak_query.js"); +fs.writeFileSync( + entry, + [ + 'import init from "./yaak_query_bg.wasm?init";', + 'export * from "./yaak_query_bg.js";', + 'import * as bg from "./yaak_query_bg.js";', + 'const instance = await init({ "./yaak_query_bg.js": bg });', + "bg.__wbg_set_wasm(instance.exports);", + "instance.exports.__wbindgen_start();", + "", + ].join("\n"), +); diff --git a/crates/yaak-query/index.ts b/crates/yaak-query/index.ts new file mode 100644 index 000000000..14ca5657a --- /dev/null +++ b/crates/yaak-query/index.ts @@ -0,0 +1,85 @@ +export * from "./bindings/query"; +import type { Completions, Diagnostic, Document, Node, Schema, Token } from "./bindings/query"; +import { + QueryMatcher as WasmQueryMatcher, + parse_query, + query_completions, + query_diagnostics, + query_format_field, + query_tokens, +} from "./pkg"; + +// Every span and offset here counts UTF-16 code units, like JavaScript strings do. + +/** The parsed tree, or `null` when the query has no terms */ +export function parseQuery(input: string): Node | null { + return parse_query(input) as Node | null; +} + +/** Highlighting spans, in order */ +export function queryTokens(input: string): Token[] { + return query_tokens(input) as Token[]; +} + +/** + * What can complete the term at `cursor`: field names, or after `@field:` the values given for + * that field in `values` + */ +export function queryCompletions( + input: string, + cursor: number, + schema: Schema, + values?: Record, +): Completions | null { + return query_completions(input, cursor, schema, values ?? {}) as Completions | null; +} + +/** The terms the schema cannot run: unknown fields and values that cannot apply */ +export function queryDiagnostics(input: string, schema: Schema): Diagnostic[] { + return query_diagnostics(input, schema) as Diagnostic[]; +} + +/** `@field:value` (or `-@field:value`), with the value quoted when it needs to be */ +export function formatField(field: string, value: string, negated = false): string { + return query_format_field(field, value, negated); +} + +/** The length that marks a cell with no value (`Table::NO_VALUE` in the crate) */ +const NO_VALUE = 0xffffffff; + +/** A query parsed once, to match against many documents */ +export class QueryMatcher { + readonly #inner: WasmQueryMatcher; + + constructor(input: string, schema: Schema) { + this.#inner = new WasmQueryMatcher(input, schema); + } + + /** The query has no terms, so it matches everything */ + get isEmpty(): boolean { + return this.#inner.isEmpty; + } + + matches(doc: Document): boolean { + return this.#inner.matches(JSON.stringify(doc)); + } + + /** + * The indexes of the matching rows, for many documents in one call. `cells` is flat: for + * each row, its free text, then a value (or `null` when it has none) for each of `columns` + */ + matchingRows(columns: string[], cells: (string | null)[]): Uint32Array { + // Packed as one string plus each cell's length, which crosses into wasm far faster than + // JSON: nothing to escape or parse, and the cells are read in place + const lengths = new Uint32Array(cells.length); + for (let i = 0; i < cells.length; i++) { + lengths[i] = cells[i]?.length ?? NO_VALUE; + } + return this.#inner.matchingRows(columns, cells.join(""), lengths); + } + + /** Frees the parsed query now rather than when it is garbage collected */ + free() { + this.#inner.free(); + } +} diff --git a/crates/yaak-query/package.json b/crates/yaak-query/package.json new file mode 100644 index 000000000..da9e0b86a --- /dev/null +++ b/crates/yaak-query/package.json @@ -0,0 +1,15 @@ +{ + "name": "@yaakapp-internal/query", + "version": "1.0.0", + "private": true, + "main": "index.ts", + "scripts": { + "bootstrap": "npm run build", + "build": "run-s build:*", + "build:pack": "node build-wasm.cjs", + "build:clean": "rimraf ./pkg/.gitignore" + }, + "devDependencies": { + "rimraf": "^6.1.2" + } +} diff --git a/crates/yaak-query/pkg/package.json b/crates/yaak-query/pkg/package.json new file mode 100644 index 000000000..a51f42d53 --- /dev/null +++ b/crates/yaak-query/pkg/package.json @@ -0,0 +1,17 @@ +{ + "name": "yaak-query", + "type": "module", + "version": "0.1.0", + "files": [ + "yaak_query_bg.wasm", + "yaak_query.js", + "yaak_query_bg.js", + "yaak_query.d.ts" + ], + "main": "yaak_query.js", + "types": "yaak_query.d.ts", + "sideEffects": [ + "./yaak_query.js", + "./snippets/*" + ] +} \ No newline at end of file diff --git a/crates/yaak-query/pkg/yaak_query.d.ts b/crates/yaak-query/pkg/yaak_query.d.ts new file mode 100644 index 000000000..d326c7931 --- /dev/null +++ b/crates/yaak-query/pkg/yaak_query.d.ts @@ -0,0 +1,49 @@ +/* tslint:disable */ +/* eslint-disable */ + +/** + * A query parsed and resolved once, then matched against many documents. + */ +export class QueryMatcher { + free(): void; + [Symbol.dispose](): void; + /** + * Whether a `Document` (`{ text, fields }`), as JSON, matches. JSON crosses into wasm far + * faster than walking a JS object property by property. + */ + matches(doc_json: string): boolean; + /** + * The indexes of the matching rows, for many documents in one call. The cells (per row, + * its free text, then one per column) are packed as in [`Table::from_packed`]: joined into + * one string, with each one's UTF-16 length, or `Table::NO_VALUE` for no value. + */ + matchingRows(columns: string[], joined: string, lengths: Uint32Array): Uint32Array; + constructor(input: string, schema: any); + /** + * Whether the query has no terms, so it matches everything. + */ + readonly isEmpty: boolean; +} + +/** + * The parsed tree (`Node | null`), with UTF-16 spans. + */ +export function parse_query(input: string): any; + +/** + * `Completions | null` at UTF-16 offset `cursor`. `values` maps field paths to the values seen + * for them (`Record`). + */ +export function query_completions(input: string, cursor: number, schema: any, values: any): any; + +/** + * The terms the schema cannot run (`Diagnostic[]`), with UTF-16 spans. + */ +export function query_diagnostics(input: string, schema: any): any; + +export function query_format_field(field: string, value: string, negated: boolean): string; + +/** + * Highlighting spans (`Token[]`), in UTF-16 offsets. + */ +export function query_tokens(input: string): any; diff --git a/crates/yaak-query/pkg/yaak_query.js b/crates/yaak-query/pkg/yaak_query.js new file mode 100644 index 000000000..d3d8a23f4 --- /dev/null +++ b/crates/yaak-query/pkg/yaak_query.js @@ -0,0 +1,6 @@ +import init from "./yaak_query_bg.wasm?init"; +export * from "./yaak_query_bg.js"; +import * as bg from "./yaak_query_bg.js"; +const instance = await init({ "./yaak_query_bg.js": bg }); +bg.__wbg_set_wasm(instance.exports); +instance.exports.__wbindgen_start(); diff --git a/crates/yaak-query/pkg/yaak_query_bg.js b/crates/yaak-query/pkg/yaak_query_bg.js new file mode 100644 index 000000000..8b94229f4 --- /dev/null +++ b/crates/yaak-query/pkg/yaak_query_bg.js @@ -0,0 +1,609 @@ +/** + * A query parsed and resolved once, then matched against many documents. + */ +export class QueryMatcher { + __destroy_into_raw() { + const ptr = this.__wbg_ptr; + this.__wbg_ptr = 0; + QueryMatcherFinalization.unregister(this); + return ptr; + } + free() { + const ptr = this.__destroy_into_raw(); + wasm.__wbg_querymatcher_free(ptr, 0); + } + /** + * Whether the query has no terms, so it matches everything. + * @returns {boolean} + */ + get isEmpty() { + const ret = wasm.querymatcher_isEmpty(this.__wbg_ptr); + return ret !== 0; + } + /** + * Whether a `Document` (`{ text, fields }`), as JSON, matches. JSON crosses into wasm far + * faster than walking a JS object property by property. + * @param {string} doc_json + * @returns {boolean} + */ + matches(doc_json) { + const ptr0 = passStringToWasm0(doc_json, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.querymatcher_matches(this.__wbg_ptr, ptr0, len0); + if (ret[2]) { + throw takeFromExternrefTable0(ret[1]); + } + return ret[0] !== 0; + } + /** + * The indexes of the matching rows, for many documents in one call. The cells (per row, + * its free text, then one per column) are packed as in [`Table::from_packed`]: joined into + * one string, with each one's UTF-16 length, or `Table::NO_VALUE` for no value. + * @param {string[]} columns + * @param {string} joined + * @param {Uint32Array} lengths + * @returns {Uint32Array} + */ + matchingRows(columns, joined, lengths) { + const ptr0 = passArrayJsValueToWasm0(columns, wasm.__wbindgen_malloc); + const len0 = WASM_VECTOR_LEN; + const ptr1 = passStringToWasm0(joined, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len1 = WASM_VECTOR_LEN; + const ptr2 = passArray32ToWasm0(lengths, wasm.__wbindgen_malloc); + const len2 = WASM_VECTOR_LEN; + const ret = wasm.querymatcher_matchingRows(this.__wbg_ptr, ptr0, len0, ptr1, len1, ptr2, len2); + if (ret[3]) { + throw takeFromExternrefTable0(ret[2]); + } + var v4 = getArrayU32FromWasm0(ret[0], ret[1]).slice(); + wasm.__wbindgen_free(ret[0], ret[1] * 4, 4); + return v4; + } + /** + * @param {string} input + * @param {any} schema + */ + constructor(input, schema) { + const ptr0 = passStringToWasm0(input, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.querymatcher_new(ptr0, len0, schema); + if (ret[2]) { + throw takeFromExternrefTable0(ret[1]); + } + this.__wbg_ptr = ret[0]; + QueryMatcherFinalization.register(this, this.__wbg_ptr, this); + return this; + } +} +if (Symbol.dispose) QueryMatcher.prototype[Symbol.dispose] = QueryMatcher.prototype.free; + +/** + * The parsed tree (`Node | null`), with UTF-16 spans. + * @param {string} input + * @returns {any} + */ +export function parse_query(input) { + const ptr0 = passStringToWasm0(input, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_query(ptr0, len0); + if (ret[2]) { + throw takeFromExternrefTable0(ret[1]); + } + return takeFromExternrefTable0(ret[0]); +} + +/** + * `Completions | null` at UTF-16 offset `cursor`. `values` maps field paths to the values seen + * for them (`Record`). + * @param {string} input + * @param {number} cursor + * @param {any} schema + * @param {any} values + * @returns {any} + */ +export function query_completions(input, cursor, schema, values) { + const ptr0 = passStringToWasm0(input, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.query_completions(ptr0, len0, cursor, schema, values); + if (ret[2]) { + throw takeFromExternrefTable0(ret[1]); + } + return takeFromExternrefTable0(ret[0]); +} + +/** + * The terms the schema cannot run (`Diagnostic[]`), with UTF-16 spans. + * @param {string} input + * @param {any} schema + * @returns {any} + */ +export function query_diagnostics(input, schema) { + const ptr0 = passStringToWasm0(input, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.query_diagnostics(ptr0, len0, schema); + if (ret[2]) { + throw takeFromExternrefTable0(ret[1]); + } + return takeFromExternrefTable0(ret[0]); +} + +/** + * @param {string} field + * @param {string} value + * @param {boolean} negated + * @returns {string} + */ +export function query_format_field(field, value, negated) { + let deferred3_0; + let deferred3_1; + try { + const ptr0 = passStringToWasm0(field, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ptr1 = passStringToWasm0(value, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len1 = WASM_VECTOR_LEN; + const ret = wasm.query_format_field(ptr0, len0, ptr1, len1, negated); + deferred3_0 = ret[0]; + deferred3_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred3_0, deferred3_1, 1); + } +} + +/** + * Highlighting spans (`Token[]`), in UTF-16 offsets. + * @param {string} input + * @returns {any} + */ +export function query_tokens(input) { + const ptr0 = passStringToWasm0(input, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.query_tokens(ptr0, len0); + if (ret[2]) { + throw takeFromExternrefTable0(ret[1]); + } + return takeFromExternrefTable0(ret[0]); +} +export function __wbg_Error_bce6d499ff0a4aff(arg0, arg1) { + const ret = Error(getStringFromWasm0(arg0, arg1)); + return ret; +} +export function __wbg_String_8564e559799eccda(arg0, arg1) { + const ret = String(arg1); + const ptr1 = passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len1 = WASM_VECTOR_LEN; + getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true); + getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true); +} +export function __wbg___wbindgen_bigint_get_as_i64_410e28c7b761ad83(arg0, arg1) { + const v = arg1; + const ret = typeof(v) === 'bigint' ? v : undefined; + getDataViewMemory0().setBigInt64(arg0 + 8 * 1, isLikeNone(ret) ? BigInt(0) : ret, true); + getDataViewMemory0().setInt32(arg0 + 4 * 0, !isLikeNone(ret), true); +} +export function __wbg___wbindgen_boolean_get_2304fb8c853028c8(arg0) { + const v = arg0; + const ret = typeof(v) === 'boolean' ? v : undefined; + return isLikeNone(ret) ? 0xFFFFFF : ret ? 1 : 0; +} +export function __wbg___wbindgen_debug_string_edece8177ad01481(arg0, arg1) { + const ret = debugString(arg1); + const ptr1 = passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len1 = WASM_VECTOR_LEN; + getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true); + getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true); +} +export function __wbg___wbindgen_in_07056af4f902c445(arg0, arg1) { + const ret = arg0 in arg1; + return ret; +} +export function __wbg___wbindgen_is_bigint_aeae3893f30ed54e(arg0) { + const ret = typeof(arg0) === 'bigint'; + return ret; +} +export function __wbg___wbindgen_is_function_5cd60d5cf78b4eef(arg0) { + const ret = typeof(arg0) === 'function'; + return ret; +} +export function __wbg___wbindgen_is_null_2042690d351e14f0(arg0) { + const ret = arg0 === null; + return ret; +} +export function __wbg___wbindgen_is_object_b4593df85baada48(arg0) { + const val = arg0; + const ret = typeof(val) === 'object' && val !== null; + return ret; +} +export function __wbg___wbindgen_is_string_dde0fd9020db4434(arg0) { + const ret = typeof(arg0) === 'string'; + return ret; +} +export function __wbg___wbindgen_is_undefined_35bb9f4c7fd651d5(arg0) { + const ret = arg0 === undefined; + return ret; +} +export function __wbg___wbindgen_jsval_eq_c0ed08b3e0f393b9(arg0, arg1) { + const ret = arg0 === arg1; + return ret; +} +export function __wbg___wbindgen_jsval_loose_eq_0ad77b7717db155c(arg0, arg1) { + const ret = arg0 == arg1; + return ret; +} +export function __wbg___wbindgen_number_get_f73a1244370fcc2c(arg0, arg1) { + const obj = arg1; + const ret = typeof(obj) === 'number' ? obj : undefined; + getDataViewMemory0().setFloat64(arg0 + 8 * 1, isLikeNone(ret) ? 0 : ret, true); + getDataViewMemory0().setInt32(arg0 + 4 * 0, !isLikeNone(ret), true); +} +export function __wbg___wbindgen_string_get_d109740c0d18f4d7(arg0, arg1) { + const obj = arg1; + const ret = typeof(obj) === 'string' ? obj : undefined; + var ptr1 = isLikeNone(ret) ? 0 : passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + var len1 = WASM_VECTOR_LEN; + getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true); + getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true); +} +export function __wbg___wbindgen_throw_9c31b086c2b26051(arg0, arg1) { + throw new Error(getStringFromWasm0(arg0, arg1)); +} +export function __wbg_call_13665d9f14390edc() { return handleError(function (arg0, arg1) { + const ret = arg0.call(arg1); + return ret; +}, arguments); } +export function __wbg_done_54b8da57023b7ed2(arg0) { + const ret = arg0.done; + return ret; +} +export function __wbg_entries_564a7e8b1e54ede5(arg0) { + const ret = Object.entries(arg0); + return ret; +} +export function __wbg_get_3e9a707ab7d352eb() { return handleError(function (arg0, arg1) { + const ret = Reflect.get(arg0, arg1); + return ret; +}, arguments); } +export function __wbg_get_98fdf51d029a75eb(arg0, arg1) { + const ret = arg0[arg1 >>> 0]; + return ret; +} +export function __wbg_get_unchecked_1dfe6d05ad91d9b7(arg0, arg1) { + const ret = arg0[arg1 >>> 0]; + return ret; +} +export function __wbg_get_with_ref_key_6412cf3094599694(arg0, arg1) { + const ret = arg0[arg1]; + return ret; +} +export function __wbg_instanceof_ArrayBuffer_53db37b06f6b9afe(arg0) { + let result; + try { + result = arg0 instanceof ArrayBuffer; + } catch (_) { + result = false; + } + const ret = result; + return ret; +} +export function __wbg_instanceof_Map_16f217b9a2a08d8c(arg0) { + let result; + try { + result = arg0 instanceof Map; + } catch (_) { + result = false; + } + const ret = result; + return ret; +} +export function __wbg_instanceof_Uint8Array_abd07d4bd221d50b(arg0) { + let result; + try { + result = arg0 instanceof Uint8Array; + } catch (_) { + result = false; + } + const ret = result; + return ret; +} +export function __wbg_isArray_94898ed3aad6947b(arg0) { + const ret = Array.isArray(arg0); + return ret; +} +export function __wbg_isSafeInteger_01e964d144ad3a55(arg0) { + const ret = Number.isSafeInteger(arg0); + return ret; +} +export function __wbg_iterator_1441b47f341dc34f() { + const ret = Symbol.iterator; + return ret; +} +export function __wbg_length_2591a0f4f659a55c(arg0) { + const ret = arg0.length; + return ret; +} +export function __wbg_length_56fcd3e2b7e0299d(arg0) { + const ret = arg0.length; + return ret; +} +export function __wbg_new_02d162bc6cf02f60() { + const ret = new Object(); + return ret; +} +export function __wbg_new_310879b66b6e95e1() { + const ret = new Array(); + return ret; +} +export function __wbg_new_7ddec6de44ff8f5d(arg0) { + const ret = new Uint8Array(arg0); + return ret; +} +export function __wbg_next_2a4e19f4f5083b0f(arg0) { + const ret = arg0.next; + return ret; +} +export function __wbg_next_6429a146bf756f93() { return handleError(function (arg0) { + const ret = arg0.next(); + return ret; +}, arguments); } +export function __wbg_prototypesetcall_5f9bdc8d75e07276(arg0, arg1, arg2) { + Uint8Array.prototype.set.call(getArrayU8FromWasm0(arg0, arg1), arg2); +} +export function __wbg_set_6be42768c690e380(arg0, arg1, arg2) { + arg0[arg1] = arg2; +} +export function __wbg_set_78ea6a19f4818587(arg0, arg1, arg2) { + arg0[arg1 >>> 0] = arg2; +} +export function __wbg_value_9cc0518af87a489c(arg0) { + const ret = arg0.value; + return ret; +} +export function __wbindgen_cast_0000000000000001(arg0) { + // Cast intrinsic for `F64 -> Externref`. + const ret = arg0; + return ret; +} +export function __wbindgen_cast_0000000000000002(arg0) { + // Cast intrinsic for `I64 -> Externref`. + const ret = arg0; + return ret; +} +export function __wbindgen_cast_0000000000000003(arg0, arg1) { + // Cast intrinsic for `Ref(String) -> Externref`. + const ret = getStringFromWasm0(arg0, arg1); + return ret; +} +export function __wbindgen_cast_0000000000000004(arg0) { + // Cast intrinsic for `U64 -> Externref`. + const ret = BigInt.asUintN(64, arg0); + return ret; +} +export function __wbindgen_init_externref_table() { + const table = wasm.__wbindgen_externrefs; + const offset = table.grow(4); + table.set(0, undefined); + table.set(offset + 0, undefined); + table.set(offset + 1, null); + table.set(offset + 2, true); + table.set(offset + 3, false); +} +const QueryMatcherFinalization = (typeof FinalizationRegistry === 'undefined') + ? { register: () => {}, unregister: () => {} } + : new FinalizationRegistry(ptr => wasm.__wbg_querymatcher_free(ptr, 1)); + +function addToExternrefTable0(obj) { + const idx = wasm.__externref_table_alloc(); + wasm.__wbindgen_externrefs.set(idx, obj); + return idx; +} + +function debugString(val) { + // primitive types + const type = typeof val; + if (type == 'number' || type == 'boolean' || val == null) { + return `${val}`; + } + if (type == 'string') { + return `"${val}"`; + } + if (type == 'symbol') { + const description = val.description; + if (description == null) { + return 'Symbol'; + } else { + return `Symbol(${description})`; + } + } + if (type == 'function') { + const name = val.name; + if (typeof name == 'string' && name.length > 0) { + return `Function(${name})`; + } else { + return 'Function'; + } + } + // objects + if (Array.isArray(val)) { + const length = val.length; + let debug = '['; + if (length > 0) { + debug += debugString(val[0]); + } + for(let i = 1; i < length; i++) { + debug += ', ' + debugString(val[i]); + } + debug += ']'; + return debug; + } + // Test for built-in + const builtInMatches = /\[object ([^\]]+)\]/.exec(toString.call(val)); + let className; + if (builtInMatches && builtInMatches.length > 1) { + className = builtInMatches[1]; + } else { + // Failed to match the standard '[object ClassName]' + return toString.call(val); + } + if (className == 'Object') { + // we're a user defined class or Object + // JSON.stringify avoids problems with cycles, and is generally much + // easier than looping through ownProperties of `val`. + try { + return 'Object(' + JSON.stringify(val) + ')'; + } catch (_) { + return 'Object'; + } + } + // errors + if (val instanceof Error) { + return `${val.name}: ${val.message}\n${val.stack}`; + } + // TODO we could test for more things here, like `Set`s and `Map`s. + return className; +} + +function getArrayU32FromWasm0(ptr, len) { + ptr = ptr >>> 0; + return getUint32ArrayMemory0().subarray(ptr / 4, ptr / 4 + len); +} + +function getArrayU8FromWasm0(ptr, len) { + ptr = ptr >>> 0; + return getUint8ArrayMemory0().subarray(ptr / 1, ptr / 1 + len); +} + +let cachedDataViewMemory0 = null; +function getDataViewMemory0() { + if (cachedDataViewMemory0 === null || cachedDataViewMemory0.buffer.detached === true || (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer)) { + cachedDataViewMemory0 = new DataView(wasm.memory.buffer); + } + return cachedDataViewMemory0; +} + +function getStringFromWasm0(ptr, len) { + return decodeText(ptr >>> 0, len); +} + +let cachedUint32ArrayMemory0 = null; +function getUint32ArrayMemory0() { + if (cachedUint32ArrayMemory0 === null || cachedUint32ArrayMemory0.byteLength === 0) { + cachedUint32ArrayMemory0 = new Uint32Array(wasm.memory.buffer); + } + return cachedUint32ArrayMemory0; +} + +let cachedUint8ArrayMemory0 = null; +function getUint8ArrayMemory0() { + if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) { + cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer); + } + return cachedUint8ArrayMemory0; +} + +function handleError(f, args) { + try { + return f.apply(this, args); + } catch (e) { + const idx = addToExternrefTable0(e); + wasm.__wbindgen_exn_store(idx); + } +} + +function isLikeNone(x) { + return x === undefined || x === null; +} + +function passArray32ToWasm0(arg, malloc) { + const ptr = malloc(arg.length * 4, 4) >>> 0; + getUint32ArrayMemory0().set(arg, ptr / 4); + WASM_VECTOR_LEN = arg.length; + return ptr; +} + +function passArrayJsValueToWasm0(array, malloc) { + const ptr = malloc(array.length * 4, 4) >>> 0; + for (let i = 0; i < array.length; i++) { + const add = addToExternrefTable0(array[i]); + getDataViewMemory0().setUint32(ptr + 4 * i, add, true); + } + WASM_VECTOR_LEN = array.length; + return ptr; +} + +function passStringToWasm0(arg, malloc, realloc) { + if (realloc === undefined) { + const buf = cachedTextEncoder.encode(arg); + const ptr = malloc(buf.length, 1) >>> 0; + getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf); + WASM_VECTOR_LEN = buf.length; + return ptr; + } + + let len = arg.length; + let ptr = malloc(len, 1) >>> 0; + + const mem = getUint8ArrayMemory0(); + + let offset = 0; + + for (; offset < len; offset++) { + const code = arg.charCodeAt(offset); + if (code > 0x7F) break; + mem[ptr + offset] = code; + } + if (offset !== len) { + if (offset !== 0) { + arg = arg.slice(offset); + } + ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0; + const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len); + const ret = cachedTextEncoder.encodeInto(arg, view); + + offset += ret.written; + ptr = realloc(ptr, len, offset, 1) >>> 0; + } + + WASM_VECTOR_LEN = offset; + return ptr; +} + +function takeFromExternrefTable0(idx) { + const value = wasm.__wbindgen_externrefs.get(idx); + wasm.__externref_table_dealloc(idx); + return value; +} + +let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }); +cachedTextDecoder.decode(); +const MAX_SAFARI_DECODE_BYTES = 2146435072; +let numBytesDecoded = 0; +function decodeText(ptr, len) { + numBytesDecoded += len; + if (numBytesDecoded >= MAX_SAFARI_DECODE_BYTES) { + cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }); + cachedTextDecoder.decode(); + numBytesDecoded = len; + } + return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len)); +} + +const cachedTextEncoder = new TextEncoder(); + +if (!('encodeInto' in cachedTextEncoder)) { + cachedTextEncoder.encodeInto = function (arg, view) { + const buf = cachedTextEncoder.encode(arg); + view.set(buf); + return { + read: arg.length, + written: buf.length + }; + }; +} + +let WASM_VECTOR_LEN = 0; + + +let wasm; +export function __wbg_set_wasm(val) { + wasm = val; +} diff --git a/crates/yaak-query/pkg/yaak_query_bg.wasm b/crates/yaak-query/pkg/yaak_query_bg.wasm new file mode 100644 index 000000000..c71e7d627 Binary files /dev/null and b/crates/yaak-query/pkg/yaak_query_bg.wasm differ diff --git a/crates/yaak-query/pkg/yaak_query_bg.wasm.d.ts b/crates/yaak-query/pkg/yaak_query_bg.wasm.d.ts new file mode 100644 index 000000000..ad8a62c71 --- /dev/null +++ b/crates/yaak-query/pkg/yaak_query_bg.wasm.d.ts @@ -0,0 +1,21 @@ +/* tslint:disable */ +/* eslint-disable */ +export const memory: WebAssembly.Memory; +export const __wbg_querymatcher_free: (a: number, b: number) => void; +export const parse_query: (a: number, b: number) => [number, number, number]; +export const query_completions: (a: number, b: number, c: number, d: any, e: any) => [number, number, number]; +export const query_diagnostics: (a: number, b: number, c: any) => [number, number, number]; +export const query_format_field: (a: number, b: number, c: number, d: number, e: number) => [number, number]; +export const query_tokens: (a: number, b: number) => [number, number, number]; +export const querymatcher_isEmpty: (a: number) => number; +export const querymatcher_matches: (a: number, b: number, c: number) => [number, number, number]; +export const querymatcher_matchingRows: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => [number, number, number, number]; +export const querymatcher_new: (a: number, b: number, c: any) => [number, number, number]; +export const __wbindgen_malloc: (a: number, b: number) => number; +export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number; +export const __wbindgen_exn_store: (a: number) => void; +export const __externref_table_alloc: () => number; +export const __wbindgen_externrefs: WebAssembly.Table; +export const __externref_table_dealloc: (a: number) => void; +export const __wbindgen_free: (a: number, b: number, c: number) => void; +export const __wbindgen_start: () => void; diff --git a/crates/yaak-query/src/ast.rs b/crates/yaak-query/src/ast.rs new file mode 100644 index 000000000..0b1164154 --- /dev/null +++ b/crates/yaak-query/src/ast.rs @@ -0,0 +1,195 @@ +//! The parsed query tree. +//! +//! Every node carries the byte span of its source text, so a consumer can highlight it, point +//! at it in a diagnostic, or rewrite it in place. A consumer that compiles queries (to SQL, say) +//! walks [`Node`] directly; nothing here depends on how the query is evaluated. + +use serde::{Deserialize, Serialize}; + +/// A half-open byte range `start..end` in the query source. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub struct Span { + pub start: u32, + pub end: u32, +} + +impl Span { + pub fn new(start: usize, end: usize) -> Self { + Self { start: start as u32, end: end as u32 } + } + + pub fn start(&self) -> usize { + self.start as usize + } + + pub fn end(&self) -> usize { + self.end as usize + } + + /// The source text this span covers. + pub fn slice<'a>(&self, source: &'a str) -> &'a str { + &source[self.start()..self.end()] + } + + pub(crate) fn to(&self, other: Span) -> Span { + Span { start: self.start.min(other.start), end: self.end.max(other.end) } + } +} + +/// A node of the parsed query. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type")] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub enum Node { + /// Every child matches: terms side by side, or joined with `AND`. + And { children: Vec, span: Span }, + /// Any child matches: terms joined with `OR`. + Or { children: Vec, span: Span }, + /// The node does not match: `-term`, `-(group)`, `-@field:value`. + Not { node: Box, span: Span }, + /// A bare word, matched against the document's free text. + Term { text: String, span: Span }, + /// A double-quoted string, matched against the free text with its spaces kept. + Phrase { text: String, span: Span }, + /// `@path:value`. + Field { + /// The dotted path, one entry per identifier: `@req.header.accept` is + /// `["req", "header", "accept"]`. + path: Vec, + value: FieldValue, + span: Span, + /// `@path:`, from the `@` through the colon. + #[serde(rename = "pathSpan")] + path_span: Span, + /// The value as written, quotes included. Empty (and at the end of the field) when + /// nothing follows the colon. + #[serde(rename = "valueSpan")] + value_span: Span, + }, +} + +impl Node { + pub fn span(&self) -> Span { + match self { + Node::And { span, .. } + | Node::Or { span, .. } + | Node::Not { span, .. } + | Node::Term { span, .. } + | Node::Phrase { span, .. } + | Node::Field { span, .. } => *span, + } + } + + /// Calls `f` on this node and every node below it, parents first. + pub fn walk<'a>(&'a self, f: &mut impl FnMut(&'a Node)) { + f(self); + match self { + Node::And { children, .. } | Node::Or { children, .. } => { + for child in children { + child.walk(f); + } + } + Node::Not { node, .. } => node.walk(f), + Node::Term { .. } | Node::Phrase { .. } | Node::Field { .. } => {} + } + } + + /// Rewrites every span in the tree, for a consumer that counts positions differently (the + /// WebAssembly package hands JavaScript UTF-16 offsets). + pub fn map_spans(&mut self, f: &impl Fn(Span) -> Span) { + match self { + Node::And { children, span } | Node::Or { children, span } => { + *span = f(*span); + for child in children { + child.map_spans(f); + } + } + Node::Not { node, span } => { + *span = f(*span); + node.map_spans(f); + } + Node::Term { span, .. } | Node::Phrase { span, .. } => *span = f(*span), + Node::Field { span, path_span, value_span, .. } => { + *span = f(*span); + *path_span = f(*path_span); + *value_span = f(*value_span); + } + } + } +} + +/// What a field term's value asks for. The shape is decided by the parser alone; the schema +/// decides what it means for a given field. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub enum FieldValue { + /// Plain text, matched as a case-insensitive substring. `quoted` when it was written in + /// double quotes (escapes already removed). Empty for a trailing `@field:`. + Text { text: String, quoted: bool }, + /// A bare value containing `*`, which must match the whole field value. `*` matches any + /// run of characters, including none. + Wildcard { pattern: String }, + /// `*` alone: the field has a value. Negated, it has none. + Exists, + /// `>`, `>=`, `<` or `<=` followed by a number. + Compare { + op: CompareOp, + number: NumberLiteral, + }, + /// `[low TO high]`, inclusive. + Range { + low: NumberLiteral, + high: NumberLiteral, + }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub enum CompareOp { + #[serde(rename = ">")] + Gt, + #[serde(rename = ">=")] + Gte, + #[serde(rename = "<")] + Lt, + #[serde(rename = "<=")] + Lte, +} + +impl CompareOp { + pub fn as_str(&self) -> &'static str { + match self { + CompareOp::Gt => ">", + CompareOp::Gte => ">=", + CompareOp::Lt => "<", + CompareOp::Lte => "<=", + } + } + + pub fn test(&self, left: f64, right: f64) -> bool { + match self { + CompareOp::Gt => left > right, + CompareOp::Gte => left >= right, + CompareOp::Lt => left < right, + CompareOp::Lte => left <= right, + } + } +} + +/// A number as written in a comparison or range, with its unit when it has one (`500ms`, +/// `2.5mb`). The schema says which units a field accepts and what they are worth. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub struct NumberLiteral { + pub value: f64, + pub unit: Option, + /// The number as it appears in the source, unit included. + pub raw: String, +} diff --git a/crates/yaak-query/src/complete.rs b/crates/yaak-query/src/complete.rs new file mode 100644 index 000000000..9849c50c2 --- /dev/null +++ b/crates/yaak-query/src/complete.rs @@ -0,0 +1,378 @@ +//! What can complete the term at the cursor. + +use crate::ast::Span; +use crate::format::{format_value, is_valid_path}; +use crate::lexer::{TokKind, is_ident_char, lex, lex_quoted}; +use crate::schema::{FieldKind, Schema}; +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; + +/// Completions for the term at the cursor, all replacing `span`. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub struct Completions { + pub span: Span, + pub items: Vec, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub struct Completion { + /// What to show: a field's path, or a value as it is. + pub label: String, + /// What replaces the span: `@method:`, `@header.`, or a value quoted when it needs to be. + pub insert: String, + pub kind: CompletionKind, + pub info: Option, + /// The term is not finished after inserting (a field still needs its value, a map field its + /// key), so completion should open again. + pub reopen: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub enum CompletionKind { + Field, + Value, +} + +/// Completions at byte offset `cursor` of `input`. +/// +/// At the start of a term, or in a word made of identifier characters (with or without its +/// `@`), these are the schema's fields whose path contains what is typed, those starting with +/// it first. After `@map.` they are the keys given in `values` under the map's name. After +/// `@field:` they are the values given in `values` under the field's path that contain what is +/// typed (quotes and escapes aside); the whole value is replaced. A field written without its +/// `@` (`method:post`) gets the term with the `@`. Inside a phrase, after a `)`, or in other +/// text there are none. +pub fn completions( + input: &str, + cursor: usize, + schema: &Schema, + values: &HashMap>, +) -> Option { + let mut cursor = cursor.min(input.len()); + while !input.is_char_boundary(cursor) { + cursor -= 1; + } + + let toks = lex(input); + let Some(tok) = toks.iter().find(|t| t.span.start() < cursor && cursor <= t.span.end()) else { + // Between terms: anything may start here + let at_term_start = input[..cursor].chars().next_back().is_none_or(char::is_whitespace); + return at_term_start + .then(|| field_completions(schema, values, "", Span::new(cursor, cursor))) + .flatten(); + }; + + match &tok.kind { + TokKind::Minus | TokKind::LParen if cursor == tok.span.end() => { + field_completions(schema, values, "", Span::new(cursor, cursor)) + } + TokKind::Word(word) => { + let typed = &input[tok.span.start()..cursor]; + if let Some(fix) = missing_at(word, typed, schema) { + return Some(Completions { span: tok.span, items: vec![fix] }); + } + let partial = typed.strip_prefix('@').unwrap_or(typed); + if !partial.chars().all(|c| is_ident_char(c) || c == '.') { + return None; + } + field_completions(schema, values, partial, Span::new(tok.span.start(), cursor)) + } + TokKind::Field { path, path_span, value_span, .. } => { + if cursor < value_span.start() { + // In the path: replace it through the colon + let partial = &input[tok.span.start() + 1..cursor]; + return field_completions(schema, values, partial, *path_span); + } + value_completions(input, cursor, path, *value_span, schema, values) + } + _ => None, + } +} + +/// A word written like a field but without its `@` (`method:post`), once the cursor is past the +/// colon: the same term with the `@`, replacing the word. Words like `localhost:3000`, whose +/// part before the colon is no field, get nothing. +fn missing_at(word: &str, typed: &str, schema: &Schema) -> Option { + if word.starts_with('@') || !typed.contains(':') { + return None; + } + let (path, value) = word.split_once(':')?; + if !is_valid_path(path) { + return None; + } + let path: Vec = path.split('.').map(String::from).collect(); + schema.resolve(&path)?; + let term = format!("@{word}"); + Some(Completion { + label: term.clone(), + insert: term, + kind: CompletionKind::Field, + info: None, + // With no value yet, go on to the field's values + reopen: value.is_empty(), + }) +} + +fn field_completions( + schema: &Schema, + values: &HashMap>, + partial: &str, + span: Span, +) -> Option { + let needle = partial.to_lowercase(); + let mut items = Vec::new(); + + // Keys under a map: `@header.x-` offers `@header.x-cache:` + if let Some((map_name, key_partial)) = needle.rsplit_once('.') + && let Some(map) = schema.fields.iter().find(|f| f.map && f.name.to_lowercase() == map_name) + { + let keys = lookup(values, map_name).into_iter().flatten(); + for key in ranked(keys, key_partial) { + let path = format!("{}.{}", map.name, key); + if !is_valid_path(&path) { + continue; + } + items.push(Completion { + insert: format!("@{path}:"), + label: path, + kind: CompletionKind::Field, + info: None, + reopen: true, + }); + } + } + + for field in ranked_by(&schema.fields, &needle, |f| f.name.as_str()) { + let insert = + if field.map { format!("@{}.", field.name) } else { format!("@{}:", field.name) }; + items.push(Completion { + label: field.name.clone(), + insert, + kind: CompletionKind::Field, + info: field.info.clone(), + reopen: true, + }); + } + + (!items.is_empty()).then_some(Completions { span, items }) +} + +fn value_completions( + input: &str, + cursor: usize, + path: &[String], + value_span: Span, + schema: &Schema, + values: &HashMap>, +) -> Option { + let field = schema.resolve(path)?; + let typed = &input[value_span.start()..cursor]; + let needle = if typed.starts_with('"') { lex_quoted(typed, 0).0 } else { typed.to_string() }; + + let joined = path.join("."); + let flags = ["true".to_string(), "false".to_string()]; + let candidates: Vec<&String> = match lookup(values, &joined) { + Some(given) => given.iter().collect(), + None if field.def.kind == FieldKind::Flag => flags.iter().collect(), + None => return None, + }; + + let items: Vec = ranked(candidates, &needle.to_lowercase()) + .into_iter() + .map(|value| Completion { + label: value.clone(), + insert: format_value(value), + kind: CompletionKind::Value, + info: None, + reopen: false, + }) + .collect(); + (!items.is_empty()).then_some(Completions { span: value_span, items }) +} + +/// The values a consumer gave for `path`, matched case-insensitively. +fn lookup<'a>(values: &'a HashMap>, path: &str) -> Option<&'a Vec> { + values + .get(path) + .or_else(|| values.iter().find(|(k, _)| k.eq_ignore_ascii_case(path)).map(|(_, v)| v)) +} + +/// The strings containing `needle` (lowercase), those starting with it first, each once. +fn ranked<'a>(items: impl IntoIterator, needle: &str) -> Vec<&'a String> { + let items: Vec<&String> = items.into_iter().collect(); + let mut seen = HashSet::new(); + ranked_by(&items, needle, |s| s.as_str()) + .into_iter() + .copied() + .filter(|s| seen.insert(*s)) + .collect() +} + +fn ranked_by<'a, T>(items: &'a [T], needle: &str, key: impl Fn(&T) -> &str) -> Vec<&'a T> { + let lower = |item: &T| key(item).to_lowercase(); + let starts = items.iter().filter(|i| lower(i).starts_with(needle)); + let contains = items.iter().filter(|i| { + let l = lower(i); + !l.starts_with(needle) && l.contains(needle) + }); + starts.chain(contains).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::schema::{FieldDef, FreeText}; + + fn schema() -> Schema { + let text = |name: &str| FieldDef { + name: name.into(), + kind: FieldKind::Text, + map: false, + info: None, + }; + Schema::new( + vec![ + text("method"), + text("url"), + text("name"), + FieldDef { name: "header".into(), kind: FieldKind::Text, map: true, info: None }, + FieldDef { name: "error".into(), kind: FieldKind::Flag, map: false, info: None }, + ], + FreeText::Fuzzy, + ) + } + + fn values() -> HashMap> { + HashMap::from([ + ( + "method".to_string(), + vec!["GET".to_string(), "POST".to_string(), "PATCH".to_string()], + ), + ("name".to_string(), vec!["My Request".to_string(), "Other".to_string()]), + ( + "header".to_string(), + vec![ + "x-cache".to_string(), + "Accept".to_string(), + "bad key".to_string(), + ], + ), + ]) + } + + /// The completion at the `|` in `input`: the replaced text and the inserts. + fn at(input: &str) -> Option<(String, Vec)> { + let cursor = input.find('|').unwrap(); + let input = input.replacen('|', "", 1); + let c = completions(&input, cursor, &schema(), &values())?; + Some((c.span.slice(&input).to_string(), c.items.into_iter().map(|i| i.insert).collect())) + } + + fn inserts(list: &[&str]) -> Vec { + list.iter().map(|s| s.to_string()).collect() + } + + #[test] + fn fields_at_a_term_start() { + let all = inserts(&["@method:", "@url:", "@name:", "@header.", "@error:"]); + assert_eq!(at("|"), Some(("".into(), all.clone()))); + assert_eq!(at("a |"), Some(("".into(), all.clone()))); + assert_eq!(at("-|"), Some(("".into(), all.clone()))); + assert_eq!(at("(|"), Some(("".into(), all))); + } + + #[test] + fn fields_for_a_partial_word() { + assert_eq!(at("@met|"), Some(("@met".into(), inserts(&["@method:"])))); + assert_eq!(at("met|"), Some(("met".into(), inserts(&["@method:"])))); + // Starts-with first, then contains + assert_eq!(at("@me|"), Some(("@me".into(), inserts(&["@method:", "@name:"])))); + assert_eq!(at("-@ur|"), Some(("@ur".into(), inserts(&["@url:"])))); + assert_eq!(at("x (@na|"), Some(("@na".into(), inserts(&["@name:"])))); + assert_eq!( + at("@e|"), + Some(("@e".into(), inserts(&["@error:", "@method:", "@name:", "@header."]))) + ); + assert_eq!(at("@zz|"), None); + assert_eq!(at("localhost:30|"), None); + } + + #[test] + fn map_keys() { + assert_eq!( + at("@header.|"), + Some(("@header.".into(), inserts(&["@header.x-cache:", "@header.Accept:"]))) + ); + assert_eq!( + at("@header.ac|"), + Some(("@header.ac".into(), inserts(&["@header.Accept:", "@header.x-cache:"]))) + ); + } + + #[test] + fn values_after_the_colon() { + assert_eq!(at("@method:|"), Some(("".into(), inserts(&["GET", "POST", "PATCH"])))); + assert_eq!(at("@method:p|"), Some(("p".into(), inserts(&["POST", "PATCH"])))); + assert_eq!(at("@method:t|"), Some(("t".into(), inserts(&["GET", "POST", "PATCH"])))); + assert_eq!(at("@name:my|"), Some(("my".into(), inserts(&["\"My Request\""])))); + assert_eq!(at("@name:\"my r|"), Some(("\"my r".into(), inserts(&["\"My Request\""])))); + // The whole value is replaced, even past the cursor + assert_eq!(at("@method:p|os x"), Some(("pos".into(), inserts(&["POST", "PATCH"])))); + assert_eq!(at("@method:zz|"), None); + // Fields with no values given offer none, unknown fields neither + assert_eq!(at("@url:|"), None); + assert_eq!(at("@nope:|"), None); + // Flags offer true and false + assert_eq!(at("@error:|"), Some(("".into(), inserts(&["true", "false"])))); + } + + #[test] + fn editing_a_field_name() { + assert_eq!(at("@met|hod:post"), Some(("@method:".into(), inserts(&["@method:"])))); + } + + #[test] + fn suggests_the_at_for_a_field_written_without_it() { + assert_eq!(at("method:post|"), Some(("method:post".into(), inserts(&["@method:post"])))); + assert_eq!(at("x method:p|"), Some(("method:p".into(), inserts(&["@method:p"])))); + assert_eq!(at("-method:po|st"), Some(("method:post".into(), inserts(&["@method:post"])))); + assert_eq!(at("(METHOD:get|"), Some(("METHOD:get".into(), inserts(&["@METHOD:get"])))); + assert_eq!( + at("header.accept:json|"), + Some(("header.accept:json".into(), inserts(&["@header.accept:json"]))) + ); + // With nothing after the colon, the field's values come next + let c = completions("method:", 7, &schema(), &values()).unwrap(); + assert_eq!(c.items[0].insert, "@method:"); + assert!(c.items[0].reopen); + assert!(!completions("method:x", 8, &schema(), &values()).unwrap().items[0].reopen); + // Not a field: plain text, nothing to suggest + assert_eq!(at("localhost:3000|"), None); + assert_eq!(at("nope:x|"), None); + assert_eq!(at("http://x|"), None); + // Before the colon, the word still completes field names + assert_eq!(at("met|hod:post"), Some(("met".into(), inserts(&["@method:"])))); + } + + #[test] + fn nothing_inside_phrases_or_after_parens() { + assert_eq!(at("\"my |"), None); + assert_eq!(at("(a)|"), None); + // A word is replaced up to the cursor + assert_eq!(at("a|b"), Some(("a".into(), inserts(&["@name:", "@header."])))); + } + + #[test] + fn cursor_is_clamped() { + let c = completions("é", 1, &schema(), &values()); + assert!(c.is_none() || c.unwrap().span.end <= 2); + assert!(completions("", 99, &schema(), &values()).is_some()); + } +} diff --git a/crates/yaak-query/src/eval.rs b/crates/yaak-query/src/eval.rs new file mode 100644 index 000000000..85dd998e8 --- /dev/null +++ b/crates/yaak-query/src/eval.rs @@ -0,0 +1,1055 @@ +//! Runs a parsed query in memory, against one document or a whole table of them. + +use crate::ast::{FieldValue, Node}; +use crate::fuzzy::{FuzzyScratch, fuzzy_match_in}; +use crate::schema::{FieldKind, FreeText, Schema, parse_flag, to_base}; +use serde::{Deserialize, Serialize}; +use std::borrow::Cow; +use std::collections::HashMap; + +/// What a query is matched against: free text for bare words and phrases, and field values by +/// lowercase dotted path. A map field's values sit under `name.key` (`header.x-cache`). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub struct Document { + #[serde(default)] + #[cfg_attr(test, ts(optional, as = "Option"))] + pub text: String, + #[serde(default)] + #[cfg_attr(test, ts(optional, as = "Option>"))] + pub fields: HashMap, +} + +/// One value or several. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub enum DocValues { + One(DocValue), + Many(Vec), +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub enum DocValue { + Flag(bool), + Number(f64), + Text(String), +} + +impl Document { + pub fn new(text: impl Into) -> Self { + Self { text: text.into(), fields: HashMap::new() } + } + + /// Adds a value under `path`, keeping any already there. + pub fn with(mut self, path: &str, value: impl Into) -> Self { + let value = value.into(); + let path = path.to_lowercase(); + match self.fields.remove(&path) { + None => self.fields.insert(path, DocValues::One(value)), + Some(DocValues::One(first)) => { + self.fields.insert(path, DocValues::Many(vec![first, value])) + } + Some(DocValues::Many(mut values)) => { + values.push(value); + self.fields.insert(path, DocValues::Many(values)) + } + }; + self + } +} + +impl From<&str> for DocValue { + fn from(value: &str) -> Self { + DocValue::Text(value.to_string()) + } +} + +impl From for DocValue { + fn from(value: String) -> Self { + DocValue::Text(value) + } +} + +impl From for DocValue { + fn from(value: f64) -> Self { + DocValue::Number(value) + } +} + +impl From for DocValue { + fn from(value: bool) -> Self { + DocValue::Flag(value) + } +} + +impl DocValue { + fn view(&self) -> Val<'_> { + match self { + DocValue::Text(text) => Val::Text(text), + DocValue::Number(n) => Val::Number(*n), + DocValue::Flag(b) => Val::Flag(*b), + } + } +} + +/// Many documents with the same fields, as one flat run of cells: for each row, its free text, +/// then a text value (or none) per column. Cells may borrow from the input they were read from. +/// Matching a table is much cheaper than matching its rows one document at a time. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Table<'a> { + /// Lowercase field paths, one per column. + columns: Vec, + cells: Vec>>, +} + +impl<'a> Table<'a> { + /// An empty table whose columns hold the fields at these paths. + pub fn new(columns: impl IntoIterator>) -> Self { + let columns = columns.into_iter().map(|c| c.into().to_lowercase()).collect(); + Self { columns, cells: Vec::new() } + } + + /// A table from its cells, already flat (text first, then a cell per column, row after + /// row). `None` when the cells don't fill whole rows. + pub fn from_cells( + columns: impl IntoIterator>, + cells: Vec>>, + ) -> Option { + let table = Self { cells, ..Self::new(columns) }; + table.cells.len().is_multiple_of(table.stride()).then_some(table) + } + + /// A table from cells packed the way JavaScript can produce them cheaply: every cell joined + /// into `joined`, with each cell's length in UTF-16 code units in `lengths` (or + /// [`Table::NO_VALUE`] for a cell with no value). Cells borrow from `joined`. `None` when + /// the lengths don't add up to `joined` or the cells don't fill whole rows. + pub fn from_packed( + columns: impl IntoIterator>, + joined: &'a str, + lengths: &[u32], + ) -> Option { + let mut cells = Vec::with_capacity(lengths.len()); + let ascii = joined.is_ascii(); + let mut start = 0; + for &len in lengths { + if len == Self::NO_VALUE { + cells.push(None); + continue; + } + let end = if ascii { + start + len as usize + } else { + // Walk `len` UTF-16 units forward + let mut units = 0; + let mut end = start; + for c in joined.get(start..)?.chars() { + if units >= len as usize { + break; + } + units += c.len_utf16(); + end += c.len_utf8(); + } + if units != len as usize { + return None; + } + end + }; + cells.push(Some(Cow::Borrowed(joined.get(start..end)?))); + start = end; + } + if start != joined.len() { + return None; + } + Self::from_cells(columns, cells) + } + + /// The length [`Table::from_packed`] reads as a cell with no value. + pub const NO_VALUE: u32 = u32::MAX; + + /// Adds a row: its free text, and a value or `None` per column, in column order. Missing + /// trailing cells are `None`; extra ones are dropped. + pub fn push( + &mut self, + text: impl Into>, + cells: impl IntoIterator>>, + ) { + self.cells.push(Some(text.into())); + let mut cells = cells.into_iter(); + for _ in 0..self.columns.len() { + self.cells.push(cells.next().flatten()); + } + } + + pub fn len(&self) -> usize { + self.cells.len() / self.stride() + } + + pub fn is_empty(&self) -> bool { + self.cells.is_empty() + } + + fn stride(&self) -> usize { + self.columns.len() + 1 + } +} + +/// A field value as matching sees it, borrowed from a document or a table. +#[derive(Clone, Copy, Debug)] +enum Val<'a> { + Text(&'a str), + Number(f64), + Flag(bool), +} + +impl Val<'_> { + /// The value as lowercase text. + fn lower(&self) -> Cow<'_, str> { + match self { + Val::Text(text) => Cow::Owned(text.to_lowercase()), + Val::Number(n) => Cow::Owned(format_number(*n)), + Val::Flag(b) => Cow::Borrowed(if *b { "true" } else { "false" }), + } + } + + /// Whether the value, lowercased, contains `needle` (lowercase). + fn contains(&self, needle: &str) -> bool { + match self { + Val::Text(text) => contains_ignore_case(text, needle), + _ => self.lower().contains(needle), + } + } + + fn as_number(&self) -> Option { + match self { + Val::Number(n) => Some(*n), + Val::Text(text) => text.trim().parse().ok(), + Val::Flag(_) => None, + } + } + + fn as_flag(&self) -> Option { + match self { + Val::Flag(b) => Some(*b), + Val::Text(text) => parse_flag(text), + Val::Number(n) => Some(*n != 0.0), + } + } + + fn is_present(&self) -> bool { + match self { + Val::Text(text) => !text.is_empty(), + Val::Number(_) => true, + Val::Flag(b) => *b, + } + } +} + +/// Whether `hay`, lowercased, contains `needle` (lowercase), without copying ASCII text. +fn contains_ignore_case(hay: &str, needle: &str) -> bool { + if !(hay.is_ascii() && needle.is_ascii()) { + return hay.to_lowercase().contains(needle); + } + let (hay, needle) = (hay.as_bytes(), needle.as_bytes()); + needle.is_empty() || hay.windows(needle.len()).any(|w| w.eq_ignore_ascii_case(needle)) +} + +/// `text` lowercased into `out`, without a fresh allocation per call. +fn lowercase_into(text: &str, out: &mut String) { + out.clear(); + if text.is_ascii() { + out.push_str(text); + out.make_ascii_lowercase(); + } else { + out.push_str(&text.to_lowercase()); + } +} + +/// `200`, not `200.0`, so a number field's text matches what the user sees. +fn format_number(n: f64) -> String { + if n.fract() == 0.0 && n.abs() < 1e15 { format!("{}", n as i64) } else { n.to_string() } +} + +/// Where field values come from: a [`Document`], or a row of a [`Table`]. +trait FieldSource { + /// Whether any value under `path` (lowercase) passes `test`. With `any_key`, `path` is a + /// map field and the values of every key under it count. + fn any_value(&self, path: &str, any_key: bool, test: &mut dyn FnMut(Val) -> bool) -> bool; +} + +impl FieldSource for Document { + fn any_value(&self, path: &str, any_key: bool, test: &mut dyn FnMut(Val) -> bool) -> bool { + self.fields.iter().filter(|(key, _)| path_matches(key, path, any_key)).any(|(_, values)| { + match values { + DocValues::One(v) => test(v.view()), + DocValues::Many(vs) => vs.iter().any(|v| test(v.view())), + } + }) + } +} + +struct RowSource<'t, 'a> { + columns: &'t [String], + /// The row's cells, after its text. + cells: &'t [Option>], +} + +impl FieldSource for RowSource<'_, '_> { + fn any_value(&self, path: &str, any_key: bool, test: &mut dyn FnMut(Val) -> bool) -> bool { + self.columns.iter().zip(self.cells).any(|(column, cell)| { + path_matches(column, path, any_key) + && cell.as_deref().is_some_and(|v| test(Val::Text(v))) + }) + } +} + +/// Whether a document key is `path`, or with `any_key`, any key under it (`path.`). Paths +/// are ASCII identifiers, so comparing ASCII case-insensitively is exact. +fn path_matches(key: &str, path: &str, any_key: bool) -> bool { + let (key, path) = (key.as_bytes(), path.as_bytes()); + if key.eq_ignore_ascii_case(path) { + return true; + } + any_key + && key.len() > path.len() + 1 + && key[..path.len()].eq_ignore_ascii_case(path) + && key[path.len()] == b'.' +} + +/// Whether `doc` matches the query. `None` (an empty query) matches everything. To match many +/// documents, build a [`Matcher`] once instead. +pub fn evaluate(root: Option<&Node>, schema: &Schema, doc: &Document) -> bool { + Matcher::new(root, schema).matches(doc) +} + +/// A query resolved against a schema once (fields looked up, text lowercased), to match many +/// documents. +#[derive(Clone, Debug, PartialEq)] +pub struct Matcher { + plan: Option, + fuzzy: bool, + /// Whether any term reads the free text, which then has to be lowercased per document. + uses_text: bool, +} + +#[derive(Clone, Debug, PartialEq)] +enum Plan { + And(Vec), + Or(Vec), + Not(Box), + /// A bare word, lowercased and trimmed. + Term(String), + /// A phrase, lowercased. + Phrase(String), + Field { + /// The lowercase path the document stores the values under. + path: String, + /// A map field with no key: every key's values. + any_key: bool, + kind: FieldKind, + /// With its text and wildcard pattern lowercased. + value: FieldValue, + }, + /// A term that can match nothing: an unknown field, an empty word or phrase. + Never, +} + +/// Buffers reused from one document to the next. +#[derive(Default)] +struct Scratch { + text: String, + fuzzy: FuzzyScratch, +} + +impl Matcher { + pub fn new(root: Option<&Node>, schema: &Schema) -> Self { + let mut uses_text = false; + let plan = root.map(|root| plan_node(root, schema, &mut uses_text)); + Self { plan, fuzzy: schema.free_text == FreeText::Fuzzy, uses_text } + } + + /// Whether the query has no terms, so it matches everything. + pub fn is_empty(&self) -> bool { + self.plan.is_none() + } + + pub fn matches(&self, doc: &Document) -> bool { + self.matches_source(&doc.text, doc, &mut Scratch::default()) + } + + /// The indexes of the rows that match, in order. + pub fn matching_rows(&self, table: &Table) -> Vec { + let mut scratch = Scratch::default(); + let rows = table.cells.chunks_exact(table.stride()); + rows.enumerate() + .filter(|(_, row)| { + let text = row[0].as_deref().unwrap_or_default(); + let source = RowSource { columns: &table.columns, cells: &row[1..] }; + self.matches_source(text, &source, &mut scratch) + }) + .map(|(i, _)| i as u32) + .collect() + } + + fn matches_source(&self, text: &str, source: &impl FieldSource, scratch: &mut Scratch) -> bool { + let Some(plan) = &self.plan else { return true }; + if self.uses_text { + lowercase_into(text, &mut scratch.text); + } + let Scratch { text, fuzzy } = scratch; + self.eval(plan, text, source, fuzzy) + } + + fn eval( + &self, + plan: &Plan, + text: &str, + source: &impl FieldSource, + fuzzy: &mut FuzzyScratch, + ) -> bool { + match plan { + Plan::And(children) => children.iter().all(|c| self.eval(c, text, source, fuzzy)), + Plan::Or(children) => children.iter().any(|c| self.eval(c, text, source, fuzzy)), + Plan::Not(node) => !self.eval(node, text, source, fuzzy), + Plan::Term(term) if self.fuzzy => fuzzy_match_in(text, term, fuzzy), + Plan::Term(term) | Plan::Phrase(term) => text.contains(term.as_str()), + Plan::Field { path, any_key, kind, value } => { + let mut any = + |test: &mut dyn FnMut(Val) -> bool| source.any_value(path, *any_key, test); + match kind { + FieldKind::Text => match_text(&mut any, value), + FieldKind::Number { units } => match_number(&mut any, value, units), + FieldKind::Flag => match_flag(&mut any, value), + } + } + Plan::Never => false, + } + } +} + +fn plan_node(node: &Node, schema: &Schema, uses_text: &mut bool) -> Plan { + let mut children = + |nodes: &[Node]| nodes.iter().map(|n| plan_node(n, schema, uses_text)).collect(); + match node { + Node::And { children: nodes, .. } => Plan::And(children(nodes)), + Node::Or { children: nodes, .. } => Plan::Or(children(nodes)), + Node::Not { node, .. } => Plan::Not(Box::new(plan_node(node, schema, uses_text))), + Node::Term { text, .. } => { + let term = text.trim().to_lowercase(); + if term.is_empty() { + return Plan::Never; + } + *uses_text = true; + Plan::Term(term) + } + Node::Phrase { text, .. } => { + if text.is_empty() { + return Plan::Never; + } + *uses_text = true; + Plan::Phrase(text.to_lowercase()) + } + Node::Field { path, value, .. } => { + let Some(field) = schema.resolve(path) else { + return Plan::Never; + }; + let value = match value { + FieldValue::Text { text, quoted } => { + FieldValue::Text { text: text.to_lowercase(), quoted: *quoted } + } + FieldValue::Wildcard { pattern } => { + FieldValue::Wildcard { pattern: pattern.to_lowercase() } + } + other => other.clone(), + }; + Plan::Field { + path: field.document_path(), + any_key: field.def.map && field.key.is_none(), + kind: field.def.kind.clone(), + value, + } + } + } +} + +/// Asks whether any of a field's values passes a test. +type AnyValue<'s> = dyn FnMut(&mut dyn FnMut(Val) -> bool) -> bool + 's; + +/// `value`'s text and wildcard pattern are already lowercase (see [`plan_node`]). +fn match_text(any: &mut AnyValue, value: &FieldValue) -> bool { + match value { + FieldValue::Text { text: needle, .. } => { + !needle.is_empty() && any(&mut |v| v.contains(needle)) + } + FieldValue::Wildcard { pattern } => any(&mut |v| wildcard_match(pattern, &v.lower())), + FieldValue::Exists => any(&mut |v| v.is_present()), + // A text field holding numbers, such as a status shown as text + FieldValue::Compare { op, number } => { + number.unit.is_none() + && any(&mut |v| v.as_number().is_some_and(|n| op.test(n, number.value))) + } + FieldValue::Range { low, high } => { + low.unit.is_none() + && high.unit.is_none() + && any(&mut |v| v.as_number().is_some_and(|n| n >= low.value && n <= high.value)) + } + } +} + +fn match_number(any: &mut AnyValue, value: &FieldValue, units: &[crate::Unit]) -> bool { + match value { + FieldValue::Text { text, quoted } => { + if text.is_empty() { + return false; + } + // `@status:404` is equality; anything else matches the number as text + match (!quoted).then(|| crate::lexer::parse_number(text)).flatten() { + Some(number) => to_base(&number, units) + .is_some_and(|target| any(&mut |v| v.as_number() == Some(target))), + None => match_text(any, value), + } + } + FieldValue::Wildcard { .. } => match_text(any, value), + FieldValue::Exists => any(&mut |v| v.as_number().is_some()), + FieldValue::Compare { op, number } => to_base(number, units) + .is_some_and(|target| any(&mut |v| v.as_number().is_some_and(|n| op.test(n, target)))), + FieldValue::Range { low, high } => match (to_base(low, units), to_base(high, units)) { + (Some(low), Some(high)) => { + any(&mut |v| v.as_number().is_some_and(|n| n >= low && n <= high)) + } + _ => false, + }, + } +} + +fn match_flag(any: &mut AnyValue, value: &FieldValue) -> bool { + let mut is_set = || any(&mut |v| v.as_flag() == Some(true)); + match value { + FieldValue::Exists => is_set(), + FieldValue::Text { text, .. } => match parse_flag(text) { + Some(true) => is_set(), + Some(false) => !is_set(), + None => false, + }, + _ => false, + } +} + +/// Whether `pattern`, where `*` matches any run of characters, matches the whole of `text`. +pub(crate) fn wildcard_match(pattern: &str, text: &str) -> bool { + let mut parts = pattern.split('*'); + let first = parts.next().unwrap_or_default(); + let Some(mut rest) = text.strip_prefix(first) else { + return false; + }; + let parts: Vec<&str> = parts.collect(); + let Some((last, middle)) = parts.split_last() else { + // No `*` at all: the whole text must equal the pattern + return rest.is_empty(); + }; + for part in middle { + match rest.find(part) { + Some(i) => rest = &rest[i + part.len()..], + None => return false, + } + } + rest.len() >= last.len() && rest.ends_with(last) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::schema::{FieldDef, Unit}; + use crate::{format_field, parse}; + + fn text_field(name: &str) -> FieldDef { + FieldDef { name: name.into(), kind: FieldKind::Text, map: false, info: None } + } + + /// The fields and free-text mode of Yaak's sidebar. + fn sidebar() -> Schema { + Schema::new( + [ + "method", + "url", + "name", + "type", + "folder", + "grpc_service", + "grpc_method", + ] + .into_iter() + .map(text_field) + .collect(), + FreeText::Fuzzy, + ) + } + + fn proxy_like() -> Schema { + Schema::new( + vec![ + text_field("url"), + FieldDef { name: "header".into(), kind: FieldKind::Text, map: true, info: None }, + FieldDef { + name: "status".into(), + kind: FieldKind::Number { units: vec![] }, + map: false, + info: None, + }, + FieldDef { + name: "duration".into(), + kind: FieldKind::Number { + units: vec![ + Unit { name: "ms".into(), factor: 1.0 }, + Unit { name: "s".into(), factor: 1000.0 }, + ], + }, + map: false, + info: None, + }, + FieldDef { name: "error".into(), kind: FieldKind::Flag, map: false, info: None }, + ], + FreeText::Substring, + ) + } + + /// An HTTP request in Yaak's sidebar, built the way `getItemFields` and `getItemText` do. + fn request(method: &str, name: &str, url: &str, folder: Option<&str>) -> Document { + let mut doc = Document::new(format!("{method} {name}")) + .with("method", method.to_uppercase()) + .with("url", url) + .with("name", name) + .with("type", "http"); + if let Some(folder) = folder { + doc = doc.with("folder", folder); + } + doc + } + + fn grpc(name: &str, service: &str, method: &str) -> Document { + Document::new(name) + .with("grpc_method", method) + .with("grpc_service", service) + .with("url", "localhost:50051") + .with("name", name) + .with("type", "grpc") + } + + fn matches(query: &str, schema: &Schema, doc: &Document) -> bool { + evaluate(parse(query).as_ref(), schema, doc) + } + + fn sidebar_matches(query: &str, doc: &Document) -> bool { + matches(query, &sidebar(), doc) + } + + #[test] + fn empty_query_matches_everything() { + assert!(sidebar_matches("", &Document::default())); + assert!(sidebar_matches(" ", &Document::default())); + } + + #[test] + fn bare_words_are_fuzzy_in_the_sidebar() { + let doc = request("GET", "My Request", "https://yaak.app", None); + assert!(sidebar_matches("my", &doc)); + assert!(sidebar_matches("REQ", &doc)); + assert!(sidebar_matches("mr", &doc)); + assert!(sidebar_matches("get", &doc)); + assert!(!sidebar_matches("rq", &doc)); + // Bare words match the free text only, not the URL + assert!(!sidebar_matches("yaak", &doc)); + } + + #[test] + fn bare_words_are_substrings_in_substring_mode() { + let doc = Document::new("GET /v1/charges 200"); + let schema = proxy_like(); + assert!(matches("rges", &schema, &doc)); + assert!(matches("v1 charges", &schema, &doc)); + assert!(!matches("gv", &schema, &doc)); + } + + #[test] + fn phrases_are_case_insensitive_substrings() { + let doc = request("GET", "My Request", "", None); + assert!(sidebar_matches("\"my req\"", &doc)); + assert!(sidebar_matches("\"MY REQ\"", &doc)); + assert!(!sidebar_matches("\"mr\"", &doc)); + assert!(!sidebar_matches("\"\"", &doc)); + } + + #[test] + fn fields() { + let post = request("post", "Create user", "https://api.yaak.app/users", Some("Users")); + let get = request("get", "List users", "https://api.yaak.app/users", Some("Admin")); + let rpc = grpc("Say hello", "helloworld.Greeter", "SayHello"); + + assert!(sidebar_matches("@method:post", &post)); + assert!(sidebar_matches("@METHOD:Po", &post)); + assert!(!sidebar_matches("@method:post", &get)); + assert!(sidebar_matches("@url:yaak.app/users", &post)); + assert!(sidebar_matches("@folder:users", &post)); + assert!(!sidebar_matches("@folder:users", &get)); + assert!(sidebar_matches("@folder:users OR @folder:admin", &get)); + assert!(sidebar_matches("@type:grpc", &rpc)); + assert!(!sidebar_matches("-@type:grpc", &rpc)); + assert!(sidebar_matches("-@type:grpc", &post)); + assert!(sidebar_matches("@grpc_service:greeter @grpc_method:hello", &rpc)); + // A field the document lacks matches nothing, and its negation everything + assert!(!sidebar_matches("@grpc_method:hello", &post)); + assert!(sidebar_matches("-@grpc_method:hello", &post)); + // `@` is required: without it, the term is free text + assert!(!sidebar_matches("method:post", &post)); + assert!(sidebar_matches("localhost:50051", &Document::new("localhost:50051"))); + // An unknown field matches nothing + assert!(!sidebar_matches("@nope:post", &post)); + // A trailing field with no value yet matches nothing, as before + assert!(!sidebar_matches("@method:", &post)); + } + + #[test] + fn groups_and_negation() { + let a = Document::new("alpha"); + let b = Document::new("beta"); + let ac = Document::new("alpha charlie"); + assert!(sidebar_matches("(alpha OR beta) -charlie", &a)); + assert!(sidebar_matches("(alpha OR beta) -charlie", &b)); + assert!(!sidebar_matches("(alpha OR beta) -charlie", &ac)); + assert!(sidebar_matches("-(alpha OR beta)", &Document::new("delta"))); + assert!(sidebar_matches("alpha charlie", &ac)); + assert!(!sidebar_matches("alpha charlie", &a)); + } + + #[test] + fn wildcards() { + assert!(wildcard_match("*.github.com", "api.github.com")); + assert!(!wildcard_match("*.github.com", "github.com")); + assert!(wildcard_match("a*", "a")); + assert!(wildcard_match("*", "")); + assert!(wildcard_match("a*b*c", "aXXbYYc")); + assert!(!wildcard_match("a*b*c", "aXXbYYcd")); + assert!(wildcard_match("a**c", "ac")); + assert!(!wildcard_match("ab*ba", "aba")); + + let doc = Document::default().with("url", "https://API.github.com/repos"); + let schema = proxy_like(); + assert!(matches("@url:https://api.github.com/*", &schema, &doc)); + assert!(matches("@url:*GITHUB*", &schema, &doc)); + assert!(!matches("@url:github*", &schema, &doc)); + // Quoted, `*` is literal + assert!(!matches("@url:\"*github*\"", &schema, &doc)); + assert!(matches("@url:\"*\"", &schema, &Document::default().with("url", "a*b"))); + } + + #[test] + fn presence() { + let schema = proxy_like(); + let with = Document::default().with("header.x-cache", "HIT").with("status", 200.0); + let without = Document::default().with("url", "x").with("status", 200.0); + assert!(matches("@header.x-cache:*", &schema, &with)); + assert!(!matches("@header.x-cache:*", &schema, &without)); + assert!(matches("-@header.x-cache:*", &schema, &without)); + assert!(matches("@status:*", &schema, &with)); + assert!(!matches("@url:*", &schema, &Document::default().with("url", ""))); + } + + #[test] + fn map_fields() { + let schema = proxy_like(); + let doc = Document::default() + .with("header.content-type", "application/json") + .with("header.x-cache", "HIT"); + assert!(matches("@header.content-type:json", &schema, &doc)); + assert!(matches("@header.Content-Type:JSON", &schema, &doc)); + assert!(!matches("@header.content-type:xml", &schema, &doc)); + assert!(!matches("@header.accept:json", &schema, &doc)); + // The map with no key looks at every key + assert!(matches("@header:hit", &schema, &doc)); + assert!(matches("@header:*", &schema, &doc)); + } + + #[test] + fn numbers() { + let schema = proxy_like(); + let doc = Document::default().with("status", 404.0).with("duration", 1500.0); + assert!(matches("@status:404", &schema, &doc)); + assert!(!matches("@status:40", &schema, &doc)); + assert!(matches("@status:40*", &schema, &doc)); + assert!(matches("@status:>=400", &schema, &doc)); + assert!(!matches("@status:<400", &schema, &doc)); + assert!(matches("@status:[400 TO 499]", &schema, &doc)); + assert!(matches("@status:[404 TO 404]", &schema, &doc)); + assert!(!matches("@status:[200 TO 299]", &schema, &doc)); + assert!(matches("@duration:>1s", &schema, &doc)); + assert!(matches("@duration:>1000ms", &schema, &doc)); + assert!(matches("@duration:1.5s", &schema, &doc)); + assert!(matches("@duration:[1s TO 2s]", &schema, &doc)); + assert!(!matches("@duration:>2s", &schema, &doc)); + // A unit the field does not take matches nothing + assert!(!matches("@duration:>1m", &schema, &doc)); + assert!(!matches("@status:>1ms", &schema, &doc)); + } + + #[test] + fn comparisons_on_text_fields() { + let doc = Document::default().with("url", "404"); + assert!(matches("@url:>400", &proxy_like(), &doc)); + assert!(!matches("@url:>400", &proxy_like(), &Document::default().with("url", "x"))); + } + + #[test] + fn flags() { + let schema = proxy_like(); + let failed = Document::default().with("error", true); + let ok = Document::default().with("error", false); + assert!(matches("@error:*", &schema, &failed)); + assert!(!matches("@error:*", &schema, &ok)); + assert!(matches("@error:true", &schema, &failed)); + assert!(matches("@error:false", &schema, &ok)); + assert!(matches("@error:false", &schema, &Document::default())); + assert!(!matches("@error:maybe", &schema, &failed)); + } + + #[test] + fn multiple_values() { + let doc = Document::default().with("url", "a").with("url", "b"); + assert!(matches("@url:b", &proxy_like(), &doc)); + } + + /// A table gives the same answers as matching its rows one document at a time. + #[test] + fn tables_match_like_documents() { + let columns = [ + "method", + "url", + "name", + "type", + "folder", + "grpc_service", + "grpc_method", + ]; + let docs = [ + request("post", "Create user", "https://api.yaak.app/users", Some("Users")), + request("get", "List users", "https://api.yaak.app/users", Some("Admin")), + request("GET", "My Request", "https://yaak.app", None), + grpc("Say hello", "helloworld.Greeter", "SayHello"), + Document::new("Folder"), + ]; + let mut table = Table::new(columns); + for doc in &docs { + let cells = columns.iter().map(|c| match doc.fields.get(*c) { + Some(DocValues::One(DocValue::Text(v))) => Some(Cow::Borrowed(v.as_str())), + _ => None, + }); + table.push(doc.text.as_str(), cells); + } + assert_eq!(table.len(), docs.len()); + + for query in [ + "", + "user", + "mr", + "\"my req\"", + "@method:post", + "-@type:grpc", + "@folder:users OR @folder:admin", + "(users OR hello) -create", + "@grpc_method:*", + "-@folder:*", + "@url:*yaak.app*", + "@nope:x", + "-@nope:x", + "method:post", + ] { + let matcher = Matcher::new(parse(query).as_ref(), &sidebar()); + let expected: Vec = (0..docs.len() as u32) + .filter(|&i| sidebar_matches(query, &docs[i as usize])) + .collect(); + assert_eq!(matcher.matching_rows(&table), expected, "{query:?}"); + } + assert!(Matcher::new(None, &sidebar()).is_empty()); + } + + #[test] + fn table_cells_line_up_with_columns() { + let mut table = Table::new(["Method", "url"]); + table.push("a", [Some("GET".into())]); + table.push("b", [None, Some("x".into()), Some("ignored".into())]); + assert_eq!(table.len(), 2); + let rows = |q: &str| Matcher::new(parse(q).as_ref(), &sidebar()).matching_rows(&table); + assert_eq!(rows("@method:get"), vec![0]); + assert_eq!(rows("@url:x"), vec![1]); + assert_eq!(rows("-@url:*"), vec![0]); + assert_eq!(rows("b"), vec![1]); + } + + #[test] + fn tables_from_flat_cells() { + let cells = vec![ + Some("GET List".into()), + Some("GET".into()), + None, + Some("POST".into()), + ]; + let table = Table::from_cells(["method"], cells.clone()).unwrap(); + assert_eq!(table.len(), 2); + let rows = |q: &str| Matcher::new(parse(q).as_ref(), &sidebar()).matching_rows(&table); + assert_eq!(rows("@method:post"), vec![1]); + assert_eq!(rows("list"), vec![0]); + // Cells that don't fill whole rows + assert!(Table::from_cells(["method", "url"], cells).is_none()); + assert!(Table::from_cells(["method"], Vec::new()).unwrap().is_empty()); + } + + /// Packs cells the way the JavaScript side does: joined, with UTF-16 lengths. + fn pack(cells: &[Option<&str>]) -> (String, Vec) { + let joined = cells.iter().flatten().copied().collect(); + let lengths = cells + .iter() + .map(|c| c.map_or(Table::NO_VALUE, |c| c.encode_utf16().count() as u32)) + .collect(); + (joined, lengths) + } + + #[test] + fn tables_from_packed_cells() { + let rows = |table: &Table, q: &str| { + Matcher::new(parse(q).as_ref(), &sidebar()).matching_rows(table) + }; + + let (joined, lengths) = pack(&[ + Some("GET List"), + Some("GET"), + Some(""), + None, + Some("POST"), + Some("x"), + ]); + let table = Table::from_packed(["method", "url"], &joined, &lengths).unwrap(); + assert_eq!(table.len(), 2); + assert_eq!(rows(&table, "@method:post"), vec![1]); + assert_eq!(rows(&table, "@url:x"), vec![1]); + assert_eq!(rows(&table, "list"), vec![0]); + assert_eq!(rows(&table, "-@method:*"), Vec::::new()); + + // Non-ASCII text, including a character outside the BMP (two UTF-16 units) + let (joined, lengths) = pack(&[ + Some("Über 🦀 crab"), + Some("GET"), + Some("Café"), + Some("x"), + None, + Some("y"), + ]); + let table = Table::from_packed(["method", "name"], &joined, &lengths).unwrap(); + assert_eq!(rows(&table, "🦀"), vec![0]); + assert_eq!(rows(&table, "@name:café"), vec![0]); + assert_eq!(rows(&table, "@name:y"), vec![1]); + + // Lengths that don't add up, or don't fill whole rows + assert!(Table::from_packed(["method"], "abc", &[1, 1]).is_none()); + assert!(Table::from_packed(["method"], "abc", &[2, 2]).is_none()); + assert!(Table::from_packed(["method"], "abc", &[3]).is_none()); + assert!(Table::from_packed(["method"], "é", &[2, 0]).is_none()); + assert!(Table::from_packed(["method"], "", &[]).unwrap().is_empty()); + } + + #[test] + fn non_ascii_text_matches_case_insensitively() { + let doc = Document::new("ÜBER Café").with("name", "ÜBER Café"); + assert!(sidebar_matches("über", &doc)); + assert!(sidebar_matches("@name:café", &doc)); + assert!(sidebar_matches("\"über c\"", &doc)); + assert!(!sidebar_matches("@name:cafe", &doc)); + } + + #[test] + fn document_from_json() { + let doc: Document = serde_json::from_str( + r#"{ "text": "x", "fields": { "url": "a", "tags": ["b", "c"], "status": 200, "error": true } }"#, + ) + .unwrap(); + assert_eq!(doc.fields["url"], DocValues::One(DocValue::Text("a".into()))); + assert_eq!(doc.fields["status"], DocValues::One(DocValue::Number(200.0))); + assert_eq!(doc.fields["error"], DocValues::One(DocValue::Flag(true))); + assert!(matches("@status:200 @error:true", &proxy_like(), &doc)); + } + + // The cases from Yaak's former `format.test.ts`: a formatted field filter matches the value + // it was made from. Values that contain query syntax are now quoted so they stay literal. + + fn matches_formatted_url(value: &str) -> bool { + matches( + &format_field("url", value, false), + &sidebar(), + &Document::default().with("url", value), + ) + } + + #[test] + fn format_keeps_url_like_values_bare() { + assert_eq!(format_field("url", "yaak.app/foo-bar", false), "@url:yaak.app/foo-bar"); + assert!(matches_formatted_url("yaak.app/foo-bar")); + } + + #[test] + fn format_quotes_values_with_query_syntax() { + // `*` is a wildcard and `)` closes a group, so this one is quoted now + assert_eq!(format_field("url", "yaa$&#*@tsrna(*)", false), "@url:\"yaa$&#*@tsrna(*)\""); + assert!(matches_formatted_url("yaa$&#*@tsrna(*)")); + assert_eq!(format_field("url", "yaa$&#@tsrna(", false), "@url:yaa$&#@tsrna("); + assert!(matches_formatted_url("yaa$&#@tsrna(")); + } + + #[test] + fn format_keeps_values_that_start_with_an_operator_token_bare() { + assert_eq!(format_field("url", "-foo", false), "@url:-foo"); + assert!(matches_formatted_url("-foo")); + } + + #[test] + fn format_keeps_boolean_operator_words_bare() { + assert_eq!(format_field("url", "AND", false), "@url:AND"); + assert_eq!(format_field("url", "or", false), "@url:or"); + assert_eq!(format_field("url", "Not", false), "@url:Not"); + assert!(matches_formatted_url("AND")); + } + + #[test] + fn format_escapes_quoted_values() { + assert_eq!(format_field("url", "say \"hi\"", false), "@url:\"say \\\"hi\\\"\""); + assert!(matches_formatted_url("say \"hi\"")); + } + + #[test] + fn format_quotes_values_that_start_with_a_quote() { + assert_eq!(format_field("url", "\"hi\"", false), "@url:\"\\\"hi\\\"\""); + assert!(matches_formatted_url("\"hi\"")); + } + + #[test] + fn formatted_values_round_trip() { + for value in [ + "a b", + "*", + ">5", + "[1 TO 2]", + "x)", + "back\\slash", + "\\\"", + "tab\there", + "é ü", + "a*b", + "<=3ms", + "\\", + ] { + assert!(matches_formatted_url(value), "{value:?} did not round-trip"); + } + } +} diff --git a/crates/yaak-query/src/format.rs b/crates/yaak-query/src/format.rs new file mode 100644 index 000000000..a6cc2d8ba --- /dev/null +++ b/crates/yaak-query/src/format.rs @@ -0,0 +1,89 @@ +//! Writes field terms back out as query text. + +use crate::ast::FieldValue; +use crate::lexer::{classify_bare, is_ident_char}; + +/// A term matching `value` in `field`: `@field:value`, or `-@field:value` when `negated`. The +/// value stays bare when it would read back as the same literal text, and is quoted (with `"` +/// and `\` escaped) when it holds whitespace, a quote, a `)`, a `*`, or the shape of a +/// comparison or range. +pub fn format_field(field: &str, value: &str, negated: bool) -> String { + let sign = if negated { "-" } else { "" }; + format!("{sign}@{field}:{}", format_value(value)) +} + +/// A field value as it should be written after the colon. +pub fn format_value(value: &str) -> String { + if is_bare_value(value) { value.to_string() } else { quote(value) } +} + +/// `value` in double quotes, with `"` and `\` escaped. +pub fn quote(value: &str) -> String { + let mut out = String::with_capacity(value.len() + 2); + out.push('"'); + for c in value.chars() { + if matches!(c, '"' | '\\') { + out.push('\\'); + } + out.push(c); + } + out.push('"'); + out +} + +fn is_bare_value(value: &str) -> bool { + let Some(first) = value.chars().next() else { + return false; + }; + if first == '"' || first == '[' { + return false; + } + if value.chars().any(|c| c.is_whitespace() || c == ')') { + return false; + } + matches!(classify_bare(value), FieldValue::Text { .. }) +} + +/// Whether `name` can be written as a field path: identifiers joined by dots. +pub fn is_valid_path(name: &str) -> bool { + !name.is_empty() + && name.split('.').all(|part| !part.is_empty() && part.chars().all(is_ident_char)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bare_or_quoted() { + assert_eq!(format_value("post"), "post"); + assert_eq!(format_value("a b"), "\"a b\""); + assert_eq!(format_value(""), "\"\""); + assert_eq!(format_value("*"), "\"*\""); + assert_eq!(format_value("a*"), "\"a*\""); + assert_eq!(format_value(">5"), "\">5\""); + assert_eq!(format_value(">x"), ">x"); + assert_eq!(format_value("[1 TO 2]"), "\"[1 TO 2]\""); + assert_eq!(format_value("[x"), "\"[x\""); + assert_eq!(format_value("a)"), "\"a)\""); + assert_eq!(format_value("a(b"), "a(b"); + assert_eq!(format_value("a\"b"), "a\"b"); + assert_eq!(format_value("a\\b"), "a\\b"); + assert_eq!(format_value("x\\ y"), "\"x\\\\ y\""); + } + + #[test] + fn negated() { + assert_eq!(format_field("type", "grpc", true), "-@type:grpc"); + assert_eq!(format_field("name", "My Request", true), "-@name:\"My Request\""); + } + + #[test] + fn paths() { + assert!(is_valid_path("method")); + assert!(is_valid_path("req.header.x-cache")); + assert!(!is_valid_path("")); + assert!(!is_valid_path("a..b")); + assert!(!is_valid_path("a b")); + } +} diff --git a/crates/yaak-query/src/fuzzy.rs b/crates/yaak-query/src/fuzzy.rs new file mode 100644 index 000000000..153606cc8 --- /dev/null +++ b/crates/yaak-query/src/fuzzy.rs @@ -0,0 +1,206 @@ +//! Word-prefix matching, ported from fuzzbunny's `fuzzyMatchSanitized` so Yaak's sidebar keeps +//! the behaviour it had when it called fuzzbunny directly. +//! +//! A search matches when it is a substring of the target. Failing that, it matches when it can +//! be split into runs that each start a word or sit at a punctuation mark, in order: `mr` finds +//! "My Request", `usam` finds "United States of America", but `rq` does not find "request". +//! Spaces in either string never break a run. +//! +//! Only whether it matches is kept; fuzzbunny's scores and highlight ranges are not used. + +/// Whether `search` matches `target`, both already lowercased and trimmed. +pub fn fuzzy_match(target: &str, search: &str) -> bool { + fuzzy_match_in(target, search, &mut FuzzyScratch::default()) +} + +/// Buffers [`fuzzy_match_in`] reuses from one call to the next. +#[derive(Default)] +pub(crate) struct FuzzyScratch { + target: Vec, + search: Vec, + skips: Vec, +} + +/// [`fuzzy_match`], reusing `scratch` instead of allocating per call. ASCII text, the usual case, +/// is matched byte by byte without copying. +pub(crate) fn fuzzy_match_in(target: &str, search: &str, scratch: &mut FuzzyScratch) -> bool { + if target.is_empty() { + return false; + } + if search.is_empty() { + return true; + } + + // A search starting with a quote wants a substring only; the closing quote is optional + let quoted = search.starts_with('"'); + let search = if quoted { + let inner = &search[1..]; + inner.strip_suffix('"').unwrap_or(inner) + } else { + search + }; + + if target.contains(search) { + return true; + } + + // One character that is not a substring can't match fuzzily either + let mut chars = search.chars(); + if quoted || (chars.next().is_some() && chars.next().is_none()) { + return false; + } + + let skips = &mut scratch.skips; + if target.is_ascii() && search.is_ascii() { + return word_prefix_match(target.as_bytes(), search.as_bytes(), skips); + } + scratch.target.clear(); + scratch.target.extend(target.chars()); + scratch.search.clear(); + scratch.search.extend(search.chars()); + word_prefix_match(&scratch.target, &scratch.search, skips) +} + +/// A unit of text the matcher walks: a byte of ASCII text, or a char. +trait Unit: Copy + PartialEq { + /// fuzzbunny counts lowercase letters, digits, and every char code from 127 up. + fn is_alpha_num(self) -> bool; + /// ASCII punctuation: `!` to `/`, `:` to `@`, `[` to `` ` ``, `{` to `~`. + fn is_punctuation(self) -> bool; + fn is_space(self) -> bool; +} + +impl Unit for char { + fn is_alpha_num(self) -> bool { + self.is_ascii_lowercase() || self.is_ascii_digit() || self as u32 >= 127 + } + + fn is_punctuation(self) -> bool { + matches!(self, '!'..='/' | ':'..='@' | '['..='`' | '{'..='~') + } + + fn is_space(self) -> bool { + self == ' ' + } +} + +impl Unit for u8 { + fn is_alpha_num(self) -> bool { + (self as char).is_alpha_num() + } + + fn is_punctuation(self) -> bool { + (self as char).is_punctuation() + } + + fn is_space(self) -> bool { + self == b' ' + } +} + +/// The word-prefix fallback, once `search` is known not to be a plain substring. +fn word_prefix_match(target: &[T], search: &[T], skips: &mut Vec) -> bool { + target_skips(target, skips); + (0..skips.len() - 1) + .any(|skip| target[skips[skip]] == search[0] && prefix_match(skip, search, target, skips)) +} + +/// The indices where a word starts or a punctuation mark sits, then the target's length, so +/// every run lies between `skips[i]` and `skips[i + 1]`. +fn target_skips(target: &[T], skips: &mut Vec) { + skips.clear(); + let mut was_alpha_num = false; + for (i, &c) in target.iter().enumerate() { + let alpha_num = c.is_alpha_num(); + if (alpha_num && !was_alpha_num) || c.is_punctuation() { + skips.push(i); + } + was_alpha_num = alpha_num; + } + skips.push(target.len()); +} + +fn prefix_match(mut skip: usize, search: &[T], target: &[T], skips: &[usize]) -> bool { + let mut search_idx = 0; + while skip < skips.len() - 1 { + let mut target_idx = skips[skip]; + let end = skips[skip + 1]; + while target_idx < end && search_idx < search.len() { + let t = target[target_idx]; + let s = search[search_idx]; + if t == s { + target_idx += 1; + search_idx += 1; + } else if t.is_space() { + target_idx += 1; + } else if s.is_space() { + search_idx += 1; + } else { + break; + } + } + if search_idx == search.len() { + return true; + } + skip += 1; + } + false +} + +#[cfg(test)] +mod tests { + use super::fuzzy_match; + + fn m(target: &str, search: &str) -> bool { + fuzzy_match(&target.to_lowercase(), &search.trim().to_lowercase()) + } + + #[test] + fn substring_first() { + assert!(m("My Request", "req")); + assert!(m("My Request", "quest")); + assert!(m("/v1/charges", "rges")); + assert!(m("abc", "abc")); + } + + #[test] + fn word_prefixes() { + assert!(m("My Request", "mr")); + assert!(m("My Request", "myreq")); + assert!(m("United States of America", "usam")); + assert!(m("api.github.com", "apigi")); + // Each run must continue from the start of a word: "gh" is not a prefix of "github" + assert!(!m("api.github.com", "apigh")); + assert!(m("POST Create User", "postcu")); + assert!(m("get_user_by_id", "gubi")); + } + + #[test] + fn not_any_subsequence() { + assert!(!m("My Request", "rq")); + assert!(!m("request", "rqst")); + assert!(!m("My Request", "mx")); + assert!(!m("abc", "x")); + } + + #[test] + fn spaces_do_not_break_a_run() { + assert!(m("My Request", "my r")); + assert!(m("foo bar", "f b")); + } + + #[test] + fn empty_and_quoted() { + assert!(!m("", "a")); + assert!(m("abc", "")); + assert!(m("las vegas", "\"la")); + assert!(!m("los angeles", "\"la\"")); + assert!(m("los angeles", "la")); + } + + #[test] + fn unicode_is_alphanumeric() { + assert!(m("Über Café", "üc")); + assert!(m("日本 語", "日語")); + } +} diff --git a/crates/yaak-query/src/lexer.rs b/crates/yaak-query/src/lexer.rs new file mode 100644 index 000000000..b28ae24ee --- /dev/null +++ b/crates/yaak-query/src/lexer.rs @@ -0,0 +1,320 @@ +//! Splits a query into tokens. Never fails: anything it does not recognise is text. + +use crate::ast::{CompareOp, FieldValue, NumberLiteral, Span}; + +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum TokKind { + LParen, + RParen, + /// `-` at the start of a term. It negates only what follows it directly. + Minus, + Or, + And, + Word(String), + Phrase { + text: String, + }, + Field { + path: Vec, + path_span: Span, + value: FieldValue, + value_span: Span, + }, +} + +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct Tok { + pub kind: TokKind, + pub span: Span, +} + +pub(crate) fn lex(input: &str) -> Vec { + let mut toks = Vec::new(); + let mut i = 0; + while let Some(c) = input[i..].chars().next() { + if c.is_whitespace() { + i += c.len_utf8(); + continue; + } + let (kind, end) = match c { + '(' => (TokKind::LParen, i + 1), + ')' => (TokKind::RParen, i + 1), + '-' => (TokKind::Minus, i + 1), + '"' => { + let (text, end) = lex_quoted(input, i); + (TokKind::Phrase { text }, end) + } + '@' => match lex_field(input, i) { + Some(field) => field, + None => lex_word(input, i), + }, + _ => lex_word(input, i), + }; + toks.push(Tok { kind, span: Span::new(i, end) }); + i = end; + } + toks +} + +/// Whether `c` ends a bare free-text word. +fn ends_word(c: char) -> bool { + c.is_whitespace() || matches!(c, '(' | ')' | '"') +} + +/// Whether `c` ends a bare field value. Parentheses inside a value are kept, except a closing +/// one, which closes the group the field sits in. +fn ends_value(c: char) -> bool { + c.is_whitespace() || c == ')' +} + +fn run_until(input: &str, start: usize, stop: impl Fn(char) -> bool) -> usize { + input[start..].find(stop).map_or(input.len(), |n| start + n) +} + +fn lex_word(input: &str, start: usize) -> (TokKind, usize) { + // The first character is never a word end (the caller checked), so a word is never empty + let first = input[start..].chars().next().map_or(0, char::len_utf8); + let end = run_until(input, start + first, ends_word); + let text = &input[start..end]; + let kind = match text { + "OR" => TokKind::Or, + "AND" => TokKind::And, + _ => TokKind::Word(text.to_string()), + }; + (kind, end) +} + +/// A double-quoted string starting at `start`. `\"` and `\\` are escapes; any other backslash +/// is kept. Without a closing quote it runs to the end of the input. +pub(crate) fn lex_quoted(input: &str, start: usize) -> (String, usize) { + let mut text = String::new(); + let mut chars = input[start + 1..].char_indices(); + while let Some((n, c)) = chars.next() { + match c { + '"' => return (text, start + 1 + n + 1), + '\\' => match input[start + 1 + n + 1..].chars().next() { + Some(next @ ('"' | '\\')) => { + text.push(next); + chars.next(); + } + _ => text.push('\\'), + }, + c => text.push(c), + } + } + (text, input.len()) +} + +pub(crate) fn is_ident_char(c: char) -> bool { + c.is_ascii_alphanumeric() || c == '_' || c == '-' +} + +/// `@path:value`, or `None` when the text at `start` is not one (no colon after the path). +fn lex_field(input: &str, start: usize) -> Option<(TokKind, usize)> { + let mut path = Vec::new(); + let mut i = start + 1; + loop { + let end = run_until(input, i, |c| !is_ident_char(c)); + if end == i { + return None; + } + path.push(input[i..end].to_string()); + i = end; + match input[i..].chars().next() { + Some('.') => i += 1, + Some(':') => break, + _ => return None, + } + } + let path_span = Span::new(start, i + 1); + let (value, value_end) = lex_value(input, i + 1); + let kind = TokKind::Field { path, path_span, value, value_span: Span::new(i + 1, value_end) }; + Some((kind, value_end)) +} + +fn lex_value(input: &str, start: usize) -> (FieldValue, usize) { + match input[start..].chars().next() { + None => return (FieldValue::Text { text: String::new(), quoted: false }, start), + Some(c) if ends_value(c) => { + return (FieldValue::Text { text: String::new(), quoted: false }, start); + } + Some('"') => { + let (text, end) = lex_quoted(input, start); + return (FieldValue::Text { text, quoted: true }, end); + } + Some('[') => { + if let Some((low, high, end)) = lex_range(input, start) { + return (FieldValue::Range { low, high }, end); + } + } + Some(_) => {} + } + let end = run_until(input, start, ends_value); + (classify_bare(&input[start..end]), end) +} + +/// What a bare (unquoted) field value means: presence, a comparison, a wildcard pattern, or +/// plain text. +pub(crate) fn classify_bare(raw: &str) -> FieldValue { + if raw == "*" { + return FieldValue::Exists; + } + if let Some((op, number)) = parse_compare(raw) { + return FieldValue::Compare { op, number }; + } + if raw.contains('*') { + return FieldValue::Wildcard { pattern: raw.to_string() }; + } + FieldValue::Text { text: raw.to_string(), quoted: false } +} + +fn parse_compare(raw: &str) -> Option<(CompareOp, NumberLiteral)> { + let (op, rest) = if let Some(rest) = raw.strip_prefix(">=") { + (CompareOp::Gte, rest) + } else if let Some(rest) = raw.strip_prefix("<=") { + (CompareOp::Lte, rest) + } else if let Some(rest) = raw.strip_prefix('>') { + (CompareOp::Gt, rest) + } else if let Some(rest) = raw.strip_prefix('<') { + (CompareOp::Lt, rest) + } else { + return None; + }; + Some((op, parse_number(rest)?)) +} + +/// A number with an optional unit: `-?digits(.digits)?letters?`, the whole of `raw`. +pub(crate) fn parse_number(raw: &str) -> Option { + let body = raw.strip_prefix('-').unwrap_or(raw); + let digits_end = body.find(|c: char| !c.is_ascii_digit()).unwrap_or(body.len()); + if digits_end == 0 { + return None; + } + let mut end = digits_end; + if let Some(fraction) = body[end..].strip_prefix('.') { + let fraction_len = fraction.find(|c: char| !c.is_ascii_digit()).unwrap_or(fraction.len()); + if fraction_len == 0 { + return None; + } + end += 1 + fraction_len; + } + let unit = &body[end..]; + if !unit.chars().all(|c| c.is_ascii_alphabetic()) { + return None; + } + let number_len = raw.len() - body.len() + end; + let value: f64 = raw[..number_len].parse().ok()?; + Some(NumberLiteral { + value, + unit: (!unit.is_empty()).then(|| unit.to_string()), + raw: raw.to_string(), + }) +} + +/// `[low TO high]` starting at `start`, or `None` when the text does not have exactly that +/// shape (it is then a literal value). +fn lex_range(input: &str, start: usize) -> Option<(NumberLiteral, NumberLiteral, usize)> { + let skip_ws = |i: usize| run_until(input, i, |c| !c.is_whitespace()); + let mut i = skip_ws(start + 1); + let low_end = run_until(input, i, |c| c.is_whitespace() || c == ']'); + let low = parse_number(&input[i..low_end])?; + i = skip_ws(low_end); + if i == low_end { + return None; + } + input[i..].strip_prefix("TO")?; + i += 2; + let after_to = skip_ws(i); + if after_to == i { + return None; + } + i = after_to; + let high_end = run_until(input, i, |c| c.is_whitespace() || c == ']'); + let high = parse_number(&input[i..high_end])?; + i = skip_ws(high_end); + input[i..].strip_prefix(']')?; + let end = i + 1; + match input[end..].chars().next() { + None => Some((low, high, end)), + Some(c) if ends_value(c) => Some((low, high, end)), + Some(_) => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn kinds(input: &str) -> Vec { + lex(input).into_iter().map(|t| t.kind).collect() + } + + #[test] + fn numbers() { + let n = parse_number("500ms").unwrap(); + assert_eq!(n.value, 500.0); + assert_eq!(n.unit.as_deref(), Some("ms")); + assert_eq!(parse_number("-2.5").unwrap().value, -2.5); + assert_eq!(parse_number("400").unwrap().unit, None); + assert!(parse_number("").is_none()); + assert!(parse_number("ms").is_none()); + assert!(parse_number("1.").is_none()); + assert!(parse_number("1.5.2").is_none()); + assert!(parse_number("12a3").is_none()); + assert!(parse_number("-").is_none()); + } + + #[test] + fn words_and_operators() { + assert_eq!( + kinds("a OR b AND or"), + vec![ + TokKind::Word("a".into()), + TokKind::Or, + TokKind::Word("b".into()), + TokKind::And, + TokKind::Word("or".into()), + ] + ); + } + + #[test] + fn words_stop_at_parens_and_quotes() { + assert_eq!( + kinds("(a)b\"c d\""), + vec![ + TokKind::LParen, + TokKind::Word("a".into()), + TokKind::RParen, + TokKind::Word("b".into()), + TokKind::Phrase { text: "c d".into() }, + ] + ); + } + + #[test] + fn quoted_escapes() { + assert_eq!(lex_quoted(r#""a \"b\" \\ \n""#, 0), (r#"a "b" \ \n"#.to_string(), 15)); + assert_eq!(lex_quoted(r#""open"#, 0), ("open".to_string(), 5)); + assert_eq!(lex_quoted(r#""trailing\"#, 0), ("trailing\\".to_string(), 10)); + } + + #[test] + fn spans_are_bytes() { + let toks = lex("é @url:ü"); + assert_eq!(toks[0].span, Span::new(0, 2)); + assert_eq!(toks[1].span, Span::new(3, 10)); + } + + #[test] + fn range_needs_its_whole_shape() { + assert!(lex_range("[1 TO 2]", 0).is_some()); + assert!(lex_range("[ 1 TO 2 ]", 0).is_some()); + assert!(lex_range("[1 to 2]", 0).is_none()); + assert!(lex_range("[1TO 2]", 0).is_none()); + assert!(lex_range("[1 TO 2", 0).is_none()); + assert!(lex_range("[1 TO 2]x", 0).is_none()); + assert!(lex_range("[a TO b]", 0).is_none()); + assert!(lex_range("[ base_url ]", 0).is_none()); + } +} diff --git a/crates/yaak-query/src/lib.rs b/crates/yaak-query/src/lib.rs new file mode 100644 index 000000000..14cc87e66 --- /dev/null +++ b/crates/yaak-query/src/lib.rs @@ -0,0 +1,40 @@ +//! The search query language of Yaak and Yaak Proxy. +//! +//! The syntax follows Datadog's log search. Terms side by side must all match; `OR` (upper +//! case) between terms or groups means either; parentheses group; `-` directly before a term or +//! group excludes it. A bare word matches the free text, a double-quoted phrase matches it with +//! its spaces kept, and `@path:value` matches a field. A value may be `*` (the field is present), +//! a `*` wildcard pattern, a comparison like `>=400` or `>500ms`, or a range like +//! `[400 TO 499]`. Everything matches case-insensitively, and nothing in the input is an error. +//! +//! [`parse`] gives the tree, which a consumer may walk to compile it elsewhere (to SQL, say). +//! [`evaluate`] runs it in memory against a [`Document`], with a [`Schema`] saying what the +//! fields are. [`tokens`], [`completions`] and [`format_field`] serve the search box. + +// The wasm-bindgen exports expand to unsafe glue; nothing else in the crate may use unsafe +#![cfg_attr( + not(all(feature = "wasm", target_arch = "wasm32")), + forbid(unsafe_code) +)] +#![cfg_attr(all(feature = "wasm", target_arch = "wasm32"), deny(unsafe_code))] + +pub mod ast; +mod complete; +mod eval; +mod format; +pub mod fuzzy; +mod lexer; +mod parser; +mod schema; +mod tokens; +#[cfg(all(feature = "wasm", target_arch = "wasm32"))] +#[allow(unsafe_code)] +mod wasm; + +pub use ast::{CompareOp, FieldValue, Node, NumberLiteral, Span}; +pub use complete::{Completion, CompletionKind, Completions, completions}; +pub use eval::{DocValue, DocValues, Document, Matcher, Table, evaluate}; +pub use format::{format_field, format_value, is_valid_path, quote}; +pub use parser::parse; +pub use schema::{Diagnostic, FieldDef, FieldKind, FreeText, ResolvedField, Schema, Unit}; +pub use tokens::{Token, TokenKind, tokens}; diff --git a/crates/yaak-query/src/parser.rs b/crates/yaak-query/src/parser.rs new file mode 100644 index 000000000..3ef505af7 --- /dev/null +++ b/crates/yaak-query/src/parser.rs @@ -0,0 +1,393 @@ +//! Builds the [`Node`] tree from tokens. +//! +//! Precedence, tightest first: `-`, then AND (implicit or written), then `OR`. The parser is +//! forgiving and never fails: a stray `)` or a dangling `OR`, `AND` or `-` is ignored, an +//! unclosed `(` closes at the end, and an empty group is dropped. Groups and negations nest +//! at most [`MAX_DEPTH`] deep; past that, `(` and `-` are ignored, so pathological input can't +//! exhaust the stack (the WebAssembly build has a small one). + +use crate::ast::{Node, Span}; +use crate::lexer::{Tok, TokKind, lex}; + +/// How deep groups and negations may nest, together. +pub const MAX_DEPTH: usize = 32; + +/// Parses a query. `None` when it has no terms (empty, only whitespace, or only operators). +pub fn parse(input: &str) -> Option { + let toks = lex(input); + let mut parser = Parser { toks: &toks, i: 0 }; + let mut parts = Vec::new(); + // A `)` with no group to close ends parse_or early; skip it and carry on + loop { + if let Some(node) = parser.parse_or(0) { + parts.push(node); + } + if parser.peek().is_none() { + break; + } + parser.i += 1; + } + join(parts, |children, span| Node::And { children, span }) +} + +struct Parser<'a> { + toks: &'a [Tok], + i: usize, +} + +impl<'a> Parser<'a> { + fn peek(&self) -> Option<&'a Tok> { + self.toks.get(self.i) + } + + fn parse_or(&mut self, depth: usize) -> Option { + let mut parts = Vec::new(); + loop { + if let Some(node) = self.parse_and(depth) { + parts.push(node); + } + match self.peek() { + Some(Tok { kind: TokKind::Or, .. }) => self.i += 1, + _ => break, + } + } + join(parts, |children, span| Node::Or { children, span }) + } + + fn parse_and(&mut self, depth: usize) -> Option { + let mut parts = Vec::new(); + while let Some(tok) = self.peek() { + match tok.kind { + TokKind::Or | TokKind::RParen => break, + TokKind::And => self.i += 1, + _ => { + if let Some(node) = self.parse_unary(depth) { + parts.push(node); + } + } + } + } + join(parts, |children, span| Node::And { children, span }) + } + + /// Always consumes at least one token. + fn parse_unary(&mut self, depth: usize) -> Option { + let tok = self.peek()?; + if tok.kind != TokKind::Minus { + return self.parse_primary(depth); + } + self.i += 1; + // `-` negates only a term, group or another `-` written directly after it + let next = self.peek().filter(|next| { + next.span.start == tok.span.end + && !matches!(next.kind, TokKind::RParen | TokKind::Or | TokKind::And) + }); + if next.is_none() || depth >= MAX_DEPTH { + return None; + } + let node = self.parse_unary(depth + 1)?; + let span = tok.span.to(node.span()); + Some(Node::Not { node: Box::new(node), span }) + } + + /// Always consumes at least one token. + fn parse_primary(&mut self, depth: usize) -> Option { + let tok = self.peek()?; + self.i += 1; + match &tok.kind { + TokKind::Word(text) => Some(Node::Term { text: text.clone(), span: tok.span }), + TokKind::Phrase { text } => Some(Node::Phrase { text: text.clone(), span: tok.span }), + TokKind::Field { path, path_span, value, value_span } => Some(Node::Field { + path: path.clone(), + value: value.clone(), + span: tok.span, + path_span: *path_span, + value_span: *value_span, + }), + // Too deep: the group's contents carry on at this level instead + TokKind::LParen if depth >= MAX_DEPTH => None, + TokKind::LParen => { + let inner = self.parse_or(depth + 1); + if let Some(Tok { kind: TokKind::RParen, .. }) = self.peek() { + self.i += 1; + } + inner + } + // Operators are handled by the callers; reaching one here means it is out of place + TokKind::Minus | TokKind::Or | TokKind::And | TokKind::RParen => None, + } + } +} + +/// One node stays itself; several are joined by `make` with a span covering them all. +fn join(mut parts: Vec, make: impl Fn(Vec, Span) -> Node) -> Option { + match parts.len() { + 0 => None, + 1 => parts.pop(), + _ => { + let span = parts[0].span().to(parts[parts.len() - 1].span()); + Some(make(parts, span)) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ast::{CompareOp, FieldValue}; + + /// A compact rendering of the tree, to keep the tests readable. + fn show(input: &str) -> String { + fn go(node: &Node) -> String { + match node { + Node::And { children, .. } => { + format!("(and {})", children.iter().map(go).collect::>().join(" ")) + } + Node::Or { children, .. } => { + format!("(or {})", children.iter().map(go).collect::>().join(" ")) + } + Node::Not { node, .. } => format!("(not {})", go(node)), + Node::Term { text, .. } => text.clone(), + Node::Phrase { text, .. } => format!("{text:?}"), + Node::Field { path, value, .. } => { + let value = match value { + FieldValue::Text { text, quoted: false } => text.clone(), + FieldValue::Text { text, quoted: true } => format!("{text:?}"), + FieldValue::Wildcard { pattern } => format!("wild({pattern})"), + FieldValue::Exists => "exists".to_string(), + FieldValue::Compare { op, number } => { + format!("{}{}", op.as_str(), number.raw) + } + FieldValue::Range { low, high } => format!("[{} TO {}]", low.raw, high.raw), + }; + format!("@{}={}", path.join("."), value) + } + } + } + parse(input).map_or("none".to_string(), |n| go(&n)) + } + + #[test] + fn empty() { + assert_eq!(show(""), "none"); + assert_eq!(show(" "), "none"); + assert_eq!(show("OR AND"), "none"); + assert_eq!(show("()"), "none"); + assert_eq!(show("-"), "none"); + assert_eq!(show(")"), "none"); + } + + #[test] + fn terms_and_phrases() { + assert_eq!(show("foo"), "foo"); + assert_eq!(show("foo bar"), "(and foo bar)"); + assert_eq!(show("\"hi there\""), "\"hi there\""); + assert_eq!(show("\"hi there"), "\"hi there\""); + assert_eq!(show("a\"b c\"d"), "(and a \"b c\" d)"); + assert_eq!(show(r#""say \"hi\"""#), r#""say \"hi\"""#); + } + + #[test] + fn boolean_operators() { + assert_eq!(show("a OR b"), "(or a b)"); + assert_eq!(show("a AND b"), "(and a b)"); + assert_eq!(show("a b OR c"), "(or (and a b) c)"); + assert_eq!(show("a OR b c"), "(or a (and b c))"); + assert_eq!(show("a OR b OR c"), "(or a b c)"); + // Case-sensitive, as Datadog: lowercase is a word + assert_eq!(show("a or b"), "(and a or b)"); + assert_eq!(show("a and b"), "(and a and b)"); + // NOT is not a keyword + assert_eq!(show("NOT a"), "(and NOT a)"); + } + + #[test] + fn dangling_operators_are_ignored() { + assert_eq!(show("a OR"), "a"); + assert_eq!(show("OR a"), "a"); + assert_eq!(show("a OR OR b"), "(or a b)"); + assert_eq!(show("a AND"), "a"); + assert_eq!(show("a -"), "a"); + assert_eq!(show("- a"), "a"); + assert_eq!(show("(a -)"), "a"); + } + + #[test] + fn groups() { + assert_eq!(show("(a OR b) -c"), "(and (or a b) (not c))"); + assert_eq!(show("-(a OR b)"), "(not (or a b))"); + assert_eq!(show("(a OR b"), "(or a b)"); + assert_eq!(show("a) b"), "(and a b)"); + assert_eq!(show("((a))"), "a"); + assert_eq!(show("x (a (b OR c))"), "(and x (and a (or b c)))"); + } + + #[test] + fn negation() { + assert_eq!(show("-foo"), "(not foo)"); + assert_eq!(show("--foo"), "(not (not foo))"); + assert_eq!(show("-\"a b\""), "(not \"a b\")"); + assert_eq!(show("-@type:grpc"), "(not @type=grpc)"); + assert_eq!(show("a-b"), "a-b"); + assert_eq!(show("-OR a"), "a"); + } + + #[test] + fn fields() { + assert_eq!(show("@method:post"), "@method=post"); + assert_eq!(show("@url:\"a b\""), "@url=\"a b\""); + assert_eq!(show("@header.x-cache:HIT"), "@header.x-cache=HIT"); + assert_eq!(show("@req.header.content-type:json"), "@req.header.content-type=json"); + assert_eq!(show("@folder:x OR @folder:y"), "(or @folder=x @folder=y)"); + assert_eq!(show("@url:https://a.b/c?d=e"), "@url=https://a.b/c?d=e"); + } + + #[test] + fn at_is_required() { + assert_eq!(show("method:post"), "method:post"); + assert_eq!(show("localhost:3000"), "localhost:3000"); + assert_eq!(show("@method"), "@method"); + assert_eq!(show("@"), "@"); + assert_eq!(show("@.a:b"), "@.a:b"); + assert_eq!(show("@a.:b"), "@a.:b"); + assert_eq!(show("@a b:c"), "(and @a b:c)"); + } + + #[test] + fn trailing_field_has_empty_value() { + assert_eq!(show("@method:"), "@method="); + assert_eq!(show("@method: post"), "(and @method= post)"); + assert_eq!(show("(@method:)"), "@method="); + } + + #[test] + fn field_values() { + assert_eq!(show("@url:-foo"), "@url=-foo"); + assert_eq!(show("@url:AND"), "@url=AND"); + assert_eq!(show("@url:a(b"), "@url=a(b"); + assert_eq!(show("(@url:a OR @url:b)"), "(or @url=a @url=b)"); + assert_eq!(show("@url:a\"b"), "@url=a\"b"); + assert_eq!(show("@url:\"a"), "@url=\"a\""); + } + + #[test] + fn special_values() { + assert_eq!(show("@status:*"), "@status=exists"); + assert_eq!(show("-@status:*"), "(not @status=exists)"); + assert_eq!(show("@url:*.github.com"), "@url=wild(*.github.com)"); + assert_eq!(show("@url:\"*.github.com\""), "@url=\"*.github.com\""); + assert_eq!(show("@status:>=400"), "@status=>=400"); + assert_eq!(show("@duration:>500ms"), "@duration=>500ms"); + assert_eq!(show("@status:<1"), "@status=<1"); + assert_eq!(show("@status:[400 TO 499]"), "@status=[400 TO 499]"); + assert_eq!(show("(@status:[400 TO 499])"), "@status=[400 TO 499]"); + // Not quite the shape: literal text or a wildcard + assert_eq!(show("@status:>="), "@status=>="); + assert_eq!(show("@status:>abc"), "@status=>abc"); + assert_eq!(show("@status:**"), "@status=wild(**)"); + assert_eq!(show("@status:[400"), "@status=[400"); + } + + #[test] + fn templates_parse_sanely() { + assert_eq!(show("@url:${[ base_url ]}"), "(and @url=${[ base_url ]})"); + assert_eq!(show("@url:[ base_url ]"), "(and @url=[ base_url ])"); + assert_eq!(show("@url:\"${[ base_url ]}/x\""), "@url=\"${[ base_url ]}/x\""); + } + + #[test] + fn compare_value_details() { + let Some(Node::Field { value: FieldValue::Compare { op, number }, .. }) = + parse("@duration:<=1.5s") + else { + panic!("not a comparison"); + }; + assert_eq!(op, CompareOp::Lte); + assert_eq!(number.value, 1.5); + assert_eq!(number.unit.as_deref(), Some("s")); + } + + #[test] + fn spans() { + let input = "-@url:\"a b\" OR (c"; + let node = parse(input).unwrap(); + let Node::Or { children, span } = &node else { + panic!() + }; + assert_eq!(span.slice(input), input); + let Node::Not { node: field, span } = &children[0] else { + panic!() + }; + assert_eq!(span.slice(input), "-@url:\"a b\""); + let Node::Field { span, path_span, value_span, .. } = field.as_ref() else { + panic!() + }; + assert_eq!(span.slice(input), "@url:\"a b\""); + assert_eq!(path_span.slice(input), "@url:"); + assert_eq!(value_span.slice(input), "\"a b\""); + assert_eq!(children[1].span().slice(input), "c"); + + let input = "@method:"; + let Some(Node::Field { value_span, .. }) = parse(input) else { + panic!() + }; + assert_eq!(value_span, Span::new(8, 8)); + } + + #[test] + fn nesting_is_bounded() { + fn depth(node: &Node) -> usize { + match node { + Node::And { children, .. } | Node::Or { children, .. } => { + children.iter().map(depth).max().unwrap_or(0) + 1 + } + Node::Not { node, .. } => depth(node) + 1, + _ => 1, + } + } + let parens = format!("{}a OR b{}", "(".repeat(5000), ")".repeat(5000)); + let negations = format!("{}a", "-".repeat(5000)); + let mixed = format!("{}a{}", "-(".repeat(5000), ")".repeat(5000)); + for input in [parens, negations, mixed] { + let node = parse(&input).unwrap(); + assert!(depth(&node) <= MAX_DEPTH + 2, "too deep for {:?}", &input[..10]); + } + // Within the limit, nesting is kept exactly + let input = format!("{}a{}", "(".repeat(MAX_DEPTH), ")".repeat(MAX_DEPTH)); + assert_eq!(show(&input), "a"); + assert_eq!(show(&format!("{}a", "-".repeat(2))), "(not (not a))"); + } + + #[test] + fn never_panics_on_odd_input() { + for input in [ + "\"", + "\\", + "@", + "@:", + "@a:", + "-", + "--", + "(", + ")", + "((", + "))", + "[", + "@a:[", + "@a:[1 TO", + "@a:\"", + "@a:\"\\", + "é@ü:ö", + "@a.b.c:", + "@a:>", + "@a:<=", + "-(", + "OR", + "AND OR", + "@a:)", + "🦀 @x:🦀 \"🦀", + ] { + let _ = parse(input); + } + } +} diff --git a/crates/yaak-query/src/schema.rs b/crates/yaak-query/src/schema.rs new file mode 100644 index 000000000..17a1c02fb --- /dev/null +++ b/crates/yaak-query/src/schema.rs @@ -0,0 +1,307 @@ +//! What a consumer's fields are, and how its bare words match. + +use crate::ast::{FieldValue, Node, NumberLiteral, Span}; +use crate::lexer::parse_number; +use serde::{Deserialize, Serialize}; + +/// A consumer's searchable fields. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub struct Schema { + pub fields: Vec, + /// Substring when left out. + #[serde(default)] + #[cfg_attr(test, ts(optional, as = "Option"))] + pub free_text: FreeText, +} + +/// How a bare word matches the document's free text. A quoted phrase is always a substring. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub enum FreeText { + /// A substring, falling back to word-prefix matching (`mr` finds "My Request"), as + /// fuzzbunny does. + Fuzzy, + /// A case-insensitive substring. + #[default] + Substring, +} + +/// One field: `@name:value`. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub struct FieldDef { + /// The field's path, dotted when nested: `method`, `req.header`. + pub name: String, + pub kind: FieldKind, + /// A map of named values, such as headers: `@header.:value` is valid for any key, and + /// the key is matched against the document's keys (case-insensitively). + #[serde(default)] + #[cfg_attr(test, ts(optional, as = "Option"))] + pub map: bool, + /// A short note shown with the field in completions. + #[serde(default)] + #[cfg_attr(test, ts(optional))] + pub info: Option, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub enum FieldKind { + Text, + /// A number, with the units a comparison or range may be written in. A number written + /// without a unit is in the field's base unit. + Number { + #[serde(default)] + units: Vec, + }, + /// True or false: `@name:true`, `@name:false`, and `@name:*` for true. + Flag, +} + +/// A unit a number may be written in, and how many of the field's base unit it is worth: +/// `{ name: "s", factor: 1000 }` for a field counted in milliseconds. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub struct Unit { + pub name: String, + pub factor: f64, +} + +/// A field path resolved against the schema. +#[derive(Clone, Debug, PartialEq)] +pub struct ResolvedField<'a> { + pub def: &'a FieldDef, + /// The key under a map field: `x-cache` for `@header.x-cache`. `None` for the map field + /// itself (`@header:...`, which matches any key) and for plain fields. + pub key: Option, +} + +impl ResolvedField<'_> { + /// The lowercase dotted path a document stores this field's values under. + pub fn document_path(&self) -> String { + match &self.key { + Some(key) => format!("{}.{}", self.def.name.to_lowercase(), key.to_lowercase()), + None => self.def.name.to_lowercase(), + } + } +} + +/// Something in a query the schema does not accept. The query still parses and runs; the +/// offending term matches nothing. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub enum Diagnostic { + /// No field has this path. `span` covers `@path:`. + UnknownField { path: String, span: Span }, + /// The value cannot apply to this field: a unit the field does not take, a number for a + /// text-only flag, and so on. `span` covers the value. + InvalidValue { path: String, span: Span }, +} + +impl Schema { + pub fn new(fields: Vec, free_text: FreeText) -> Self { + Self { fields, free_text } + } + + /// Finds the field a path names, case-insensitively. A path under a map field resolves to + /// that field with the rest of the path as the key; the longest matching map wins. + pub fn resolve(&self, path: &[String]) -> Option> { + let joined = path.join(".").to_lowercase(); + if let Some(def) = self.fields.iter().find(|f| f.name.to_lowercase() == joined) { + return Some(ResolvedField { def, key: None }); + } + self.fields + .iter() + .filter(|f| f.map) + .filter_map(|f| { + let name = f.name.to_lowercase(); + let key = joined.strip_prefix(&name)?.strip_prefix('.')?; + (!key.is_empty()).then_some((name.len(), f, key)) + }) + .max_by_key(|(len, ..)| *len) + .map(|(_, def, key)| ResolvedField { def, key: Some(key.to_string()) }) + } + + /// The terms in a query this schema cannot run, for the UI to mark. + pub fn diagnostics(&self, root: &Node) -> Vec { + let mut out = Vec::new(); + root.walk(&mut |node| { + let Node::Field { path, value, path_span, value_span, .. } = node else { + return; + }; + let joined = path.join("."); + match self.resolve(path) { + None => out.push(Diagnostic::UnknownField { path: joined, span: *path_span }), + Some(field) => { + if !value_fits(&field.def.kind, value) { + out.push(Diagnostic::InvalidValue { path: joined, span: *value_span }); + } + } + } + }); + out + } +} + +/// Whether a value can mean anything for a field of this kind. +fn value_fits(kind: &FieldKind, value: &FieldValue) -> bool { + match (kind, value) { + (FieldKind::Number { units }, FieldValue::Compare { number, .. }) => { + to_base(number, units).is_some() + } + (FieldKind::Number { units }, FieldValue::Range { low, high }) => { + to_base(low, units).is_some() && to_base(high, units).is_some() + } + // A bare number is equality, so its unit has to be one the field takes + (FieldKind::Number { units }, FieldValue::Text { text, quoted: false }) => { + parse_number(text).is_none_or(|number| to_base(&number, units).is_some()) + } + // Text holding numbers compares them as plain numbers, without units + (FieldKind::Text, FieldValue::Compare { number, .. }) => number.unit.is_none(), + (FieldKind::Text, FieldValue::Range { low, high }) => { + low.unit.is_none() && high.unit.is_none() + } + (FieldKind::Flag, FieldValue::Text { text, .. }) => { + text.is_empty() || parse_flag(text).is_some() + } + (FieldKind::Flag, FieldValue::Exists) => true, + (FieldKind::Flag, _) => false, + _ => true, + } +} + +/// A number in the field's base unit, or `None` when its unit is not one the field takes. +pub(crate) fn to_base(number: &NumberLiteral, units: &[Unit]) -> Option { + match &number.unit { + None => Some(number.value), + Some(unit) => units + .iter() + .find(|u| u.name.eq_ignore_ascii_case(unit)) + .map(|u| number.value * u.factor), + } +} + +pub(crate) fn parse_flag(text: &str) -> Option { + match text.to_lowercase().as_str() { + "true" | "yes" | "1" => Some(true), + "false" | "no" | "0" => Some(false), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parse; + + fn field(name: &str, kind: FieldKind, map: bool) -> FieldDef { + FieldDef { name: name.to_string(), kind, map, info: None } + } + + fn schema() -> Schema { + Schema::new( + vec![ + field("url", FieldKind::Text, false), + field("header", FieldKind::Text, true), + field("req.header", FieldKind::Text, true), + field( + "duration", + FieldKind::Number { units: vec![Unit { name: "s".into(), factor: 1000.0 }] }, + false, + ), + field("error", FieldKind::Flag, false), + ], + FreeText::Substring, + ) + } + + fn path(p: &str) -> Vec { + p.split('.').map(String::from).collect() + } + + #[test] + fn resolves_plain_and_map_fields() { + let s = schema(); + assert_eq!(s.resolve(&path("URL")).unwrap().def.name, "url"); + assert_eq!(s.resolve(&path("url")).unwrap().key, None); + + let h = s.resolve(&path("header.X-Cache")).unwrap(); + assert_eq!(h.def.name, "header"); + assert_eq!(h.key.as_deref(), Some("x-cache")); + assert_eq!(h.document_path(), "header.x-cache"); + + let r = s.resolve(&path("req.header.accept")).unwrap(); + assert_eq!(r.def.name, "req.header"); + assert_eq!(r.key.as_deref(), Some("accept")); + + // The map itself, with no key + assert_eq!(s.resolve(&path("header")).unwrap().key, None); + + assert!(s.resolve(&path("nope")).is_none()); + assert!(s.resolve(&path("url.x")).is_none()); + assert!(s.resolve(&path("req")).is_none()); + } + + #[test] + fn reports_unknown_fields_and_bad_values() { + let s = schema(); + let input = "@nope:x @url:y @duration:>5m @duration:>5s @error:maybe @error:true"; + let diagnostics = s.diagnostics(&parse(input).unwrap()); + assert_eq!( + diagnostics, + vec![ + Diagnostic::UnknownField { path: "nope".into(), span: Span::new(0, 6) }, + Diagnostic::InvalidValue { path: "duration".into(), span: Span::new(25, 28) }, + Diagnostic::InvalidValue { path: "error".into(), span: Span::new(50, 55) }, + ] + ); + } + + /// Diagnostics flag exactly the unit mistakes the evaluator rejects. + #[test] + fn reports_units_the_evaluator_rejects() { + let s = schema(); + let invalid = |input: &str| !s.diagnostics(&parse(input).unwrap()).is_empty(); + assert!(invalid("@duration:1m")); + assert!(invalid("@duration:[1m TO 2s]")); + assert!(invalid("@url:>1s")); + assert!(invalid("@url:[1 TO 2s]")); + assert!(!invalid("@duration:1s")); + assert!(!invalid("@duration:1500")); + assert!(!invalid("@duration:abc")); + assert!(!invalid("@duration:\"1m\"")); + assert!(!invalid("@url:>1")); + assert!(!invalid("@url:1m")); + } + + #[test] + fn deserializes_from_json() { + let s: Schema = serde_json::from_str( + r#"{ + "fields": [ + { "name": "method", "kind": { "type": "text" } }, + { "name": "header", "kind": { "type": "text" }, "map": true }, + { "name": "size", "kind": { "type": "number", "units": [{ "name": "kb", "factor": 1024 }] } }, + { "name": "error", "kind": { "type": "flag" }, "info": "Failed" } + ], + "freeText": "fuzzy" + }"#, + ) + .unwrap(); + assert_eq!(s.free_text, FreeText::Fuzzy); + assert!(s.fields[1].map); + assert_eq!(s.fields[3].info.as_deref(), Some("Failed")); + } +} diff --git a/crates/yaak-query/src/tokens.rs b/crates/yaak-query/src/tokens.rs new file mode 100644 index 000000000..11ff84243 --- /dev/null +++ b/crates/yaak-query/src/tokens.rs @@ -0,0 +1,100 @@ +//! Spans for syntax highlighting. + +use crate::ast::Span; +use crate::lexer::{TokKind, lex}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub enum TokenKind { + /// `@path:`, through the colon. + Field, + /// A field's value, quoted or not. + Value, + /// `OR` and `AND`. + Operator, + /// A quoted phrase in free text. + Quote, + Paren, + /// A `-` that negates what follows it. + Negation, + /// A bare free-text word. + Text, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub struct Token { + pub kind: TokenKind, + pub span: Span, +} + +/// The highlighted pieces of `input`, in order. Whitespace is not covered. +pub fn tokens(input: &str) -> Vec { + let mut out = Vec::new(); + let toks = lex(input); + for (i, tok) in toks.iter().enumerate() { + let kind = match &tok.kind { + TokKind::LParen | TokKind::RParen => TokenKind::Paren, + TokKind::Minus => { + // The parser ignores a `-` with nothing to negate directly after it + let negates = toks.get(i + 1).is_some_and(|next| { + next.span.start == tok.span.end + && !matches!(next.kind, TokKind::RParen | TokKind::Or | TokKind::And) + }); + if negates { TokenKind::Negation } else { TokenKind::Text } + } + TokKind::Or | TokKind::And => TokenKind::Operator, + TokKind::Word(_) => TokenKind::Text, + TokKind::Phrase { .. } => TokenKind::Quote, + TokKind::Field { path_span, value_span, .. } => { + out.push(Token { kind: TokenKind::Field, span: *path_span }); + if value_span.start != value_span.end { + out.push(Token { kind: TokenKind::Value, span: *value_span }); + } + continue; + } + }; + out.push(Token { kind, span: tok.span }); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn show(input: &str) -> Vec<(TokenKind, &str)> { + tokens(input).into_iter().map(|t| (t.kind, t.span.slice(input))).collect() + } + + #[test] + fn kinds_and_spans() { + use TokenKind::*; + assert_eq!( + show("-@method:post OR (\"a b\" c) @url:"), + vec![ + (Negation, "-"), + (Field, "@method:"), + (Value, "post"), + (Operator, "OR"), + (Paren, "("), + (Quote, "\"a b\""), + (Text, "c"), + (Paren, ")"), + (Field, "@url:"), + ] + ); + assert_eq!( + show("@status:[400 TO 499]"), + vec![(Field, "@status:"), (Value, "[400 TO 499]")] + ); + assert_eq!(show("localhost:3000"), vec![(Text, "localhost:3000")]); + assert_eq!(show(""), vec![]); + // A `-` with nothing directly after it negates nothing + assert_eq!(show("- a -"), vec![(Text, "-"), (Text, "a"), (Text, "-")]); + } +} diff --git a/crates/yaak-query/src/wasm.rs b/crates/yaak-query/src/wasm.rs new file mode 100644 index 000000000..fbf0089b3 --- /dev/null +++ b/crates/yaak-query/src/wasm.rs @@ -0,0 +1,161 @@ +//! The `@yaakapp-internal/query` exports. +//! +//! JavaScript counts string positions in UTF-16 code units, so every span going out is +//! converted from bytes, and the cursor coming in is converted to bytes. + +use crate::{Document, Matcher, Schema, Span, Table}; +use serde::Serialize; +use serde::de::DeserializeOwned; +use std::collections::HashMap; +use wasm_bindgen::JsValue; +use wasm_bindgen::prelude::wasm_bindgen; + +/// Byte offset to UTF-16 offset, and back, for one input. +struct Offsets { + /// `utf16[i]` is the UTF-16 offset of byte `i`, for every char boundary and the end. + utf16: Vec, +} + +impl Offsets { + fn new(input: &str) -> Self { + let mut utf16 = vec![0; input.len() + 1]; + let mut units = 0u32; + for (i, c) in input.char_indices() { + for b in 0..c.len_utf8() { + utf16[i + b] = units; + } + units += c.len_utf16() as u32; + } + utf16[input.len()] = units; + Self { utf16 } + } + + fn span(&self, span: Span) -> Span { + Span { start: self.utf16[span.start()], end: self.utf16[span.end()] } + } + + /// The byte offset of UTF-16 offset `pos`, or of the char it falls inside. + fn byte(&self, pos: u32) -> usize { + self.utf16.partition_point(|&u| u < pos).min(self.utf16.len() - 1) + } +} + +fn to_js(value: &T) -> Result { + value + .serialize(&serde_wasm_bindgen::Serializer::json_compatible()) + .map_err(|e| JsValue::from_str(&e.to_string())) +} + +fn from_js(value: JsValue) -> Result { + serde_wasm_bindgen::from_value(value).map_err(|e| JsValue::from_str(&e.to_string())) +} + +/// The parsed tree (`Node | null`), with UTF-16 spans. +#[wasm_bindgen] +pub fn parse_query(input: &str) -> Result { + let offsets = Offsets::new(input); + let mut root = crate::parse(input); + if let Some(root) = &mut root { + root.map_spans(&|s| offsets.span(s)); + } + to_js(&root) +} + +/// Highlighting spans (`Token[]`), in UTF-16 offsets. +#[wasm_bindgen] +pub fn query_tokens(input: &str) -> Result { + let offsets = Offsets::new(input); + let mut tokens = crate::tokens(input); + for token in &mut tokens { + token.span = offsets.span(token.span); + } + to_js(&tokens) +} + +/// `Completions | null` at UTF-16 offset `cursor`. `values` maps field paths to the values seen +/// for them (`Record`). +#[wasm_bindgen] +pub fn query_completions( + input: &str, + cursor: u32, + schema: JsValue, + values: JsValue, +) -> Result { + let schema: Schema = from_js(schema)?; + let values: HashMap> = + if values.is_undefined() || values.is_null() { HashMap::new() } else { from_js(values)? }; + let offsets = Offsets::new(input); + let mut result = crate::completions(input, offsets.byte(cursor), &schema, &values); + if let Some(result) = &mut result { + result.span = offsets.span(result.span); + } + to_js(&result) +} + +/// The terms the schema cannot run (`Diagnostic[]`), with UTF-16 spans. +#[wasm_bindgen] +pub fn query_diagnostics(input: &str, schema: JsValue) -> Result { + let schema: Schema = from_js(schema)?; + let offsets = Offsets::new(input); + let mut diagnostics = + crate::parse(input).map(|root| schema.diagnostics(&root)).unwrap_or_default(); + for diagnostic in &mut diagnostics { + match diagnostic { + crate::Diagnostic::UnknownField { span, .. } + | crate::Diagnostic::InvalidValue { span, .. } => *span = offsets.span(*span), + } + } + to_js(&diagnostics) +} + +#[wasm_bindgen] +pub fn query_format_field(field: &str, value: &str, negated: bool) -> String { + crate::format_field(field, value, negated) +} + +/// A query parsed and resolved once, then matched against many documents. +#[wasm_bindgen] +pub struct QueryMatcher { + matcher: Matcher, +} + +#[wasm_bindgen] +impl QueryMatcher { + #[wasm_bindgen(constructor)] + pub fn new(input: &str, schema: JsValue) -> Result { + let schema: Schema = from_js(schema)?; + Ok(QueryMatcher { matcher: Matcher::new(crate::parse(input).as_ref(), &schema) }) + } + + /// Whether the query has no terms, so it matches everything. + #[wasm_bindgen(getter, js_name = isEmpty)] + pub fn is_empty(&self) -> bool { + self.matcher.is_empty() + } + + /// Whether a `Document` (`{ text, fields }`), as JSON, matches. JSON crosses into wasm far + /// faster than walking a JS object property by property. + pub fn matches(&self, doc_json: &str) -> Result { + let doc: Document = serde_json::from_str(doc_json).map_err(js_error)?; + Ok(self.matcher.matches(&doc)) + } + + /// The indexes of the matching rows, for many documents in one call. The cells (per row, + /// its free text, then one per column) are packed as in [`Table::from_packed`]: joined into + /// one string, with each one's UTF-16 length, or `Table::NO_VALUE` for no value. + #[wasm_bindgen(js_name = matchingRows)] + pub fn matching_rows( + &self, + columns: Vec, + joined: &str, + lengths: &[u32], + ) -> Result, JsValue> { + let table = Table::from_packed(columns, joined, lengths) + .ok_or_else(|| JsValue::from_str("cells do not line up with the columns"))?; + Ok(self.matcher.matching_rows(&table)) + } +} + +fn js_error(e: impl std::fmt::Display) -> JsValue { + JsValue::from_str(&e.to_string()) +} diff --git a/package-lock.json b/package-lock.json index a4539a7a3..9a34d7914 100644 --- a/package-lock.json +++ b/package-lock.json @@ -64,6 +64,7 @@ "crates/yaak-git", "crates/yaak-models", "crates/yaak-plugins", + "crates/yaak-query", "crates/yaak-sse", "crates/yaak-sync", "crates/yaak-templates", @@ -322,6 +323,13 @@ "name": "@yaakapp-internal/plugins", "version": "1.0.0" }, + "crates/yaak-query": { + "name": "@yaakapp-internal/query", + "version": "1.0.0", + "devDependencies": { + "rimraf": "^6.1.2" + } + }, "crates/yaak-sse": { "name": "@yaakapp-internal/sse", "version": "1.0.0", @@ -5241,6 +5249,10 @@ "resolved": "crates-proxy/yaak-proxy-lib", "link": true }, + "node_modules/@yaakapp-internal/query": { + "resolved": "crates/yaak-query", + "link": true + }, "node_modules/@yaakapp-internal/rpc-schema": { "resolved": "crates/common/yaak-rpc-schema", "link": true diff --git a/package.json b/package.json index 9a83a9673..9361af88a 100644 --- a/package.json +++ b/package.json @@ -63,6 +63,7 @@ "crates/yaak-git", "crates/yaak-models", "crates/yaak-plugins", + "crates/yaak-query", "crates/yaak-sse", "crates/yaak-sync", "crates/yaak-templates", diff --git a/vite.config.ts b/vite.config.ts index d009150e4..88ca8c9af 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -10,6 +10,7 @@ export default defineConfig({ ignorePatterns: [ "**/bindings/**", "**/routeTree.gen.ts", + "crates/yaak-query/pkg/**", "crates/yaak-templates/pkg/**", "crates/yaak-wasm/pkg/**", "plugins/importer-bruno/tests/fixtures/upstream/**", @@ -18,6 +19,7 @@ export default defineConfig({ lint: { ignorePatterns: [ "npm/**", + "crates/yaak-query/pkg/**", "crates/yaak-templates/pkg/**", "crates/yaak-wasm/pkg/**", "**/bindings/gen_*.ts",