From 161db4de466c439cb40f56805dcbb06d76c29d12 Mon Sep 17 00:00:00 2001 From: Gregory Schier Date: Sat, 20 Jun 2026 00:30:37 -0700 Subject: [PATCH] Fix filter field value highlighting --- .../core/Editor/filter/extension.ts | 4 +- .../core/Editor/filter/filter.grammar | 10 +++-- .../core/Editor/filter/filter.test.ts | 42 +++++++++++++++++++ .../components/core/Editor/filter/filter.ts | 16 +++---- .../core/Editor/filter/format.test.ts | 22 +++++++--- .../components/core/Editor/filter/format.ts | 6 +-- .../core/Editor/filter/highlight.ts | 2 +- .../components/core/Editor/filter/query.ts | 16 +++++-- 8 files changed, 90 insertions(+), 28 deletions(-) create mode 100644 apps/yaak-client/components/core/Editor/filter/filter.test.ts diff --git a/apps/yaak-client/components/core/Editor/filter/extension.ts b/apps/yaak-client/components/core/Editor/filter/extension.ts index 722c442a..f98c608c 100644 --- a/apps/yaak-client/components/core/Editor/filter/extension.ts +++ b/apps/yaak-client/components/core/Editor/filter/extension.ts @@ -16,8 +16,8 @@ export interface FilterOptions { } const FIELD_IDENT = /[A-Za-z0-9_/]+$/; -const VALUE_IDENT = /[A-Za-z0-9_\-./]+$/; -const VALUE_IDENT_ONLY = /^[A-Za-z0-9_\-./]+$/; +const VALUE_IDENT = /\S+$/; +const VALUE_IDENT_ONLY = /^\S+$/; function normalizeFields(fields: FieldDef[]): { fieldNames: string[]; diff --git a/apps/yaak-client/components/core/Editor/filter/filter.grammar b/apps/yaak-client/components/core/Editor/filter/filter.grammar index 077ab945..dcf8b81e 100644 --- a/apps/yaak-client/components/core/Editor/filter/filter.grammar +++ b/apps/yaak-client/components/core/Editor/filter/filter.grammar @@ -2,7 +2,7 @@ @skip { space+ } @tokens { - space { std.whitespace+ } + space { $[ \t\r\n]+ } LParen { "(" } RParen { ")" } @@ -17,8 +17,10 @@ // "quoted phrase" with simple escapes: \" and \\ Phrase { '"' (!["\\] | "\\" _)* '"' } - // field/word characters (keep generous for URLs/paths) - Word { $[A-Za-z0-9_]+ } + // Bare words run until filter syntax or whitespace. Leading '-' remains unary + // negation, but '-' may appear after the first character. + Word { ![ \t\r\n():"@-] ![ \t\r\n():"@]* } + FieldValueWord { ![ \t\r\n"] ![ \t\r\n]* } @precedence { Not, And, Or, Word } } @@ -66,7 +68,7 @@ FieldName { FieldValue { Phrase -| Term +| FieldValueWord } Term { diff --git a/apps/yaak-client/components/core/Editor/filter/filter.test.ts b/apps/yaak-client/components/core/Editor/filter/filter.test.ts new file mode 100644 index 00000000..6002175f --- /dev/null +++ b/apps/yaak-client/components/core/Editor/filter/filter.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, test } from "vite-plus/test"; +import { parser } from "./filter"; + +function getNodeNames(input: string): string[] { + const tree = parser.parse(input); + const nodes: string[] = []; + const cursor = tree.cursor(); + do { + if (cursor.name !== "Query") { + nodes.push(cursor.name); + } + } while (cursor.next()); + return nodes; +} + +describe("filter grammar", () => { + test("parses URL-like field values as one value", () => { + const nodes = getNodeNames("@url:yaak.app/foo-bar"); + + expect(nodes).not.toContain("⚠"); + expect(nodes).toContain("FieldValue"); + expect(nodes).toContain("FieldValueWord"); + }); + + test("parses punctuation-heavy field values as one value", () => { + const nodes = getNodeNames("@url:yaa$&#*@tsrna(*)"); + + expect(nodes).not.toContain("⚠"); + expect(nodes).toContain("FieldValue"); + expect(nodes).toContain("FieldValueWord"); + }); + + test("parses operator-looking field values as one value", () => { + const negativeValueNodes = getNodeNames("@url:-foo"); + const operatorWordNodes = getNodeNames("@url:AND"); + + expect(negativeValueNodes).not.toContain("⚠"); + expect(negativeValueNodes).toContain("FieldValueWord"); + expect(operatorWordNodes).not.toContain("⚠"); + expect(operatorWordNodes).toContain("FieldValueWord"); + }); +}); diff --git a/apps/yaak-client/components/core/Editor/filter/filter.ts b/apps/yaak-client/components/core/Editor/filter/filter.ts index 65767606..717a6daa 100644 --- a/apps/yaak-client/components/core/Editor/filter/filter.ts +++ b/apps/yaak-client/components/core/Editor/filter/filter.ts @@ -3,20 +3,20 @@ import {LRParser} from "@lezer/lr" import {highlight} from "./highlight" export const parser = LRParser.deserialize({ version: 14, - states: "%^OVQPOOPhOPOOOVQPO'#CfOmQPO'#ChO!_QPO'#ChO!dQPO'#CgOOQO'#Cc'#CcOVQPO'#CaOOQO'#Ca'#CaO!iQPO'#C`O!yQPO'#C_OOQO'#C^'#C^QOQPOOPOOO'#Cq'#CqP#UOPO)C>kO#]QPO,59QOOQO,59S,59SO#bQPO,59ROOQO,58{,58{OVQPO'#CrOOQO'#Cr'#CrO#jQPO,58zOVQPO'#CsO#zQPO,58yPOOO-E6o-E6oOOQO1G.l1G.lOOQO'#Cn'#CnOOQO'#Cl'#ClOOQO1G.m1G.mOOQO,59^,59^OOQO-E6p-E6pOOQO,59_,59_OOQO-E6q-E6q", - stateData: "$]~OjPQ~OUVOXQO]SO^ROaUO~Oj]O~OUbXXbX]bX^bX_[XabXcbXdbXhbXWbX~O^`O~O_aO~OccOdSXhSXWSX~PVOdfOhRXWRX~Oj]O~Qi]WiO~O^jOakO~OccOdSahSaWSa~PVOdfOhRaWRa~OUcd^d~", - goto: "#ihPPioszP!ZPP!d!d!mPPP!vP!yPP#V#]#cQ[OR_QTZOQSYOQRofUXOQfQbVSdXeRmc_WOQVXcef_UOQVXcef_TOQVXcefRla^UOQVXcefRkaQ^PRh^QeXRneQgYRpg", - nodeNames: "⚠ Query Expr OrExpr AndExpr Unary Not Primary RParen LParen Group Field FieldName At Word Colon FieldValue Phrase Term And Or", - maxTerm: 26, + states: "%WOVQPOOPhOPOOOVQPO'#CfOmQPO'#ChO!_QPO'#ChO!dQPO'#CgOOQO'#Cc'#CcOVQPO'#CaOOQO'#Ca'#CaO!iQPO'#C`O!yQPO'#C_OOQO'#C^'#C^QOQPOOPOOO'#Cr'#CrP#UOPO)C>lO#]QPO,59QOOQO,59S,59SO#bQQO,59ROOQO,58{,58{OVQPO'#CsOOQO'#Cs'#CsO#jQPO,58zOVQPO'#CtO#zQPO,58yPOOO-E6p-E6pOOQO1G.l1G.lOOQO'#Cl'#ClOOQO1G.m1G.mOOQO,59_,59_OOQO-E6q-E6qOOQO,59`,59`OOQO-E6r-E6r", + stateData: "$]~OkPQ~OUVOXQO]SO^ROaUO~Ok]O~OUcXXcX]cX^cX_[XacXdcXecXicXWcX~O^`O~O_aO~OdcOeSXiSXWSX~PVOefOiRXWRX~Ok]O~Qj]WiO~OajObjO~OdcOeSaiSaWSa~PVOefOiRaWRa~OUde^e~", + goto: "#^iPPjpt{P![PP!e!e!nPPP!wPP!ePP!z#Q#WQ[OR_QTZOQSYOQRnfUXOQfQbVSdXeRlc_WOQVXcef_UOQVXcef_TOQVXcefRkaQ^PRh^QeXRmeQgYRog", + nodeNames: "⚠ Query Expr OrExpr AndExpr Unary Not Primary RParen LParen Group Field FieldName At Word Colon FieldValue Phrase FieldValueWord Term And Or", + maxTerm: 27, nodeProps: [ ["openedBy", 8,"LParen"], ["closedBy", 9,"RParen"] ], propSources: [highlight], - skippedNodes: [0,21], + skippedNodes: [0,22], repeatNodeCount: 3, - tokenData: ")n~RhX^!mpq!mrs#bxy%Oyz%T}!O%Y!Q![%_![!]%p!b!c%u!c!d%z!d!p%_!p!q'_!q!r(r!r!}%_#R#S%_#T#o%_#y#z!m$f$g!m#BY#BZ!m$IS$I_!m$I|$JO!m$JT$JU!m$KV$KW!m&FU&FV!m~!rYj~X^!mpq!m#y#z!m$f$g!m#BY#BZ!m$IS$I_!m$I|$JO!m$JT$JU!m$KV$KW!m&FU&FV!m~#eVOr#brs#zs#O#b#O#P$P#P;'S#b;'S;=`$x<%lO#b~$POa~~$SRO;'S#b;'S;=`$];=`O#b~$`WOr#brs#zs#O#b#O#P$P#P;'S#b;'S;=`$x;=`<%l#b<%lO#b~${P;=`<%l#b~%TOX~~%YOW~~%_OU~~%dS^~!Q![%_!c!}%_#R#S%_#T#o%_~%uO_~~%zO]~~&PU^~!Q![%_!c!p%_!p!q&c!q!}%_#R#S%_#T#o%_~&hU^~!Q![%_!c!f%_!f!g&z!g!}%_#R#S%_#T#o%_~'RSc~^~!Q![%_!c!}%_#R#S%_#T#o%_~'dU^~!Q![%_!c!q%_!q!r'v!r!}%_#R#S%_#T#o%_~'{U^~!Q![%_!c!v%_!v!w(_!w!}%_#R#S%_#T#o%_~(fSU~^~!Q![%_!c!}%_#R#S%_#T#o%_~(wU^~!Q![%_!c!t%_!t!u)Z!u!}%_#R#S%_#T#o%_~)bSd~^~!Q![%_!c!}%_#R#S%_#T#o%_", - tokenizers: [0], + tokenData: "2h~RiOX!pXY$hYZ$hZ]!p]^$h^p!ppq$hqr!prs$ysx!pxy&gyz'Qz}!p}!O'k!O![!p![!](U!]!b!p!b!c(o!c!d)Y!d!p!p!p!q,q!q!r0Y!r;'S!p;'S;=`$b<%lO!pR!w^bQ^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c;'S!p;'S;=`$b<%lO!pQ#xUbQOX#sZ]#s^p#sq;'S#s;'S;=`$[<%lO#sQ$_P;=`<%l#sR$eP;=`<%l!p~$mSk~XY$hYZ$h]^$hpq$h~$|VOr$yrs%cs#O$y#O#P%h#P;'S$y;'S;=`&a<%lO$y~%hOa~~%kRO;'S$y;'S;=`%t;=`O$y~%wWOr$yrs%cs#O$y#O#P%h#P;'S$y;'S;=`&a;=`<%l$y<%lO$y~&dP;=`<%l$yR&nUbQXPOX#sZ]#s^p#sq;'S#s;'S;=`$[<%lO#sR'XUbQWPOX#sZ]#s^p#sq;'S#s;'S;=`$[<%lO#sR'rUbQUPOX#sZ]#s^p#sq;'S#s;'S;=`$[<%lO#sR(]U_PbQOX#sZ]#s^p#sq;'S#s;'S;=`$[<%lO#sR(vU]PbQOX#sZ]#s^p#sq;'S#s;'S;=`$[<%lO#sR)a`bQ^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c!p!p!p!q*c!q;'S!p;'S;=`$b<%lO!pR*j`bQ^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c!f!p!f!g+l!g;'S!p;'S;=`$b<%lO!pR+u^bQdP^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c;'S!p;'S;=`$b<%lO!pR,x`bQ^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c!q!p!q!r-z!r;'S!p;'S;=`$b<%lO!pR.R`bQ^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c!v!p!v!w/T!w;'S!p;'S;=`$b<%lO!pR/^^bQUP^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c;'S!p;'S;=`$b<%lO!pR0a`bQ^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c!t!p!t!u1c!u;'S!p;'S;=`$b<%lO!pR1l^bQeP^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c;'S!p;'S;=`$b<%lO!p", + tokenizers: [0, 1], topRules: {"Query":[0,1]}, tokenPrec: 145 }) diff --git a/apps/yaak-client/components/core/Editor/filter/format.test.ts b/apps/yaak-client/components/core/Editor/filter/format.test.ts index c7f90a5c..07ba7afe 100644 --- a/apps/yaak-client/components/core/Editor/filter/format.test.ts +++ b/apps/yaak-client/components/core/Editor/filter/format.test.ts @@ -14,15 +14,20 @@ describe("formatFieldFilter", () => { expect(matchesFormattedUrl("yaak.app/foo-bar")).toBe(true); }); - test("quotes values that start with an operator token", () => { - expect(formatFieldFilter("url", "-foo")).toBe('@url:"-foo"'); + test("keeps non-syntax punctuation bare", () => { + expect(formatFieldFilter("url", "yaa$&#*@tsrna(*)")).toBe("@url:yaa$&#*@tsrna(*)"); + expect(matchesFormattedUrl("yaa$&#*@tsrna(*)")).toBe(true); + }); + + test("keeps values that start with an operator token bare", () => { + expect(formatFieldFilter("url", "-foo")).toBe("@url:-foo"); expect(matchesFormattedUrl("-foo")).toBe(true); }); - test("quotes boolean operator words", () => { - expect(formatFieldFilter("url", "AND")).toBe('@url:"AND"'); - expect(formatFieldFilter("url", "or")).toBe('@url:"or"'); - expect(formatFieldFilter("url", "Not")).toBe('@url:"Not"'); + test("keeps boolean operator words bare", () => { + expect(formatFieldFilter("url", "AND")).toBe("@url:AND"); + expect(formatFieldFilter("url", "or")).toBe("@url:or"); + expect(formatFieldFilter("url", "Not")).toBe("@url:Not"); expect(matchesFormattedUrl("AND")).toBe(true); }); @@ -30,4 +35,9 @@ describe("formatFieldFilter", () => { expect(formatFieldFilter("url", 'say "hi"')).toBe('@url:"say \\"hi\\""'); expect(matchesFormattedUrl('say "hi"')).toBe(true); }); + + test("quotes values that start with a quote", () => { + expect(formatFieldFilter("url", '"hi"')).toBe('@url:"\\"hi\\""'); + expect(matchesFormattedUrl('"hi"')).toBe(true); + }); }); diff --git a/apps/yaak-client/components/core/Editor/filter/format.ts b/apps/yaak-client/components/core/Editor/filter/format.ts index 96173353..f1fc7215 100644 --- a/apps/yaak-client/components/core/Editor/filter/format.ts +++ b/apps/yaak-client/components/core/Editor/filter/format.ts @@ -1,9 +1,7 @@ -const bareFieldValue = /^[A-Za-z0-9_./][A-Za-z0-9_\-./]*$/; -const operatorWord = /^(?:AND|OR|NOT)$/i; +const bareFieldValue = /^[^\s"]\S*$/; export function formatFieldFilter(field: string, value: string) { const escapedValue = value.replace(/\\/g, "\\\\").replace(/"/g, '\\"'); - const filterValue = - bareFieldValue.test(value) && !operatorWord.test(value) ? value : `"${escapedValue}"`; + const filterValue = bareFieldValue.test(value) ? value : `"${escapedValue}"`; return `@${field}:${filterValue}`; } diff --git a/apps/yaak-client/components/core/Editor/filter/highlight.ts b/apps/yaak-client/components/core/Editor/filter/highlight.ts index c6092819..9c05d726 100644 --- a/apps/yaak-client/components/core/Editor/filter/highlight.ts +++ b/apps/yaak-client/components/core/Editor/filter/highlight.ts @@ -18,5 +18,5 @@ export const highlight = styleTags({ // Fields "FieldName/At": t.attributeName, "FieldName/Word": t.attributeName, - "FieldValue/Term/Word": t.attributeValue, + "FieldValue/FieldValueWord": t.attributeValue, }); diff --git a/apps/yaak-client/components/core/Editor/filter/query.ts b/apps/yaak-client/components/core/Editor/filter/query.ts index 6c3eb5be..12bf4a4b 100644 --- a/apps/yaak-client/components/core/Editor/filter/query.ts +++ b/apps/yaak-client/components/core/Editor/filter/query.ts @@ -30,7 +30,8 @@ type Tok = | { kind: "EOF" }; const isSpace = (c: string) => /\s/.test(c); -const isIdent = (c: string) => /[A-Za-z0-9_\-./]/.test(c); +const isWordStart = (c: string) => c !== "" && !isSpace(c) && !/[():"@-]/.test(c); +const isWordChar = (c: string) => c !== "" && !isSpace(c) && !/[():"@]/.test(c); export function tokenize(input: string): Tok[] { const toks: Tok[] = []; @@ -42,7 +43,13 @@ export function tokenize(input: string): Tok[] { const readWord = () => { let s = ""; - while (i < n && isIdent(peek())) s += advance(); + while (i < n && isWordChar(peek())) s += advance(); + return s; + }; + + const readFieldValue = () => { + let s = ""; + while (i < n && !isSpace(peek())) s += advance(); return s; }; @@ -85,6 +92,9 @@ export function tokenize(input: string): Tok[] { if (c === ":") { toks.push({ kind: "COLON" }); i++; + if (peek() && !isSpace(peek()) && peek() !== `"`) { + toks.push({ kind: "WORD", text: readFieldValue() }); + } continue; } if (c === `"`) { @@ -99,7 +109,7 @@ export function tokenize(input: string): Tok[] { } // WORD / AND / OR / NOT - if (isIdent(c)) { + if (isWordStart(c)) { const w = readWord(); const upper = w.toUpperCase(); if (upper === "AND") toks.push({ kind: "AND" });