Fix JSON breadcrumb filters for literal keys (#689)

apps/yaak-client/components/core/Editor/json/jsonPath.test.ts

@@ -116,7 +116,20 @@ describe("jsonPathToSegments", () => {
   });
 
   test("returns null for paths that don't name a single location", () => {
-    for (const path of ["$..id", "$[*]", "$.items[?(@.id)]", "$[0:2]", "items[0]", "$.a b"]) {
+    for (const path of [
+      "$..id",
+      "$[*]",
+      "$.items[?(@.id)]",
+      "$[0:2]",
+      "items[0]",
+      "$.a b",
+      '$["\\q"]',
+      '$["unterminated]',
+      '$["line\nbreak"]',
+      "$[01]",
+      "$[9007199254740993]",
+      "$.$",
+    ]) {
       expect(jsonPathToSegments(path)).toBeNull();
     }
   });

apps/yaak-client/components/core/Editor/json/jsonPath.ts

@@ -1,15 +1,10 @@
 import { syntaxTree } from "@codemirror/language";
 import type { EditorState } from "@codemirror/state";
 import type { SyntaxNode } from "@lezer/common";
+import type { JsonPathSegment } from "@yaakapp-internal/lib/jsonPath";
 
-/**
- * One step of a path into a JSON document. A `key` names an object member; an
- * `index` names a position within an array. They are kept separate so the UI
- * can show `items > 2` rather than folding the index into the key name.
- */
-export type JsonPathSegment =
-  | { readonly kind: "key"; readonly key: string }
-  | { readonly kind: "index"; readonly index: number };
+export { jsonPathToSegments } from "@yaakapp-internal/lib/jsonPath";
+export type { JsonPathSegment } from "@yaakapp-internal/lib/jsonPath";
 
 // Lezer node names from the JSONC grammar (@shopify/lang-jsonc). A JSON value is
 // exactly one of these; everything else in the tree is punctuation or a comment.
@@ -90,7 +85,8 @@ const BARE_KEY = /^[A-Za-z_$][A-Za-z0-9_$]*$/;
 /** A single segment as JSONPath, choosing dot vs. bracket form for a key. */
 function segmentToJsonPath(segment: JsonPathSegment): string {
   if (segment.kind === "index") return `[${segment.index}]`;
-  if (BARE_KEY.test(segment.key)) return `.${segment.key}`;
+  // jsonpath-plus interprets a bare `$` as its root operator.
+  if (BARE_KEY.test(segment.key) && segment.key !== "$") return `.${segment.key}`;
   // Anything with a dot, space, quote, etc. must be a quoted bracket accessor.
   // JSON.stringify gives correct double-quoting and escaping, matching the
   // convention the JSONPath filter box already displays.
@@ -104,31 +100,3 @@ function segmentToJsonPath(segment: JsonPathSegment): string {
 export function segmentsToJsonPath(segments: JsonPathSegment[], count = segments.length): string {
   return "$" + segments.slice(0, count).map(segmentToJsonPath).join("");
 }
-
-const PATH_SEGMENT = /^(?:\.([A-Za-z_$][A-Za-z0-9_$]*)|\[(\d+)\]|\[("(?:[^"\\]|\\.)*")\])/;
-
-/**
- * Parse a JSONPath that names a single location (`$.a[0]["b.c"]`) back into
- * segments. Returns `null` for anything else, like wildcards or recursive descent.
- */
-export function jsonPathToSegments(path: string): JsonPathSegment[] | null {
-  let rest = path.trim();
-  if (!rest.startsWith("$")) return null;
-  rest = rest.slice(1);
-
-  const segments: JsonPathSegment[] = [];
-  while (rest.length > 0) {
-    const match = PATH_SEGMENT.exec(rest);
-    if (match == null) return null;
-    const [whole, bareKey, index, quotedKey] = match;
-    if (bareKey != null) {
-      segments.push({ kind: "key", key: bareKey });
-    } else if (index != null) {
-      segments.push({ kind: "index", index: Number(index) });
-    } else if (quotedKey != null) {
-      segments.push({ kind: "key", key: JSON.parse(quotedKey) as string });
-    }
-    rest = rest.slice(whole.length);
-  }
-  return segments;
-}

package-lock.json

@@ -15101,6 +15101,7 @@
       "name": "@yaak/filter-jsonpath",
       "version": "0.1.0",
       "dependencies": {
+        "@yaakapp-internal/lib": "^1.0.0",
         "jsonpath-plus": "^10.3.0"
       },
       "devDependencies": {

packages/common-lib/jsonPath.ts

@@ -0,0 +1,39 @@
+/** Literal steps into a JSON response, distinct from JSONPath operators. */
+export type JsonPathSegment =
+  | { readonly kind: "key"; readonly key: string }
+  | { readonly kind: "index"; readonly index: number };
+
+const PATH_SEGMENT = /^(?:\.([A-Za-z_$][A-Za-z0-9_$]*)|\[(0|[1-9]\d*)\]|\[("(?:[^"\\]|\\.)*")\])/;
+
+/**
+ * Parse a single location with JSON-quoted keys (`$.a[0]["b.c"]`). Other
+ * JSONPath expressions belong to the filter engine, not to the breadcrumb trail.
+ */
+export function jsonPathToSegments(path: string): JsonPathSegment[] | null {
+  let rest = path.trim();
+  if (!rest.startsWith("$")) return null;
+  rest = rest.slice(1);
+
+  const segments: JsonPathSegment[] = [];
+  while (rest.length > 0) {
+    const match = PATH_SEGMENT.exec(rest);
+    if (match == null) return null;
+    const [whole, bareKey, index, quotedKey] = match;
+    if (bareKey != null) {
+      if (bareKey === "$") return null; // The library's root operator, not a literal key.
+      segments.push({ kind: "key", key: bareKey });
+    } else if (index != null) {
+      const value = Number(index);
+      if (!Number.isSafeInteger(value)) return null;
+      segments.push({ kind: "index", index: value });
+    } else if (quotedKey != null) {
+      try {
+        segments.push({ kind: "key", key: JSON.parse(quotedKey) as string });
+      } catch {
+        return null;
+      }
+    }
+    rest = rest.slice(whole.length);
+  }
+  return segments;
+}

plugins/filter-jsonpath/package.json

@@ -11,9 +11,11 @@
   },
   "scripts": {
     "build": "yaakcli build",
-    "dev": "yaakcli dev"
+    "dev": "yaakcli dev",
+    "test": "vp test --run tests"
   },
   "dependencies": {
+    "@yaakapp-internal/lib": "^1.0.0",
     "jsonpath-plus": "^10.3.0"
   },
   "devDependencies": {

plugins/filter-jsonpath/src/index.ts

@@ -1,4 +1,5 @@
 import type { PluginDefinition } from "@yaakapp/api";
+import { jsonPathToSegments } from "@yaakapp-internal/lib/jsonPath";
 import { JSONPath } from "jsonpath-plus";
 
 export const plugin: PluginDefinition = {
@@ -8,7 +9,24 @@ export const plugin: PluginDefinition = {
     onFilter(_ctx, args) {
       const parsed = JSON.parse(args.payload);
       try {
-        const filtered = JSONPath({ path: args.filter, json: parsed });
+        // jsonpath-plus doesn't decode JSON-quoted keys reliably and can treat
+        // their contents as operators. Only complete literal paths take this
+        // route; all other expressions retain the library's existing behavior.
+        const segments = args.filter.includes('["') ? jsonPathToSegments(args.filter) : null;
+        let filtered: unknown;
+        if (segments != null) {
+          let value: unknown = parsed;
+          for (const segment of segments) {
+            const key = segment.kind === "key" ? segment.key : segment.index;
+            if (value == null || !Object.prototype.hasOwnProperty.call(value, key)) {
+              return { content: "[]" };
+            }
+            value = (value as Record<string, unknown>)[key];
+          }
+          filtered = [value];
+        } else {
+          filtered = JSONPath({ path: args.filter, json: parsed });
+        }
         return { content: JSON.stringify(filtered, null, 2) };
       } catch (err) {
         return {

plugins/filter-jsonpath/tests/breadcrumbs.test.ts

@@ -0,0 +1,130 @@
+import { forceParsing } from "@codemirror/language";
+import { EditorState } from "@codemirror/state";
+import { jsonc } from "@shopify/lang-jsonc";
+import { JSONPath } from "jsonpath-plus";
+import { describe, expect, test } from "vite-plus/test";
+import {
+  jsonPathSegmentsAt,
+  jsonPathToSegments,
+  segmentsToJsonPath,
+} from "../../../apps/yaak-client/components/core/Editor/json/jsonPath";
+import { plugin } from "../src";
+
+async function filter(json: unknown, path: string) {
+  const result = await plugin.filter!.onFilter({} as never, {
+    payload: JSON.stringify(json),
+    filter: path,
+    mimeType: "application/json",
+  });
+  expect(result.error).toBeUndefined();
+  return JSON.parse(result.content);
+}
+
+/** Exercise the caret path and expression the breadcrumb button actually sends. */
+function breadcrumb(json: unknown) {
+  const doc = JSON.stringify(json, null, 2);
+  const state = EditorState.create({ doc, extensions: [jsonc()] });
+  forceParsing({ state } as never, doc.length, 5000);
+  const segments = jsonPathSegmentsAt(state, doc.indexOf('"needle-value"') + 4);
+  expect(segments).not.toBeNull();
+  const path = segmentsToJsonPath(segments ?? []);
+  expect(jsonPathToSegments(path)).toEqual(segments);
+  return path;
+}
+
+describe("breadcrumb paths through the real filter plugin", () => {
+  test.each([
+    "identifier",
+    "a.b",
+    "space key",
+    "",
+    'quote"key',
+    "single'key",
+    "back\\slash",
+    "bracket]key",
+    "combined\"\\]'key",
+    "*",
+    "..",
+    "$",
+    "^",
+    "~",
+    "a,b",
+    "0:2",
+    "?(true)",
+    "(@.length-1)",
+    "@number()",
+    "`key",
+    "x)]y",
+    "x)']y",
+    "semi;colon",
+    "percent%@%key",
+    "line\nbreak\t\u0000",
+    "emoji 🤔",
+    "lone surrogate \ud800",
+    "__proto__",
+    "constructor",
+  ])("selects the literal key %j, not an operator or expression", async (key) => {
+    const json = Object.fromEntries([
+      [key, "needle-value"],
+      ["decoy", "wrong-value"],
+    ]);
+    expect(await filter(json, breadcrumb(json))).toEqual(["needle-value"]);
+  });
+
+  test("nested unusual keys and arrays round-trip and filter at every crumb", async () => {
+    const inner = Object.fromEntries([["inner'\\]", { "": "needle-value" }]]);
+    const json = { outer: Object.fromEntries([['quote"key', ["decoy", inner]]]) };
+    const path = breadcrumb(json);
+    const segments = jsonPathToSegments(path)!;
+    expect(await filter(json, path)).toEqual(["needle-value"]);
+    expect(await filter(json, segmentsToJsonPath(segments, 4))).toEqual([{ "": "needle-value" }]);
+    expect(await filter(json, segmentsToJsonPath(segments, 3))).toEqual([inner]);
+    expect(await filter(json, segmentsToJsonPath(segments, 2))).toEqual([["decoy", inner]]);
+    expect(await filter(json, segmentsToJsonPath(segments, 1))).toEqual([json.outer]);
+    expect(await filter(json, segmentsToJsonPath(segments, 0))).toEqual([json]);
+  });
+
+  test("root arrays and quoted numeric object keys remain distinct", async () => {
+    const json = ["decoy", { "01": { "1": "needle-value" } }];
+    expect(await filter(json, breadcrumb(json))).toEqual(["needle-value"]);
+    expect(await filter(json, '$[1]["01"]["missing"]')).toEqual([]);
+  });
+
+  test.each([null, false, 0, "", [1, 2], { value: 3 }])(
+    "retains the matched value %j",
+    async (value) => {
+      expect(await filter({ 'quote"key': value }, '$["quote\\\"key"]')).toEqual([value]);
+    },
+  );
+
+  test("only follows own properties, while allowing actual prototype-named JSON keys", async () => {
+    const json = JSON.parse('{"a.b":{"__proto__":{"constructor":"own-value"}}}');
+    expect(await filter(json, '$["a.b"].__proto__.constructor')).toEqual(["own-value"]);
+    expect(await filter(json, '$["a.b"].constructor')).toEqual([]);
+    expect(await filter(json, '$["a.b"].__proto__.toString')).toEqual([]);
+    expect(await filter(json, '$["a.b"].__proto__.constructor.__proto__')).toEqual([]);
+  });
+});
+
+describe("existing JSONPath expressions", () => {
+  const json = { items: [{ id: 1 }, { id: 2 }, { id: 3 }], "space key": 4 };
+  test.each([
+    ["$", [json]],
+    ["$.items[1].id", [2]],
+    ['$["space key"]', [4]],
+    ["$['space key']", [4]],
+    ["$.items[*].id", [1, 2, 3]],
+    ["$..id", [1, 2, 3]],
+    ["$.items[0:2].id", [1, 2]],
+    ["$.items[0,2].id", [1, 3]],
+    ["$.items[?(@.id > 1)].id", [2, 3]],
+    ['$.items[?(@["id"] > 1)].id', [2, 3]],
+    ["$.items[(@.length-1)].id", [3]],
+    ['$["items"][*].id', [1, 2, 3]],
+    ['$["items"].$', [json.items]],
+    ["$.missing", []],
+  ])("preserves %s", async (path, expected) => {
+    expect(await filter(json, path as string)).toEqual(expected);
+    expect(await filter(json, path as string)).toEqual(JSONPath({ path: path as string, json }));
+  });
+});