From 1129f7aa939458a31f6f3eedd8746ee89e9559d7 Mon Sep 17 00:00:00 2001 From: Gregory Schier Date: Fri, 6 Nov 2026 09:53:56 -0700 Subject: [PATCH 01/45] Capture WebSocket, SSE and gRPC (#814) Co-authored-by: Claude Opus 5.5 --- Cargo.lock | 71 ++ apps/yaak-proxy/harness/fixtures.ts | 6 + apps/yaak-proxy/lib/flowViews.test.ts | 6 + crates-proxy/yaak-proxy-lib/Cargo.toml | 11 +- .../yaak-proxy-lib/bindings/gen_models.ts | 137 ++- .../yaak-proxy-lib/bindings/gen_rpc.ts | 92 +- .../migrations/00000006_flow_messages.sql | 44 + crates-proxy/yaak-proxy-lib/src/client.rs | 2 +- crates-proxy/yaak-proxy-lib/src/control.rs | 92 +- crates-proxy/yaak-proxy-lib/src/db.rs | 19 +- crates-proxy/yaak-proxy-lib/src/flows.rs | 24 +- crates-proxy/yaak-proxy-lib/src/grpc.rs | 595 +++++++++++++ crates-proxy/yaak-proxy-lib/src/har.rs | 78 +- crates-proxy/yaak-proxy-lib/src/lib.rs | 826 +++++++++++++++++- crates-proxy/yaak-proxy-lib/src/messages.rs | 368 ++++++++ crates-proxy/yaak-proxy-lib/src/models.rs | 77 ++ crates-proxy/yaak-proxy-lib/tests/streams.rs | 437 +++++++++ crates-tauri/yaak-app-proxy/src/lib.rs | 46 +- .../yaak-app-proxy/tauri.bundle.conf.json | 4 + crates/yaak-grpc/src/lib.rs | 8 + crates/yaak-proxy/Cargo.toml | 4 +- crates/yaak-proxy/src/body.rs | 66 +- crates/yaak-proxy/src/decode.rs | 378 ++++++++ crates/yaak-proxy/src/grpc.rs | 346 ++++++++ crates/yaak-proxy/src/lib.rs | 34 +- crates/yaak-proxy/src/messages.rs | 113 +++ crates/yaak-proxy/src/request.rs | 110 ++- crates/yaak-proxy/src/sse.rs | 306 +++++++ crates/yaak-proxy/src/ws.rs | 419 +++++++++ crates/yaak-proxy/tests/proxy.rs | 570 +++++++++++- package.json | 3 +- scripts/sign-proxy-vendored.cjs | 36 + 32 files changed, 5228 insertions(+), 100 deletions(-) create mode 100644 crates-proxy/yaak-proxy-lib/migrations/00000006_flow_messages.sql create mode 100644 crates-proxy/yaak-proxy-lib/src/grpc.rs create mode 100644 crates-proxy/yaak-proxy-lib/src/messages.rs create mode 100644 crates-proxy/yaak-proxy-lib/tests/streams.rs create mode 100644 crates/yaak-proxy/src/decode.rs create mode 100644 crates/yaak-proxy/src/grpc.rs create mode 100644 crates/yaak-proxy/src/messages.rs create mode 100644 crates/yaak-proxy/src/sse.rs create mode 100644 crates/yaak-proxy/src/ws.rs create mode 100644 scripts/sign-proxy-vendored.cjs diff --git a/Cargo.lock b/Cargo.lock index 9ff5f085a..005fe1520 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6480,6 +6480,70 @@ dependencies = [ "prost", ] +[[package]] +name = "protoc-bin-vendored" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8760a25b6ff9c620324822737e468478fa092234190d2e449760344354896ed9" +dependencies = [ + "protoc-bin-vendored-linux-aarch_64", + "protoc-bin-vendored-linux-ppcle_64", + "protoc-bin-vendored-linux-s390_64", + "protoc-bin-vendored-linux-x86_32", + "protoc-bin-vendored-linux-x86_64", + "protoc-bin-vendored-macos-aarch_64", + "protoc-bin-vendored-macos-x86_64", + "protoc-bin-vendored-win32", +] + +[[package]] +name = "protoc-bin-vendored-linux-aarch_64" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73fa2624782ca04cd44f51554566717377acd240e4c0016d757dd74fccc9324f" + +[[package]] +name = "protoc-bin-vendored-linux-ppcle_64" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2417e9817fa237dab803ad4dda7357a111656e242959cc6b8f9a1a583367d42" + +[[package]] +name = "protoc-bin-vendored-linux-s390_64" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d189c34636356a46a7ed3188233dc8a88c431278cc54d4a19b096a2d270e985" + +[[package]] +name = "protoc-bin-vendored-linux-x86_32" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171e39f1e846e5f322ced1ac3b8d4cd3a3833ca24b6e5d58b3632574fe6204fa" + +[[package]] +name = "protoc-bin-vendored-linux-x86_64" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "873cdcc097593432086661aa432b8078f1cd87bfb02847c332e98ae2c119e966" + +[[package]] +name = "protoc-bin-vendored-macos-aarch_64" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeb72df001783b8297847fe8f5f874ee400fd742c843d60583e8c23d96977c7f" + +[[package]] +name = "protoc-bin-vendored-macos-x86_64" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b04652167eca899dda05f32f5481adeaf25c623a98ce2fc146a001cc59a2add7" + +[[package]] +name = "protoc-bin-vendored-win32" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "263a3f48f01e7309e857138bd47f785585b4a005e8e56c6d2824ce91195999c3" + [[package]] name = "quick-error" version = "1.2.3" @@ -11813,7 +11877,9 @@ dependencies = [ name = "yaak-proxy" version = "0.1.0" dependencies = [ + "base64 0.23.1", "bytes", + "flate2", "futures-util", "http", "http-body-util", @@ -11854,6 +11920,7 @@ dependencies = [ "chrono", "encoding_rs", "flate2", + "futures-util", "http-body-util", "hyper", "hyper-util", @@ -11865,6 +11932,7 @@ dependencies = [ "objc2-foundation 0.3.2", "objc2-service-management", "plist", + "protoc-bin-vendored", "r2d2", "r2d2_sqlite", "reqwest", @@ -11874,11 +11942,14 @@ dependencies = [ "serde_json", "tempfile", "tokio", + "tonic", + "tonic-reflection", "ts-rs", "url", "yaak-common", "yaak-database", "yaak-filter", + "yaak-grpc", "yaak-http", "yaak-lifecycle", "yaak-models", diff --git a/apps/yaak-proxy/harness/fixtures.ts b/apps/yaak-proxy/harness/fixtures.ts index bc6966b45..57086aefc 100644 --- a/apps/yaak-proxy/harness/fixtures.ts +++ b/apps/yaak-proxy/harness/fixtures.ts @@ -42,6 +42,12 @@ const base: Flow = { clientName: "curl", clientBundleId: null, clientPath: "/usr/bin/curl", + streamKind: null, + messageCount: 0, + grpcService: null, + grpcMethod: null, + grpcStatus: null, + grpcMessage: null, }; const jsonRequest = JSON.stringify({ amount: 4200, currency: "usd", capture: true }); diff --git a/apps/yaak-proxy/lib/flowViews.test.ts b/apps/yaak-proxy/lib/flowViews.test.ts index 0902ca2bf..23c6bc288 100644 --- a/apps/yaak-proxy/lib/flowViews.test.ts +++ b/apps/yaak-proxy/lib/flowViews.test.ts @@ -50,6 +50,12 @@ const flow: Flow = { clientName: null, clientBundleId: null, clientPath: null, + streamKind: null, + messageCount: 0, + grpcService: null, + grpcMethod: null, + grpcStatus: null, + grpcMessage: null, }; function bytes(text: string): FlowBodyData { diff --git a/crates-proxy/yaak-proxy-lib/Cargo.toml b/crates-proxy/yaak-proxy-lib/Cargo.toml index 05845b2c7..e4a62cf6e 100644 --- a/crates-proxy/yaak-proxy-lib/Cargo.toml +++ b/crates-proxy/yaak-proxy-lib/Cargo.toml @@ -22,12 +22,13 @@ rusqlite = { version = "0.38", features = ["bundled", "chrono", "functions"] } sea-query = { version = "1.0", features = ["with-chrono", "attr"] } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } -tokio = { workspace = true, features = ["io-util", "macros", "net", "process", "rt-multi-thread"] } +tokio = { workspace = true, features = ["io-util", "macros", "net", "process", "rt-multi-thread", "sync", "time"] } ts-rs = { workspace = true, features = ["chrono-impl"] } url = "2" yaak-common = { workspace = true } yaak-database = { workspace = true } yaak-filter = { workspace = true } +yaak-grpc = { workspace = true } yaak-http = { workspace = true } yaak-lifecycle = { workspace = true } yaak-models = { workspace = true } @@ -71,9 +72,13 @@ plist = "1.7" base64 = "0.23.1" bytes = "1" flate2 = "1" +futures-util = "0.3" http-body-util = "0.1" -hyper = { version = "1", features = ["http1", "server"] } -hyper-util = { version = "0.1", features = ["tokio"] } +hyper = { version = "1", features = ["http1", "http2", "server"] } +hyper-util = { version = "0.1", features = ["tokio", "server-auto"] } +protoc-bin-vendored = "3" +tonic = { version = "0.14.6", default-features = false } +tonic-reflection = "0.14.6" reqwest = { workspace = true } tempfile = "3" tokio = { workspace = true, features = ["macros", "net", "rt-multi-thread", "time"] } diff --git a/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts b/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts index 153dd80f9..8acd262a0 100644 --- a/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts +++ b/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts @@ -65,10 +65,145 @@ clientBundleId: string | null, /** * Its app bundle's path, or its executable's outside an app. */ -clientPath: string | null, }; +clientPath: string | null, +/** + * The flow carries a stream of messages, listed with `list_messages`. It is written as the + * stream opens and again as messages land; it is open until `elapsedMs`, `error` or + * `cancelled` is set. + */ +streamKind: StreamKind | null, +/** + * Messages recorded so far. + */ +messageCount: number, +/** + * A gRPC call's service, from its path: `package.Service`. + */ +grpcService: string | null, +/** + * A gRPC call's method, from its path. + */ +grpcMethod: string | null, +/** + * A gRPC call's status, from its trailers. Not 0 is a failed call. + */ +grpcStatus: number | null, +/** + * A gRPC call's `grpc-message`, decoded. + */ +grpcMessage: string | null, }; export type FlowBody = "request" | "response"; +/** + * One message of a streaming flow. Its payload is fetched on its own. + */ +export type FlowMessage = { id: string, flowId: string, +/** + * Its place in the flow, from 0, in the order messages passed the proxy. + */ +seq: number, direction: MessageDirection, kind: MessageKind, +/** + * When it had passed the proxy whole. + */ +timestamp: string, +/** + * Bytes it took on the wire: the frames' payloads, the event's lines, the gRPC message + * without its prefix. + */ +size: number, +/** + * Bytes of payload stored: unmasked, reassembled, inflated or decompressed, and cut at the + * capture limit. + */ +storedSize: number, +/** + * The stored payload is cut short at the capture limit. + */ +truncated: boolean, +/** + * It was compressed on the wire (permessage-deflate, gRPC's compressed flag); the payload is + * stored decompressed unless `error` says otherwise. + */ +compressed: boolean, +/** + * Why the payload could not be decoded or stored. + */ +error: string | null, meta: MessageMeta, +/** + * The payload's start as text, when it is text. + */ +preview: string | null, }; + +/** + * Descriptors for decoding the gRPC calls to hosts that match `hostPattern`. + */ +export type GrpcSchema = { id: string, createdAt: string, updatedAt: string, +/** + * A host (`api.example.com`), a host and port (`localhost:50051`), any subdomain of one + * (`*.example.com`), or every host (`*`). + */ +hostPattern: string, source: GrpcSchemaSource, +/** + * The `.proto` files and include directories compiled, for a `files` schema. + */ +protoPaths: Array, +/** + * The services the descriptors define, by full name. + */ +services: Array, }; + +/** + * Where a schema's descriptors came from. + */ +export type GrpcSchemaSource = "files" | "reflection"; + +export type MessageDirection = "client" | "server"; + +export type MessageKind = "websocket_text" | "websocket_binary" | "websocket_ping" | "websocket_pong" | "websocket_close" | "sse_event" | "grpc_message" | "grpc_trailers"; + +/** + * What a kind of message carries beside its payload. Fields that do not apply are left out. + */ +export type MessageMeta = { +/** + * A WebSocket close frame's status code. + */ +closeCode?: number, +/** + * A WebSocket close frame's reason. + */ +closeReason?: string, +/** + * An SSE event's `event` field. + */ +event?: string, +/** + * An SSE event's `id` field. + */ +eventId?: string, +/** + * An SSE event's `retry` field, in milliseconds. + */ +retry?: number, +/** + * An SSE event's lines as they came, comments included, up to 4 KB. + */ +raw?: string, +/** + * gRPC trailers' `grpc-status`. + */ +grpcStatus?: number, +/** + * gRPC trailers' `grpc-message`, decoded. + */ +grpcMessage?: string, }; + export type ModelPayload = { model: Flow, change: ModelChangeEvent, }; export type ProxyHeader = { name: string, value: string, }; + +/** + * The streaming protocol a flow carries. + */ +export type StreamKind = "websocket" | "sse" | "grpc" | "grpc_web"; diff --git a/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts b/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts index 629597ae7..d80119161 100644 --- a/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts +++ b/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts @@ -1,10 +1,20 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { Flow, FlowBody, ModelPayload } from "./gen_models"; +import type { Flow, FlowBody, FlowMessage, GrpcSchema, ModelPayload } from "./gen_models"; export type ActionInvocation = { "scope": "global", action: GlobalAction, }; export type ActionMetadata = { label: string, defaultHotkey: string | null, }; +export type AddGrpcSchemaRequest = { +/** + * `api.example.com`, `localhost:50051`, `*.example.com` or `*`. + */ +hostPattern: string, +/** + * `.proto` files, and directories they import from. + */ +protoPaths: Array, }; + export type AppIconRequest = { /** * A flow's `clientPath`. @@ -44,6 +54,31 @@ bundleId: string | null, */ path: string, }; +export type DecodeGrpcMessageRequest = { flowId: string, seq: number, }; + +/** + * A captured gRPC message as JSON. + */ +export type DecodedGrpcMessage = { +/** + * Pretty JSON: the message's fields by name with a schema; without one, an array of its + * fields as `{ field, wireType, value }`, a length-delimited one with its guesses (`string`, + * `message`, `bytes` in base64). + */ +json: string, +/** + * The message type decoded with: `package.HelloReply`. None when schemaless. + */ +messageType: string | null, +/** + * The schema decoded with. None when schemaless. + */ +schema: GrpcSchema | null, +/** + * Why it decoded schemaless, or what to know about the decode. + */ +note: string | null, }; + export type DropReason = "hop_by_hop" | "set_by_sending" | "body_decoded"; export type DroppedHeader = { name: string, value: string, reason: DropReason, }; @@ -100,7 +135,8 @@ app?: Array, */ method?: string, /** - * A status (`404`), a class of them (`4xx`), or `error` for flows that failed. + * A status (`404`), a class of them (`4xx`), or `error` for flows that failed, gRPC calls + * that ended with a status other than OK among them. */ status?: string, /** @@ -149,6 +185,18 @@ storedSize: number, */ decodeError?: string, }; +export type GetMessagePayloadRequest = { flowId: string, seq: number, }; + +export type GetMessagePayloadResponse = { +/** + * The payload as stored (unmasked, inflated, decompressed), in base64. + */ +data: string, +/** + * It is cut at the capture limit. + */ +truncated: boolean, }; + export type GetProxyStateRequest = Record; export type GetProxyStateResponse = { state: ProxyState, @@ -214,6 +262,28 @@ total: number, */ matching: number | null, }; +export type ListGrpcSchemasRequest = Record; + +export type ListMessagesRequest = { flowId: string, +/** + * Only messages after this `seq`, for following a stream as it grows. + */ +afterSeq?: number, +/** + * At most this many, the first after `afterSeq`. + */ +limit?: number, }; + +export type ListMessagesResponse = { +/** + * In order, by `seq`. + */ +messages: Array, +/** + * Messages the flow has in all. + */ +total: number, }; + export type ListWorkspacesRequest = Record; export type ListWorkspacesResponse = { @@ -222,6 +292,11 @@ export type ListWorkspacesResponse = { */ yaakFound: boolean, workspaces: Array, }; +/** + * A message of a streaming flow was recorded. + */ +export type MessagePayload = { message: FlowMessage, }; + export type OtherOwner = { owner: Owner, pid: number, startedAt: string, /** * The engine's port while it runs. @@ -241,9 +316,18 @@ export type ProxyState = "running" | "stopped"; export type ProxyStatePayload = { state: ProxyState, }; -export type RpcEventSchema = { flows_cleared: FlowsClearedPayload, model_write: ModelPayload, proxy_state_changed: ProxyStatePayload, }; +export type ReflectGrpcHostRequest = { +/** + * `host` or `host:port`, over TLS on 443 unless a port is given, or either after + * `http://` to ask without TLS. + */ +host: string, }; -export type RpcSchema = { execute_action: [ActionInvocation, boolean], format_body: [FormatBodyRequest, string], list_actions: [ListActionsRequest, ListActionsResponse], list_flows: [FlowFilter, ListFlowsResponse], set_recording: [SetRecordingRequest, boolean], set_system_proxy_at_launch: [SetSystemProxyAtLaunchRequest, boolean], set_http_version: [SetHttpVersionRequest, HttpVersion], app_icon: [AppIconRequest, string | null], ca_status: [CaStatusRequest, CaStatus], ca_trust: [CaTrustRequest, CaStatus], clear_flows: [ClearFlowsRequest, number], export_har: [ExportHarRequest, number], filter_body: [FilterBodyRequest, FilterBodyResponse], get_flow_body: [GetFlowBodyRequest, GetFlowBodyResponse], get_proxy_state: [GetProxyStateRequest, GetProxyStateResponse], helper_register: [HelperRegisterRequest, SystemProxyStatus], helper_unregister: [HelperUnregisterRequest, SystemProxyStatus], list_workspaces: [ListWorkspacesRequest, ListWorkspacesResponse], preview_save: [PreviewSaveRequest, SavePreview], save_flow: [SaveFlowToYaakRequest, SavedFlow], save_flow_body: [SaveFlowBodyRequest, SaveFlowBodyResponse], system_proxy_set: [SystemProxySetRequest, SystemProxyStatus], system_proxy_status: [SystemProxyStatusRequest, SystemProxyStatus], take_over: [TakeOverRequest, boolean], }; +export type RemoveGrpcSchemaRequest = { id: string, }; + +export type RpcEventSchema = { flows_cleared: FlowsClearedPayload, message_write: MessagePayload, model_write: ModelPayload, proxy_state_changed: ProxyStatePayload, }; + +export type RpcSchema = { execute_action: [ActionInvocation, boolean], format_body: [FormatBodyRequest, string], get_message_payload: [GetMessagePayloadRequest, GetMessagePayloadResponse], list_actions: [ListActionsRequest, ListActionsResponse], list_flows: [FlowFilter, ListFlowsResponse], list_grpc_schemas: [ListGrpcSchemasRequest, Array], list_messages: [ListMessagesRequest, ListMessagesResponse], remove_grpc_schema: [RemoveGrpcSchemaRequest, boolean], set_recording: [SetRecordingRequest, boolean], set_system_proxy_at_launch: [SetSystemProxyAtLaunchRequest, boolean], set_http_version: [SetHttpVersionRequest, HttpVersion], add_grpc_schema: [AddGrpcSchemaRequest, GrpcSchema], app_icon: [AppIconRequest, string | null], ca_status: [CaStatusRequest, CaStatus], ca_trust: [CaTrustRequest, CaStatus], clear_flows: [ClearFlowsRequest, number], decode_grpc_message: [DecodeGrpcMessageRequest, DecodedGrpcMessage], export_har: [ExportHarRequest, number], filter_body: [FilterBodyRequest, FilterBodyResponse], get_flow_body: [GetFlowBodyRequest, GetFlowBodyResponse], get_proxy_state: [GetProxyStateRequest, GetProxyStateResponse], helper_register: [HelperRegisterRequest, SystemProxyStatus], helper_unregister: [HelperUnregisterRequest, SystemProxyStatus], list_workspaces: [ListWorkspacesRequest, ListWorkspacesResponse], preview_save: [PreviewSaveRequest, SavePreview], reflect_grpc_host: [ReflectGrpcHostRequest, GrpcSchema], save_flow: [SaveFlowToYaakRequest, SavedFlow], save_flow_body: [SaveFlowBodyRequest, SaveFlowBodyResponse], system_proxy_set: [SystemProxySetRequest, SystemProxyStatus], system_proxy_status: [SystemProxyStatusRequest, SystemProxyStatus], take_over: [TakeOverRequest, boolean], }; export type SaveFlowBodyRequest = { flowId: string, which: FlowBody, /** diff --git a/crates-proxy/yaak-proxy-lib/migrations/00000006_flow_messages.sql b/crates-proxy/yaak-proxy-lib/migrations/00000006_flow_messages.sql new file mode 100644 index 000000000..7852d5f90 --- /dev/null +++ b/crates-proxy/yaak-proxy-lib/migrations/00000006_flow_messages.sql @@ -0,0 +1,44 @@ +-- Streaming flows: WebSocket, server-sent events, gRPC and gRPC-Web. +ALTER TABLE flows ADD COLUMN stream_kind TEXT; +ALTER TABLE flows ADD COLUMN message_count INTEGER NOT NULL DEFAULT 0; +-- A gRPC call's service and method, from its path, and its status, from its trailers. +ALTER TABLE flows ADD COLUMN grpc_service TEXT; +ALTER TABLE flows ADD COLUMN grpc_method TEXT; +ALTER TABLE flows ADD COLUMN grpc_status INTEGER; +ALTER TABLE flows ADD COLUMN grpc_message TEXT; + +-- The messages of streaming flows, in the order they passed. A payload over a few KB is in +-- blobs.sqlite, under `.msg.`, and `data` is NULL. +CREATE TABLE flow_messages +( + id TEXT NOT NULL PRIMARY KEY, + flow_id TEXT NOT NULL, + seq INTEGER NOT NULL, + direction TEXT NOT NULL, + kind TEXT NOT NULL, + timestamp DATETIME NOT NULL, + size INTEGER NOT NULL DEFAULT 0, + stored_size INTEGER NOT NULL DEFAULT 0, + truncated BOOLEAN NOT NULL DEFAULT FALSE, + compressed BOOLEAN NOT NULL DEFAULT FALSE, + error TEXT, + meta TEXT NOT NULL DEFAULT '{}', + preview TEXT, + data BLOB +); + +CREATE UNIQUE INDEX idx_flow_messages_flow_id_seq ON flow_messages (flow_id, seq); + +-- Where gRPC messages are decoded from: protos attached to a host pattern, or a server's +-- reflection, with the descriptors compiled from them. +CREATE TABLE grpc_schemas +( + id TEXT NOT NULL PRIMARY KEY, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL, + host_pattern TEXT NOT NULL, + source TEXT NOT NULL, + proto_paths TEXT NOT NULL DEFAULT '[]', + services TEXT NOT NULL DEFAULT '[]', + descriptor BLOB NOT NULL +); diff --git a/crates-proxy/yaak-proxy-lib/src/client.rs b/crates-proxy/yaak-proxy-lib/src/client.rs index 914fa4c3a..099c94a82 100644 --- a/crates-proxy/yaak-proxy-lib/src/client.rs +++ b/crates-proxy/yaak-proxy-lib/src/client.rs @@ -230,6 +230,6 @@ impl OfflineStore { pub async fn har(&self, filter: &FlowFilter) -> Result { let flows = flows::list(&self.db, filter)?; - har::export(&self.blobs, &flows, self.capture_body_limit).await + har::export(&self.db, &self.blobs, &flows, self.capture_body_limit).await } } diff --git a/crates-proxy/yaak-proxy-lib/src/control.rs b/crates-proxy/yaak-proxy-lib/src/control.rs index fca098329..9355a43b2 100644 --- a/crates-proxy/yaak-proxy-lib/src/control.rs +++ b/crates-proxy/yaak-proxy-lib/src/control.rs @@ -4,12 +4,17 @@ //! time: a file whose process is gone, or whose port no longer answers, is stale and replaced. use crate::ca::{self, CaStatus}; -use crate::flows::{self, FlowFilter}; +use crate::flows::{self, BodyPreview, FlowFilter}; +use crate::grpc::{self, DecodedGrpcMessage}; use crate::har; use crate::macos::SystemProxyStatus; +use crate::messages::{self, FlowMessage, MessageKind}; use crate::models::{FlowBody, find_header}; use crate::save::SaveFlowRequest; -use crate::{ProxyCtx, ProxyState, bodies}; +use crate::{ + AddGrpcSchemaRequest, ListMessagesResponse, ProxyCtx, ProxyState, ReflectGrpcHostRequest, + bodies, +}; use bytes::Bytes; use chrono::{DateTime, Utc}; use http_body_util::{BodyExt, Full, Limited}; @@ -240,6 +245,25 @@ pub struct ErrorBody { pub error: String, } +/// `GET /flows/{id}/messages/{seq}`: a message with its payload, and a gRPC message decoded. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MessageDetail { + pub message: FlowMessage, + pub payload: BodyPreview, + /// A gRPC message as JSON, with its schema or without one. + #[serde(skip_serializing_if = "Option::is_none")] + pub decoded: Option, + /// Why a gRPC message could not be decoded. + #[serde(skip_serializing_if = "Option::is_none")] + pub decode_error: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RemovedResponse { + pub removed: bool, +} + /// Raw body headers: whether the body is cut short, at the capture limit or on the wire. pub const TRUNCATED_HEADER: &str = "x-yaak-truncated"; @@ -504,6 +528,47 @@ async fn route(state: &ServerState, req: Request) -> Result { raw_body(ctx, id, FlowBody::Response).await } + (&Method::GET, ["flows", id, "messages"]) => { + found(ctx, id)?; + let (after_seq, limit) = message_page(&query)?; + let messages = messages::list(&ctx.db, id, after_seq, limit)?; + let total = messages::count(&ctx.db, id)?; + json(&ListMessagesResponse { messages, total }) + } + (&Method::GET, ["flows", id, "messages", seq]) => { + let flow = flows::find(&ctx.db, id) + .ok_or_else(|| ApiError::new(StatusCode::NOT_FOUND, format!("No flow {id}")))?; + let seq: i64 = seq.parse().map_err(|_| bad_request(format!("Invalid seq {seq:?}")))?; + let message = messages::find(&ctx.db, id, seq)?.ok_or_else(|| { + ApiError::new(StatusCode::NOT_FOUND, format!("No message {seq} in flow {id}")) + })?; + let payload = messages::payload(&ctx.db, &ctx.blobs, &message)?; + let (decoded, decode_error) = if message.kind == MessageKind::GrpcMessage { + match grpc::decode(&ctx.db, &ctx.blobs, &ctx.grpc, &flow, seq).await { + Ok(decoded) => (Some(decoded), None), + Err(e) => (None, Some(e)), + } + } else { + (None, None) + }; + let payload = BodyPreview::new(&payload, message.truncated); + json(&MessageDetail { message, payload, decoded, decode_error }) + } + (&Method::GET, ["grpc-schemas"]) => json(&grpc::list(&ctx.db)?), + (&Method::POST, ["grpc-schemas"]) => { + let body: AddGrpcSchemaRequest = read_json(req).await?; + let added = grpc::add(&ctx.db, &ctx.grpc, &body.host_pattern, &body.proto_paths); + json(&added.await.map_err(bad_request)?) + } + (&Method::POST, ["grpc-schemas", "reflect"]) => { + let body: ReflectGrpcHostRequest = read_json(req).await?; + let (host, plaintext) = grpc::reflection_host(&body.host); + let reflected = grpc::reflect(&ctx.db, &ctx.grpc, &host, plaintext).await; + json(&reflected.map_err(bad_request)?) + } + (&Method::DELETE, ["grpc-schemas", id]) => { + json(&RemovedResponse { removed: grpc::remove(&ctx.db, &ctx.grpc, id)? }) + } (&Method::POST, ["flows", id, "save"]) => { found(ctx, id)?; let id = id.to_string(); @@ -513,7 +578,7 @@ async fn route(state: &ServerState, req: Request) -> Result { let filter = FlowFilter::from_query(&query).map_err(bad_request)?; let flows = flows::list(&ctx.db, &filter)?; - json(&har::export(&ctx.blobs, &flows, ctx.capture_body_limit()?).await?) + json(&har::export(&ctx.db, &ctx.blobs, &flows, ctx.capture_body_limit()?).await?) } (&Method::GET, ["ca"]) => { let ca = CaStore::load_or_create(&ctx.ca_dir()) @@ -582,6 +647,27 @@ async fn raw_body(ctx: &ProxyCtx, id: &str, which: FlowBody) -> Result Result<(Option, Option), ApiError> { + let (mut after, mut limit) = (None, None); + for (key, value) in url::form_urlencoded::parse(query.as_bytes()) { + match key.as_ref() { + "after" => { + after = Some( + value.parse().map_err(|_| bad_request(format!("Invalid after {value:?}")))?, + ) + } + "limit" => { + limit = Some( + value.parse().map_err(|_| bad_request(format!("Invalid limit {value:?}")))?, + ) + } + _ => {} + } + } + Ok((after, limit)) +} + fn body_limit(query: &str) -> Result { for (key, value) in url::form_urlencoded::parse(query.as_bytes()) { if key == "bodyLimit" { diff --git a/crates-proxy/yaak-proxy-lib/src/db.rs b/crates-proxy/yaak-proxy-lib/src/db.rs index dcbb88a59..012aba937 100644 --- a/crates-proxy/yaak-proxy-lib/src/db.rs +++ b/crates-proxy/yaak-proxy-lib/src/db.rs @@ -141,6 +141,12 @@ mod tests { ALTER TABLE flows DROP COLUMN client_bundle_id; ALTER TABLE flows DROP COLUMN client_path; ALTER TABLE flows DROP COLUMN cancelled; + ALTER TABLE flows DROP COLUMN stream_kind; + ALTER TABLE flows DROP COLUMN message_count; + ALTER TABLE flows DROP COLUMN grpc_service; + ALTER TABLE flows DROP COLUMN grpc_method; + ALTER TABLE flows DROP COLUMN grpc_status; + ALTER TABLE flows DROP COLUMN grpc_message; INSERT INTO flows (id, url, method) VALUES ('fl_old', 'https://example.com', 'GET');", ) .unwrap(); @@ -152,6 +158,11 @@ mod tests { assert_eq!((flows[0].id.as_str(), flows[0].saved_request_id.as_deref()), ("fl_old", None)); assert_eq!((flows[0].client_pid, flows[0].client_name.as_deref()), (None, None)); assert!(!flows[0].cancelled); + assert_eq!((flows[0].stream_kind, flows[0].message_count), (None, 0)); + // Failed flows are found without the gRPC status column too. + let failed = + crate::flows::FlowFilter { status: Some("error".into()), ..Default::default() }; + assert_eq!(crate::flows::list(&db, &failed).unwrap().len(), 0); } #[test] @@ -187,7 +198,7 @@ mod tests { .unwrap() .map(|t| t.unwrap()) .collect(); - assert_eq!(tables, vec!["_sqlx_migrations", "flows"]); + assert_eq!(tables, vec!["_sqlx_migrations", "flow_messages", "flows", "grpc_schemas"]); let flow_columns = columns(&conn, "flows"); assert!(flow_columns.contains(&"response_truncated".to_string())); @@ -197,6 +208,12 @@ mod tests { "client_bundle_id", "client_path", "cancelled", + "stream_kind", + "message_count", + "grpc_service", + "grpc_method", + "grpc_status", + "grpc_message", ] { assert!(flow_columns.contains(&column.to_string()), "{column} in {flow_columns:?}"); } diff --git a/crates-proxy/yaak-proxy-lib/src/flows.rs b/crates-proxy/yaak-proxy-lib/src/flows.rs index 0ae72f4c5..46ea4db86 100644 --- a/crates-proxy/yaak-proxy-lib/src/flows.rs +++ b/crates-proxy/yaak-proxy-lib/src/flows.rs @@ -32,7 +32,8 @@ pub struct FlowFilter { /// The method, any case. #[ts(optional)] pub method: Option, - /// A status (`404`), a class of them (`4xx`), or `error` for flows that failed. + /// A status (`404`), a class of them (`4xx`), or `error` for flows that failed, gRPC calls + /// that ended with a status other than OK among them. #[ts(optional)] pub status: Option, /// At most this many flows, the newest. @@ -114,7 +115,9 @@ impl FlowFilter { } /// The SQL condition for every field but `after`, `before` and `limit`, and its parameters. - fn condition(&self) -> (String, Vec) { + /// `grpc` says the database has gRPC statuses: one opened read-only before its migrations + /// have run may not. + fn condition(&self, grpc: bool) -> (String, Vec) { let mut sql = String::from("1"); let mut params: Vec = Vec::new(); for term in self.q.iter().flat_map(|q| q.split_whitespace()) { @@ -165,6 +168,10 @@ impl FlowFilter { } match self.status.as_deref().map(str::trim) { None | Some("") => {} + // A gRPC call that ended with a status other than OK failed too. + Some(s) if s.eq_ignore_ascii_case("error") && grpc => { + sql.push_str(" AND (error IS NOT NULL OR coalesce(grpc_status, 0) != 0)") + } Some(s) if s.eq_ignore_ascii_case("error") => sql.push_str(" AND error IS NOT NULL"), // A whole status or a class, not the start of one as a search term may be. Some(s) if s.len() == 3 => match status_condition(&s.to_ascii_lowercase(), &mut params) @@ -268,8 +275,13 @@ pub fn page( }) } +/// The database has the gRPC status column, added by a later migration. +fn has_grpc_status(conn: &rusqlite::Connection) -> bool { + conn.prepare("SELECT grpc_status FROM flows LIMIT 0").is_ok() +} + fn list_in(conn: &rusqlite::Connection, filter: &FlowFilter) -> Result, String> { - let (condition, mut params) = filter.condition(); + let (condition, mut params) = filter.condition(has_grpc_status(conn)); let mut sql = format!("SELECT * FROM flows WHERE {condition}"); if let Some(id) = &filter.after { // By rowid, which only grows as flows are written. @@ -300,7 +312,7 @@ fn list_in(conn: &rusqlite::Connection, filter: &FlowFilter) -> Result } fn count_matching(conn: &rusqlite::Connection, filter: &FlowFilter) -> Result { - let (condition, params) = filter.condition(); + let (condition, params) = filter.condition(has_grpc_status(conn)); conn.query_row( &format!("SELECT COUNT(*) FROM flows WHERE {condition}"), rusqlite::params_from_iter(params), @@ -319,12 +331,14 @@ pub fn clear(db: &ProxyQueryManager, blobs: &BlobManager) -> Result, _>>()?; conn.execute("DELETE FROM flows", [])?; + conn.execute("DELETE FROM flow_messages", [])?; // Only the deleted flows' bodies, so a flow still in flight keeps what it has written. blobs .with_tx(|b| { for id in &ids { b.delete_chunks(&Flow::body_id(id, FlowBody::Request))?; b.delete_chunks(&Flow::body_id(id, FlowBody::Response))?; + b.delete_chunks_like(&format!("{id}.msg.%"))?; } Ok::<_, yaak_models::error::Error>(()) }) @@ -634,7 +648,7 @@ mod tests { let plan: Vec = db.with_conn(|c| { let conn = c.conn().resolve(); let (condition, params) = - FlowFilter { q: Some("stripe".into()), ..Default::default() }.condition(); + FlowFilter { q: Some("stripe".into()), ..Default::default() }.condition(true); let sql = format!( "EXPLAIN QUERY PLAN SELECT * FROM flows WHERE {condition} \ ORDER BY started_at DESC, id DESC LIMIT 500" diff --git a/crates-proxy/yaak-proxy-lib/src/grpc.rs b/crates-proxy/yaak-proxy-lib/src/grpc.rs new file mode 100644 index 000000000..1d0087cb0 --- /dev/null +++ b/crates-proxy/yaak-proxy-lib/src/grpc.rs @@ -0,0 +1,595 @@ +//! Decoding captured gRPC messages. A message decodes to JSON with the schema of its call's +//! service, found first in the protos attached to a host pattern, then in the server's own +//! reflection, asked once per host. Without one it decodes schemaless: field numbers, wire +//! types and values, with nested messages guessed. + +use crate::db::ProxyQueryManager; +use crate::messages::{self, FlowMessage, MessageDirection, MessageKind}; +use crate::models::Flow; +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64; +use chrono::NaiveDateTime; +use rusqlite::{Row, params}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; +use ts_rs::TS; +use yaak_database::generate_prefixed_id; +use yaak_grpc::GrpcConfig; +use yaak_grpc::prost_reflect::{DescriptorPool, DynamicMessage, MessageDescriptor}; +use yaak_models::blob_manager::BlobManager; + +/// Where a schema's descriptors came from. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "snake_case")] +#[ts(export, export_to = "gen_models.ts")] +pub enum GrpcSchemaSource { + /// `.proto` files the user attached to a host pattern. + Files, + /// The server's reflection service. + Reflection, +} + +/// Descriptors for decoding the gRPC calls to hosts that match `hostPattern`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_models.ts")] +pub struct GrpcSchema { + pub id: String, + pub created_at: NaiveDateTime, + pub updated_at: NaiveDateTime, + /// A host (`api.example.com`), a host and port (`localhost:50051`), any subdomain of one + /// (`*.example.com`), or every host (`*`). + pub host_pattern: String, + pub source: GrpcSchemaSource, + /// The `.proto` files and include directories compiled, for a `files` schema. + pub proto_paths: Vec, + /// The services the descriptors define, by full name. + pub services: Vec, +} + +/// A captured gRPC message as JSON. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct DecodedGrpcMessage { + /// Pretty JSON: the message's fields by name with a schema; without one, an array of its + /// fields as `{ field, wireType, value }`, a length-delimited one with its guesses (`string`, + /// `message`, `bytes` in base64). + pub json: String, + /// The message type decoded with: `package.HelloReply`. None when schemaless. + pub message_type: Option, + /// The schema decoded with. None when schemaless. + pub schema: Option, + /// Why it decoded schemaless, or what to know about the decode. + pub note: Option, +} + +/// What a host's reflection answered, once it has; decodes that need it while it is being asked +/// wait for the same answer. +type Lookup = Arc>>; + +/// The decoded descriptor pools, by schema id, and the hosts reflection was asked of. +#[derive(Clone, Default)] +pub struct GrpcState { + protoc: Arc>>, + pools: Arc>>, + /// By `host:port`. + lookups: Arc>>, +} + +impl GrpcState { + pub fn set_protoc(&self, config: GrpcConfig) { + if let Ok(mut protoc) = self.protoc.lock() { + *protoc = Some(config); + } + } + + /// The protoc set with [`Self::set_protoc`], or else one on `PATH`, with the `include` + /// directory installed beside it. + fn protoc(&self) -> Result { + if let Some(config) = self.protoc.lock().ok().and_then(|p| p.clone()) { + return Ok(config); + } + protoc_on_path(std::env::var_os("PATH")) + } +} + +/// What to do when there is no protoc to compile attached `.proto` files with. +pub const NO_PROTOC: &str = "Yaak Proxy has no protoc to compile .proto files with. A packaged \ + Yaak Proxy includes one; otherwise install protoc (on macOS, `brew install protobuf`) so it \ + is on PATH, or let the server's reflection supply the schema."; + +/// A `protoc` in one of the directories of `path`, a `PATH` value, with the `include` directory +/// installed beside it. +fn protoc_on_path(path: Option) -> Result { + let name = if cfg!(windows) { "protoc.exe" } else { "protoc" }; + let bin = path + .iter() + .flat_map(std::env::split_paths) + .map(|dir| dir.join(name)) + .find(|p| p.is_file()) + .ok_or(NO_PROTOC)?; + let include = bin.parent().and_then(|d| d.parent()).map(|d| d.join("include")); + Ok(GrpcConfig { protoc_include_dir: include.unwrap_or_default(), protoc_bin_path: bin }) +} + +const COLUMNS: &str = "id, created_at, updated_at, host_pattern, source, proto_paths, services"; + +fn schema_from_row(r: &Row) -> rusqlite::Result { + let source: String = r.get("source")?; + let proto_paths: String = r.get("proto_paths")?; + let services: String = r.get("services")?; + Ok(GrpcSchema { + id: r.get("id")?, + created_at: r.get("created_at")?, + updated_at: r.get("updated_at")?, + host_pattern: r.get("host_pattern")?, + source: if source == "reflection" { + GrpcSchemaSource::Reflection + } else { + GrpcSchemaSource::Files + }, + proto_paths: serde_json::from_str(&proto_paths).unwrap_or_default(), + services: serde_json::from_str(&services).unwrap_or_default(), + }) +} + +pub fn list(db: &ProxyQueryManager) -> Result, String> { + db.with_conn(|c| { + let conn = c.conn().resolve(); + let mut stmt = conn + .prepare(&format!("SELECT {COLUMNS} FROM grpc_schemas ORDER BY created_at, id")) + .map_err(|e| e.to_string())?; + let rows = stmt.query_map([], schema_from_row).map_err(|e| e.to_string())?; + rows.collect::, _>>().map_err(|e| e.to_string()) + }) +} + +fn get(db: &ProxyQueryManager, id: &str) -> Result { + db.with_conn(|c| { + c.conn() + .resolve() + .query_row( + &format!("SELECT {COLUMNS} FROM grpc_schemas WHERE id = ?1"), + [id], + schema_from_row, + ) + .map_err(|_| format!("No gRPC schema {id}")) + }) +} + +fn insert( + db: &ProxyQueryManager, + host_pattern: &str, + source: GrpcSchemaSource, + proto_paths: &[String], + pool: &DescriptorPool, +) -> Result { + let id = generate_prefixed_id("gs"); + let services: Vec = pool.services().map(|s| s.full_name().to_string()).collect(); + let source_name = match source { + GrpcSchemaSource::Files => "files", + GrpcSchemaSource::Reflection => "reflection", + }; + db.with_tx(|c| { + let conn = c.conn().resolve(); + // A host's reflection is kept once, the latest. + if source == GrpcSchemaSource::Reflection { + conn.execute( + "DELETE FROM grpc_schemas WHERE source = 'reflection' AND host_pattern = ?1", + [host_pattern], + )?; + } + conn.execute( + "INSERT INTO grpc_schemas (id, host_pattern, source, proto_paths, services, descriptor) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + params![ + id, + host_pattern, + source_name, + serde_json::to_string(proto_paths).unwrap_or_default(), + serde_json::to_string(&services).unwrap_or_default(), + pool.encode_to_vec(), + ], + )?; + Ok::<_, yaak_database::Error>(()) + }) + .map_err(|e| format!("Failed to save the gRPC schema: {e}"))?; + get(db, &id) +} + +/// Compiles `proto_paths` (files, and directories to import from) and attaches them to +/// `host_pattern`. +pub async fn add( + db: &ProxyQueryManager, + state: &GrpcState, + host_pattern: &str, + proto_paths: &[String], +) -> Result { + let host_pattern = host_pattern.trim().to_ascii_lowercase(); + if host_pattern.is_empty() { + return Err("A host pattern is required".into()); + } + if proto_paths.is_empty() { + return Err("At least one .proto file is required".into()); + } + let paths: Vec = proto_paths.iter().map(PathBuf::from).collect(); + if let Some(missing) = paths.iter().find(|p| !p.exists()) { + return Err(format!("{} does not exist", missing.display())); + } + let config = state.protoc()?; + let pool = yaak_grpc::fill_pool_from_files(&config, &paths) + .await + .map_err(|e| format!("Failed to compile the protos: {e}"))?; + insert(db, &host_pattern, GrpcSchemaSource::Files, proto_paths, &pool) +} + +pub fn remove(db: &ProxyQueryManager, state: &GrpcState, id: &str) -> Result { + let removed = db + .with_tx(|c| { + let removed = + c.conn().resolve().execute("DELETE FROM grpc_schemas WHERE id = ?1", [id])?; + Ok::<_, yaak_database::Error>(removed) + }) + .map_err(|e| e.to_string())?; + if let Ok(mut pools) = state.pools.lock() { + pools.remove(id); + } + Ok(removed > 0) +} + +/// How long a server has to answer reflection, all of it. +const REFLECTION_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(15); + +/// A host to ask for reflection, as given to `reflect_grpc_host`: `host`, `host:port`, or +/// either after `http://` or `https://`. Returns `host:port` and whether to ask without TLS; +/// without a scheme, over TLS on 443. +pub fn reflection_host(host: &str) -> (String, bool) { + let host = host.trim().trim_end_matches('/').to_ascii_lowercase(); + let (host, plaintext) = match host.strip_prefix("http://") { + Some(rest) => (rest.to_string(), true), + None => (host.strip_prefix("https://").unwrap_or(&host).to_string(), false), + }; + let has_port = host.rsplit_once(':').is_some_and(|(_, port)| port.parse::().is_ok()); + if has_port { + return (host, plaintext); + } + (format!("{host}:{}", if plaintext { 80 } else { 443 }), plaintext) +} + +/// Asks the server at `authority` (`host:port`) for its services by reflection, without TLS +/// when `plaintext`, and keeps them as the host's schema, also for decodes to come. Fails when +/// it offers none. +pub async fn reflect( + db: &ProxyQueryManager, + state: &GrpcState, + authority: &str, + plaintext: bool, +) -> Result { + let authority = authority.trim().to_ascii_lowercase(); + let reflected = reflect_uncached(db, &authority, plaintext).await; + if let Ok(mut lookups) = state.lookups.lock() { + let answered = tokio::sync::OnceCell::new_with(Some(reflected.clone())); + lookups.insert(authority, Arc::new(answered)); + } + reflected +} + +async fn reflect_uncached( + db: &ProxyQueryManager, + authority: &str, + plaintext: bool, +) -> Result { + let scheme = if plaintext { "http" } else { "https" }; + let uri: hyper::Uri = format!("{scheme}://{authority}") + .parse() + .map_err(|e| format!("Invalid host {authority}: {e}"))?; + // Reflection's client panics on an answer it does not expect; that ends this task alone. + // Timed out inside the task, so a server that stalls leaves nothing running. + let timeout_error = format!("{authority} did not answer reflection in time"); + let reflected = tokio::spawn(async move { + let metadata = Default::default(); + let reflecting = + yaak_grpc::fill_pool_from_reflection(&uri, &metadata, true, None, 64 << 20); + match tokio::time::timeout(REFLECTION_TIMEOUT, reflecting).await { + Ok(reflected) => reflected.map_err(|e| e.to_string()), + Err(_) => Err(timeout_error), + } + }) + .await + .map_err(|_| format!("{authority} answered reflection with something unexpected"))??; + if reflected.services().next().is_none() { + return Err(format!("{authority} offers no services by reflection")); + } + insert(db, authority, GrpcSchemaSource::Reflection, &[], &reflected) +} + +/// Whether `pattern` covers `host` (lowercase, no port) at `authority` (`host:port`). +fn matches(pattern: &str, host: &str, authority: &str) -> bool { + if pattern == "*" { + return true; + } + let target = if pattern.contains(':') { authority } else { host }; + match pattern.strip_prefix("*.") { + Some(domain) => target.strip_suffix(domain).is_some_and(|rest| rest.ends_with('.')), + None => pattern == target, + } +} + +impl GrpcState { + fn pool(&self, db: &ProxyQueryManager, id: &str) -> Result { + if let Some(pool) = self.pools.lock().ok().and_then(|p| p.get(id).cloned()) { + return Ok(pool); + } + let bytes: Vec = db.with_conn(|c| { + c.conn() + .resolve() + .query_row("SELECT descriptor FROM grpc_schemas WHERE id = ?1", [id], |r| r.get(0)) + .map_err(|e| e.to_string()) + })?; + let pool = DescriptorPool::decode(bytes.as_slice()).map_err(|e| e.to_string())?; + if let Ok(mut pools) = self.pools.lock() { + pools.insert(id.to_string(), pool.clone()); + } + Ok(pool) + } +} + +/// The schema that defines `service` for the host at `authority`: attached protos first, the +/// most specific pattern first, then the host's reflection, asked of the server once, without +/// TLS when `plaintext`, as the call was made. +async fn find_schema( + db: &ProxyQueryManager, + state: &GrpcState, + authority: &str, + plaintext: bool, + service: &str, +) -> Result<(Option<(GrpcSchema, DescriptorPool)>, Option), String> { + let host = authority.rsplit_once(':').map_or(authority, |(h, _)| h); + let mut schemas: Vec = + list(db)?.into_iter().filter(|s| matches(&s.host_pattern, host, authority)).collect(); + // Files before reflection; then exact before wildcards, the narrower first. + schemas.sort_by_key(|s| { + let wildcard = s.host_pattern.starts_with('*'); + ( + s.source == GrpcSchemaSource::Reflection, + wildcard, + std::cmp::Reverse(s.host_pattern.len()), + ) + }); + for schema in &schemas { + let pool = state.pool(db, &schema.id)?; + if pool.get_service_by_name(service).is_some() { + return Ok((Some((schema.clone(), pool)), None)); + } + } + if schemas.iter().any(|s| s.source == GrpcSchemaSource::Reflection) { + return Ok((None, Some(format!("No schema for {service} on {authority}")))); + } + // Asked once: decodes that come while it is being asked wait for the same answer. + let Some(lookup) = + state.lookups.lock().ok().map(|mut l| l.entry(authority.into()).or_default().clone()) + else { + return Ok((None, Some(format!("No schema for {service} on {authority}")))); + }; + let answer = lookup.get_or_init(|| reflect_uncached(db, authority, plaintext)).await.clone(); + match answer { + // Unless it was removed since. + Ok(schema) => match state.pool(db, &schema.id) { + Ok(pool) if pool.get_service_by_name(service).is_some() => { + Ok((Some((schema, pool)), None)) + } + Ok(_) => { + Ok((None, Some(format!("{authority}'s reflection does not include {service}")))) + } + Err(_) => Ok((None, Some(format!("No schema for {service} on {authority}")))), + }, + Err(e) => Ok((None, Some(format!("No schema for {service}; reflection failed: {e}")))), + } +} + +/// Decodes a captured gRPC message. +pub async fn decode( + db: &ProxyQueryManager, + blobs: &BlobManager, + state: &GrpcState, + flow: &Flow, + seq: i64, +) -> Result { + let message = messages::get(db, &flow.id, seq)?; + if message.kind != MessageKind::GrpcMessage { + return Err(format!("Message {seq} is not a gRPC message")); + } + let payload = messages::payload(db, blobs, &message)?; + if message.compressed && message.error.is_some() { + return Err(message.error.unwrap_or_default()); + } + let (Some(service), Some(method)) = (&flow.grpc_service, &flow.grpc_method) else { + return Ok(schemaless(&payload, &message, Some("The call's path names no service".into()))); + }; + let url = url::Url::parse(&flow.url).map_err(|e| e.to_string())?; + let host = url.host_str().unwrap_or_default().to_ascii_lowercase(); + let authority = format!("{host}:{}", url.port_or_known_default().unwrap_or(443)); + let plaintext = url.scheme() == "http"; + let (found, note) = find_schema(db, state, &authority, plaintext, service).await?; + let Some((schema, pool)) = found else { + return Ok(schemaless(&payload, &message, note)); + }; + let Some(descriptor) = pool + .get_service_by_name(service) + .and_then(|s| s.methods().find(|m| m.name() == method)) + .map(|m| match message.direction { + MessageDirection::Client => m.input(), + MessageDirection::Server => m.output(), + }) + else { + let note = format!("{} has no method {method}", service); + return Ok(schemaless(&payload, &message, Some(note))); + }; + decode_with(&payload, &message, descriptor, schema) +} + +fn decode_with( + payload: &[u8], + message: &FlowMessage, + descriptor: MessageDescriptor, + schema: GrpcSchema, +) -> Result { + let message_type = descriptor.full_name().to_string(); + match DynamicMessage::decode(descriptor, payload) { + Ok(decoded) => Ok(DecodedGrpcMessage { + json: yaak_grpc::message_to_json(&decoded)?, + message_type: Some(message_type), + schema: Some(schema), + note: message.truncated.then(|| "The message is cut at the capture limit".into()), + }), + Err(e) => { + Ok(schemaless(payload, message, Some(format!("Did not decode as {message_type}: {e}")))) + } + } +} + +fn schemaless(payload: &[u8], message: &FlowMessage, reason: Option) -> DecodedGrpcMessage { + let mut notes: Vec = reason.into_iter().collect(); + notes.push("Decoded without a schema: field names and types are guesses".into()); + if message.truncated { + notes.push("The message is cut at the capture limit".into()); + } + let fields = decode_fields(payload, 0).unwrap_or_else(|| { + notes.push("Not a valid protobuf message: shown as bytes".into()); + vec![json!({ "bytes": BASE64.encode(payload) })] + }); + DecodedGrpcMessage { + json: serde_json::to_string_pretty(&fields).unwrap_or_default(), + message_type: None, + schema: None, + note: Some(notes.join(". ")), + } +} + +const MAX_DEPTH: usize = 16; + +/// A message's fields from the protobuf wire format, None when the bytes are not one. +pub fn decode_fields(mut bytes: &[u8], depth: usize) -> Option> { + let mut fields = Vec::new(); + while !bytes.is_empty() { + let key = read_varint(&mut bytes)?; + let field = key >> 3; + if field == 0 || field > 536_870_911 { + return None; + } + let value = match key & 7 { + 0 => { + let v = read_varint(&mut bytes)?; + json!({ "field": field, "wireType": "varint", "value": number(v) }) + } + 1 => { + let raw: [u8; 8] = bytes.get(..8)?.try_into().ok()?; + bytes = &bytes[8..]; + let v = u64::from_le_bytes(raw); + json!({ "field": field, "wireType": "fixed64", "value": number(v), "double": f64::from_le_bytes(raw) }) + } + 2 => { + let len = usize::try_from(read_varint(&mut bytes)?).ok()?; + let data = bytes.get(..len)?; + bytes = &bytes[len..]; + let mut value = json!({ "field": field, "wireType": "len" }); + let text = std::str::from_utf8(data) + .ok() + .filter(|s| s.chars().all(|c| !c.is_control() || c.is_whitespace())); + if let Some(text) = text { + value["string"] = json!(text); + } + if depth < MAX_DEPTH + && !data.is_empty() + && let Some(nested) = decode_fields(data, depth + 1) + { + value["message"] = json!(nested); + } + if text.is_none() { + value["bytes"] = json!(BASE64.encode(data)); + } + value + } + 5 => { + let raw: [u8; 4] = bytes.get(..4)?.try_into().ok()?; + bytes = &bytes[4..]; + let v = u32::from_le_bytes(raw); + json!({ "field": field, "wireType": "fixed32", "value": v, "float": f32::from_le_bytes(raw) }) + } + // Groups are long deprecated; 6 and 7 are no wire type. + _ => return None, + }; + fields.push(value); + } + Some(fields) +} + +/// A u64 as a JSON number when JavaScript can hold it exactly, else as a string. +fn number(v: u64) -> Value { + if v <= (1 << 53) { json!(v) } else { json!(v.to_string()) } +} + +fn read_varint(bytes: &mut &[u8]) -> Option { + let mut value = 0u64; + for i in 0..10 { + let byte = *bytes.get(i)?; + value |= u64::from(byte & 0x7f) << (7 * i); + if byte & 0x80 == 0 { + *bytes = &bytes[i + 1..]; + return Some(value); + } + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn without_protoc_the_error_says_what_to_install() { + let empty = tempfile::tempdir().unwrap(); + let err = protoc_on_path(Some(empty.path().as_os_str().to_owned())).err().unwrap(); + assert_eq!(err, NO_PROTOC); + assert!(err.contains("brew install protobuf"), "{err}"); + assert!(protoc_on_path(None).is_err()); + } + + #[test] + fn reflection_hosts() { + assert_eq!(reflection_host("api.example.com"), ("api.example.com:443".into(), false)); + assert_eq!(reflection_host("localhost:50051"), ("localhost:50051".into(), false)); + assert_eq!(reflection_host("http://localhost:50051"), ("localhost:50051".into(), true)); + assert_eq!(reflection_host("http://Example.com/"), ("example.com:80".into(), true)); + assert_eq!(reflection_host("https://example.com"), ("example.com:443".into(), false)); + } + + #[test] + fn host_patterns() { + assert!(matches("api.example.com", "api.example.com", "api.example.com:443")); + assert!(!matches("api.example.com", "www.example.com", "www.example.com:443")); + assert!(matches("*.example.com", "a.b.example.com", "a.b.example.com:443")); + assert!(!matches("*.example.com", "example.com", "example.com:443")); + assert!(!matches("*.example.com", "badexample.com", "badexample.com:443")); + assert!(matches("localhost:50051", "localhost", "localhost:50051")); + assert!(!matches("localhost:50051", "localhost", "localhost:50052")); + assert!(matches("*", "anything", "anything:1")); + } + + #[test] + fn decodes_the_wire_format_without_a_schema() { + // field 1: varint 150; field 2: "hi"; field 3: nested { field 1: 1 } + let bytes = [ + 0x08, 0x96, 0x01, 0x12, 0x02, b'h', b'i', 0x1a, 0x02, 0x08, 0x01, + ]; + let fields = decode_fields(&bytes, 0).unwrap(); + assert_eq!(fields[0], json!({ "field": 1, "wireType": "varint", "value": 150 })); + assert_eq!(fields[1]["string"], "hi"); + assert_eq!(fields[2]["message"][0]["value"], 1); + assert!(decode_fields(&[0x08], 0).is_none(), "a varint cut short"); + } +} diff --git a/crates-proxy/yaak-proxy-lib/src/har.rs b/crates-proxy/yaak-proxy-lib/src/har.rs index fc08df008..49409a2aa 100644 --- a/crates-proxy/yaak-proxy-lib/src/har.rs +++ b/crates-proxy/yaak-proxy-lib/src/har.rs @@ -4,8 +4,12 @@ //! at the capture limit and `_error` on a flow that failed. use crate::bodies::{self, DecodedBody}; +use crate::db::ProxyQueryManager; use crate::flows::{BodyEncoding, BodyPreview}; -use crate::models::{Flow, FlowBody, ProxyHeader, find_header}; +use crate::messages::{self, MessageDirection, MessageKind}; +use crate::models::{Flow, FlowBody, ProxyHeader, StreamKind, find_header}; +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64; use serde::{Deserialize, Serialize}; use yaak_models::blob_manager::BlobManager; @@ -40,6 +44,26 @@ pub struct HarEntry { pub server_ip_address: Option, #[serde(rename = "_error", skip_serializing_if = "Option::is_none")] pub error: Option, + /// A WebSocket's data messages, as Chrome's devtools export them. + #[serde( + rename = "_webSocketMessages", + skip_serializing_if = "Option::is_none", + default + )] + pub web_socket_messages: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HarWebSocketMessage { + /// `send` from the client, `receive` from the server. + #[serde(rename = "type")] + pub kind: String, + /// Seconds since the epoch. + pub time: f64, + /// 1 for text, 2 for binary. + pub opcode: u8, + /// The text, or a binary message in base64. + pub data: String, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -118,16 +142,59 @@ pub struct HarTimings { } /// A HAR of `flows`, reading each body from `blobs` and keeping at most `limit` bytes of it. -pub async fn export(blobs: &BlobManager, flows: &[Flow], limit: usize) -> Result { +/// A WebSocket's messages are in its entry's `_webSocketMessages`; an event stream's events are +/// its response body, as they came. +pub async fn export( + db: &ProxyQueryManager, + blobs: &BlobManager, + flows: &[Flow], + limit: usize, +) -> Result { let mut entries = Vec::with_capacity(flows.len()); for flow in flows { let request = bodies::read_body(blobs, flow, FlowBody::Request, limit).await?; let response = bodies::read_body(blobs, flow, FlowBody::Response, limit).await?; - entries.push(entry(flow, &request, &response)); + let mut entry = entry(flow, &request, &response); + if flow.stream_kind == Some(StreamKind::Websocket) { + entry.web_socket_messages = Some(websocket_messages(db, blobs, flow)?); + } + entries.push(entry); } Ok(har(entries)) } +fn websocket_messages( + db: &ProxyQueryManager, + blobs: &BlobManager, + flow: &Flow, +) -> Result, String> { + let mut out = Vec::new(); + for message in messages::list(db, &flow.id, None, None)? { + let opcode = match message.kind { + MessageKind::WebsocketText => 1, + MessageKind::WebsocketBinary => 2, + _ => continue, + }; + let payload = messages::payload(db, blobs, &message)?; + let data = if opcode == 1 { + String::from_utf8_lossy(&payload).into_owned() + } else { + BASE64.encode(&payload) + }; + out.push(HarWebSocketMessage { + kind: match message.direction { + MessageDirection::Client => "send", + MessageDirection::Server => "receive", + } + .into(), + time: message.timestamp.and_utc().timestamp_micros() as f64 / 1_000_000.0, + opcode, + data, + }); + } + Ok(out) +} + pub fn har(entries: Vec) -> Har { Har { log: HarLog { @@ -186,6 +253,7 @@ pub fn entry(flow: &Flow, request_body: &DecodedBody, response_body: &DecodedBod .clone() .or_else(|| response_body.decode_error.clone()) .or_else(|| request_body.decode_error.clone()), + web_socket_messages: None, } } @@ -394,7 +462,9 @@ mod tests { ..Default::default() }; - let har = export(&blobs, &[flow], 1024 * 1024).await.unwrap(); + let dir = tempfile::tempdir().unwrap(); + let db = ProxyQueryManager::new(&dir.path().join("proxy.db")).unwrap(); + let har = export(&db, &blobs, &[flow], 1024 * 1024).await.unwrap(); let entry = &har.log.entries[0]; let text_kept = entry.response.content.text.as_deref().unwrap(); assert!(!text_kept.is_empty() && text.starts_with(text_kept.as_bytes())); diff --git a/crates-proxy/yaak-proxy-lib/src/lib.rs b/crates-proxy/yaak-proxy-lib/src/lib.rs index f27f7277c..ff169f997 100644 --- a/crates-proxy/yaak-proxy-lib/src/lib.rs +++ b/crates-proxy/yaak-proxy-lib/src/lib.rs @@ -7,9 +7,11 @@ pub mod control; pub mod convert; pub mod db; pub mod flows; +pub mod grpc; pub mod har; pub mod helper; pub mod macos; +pub mod messages; pub mod models; pub mod save; pub mod settings; @@ -21,9 +23,11 @@ use crate::client::ControlClient; use crate::control::{ControlFile, ControlInfo, ControlServer, Owner}; use crate::convert::DroppedHeader; use crate::db::ProxyQueryManager; +use crate::grpc::{DecodedGrpcMessage, GrpcSchema, GrpcState}; use crate::helper::HelperState; use crate::macos::{SystemProxyBackend, SystemProxyStatus}; -use crate::models::{Flow, FlowBody, FlowIden, ModelPayload, ProxyHeader, find_header}; +use crate::messages::FlowMessage; +use crate::models::{Flow, FlowBody, FlowIden, ModelPayload, ProxyHeader, StreamKind, find_header}; use crate::save::{ClientDb, SaveFlowRequest, SaveFlowResponse, WorkspaceSummary}; use crate::settings::{HttpVersion, ProxySettings}; use base64::Engine; @@ -40,6 +44,7 @@ use std::sync::{Arc, Mutex, OnceLock}; use std::time::{Duration, SystemTime}; use ts_rs::TS; use yaak_database::{ModelChangeEvent, UpdateSource, UpsertModelInfo}; +pub use yaak_grpc::GrpcConfig; use yaak_models::blob_manager::{BlobManager, BodyChunkWriter}; use yaak_proxy::{CaStore, ProxyConfig, ProxyEvent, ProxyHandle}; use yaak_rpc::{RpcError, RpcEventEmitter, define_rpc}; @@ -70,6 +75,8 @@ pub struct ProxyCtx { pub db: ProxyQueryManager, pub blobs: BlobManager, pub events: RpcEventEmitter, + /// gRPC schemas as decoded, and the protoc that compiles attached protos. + pub grpc: GrpcState, } impl ProxyCtx { @@ -94,9 +101,17 @@ impl ProxyCtx { db, blobs, events, + grpc: GrpcState::default(), }) } + /// The protoc that compiles `.proto` files attached for gRPC decoding, as the app bundles + /// it. Without one, a `protoc` on `PATH` is used. + pub fn with_protoc(self, config: yaak_grpc::GrpcConfig) -> Self { + self.grpc.set_protoc(config); + self + } + /// The process this runs in, which decides how the system proxy gets admin rights. pub fn with_owner(mut self, owner: Owner) -> Self { self.owner = owner; @@ -167,6 +182,7 @@ impl ProxyCtx { } let settings = self.settings()?; + self.end_stale_streams(); let ca = CaStore::load_or_create(&self.ca_dir()) .map_err(|e| format!("Failed to load proxy CA: {e}"))?; let mut config = ProxyConfig::new(port.unwrap_or(settings.port), Arc::new(ca)); @@ -223,6 +239,22 @@ impl ProxyCtx { Ok(()) } + /// Marks streams an engine left open, as one that crashed does, as ended. Only the engine's + /// owner, about to start it, may: until then another process's streams may be open. + fn end_stale_streams(&self) { + let ended = self.db.with_tx(|c| { + c.conn().resolve().execute( + "UPDATE flows SET error = ?1 WHERE stream_kind IS NOT NULL \ + AND elapsed_ms IS NULL AND error IS NULL AND NOT cancelled", + [STOPPED_MID_STREAM], + )?; + Ok::<_, yaak_database::Error>(()) + }); + if let Err(e) = ended { + warn!("Failed to end streams left open: {e}"); + } + } + /// Stops the engine and turns off a system proxy that points at it. pub fn stop(&self) -> Result<(), String> { self.system_proxy_on.store(false, Ordering::SeqCst); @@ -947,8 +979,146 @@ pub struct HelperRegisterRequest {} #[ts(export, export_to = "gen_rpc.ts")] pub struct HelperUnregisterRequest {} +#[derive(Debug, Clone, Default, Serialize, Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +#[serde(rename_all = "camelCase")] +pub struct ListMessagesRequest { + pub flow_id: String, + /// Only messages after this `seq`, for following a stream as it grows. + #[ts(optional, type = "number")] + pub after_seq: Option, + /// At most this many, the first after `afterSeq`. + #[ts(optional, type = "number")] + pub limit: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +#[serde(rename_all = "camelCase")] +pub struct ListMessagesResponse { + /// In order, by `seq`. + pub messages: Vec, + /// Messages the flow has in all. + #[ts(type = "number")] + pub total: i64, +} + +#[derive(Debug, Clone, Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +#[serde(rename_all = "camelCase")] +pub struct GetMessagePayloadRequest { + pub flow_id: String, + #[ts(type = "number")] + pub seq: i64, +} + +#[derive(Debug, Clone, Serialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +#[serde(rename_all = "camelCase")] +pub struct GetMessagePayloadResponse { + /// The payload as stored (unmasked, inflated, decompressed), in base64. + pub data: String, + /// It is cut at the capture limit. + pub truncated: bool, +} + +#[derive(Debug, Clone, Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +#[serde(rename_all = "camelCase")] +pub struct DecodeGrpcMessageRequest { + pub flow_id: String, + #[ts(type = "number")] + pub seq: i64, +} + +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct ListGrpcSchemasRequest {} + +#[derive(Debug, Clone, Serialize, Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +#[serde(rename_all = "camelCase")] +pub struct AddGrpcSchemaRequest { + /// `api.example.com`, `localhost:50051`, `*.example.com` or `*`. + pub host_pattern: String, + /// `.proto` files, and directories they import from. + pub proto_paths: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct RemoveGrpcSchemaRequest { + pub id: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct ReflectGrpcHostRequest { + /// `host` or `host:port`, over TLS on 443 unless a port is given, or either after + /// `http://` to ask without TLS. + pub host: String, +} + +/// A message of a streaming flow was recorded. +#[derive(Debug, Clone, Serialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct MessagePayload { + pub message: FlowMessage, +} + // -- Handlers -- +fn list_messages( + ctx: &ProxyCtx, + req: ListMessagesRequest, +) -> Result { + find_flow(ctx, &req.flow_id)?; + let messages = + messages::list(&ctx.db, &req.flow_id, req.after_seq, req.limit).map_err(rpc_error)?; + let total = messages::count(&ctx.db, &req.flow_id).map_err(rpc_error)?; + Ok(ListMessagesResponse { messages, total }) +} + +fn get_message_payload( + ctx: &ProxyCtx, + req: GetMessagePayloadRequest, +) -> Result { + let message = messages::get(&ctx.db, &req.flow_id, req.seq).map_err(rpc_error)?; + let data = messages::payload(&ctx.db, &ctx.blobs, &message).map_err(rpc_error)?; + Ok(GetMessagePayloadResponse { data: BASE64.encode(data), truncated: message.truncated }) +} + +async fn decode_grpc_message( + ctx: ProxyCtx, + req: DecodeGrpcMessageRequest, +) -> Result { + let flow = find_flow(&ctx, &req.flow_id)?; + grpc::decode(&ctx.db, &ctx.blobs, &ctx.grpc, &flow, req.seq).await.map_err(rpc_error) +} + +fn list_grpc_schemas( + ctx: &ProxyCtx, + _req: ListGrpcSchemasRequest, +) -> Result, RpcError> { + grpc::list(&ctx.db).map_err(rpc_error) +} + +async fn add_grpc_schema(ctx: ProxyCtx, req: AddGrpcSchemaRequest) -> Result { + grpc::add(&ctx.db, &ctx.grpc, &req.host_pattern, &req.proto_paths).await.map_err(rpc_error) +} + +fn remove_grpc_schema(ctx: &ProxyCtx, req: RemoveGrpcSchemaRequest) -> Result { + grpc::remove(&ctx.db, &ctx.grpc, &req.id).map_err(rpc_error) +} + +async fn reflect_grpc_host( + ctx: ProxyCtx, + req: ReflectGrpcHostRequest, +) -> Result { + let (host, plaintext) = grpc::reflection_host(&req.host); + grpc::reflect(&ctx.db, &ctx.grpc, &host, plaintext).await.map_err(rpc_error) +} + fn execute_action(ctx: &ProxyCtx, invocation: ActionInvocation) -> Result { match invocation { ActionInvocation::Global { action } => match action { @@ -1086,7 +1256,7 @@ async fn export_har(ctx: ProxyCtx, req: ExportHarRequest) -> Result, response_body: Option, + /// Written to the database already, as a streaming flow is when its stream opens. + persisted: bool, + /// Cleared while it streamed: nothing more of it is kept. + cleared: bool, + next_seq: i64, + /// Changed since it was last written, by a message or a status. + dirty: bool, + last_write: Option, +} + +impl InFlight { + /// Something shown of the flow changed: one written while it streams is written again on + /// the stream flow interval, so it does not wait for a message to show it. + fn changed(&mut self) { + if self.persisted { + self.dirty = true; + } + } +} + +/// A message waiting to be written, with its payload. +struct PendingMessage { + id: u64, + message: messages::FlowMessage, + data: bytes::Bytes, } /// Where the event loop stores flows, and whether it does. @@ -1289,10 +1499,108 @@ struct FlowSink { } fn run_event_loop(rx: std::sync::mpsc::Receiver, sink: FlowSink) { - let FlowSink { db, blobs, events, attribution, recording } = sink; + use std::sync::mpsc::{RecvTimeoutError, TryRecvError}; let mut in_flight: HashMap = HashMap::new(); + let mut pending: Vec = Vec::new(); + let mut last_due_check = std::time::Instant::now(); - while let Ok(event) = rx.recv() { + loop { + // Messages are written once no more are waiting, or a batch is full; flows changed by + // them once their interval is up. + let event = if !pending.is_empty() { + match rx.try_recv() { + Ok(event) => event, + Err(TryRecvError::Empty) => { + sink.write_messages(&mut in_flight, &mut pending); + sink.write_due_flows(&mut in_flight); + continue; + } + Err(TryRecvError::Disconnected) => break, + } + } else if let Some(due) = next_flow_write(&in_flight) { + let wait = due.saturating_duration_since(std::time::Instant::now()); + match rx.recv_timeout(wait) { + Ok(event) => event, + Err(RecvTimeoutError::Timeout) => { + sink.write_due_flows(&mut in_flight); + continue; + } + Err(RecvTimeoutError::Disconnected) => break, + } + } else { + match rx.recv() { + Ok(event) => event, + Err(_) => break, + } + }; + if let ProxyEvent::Message { id, message } = event { + record_message(&mut in_flight, &mut pending, id, &message); + if pending.len() >= MESSAGE_BATCH { + sink.write_messages(&mut in_flight, &mut pending); + } + } else { + // Before anything else of the flows, so a flow's messages are in before it ends. + sink.write_messages(&mut in_flight, &mut pending); + sink.handle(&mut in_flight, event); + } + // Traffic that never lets up never empties the queue, so flows due to be written are + // looked for here too, every so often, their messages written first. + if last_due_check.elapsed() >= DUE_CHECK_INTERVAL { + last_due_check = std::time::Instant::now(); + sink.write_messages(&mut in_flight, &mut pending); + sink.write_due_flows(&mut in_flight); + } + } + + sink.write_messages(&mut in_flight, &mut pending); + // The engine stopped: a stream still open ends with it. Other flows in flight are dropped, + // as they always have been. + for (_, mut f) in in_flight.drain() { + if f.persisted { + f.flow.error = Some(STOPPED_MID_STREAM.to_string()); + sink.finish_flow(f); + } + } +} + +/// When the next changed streaming flow is due to be written. +fn next_flow_write(in_flight: &HashMap) -> Option { + in_flight + .values() + .filter(|f| f.dirty) + .map(|f| { + f.last_write.map_or_else(std::time::Instant::now, |t| t + STREAM_FLOW_WRITE_INTERVAL) + }) + .min() +} + +fn record_message( + in_flight: &mut HashMap, + pending: &mut Vec, + id: u64, + message: &yaak_proxy::StreamMessage, +) { + let Some(f) = in_flight.get_mut(&id).filter(|f| !f.cleared) else { + return; + }; + if let yaak_proxy::MessageKind::GrpcTrailers { status, message } = &message.kind { + f.flow.grpc_status = *status; + f.flow.grpc_message = message.clone(); + } + let seq = f.next_seq; + f.next_seq += 1; + f.flow.message_count += 1; + f.dirty = true; + pending.push(PendingMessage { + id, + message: messages::FlowMessage::from_engine(&f.flow.id, seq, message), + data: message.data.clone(), + }); +} + +impl FlowSink { + fn handle(&self, in_flight: &mut HashMap, event: ProxyEvent) { + let FlowSink { blobs, attribution, recording, .. } = self; match event { ProxyEvent::RequestStart { id, @@ -1306,7 +1614,7 @@ fn run_event_loop(rx: std::sync::mpsc::Receiver, sink: FlowSink) { // Paused when it started: it passes through, and none of its events are kept. // Judged by its start rather than now, as the loop can run behind. if !recording.lock().map(|r| r.was_on_at(started_at)).unwrap_or(true) { - continue; + return; } // Begun as the connection was accepted; this covers one accepted before the // engine's port was known. @@ -1329,9 +1637,38 @@ fn run_event_loop(rx: std::sync::mpsc::Receiver, sink: FlowSink) { connection, request_body: Some(BodyChunkWriter::new(blobs.clone(), request_body)), response_body: Some(BodyChunkWriter::new(blobs.clone(), response_body)), + persisted: false, + cleared: false, + next_seq: 0, + dirty: false, + last_write: None, }, ); } + ProxyEvent::StreamStart { id, kind } => { + if let Some(f) = in_flight.get_mut(&id) { + let kind = StreamKind::from(kind); + f.flow.stream_kind = Some(kind); + if kind.is_grpc() + && let Ok(url) = url::Url::parse(&f.flow.url) + && let Some((service, method)) = yaak_proxy::service_and_method(url.path()) + { + f.flow.grpc_service = Some(service.to_string()); + f.flow.grpc_method = Some(method.to_string()); + } + // Shown while it streams, rather than only once it ends. + self.write_stream_flow(f); + } + } + // Recorded by the loop, which batches them. + ProxyEvent::Message { .. } => {} + ProxyEvent::StreamEnd { id, elapsed_ms, error } => { + if let Some(mut f) = in_flight.remove(&id) { + f.flow.elapsed_ms = Some(clamp_ms(elapsed_ms)); + f.flow.error = error; + self.finish_flow(f); + } + } ProxyEvent::RequestHeader { id, name, value } => { if let Some(f) = in_flight.get_mut(&id) { f.flow.request_headers.push(ProxyHeader { name, value }); @@ -1344,6 +1681,7 @@ fn run_event_loop(rx: std::sync::mpsc::Receiver, sink: FlowSink) { f.flow.request_size += len as i64; f.flow.request_truncated |= data.len() < len; write_body_chunk(&mut f.request_body, &data); + f.changed(); } } ProxyEvent::RequestBodyComplete { id, size, truncated } => { @@ -1351,6 +1689,7 @@ fn run_event_loop(rx: std::sync::mpsc::Receiver, sink: FlowSink) { f.flow.request_size = size as i64; f.flow.request_truncated = truncated; finish_body(f.request_body.take()); + f.changed(); } } ProxyEvent::ResponseStart { id, status, http_version, remote_addr, elapsed_ms } => { @@ -1359,16 +1698,22 @@ fn run_event_loop(rx: std::sync::mpsc::Receiver, sink: FlowSink) { f.flow.upstream_http_version = Some(http_version); f.flow.remote_addr = remote_addr.map(|a| a.to_string()); f.flow.elapsed_headers_ms = Some(clamp_ms(elapsed_ms)); + f.changed(); } } ProxyEvent::ResponseHeader { id, name, value } => { if let Some(f) = in_flight.get_mut(&id) { f.flow.response_headers.push(ProxyHeader { name, value }); + f.changed(); } } - ProxyEvent::ResponseBodyChunk { id, data, .. } => { + // Counted per chunk too, for a stream written while it is open. + ProxyEvent::ResponseBodyChunk { id, len, data } => { if let Some(f) = in_flight.get_mut(&id) { + f.flow.response_size += len as i64; + f.flow.response_truncated |= data.len() < len; write_body_chunk(&mut f.response_body, &data); + f.changed(); } } ProxyEvent::ResponseBodyComplete { @@ -1385,25 +1730,204 @@ fn run_event_loop(rx: std::sync::mpsc::Receiver, sink: FlowSink) { f.flow.elapsed_ms = Some(clamp_ms(elapsed_ms)); f.flow.error = error; f.flow.cancelled = cancelled; - finish_flow(&db, &events, &attribution, f); + self.finish_flow(f); } } ProxyEvent::Cancelled { id, elapsed_ms } => { if let Some(mut f) = in_flight.remove(&id) { f.flow.elapsed_ms = Some(clamp_ms(elapsed_ms)); f.flow.cancelled = true; - finish_flow(&db, &events, &attribution, f); + self.finish_flow(f); } } ProxyEvent::Error { id, error, elapsed_ms } => { if let Some(mut f) = in_flight.remove(&id) { f.flow.elapsed_ms = Some(clamp_ms(elapsed_ms)); f.flow.error = Some(error); - finish_flow(&db, &events, &attribution, f); + self.finish_flow(f); } } } } + + /// Writes `pending` in one transaction, each payload over [`messages::INLINE_LIMIT`] first + /// to `blobs.sqlite`, and tells the app of each. A flow cleared meanwhile keeps nothing more. + fn write_messages( + &self, + in_flight: &mut HashMap, + pending: &mut Vec, + ) { + if pending.is_empty() { + return; + } + let batch = std::mem::take(pending); + let mut rows = Vec::with_capacity(batch.len()); + for PendingMessage { id, mut message, data } in batch { + let inline = if data.len() > messages::INLINE_LIMIT { + let mut writer = BodyChunkWriter::new(self.blobs.clone(), message.blob_id()); + if let Err(e) = writer.write(&data).and_then(|_| writer.finish()) { + warn!("Failed to store a proxy message payload: {e}"); + message.stored_size = 0; + message + .error + .get_or_insert_with(|| format!("Failed to store the payload: {e}")); + } + None + } else { + Some(data) + }; + rows.push((id, message, inline)); + } + + let mut orphaned = Vec::new(); + let mut newly_cleared = Vec::new(); + let written = self.db.with_tx(|ctx| { + let conn = ctx.conn().resolve(); + let mut checked = std::collections::HashSet::new(); + for (id, message, inline) in &rows { + let Some(f) = in_flight.get_mut(id) else { + continue; + }; + // Cleared since it was written: what comes after is not kept either. + if f.persisted && checked.insert(*id) && !flow_exists(conn, &f.flow.id) { + f.cleared = true; + newly_cleared.push(*id); + } + if f.cleared { + orphaned.push(message.blob_id()); + continue; + } + messages::insert(conn, message, inline.as_deref())?; + // Before the commit, as flows are, so a clear's event comes after it. + self.events.emit("message_write", &MessagePayload { message: message.clone() }); + } + Ok::<_, yaak_database::Error>(()) + }); + if let Err(e) = written { + warn!("Failed to write proxy messages: {e}"); + } + if !orphaned.is_empty() { + let _ = self.blobs.with_tx(|b| orphaned.iter().try_for_each(|id| b.delete_chunks(id))); + } + for id in newly_cleared { + if let Some(f) = in_flight.get_mut(&id) { + self.stop_cleared(f); + } + } + } + + /// A stream cleared while open keeps nothing more: its bodies stop being written, and what + /// they wrote since the clear is deleted, as the stream may run long before it ends. + fn stop_cleared(&self, f: &mut InFlight) { + f.cleared = true; + f.request_body = None; + f.response_body = None; + self.forget_cleared(&f.flow.id); + } + + /// Writes the streaming flows whose messages or status changed, at most once per + /// [`STREAM_FLOW_WRITE_INTERVAL`] each. + fn write_due_flows(&self, in_flight: &mut HashMap) { + let now = std::time::Instant::now(); + for f in in_flight.values_mut() { + let due = f.last_write.is_none_or(|t| now >= t + STREAM_FLOW_WRITE_INTERVAL); + if f.dirty && due { + self.write_stream_flow(f); + } + } + } + + /// Writes a streaming flow as it is so far, for the app to show while it streams. Unless it + /// was cleared since it was last written: then nothing more of it is kept. + fn write_stream_flow(&self, f: &mut InFlight) { + f.dirty = false; + f.last_write = Some(std::time::Instant::now()); + if f.cleared { + return; + } + let mut flow = f.flow.clone(); + flow.content_type = find_header(&flow.response_headers, "content-type").map(String::from); + // Not waited for here; the flow's end waits for it. + if let Some(app) = self.attribution.get(f.connection, Duration::ZERO) { + flow.client_pid = Some(app.pid); + flow.client_name = Some(app.name); + flow.client_bundle_id = app.bundle_id; + flow.client_path = Some(app.path); + } + let persisted = f.persisted; + let written = self.db.with_tx(|ctx| self.write_flow(ctx, flow, persisted)); + match written { + Ok(true) => f.persisted = true, + Ok(false) => self.stop_cleared(f), + Err(e) => warn!("Failed to write proxy flow: {e}"), + } + } + + fn finish_flow(&self, f: InFlight) { + finish_body(f.request_body); + finish_body(f.response_body); + let id = f.flow.id.clone(); + if f.cleared { + self.forget_cleared(&id); + return; + } + + let mut flow = f.flow; + flow.content_type = find_header(&flow.response_headers, "content-type").map(String::from); + if let Some(app) = self.attribution.get(f.connection, ATTRIBUTION_WAIT) { + flow.client_pid = Some(app.pid); + flow.client_name = Some(app.name); + flow.client_bundle_id = app.bundle_id; + flow.client_path = Some(app.path); + } + let written = self.db.with_tx(|ctx| self.write_flow(ctx, flow, f.persisted)); + match written { + Ok(true) => {} + Ok(false) => self.forget_cleared(&id), + Err(e) => warn!("Failed to write proxy flow: {e}"), + } + } + + /// Writes `flow` and tells the app. One written before, as a stream is while open, keeps + /// what was recorded on it since, the request it was saved as; one cleared since stays + /// cleared, and this returns false. + fn write_flow( + &self, + ctx: &yaak_database::DbContext, + mut flow: Flow, + persisted: bool, + ) -> Result { + if persisted { + let Some(stored) = ctx.find_optional::(FlowIden::Id, &flow.id) else { + return Ok(false); + }; + flow.saved_request_id = stored.saved_request_id; + } + let (saved, created) = ctx.upsert(&flow, &UpdateSource::Background)?; + // Sent while this holds the writer, before the commit, so a clear's event, sent after + // its own commit, always comes after it. + self.events.emit( + "model_write", + &ModelPayload { model: saved, change: ModelChangeEvent::Upsert { created } }, + ); + Ok(true) + } + + /// Deletes what a flow cleared while it streamed wrote after the clear. + fn forget_cleared(&self, flow_id: &str) { + let deleted = self.blobs.with_tx(|b| { + b.delete_chunks(&Flow::body_id(flow_id, FlowBody::Request))?; + b.delete_chunks(&Flow::body_id(flow_id, FlowBody::Response))?; + b.delete_chunks_like(&format!("{flow_id}.msg.%")) + }); + if let Err(e) = deleted { + warn!("Failed to delete a cleared flow's bodies: {e}"); + } + } +} + +fn flow_exists(conn: &rusqlite::Connection, flow_id: &str) -> bool { + conn.query_row("SELECT 1 FROM flows WHERE id = ?1", [flow_id], |_| Ok(())).is_ok() } fn clamp_ms(ms: u64) -> i32 { @@ -1431,38 +1955,6 @@ fn finish_body(writer: Option) { } } -fn finish_flow( - db: &ProxyQueryManager, - events: &RpcEventEmitter, - attribution: &Attribution, - f: InFlight, -) { - finish_body(f.request_body); - finish_body(f.response_body); - - let mut flow = f.flow; - flow.content_type = find_header(&flow.response_headers, "content-type").map(String::from); - if let Some(app) = attribution.get(f.connection, ATTRIBUTION_WAIT) { - flow.client_pid = Some(app.pid); - flow.client_name = Some(app.name); - flow.client_bundle_id = app.bundle_id; - flow.client_path = Some(app.path); - } - let written = db.with_tx(|ctx| { - let (saved, created) = ctx.upsert(&flow, &UpdateSource::Background)?; - // Sent while this holds the writer, before the commit, so a clear's event, sent after - // its own commit, always comes after it. - events.emit( - "model_write", - &ModelPayload { model: saved, change: ModelChangeEvent::Upsert { created } }, - ); - Ok::<_, yaak_database::Error>(()) - }); - if let Err(e) = written { - warn!("Failed to write proxy flow: {e}"); - } -} - // -- Router + Schema -- define_rpc! { @@ -1470,17 +1962,23 @@ define_rpc! { commands { execute_action(ActionInvocation) -> bool, format_body(FormatBodyRequest) -> String, + get_message_payload(GetMessagePayloadRequest) -> GetMessagePayloadResponse, list_actions(ListActionsRequest) -> ListActionsResponse, list_flows(flows::FlowFilter) -> ListFlowsResponse, + list_grpc_schemas(ListGrpcSchemasRequest) -> Vec, + list_messages(ListMessagesRequest) -> ListMessagesResponse, + remove_grpc_schema(RemoveGrpcSchemaRequest) -> bool, set_recording(SetRecordingRequest) -> bool, set_system_proxy_at_launch(SetSystemProxyAtLaunchRequest) -> bool, set_http_version(SetHttpVersionRequest) -> HttpVersion, } async_commands { + add_grpc_schema(AddGrpcSchemaRequest) -> GrpcSchema, app_icon(AppIconRequest) -> Option, ca_status(CaStatusRequest) -> CaStatus, ca_trust(CaTrustRequest) -> CaStatus, clear_flows(ClearFlowsRequest) -> usize, + decode_grpc_message(DecodeGrpcMessageRequest) -> DecodedGrpcMessage, export_har(ExportHarRequest) -> usize, filter_body(FilterBodyRequest) -> FilterBodyResponse, get_flow_body(GetFlowBodyRequest) -> GetFlowBodyResponse, @@ -1489,6 +1987,7 @@ define_rpc! { helper_unregister(HelperUnregisterRequest) -> SystemProxyStatus, list_workspaces(ListWorkspacesRequest) -> ListWorkspacesResponse, preview_save(PreviewSaveRequest) -> SavePreview, + reflect_grpc_host(ReflectGrpcHostRequest) -> GrpcSchema, save_flow(SaveFlowToYaakRequest) -> SavedFlow, save_flow_body(SaveFlowBodyRequest) -> SaveFlowBodyResponse, system_proxy_set(SystemProxySetRequest) -> SystemProxyStatus, @@ -1497,6 +1996,7 @@ define_rpc! { } events { flows_cleared(FlowsClearedPayload), + message_write(MessagePayload), model_write(ModelPayload), proxy_state_changed(ProxyStatePayload), } @@ -1545,6 +2045,250 @@ mod tests { assert!(log.was_on_at(SystemTime::now())); } + /// An event loop writing to `ctx`'s databases, fed by the returned sender. + fn event_loop( + ctx: &ProxyCtx, + ) -> (std::sync::mpsc::Sender, std::thread::JoinHandle<()>) { + let (tx, rx) = std::sync::mpsc::channel(); + let sink = FlowSink { + db: ctx.db.clone(), + blobs: ctx.blobs.clone(), + events: ctx.events.clone(), + attribution: Attribution::new(0), + recording: ctx.recording.clone(), + }; + (tx, std::thread::spawn(move || run_event_loop(rx, sink))) + } + + fn open_websocket(tx: &std::sync::mpsc::Sender, id: u64, url: &str) { + tx.send(ProxyEvent::RequestStart { + id, + started_at: SystemTime::now(), + method: "GET".into(), + url: url.into(), + http_version: "HTTP/1.1".into(), + connection: id, + client_addr: "127.0.0.1:1".parse().unwrap(), + }) + .unwrap(); + tx.send(ProxyEvent::ResponseStart { + id, + status: 101, + http_version: "HTTP/1.1".into(), + remote_addr: None, + elapsed_ms: 1, + }) + .unwrap(); + tx.send(ProxyEvent::StreamStart { id, kind: yaak_proxy::StreamKind::WebSocket }).unwrap(); + } + + fn ws_message( + id: u64, + direction: yaak_proxy::MessageDirection, + kind: yaak_proxy::MessageKind, + data: &[u8], + ) -> ProxyEvent { + ProxyEvent::Message { + id, + message: yaak_proxy::StreamMessage { + direction, + kind, + at: SystemTime::now(), + size: data.len() as u64, + data: bytes::Bytes::copy_from_slice(data), + truncated: false, + compressed: false, + error: None, + }, + } + } + + fn wait_for_flow(ctx: &ProxyCtx, check: impl Fn(&Flow) -> bool) -> Flow { + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + let flows = ctx.db.with_conn(|c| c.find_all::().unwrap()); + if let Some(flow) = flows.into_iter().find(&check) { + return flow; + } + assert!(std::time::Instant::now() < deadline, "no flow as expected"); + std::thread::sleep(Duration::from_millis(10)); + } + } + + #[tokio::test(flavor = "multi_thread")] + async fn websocket_messages_are_stored_and_exported() { + use yaak_proxy::{MessageDirection as D, MessageKind as K}; + let dir = tempfile::tempdir().unwrap(); + let (events, event_rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events).unwrap(); + let (tx, event_loop) = event_loop(&ctx); + open_websocket(&tx, 1, "https://example.com/socket"); + let big: Vec = (0..10_000u32).map(|i| i as u8).collect(); + tx.send(ws_message(1, D::Client, K::WebSocketText, b"hi")).unwrap(); + tx.send(ws_message(1, D::Server, K::WebSocketBinary, &big)).unwrap(); + let close = K::WebSocketClose { code: Some(1000), reason: "bye".into() }; + tx.send(ws_message(1, D::Client, close, b"\x03\xe8bye")).unwrap(); + + // Written while open, its count following its messages. + let open = wait_for_flow(&ctx, |f| f.message_count == 3); + assert_eq!((open.stream_kind, open.elapsed_ms), (Some(StreamKind::Websocket), None)); + tx.send(ProxyEvent::StreamEnd { id: 1, elapsed_ms: 50, error: None }).unwrap(); + drop(tx); + event_loop.join().unwrap(); + + let flow = flows::list(&ctx.db, &Default::default()).unwrap().remove(0); + assert_eq!((flow.elapsed_ms, flow.error.as_deref()), (Some(50), None)); + let stored = messages::list(&ctx.db, &flow.id, None, None).unwrap(); + let kinds: Vec<_> = stored.iter().map(|m| (m.seq, m.kind, m.direction)).collect(); + assert_eq!( + kinds, + vec![ + (0, messages::MessageKind::WebsocketText, messages::MessageDirection::Client), + (1, messages::MessageKind::WebsocketBinary, messages::MessageDirection::Server), + (2, messages::MessageKind::WebsocketClose, messages::MessageDirection::Client), + ] + ); + assert_eq!(stored[0].preview.as_deref(), Some("hi")); + assert_eq!(stored[1].preview, None); + assert_eq!( + (stored[2].meta.close_code, stored[2].meta.close_reason.as_deref()), + (Some(1000), Some("bye")) + ); + // The large payload is in the blob store, the small ones in their rows. + assert_eq!(messages::payload(&ctx.db, &ctx.blobs, &stored[1]).unwrap(), big); + assert!(ctx.blobs.connect().body_exists(&stored[1].blob_id()).unwrap()); + assert!(!ctx.blobs.connect().body_exists(&stored[0].blob_id()).unwrap()); + assert_eq!(messages::payload(&ctx.db, &ctx.blobs, &stored[0]).unwrap(), b"hi"); + + let har = + har::export(&ctx.db, &ctx.blobs, std::slice::from_ref(&flow), 1 << 20).await.unwrap(); + let har = serde_json::to_value(&har).unwrap(); + let sent = &har["log"]["entries"][0]["_webSocketMessages"]; + assert_eq!(sent.as_array().unwrap().len(), 2, "data messages only, as devtools exports"); + assert_eq!((sent[0]["type"].as_str(), sent[0]["opcode"].as_u64()), (Some("send"), Some(1))); + assert_eq!(sent[0]["data"], "hi"); + assert_eq!( + (sent[1]["type"].as_str(), sent[1]["opcode"].as_u64()), + (Some("receive"), Some(2)) + ); + assert_eq!(sent[1]["data"], BASE64.encode(&big)); + + let names: Vec<_> = event_rx.try_iter().map(|e| e.event).collect(); + assert_eq!(names.iter().filter(|n| **n == "message_write").count(), 3, "{names:?}"); + + ctx.clear_flows().unwrap(); + assert!(!ctx.blobs.connect().body_exists(&stored[1].blob_id()).unwrap()); + } + + #[test] + fn a_stream_cleared_while_open_stays_cleared() { + use yaak_proxy::{MessageDirection as D, MessageKind as K}; + let dir = tempfile::tempdir().unwrap(); + let (events, _event_rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events).unwrap(); + let (tx, event_loop) = event_loop(&ctx); + open_websocket(&tx, 1, "https://example.com/socket"); + tx.send(ws_message(1, D::Client, K::WebSocketText, b"before")).unwrap(); + let flow = wait_for_flow(&ctx, |f| f.message_count == 1); + ctx.clear_flows().unwrap(); + + tx.send(ws_message(1, D::Server, K::WebSocketBinary, &[7; 8192])).unwrap(); + // A body chunk past a blob chunk's size would be written at once, were it kept. + let data = bytes::Bytes::from(vec![b'x'; 2 << 20]); + tx.send(ProxyEvent::ResponseBodyChunk { id: 1, len: data.len(), data }).unwrap(); + std::thread::sleep(Duration::from_millis(500)); + let blobs = ctx.blobs.connect().list_body_ids().unwrap(); + assert!(blobs.iter().all(|b| !b.starts_with(&flow.id)), "while open: {blobs:?}"); + + tx.send(ProxyEvent::StreamEnd { id: 1, elapsed_ms: 9, error: None }).unwrap(); + drop(tx); + event_loop.join().unwrap(); + assert_eq!(flows::count(&ctx.db).unwrap(), 0, "not brought back by its end"); + assert_eq!(messages::count(&ctx.db, &flow.id).unwrap(), 0); + let blobs = ctx.blobs.connect().list_body_ids().unwrap(); + assert!(blobs.iter().all(|b| !b.starts_with(&flow.id)), "{blobs:?}"); + } + + #[test] + fn an_open_stream_is_written_while_events_never_let_up() { + use yaak_proxy::{MessageDirection as D, MessageKind as K}; + let dir = tempfile::tempdir().unwrap(); + let (events, _event_rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events).unwrap(); + let (tx, event_loop) = event_loop(&ctx); + open_websocket(&tx, 1, "https://example.com/socket"); + wait_for_flow(&ctx, |f| f.stream_kind.is_some()); + + // A backlog of messages, each followed by another event, which costs the loop a + // transaction for the message before it, so its queue does not empty for a while. + const SENT: i64 = 10_000; + for _ in 0..SENT { + tx.send(ws_message(1, D::Client, K::WebSocketText, b"tick")).unwrap(); + let data = bytes::Bytes::from_static(b"x"); + tx.send(ProxyEvent::ResponseBodyChunk { id: 1, len: 1, data }).unwrap(); + } + let counted = wait_for_flow(&ctx, |f| f.message_count > 0); + assert!(counted.message_count < SENT, "written before the backlog was through"); + drop(tx); + event_loop.join().unwrap(); + } + + #[test] + fn an_open_call_shows_its_response_before_any_message() { + let dir = tempfile::tempdir().unwrap(); + let (events, _event_rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events).unwrap(); + let (tx, event_loop) = event_loop(&ctx); + tx.send(ProxyEvent::RequestStart { + id: 1, + started_at: SystemTime::now(), + method: "POST".into(), + url: "https://example.com/pkg.Greeter/Watch".into(), + http_version: "HTTP/2".into(), + connection: 1, + client_addr: "127.0.0.1:1".parse().unwrap(), + }) + .unwrap(); + tx.send(ProxyEvent::StreamStart { id: 1, kind: yaak_proxy::StreamKind::Grpc }).unwrap(); + wait_for_flow(&ctx, |f| f.stream_kind.is_some()); + tx.send(ProxyEvent::ResponseStart { + id: 1, + status: 200, + http_version: "HTTP/2".into(), + remote_addr: None, + elapsed_ms: 3, + }) + .unwrap(); + tx.send(ProxyEvent::ResponseHeader { + id: 1, + name: "content-type".into(), + value: "application/grpc".into(), + }) + .unwrap(); + // No message yet, and the server may send none for a long while. + let open = wait_for_flow(&ctx, |f| f.status == Some(200)); + assert_eq!( + (open.elapsed_ms, open.content_type.as_deref()), + (None, Some("application/grpc")) + ); + drop(tx); + event_loop.join().unwrap(); + } + + #[test] + fn streams_still_open_end_when_the_engine_stops() { + let dir = tempfile::tempdir().unwrap(); + let (events, _event_rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events).unwrap(); + let (tx, event_loop) = event_loop(&ctx); + open_websocket(&tx, 1, "https://example.com/socket"); + wait_for_flow(&ctx, |f| f.stream_kind.is_some()); + drop(tx); + event_loop.join().unwrap(); + let flow = flows::list(&ctx.db, &Default::default()).unwrap().remove(0); + assert_eq!(flow.error.as_deref(), Some(STOPPED_MID_STREAM)); + } + /// A flow whose response is `body` under `content_type`, stored in `ctx` fn flow_with_response(ctx: &ProxyCtx, id: &str, content_type: &str, body: &[u8]) { let flow = Flow { diff --git a/crates-proxy/yaak-proxy-lib/src/messages.rs b/crates-proxy/yaak-proxy-lib/src/messages.rs new file mode 100644 index 000000000..190f7715d --- /dev/null +++ b/crates-proxy/yaak-proxy-lib/src/messages.rs @@ -0,0 +1,368 @@ +//! The messages of streaming flows: WebSocket frames, server-sent events, and gRPC messages and +//! trailers. Each is a row of `flow_messages`, its payload inline when small, else in +//! `blobs.sqlite`. Like flows, they are never pruned; clearing removes them with their flow. + +use crate::db::ProxyQueryManager; +use chrono::{DateTime, NaiveDateTime, Utc}; +use rusqlite::{Row, params}; +use serde::{Deserialize, Serialize}; +use ts_rs::TS; +use yaak_database::generate_prefixed_id; +use yaak_models::blob_manager::BlobManager; + +/// Payloads up to this many bytes are kept in the row; larger ones in `blobs.sqlite`. +pub const INLINE_LIMIT: usize = 4096; +/// Characters of a text payload kept as its preview. +pub const PREVIEW_CHARS: usize = 200; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "snake_case")] +#[ts(export, export_to = "gen_models.ts")] +pub enum MessageDirection { + Client, + Server, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "snake_case")] +#[ts(export, export_to = "gen_models.ts")] +pub enum MessageKind { + WebsocketText, + WebsocketBinary, + WebsocketPing, + WebsocketPong, + WebsocketClose, + SseEvent, + GrpcMessage, + GrpcTrailers, +} + +const KINDS: [MessageKind; 8] = [ + MessageKind::WebsocketText, + MessageKind::WebsocketBinary, + MessageKind::WebsocketPing, + MessageKind::WebsocketPong, + MessageKind::WebsocketClose, + MessageKind::SseEvent, + MessageKind::GrpcMessage, + MessageKind::GrpcTrailers, +]; + +impl MessageKind { + pub fn as_str(self) -> &'static str { + match self { + Self::WebsocketText => "websocket_text", + Self::WebsocketBinary => "websocket_binary", + Self::WebsocketPing => "websocket_ping", + Self::WebsocketPong => "websocket_pong", + Self::WebsocketClose => "websocket_close", + Self::SseEvent => "sse_event", + Self::GrpcMessage => "grpc_message", + Self::GrpcTrailers => "grpc_trailers", + } + } + + pub fn parse(s: &str) -> Option { + KINDS.into_iter().find(|k| k.as_str() == s) + } + + /// The WebSocket opcode, for HAR's `_webSocketMessages`. + pub fn websocket_opcode(self) -> Option { + match self { + Self::WebsocketText => Some(1), + Self::WebsocketBinary => Some(2), + Self::WebsocketClose => Some(8), + Self::WebsocketPing => Some(9), + Self::WebsocketPong => Some(10), + _ => None, + } + } +} + +/// What a kind of message carries beside its payload. Fields that do not apply are left out. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(default, rename_all = "camelCase")] +#[ts(export, export_to = "gen_models.ts")] +pub struct MessageMeta { + /// A WebSocket close frame's status code. + #[serde(skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub close_code: Option, + /// A WebSocket close frame's reason. + #[serde(skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub close_reason: Option, + /// An SSE event's `event` field. + #[serde(skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub event: Option, + /// An SSE event's `id` field. + #[serde(skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub event_id: Option, + /// An SSE event's `retry` field, in milliseconds. + #[serde(skip_serializing_if = "Option::is_none")] + #[ts(optional, type = "number")] + pub retry: Option, + /// An SSE event's lines as they came, comments included, up to 4 KB. + #[serde(skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub raw: Option, + /// gRPC trailers' `grpc-status`. + #[serde(skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub grpc_status: Option, + /// gRPC trailers' `grpc-message`, decoded. + #[serde(skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub grpc_message: Option, +} + +/// One message of a streaming flow. Its payload is fetched on its own. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_models.ts")] +pub struct FlowMessage { + pub id: String, + pub flow_id: String, + /// Its place in the flow, from 0, in the order messages passed the proxy. + #[ts(type = "number")] + pub seq: i64, + pub direction: MessageDirection, + pub kind: MessageKind, + /// When it had passed the proxy whole. + pub timestamp: NaiveDateTime, + /// Bytes it took on the wire: the frames' payloads, the event's lines, the gRPC message + /// without its prefix. + #[ts(type = "number")] + pub size: i64, + /// Bytes of payload stored: unmasked, reassembled, inflated or decompressed, and cut at the + /// capture limit. + #[ts(type = "number")] + pub stored_size: i64, + /// The stored payload is cut short at the capture limit. + pub truncated: bool, + /// It was compressed on the wire (permessage-deflate, gRPC's compressed flag); the payload is + /// stored decompressed unless `error` says otherwise. + pub compressed: bool, + /// Why the payload could not be decoded or stored. + pub error: Option, + pub meta: MessageMeta, + /// The payload's start as text, when it is text. + pub preview: Option, +} + +impl FlowMessage { + /// The message as it is stored, from the engine's. + pub fn from_engine(flow_id: &str, seq: i64, message: &yaak_proxy::StreamMessage) -> Self { + use yaak_proxy::MessageKind as K; + let mut meta = MessageMeta::default(); + let kind = match &message.kind { + K::WebSocketText => MessageKind::WebsocketText, + K::WebSocketBinary => MessageKind::WebsocketBinary, + K::WebSocketPing => MessageKind::WebsocketPing, + K::WebSocketPong => MessageKind::WebsocketPong, + K::WebSocketClose { code, reason } => { + meta.close_code = *code; + meta.close_reason = Some(reason.clone()).filter(|r| !r.is_empty()); + MessageKind::WebsocketClose + } + K::SseEvent { event, id, retry, raw } => { + meta.event = event.clone(); + meta.event_id = id.clone(); + meta.retry = *retry; + meta.raw = Some(raw.clone()); + MessageKind::SseEvent + } + K::GrpcMessage => MessageKind::GrpcMessage, + K::GrpcTrailers { status, message } => { + meta.grpc_status = *status; + meta.grpc_message = message.clone(); + MessageKind::GrpcTrailers + } + }; + let direction = match message.direction { + yaak_proxy::MessageDirection::Client => MessageDirection::Client, + yaak_proxy::MessageDirection::Server => MessageDirection::Server, + }; + let preview = match kind { + // Binary by nature, however it happens to read. + MessageKind::WebsocketBinary | MessageKind::GrpcMessage => None, + MessageKind::WebsocketClose => meta.close_reason.clone(), + _ => text_preview(&message.data, message.truncated), + }; + Self { + id: generate_prefixed_id("fm"), + flow_id: flow_id.to_string(), + seq, + direction, + kind, + timestamp: DateTime::::from(message.at).naive_utc(), + size: i64::try_from(message.size).unwrap_or(i64::MAX), + stored_size: message.data.len() as i64, + truncated: message.truncated, + compressed: message.compressed, + error: message.error.clone(), + meta, + preview, + } + } + + /// Where a payload too large for the row is kept in `blobs.sqlite`. + pub fn blob_id(&self) -> String { + format!("{}.msg.{}", self.flow_id, self.id) + } + + fn from_row(r: &Row) -> rusqlite::Result { + let direction: String = r.get("direction")?; + let kind: String = r.get("kind")?; + let meta: String = r.get("meta")?; + Ok(Self { + id: r.get("id")?, + flow_id: r.get("flow_id")?, + seq: r.get("seq")?, + direction: if direction == "client" { + MessageDirection::Client + } else { + MessageDirection::Server + }, + kind: MessageKind::parse(&kind).unwrap_or(MessageKind::WebsocketBinary), + timestamp: r.get("timestamp")?, + size: r.get("size")?, + stored_size: r.get("stored_size")?, + truncated: r.get("truncated")?, + compressed: r.get("compressed")?, + error: r.get("error")?, + meta: serde_json::from_str(&meta).unwrap_or_default(), + preview: r.get("preview")?, + }) + } +} + +/// The start of `data` as text, if it is UTF-8: a cut can split a character at the end. +fn text_preview(data: &[u8], truncated: bool) -> Option { + let text = match std::str::from_utf8(data) { + Ok(text) => text, + Err(e) if truncated && e.error_len().is_none() => { + std::str::from_utf8(&data[..e.valid_up_to()]).ok()? + } + Err(_) => return None, + }; + Some(text.chars().take(PREVIEW_CHARS).collect()) +} + +/// Writes `message`, with its payload in the row when `inline` has it. +pub fn insert( + conn: &rusqlite::Connection, + message: &FlowMessage, + inline: Option<&[u8]>, +) -> rusqlite::Result<()> { + let meta = serde_json::to_string(&message.meta).unwrap_or_else(|_| "{}".into()); + let direction = match message.direction { + MessageDirection::Client => "client", + MessageDirection::Server => "server", + }; + conn.execute( + "INSERT INTO flow_messages (id, flow_id, seq, direction, kind, timestamp, size, \ + stored_size, truncated, compressed, error, meta, preview, data) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)", + params![ + message.id, + message.flow_id, + message.seq, + direction, + message.kind.as_str(), + message.timestamp, + message.size, + message.stored_size, + message.truncated, + message.compressed, + message.error, + meta, + message.preview, + inline, + ], + )?; + Ok(()) +} + +const COLUMNS: &str = "id, flow_id, seq, direction, kind, timestamp, size, stored_size, \ + truncated, compressed, error, meta, preview"; + +/// A flow's messages in order, after `after_seq` when given, at most `limit` of them. +pub fn list( + db: &ProxyQueryManager, + flow_id: &str, + after_seq: Option, + limit: Option, +) -> Result, String> { + db.with_conn(|c| { + let conn = c.conn().resolve(); + let limit = limit.map_or(-1, |l| l.min(i64::MAX as usize) as i64); + let mut stmt = conn + .prepare(&format!( + "SELECT {COLUMNS} FROM flow_messages WHERE flow_id = ?1 AND seq > ?2 \ + ORDER BY seq LIMIT ?3" + )) + .map_err(|e| e.to_string())?; + let rows = stmt + .query_map(params![flow_id, after_seq.unwrap_or(-1), limit], FlowMessage::from_row) + .map_err(|e| e.to_string())?; + rows.collect::, _>>().map_err(|e| e.to_string()) + }) +} + +/// Messages recorded for a flow. +pub fn count(db: &ProxyQueryManager, flow_id: &str) -> Result { + db.with_conn(|c| { + c.conn() + .resolve() + .query_row("SELECT COUNT(*) FROM flow_messages WHERE flow_id = ?1", [flow_id], |r| { + r.get(0) + }) + .map_err(|e| e.to_string()) + }) +} + +pub fn find( + db: &ProxyQueryManager, + flow_id: &str, + seq: i64, +) -> Result, String> { + db.with_conn(|c| { + let conn = c.conn().resolve(); + let found = conn.query_row( + &format!("SELECT {COLUMNS} FROM flow_messages WHERE flow_id = ?1 AND seq = ?2"), + params![flow_id, seq], + FlowMessage::from_row, + ); + match found { + Ok(message) => Ok(Some(message)), + Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None), + Err(e) => Err(e.to_string()), + } + }) +} + +pub fn get(db: &ProxyQueryManager, flow_id: &str, seq: i64) -> Result { + find(db, flow_id, seq)?.ok_or_else(|| format!("No message {seq} in flow {flow_id}")) +} + +/// A message's stored payload: decoded, and cut at the capture limit. +pub fn payload( + db: &ProxyQueryManager, + blobs: &BlobManager, + message: &FlowMessage, +) -> Result, String> { + let inline: Option> = db.with_conn(|c| { + c.conn() + .resolve() + .query_row("SELECT data FROM flow_messages WHERE id = ?1", [&message.id], |r| r.get(0)) + .map_err(|e| e.to_string()) + })?; + if let Some(data) = inline { + return Ok(data); + } + let chunks = blobs.connect().get_chunks(&message.blob_id()).map_err(|e| e.to_string())?; + Ok(chunks.into_iter().flat_map(|c| c.data).collect()) +} diff --git a/crates-proxy/yaak-proxy-lib/src/models.rs b/crates-proxy/yaak-proxy-lib/src/models.rs index 15d35747e..606323b38 100644 --- a/crates-proxy/yaak-proxy-lib/src/models.rs +++ b/crates-proxy/yaak-proxy-lib/src/models.rs @@ -66,6 +66,64 @@ pub struct Flow { pub client_bundle_id: Option, /// Its app bundle's path, or its executable's outside an app. pub client_path: Option, + /// The flow carries a stream of messages, listed with `list_messages`. It is written as the + /// stream opens and again as messages land; it is open until `elapsedMs`, `error` or + /// `cancelled` is set. + pub stream_kind: Option, + /// Messages recorded so far. + #[ts(type = "number")] + pub message_count: i64, + /// A gRPC call's service, from its path: `package.Service`. + pub grpc_service: Option, + /// A gRPC call's method, from its path. + pub grpc_method: Option, + /// A gRPC call's status, from its trailers. Not 0 is a failed call. + pub grpc_status: Option, + /// A gRPC call's `grpc-message`, decoded. + pub grpc_message: Option, +} + +/// The streaming protocol a flow carries. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "snake_case")] +#[ts(export, export_to = "gen_models.ts")] +pub enum StreamKind { + Websocket, + Sse, + Grpc, + GrpcWeb, +} + +impl StreamKind { + pub fn as_str(self) -> &'static str { + match self { + Self::Websocket => "websocket", + Self::Sse => "sse", + Self::Grpc => "grpc", + Self::GrpcWeb => "grpc_web", + } + } + + pub fn parse(s: &str) -> Option { + [Self::Websocket, Self::Sse, Self::Grpc, Self::GrpcWeb] + .into_iter() + .find(|k| k.as_str() == s) + } + + pub fn is_grpc(self) -> bool { + matches!(self, Self::Grpc | Self::GrpcWeb) + } +} + +impl From for StreamKind { + fn from(kind: yaak_proxy::StreamKind) -> Self { + match kind { + yaak_proxy::StreamKind::WebSocket => Self::Websocket, + yaak_proxy::StreamKind::Sse => Self::Sse, + yaak_proxy::StreamKind::Grpc => Self::Grpc, + yaak_proxy::StreamKind::GrpcWeb => Self::GrpcWeb, + } + } } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, TS)] @@ -164,6 +222,12 @@ impl UpsertModelInfo for Flow { (ClientName, self.client_name.into()), (ClientBundleId, self.client_bundle_id.into()), (ClientPath, self.client_path.into()), + (FlowIden::StreamKind, self.stream_kind.map(|k| k.as_str()).into()), + (MessageCount, self.message_count.into()), + (GrpcService, self.grpc_service.into()), + (GrpcMethod, self.grpc_method.into()), + (GrpcStatus, self.grpc_status.into()), + (GrpcMessage, self.grpc_message.into()), ]) } @@ -193,6 +257,12 @@ impl UpsertModelInfo for Flow { FlowIden::ClientName, FlowIden::ClientBundleId, FlowIden::ClientPath, + FlowIden::StreamKind, + FlowIden::MessageCount, + FlowIden::GrpcService, + FlowIden::GrpcMethod, + FlowIden::GrpcStatus, + FlowIden::GrpcMessage, ] } @@ -229,6 +299,13 @@ impl UpsertModelInfo for Flow { client_name: added_column(r, "client_name")?, client_bundle_id: added_column(r, "client_bundle_id")?, client_path: added_column(r, "client_path")?, + stream_kind: added_column::(r, "stream_kind")? + .and_then(|k| StreamKind::parse(&k)), + message_count: added_column(r, "message_count")?.unwrap_or_default(), + grpc_service: added_column(r, "grpc_service")?, + grpc_method: added_column(r, "grpc_method")?, + grpc_status: added_column(r, "grpc_status")?, + grpc_message: added_column(r, "grpc_message")?, }) } } diff --git a/crates-proxy/yaak-proxy-lib/tests/streams.rs b/crates-proxy/yaak-proxy-lib/tests/streams.rs new file mode 100644 index 000000000..42a13462c --- /dev/null +++ b/crates-proxy/yaak-proxy-lib/tests/streams.rs @@ -0,0 +1,437 @@ +//! Streaming flows through a running engine: server-sent events and gRPC-Web over plain +//! HTTP, stored as messages while they stream, and gRPC messages decoded with and without a +//! schema. + +use std::convert::Infallible; +use std::net::SocketAddr; +use std::path::Path; +use std::time::{Duration, Instant}; + +use bytes::Bytes; +use http_body_util::combinators::UnsyncBoxBody; +use http_body_util::{BodyExt, Full, StreamBody}; +use hyper::body::{Frame, Incoming}; +use hyper::service::service_fn; +use hyper::{Request, Response}; +use hyper_util::rt::{TokioExecutor, TokioIo}; +use hyper_util::server::conn::auto; +use serde_json::{Value, json}; +use tokio::net::TcpListener; +use yaak_proxy_lib::models::Flow; +use yaak_proxy_lib::settings::ProxySettings; +use yaak_proxy_lib::{GrpcConfig, ProxyCtx}; +use yaak_rpc::RpcEventEmitter; + +type BoxError = Box; +type TestBody = UnsyncBoxBody; + +const PROTO: &str = r#" +syntax = "proto3"; +package pkg; +service Greeter { + rpc SayHello (HelloRequest) returns (HelloReply); +} +message HelloRequest { string name = 1; } +message HelloReply { string message = 1; int32 count = 2; } +"#; + +/// The longest a test here may run: one that hangs fails with a message instead of holding CI +/// until its job is cancelled. +const TEST_TIMEOUT: Duration = Duration::from_secs(60); + +async fn within(test: impl Future) -> T { + match tokio::time::timeout(TEST_TIMEOUT, test).await { + Ok(result) => result, + Err(_) => panic!("the test did not finish within {TEST_TIMEOUT:?}"), + } +} + +fn full(data: impl Into) -> TestBody { + Full::new(data.into()).map_err(|never| match never {}).boxed_unsync() +} + +fn grpc_frame(flag: u8, payload: &[u8]) -> Vec { + let mut out = vec![flag]; + out.extend((payload.len() as u32).to_be_bytes()); + out.extend(payload); + out +} + +/// A protobuf message with a string in field 1, and a varint in field 2 when given. +fn proto(text: &str, count: Option) -> Vec { + let mut out = vec![0x0a, text.len() as u8]; + out.extend(text.as_bytes()); + if let Some(count) = count { + out.extend([0x10, count]); + } + out +} + +fn protoc() -> GrpcConfig { + GrpcConfig { + protoc_bin_path: protoc_bin_vendored::protoc_bin_path().unwrap(), + protoc_include_dir: protoc_bin_vendored::include_path().unwrap(), + } +} + +fn proto_file(dir: &Path) -> String { + let path = dir.join("greeter.proto"); + std::fs::write(&path, PROTO).unwrap(); + path.to_string_lossy().into_owned() +} + +type ReflectionFuture = std::pin::Pin> + Send>>; +/// A gRPC reflection service, its type erased. +type Reflection = + std::sync::Arc) -> ReflectionFuture + Send + Sync>; + +fn reflection_service(service: S) -> Reflection +where + S: tonic::codegen::Service< + Request, + Response = Response, + Error = Infallible, + > + Clone + + Send + + Sync + + 'static, + S::Future: Send, +{ + std::sync::Arc::new(move |req| { + let mut service = service.clone(); + Box::pin(async move { + std::future::poll_fn(|cx| service.poll_ready(cx)).await.unwrap(); + service.call(req).await.unwrap() + }) + }) +} + +/// Serves the upstream's routes, and gRPC reflection for `pkg.Greeter` over HTTP/2 when given +/// one. +async fn route( + req: Request, + reflection: Option, +) -> Result, Infallible> { + let path = req.uri().path().to_string(); + if path.starts_with("/grpc.reflection.v1.ServerReflection/") { + let Some(reflection) = reflection else { + return Ok(Response::builder().status(404).body(full("")).unwrap()); + }; + let resp = reflection(req.map(tonic::body::Body::new)).await; + return Ok(resp.map(|b| b.map_err(|e| Box::new(e) as BoxError).boxed_unsync())); + } + Ok(match path.as_str() { + // Two events a while apart, so the first is stored while the stream is open. + "/events" => { + let frames = futures_util::stream::unfold(0, |i| async move { + let chunk = match i { + 0 => "event: tick\nid: 1\ndata: first\n\n", + 1 => { + tokio::time::sleep(Duration::from_millis(1500)).await; + "data: second\n\n" + } + _ => return None, + }; + Some((Ok::<_, BoxError>(Frame::data(Bytes::from(chunk))), i + 1)) + }); + Response::builder() + .header("content-type", "text/event-stream") + .body(StreamBody::new(frames).boxed_unsync()) + .unwrap() + } + "/pkg.Greeter/SayHello" => { + let body = req.into_body().collect().await.unwrap().to_bytes(); + let name = String::from_utf8_lossy(&body[7..]).into_owned(); + let mut out = grpc_frame(0, &proto(&format!("Hello {name}"), Some(3))); + out.extend(grpc_frame(0x80, b"grpc-status: 0\r\n")); + Response::builder() + .header("content-type", "application/grpc-web+proto") + .body(full(out)) + .unwrap() + } + _ => Response::builder() + .header("content-type", "application/grpc-web+proto") + .body(full(grpc_frame(0x80, b"grpc-status: 12\r\ngrpc-message: not%20here\r\n"))) + .unwrap(), + }) +} + +async fn spawn_upstream(reflection: Option) -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + loop { + let (tcp, _) = listener.accept().await.unwrap(); + let reflection = reflection.clone(); + tokio::spawn(async move { + let svc = service_fn(move |req| route(req, reflection.clone())); + let _ = auto::Builder::new(TokioExecutor::new()) + .serve_connection(TokioIo::new(tcp), svc) + .await; + }); + } + }); + addr +} + +fn start(dir: &Path) -> (ProxyCtx, std::sync::mpsc::Receiver) { + let settings = ProxySettings { + port: 0, + capture_body_limit: 1024 * 1024, + system_proxy_at_launch: false, + ..Default::default() + }; + settings.save(&dir.join("settings.json")).unwrap(); + let (events, event_rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir, events).unwrap().with_protoc(protoc()); + ctx.start().unwrap(); + (ctx, event_rx) +} + +fn client(ctx: &ProxyCtx) -> reqwest::Client { + let proxy = format!("http://127.0.0.1:{}", ctx.port().unwrap()); + reqwest::Client::builder().proxy(reqwest::Proxy::all(proxy).unwrap()).build().unwrap() +} + +async fn rpc(ctx: &ProxyCtx, cmd: &str, payload: Value) -> Result { + yaak_proxy_lib::build_router().dispatch(cmd, payload, ctx).await.map_err(|e| e.message) +} + +/// Waits until `check` holds for the flow of `url`, as stored. +async fn wait_for(ctx: &ProxyCtx, url: &str, check: impl Fn(&Flow) -> bool) -> Flow { + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let flows = ctx.db.with_conn(|db| db.find_all::().unwrap()); + if let Some(flow) = flows.into_iter().find(|f| f.url == url && check(f)) { + return flow; + } + assert!(Instant::now() < deadline, "no flow for {url} as expected"); + tokio::time::sleep(Duration::from_millis(20)).await; + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn server_sent_events_are_stored_while_the_stream_is_open() { + within(async { + let dir = tempfile::tempdir().unwrap(); + let (ctx, event_rx) = start(dir.path()); + let upstream = spawn_upstream(None).await; + let url = format!("http://{upstream}/events"); + let reading = tokio::spawn({ + let client = client(&ctx); + let url = url.clone(); + async move { client.get(&url).send().await.unwrap().text().await.unwrap() } + }); + + // The flow is in the database with its first event while the second is still to come. + let open = wait_for(&ctx, &url, |f| f.message_count == 1).await; + assert_eq!(open.stream_kind.map(|k| k.as_str()), Some("sse")); + assert_eq!(open.elapsed_ms, None, "still streaming"); + let listed = rpc(&ctx, "list_messages", json!({ "flowId": open.id })).await.unwrap(); + assert_eq!(listed["total"], 1); + let first = &listed["messages"][0]; + assert_eq!( + (first["kind"].as_str(), first["direction"].as_str()), + (Some("sse_event"), Some("server")) + ); + assert_eq!(first["preview"], "first"); + assert_eq!( + first["meta"], + json!({ "event": "tick", "eventId": "1", "raw": "event: tick\nid: 1\ndata: first\n\n" }) + ); + + assert_eq!(reading.await.unwrap(), "event: tick\nid: 1\ndata: first\n\ndata: second\n\n"); + let ended = wait_for(&ctx, &url, |f| f.elapsed_ms.is_some()).await; + assert_eq!((ended.message_count, ended.error), (2, None)); + let after = + rpc(&ctx, "list_messages", json!({ "flowId": ended.id, "afterSeq": 0 })).await.unwrap(); + assert_eq!(after["messages"].as_array().unwrap().len(), 1); + let payload = rpc(&ctx, "get_message_payload", json!({ "flowId": ended.id, "seq": 1 })) + .await + .unwrap(); + assert_eq!(payload["data"], "c2Vjb25k", "\"second\" in base64"); + + // The app heard of the open flow, of each message, and of the end. + let events: Vec<_> = event_rx.try_iter().map(|e| e.event).collect(); + let writes = |name: &str| events.iter().filter(|e| **e == name).count(); + assert_eq!(writes("message_write"), 2, "{events:?}"); + assert!(writes("model_write") >= 2, "{events:?}"); + }) + .await +} + +#[tokio::test(flavor = "multi_thread")] +async fn grpc_web_calls_are_stored_and_decoded() { + within(async { + let dir = tempfile::tempdir().unwrap(); + let (ctx, _event_rx) = start(dir.path()); + let upstream = spawn_upstream(None).await; + let client = client(&ctx); + + let url = format!("http://{upstream}/pkg.Greeter/SayHello"); + let body = client + .post(&url) + .header("content-type", "application/grpc-web+proto") + .body(grpc_frame(0, &proto("Ada", None))) + .send() + .await + .unwrap() + .bytes() + .await + .unwrap(); + assert!(body.starts_with(&grpc_frame(0, &proto("Hello Ada", Some(3))))); + let flow = wait_for(&ctx, &url, |f| f.elapsed_ms.is_some()).await; + assert_eq!(flow.stream_kind.map(|k| k.as_str()), Some("grpc_web")); + assert_eq!( + (flow.grpc_service.as_deref(), flow.grpc_method.as_deref()), + (Some("pkg.Greeter"), Some("SayHello")) + ); + assert_eq!((flow.grpc_status, flow.message_count), (Some(0), 3)); + + let failing = format!("http://{upstream}/pkg.Greeter/Missing"); + client + .post(&failing) + .header("content-type", "application/grpc-web+proto") + .body(grpc_frame(0, &proto("Bo", None))) + .send() + .await + .unwrap(); + let failed = wait_for(&ctx, &failing, |f| f.elapsed_ms.is_some()).await; + assert_eq!( + (failed.grpc_status, failed.grpc_message.as_deref()), + (Some(12), Some("not here")) + ); + let errors = rpc(&ctx, "list_flows", json!({ "status": "error" })).await.unwrap(); + let ids: Vec<_> = + errors["flows"].as_array().unwrap().iter().map(|f| f["id"].clone()).collect(); + assert_eq!(ids, vec![json!(failed.id)], "a non-zero status is a failed call"); + + // No schema, and reflection against the server fails: decoded by wire type alone. + let decode = json!({ "flowId": flow.id, "seq": 1 }); + let schemaless = rpc(&ctx, "decode_grpc_message", decode.clone()).await.unwrap(); + assert_eq!(schemaless["messageType"], Value::Null); + assert!(schemaless["note"].as_str().unwrap().contains("reflection failed"), "{schemaless}"); + let fields: Value = serde_json::from_str(schemaless["json"].as_str().unwrap()).unwrap(); + assert_eq!(fields[0]["string"], "Hello Ada"); + assert_eq!(fields[1], json!({ "field": 2, "wireType": "varint", "value": 3 })); + + // With the proto attached to the host, by name. + let protos = tempfile::tempdir().unwrap(); + let added = rpc( + &ctx, + "add_grpc_schema", + json!({ "hostPattern": "127.0.0.1", "protoPaths": [proto_file(protos.path())] }), + ) + .await + .unwrap(); + assert_eq!(added["services"], json!(["pkg.Greeter"])); + assert_eq!(added["source"], "files"); + let decoded = rpc(&ctx, "decode_grpc_message", decode.clone()).await.unwrap(); + assert_eq!(decoded["messageType"], "pkg.HelloReply"); + let message: Value = serde_json::from_str(decoded["json"].as_str().unwrap()).unwrap(); + assert_eq!(message, json!({ "message": "Hello Ada", "count": 3 })); + let request = json!({ "flowId": flow.id, "seq": 0 }); + let decoded = rpc(&ctx, "decode_grpc_message", request).await.unwrap(); + assert_eq!(decoded["messageType"], "pkg.HelloRequest"); + let trailers = json!({ "flowId": flow.id, "seq": 2 }); + assert!( + rpc(&ctx, "decode_grpc_message", trailers).await.is_err(), + "trailers are no message" + ); + + // The control API returns the same, and lists the messages. + ctx.serve_control_api().unwrap(); + let info = ctx.control_info().unwrap(); + let api = reqwest::Client::new(); + let get = |path: String| { + api.get(format!("{}{path}", info.base_url())).bearer_auth(&info.token).send() + }; + let listed: Value = get(format!("/flows/{}/messages?after=0&limit=1", flow.id)) + .await + .unwrap() + .json() + .await + .unwrap(); + assert_eq!( + (listed["total"].as_i64(), listed["messages"][0]["seq"].as_i64()), + (Some(3), Some(1)) + ); + let detail: Value = + get(format!("/flows/{}/messages/1", flow.id)).await.unwrap().json().await.unwrap(); + assert_eq!(detail["decoded"]["messageType"], "pkg.HelloReply"); + assert_eq!(detail["payload"]["encoding"], "utf8"); + let schemas: Value = get("/grpc-schemas".into()).await.unwrap().json().await.unwrap(); + assert_eq!(schemas.as_array().unwrap().len(), 1); + + let removed = rpc(&ctx, "remove_grpc_schema", json!({ "id": added["id"] })).await.unwrap(); + assert_eq!(removed, true); + let after = rpc(&ctx, "decode_grpc_message", decode).await.unwrap(); + assert_eq!(after["messageType"], Value::Null); + + // Clearing removes the messages with their flows. + rpc(&ctx, "clear_flows", json!({})).await.unwrap(); + assert!(rpc(&ctx, "list_messages", json!({ "flowId": flow.id })).await.is_err()); + let left: i64 = ctx.db.with_conn(|c| { + c.conn() + .resolve() + .query_row("SELECT COUNT(*) FROM flow_messages", [], |r| r.get(0)) + .unwrap() + }); + assert_eq!(left, 0); + ctx.stop_control_api(); + }) + .await +} + +#[tokio::test(flavor = "multi_thread")] +async fn grpc_schemas_come_from_the_servers_reflection() { + within(async { + let protos = tempfile::tempdir().unwrap(); + let pool = + yaak_grpc::fill_pool_from_files(&protoc(), &vec![proto_file(protos.path()).into()]) + .await + .unwrap(); + let descriptors = pool.encode_to_vec(); + let reflection = tonic_reflection::server::Builder::configure() + .register_encoded_file_descriptor_set(&descriptors) + .build_v1() + .unwrap(); + let upstream = spawn_upstream(Some(reflection_service(reflection))).await; + + let dir = tempfile::tempdir().unwrap(); + let (ctx, _event_rx) = start(dir.path()); + // Plain HTTP, so reflection is asked without TLS, as the call was made. + let url = format!("http://{upstream}/pkg.Greeter/SayHello"); + client(&ctx) + .post(&url) + .header("content-type", "application/grpc-web+proto") + .body(grpc_frame(0, &proto("Cy", None))) + .send() + .await + .unwrap(); + let flow = wait_for(&ctx, &url, |f| f.elapsed_ms.is_some()).await; + + // Nothing attached: the server is asked once, and its answer kept for the host. Two + // decodes at once both wait for that answer. + let request = json!({ "flowId": flow.id, "seq": 1 }); + let (decoded, alongside) = tokio::join!( + rpc(&ctx, "decode_grpc_message", request.clone()), + rpc(&ctx, "decode_grpc_message", request) + ); + let (decoded, alongside) = (decoded.unwrap(), alongside.unwrap()); + assert_eq!(decoded["messageType"], "pkg.HelloReply", "{decoded}"); + assert_eq!(alongside["messageType"], "pkg.HelloReply", "{alongside}"); + assert_eq!(decoded["schema"]["source"], "reflection"); + assert_eq!(decoded["schema"]["hostPattern"], format!("127.0.0.1:{}", upstream.port())); + let schemas = rpc(&ctx, "list_grpc_schemas", json!({})).await.unwrap(); + assert_eq!(schemas.as_array().unwrap().len(), 1); + + // Asked again on demand, it replaces the host's earlier answer. + let host = format!("http://127.0.0.1:{}", upstream.port()); + let again = rpc(&ctx, "reflect_grpc_host", json!({ "host": host })).await.unwrap(); + assert_eq!(again["services"], json!(["pkg.Greeter"])); + let schemas = rpc(&ctx, "list_grpc_schemas", json!({})).await.unwrap(); + assert_eq!(schemas.as_array().unwrap().len(), 1); + }) + .await +} diff --git a/crates-tauri/yaak-app-proxy/src/lib.rs b/crates-tauri/yaak-app-proxy/src/lib.rs index ac694c4b2..5856a3c8e 100644 --- a/crates-tauri/yaak-app-proxy/src/lib.rs +++ b/crates-tauri/yaak-app-proxy/src/lib.rs @@ -1,5 +1,6 @@ use log::{error, info, warn}; use tauri::Runtime; +use tauri::path::BaseDirectory; use tauri::{Emitter, Manager, RunEvent, State, WebviewWindow}; use yaak_proxy_lib::ProxyCtx; use yaak_rpc::{RpcEventEmitter, RpcRouter}; @@ -48,6 +49,24 @@ fn setup_window_menu(win: &WebviewWindow) { }); } +/// Where `tauri.bundle.conf.json` puts the `yaakprotoc` that Yaak's vendor script fetches, in +/// the app's resources. +const PROTOC_DIR: &str = "vendored/protoc"; +/// Where it puts protoc's include files. +const PROTOC_INCLUDE_DIR: &str = "vendored/protoc/include"; + +/// The `yaakprotoc` and its includes in the app's resources, if the bundle has them, as a +/// packaged app's does; a development build has none. +fn bundled_protoc(app: &tauri::AppHandle) -> Option { + let bin_name = if cfg!(windows) { "yaakprotoc.exe" } else { "yaakprotoc" }; + let resource = |path: &str| app.path().resolve(path, BaseDirectory::Resource).ok(); + let protoc_bin_path = resource(&format!("{PROTOC_DIR}/{bin_name}"))?; + let protoc_include_dir = resource(PROTOC_INCLUDE_DIR)?; + protoc_bin_path + .is_file() + .then_some(yaak_proxy_lib::GrpcConfig { protoc_include_dir, protoc_bin_path }) +} + #[tauri::command] async fn rpc( router: State<'_, RpcRouter>, @@ -69,7 +88,12 @@ pub fn run() { std::fs::create_dir_all(&data_dir).expect("failed to create app data dir"); let (emitter, event_rx) = RpcEventEmitter::new(); - let ctx = ProxyCtx::new(&data_dir, emitter)?; + let mut ctx = ProxyCtx::new(&data_dir, emitter)?; + // Compiles protos attached for gRPC decoding when the bundle has protoc, as Yaak's + // does; otherwise one on PATH is used. + if let Some(config) = bundled_protoc(app.handle()) { + ctx = ctx.with_protoc(config); + } // Not while the CLI daemon owns the engine: the window offers to take it over. match ctx.serve_control_api() { // The engine runs for as long as the app does. A failure, such as the port @@ -127,3 +151,23 @@ pub fn run() { } }); } + +#[cfg(test)] +mod tests { + use super::*; + + /// The bundle puts protoc, and its includes, where the app looks for them. + #[test] + fn the_bundle_ships_protoc_where_the_app_looks() { + let config: serde_json::Value = + serde_json::from_str(include_str!("../tauri.bundle.conf.json")).unwrap(); + let resources = config["bundle"]["resources"].as_object().unwrap(); + let target = |source: &str| resources.get(source).and_then(|t| t.as_str()); + assert_eq!( + target("../yaak-app-client/vendored/protoc/yaakprotoc*") + .map(|t| t.trim_end_matches('/')), + Some(PROTOC_DIR) + ); + assert_eq!(target("../yaak-app-client/vendored/protoc/include"), Some(PROTOC_INCLUDE_DIR)); + } +} diff --git a/crates-tauri/yaak-app-proxy/tauri.bundle.conf.json b/crates-tauri/yaak-app-proxy/tauri.bundle.conf.json index b5feca0f4..bc2dfbde3 100644 --- a/crates-tauri/yaak-app-proxy/tauri.bundle.conf.json +++ b/crates-tauri/yaak-app-proxy/tauri.bundle.conf.json @@ -4,6 +4,10 @@ }, "bundle": { "externalBin": ["vendored/helper/yaak-proxy-helper"], + "resources": { + "../yaak-app-client/vendored/protoc/include": "vendored/protoc/include", + "../yaak-app-client/vendored/protoc/yaakprotoc*": "vendored/protoc/" + }, "macOS": { "minimumSystemVersion": "13.0", "hardenedRuntime": true, diff --git a/crates/yaak-grpc/src/lib.rs b/crates/yaak-grpc/src/lib.rs index 8911108a4..7624ae7bc 100644 --- a/crates/yaak-grpc/src/lib.rs +++ b/crates/yaak-grpc/src/lib.rs @@ -11,9 +11,17 @@ pub mod manager; mod reflection; mod transport; +pub use manager::GrpcConfig; +pub use prost_reflect; +pub use reflection::{fill_pool_from_files, fill_pool_from_reflection}; pub use tonic::Code; pub use tonic::metadata::*; +/// A message as pretty JSON, its default fields included, as Yaak shows gRPC messages. +pub fn message_to_json(msg: &DynamicMessage) -> Result { + serialize_dynamic_message_json(msg) +} + pub fn serialize_options() -> SerializeOptions { SerializeOptions::new().skip_default_fields(false) } diff --git a/crates/yaak-proxy/Cargo.toml b/crates/yaak-proxy/Cargo.toml index aa055f251..736231482 100644 --- a/crates/yaak-proxy/Cargo.toml +++ b/crates/yaak-proxy/Cargo.toml @@ -5,7 +5,9 @@ edition = "2024" publish = false [dependencies] -hyper = { version = "1", features = ["http1", "http2", "server", "client"] } +base64 = "0.23.1" +flate2 = "1" +hyper ={ version = "1", features = ["http1", "http2", "server", "client"] } hyper-util = { version = "0.1", features = ["tokio", "http1", "client-legacy", "server-auto"] } http-body-util = "0.1" http = "1" diff --git a/crates/yaak-proxy/src/body.rs b/crates/yaak-proxy/src/body.rs index cc814525f..c13e54a54 100644 --- a/crates/yaak-proxy/src/body.rs +++ b/crates/yaak-proxy/src/body.rs @@ -7,6 +7,9 @@ use std::time::Instant; use bytes::Bytes; use hyper::body::{Body, Frame, Incoming, SizeHint}; +use crate::grpc::{self, GrpcParser}; +use crate::messages::StreamMessage; +use crate::sse::SseParser; use crate::{Engine, ProxyEvent}; #[derive(Clone, Copy, PartialEq)] @@ -15,6 +18,36 @@ pub(crate) enum Direction { Response, } +/// Reads a streaming body's messages as the body passes. +pub(crate) enum BodyParser { + Sse(Box), + Grpc(GrpcParser), +} + +impl BodyParser { + fn feed(&mut self, data: &[u8]) -> Vec { + match self { + Self::Sse(p) => p.feed(data), + Self::Grpc(p) => p.feed(data), + } + } + + fn trailers(&mut self, trailers: &hyper::HeaderMap) -> Option { + match self { + Self::Sse(_) => None, + Self::Grpc(_) => grpc::trailers_from_headers(trailers), + } + } + + /// The body ended: messages held back for what might have followed. + fn finish(&mut self) -> Vec { + match self { + Self::Sse(p) => p.finish(), + Self::Grpc(_) => Vec::new(), + } + } +} + /// Streams a body through untouched, reporting each frame as a chunk event (with data up to /// `capture_body_limit`) and the end as a completion event. pub(crate) struct CaptureBody { @@ -28,6 +61,7 @@ pub(crate) struct CaptureBody { finished: bool, /// Set when reading the body fails. failed: Option>, + parser: Option, } impl CaptureBody { @@ -48,6 +82,19 @@ impl CaptureBody { size: 0, finished: false, failed: None, + parser: None, + } + } + + /// Records the messages `parser` reads from the body, all of it, past the capture limit too. + pub(crate) fn with_parser(mut self, parser: Option) -> Self { + self.parser = parser; + self + } + + fn emit_messages(&self, messages: impl IntoIterator) { + for message in messages { + self.engine.emit(ProxyEvent::Message { id: self.id, message }); } } @@ -69,6 +116,11 @@ impl CaptureBody { return; } self.finished = true; + // Before the body's end, so they land while the flow is still open. + if let Some(parser) = self.parser.as_mut() { + let messages = parser.finish(); + self.emit_messages(messages); + } let truncated = self.size > self.captured as u64; let event = match self.direction { Direction::Request if self.size == 0 => return, @@ -110,8 +162,20 @@ impl Body for CaptureBody { Direction::Request => ProxyEvent::RequestBodyChunk { id, len, data }, Direction::Response => ProxyEvent::ResponseBodyChunk { id, len, data }, }); + if let Some(parser) = self.parser.as_mut() { + let messages = parser.feed(frame_data); + self.emit_messages(messages); + } + } else if let Some(trailers) = frame.trailers_ref() + && let Some(parser) = self.parser.as_mut() + { + let message = parser.trailers(trailers); + self.emit_messages(message); } - if self.complete() { + // A gRPC call's status comes in trailers after its last byte, so its body is + // done only at the stream's end. + let grpc = matches!(self.parser, Some(BodyParser::Grpc(_))); + if if grpc { self.inner.is_end_stream() } else { self.complete() } { self.finish(None, false); } } diff --git a/crates/yaak-proxy/src/decode.rs b/crates/yaak-proxy/src/decode.rs new file mode 100644 index 000000000..3488f857a --- /dev/null +++ b/crates/yaak-proxy/src/decode.rs @@ -0,0 +1,378 @@ +//! A response body's `Content-Encoding` undone as it streams, for parsers that read messages +//! from the body. Output comes in pieces of at most [`PIECE`] bytes, so a highly compressed +//! body never expands in memory all at once. + +use flate2::{Decompress, FlushDecompress, Status}; + +/// The most decompressed bytes handed on at once. +const PIECE: usize = 16 * 1024; +/// The longest gzip header waited for, its optional name, comment and extra field included. +const MAX_GZIP_HEADER: usize = 128 * 1024; + +const GZIP_FHCRC: u8 = 0x02; +const GZIP_FEXTRA: u8 = 0x04; +const GZIP_FNAME: u8 = 0x08; +const GZIP_FCOMMENT: u8 = 0x10; + +/// How a body is encoded, from its `Content-Encoding`. +pub(crate) enum Decoding { + Identity, + Decoder(Box), + /// One this cannot undo. + Unsupported, +} + +impl Decoding { + /// `limit` bounds what decoding may hold on to: the input of a deflate body kept until it + /// shows whether it is zlib-framed. + pub(crate) fn new(content_encoding: Option<&str>, limit: usize) -> Self { + match content_encoding.map(|e| e.trim().to_ascii_lowercase()).as_deref() { + None | Some("") | Some("identity") => Self::Identity, + Some("gzip") | Some("x-gzip") => Self::Decoder(Box::new(ContentDecoder::gzip(limit))), + Some("deflate") => Self::Decoder(Box::new(ContentDecoder::deflate(limit))), + Some(_) => Self::Unsupported, + } + } +} + +pub(crate) struct ContentDecoder { + decompress: Decompress, + /// gzip, which comes in members, each with a header and a trailer. + gzip: bool, + /// A gzip member's header still to be read, as much of it as has come; None while a member + /// is being inflated, and for deflate, which zlib's own header covers. + gzip_header: Option>, + /// How far the gzip header's name and comment have been searched for their ends. + header_scanned: [usize; 2], + /// Bytes of a gzip member's trailer (its CRC and length, not checked) still to skip. + trailer_left: usize, + /// gzip members read whole. + members: usize, + /// The compressed stream ended; anything after it is not read. + ended: bool, + /// Deflate's `zlib` framing, which some servers leave out. + zlib: bool, + /// No output yet, so a deflate body without zlib framing can still be tried raw. + started: bool, + /// The input until the first output, while it may yet turn out not to be zlib-framed. + unframed: Vec, + /// The most input kept in `unframed`: a body that has given no output by then is not + /// decoded further. + unframed_limit: usize, +} + +impl ContentDecoder { + fn gzip(limit: usize) -> Self { + Self { + decompress: Decompress::new(false), + gzip: true, + gzip_header: Some(Vec::new()), + header_scanned: [0; 2], + trailer_left: 0, + members: 0, + ended: false, + zlib: false, + started: false, + unframed: Vec::new(), + unframed_limit: limit, + } + } + + fn deflate(limit: usize) -> Self { + Self { + decompress: Decompress::new(true), + gzip: false, + gzip_header: None, + header_scanned: [0; 2], + trailer_left: 0, + members: 0, + ended: false, + zlib: true, + started: false, + unframed: Vec::new(), + unframed_limit: limit, + } + } + + /// Decompresses `input`, handing the output on to `out` a piece at a time. + pub(crate) fn feed(&mut self, input: &[u8], out: &mut impl FnMut(&[u8])) -> Result<(), String> { + self.feed_inner(input, out)?; + // Kept only until the body gives output; empty deflate blocks can go on without end, + // and what is kept for them cannot. + if self.zlib && !self.started && self.unframed.len() > self.unframed_limit { + self.unframed = Vec::new(); + return Err("No output from the deflate body within the capture limit".into()); + } + Ok(()) + } + + fn feed_inner(&mut self, mut input: &[u8], out: &mut impl FnMut(&[u8])) -> Result<(), String> { + if self.zlib && !self.started { + self.unframed.extend_from_slice(input); + } + let mut buf = [0u8; PIECE]; + loop { + if self.ended { + return Ok(()); + } + if self.trailer_left > 0 { + let skip = self.trailer_left.min(input.len()); + input = &input[skip..]; + self.trailer_left -= skip; + if self.trailer_left > 0 { + return Ok(()); + } + } + if let Some(header) = self.gzip_header.as_mut() { + if input.is_empty() { + return Ok(()); + } + let before = header.len(); + header.extend_from_slice(input); + match gzip_header_len(header, &mut self.header_scanned) { + // A name or comment that never ends is no header to wait for. + Ok(None) if header.len() > MAX_GZIP_HEADER => { + return Err("The gzip header is too long".into()); + } + Ok(None) => return Ok(()), + Ok(Some(len)) => { + input = &input[len - before..]; + self.gzip_header = None; + self.header_scanned = [0; 2]; + } + // What follows a whole member, padding say, is not another one. + Err(_) if self.members > 0 => { + self.ended = true; + return Ok(()); + } + Err(e) => return Err(e), + } + } + + let (read_before, written_before) = + (self.decompress.total_in(), self.decompress.total_out()); + let status = match self.decompress.decompress(input, &mut buf, FlushDecompress::None) { + Ok(status) => status, + // "deflate" is meant to be zlib-framed, but some servers send it raw: read again + // from the start as raw. + Err(_) if self.zlib && !self.started => { + self.decompress = Decompress::new(false); + self.zlib = false; + let replay = std::mem::take(&mut self.unframed); + return self.feed_inner(&replay, out); + } + Err(e) => return Err(e.to_string()), + }; + let read = (self.decompress.total_in() - read_before) as usize; + let written = (self.decompress.total_out() - written_before) as usize; + input = &input[read..]; + if written > 0 { + self.started = true; + self.unframed = Vec::new(); + out(&buf[..written]); + } + if status == Status::StreamEnd { + // A gzip body may hold more members, each read in turn. + if self.gzip { + self.members += 1; + self.decompress.reset(false); + self.trailer_left = 8; + self.gzip_header = Some(Vec::new()); + continue; + } + self.ended = true; + return Ok(()); + } + // A full piece may have more output behind it, even once the input is used up. + if (input.is_empty() && written < PIECE) || (read == 0 && written == 0) { + return Ok(()); + } + } + } +} + +/// The length of the gzip header `bytes` starts with (RFC 1952 2.3), None while more of it is +/// still to come. +/// `scanned` keeps how far the name and the comment have been searched for their ends, so a +/// header that comes a byte at a time is not searched again from its start each time. +fn gzip_header_len(bytes: &[u8], scanned: &mut [usize; 2]) -> Result, String> { + if bytes.len() < 10 { + return Ok(None); + } + if bytes[..3] != [0x1f, 0x8b, 0x08] { + return Err("Not a gzip stream".into()); + } + let flags = bytes[3]; + let mut len = 10; + if flags & GZIP_FEXTRA != 0 { + let Some(extra) = bytes.get(len..len + 2) else { + return Ok(None); + }; + len += 2 + u16::from_le_bytes([extra[0], extra[1]]) as usize; + } + for (i, flag) in [GZIP_FNAME, GZIP_FCOMMENT].into_iter().enumerate() { + if flags & flag != 0 { + let from = len.max(scanned[i]); + match bytes.get(from..).and_then(|rest| rest.iter().position(|&b| b == 0)) { + Some(end) => len = from + end + 1, + None => { + scanned[i] = bytes.len().max(len); + return Ok(None); + } + } + } + } + if flags & GZIP_FHCRC != 0 { + len += 2; + } + Ok((bytes.len() >= len).then_some(len)) +} + +#[cfg(test)] +mod tests { + use super::*; + + const TEST_LIMIT: usize = 1 << 20; + use std::io::Write; + + fn decode_bytewise(encoding: &str, compressed: &[u8]) -> (Vec, usize) { + let Decoding::Decoder(mut decoder) = Decoding::new(Some(encoding), TEST_LIMIT) else { + panic!("no decoder for {encoding}"); + }; + let (mut out, mut largest) = (Vec::new(), 0); + for byte in compressed { + decoder + .feed(&[*byte], &mut |piece| { + largest = largest.max(piece.len()); + out.extend_from_slice(piece); + }) + .unwrap(); + } + (out, largest) + } + + #[test] + fn gzip_with_a_file_name_split_into_single_bytes() { + let text = b"data: hello\n\n".repeat(50); + let mut gz = flate2::GzBuilder::new() + .filename("events.txt") + .write(Vec::new(), flate2::Compression::default()); + gz.write_all(&text).unwrap(); + let (out, _) = decode_bytewise("gzip", &gz.finish().unwrap()); + assert_eq!(out, text); + } + + #[test] + fn deflate_framed_or_raw() { + let text = b"some text to squeeze".repeat(20); + let mut zlib = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + zlib.write_all(&text).unwrap(); + assert_eq!(decode_bytewise("deflate", &zlib.finish().unwrap()).0, text); + let mut raw = + flate2::write::DeflateEncoder::new(Vec::new(), flate2::Compression::default()); + raw.write_all(&text).unwrap(); + assert_eq!(decode_bytewise("deflate", &raw.finish().unwrap()).0, text); + } + + #[test] + fn output_buffered_past_a_piece_is_drained_when_the_input_runs_out() { + // Raw deflate inflates more than a piece from the last of its input. + let text: Vec = + b"data: ".iter().chain(&[b'x'; 40_000]).chain(b"\n\n").copied().collect(); + let mut raw = flate2::write::DeflateEncoder::new(Vec::new(), flate2::Compression::best()); + raw.write_all(&text).unwrap(); + let Decoding::Decoder(mut decoder) = Decoding::new(Some("deflate"), TEST_LIMIT) else { + panic!() + }; + let mut out = Vec::new(); + decoder.feed(&raw.finish().unwrap(), &mut |piece| out.extend_from_slice(piece)).unwrap(); + assert_eq!(out, text); + } + + #[test] + fn a_gzip_name_that_never_ends_is_refused() { + let Decoding::Decoder(mut decoder) = Decoding::new(Some("gzip"), TEST_LIMIT) else { + panic!() + }; + let header = [0x1f, 0x8b, 0x08, GZIP_FNAME, 0, 0, 0, 0, 0, 0]; + decoder.feed(&header, &mut |_| {}).unwrap(); + let mut refused = false; + for _ in 0..MAX_GZIP_HEADER / 512 + 1 { + if decoder.feed(&[b'a'; 512], &mut |_| {}).is_err() { + refused = true; + break; + } + } + assert!(refused); + } + + #[test] + fn deflate_without_output_is_not_kept_past_the_limit() { + // A zlib header, then empty stored blocks: valid deflate that never produces a byte. + let empty_block = [0x00, 0x00, 0x00, 0xff, 0xff]; + let Decoding::Decoder(mut decoder) = Decoding::new(Some("deflate"), 1024) else { + panic!() + }; + decoder.feed(&[0x78, 0x9c], &mut |_| {}).unwrap(); + let mut result = Ok(()); + for _ in 0..1024 { + result = decoder.feed(&empty_block, &mut |_| {}); + if result.is_err() { + break; + } + } + assert!(result.is_err(), "stopped once past the limit"); + assert!(decoder.unframed.len() <= 1024); + } + + #[test] + fn a_chunk_past_the_limit_is_decoded_when_it_gives_output() { + let text = b"data: tick\n\n".repeat(400); + let mut zlib = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::none()); + zlib.write_all(&text).unwrap(); + let body = zlib.finish().unwrap(); + assert!(body.len() > 1024); + let Decoding::Decoder(mut decoder) = Decoding::new(Some("deflate"), 1024) else { + panic!() + }; + let mut out = Vec::new(); + decoder.feed(&body, &mut |piece| out.extend_from_slice(piece)).unwrap(); + assert_eq!(out, text); + } + + #[test] + fn every_gzip_member_is_read() { + let mut body = Vec::new(); + for part in [&b"data: one\n\n"[..], b"data: two\n\n"] { + let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + gz.write_all(part).unwrap(); + body.extend(gz.finish().unwrap()); + } + assert_eq!(decode_bytewise("gzip", &body).0, b"data: one\n\ndata: two\n\n"); + let Decoding::Decoder(mut decoder) = Decoding::new(Some("gzip"), TEST_LIMIT) else { + panic!() + }; + let mut out = Vec::new(); + decoder.feed(&body, &mut |piece| out.extend_from_slice(piece)).unwrap(); + assert_eq!(out, b"data: one\n\ndata: two\n\n", "in one chunk"); + } + + #[test] + fn a_highly_compressed_body_comes_out_in_pieces() { + let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::best()); + gz.write_all(&vec![b'a'; 4 << 20]).unwrap(); + let compressed = gz.finish().unwrap(); + let Decoding::Decoder(mut decoder) = Decoding::new(Some("gzip"), TEST_LIMIT) else { + panic!() + }; + let (mut total, mut largest) = (0, 0); + decoder + .feed(&compressed, &mut |piece| { + total += piece.len(); + largest = largest.max(piece.len()); + }) + .unwrap(); + assert_eq!(total, 4 << 20); + assert!(largest <= PIECE, "{largest}"); + } +} diff --git a/crates/yaak-proxy/src/grpc.rs b/crates/yaak-proxy/src/grpc.rs new file mode 100644 index 000000000..d7b053ddb --- /dev/null +++ b/crates/yaak-proxy/src/grpc.rs @@ -0,0 +1,346 @@ +//! gRPC and gRPC-Web: the length-prefixed messages of each direction's body, and the call's +//! status from its trailers. + +use std::io::Read; + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64; +use bytes::Bytes; +use flate2::read::{GzDecoder, ZlibDecoder}; +use hyper::header::HeaderMap; + +use crate::decode::Decoding; +use crate::messages::{Capped, MessageDirection, MessageKind, StreamKind, StreamMessage}; + +/// gRPC's compressed flag, in the first byte of each message's prefix. +const FLAG_COMPRESSED: u8 = 0x01; +/// gRPC-Web's flag for the frame that carries the trailers. +const FLAG_TRAILERS: u8 = 0x80; +const PREFIX_LEN: usize = 5; + +/// The gRPC flavor a request's `Content-Type` names, and whether it is gRPC-Web's base64 one. +pub(crate) fn detect(content_type: Option<&str>) -> Option<(StreamKind, bool)> { + let content_type = content_type?.trim().to_ascii_lowercase(); + let essence = content_type.split(';').next().unwrap_or_default().trim(); + if let Some(rest) = essence.strip_prefix("application/grpc-web") { + return match rest { + "" | "+proto" => Some((StreamKind::GrpcWeb, false)), + "-text" | "-text+proto" => Some((StreamKind::GrpcWeb, true)), + _ => None, + }; + } + match essence.strip_prefix("application/grpc") { + Some("" | "+proto") => Some((StreamKind::Grpc, false)), + _ => None, + } +} + +/// `/package.Service/Method` as its service and method. +pub fn service_and_method(path: &str) -> Option<(&str, &str)> { + let path = path.split('?').next()?; + let (service, method) = path.strip_prefix('/')?.split_once('/')?; + (!service.is_empty() && !method.is_empty() && !method.contains('/')) + .then_some((service, method)) +} + +pub(crate) struct GrpcParser { + direction: MessageDirection, + /// `grpc-encoding`, what a compressed message is compressed with. + encoding: Option, + /// gRPC-Web's base64 flavor, with base64 not yet a whole quantum. + base64: Option>, + prefix: Vec, + /// The message being read: its flag, length, bytes left, and payload. + current: Option<(u8, u32, u32, Capped)>, + limit: usize, + broken: bool, + /// The body's `Content-Encoding`, which gRPC-Web may use, undone before anything else. + decoding: Decoding, +} + +impl GrpcParser { + /// Reads a body sent with `content_encoding`, as a gRPC-Web one can be. + pub(crate) fn with_content_encoding(mut self, content_encoding: Option<&str>) -> Self { + self.decoding = Decoding::new(content_encoding, self.limit); + self + } + + pub(crate) fn new( + direction: MessageDirection, + encoding: Option<&str>, + text: bool, + limit: usize, + ) -> Self { + Self { + direction, + encoding: encoding.map(|e| e.trim().to_ascii_lowercase()), + base64: text.then(Vec::new), + prefix: Vec::with_capacity(PREFIX_LEN), + current: None, + limit, + broken: false, + decoding: Decoding::Identity, + } + } + + pub(crate) fn feed(&mut self, bytes: &[u8]) -> Vec { + match std::mem::replace(&mut self.decoding, Decoding::Unsupported) { + Decoding::Identity => { + self.decoding = Decoding::Identity; + self.feed_decoded(bytes) + } + // An encoding this cannot undo leaves the frames unreadable. + Decoding::Unsupported => Vec::new(), + // A piece at a time, so a highly compressed body never expands all at once. + Decoding::Decoder(mut decoder) => { + let mut messages = Vec::new(); + let decoded = decoder.feed(bytes, &mut |piece| { + messages.extend(self.feed_decoded(piece)); + }); + if decoded.is_ok() { + self.decoding = Decoding::Decoder(decoder); + } + messages + } + } + } + + /// Reads bytes with any Content-Encoding undone: base64 first for gRPC-Web's text flavor, + /// then the frames. + fn feed_decoded(&mut self, bytes: &[u8]) -> Vec { + if self.broken { + return Vec::new(); + } + let Some(pending) = self.base64.as_mut() else { + return self.read(bytes); + }; + pending.extend(bytes.iter().filter(|b| !b.is_ascii_whitespace())); + // Each response chunk is base64 of its own, padded, so padding can come mid-stream: + // decoded up to and including each padded quantum. + let whole = pending.len() / 4 * 4; + let mut decoded = Vec::new(); + let mut start = 0; + for end in (4..=whole).step_by(4) { + if pending[end - 1] == b'=' || end == whole { + match BASE64.decode(&pending[start..end]) { + Ok(bytes) => decoded.extend(bytes), + Err(_) => { + self.broken = true; + return Vec::new(); + } + } + start = end; + } + } + pending.drain(..whole); + self.read(&decoded) + } + + fn read(&mut self, mut bytes: &[u8]) -> Vec { + let mut messages = Vec::new(); + while !bytes.is_empty() { + let Some((_, _, remaining, payload)) = self.current.as_mut() else { + let take = (PREFIX_LEN - self.prefix.len()).min(bytes.len()); + self.prefix.extend_from_slice(&bytes[..take]); + bytes = &bytes[take..]; + if self.prefix.len() == PREFIX_LEN { + let flag = self.prefix[0]; + let length = u32::from_be_bytes(self.prefix[1..5].try_into().unwrap()); + self.prefix.clear(); + self.current = Some((flag, length, length, Capped::new(self.limit))); + if length == 0 { + messages.push(self.finish()); + } + } + continue; + }; + let n = (*remaining as usize).min(bytes.len()); + payload.push(&bytes[..n]); + *remaining -= n as u32; + bytes = &bytes[n..]; + if *remaining == 0 { + messages.push(self.finish()); + } + } + messages + } + + fn finish(&mut self) -> StreamMessage { + let (flag, length, _, mut payload) = self.current.take().expect("a message being read"); + let (data, truncated) = payload.take(); + let compressed = flag & FLAG_COMPRESSED != 0; + let (data, truncated, error) = + if compressed { self.decompress(data, truncated) } else { (data, truncated, None) }; + // gRPC-Web's trailers, read once decompressed, as they can be. + let mut message = if flag & FLAG_TRAILERS != 0 { + // Trailers that would not decompress are kept as they came, with no status read. + let text = if error.is_none() { String::from_utf8_lossy(&data) } else { "".into() }; + let mut message = trailers_message(self.direction, &text); + message.data = data; + message + } else { + let mut message = StreamMessage::new(self.direction, MessageKind::GrpcMessage); + message.data = data; + message + }; + message.size = u64::from(length); + message.truncated = truncated; + message.compressed = compressed; + message.error = error; + message + } + + /// A compressed message's payload decompressed with `grpc-encoding`, whether that is cut at + /// the limit, and why it could not be decompressed, when it could not: then the payload as + /// it came. + fn decompress(&self, data: Bytes, truncated: bool) -> (Bytes, bool, Option) { + let mut decoded = Vec::new(); + let take = self.limit as u64 + 1; + let result = match self.encoding.as_deref() { + Some("gzip") => GzDecoder::new(&data[..]).take(take).read_to_end(&mut decoded), + Some("deflate") => ZlibDecoder::new(&data[..]).take(take).read_to_end(&mut decoded), + other => { + let error = + format!("Compressed with {}, not decoded", other.unwrap_or("no encoding")); + return (data, truncated, Some(error)); + } + }; + match result { + Ok(_) if !truncated => { + let truncated = decoded.len() > self.limit; + decoded.truncate(self.limit); + (decoded.into(), truncated, None) + } + // A payload cut at the limit cannot be decompressed whole. + Ok(_) => (data, truncated, Some("Too large to decompress".into())), + Err(e) => (data, truncated, Some(format!("Failed to decompress the message: {e}"))), + } + } +} + +/// The trailers message for `grpc-status` and `grpc-message` in `headers`, the HTTP/2 trailers +/// or a trailers-only response's headers. None when they carry no status. +pub(crate) fn trailers_from_headers(headers: &HeaderMap) -> Option { + headers.get("grpc-status")?; + let text: String = headers + .iter() + .map(|(name, value)| format!("{name}: {}\r\n", String::from_utf8_lossy(value.as_bytes()))) + .collect(); + let mut message = trailers_message(MessageDirection::Server, &text); + message.size = text.len() as u64; + Some(message) +} + +/// Trailers written as `name: value` lines, as gRPC-Web frames them. +fn trailers_message(direction: MessageDirection, text: &str) -> StreamMessage { + let mut status = None; + let mut status_message = None; + for line in text.lines() { + let Some((name, value)) = line.split_once(':') else { + continue; + }; + let value = value.trim(); + if name.trim().eq_ignore_ascii_case("grpc-status") { + status = value.parse().ok(); + } else if name.trim().eq_ignore_ascii_case("grpc-message") { + status_message = Some(percent_decode(value)); + } + } + let kind = MessageKind::GrpcTrailers { status, message: status_message }; + let mut message = StreamMessage::new(direction, kind); + message.data = text.to_string().into(); + message +} + +/// `grpc-message` is percent-encoded UTF-8. +fn percent_decode(value: &str) -> String { + let bytes = value.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'%' + && let Some(byte) = + value.get(i + 1..i + 3).and_then(|hex| u8::from_str_radix(hex, 16).ok()) + { + out.push(byte); + i += 3; + continue; + } + out.push(bytes[i]); + i += 1; + } + String::from_utf8_lossy(&out).into_owned() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn frame(flag: u8, payload: &[u8]) -> Vec { + let mut out = vec![flag]; + out.extend((payload.len() as u32).to_be_bytes()); + out.extend(payload); + out + } + + #[test] + fn detects_flavors() { + assert_eq!(detect(Some("application/grpc")), Some((StreamKind::Grpc, false))); + assert_eq!(detect(Some("application/grpc+proto")), Some((StreamKind::Grpc, false))); + assert_eq!(detect(Some("application/grpc-web+proto")), Some((StreamKind::GrpcWeb, false))); + assert_eq!(detect(Some("application/grpc-web-text")), Some((StreamKind::GrpcWeb, true))); + assert_eq!(detect(Some("application/grpc+json")), None); + assert_eq!(detect(Some("application/json")), None); + assert_eq!(service_and_method("/pkg.Greeter/SayHello"), Some(("pkg.Greeter", "SayHello"))); + assert_eq!(service_and_method("/"), None); + } + + #[test] + fn splits_messages_across_chunks() { + let mut bytes = frame(0, b"hello"); + bytes.extend(frame(0, b"")); + bytes.extend(frame(0, b"world!")); + let mut parser = GrpcParser::new(MessageDirection::Client, None, false, 1024); + let mut messages = Vec::new(); + for byte in bytes { + messages.extend(parser.feed(&[byte])); + } + let payloads: Vec<&[u8]> = messages.iter().map(|m| &m.data[..]).collect(); + assert_eq!(payloads, vec![&b"hello"[..], b"", b"world!"]); + } + + #[test] + fn decompresses_gzip_and_reads_web_trailers_in_base64() { + use flate2::write::GzEncoder; + use std::io::Write; + let mut gz = GzEncoder::new(Vec::new(), flate2::Compression::default()); + gz.write_all(b"squeezed").unwrap(); + let compressed = gz.finish().unwrap(); + // Two separately padded base64 chunks, as gRPC-Web text responses come. + let first = BASE64.encode(frame(FLAG_COMPRESSED, &compressed)); + let second = + BASE64.encode(frame(FLAG_TRAILERS, b"grpc-status: 5\r\ngrpc-message: not%20found\r\n")); + let mut parser = GrpcParser::new(MessageDirection::Server, Some("gzip"), true, 1024); + let mut messages = parser.feed(first.as_bytes()); + messages.extend(parser.feed(second.as_bytes())); + assert_eq!(&messages[0].data[..], b"squeezed"); + assert!(messages[0].compressed); + assert_eq!( + messages[1].kind, + MessageKind::GrpcTrailers { status: Some(5), message: Some("not found".into()) } + ); + } + + #[test] + fn compressed_web_trailers_are_read_once_decompressed() { + use flate2::write::GzEncoder; + use std::io::Write; + let mut gz = GzEncoder::new(Vec::new(), flate2::Compression::default()); + gz.write_all(b"grpc-status: 5\r\n").unwrap(); + let trailers = frame(FLAG_TRAILERS | FLAG_COMPRESSED, &gz.finish().unwrap()); + let mut parser = GrpcParser::new(MessageDirection::Server, Some("gzip"), false, 1024); + let messages = parser.feed(&trailers); + assert_eq!(messages[0].kind, MessageKind::GrpcTrailers { status: Some(5), message: None }); + assert!(messages[0].compressed && messages[0].error.is_none()); + } +} diff --git a/crates/yaak-proxy/src/lib.rs b/crates/yaak-proxy/src/lib.rs index f7401932e..81f70685d 100644 --- a/crates/yaak-proxy/src/lib.rs +++ b/crates/yaak-proxy/src/lib.rs @@ -1,8 +1,13 @@ mod body; mod cert; mod connection; +mod decode; +mod grpc; +mod messages; mod request; +mod sse; mod tunnel; +mod ws; use std::collections::hash_map::RandomState; use std::collections::{HashMap, HashSet}; @@ -24,6 +29,8 @@ use tokio::net::TcpListener; pub use cert::CaStore; use connection::handle_connection; +pub use grpc::service_and_method; +pub use messages::{MessageDirection, MessageKind, RAW_LIMIT, StreamKind, StreamMessage}; use request::BoxBody; const PASSTHROUGH_AFTER_FAILURES: u32 = 2; @@ -49,6 +56,9 @@ pub struct ProxyConfig { /// Offer HTTP/2 to clients in intercepted TLS, beside HTTP/1.1. Off, they all speak /// HTTP/1.1, for clients that misbehave on HTTP/2. See [`ProxyHandle::set_http2`]. pub http2: bool, + /// Record WebSocket frames as messages. Off, upgraded connections are spliced blind, as + /// other upgrades are. + pub websocket_capture: bool, } /// See [`ProxyConfig::on_accept`]. @@ -63,15 +73,17 @@ impl ProxyConfig { upstream_root_certs: Vec::new(), on_accept: None, http2: true, + websocket_capture: true, } } } /// Lifecycle of each proxied exchange (a flow), keyed by `id`. A flow starts with -/// `RequestStart` and ends with exactly one `ResponseBodyComplete`, `Error` or `Cancelled`. -/// Each HTTP/2 stream is a flow of its own. Traffic the engine tunnels without decrypting -/// (pinned clients, non-TLS CONNECT payloads, the frames after a `101 Switching Protocols`) -/// produces no events. +/// `RequestStart` and ends with exactly one `ResponseBodyComplete`, `StreamEnd`, `Error` or +/// `Cancelled`. Each HTTP/2 stream is a flow of its own. A streaming flow (WebSocket, +/// server-sent events, gRPC) has a `StreamStart` and then a `Message` for each message as it +/// passes. Traffic the engine tunnels without decrypting (pinned clients, non-TLS CONNECT +/// payloads, upgrades to protocols other than WebSocket) produces no events. #[derive(Debug, Clone)] pub enum ProxyEvent { /// Request headers arrived from the client. @@ -154,6 +166,18 @@ pub enum ProxyEvent { /// The client gave up before the response headers arrived, by resetting the stream or /// closing the connection. The request upstream is cancelled with it. Cancelled { id: u64, elapsed_ms: u64 }, + /// The flow carries a stream of messages: from its request headers for gRPC, its response + /// headers for server-sent events, its `101` for a WebSocket. + StreamStart { id: u64, kind: StreamKind }, + /// A message of a streaming flow passed through. + Message { id: u64, message: StreamMessage }, + /// A WebSocket ended: both sides sent a close frame, or the connection ended, with `error` + /// when it failed. Its flow ends here, as the `101` has no body to complete. + StreamEnd { + id: u64, + elapsed_ms: u64, + error: Option, + }, } /// A running engine. Dropping it stops the listener and joins its thread. @@ -225,6 +249,7 @@ pub(crate) struct Engine { handshake_failures: Mutex>, /// Offer HTTP/2 to clients in intercepted TLS. http2: AtomicBool, + websocket_capture: bool, /// `Via` value marking requests whose URL the engine rebuilt from `Host`, unique to this run, /// so one that comes back to the engine is refused instead of forwarded again. via: HeaderValue, @@ -308,6 +333,7 @@ pub fn start_proxy(config: ProxyConfig) -> Result { passthrough: Mutex::new(HashSet::new()), handshake_failures: Mutex::new(HashMap::new()), http2: AtomicBool::new(config.http2), + websocket_capture: config.websocket_capture, via: via_marker(), }); diff --git a/crates/yaak-proxy/src/messages.rs b/crates/yaak-proxy/src/messages.rs new file mode 100644 index 000000000..bd21f0471 --- /dev/null +++ b/crates/yaak-proxy/src/messages.rs @@ -0,0 +1,113 @@ +use std::time::SystemTime; + +use bytes::{Bytes, BytesMut}; + +/// The streaming protocol a flow carries, recorded as messages beside its bodies. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StreamKind { + /// An HTTP/1.1 WebSocket, after its `101 Switching Protocols`. + WebSocket, + /// A `text/event-stream` response. + Sse, + /// `application/grpc`, over HTTP/2. + Grpc, + /// `application/grpc-web`, binary or base64 (`-text`). + GrpcWeb, +} + +/// Which side sent a message. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MessageDirection { + Client, + Server, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum MessageKind { + WebSocketText, + WebSocketBinary, + WebSocketPing, + WebSocketPong, + WebSocketClose { + code: Option, + reason: String, + }, + SseEvent { + event: Option, + id: Option, + retry: Option, + /// The event's lines as they came, comments included, up to [`RAW_LIMIT`] bytes. + raw: String, + }, + GrpcMessage, + /// The status of a gRPC call: its HTTP/2 trailers, the headers of a trailers-only response, + /// or a gRPC-Web trailer frame. The payload is the trailers as `name: value` lines. + GrpcTrailers { + status: Option, + message: Option, + }, +} + +/// One message of a stream, as it passed through. +#[derive(Debug, Clone)] +pub struct StreamMessage { + pub direction: MessageDirection, + pub kind: MessageKind, + pub at: SystemTime, + /// Bytes it took on the wire: the frames' payloads, the event's lines, the length-prefixed + /// message without its prefix. + pub size: u64, + /// The payload decoded: unmasked, reassembled, inflated or decompressed. Cut at the + /// engine's `capture_body_limit`. + pub data: Bytes, + /// `data` is cut short. + pub truncated: bool, + /// It was compressed on the wire: permessage-deflate, or gRPC's compressed flag. + pub compressed: bool, + /// Why the payload could not be decoded; `data` is then what could be read of it. + pub error: Option, +} + +/// Bytes kept of an SSE event's raw lines. +pub const RAW_LIMIT: usize = 4096; + +impl StreamMessage { + pub(crate) fn new(direction: MessageDirection, kind: MessageKind) -> Self { + Self { + direction, + kind, + at: SystemTime::now(), + size: 0, + data: Bytes::new(), + truncated: false, + compressed: false, + error: None, + } + } +} + +/// A payload collected up to a limit, counting what is left out. +pub(crate) struct Capped { + pub(crate) data: BytesMut, + pub(crate) truncated: bool, + limit: usize, +} + +impl Capped { + pub(crate) fn new(limit: usize) -> Self { + Self { data: BytesMut::new(), truncated: false, limit } + } + + pub(crate) fn push(&mut self, bytes: &[u8]) { + let room = self.limit.saturating_sub(self.data.len()); + if bytes.len() > room { + self.truncated = true; + } + self.data.extend_from_slice(&bytes[..bytes.len().min(room)]); + } + + pub(crate) fn take(&mut self) -> (Bytes, bool) { + let truncated = std::mem::take(&mut self.truncated); + (self.data.split().freeze(), truncated) + } +} diff --git a/crates/yaak-proxy/src/request.rs b/crates/yaak-proxy/src/request.rs index 6b76ab3ff..bdc7d5ed0 100644 --- a/crates/yaak-proxy/src/request.rs +++ b/crates/yaak-proxy/src/request.rs @@ -25,9 +25,13 @@ use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite}; use tokio::net::TcpStream; use tokio_rustls::{LazyConfigAcceptor, TlsConnector}; -use crate::body::{CaptureBody, Direction}; +use crate::body::{BodyParser, CaptureBody, Direction}; use crate::connection::serve_tls; +use crate::grpc::{self, GrpcParser}; +use crate::messages::{MessageDirection, StreamKind}; +use crate::sse::SseParser; use crate::tunnel::{Rewind, splice, tunnel}; +use crate::ws::{self, WsFlow}; use crate::{ClientConn, Engine, ProxyEvent}; pub(crate) type BoxBody = http_body_util::combinators::BoxBody; @@ -65,6 +69,20 @@ fn error_chain(e: &(dyn Error + 'static)) -> String { message } +fn header_str<'a>(headers: &'a hyper::HeaderMap, name: &str) -> Option<&'a str> { + headers.get(name).and_then(|v| v.to_str().ok()) +} + +/// A gRPC call, from its request headers. +struct GrpcCall { + /// gRPC-Web's base64 flavor. + text: bool, + /// What the client's compressed messages are compressed with. + encoding: Option, + /// The request body's `Content-Encoding`, which gRPC-Web may use. + content_encoding: Option, +} + /// Per-flow bookkeeping shared by the plain HTTP and the intercepted HTTPS paths. struct Flow { id: u64, @@ -74,6 +92,7 @@ struct Flow { answered: bool, /// Reading the client's upload failed: it reset the stream or closed the connection. upload_failed: Arc, + grpc: Option, } impl Flow { @@ -83,12 +102,13 @@ impl Flow { req: &Request, url: String, ) -> Self { - let flow = Self { + let mut flow = Self { id: engine.next_flow_id(), start: Instant::now(), engine, answered: false, upload_failed: Arc::new(AtomicBool::new(false)), + grpc: None, }; flow.engine.emit(ProxyEvent::RequestStart { id: flow.id, @@ -118,20 +138,57 @@ impl Flow { value: String::from_utf8_lossy(value.as_bytes()).into_owned(), }); } + if let Some((kind, text)) = grpc::detect(header_str(req.headers(), "content-type")) { + flow.engine.emit(ProxyEvent::StreamStart { id: flow.id, kind }); + let encoding = header_str(req.headers(), "grpc-encoding").map(String::from); + let content_encoding = header_str(req.headers(), "content-encoding").map(String::from); + flow.grpc = Some(GrpcCall { text, encoding, content_encoding }); + } flow } fn capture_request(&self, body: Incoming) -> BoxBody { + let parser = self.grpc.as_ref().map(|call| { + let limit = self.engine.capture_body_limit; + let encoding = call.encoding.as_deref(); + let parser = GrpcParser::new(MessageDirection::Client, encoding, call.text, limit) + .with_content_encoding(call.content_encoding.as_deref()); + BodyParser::Grpc(parser) + }); CaptureBody::new(body, Direction::Request, self.id, self.start, self.engine.clone()) .flag_failure(self.upload_failed.clone()) + .with_parser(parser) .boxed() } - fn respond( - mut self, - resp: Response, - remote_addr: Option, - ) -> Response { + /// How the response's body is read for messages, if it streams them, after recording that + /// it does. + fn response_parser(&self, resp: &Response) -> Option { + let limit = self.engine.capture_body_limit; + let headers = resp.headers(); + if self.grpc.is_some() { + // A call that fails at once answers with its status in the headers, and no body. + if let Some(message) = grpc::trailers_from_headers(headers) { + self.engine.emit(ProxyEvent::Message { id: self.id, message }); + } + // A gateway's own error page is no gRPC body. + let (_, text) = grpc::detect(header_str(headers, "content-type"))?; + let encoding = header_str(headers, "grpc-encoding"); + let parser = GrpcParser::new(MessageDirection::Server, encoding, text, limit) + .with_content_encoding(header_str(headers, "content-encoding")); + return Some(BodyParser::Grpc(parser)); + } + let content_type = header_str(headers, "content-type").unwrap_or_default(); + let essence = content_type.split(';').next().unwrap_or_default().trim(); + if !essence.eq_ignore_ascii_case("text/event-stream") { + return None; + } + self.engine.emit(ProxyEvent::StreamStart { id: self.id, kind: StreamKind::Sse }); + let encoding = header_str(headers, "content-encoding"); + Some(BodyParser::Sse(Box::new(SseParser::new(encoding, limit)))) + } + + fn respond_head(&mut self, resp: &Response, remote_addr: Option) { self.answered = true; self.engine.emit(ProxyEvent::ResponseStart { id: self.id, @@ -147,12 +204,36 @@ impl Flow { value: String::from_utf8_lossy(value.as_bytes()).into_owned(), }); } + } + + fn respond( + mut self, + resp: Response, + remote_addr: Option, + ) -> Response { + self.respond_head(&resp, remote_addr); + let parser = self.response_parser(&resp); let (parts, body) = resp.into_parts(); let engine = self.engine.clone(); - let body = CaptureBody::new(body, Direction::Response, self.id, self.start, engine); + let body = CaptureBody::new(body, Direction::Response, self.id, self.start, engine) + .with_parser(parser); Response::from_parts(parts, body.boxed()) } + /// Answers with a WebSocket's `101`. The flow goes on in the returned [`WsFlow`], which + /// ends it, as the `101` has no body to. + fn respond_websocket( + mut self, + resp: Response, + remote_addr: Option, + ) -> (Response, Arc) { + self.respond_head(&resp, remote_addr); + self.engine.emit(ProxyEvent::StreamStart { id: self.id, kind: StreamKind::WebSocket }); + let ws = Arc::new(WsFlow::new(self.id, self.start, self.engine.clone())); + let (parts, body) = resp.into_parts(); + (Response::from_parts(parts, body.boxed()), ws) + } + /// Answers the client with `status` and `error` without contacting any server. fn reject(mut self, status: StatusCode, error: &str) -> Response { self.answered = true; @@ -683,6 +764,19 @@ async fn forward_https( { let upstream_upgrade = hyper::upgrade::on(&mut resp); let addr = target.addr.clone(); + let websocket = header_str(resp.headers(), UPGRADE.as_str()) + .is_some_and(|u| u.trim().eq_ignore_ascii_case("websocket")); + if websocket && engine.websocket_capture { + let deflate = ws::negotiated_deflate(resp.headers()); + let (resp, ws_flow) = flow.respond_websocket(resp, remote_addr); + tokio::spawn(async move { + if let Err(e) = ws::relay(client_upgrade, upstream_upgrade, ws_flow, deflate).await + { + debug!("WebSocket to {addr} ended: {e}"); + } + }); + return Ok(resp); + } tokio::spawn(async move { if let Err(e) = splice(client_upgrade, upstream_upgrade).await { debug!("Upgraded tunnel to {addr} ended: {e}"); diff --git a/crates/yaak-proxy/src/sse.rs b/crates/yaak-proxy/src/sse.rs new file mode 100644 index 000000000..e94fdaeef --- /dev/null +++ b/crates/yaak-proxy/src/sse.rs @@ -0,0 +1,306 @@ +//! Server-sent events (the WHATWG `text/event-stream` format), read from a response body as it +//! streams. + +use crate::decode::Decoding; +use crate::messages::{Capped, MessageDirection, MessageKind, RAW_LIMIT, StreamMessage}; + +const BOM: &[u8] = b"\xEF\xBB\xBF"; +/// Bytes a line may take beyond the capture limit, for its field name. +const LINE_SLACK: usize = 64; + +pub(crate) struct SseParser { + /// The response's Content-Encoding, undone before the events are read. + decoding: Decoding, + line: Capped, + /// A chunk ended in a CR whose line is not ended yet: an LF starting the next chunk makes it + /// a CRLF, which stays with the event it ends. + pending_cr: bool, + /// The stream ends lines with a bare CR, so a CR ending a chunk ends its line at once, and + /// an LF that starts the next chunk is skipped as the end of a CRLF. + bare_cr: bool, + /// The last chunk ended in a CR read as a bare one. + after_cr: bool, + event: Option, + id: Option, + retry: Option, + data: Capped, + /// `data` fields since the last event, which `data` joins with line feeds. + data_lines: usize, + /// A `data` line was longer than could be kept. + data_truncated: bool, + /// The start of the stream read so far while it may be a byte order mark; None after. + bom: Option>, + raw: Capped, + size: u64, +} + +impl SseParser { + pub(crate) fn new(content_encoding: Option<&str>, limit: usize) -> Self { + Self { + decoding: Decoding::new(content_encoding, limit), + // Room for a field name before a value up to the limit. + line: Capped::new(limit.saturating_add(LINE_SLACK)), + pending_cr: false, + bare_cr: false, + after_cr: false, + event: None, + id: None, + retry: None, + data: Capped::new(limit), + data_lines: 0, + data_truncated: false, + bom: Some(Vec::new()), + raw: Capped::new(RAW_LIMIT), + size: 0, + } + } + + pub(crate) fn feed(&mut self, bytes: &[u8]) -> Vec { + // Left unsupported if decoding fails: what follows cannot be read. + match std::mem::replace(&mut self.decoding, Decoding::Unsupported) { + Decoding::Identity => { + self.decoding = Decoding::Identity; + self.read(bytes) + } + Decoding::Unsupported => Vec::new(), + // A piece at a time, so a highly compressed body never expands all at once. + Decoding::Decoder(mut decoder) => { + let mut events = Vec::new(); + if decoder.feed(bytes, &mut |piece| events.extend(self.read(piece))).is_ok() { + self.decoding = Decoding::Decoder(decoder); + } + events + } + } + } + + /// The body ended: a line ending held back for the LF that might have followed it ends its + /// line now. + pub(crate) fn finish(&mut self) -> Vec { + if std::mem::take(&mut self.pending_cr) { + return self.end_line().into_iter().collect(); + } + Vec::new() + } + + fn read(&mut self, mut bytes: &[u8]) -> Vec { + // One UTF-8 byte order mark at the start of the stream is no part of it, however the + // chunks split it. + if let Some(seen) = self.bom.as_mut() { + let take = (BOM.len() - seen.len()).min(bytes.len()); + seen.extend_from_slice(&bytes[..take]); + if !BOM.starts_with(seen) { + let seen = self.bom.take().unwrap_or_default(); + let mut events = self.read(&seen); + events.extend(self.read(&bytes[take..])); + return events; + } + bytes = &bytes[take..]; + if seen.len() < BOM.len() { + return Vec::new(); + } + self.bom = None; + } + + let mut events = Vec::new(); + let mut i = 0; + while i < bytes.len() { + let byte = bytes[i]; + i += 1; + // A CR that ended the last chunk: an LF now makes it a CRLF, kept with its line. + if std::mem::take(&mut self.pending_cr) { + if byte == b'\n' { + self.size += 1; + self.raw.push(b"\n"); + } + events.extend(self.end_line()); + if byte == b'\n' { + continue; + } + } + // The LF of a CRLF split from a CR already read as a bare one. + if std::mem::take(&mut self.after_cr) && byte == b'\n' { + continue; + } + self.size += 1; + self.raw.push(&[byte]); + match byte { + b'\r' if bytes.get(i) == Some(&b'\n') => { + i += 1; + self.size += 1; + self.raw.push(b"\n"); + events.extend(self.end_line()); + } + // Whether an LF follows is for the next chunk to say, unless the stream has + // shown it ends lines with bare CRs; then the line ends now, not a chunk late. + b'\r' if i == bytes.len() => { + if self.bare_cr { + self.after_cr = true; + events.extend(self.end_line()); + } else { + self.pending_cr = true; + } + } + b'\r' => { + self.bare_cr = true; + events.extend(self.end_line()); + } + b'\n' => events.extend(self.end_line()), + _ => self.line.push(&[byte]), + } + } + events + } + + fn end_line(&mut self) -> Option { + let (line, line_truncated) = self.line.take(); + self.data_truncated |= line_truncated && line.starts_with(b"data"); + if line.is_empty() { + return self.dispatch(); + } + if line[0] == b':' { + return None; + } + let (name, value) = match line.iter().position(|&b| b == b':') { + Some(i) => { + let value = &line[i + 1..]; + (&line[..i], value.strip_prefix(b" ").unwrap_or(value)) + } + None => (&line[..], &b""[..]), + }; + let value = String::from_utf8_lossy(value); + match name { + b"event" => self.event = Some(value.into_owned()), + b"data" => { + if self.data_lines > 0 { + self.data.push(b"\n"); + } + self.data.push(value.as_bytes()); + self.data_lines += 1; + } + b"id" if !value.contains('\0') => self.id = Some(value.into_owned()), + b"retry" if !value.is_empty() && value.bytes().all(|b| b.is_ascii_digit()) => { + self.retry = value.parse().ok(); + } + _ => {} + } + None + } + + /// Ends the event at a blank line. As a browser does, only a block with a `data` line is an + /// event; one with only `id`, `retry` or `event` lines dispatches nothing. + fn dispatch(&mut self) -> Option { + let (raw, _) = self.raw.take(); + let size = std::mem::take(&mut self.size); + let (event, id, retry) = (self.event.take(), self.id.take(), self.retry.take()); + let (data, truncated) = self.data.take(); + let truncated = truncated || std::mem::take(&mut self.data_truncated); + if std::mem::take(&mut self.data_lines) == 0 { + return None; + } + let raw = String::from_utf8_lossy(&raw).into_owned(); + let kind = MessageKind::SseEvent { event, id, retry, raw }; + let mut message = StreamMessage::new(MessageDirection::Server, kind); + message.size = size; + message.data = data; + message.truncated = truncated; + Some(message) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn feed_bytewise(parser: &mut SseParser, stream: &[u8]) -> Vec { + let mut events: Vec<_> = stream.iter().flat_map(|b| parser.feed(&[*b])).collect(); + events.extend(parser.finish()); + events + } + + fn raw(message: &StreamMessage) -> &str { + match &message.kind { + MessageKind::SseEvent { raw, .. } => raw, + _ => panic!("not an event"), + } + } + + #[test] + fn a_byte_order_mark_split_across_chunks_is_skipped() { + let mut parser = SseParser::new(None, 1024); + let events = + feed_bytewise(&mut parser, b"\xEF\xBB\xBFdata: hello\n\ndata: \xEF\xBB\xBF\n\n"); + assert_eq!(&events[0].data[..], b"hello"); + assert_eq!(&events[1].data[..], b"\xEF\xBB\xBF", "only at the start of the stream"); + } + + #[test] + fn crlf_stays_with_the_event_it_ends_however_it_is_split() { + for split in [false, true] { + let stream = b"data: a\r\n\r\ndata: b\r\n\r\n"; + let mut parser = SseParser::new(None, 1024); + let events = + if split { feed_bytewise(&mut parser, stream) } else { parser.feed(stream) }; + assert_eq!(events.len(), 2, "split: {split}"); + assert_eq!(raw(&events[0]), "data: a\r\n\r\n", "split: {split}"); + assert_eq!(events[0].size, 11, "split: {split}"); + assert_eq!(raw(&events[1]), "data: b\r\n\r\n", "split: {split}"); + } + } + + #[test] + fn a_cr_ending_a_chunk_waits_for_the_next_unless_the_stream_uses_bare_crs() { + // A CR ending the chunk might start a CRLF, so its line waits for the next byte. + let mut parser = SseParser::new(None, 1024); + assert!(parser.feed(b"data: x\n\r").is_empty()); + let events = parser.feed(b"\n"); + assert_eq!(raw(&events[0]), "data: x\n\r\n"); + + // Bare CRs seen mid-chunk: one ending a chunk ends its event at once. + let mut parser = SseParser::new(None, 1024); + let events = parser.feed(b"data: a\r\rdata: b\r\r"); + assert_eq!(events.len(), 2); + assert_eq!((&events[0].data[..], &events[1].data[..]), (&b"a"[..], &b"b"[..])); + // An LF after it is the rest of a CRLF, no blank line of its own. + let events = parser.feed(b"\ndata: c\r\r"); + assert_eq!(events.len(), 1); + assert_eq!(&events[0].data[..], b"c"); + + // A held CR at the body's end ends its line then. + let mut parser = SseParser::new(None, 1024); + assert!(parser.feed(b"data: y\n\r").is_empty()); + assert_eq!(&parser.finish()[0].data[..], b"y"); + } + + #[test] + fn only_blocks_with_data_are_events() { + let mut parser = SseParser::new(None, 1024); + let events = parser.feed(b"retry: 1500\n\nid: 7\nevent: x\n\ndata:\n\n: comment\n\n"); + assert_eq!(events.len(), 1, "only the empty data event"); + assert_eq!(&events[0].data[..], b""); + assert_eq!(raw(&events[0]), "data:\n\n"); + } + + #[test] + fn a_data_line_past_the_limit_is_truncated() { + let mut parser = SseParser::new(None, 16); + let events = parser.feed(format!("data: {}\n\n", "x".repeat(100)).as_bytes()); + assert_eq!(events[0].data.len(), 16); + assert!(events[0].truncated); + } + + #[test] + fn gzip_events_are_read_as_they_decompress() { + use std::io::Write; + let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + gz.write_all(b"data: one\n\n").unwrap(); + gz.flush().unwrap(); + let first = gz.get_ref().len(); + gz.write_all(b"data: two\n\n").unwrap(); + let compressed = gz.finish().unwrap(); + let mut parser = SseParser::new(Some("gzip"), 1024); + let events = parser.feed(&compressed[..first]); + assert_eq!(&events[0].data[..], b"one", "before the rest of the body"); + assert_eq!(&parser.feed(&compressed[first..])[0].data[..], b"two"); + } +} diff --git a/crates/yaak-proxy/src/ws.rs b/crates/yaak-proxy/src/ws.rs new file mode 100644 index 000000000..dc5753908 --- /dev/null +++ b/crates/yaak-proxy/src/ws.rs @@ -0,0 +1,419 @@ +//! WebSocket capture: after the `101`, the upgraded connections are relayed byte for byte, and +//! a frame parser on each direction records every frame as it passes. The parser reads a copy +//! of what was already written on, so it never holds bytes back or changes them. + +use std::io; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; +use std::time::Instant; + +use flate2::{Decompress, FlushDecompress, Status}; +use hyper::header::{HeaderMap, SEC_WEBSOCKET_EXTENSIONS}; +use hyper::upgrade::OnUpgrade; +use hyper_util::rt::TokioIo; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; + +use crate::messages::{Capped, MessageDirection, MessageKind, StreamMessage}; +use crate::{Engine, ProxyEvent}; + +const OP_CONTINUATION: u8 = 0x0; +const OP_TEXT: u8 = 0x1; +const OP_BINARY: u8 = 0x2; +const OP_CLOSE: u8 = 0x8; +const OP_PING: u8 = 0x9; +const OP_PONG: u8 = 0xA; + +/// The most a control frame carries (RFC 6455 5.5). +const MAX_CONTROL_PAYLOAD: usize = 125; + +/// The tail permessage-deflate strips from each compressed message (RFC 7692 7.2.2). +const DEFLATE_TAIL: [u8; 4] = [0x00, 0x00, 0xff, 0xff]; + +/// permessage-deflate as the server accepted it. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct Deflate { + client_no_context_takeover: bool, + server_no_context_takeover: bool, +} + +/// The permessage-deflate parameters in the server's `Sec-WebSocket-Extensions`, if it accepted +/// the extension. +pub(crate) fn negotiated_deflate(headers: &HeaderMap) -> Option { + let extensions = + headers.get_all(SEC_WEBSOCKET_EXTENSIONS).iter().filter_map(|v| v.to_str().ok()); + for extension in extensions.flat_map(|v| v.split(',')) { + let mut params = extension.split(';').map(str::trim); + if !params.next().is_some_and(|name| name.eq_ignore_ascii_case("permessage-deflate")) { + continue; + } + let mut deflate = Deflate::default(); + for param in params { + let name = param.split('=').next().unwrap_or_default().trim(); + if name.eq_ignore_ascii_case("client_no_context_takeover") { + deflate.client_no_context_takeover = true; + } else if name.eq_ignore_ascii_case("server_no_context_takeover") { + deflate.server_no_context_takeover = true; + } + } + return Some(deflate); + } + None +} + +/// One WebSocket flow, shared by its two directions. It ends once both sides have sent a close +/// frame, or when the connection does, whichever is first. +pub(crate) struct WsFlow { + id: u64, + start: Instant, + engine: Arc, + /// A bit per direction that has sent a close frame. + closes: AtomicU8, + ended: AtomicBool, +} + +impl WsFlow { + pub(crate) fn new(id: u64, start: Instant, engine: Arc) -> Self { + Self { id, start, engine, closes: AtomicU8::new(0), ended: AtomicBool::new(false) } + } + + fn emit(&self, message: StreamMessage) { + if !self.ended.load(Ordering::Relaxed) { + self.engine.emit(ProxyEvent::Message { id: self.id, message }); + } + } + + fn closed(&self, direction: MessageDirection) { + let bit = match direction { + MessageDirection::Client => 1, + MessageDirection::Server => 2, + }; + if self.closes.fetch_or(bit, Ordering::Relaxed) | bit == 3 { + self.end(None); + } + } + + pub(crate) fn end(&self, error: Option) { + if !self.ended.swap(true, Ordering::Relaxed) { + self.engine.emit(ProxyEvent::StreamEnd { + id: self.id, + elapsed_ms: self.start.elapsed().as_millis() as u64, + error, + }); + } + } +} + +/// Relays an upgraded WebSocket connection, recording its frames on `flow`, and ends the flow +/// when the connection ends. +pub(crate) async fn relay( + client: OnUpgrade, + upstream: OnUpgrade, + flow: Arc, + deflate: Option, +) -> io::Result<()> { + let (client, upstream) = match tokio::try_join!(client, upstream) { + Ok(joined) => joined, + Err(e) => { + flow.end(Some(e.to_string())); + return Err(io::Error::other(e)); + } + }; + let (client_read, client_write) = tokio::io::split(TokioIo::new(client)); + let (upstream_read, upstream_write) = tokio::io::split(TokioIo::new(upstream)); + let limit = flow.engine.capture_body_limit; + let outbound = FrameParser::new( + MessageDirection::Client, + flow.clone(), + limit, + deflate.map(|d| d.client_no_context_takeover), + ); + let inbound = FrameParser::new( + MessageDirection::Server, + flow.clone(), + limit, + deflate.map(|d| d.server_no_context_takeover), + ); + let result = tokio::try_join!( + pump(client_read, upstream_write, outbound), + pump(upstream_read, client_write, inbound), + ); + flow.end(result.as_ref().err().map(ToString::to_string)); + result.map(|_| ()) +} + +/// Copies `from` to `to`, then shows the parser what went through. At the end of `from`, closes +/// `to` for writing, and the other direction carries on until it ends too. +async fn pump(mut from: R, mut to: W, mut parser: FrameParser) -> io::Result<()> +where + R: AsyncRead + Unpin, + W: AsyncWrite + Unpin, +{ + let mut buf = vec![0u8; 16 * 1024]; + loop { + let n = from.read(&mut buf).await?; + if n == 0 { + let _ = to.shutdown().await; + return Ok(()); + } + to.write_all(&buf[..n]).await?; + to.flush().await?; + parser.feed(&buf[..n]); + } +} + +struct Frame { + fin: bool, + opcode: u8, + mask: Option<[u8; 4]>, + remaining: u64, + offset: usize, +} + +/// A data message being reassembled from its frames. +struct Assembly { + opcode: u8, + compressed: bool, + size: u64, + payload: Capped, + error: Option, +} + +/// One direction's inflate context, kept across messages unless the side that sends them +/// negotiated no context takeover. +struct Inflater { + decompress: Decompress, + no_context_takeover: bool, + /// Inflating failed, so the context is lost for the messages after. + failed: bool, +} + +impl Inflater { + fn inflate(&mut self, mut input: &[u8], out: &mut Capped) -> Result<(), String> { + let mut buf = [0u8; 8192]; + loop { + let (read_before, written_before) = + (self.decompress.total_in(), self.decompress.total_out()); + let status = self + .decompress + .decompress(input, &mut buf, FlushDecompress::Sync) + .map_err(|e| e.to_string())?; + let read = (self.decompress.total_in() - read_before) as usize; + let written = (self.decompress.total_out() - written_before) as usize; + out.push(&buf[..written]); + input = &input[read..]; + if status == Status::StreamEnd { + // A final block ends the deflate stream; the next message starts a new one. + self.decompress.reset(false); + } + if (input.is_empty() && written < buf.len()) || (read == 0 && written == 0) { + return Ok(()); + } + } + } +} + +/// Reads WebSocket frames (RFC 6455 5.2) from one direction's bytes, records each message and +/// control frame. Stops recording at a frame it cannot read, as the stream is then unknown. +struct FrameParser { + direction: MessageDirection, + flow: Arc, + limit: usize, + header: Vec, + frame: Option, + message: Option, + control: Vec, + inflater: Option, + broken: bool, +} + +/// The length of the frame header that starts with `header`, once its first two bytes are in. +fn header_len(header: &[u8]) -> Option { + let second = *header.get(1)?; + let length = match second & 0x7f { + 126 => 2, + 127 => 8, + _ => 0, + }; + let mask = if second & 0x80 != 0 { 4 } else { 0 }; + Some(2 + length + mask) +} + +impl FrameParser { + fn new( + direction: MessageDirection, + flow: Arc, + limit: usize, + no_context_takeover: Option, + ) -> Self { + Self { + direction, + flow, + limit, + header: Vec::with_capacity(14), + frame: None, + message: None, + control: Vec::new(), + inflater: no_context_takeover.map(|no_context_takeover| Inflater { + decompress: Decompress::new(false), + no_context_takeover, + failed: false, + }), + broken: false, + } + } + + fn feed(&mut self, mut data: &[u8]) { + while !data.is_empty() && !self.broken { + let Some(frame) = self.frame.as_mut() else { + let need = header_len(&self.header).unwrap_or(2); + let take = (need - self.header.len()).min(data.len()); + self.header.extend_from_slice(&data[..take]); + data = &data[take..]; + if header_len(&self.header) == Some(self.header.len()) { + self.begin_frame(); + } + continue; + }; + let n = frame.remaining.min(data.len() as u64) as usize; + let mut chunk = data[..n].to_vec(); + if let Some(mask) = frame.mask { + for (i, byte) in chunk.iter_mut().enumerate() { + *byte ^= mask[(frame.offset + i) % 4]; + } + } + frame.offset += n; + frame.remaining -= n as u64; + let (fin, opcode, done) = (frame.fin, frame.opcode, frame.remaining == 0); + data = &data[n..]; + self.payload(opcode, &chunk); + if done { + self.frame = None; + self.end_frame(fin, opcode); + } + } + } + + fn begin_frame(&mut self) { + let header = std::mem::take(&mut self.header); + let (first, second) = (header[0], header[1]); + let fin = first & 0x80 != 0; + let rsv1 = first & 0x40 != 0; + let opcode = first & 0x0f; + let (length, rest) = match second & 0x7f { + 126 => (u16::from_be_bytes([header[2], header[3]]) as u64, &header[4..]), + 127 => (u64::from_be_bytes(header[2..10].try_into().unwrap()), &header[10..]), + n => (n as u64, &header[2..]), + }; + let mask = (second & 0x80 != 0).then(|| [rest[0], rest[1], rest[2], rest[3]]); + self.header = header; + self.header.clear(); + + match opcode { + OP_CLOSE | OP_PING | OP_PONG => self.control.clear(), + OP_TEXT | OP_BINARY => { + self.message = Some(Assembly { + opcode, + compressed: rsv1 && self.inflater.is_some(), + size: 0, + payload: Capped::new(self.limit), + error: None, + }); + } + OP_CONTINUATION => {} + // An opcode no extension here defines: what follows cannot be read. + _ => { + self.broken = true; + return; + } + } + self.frame = Some(Frame { fin, opcode, mask, remaining: length, offset: 0 }); + if length == 0 { + self.frame = None; + self.end_frame(fin, opcode); + } + } + + fn payload(&mut self, opcode: u8, chunk: &[u8]) { + if opcode >= OP_CLOSE { + // At most what a control frame may carry, however long a broken one claims to be. + let room = MAX_CONTROL_PAYLOAD.saturating_sub(self.control.len()); + self.control.extend_from_slice(&chunk[..chunk.len().min(room)]); + return; + } + let Some(message) = self.message.as_mut() else { + return; + }; + message.size += chunk.len() as u64; + match self.inflater.as_mut().filter(|_| message.compressed) { + Some(inflater) if !inflater.failed => { + if let Err(e) = inflater.inflate(chunk, &mut message.payload) { + inflater.failed = true; + message.error = Some(format!("Failed to inflate the message: {e}")); + } + } + Some(_) => { + message.payload.push(chunk); + message.error.get_or_insert_with(|| { + "Not inflated: an earlier message failed to, which lost the context".into() + }); + } + None => message.payload.push(chunk), + } + } + + fn end_frame(&mut self, fin: bool, opcode: u8) { + if opcode >= OP_CLOSE { + let payload = std::mem::take(&mut self.control); + let kind = match opcode { + OP_PING => MessageKind::WebSocketPing, + OP_PONG => MessageKind::WebSocketPong, + _ => { + let code = + (payload.len() >= 2).then(|| u16::from_be_bytes([payload[0], payload[1]])); + let reason = String::from_utf8_lossy(payload.get(2..).unwrap_or_default()); + MessageKind::WebSocketClose { code, reason: reason.into_owned() } + } + }; + let mut message = StreamMessage::new(self.direction, kind); + message.size = payload.len() as u64; + message.data = payload.into(); + self.flow.emit(message); + if opcode == OP_CLOSE { + self.flow.closed(self.direction); + } + return; + } + if !fin { + return; + } + let Some(mut assembly) = self.message.take() else { + return; + }; + if assembly.compressed + && let Some(inflater) = self.inflater.as_mut() + { + if !inflater.failed + && let Err(e) = inflater.inflate(&DEFLATE_TAIL, &mut assembly.payload) + { + inflater.failed = true; + assembly.error = Some(format!("Failed to inflate the message: {e}")); + } + if inflater.no_context_takeover && !inflater.failed { + inflater.decompress.reset(false); + } + } + let kind = if assembly.opcode == OP_TEXT { + MessageKind::WebSocketText + } else { + MessageKind::WebSocketBinary + }; + let mut message = StreamMessage::new(self.direction, kind); + let (data, truncated) = assembly.payload.take(); + message.size = assembly.size; + message.data = data; + message.truncated = truncated; + message.compressed = assembly.compressed; + message.error = assembly.error; + self.flow.emit(message); + } +} diff --git a/crates/yaak-proxy/tests/proxy.rs b/crates/yaak-proxy/tests/proxy.rs index 9e911d16f..322bf35dc 100644 --- a/crates/yaak-proxy/tests/proxy.rs +++ b/crates/yaak-proxy/tests/proxy.rs @@ -6,6 +6,8 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::mpsc::Receiver; use std::time::{Duration, SystemTime}; +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64; use bytes::Bytes; use futures_util::{SinkExt, StreamExt}; use http_body_util::combinators::BoxBody; @@ -28,10 +30,26 @@ use tokio_tungstenite::WebSocketStream; use tokio_tungstenite::tungstenite::Message; use tokio_tungstenite::tungstenite::handshake::derive_accept_key; use tokio_tungstenite::tungstenite::protocol::Role; -use yaak_proxy::{CaStore, ProxyConfig, ProxyEvent, ProxyHandle, start_proxy}; +use tokio_tungstenite::tungstenite::protocol::frame::CloseFrame; +use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode; +use yaak_proxy::{ + CaStore, MessageDirection, MessageKind, ProxyConfig, ProxyEvent, ProxyHandle, StreamKind, + StreamMessage, start_proxy, +}; const BIG_BODY: usize = 64 * 1024; +/// The longest a streaming test may run: one that hangs fails with a message instead of +/// holding CI until its job is cancelled. +const STREAM_TEST_TIMEOUT: Duration = Duration::from_secs(60); + +async fn within(test: impl Future) -> T { + match tokio::time::timeout(STREAM_TEST_TIMEOUT, test).await { + Ok(result) => result, + Err(_) => panic!("the test did not finish within {STREAM_TEST_TIMEOUT:?}"), + } +} + macro_rules! find { ($events:expr, $pat:pat => $out:expr) => { $events @@ -166,7 +184,7 @@ fn websocket_echo(mut req: Request) -> Response { let io = TokioIo::new(on_upgrade.await.unwrap()); let mut ws = WebSocketStream::from_raw_socket(io, Role::Server, None).await; while let Some(Ok(msg)) = ws.next().await { - if msg.is_text() { + if msg.is_text() || msg.is_binary() { ws.send(msg).await.unwrap(); } } @@ -231,6 +249,10 @@ struct Harness { impl Harness { fn start(capture_body_limit: usize) -> Self { + Self::start_with(capture_body_limit, |_| {}) + } + + fn start_with(capture_body_limit: usize, configure: impl FnOnce(&mut ProxyConfig)) -> Self { let dir = TempDir::new().unwrap(); let ca_dir = dir.path().join("ca"); let created = CaStore::load_or_create(&ca_dir).unwrap(); @@ -242,6 +264,7 @@ impl Harness { let mut config = ProxyConfig::new(0, ca.clone()); config.capture_body_limit = capture_body_limit; config.upstream_root_certs = vec![pki.ca_der.clone()]; + configure(&mut config); let mut proxy = start_proxy(config).unwrap(); let events = proxy.take_event_rx().unwrap(); Self { proxy, events, ca, pki, _dir: dir } @@ -381,6 +404,7 @@ fn is_terminal(event: &ProxyEvent) -> bool { ProxyEvent::ResponseBodyComplete { .. } | ProxyEvent::Error { .. } | ProxyEvent::Cancelled { .. } + | ProxyEvent::StreamEnd { .. } ) } @@ -395,7 +419,10 @@ fn flow_id(event: &ProxyEvent) -> u64 { | ProxyEvent::ResponseBodyChunk { id, .. } | ProxyEvent::ResponseBodyComplete { id, .. } | ProxyEvent::Error { id, .. } - | ProxyEvent::Cancelled { id, .. } => *id, + | ProxyEvent::Cancelled { id, .. } + | ProxyEvent::StreamStart { id, .. } + | ProxyEvent::Message { id, .. } + | ProxyEvent::StreamEnd { id, .. } => *id, } } @@ -677,33 +704,534 @@ async fn bodies_over_the_cap_are_truncated_but_delivered() { #[tokio::test(flavor = "multi_thread")] async fn websocket_upgrade_passes_through() { - let h = Harness::start(1024); - let upstream = spawn_upstream(Some(h.pki.server.clone())).await; - let port = upstream.port(); + within(async { + let h = Harness::start(1024); + let upstream = spawn_upstream(Some(h.pki.server.clone())).await; + let port = upstream.port(); - let tcp = h.connect_tunnel(port).await; + let tcp = h.connect_tunnel(port).await; + let tls = TlsConnector::from(h.tls_client_config()) + .connect(ServerName::try_from("localhost").unwrap(), tcp) + .await + .unwrap(); + + let url = format!("wss://localhost:{port}/ws"); + let (mut ws, _) = tokio_tungstenite::client_async(url.as_str(), tls).await.unwrap(); + ws.send(Message::text("hello")).await.unwrap(); + assert_eq!(ws.next().await.unwrap().unwrap().into_text().unwrap().as_str(), "hello"); + ws.send(Message::binary(vec![0u8, 1, 2, 255])).await.unwrap(); + assert_eq!(ws.next().await.unwrap().unwrap().into_data().as_ref(), &[0u8, 1, 2, 255]); + ws.send(Message::Ping(Bytes::from_static(b"p"))).await.unwrap(); + assert!( + matches!(ws.next().await.unwrap().unwrap(), Message::Pong(p) if p.as_ref() == b"p") + ); + ws.close(Some(CloseFrame { code: CloseCode::Normal, reason: "bye".into() })).await.unwrap(); + while let Some(Ok(_)) = ws.next().await {} + + let events = h.flow_events(); + let req_url = find!(events, ProxyEvent::RequestStart { url, .. } => url.clone()); + assert_eq!(req_url, format!("https://localhost:{port}/ws")); + let (status, version) = find!(events, + ProxyEvent::ResponseStart { status, http_version, .. } => (*status, http_version.clone())); + assert_eq!((status, version.as_str()), (101, "HTTP/1.1")); + let kind = find!(events, ProxyEvent::StreamStart { kind, .. } => *kind); + assert_eq!(kind, StreamKind::WebSocket); + // The flow ends with the close handshake, not with the 101. + assert!(!events.iter().any(|e| matches!(e, ProxyEvent::ResponseBodyComplete { .. }))); + let error = find!(events, ProxyEvent::StreamEnd { error, .. } => error.clone()); + assert_eq!(error, None); + + for direction in [MessageDirection::Client, MessageDirection::Server] { + let sent = messages(&events, direction); + let expected_control = match direction { + MessageDirection::Client => MessageKind::WebSocketPing, + MessageDirection::Server => MessageKind::WebSocketPong, + }; + let close = MessageKind::WebSocketClose { code: Some(1000), reason: "bye".into() }; + let kinds: Vec<_> = sent.iter().map(|m| m.kind.clone()).collect(); + assert_eq!( + kinds, + vec![ + MessageKind::WebSocketText, + MessageKind::WebSocketBinary, + expected_control, + close + ], + "{direction:?}" + ); + assert_eq!(&sent[0].data[..], b"hello"); + assert_eq!(&sent[1].data[..], &[0u8, 1, 2, 255]); + assert_eq!(&sent[2].data[..], b"p"); + } + }) + .await +} + +/// The messages a flow recorded from one side, in order. +fn messages(events: &[ProxyEvent], direction: MessageDirection) -> Vec { + events + .iter() + .filter_map(|e| match e { + ProxyEvent::Message { message, .. } if message.direction == direction => { + Some(message.clone()) + } + _ => None, + }) + .collect() +} + +/// A WebSocket frame as written on the wire, masked when `mask` is given. +fn ws_frame(fin: bool, rsv1: bool, opcode: u8, payload: &[u8], mask: Option<[u8; 4]>) -> Vec { + let mut out = vec![(u8::from(fin) << 7) | (u8::from(rsv1) << 6) | opcode]; + let mask_bit = if mask.is_some() { 0x80 } else { 0 }; + match payload.len() { + n if n < 126 => out.push(mask_bit | n as u8), + n if n <= u16::MAX as usize => { + out.push(mask_bit | 126); + out.extend((n as u16).to_be_bytes()); + } + n => { + out.push(mask_bit | 127); + out.extend((n as u64).to_be_bytes()); + } + } + match mask { + Some(mask) => { + out.extend(mask); + out.extend(payload.iter().enumerate().map(|(i, b)| b ^ mask[i % 4])); + } + None => out.extend(payload), + } + out +} + +/// permessage-deflate's compression of one message, in a context kept across messages. +fn deflate_message(compress: &mut flate2::Compress, data: &[u8]) -> Vec { + let mut out = Vec::with_capacity(data.len() + 64); + compress.compress_vec(data, &mut out, flate2::FlushCompress::Sync).unwrap(); + assert!(out.ends_with(&[0, 0, 0xff, 0xff])); + out.truncate(out.len() - 4); + out +} + +/// A server that switches to WebSocket with permessage-deflate and then sends back every byte +/// it gets, as it got it, keeping a copy. +fn raw_websocket_echo( + mut req: Request, + received: Arc>>, +) -> Response { + let accept = derive_accept_key(req.headers()["sec-websocket-key"].as_bytes()); + let on_upgrade = hyper::upgrade::on(&mut req); + tokio::spawn(async move { + let mut io = TokioIo::new(on_upgrade.await.unwrap()); + let mut buf = vec![0u8; 4096]; + loop { + let n = io.read(&mut buf).await.unwrap_or(0); + if n == 0 { + break; + } + received.lock().unwrap().extend_from_slice(&buf[..n]); + io.write_all(&buf[..n]).await.unwrap(); + } + }); + Response::builder() + .status(StatusCode::SWITCHING_PROTOCOLS) + .header(UPGRADE, "websocket") + .header(CONNECTION, "Upgrade") + .header("sec-websocket-accept", accept) + .header("sec-websocket-extensions", "permessage-deflate") + .body(full(Bytes::new())) + .unwrap() +} + +#[tokio::test(flavor = "multi_thread")] +async fn websocket_frames_pass_byte_for_byte() { + within(async { + let h = Harness::start(1024 * 1024); + let received = Arc::new(std::sync::Mutex::new(Vec::new())); + let handler = { + let received = received.clone(); + move |req| { + let received = received.clone(); + async move { Ok::<_, Infallible>(raw_websocket_echo(req, received)) } + } + }; + let upstream = + spawn_upstream_with(Some(h.pki.server.clone()), Arc::new(AtomicUsize::new(0)), handler) + .await; + let mut tls = h.tls_tunnel(upstream.port(), &[b"http/1.1"]).await; + let handshake = format!( + "GET /raw HTTP/1.1\r\nHost: localhost:{}\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\ + Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\nSec-WebSocket-Version: 13\r\n\ + Sec-WebSocket-Extensions: permessage-deflate\r\n\r\n", + upstream.port() + ); + tls.write_all(handshake.as_bytes()).await.unwrap(); + let mut head = Vec::new(); + while !head.ends_with(b"\r\n\r\n") { + head.push(tls.read_u8().await.unwrap()); + } + assert!(head.starts_with(b"HTTP/1.1 101"), "{}", String::from_utf8_lossy(&head)); + + let mask = Some([0x12, 0x34, 0x56, 0x78]); + let mut compress = flate2::Compress::new(flate2::Compression::default(), false); + let big: Vec = (0..200_000u32).map(|i| (i % 7) as u8).collect(); + let mut wire = Vec::new(); + // A text message in two fragments, with a ping between them. + wire.extend(ws_frame(false, false, 0x1, b"Hel", mask)); + wire.extend(ws_frame(true, false, 0x9, b"x", mask)); + wire.extend(ws_frame(true, false, 0x0, b"lo", mask)); + // Two compressed messages sharing one deflate context, the second over 64 KB inflated. + let compressed = deflate_message(&mut compress, b"compressed text compressed text"); + wire.extend(ws_frame(true, true, 0x1, &compressed, mask)); + let compressed = deflate_message(&mut compress, &big); + wire.extend(ws_frame(true, true, 0x2, &compressed, mask)); + let mut close = 4000u16.to_be_bytes().to_vec(); + close.extend(b"done"); + wire.extend(ws_frame(true, false, 0x8, &close, mask)); + + // Odd-sized writes, so frames and headers split across reads. + for piece in wire.chunks(7) { + tls.write_all(piece).await.unwrap(); + } + // TLS may hold the last records back until flushed. + tls.flush().await.unwrap(); + let mut echoed = vec![0u8; wire.len()]; + tls.read_exact(&mut echoed).await.unwrap(); + assert_eq!(echoed, wire, "the client gets back exactly what the server sent"); + assert_eq!(*received.lock().unwrap(), wire, "the server gets exactly what the client sent"); + drop(tls); + + let events = h.flow_events(); + // The server sent the client's masked frames back, so both sides recorded the same. + for direction in [MessageDirection::Client, MessageDirection::Server] { + let sent = messages(&events, direction); + let kinds: Vec<_> = sent.iter().map(|m| m.kind.clone()).collect(); + assert_eq!( + kinds, + vec![ + MessageKind::WebSocketPing, + MessageKind::WebSocketText, + MessageKind::WebSocketText, + MessageKind::WebSocketBinary, + MessageKind::WebSocketClose { code: Some(4000), reason: "done".into() }, + ], + "{direction:?}" + ); + assert_eq!(&sent[1].data[..], b"Hello"); + assert_eq!(sent[1].size, 5); + assert_eq!(&sent[2].data[..], b"compressed text compressed text"); + assert!(sent[2].compressed && sent[2].error.is_none(), "{:?}", sent[2].error); + assert_eq!(&sent[3].data[..], &big[..], "inflated with the first message's context"); + assert!(sent[3].size < big.len() as u64, "size on the wire, compressed"); + } + let error = find!(events, ProxyEvent::StreamEnd { error, .. } => error.clone()); + assert_eq!(error, None, "ended by the close frames"); + }) + .await +} + +/// Round trips over a WebSocket through the proxy, in microseconds, sorted. +async fn websocket_round_trips(websocket_capture: bool, n: usize) -> Vec { + let h = Harness::start_with(1024 * 1024, |c| c.websocket_capture = websocket_capture); + let upstream = spawn_upstream(Some(h.pki.server.clone())).await; + let tcp = h.connect_tunnel(upstream.port()).await; let tls = TlsConnector::from(h.tls_client_config()) .connect(ServerName::try_from("localhost").unwrap(), tcp) .await .unwrap(); - - let url = format!("wss://localhost:{port}/ws"); + let url = format!("wss://localhost:{}/ws", upstream.port()); let (mut ws, _) = tokio_tungstenite::client_async(url.as_str(), tls).await.unwrap(); - for text in ["ping", "pong"] { - ws.send(Message::text(text)).await.unwrap(); - let reply = ws.next().await.unwrap().unwrap(); - assert_eq!(reply.into_text().unwrap().as_str(), text); + let payload = "x".repeat(256); + let mut times = Vec::with_capacity(n); + for i in 0..n + 100 { + let start = std::time::Instant::now(); + ws.send(Message::text(payload.clone())).await.unwrap(); + ws.next().await.unwrap().unwrap(); + // The first hundred warm up. + if i >= 100 { + times.push(start.elapsed().as_micros()); + } } ws.close(None).await.unwrap(); + times.sort_unstable(); + times +} - let events = h.flow_events(); - let req_url = find!(events, ProxyEvent::RequestStart { url, .. } => url.clone()); - assert_eq!(req_url, format!("https://localhost:{port}/ws")); - let (status, version) = find!(events, - ProxyEvent::ResponseStart { status, http_version, .. } => (*status, http_version.clone())); - assert_eq!((status, version.as_str()), (101, "HTTP/1.1")); - let (size, error) = find!(events, ProxyEvent::ResponseBodyComplete { size, error, .. } => (*size, error.clone())); - assert_eq!((size, error), (0, None)); +#[tokio::test(flavor = "multi_thread")] +async fn websocket_capture_adds_no_measurable_latency() { + within(async { + const N: usize = 2000; + let blind = websocket_round_trips(false, N).await; + let captured = websocket_round_trips(true, N).await; + let pct = |t: &[u128], p: usize| t[(t.len() * p / 100).min(t.len() - 1)]; + eprintln!( + "WebSocket round trip through the proxy, {N} x 256 B text, microseconds: \ + blind splice p50 {} p90 {} p99 {}; captured p50 {} p90 {} p99 {}", + pct(&blind, 50), + pct(&blind, 90), + pct(&blind, 99), + pct(&captured, 50), + pct(&captured, 90), + pct(&captured, 99), + ); + // Generous, so a busy machine does not fail it: within a millisecond of the blind splice at + // the median. + assert!(pct(&captured, 50) < pct(&blind, 50) + 1000); + }) + .await +} + +/// An event stream whose chunks the test sends one at a time. +fn event_stream(chunks: tokio::sync::mpsc::UnboundedReceiver<&'static str>) -> Response { + let frames = futures_util::stream::unfold(chunks, |mut chunks| async move { + let chunk = chunks.recv().await?; + Some((Ok(Frame::data(Bytes::from(chunk))), chunks)) + }); + Response::builder() + .header(CONTENT_TYPE, "text/event-stream") + .body(BodyExt::boxed(StreamBody::new(frames))) + .unwrap() +} + +/// The next event of a flow, skipping body chunks and headers. +fn next_stream_message(h: &Harness) -> StreamMessage { + loop { + match h.events.recv_timeout(Duration::from_secs(10)).expect("no message") { + ProxyEvent::Message { message, .. } => return message, + ProxyEvent::ResponseBodyComplete { .. } => panic!("the stream ended first"), + _ => {} + } + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn server_sent_events_are_recorded_as_they_arrive() { + within(async { + let h = Harness::start(1024); + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + let rx = Arc::new(std::sync::Mutex::new(Some(rx))); + let handler = move |_req| { + let rx = rx.lock().unwrap().take().unwrap(); + async move { Ok::<_, Infallible>(event_stream(rx)) } + }; + let upstream = + spawn_upstream_with(Some(h.pki.server.clone()), Arc::new(AtomicUsize::new(0)), handler) + .await; + let url = format!("https://localhost:{}/events", upstream.port()); + let mut resp = h.client().get(&url).send().await.unwrap(); + + tx.send("event: greet\nid: 1\ndata: hello\n").unwrap(); + tx.send("data: world\n\n").unwrap(); + // Recorded while the stream is still open. + let first = next_stream_message(&h); + assert_eq!(&first.data[..], b"hello\nworld"); + assert_eq!( + first.kind, + MessageKind::SseEvent { + event: Some("greet".into()), + id: Some("1".into()), + retry: None, + raw: "event: greet\nid: 1\ndata: hello\ndata: world\n\n".into(), + } + ); + tokio::time::sleep(Duration::from_millis(500)).await; + tx.send(": a comment\r\nretry: 1500\r\ndata: two\r\n\r\n").unwrap(); + let second = next_stream_message(&h); + assert_eq!(&second.data[..], b"two"); + assert!(matches!(second.kind, MessageKind::SseEvent { retry: Some(1500), .. })); + drop(tx); + + let mut body = Vec::new(); + while let Some(chunk) = resp.chunk().await.unwrap() { + body.extend_from_slice(&chunk); + } + assert!(body.ends_with(b"data: two\r\n\r\n"), "the client gets the stream unchanged"); + let events = h.flow_events(); + let error = find!(events, ProxyEvent::ResponseBodyComplete { error, .. } => error.clone()); + assert_eq!(error, None); + }) + .await +} + +/// A gRPC message on the wire: its five-byte prefix, then `payload`. +fn grpc_frame(flag: u8, payload: &[u8]) -> Vec { + let mut out = vec![flag]; + out.extend((payload.len() as u32).to_be_bytes()); + out.extend(payload); + out +} + +/// A protobuf message with one string field, number 1, as `HelloRequest { name }` encodes. +fn proto_string(value: &str) -> Vec { + let mut out = vec![0x0a, value.len() as u8]; + out.extend(value.as_bytes()); + out +} + +/// A gRPC server for `pkg.Greeter`: `SayHello` answers once, `StreamHellos` three times, and +/// `Fail` at once with a NOT_FOUND status in its headers. +async fn greeter(req: Request) -> Result, Infallible> { + let path = req.uri().path().to_string(); + let web = req.headers()[CONTENT_TYPE].to_str().unwrap().starts_with("application/grpc-web"); + // Asked for by a header of the test's own: clients built with gzip support ask for it on + // every request. + let gzip = req.headers().contains_key("x-test-gzip"); + let request = req.into_body().collect().await.unwrap().to_bytes(); + let request = if web { BASE64.decode(&request).unwrap() } else { request.to_vec() }; + let name = String::from_utf8_lossy(&request[7..]).into_owned(); + let replies: Vec = match path.as_str() { + "/pkg.Greeter/SayHello" => vec![format!("Hello {name}")], + "/pkg.Greeter/StreamHellos" => (1..=3).map(|i| format!("Hello {name} #{i}")).collect(), + _ => { + return Ok(Response::builder() + .header(CONTENT_TYPE, "application/grpc") + .header("grpc-status", "5") + .header("grpc-message", "no%20such%20greeter") + .body(full(Bytes::new())) + .unwrap()); + } + }; + if web { + // gRPC-Web's text flavor: each frame base64 of its own, the trailers in the last. + let mut body = String::new(); + for reply in &replies { + body.push_str(&BASE64.encode(grpc_frame(0, &proto_string(reply)))); + } + body.push_str(&BASE64.encode(grpc_frame(0x80, b"grpc-status: 0\r\ngrpc-message: \r\n"))); + let resp = Response::builder().header(CONTENT_TYPE, "application/grpc-web-text"); + // Compressed for HTTP as a whole when the client accepts it, as a CDN does. + if gzip { + let mut encoder = + flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + std::io::Write::write_all(&mut encoder, body.as_bytes()).unwrap(); + let body = encoder.finish().unwrap(); + return Ok(resp.header("content-encoding", "gzip").body(full(body)).unwrap()); + } + return Ok(resp.body(full(body)).unwrap()); + } + let frames = futures_util::stream::iter(replies) + .then(|reply| async move { + tokio::time::sleep(Duration::from_millis(20)).await; + Ok(Frame::data(Bytes::from(grpc_frame(0, &proto_string(&reply))))) + }) + .chain(futures_util::stream::once(async { + let mut trailers = hyper::HeaderMap::new(); + trailers.insert("grpc-status", HeaderValue::from_static("0")); + Ok(Frame::trailers(trailers)) + })); + Ok(Response::builder() + .header(CONTENT_TYPE, "application/grpc") + .body(BodyExt::boxed(StreamBody::new(frames))) + .unwrap()) +} + +async fn grpc_call( + sender: &mut http2::SendRequest, + port: u16, + method: &str, + name: &str, +) -> (Vec, Option, hyper::HeaderMap) { + let req = Request::builder() + .method("POST") + .uri(format!("https://localhost:{port}/pkg.Greeter/{method}")) + .header(CONTENT_TYPE, "application/grpc") + .header("te", "trailers") + .body(full(grpc_frame(0, &proto_string(name)))) + .unwrap(); + let resp = sender.send_request(req).await.unwrap(); + let headers = resp.headers().clone(); + let collected = resp.into_body().collect().await.unwrap(); + let trailers = collected.trailers().cloned(); + (collected.to_bytes().to_vec(), trailers, headers) +} + +#[tokio::test(flavor = "multi_thread")] +async fn grpc_messages_and_status_are_recorded() { + within(async { + let h = Harness::start(1024); + let upstream = + spawn_upstream_with(Some(h.pki.server.clone()), Arc::new(AtomicUsize::new(0)), greeter) + .await; + let port = upstream.port(); + let mut sender = h.h2_client(port).await; + + let (body, trailers, _) = grpc_call(&mut sender, port, "SayHello", "Ada").await; + assert_eq!(body, grpc_frame(0, &proto_string("Hello Ada"))); + assert_eq!(trailers.unwrap()["grpc-status"], "0", "trailers pass through"); + let events = h.flow_events(); + assert_eq!(find!(events, ProxyEvent::StreamStart { kind, .. } => *kind), StreamKind::Grpc); + let sent = messages(&events, MessageDirection::Client); + assert_eq!(sent.len(), 1); + assert_eq!((&sent[0].data[..], sent[0].size), (&proto_string("Ada")[..], 5)); + let received = messages(&events, MessageDirection::Server); + assert_eq!(received.len(), 2); + assert_eq!(&received[0].data[..], &proto_string("Hello Ada")[..]); + assert_eq!(received[1].kind, MessageKind::GrpcTrailers { status: Some(0), message: None }); + + let (_, _, _) = grpc_call(&mut sender, port, "StreamHellos", "Bo").await; + let events = h.flow_events(); + let received = messages(&events, MessageDirection::Server); + let replies: Vec<_> = received.iter().map(|m| m.data.clone()).collect(); + assert_eq!(replies.len(), 4); + for (i, reply) in replies[..3].iter().enumerate() { + assert_eq!(&reply[..], &proto_string(&format!("Hello Bo #{}", i + 1))[..]); + } + + let (body, _, headers) = grpc_call(&mut sender, port, "Fail", "Cy").await; + assert!(body.is_empty()); + assert_eq!(headers["grpc-status"], "5"); + let events = h.flow_events(); + let received = messages(&events, MessageDirection::Server); + assert_eq!( + received[0].kind, + MessageKind::GrpcTrailers { status: Some(5), message: Some("no such greeter".into()) } + ); + }) + .await +} + +#[tokio::test(flavor = "multi_thread")] +async fn grpc_web_text_is_recorded() { + within(async { + let h = Harness::start(1024); + let upstream = + spawn_upstream_with(Some(h.pki.server.clone()), Arc::new(AtomicUsize::new(0)), greeter) + .await; + let url = format!("https://localhost:{}/pkg.Greeter/StreamHellos", upstream.port()); + // Plain, then with the whole response gzipped for HTTP. + for gzip in [false, true] { + let mut req = h + .client() + .post(&url) + .header(CONTENT_TYPE, "application/grpc-web-text") + .body(BASE64.encode(grpc_frame(0, &proto_string("Di")))); + if gzip { + req = req.header("x-test-gzip", "1"); + } + let resp = req.send().await.unwrap(); + assert_eq!(resp.status(), 200); + resp.bytes().await.unwrap(); + + let events = h.flow_events(); + // As the server sent it: a client with gzip support may undo it before the test sees it. + let encoding = + headers(&events, false).into_iter().find(|(n, _)| n == "content-encoding"); + assert_eq!(encoding.is_some(), gzip, "the response passed compressed"); + let kind = find!(events, ProxyEvent::StreamStart { kind, .. } => *kind); + assert_eq!(kind, StreamKind::GrpcWeb); + let sent = messages(&events, MessageDirection::Client); + assert_eq!(&sent[0].data[..], &proto_string("Di")[..]); + let received = messages(&events, MessageDirection::Server); + assert_eq!(received.len(), 4, "gzip: {gzip}"); + assert_eq!(&received[2].data[..], &proto_string("Hello Di #3")[..]); + assert_eq!( + received[3].kind, + MessageKind::GrpcTrailers { status: Some(0), message: Some(String::new()) } + ); + } + }) + .await } #[tokio::test(flavor = "multi_thread")] diff --git a/package.json b/package.json index 3e2022546..9a83a9673 100644 --- a/package.json +++ b/package.json @@ -111,7 +111,8 @@ "client:tauri-before-build": "npm run bootstrap", "client:tauri-before-dev": "node scripts/run-workspaces-dev.mjs apps/yaak-client", "proxy:tauri-before-build": "npm run bootstrap", - "proxy:tauri-before-bundle": "run-s bootstrap proxy:vendor-helper", + "proxy:tauri-before-bundle": "run-s bootstrap proxy:vendor-helper proxy:sign-vendored", + "proxy:sign-vendored": "node scripts/sign-proxy-vendored.cjs", "proxy:vendor-helper": "node scripts/vendor-proxy-helper.cjs", "proxy:tauri-before-dev": "node scripts/run-workspaces-dev.mjs apps/yaak-proxy" }, diff --git a/scripts/sign-proxy-vendored.cjs b/scripts/sign-proxy-vendored.cjs new file mode 100644 index 000000000..1735a1941 --- /dev/null +++ b/scripts/sign-proxy-vendored.cjs @@ -0,0 +1,36 @@ +#!/usr/bin/env node + +/** + * Signs the executables Yaak Proxy's bundle takes from Yaak's vendored tools with hardened + * runtime, as notarization requires of every executable in the app. Runs after vendoring, in + * the bundle's before-build step, with the identity `scripts/bundle-proxy.mjs` chose. Without + * one, or off macOS, the bundle is unsigned anyway and this does nothing. + */ + +const { execFileSync } = require("child_process"); +const { existsSync } = require("fs"); +const path = require("path"); + +const rootDir = path.join(__dirname, ".."); +const identity = process.env.APPLE_SIGNING_IDENTITY; + +if (process.platform !== "darwin" || !identity) { + console.log("No signing identity, so the vendored protoc is left unsigned"); + process.exit(0); +} + +const clientDir = path.join(rootDir, "crates-tauri", "yaak-app-client"); +const protoc = path.join(clientDir, "vendored", "protoc", "yaakprotoc"); +const entitlements = path.join(clientDir, "macos", "entitlements.yaakprotoc.plist"); + +if (!existsSync(protoc)) { + console.error(`${protoc} is missing: run \`npm run vendor\` first`); + process.exit(1); +} + +execFileSync( + "codesign", + ["--force", "--options", "runtime", "--entitlements", entitlements, "--sign", identity, protoc], + { stdio: "inherit" }, +); +console.log(`Signed ${path.relative(rootDir, protoc)}`); From 603a4ce364714bb6a0ab71128ab917c93fca8a62 Mon Sep 17 00:00:00 2001 From: Gregory Schier Date: Fri, 6 Nov 2026 10:37:42 -0700 Subject: [PATCH 02/45] Share Yaak's settings dialog (#817) Co-authored-by: Claude Opus 5.5 --- .../components/Settings/Settings.tsx | 170 +++---- .../components/Settings/SettingsGeneral.tsx | 170 +------ .../components/Settings/SettingsInterface.tsx | 195 +------ .../components/Settings/SettingsLicense.tsx | 180 +------ .../components/Settings/SettingsTheme.tsx | 168 +------ apps/yaak-client/components/core/Link.tsx | 60 ++- apps/yaak-client/components/core/Select.tsx | 150 +----- .../components/core/SettingRow.tsx | 475 +----------------- apps/yaak-client/hooks/useSettingsView.ts | 76 +++ apps/yaak-client/lib/settingsView.test.ts | 126 +++++ apps/yaak-client/lib/settingsView.ts | 84 ++++ apps/yaak-proxy/harness-settings.html | 212 ++++++++ package-lock.json | 1 + packages/ui/package.json | 1 + packages/ui/src/components/ExternalLink.tsx | 54 ++ packages/ui/src/components/Select.tsx | 149 ++++++ .../ui/src/components/settings/SettingRow.tsx | 457 +++++++++++++++++ .../components/settings/SettingsDialog.tsx | 110 ++++ .../components/settings/SettingsGeneral.tsx | 166 ++++++ .../components/settings/SettingsInterface.tsx | 201 ++++++++ .../components/settings/SettingsLicense.tsx | 204 ++++++++ .../src/components/settings/SettingsTheme.tsx | 173 +++++++ .../src/components/settings/settings.test.tsx | 296 +++++++++++ packages/ui/src/components/settings/types.ts | 124 +++++ packages/ui/src/index.ts | 51 +- packages/ui/src/lib/yaakUrl.test.ts | 26 + packages/ui/src/lib/yaakUrl.ts | 15 + 27 files changed, 2680 insertions(+), 1414 deletions(-) create mode 100644 apps/yaak-client/hooks/useSettingsView.ts create mode 100644 apps/yaak-client/lib/settingsView.test.ts create mode 100644 apps/yaak-client/lib/settingsView.ts create mode 100644 apps/yaak-proxy/harness-settings.html create mode 100644 packages/ui/src/components/ExternalLink.tsx create mode 100644 packages/ui/src/components/Select.tsx create mode 100644 packages/ui/src/components/settings/SettingRow.tsx create mode 100644 packages/ui/src/components/settings/SettingsDialog.tsx create mode 100644 packages/ui/src/components/settings/SettingsGeneral.tsx create mode 100644 packages/ui/src/components/settings/SettingsInterface.tsx create mode 100644 packages/ui/src/components/settings/SettingsLicense.tsx create mode 100644 packages/ui/src/components/settings/SettingsTheme.tsx create mode 100644 packages/ui/src/components/settings/settings.test.tsx create mode 100644 packages/ui/src/components/settings/types.ts create mode 100644 packages/ui/src/lib/yaakUrl.test.ts create mode 100644 packages/ui/src/lib/yaakUrl.ts diff --git a/apps/yaak-client/components/Settings/Settings.tsx b/apps/yaak-client/components/Settings/Settings.tsx index 1039ce74f..6ab3bed9d 100644 --- a/apps/yaak-client/components/Settings/Settings.tsx +++ b/apps/yaak-client/components/Settings/Settings.tsx @@ -1,13 +1,19 @@ import { platform } from "@yaakapp-internal/platform"; import { useLicense } from "@yaakapp-internal/license"; import { pluginsAtom, settingsAtom } from "@yaakapp-internal/models"; -import { HeaderSize, HStack, Icon, CountBadge } from "@yaakapp-internal/ui"; -import classNames from "classnames"; +import { + CountBadge, + SETTINGS_TAB_GENERAL, + SETTINGS_TAB_INTERFACE, + SETTINGS_TAB_LICENSE, + SETTINGS_TAB_THEME, + SettingsDialog, + type SettingsTabDef, +} from "@yaakapp-internal/ui"; import { useAtomValue } from "jotai"; import { useKeyPressEvent } from "react-use"; import { appInfo } from "../../lib/appInfo"; import { capitalize } from "../../lib/capitalize"; -import { TabContent, type TabItem, Tabs } from "../core/Tabs/Tabs"; import { SettingsAgents } from "./SettingsAgents"; import { SettingsCertificates } from "./SettingsCertificates"; import { SettingsGeneral } from "./SettingsGeneral"; @@ -24,15 +30,15 @@ interface Props { hide?: () => void; } -const TAB_GENERAL = "general"; -const TAB_INTERFACE = "interface"; -const TAB_THEME = "theme"; +const TAB_GENERAL = SETTINGS_TAB_GENERAL; +const TAB_INTERFACE = SETTINGS_TAB_INTERFACE; +const TAB_THEME = SETTINGS_TAB_THEME; const TAB_SHORTCUTS = "shortcuts"; const TAB_PROXY = "proxy"; const TAB_CERTIFICATES = "certificates"; const TAB_PLUGINS = "plugins"; const TAB_AGENTS = "agents"; -const TAB_LICENSE = "license"; +const TAB_LICENSE = SETTINGS_TAB_LICENSE; const tabs = [ TAB_GENERAL, TAB_THEME, @@ -60,105 +66,57 @@ export default function Settings({ tab, hide }: Props) { if (hide == null) await platform.window.close(); }); + const tabDefs: Record> = { + [TAB_GENERAL]: { icon: "settings", content: }, + [TAB_THEME]: { icon: "palette", content: }, + [TAB_INTERFACE]: { icon: "columns_2", content: }, + [TAB_SHORTCUTS]: { icon: "keyboard", content: }, + [TAB_PLUGINS]: { + icon: "puzzle", + rightSlot: p.source !== "bundled").length} />, + content: , + contentClassName: "h-full grid grid-rows-1", + }, + [TAB_AGENTS]: { + icon: "sparkles", + hidden: !platform.capabilities.agents, + content: , + }, + [TAB_CERTIFICATES]: { + icon: "shield_check", + rightSlot: , + content: , + }, + [TAB_PROXY]: { + icon: "wifi", + rightSlot: settings.proxy?.type === "enabled" ? : null, + content: , + }, + [TAB_LICENSE]: { + icon: "key_round", + hidden: !appInfo.featureLicense, + rightSlot: + licenseCheck.check.data?.status === "personal_use" ? ( + + ) : null, + content: , + }, + }; + return ( -
- {hide ? ( - - ) : ( - - -
Settings
-
-
- )} - ({ - value, - label: capitalize(value), - hidden: - (!appInfo.featureLicense && value === TAB_LICENSE) || - (!platform.capabilities.agents && value === TAB_AGENTS), - leftSlot: - value === TAB_GENERAL ? ( - - ) : value === TAB_THEME ? ( - - ) : value === TAB_INTERFACE ? ( - - ) : value === TAB_SHORTCUTS ? ( - - ) : value === TAB_CERTIFICATES ? ( - - ) : value === TAB_PROXY ? ( - - ) : value === TAB_PLUGINS ? ( - - ) : value === TAB_AGENTS ? ( - - ) : value === TAB_LICENSE ? ( - - ) : null, - rightSlot: - value === TAB_CERTIFICATES ? ( - - ) : value === TAB_PLUGINS ? ( - p.source !== "bundled").length} /> - ) : value === TAB_PROXY && settings.proxy?.type === "enabled" ? ( - - ) : value === TAB_LICENSE && licenseCheck.check.data?.status === "personal_use" ? ( - - ) : null, - }), - )} - > - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ ({ value, label: capitalize(value), ...tabDefs[value] }))} + /> ); } diff --git a/apps/yaak-client/components/Settings/SettingsGeneral.tsx b/apps/yaak-client/components/Settings/SettingsGeneral.tsx index d36758f9b..9b022b644 100644 --- a/apps/yaak-client/components/Settings/SettingsGeneral.tsx +++ b/apps/yaak-client/components/Settings/SettingsGeneral.tsx @@ -1,32 +1,17 @@ -import { patchModel, settingsAtom } from "@yaakapp-internal/models"; -import { Heading, VStack } from "@yaakapp-internal/ui"; +import { settingsAtom } from "@yaakapp-internal/models"; +import { SettingsGeneral as SharedSettingsGeneral } from "@yaakapp-internal/ui"; import { useAtomValue } from "jotai"; -import { useCheckForUpdates } from "../../hooks/useCheckForUpdates"; -import { appInfo } from "../../lib/appInfo"; -import { revealInFinderText } from "../../lib/reveal"; -import { CargoFeature } from "../CargoFeature"; +import { useSettingsView } from "../../hooks/useSettingsView"; import { CommercialUseBanner } from "../CommercialUseBanner"; import { DismissibleBanner } from "../core/DismissibleBanner"; -import { IconButton } from "../core/IconButton"; -import { platform } from "@yaakapp-internal/platform"; -import { - ModelSettingRowBoolean, - ModelSettingSelectControl, - SettingValue, - SettingRow, - SettingRowBoolean, - SettingRowSelect, - SettingsList, - SettingsSection, -} from "../core/SettingRow"; const WORKSPACE_SETTINGS_MOVED_AT = "2026-06-30"; export function SettingsGeneral() { const settings = useAtomValue(settingsAtom); - const checkForUpdates = useCheckForUpdates(); + const bound = useSettingsView(); - if (settings == null) { + if (settings == null || bound == null) { return null; } @@ -34,147 +19,24 @@ export function SettingsGeneral() { settings.createdAt.slice(0, 10) < WORKSPACE_SETTINGS_MOVED_AT; return ( - -
- General -

- Configure general settings for update behavior and more. -

-
-
- -
- - - - -
- - checkForUpdates.mutateAsync()} - /> -
-
- - - patchModel(settings, { autoupdate: v === "auto" }) - } - options={[ - { label: "Automatic", value: "auto" }, - { label: "Manual", value: "manual" }, - ]} - /> - - - - - - {}} - /> -
-
- - - - patchModel(settings, { promptFeedback })} - /> - - - - {showWorkspaceSettingsMovedBanner && ( + } + extraSections={ + showWorkspaceSettingsMovedBanner && (

- Workspace specific settings have moved to{" "} - Workspace Settings, accessible from the workspace switcher - menu. + Workspace specific settings have moved to Workspace Settings, accessible from + the workspace switcher menu.

- )} - - - - - - - platform.revealItemInDir(appInfo.appDataDir), - }, - ]} - /> - - - platform.revealItemInDir(appInfo.appLogDir), - }, - ]} - /> - - -
-
+ ) + } + /> ); } diff --git a/apps/yaak-client/components/Settings/SettingsInterface.tsx b/apps/yaak-client/components/Settings/SettingsInterface.tsx index baffa2738..6952ab511 100644 --- a/apps/yaak-client/components/Settings/SettingsInterface.tsx +++ b/apps/yaak-client/components/Settings/SettingsInterface.tsx @@ -1,59 +1,30 @@ -import { platform } from "@yaakapp-internal/platform"; -import { useFonts } from "@yaakapp-internal/fonts"; import { useLicense } from "@yaakapp-internal/license"; -import type { EditorKeymap, Settings } from "@yaakapp-internal/models"; +import type { Settings } from "@yaakapp-internal/models"; import { patchModel, settingsAtom } from "@yaakapp-internal/models"; -import { clamp, Heading, VStack } from "@yaakapp-internal/ui"; +import { SettingsInterface as SharedSettingsInterface, VStack } from "@yaakapp-internal/ui"; import { useAtomValue } from "jotai"; -import { useState } from "react"; import { activeWorkspaceAtom } from "../../hooks/useActiveWorkspace"; +import { useSettingsView } from "../../hooks/useSettingsView"; import { showConfirm } from "../../lib/confirm"; import { pricingUrl } from "../../lib/pricingUrl"; -import { rpc } from "../../lib/rpc"; import { CargoFeature } from "../CargoFeature"; -import { Button } from "../core/Button"; -import { Checkbox } from "../core/Checkbox"; import { Link } from "../core/Link"; -import { - ModelSettingRowBoolean, - ModelSettingRowSelect, - SettingRow, - SettingRowBoolean, - SettingRowSelect, - SettingSelectControl, - SettingsList, - SettingsSection, -} from "../core/SettingRow"; - -const NULL_FONT_VALUE = "__NULL_FONT__"; - -const fontSizeOptions = [ - 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, -].map((n) => ({ label: `${n}`, value: `${n}` })); - -const keymaps: { value: EditorKeymap; label: string }[] = [ - { value: "default", label: "Default" }, - { value: "vim", label: "Vim" }, - { value: "vscode", label: "VSCode" }, - { value: "emacs", label: "Emacs" }, -]; +import { SettingRowBoolean, SettingRowSelect, SettingsSection } from "../core/SettingRow"; export function SettingsInterface() { const workspace = useAtomValue(activeWorkspaceAtom); const settings = useAtomValue(settingsAtom); - const fonts = useFonts(); + const bound = useSettingsView(); - if (settings == null || workspace == null) { + if (settings == null || workspace == null || bound == null) { return null; } return ( - -
- Interface -

Tweak settings related to the user interface.

-
- + - - - - {fonts.data && ( - ({ label: f, value: f })), - ...fonts.data.editorFonts.map((f) => ({ label: f, value: f })), - ]} - onChange={async (v) => { - const interfaceFont = v === NULL_FONT_VALUE ? null : v; - await patchModel(settings, { interfaceFont }); - }} - /> - )} - patchModel(settings, { interfaceFontSize: Number.parseInt(v, 10) })} - /> - - - - {fonts.data && ( - ({ label: f, value: f })), - ]} - onChange={async (v) => { - const editorFont = v === NULL_FONT_VALUE ? null : v; - await patchModel(settings, { editorFont }); - }} - /> - )} - - patchModel(settings, { - editorFontSize: clamp(Number.parseInt(v, 10) || 14, 8, 30), - }) - } - /> - - - - - - - - - - - - {platform.osType() !== "macos" && ( - - )} - - + } + trailingSections={ - -
- ); -} - -function NativeTitlebarSetting({ settings }: { settings: Settings }) { - const [nativeTitlebar, setNativeTitlebar] = useState(settings.useNativeTitlebar); - - return ( - - - {settings.useNativeTitlebar !== nativeTitlebar && ( - - )} - + } + /> ); } diff --git a/apps/yaak-client/components/Settings/SettingsLicense.tsx b/apps/yaak-client/components/Settings/SettingsLicense.tsx index 9e72d481c..3853bd21c 100644 --- a/apps/yaak-client/components/Settings/SettingsLicense.tsx +++ b/apps/yaak-client/components/Settings/SettingsLicense.tsx @@ -1,17 +1,6 @@ -import { useLicense } from "@yaakapp-internal/license"; -import { Banner, HStack, Icon, VStack } from "@yaakapp-internal/ui"; -import { differenceInDays } from "date-fns"; -import { formatDate } from "date-fns/format"; -import { useState } from "react"; -import { useToggle } from "../../hooks/useToggle"; -import { pricingUrl } from "../../lib/pricingUrl"; +import { SettingsLicense as SharedSettingsLicense } from "@yaakapp-internal/ui"; +import { useSettingsView } from "../../hooks/useSettingsView"; import { CargoFeature } from "../CargoFeature"; -import { Button } from "../core/Button"; -import { Link } from "../core/Link"; -import { PlainInput } from "../core/PlainInput"; -import { Separator } from "../core/Separator"; -import { platform } from "@yaakapp-internal/platform"; -import { pluralizeCount } from "@yaakapp-internal/lib/pluralize"; export function SettingsLicense() { return ( @@ -22,166 +11,7 @@ export function SettingsLicense() { } function SettingsLicenseCmp() { - const { check, activate, deactivate } = useLicense(); - const [key, setKey] = useState(""); - const [activateFormVisible, toggleActivateFormVisible] = useToggle(false); - - if (check.isPending) { - return null; - } - - const renderBanner = () => { - if (!check.data) return null; - - switch (check.data.status) { - case "active": - return Your license is active 🥳; - - case "trialing": - return ( - -

- - {pluralizeCount("day", differenceInDays(check.data.data.end, new Date()))} - {" "} - left to evaluate Yaak for commercial use. -
- Personal use is always free, forever. - -

- - Learn More - -
-

-
- ); - - case "personal_use": - return ( - -

- Your commercial-use trial has ended. -
- - You may continue using Yaak for personal use only. -
A license is required for commercial use. -
- -

- - Learn More - -
-

-
- ); - - case "inactive": - return ( - - Your license is invalid. Please Sign In{" "} - for more details - - ); - - case "expired": - return ( - - Your license expired{" "} - {formatDate(check.data.data.periodEnd, "MMMM dd, yyyy")}. Please{" "} - Resubscribe to continue receiving - updates. - {check.data.data.changesUrl && ( - <> -
- What's new in latest builds - - )} -
- ); - - case "past_due": - return ( - - Your payment method needs attention. -
- To re-activate your license, please{" "} - update your billing info. -
- ); - - case "error": - return ( - - License check failed: {check.data.data.message} (Code: {check.data.data.code}) - - ); - } - }; - - return ( -
- {renderBanner()} - - {check.error && {check.error}} - {activate.error && {activate.error}} - - {check.data?.status === "active" ? ( - - - - - ) : ( - - - - - )} - - {activateFormVisible && ( - { - e.preventDefault(); - await activate.mutateAsync({ licenseKey: key }); - toggleActivateFormVisible(); - }} - > - - - - )} -
- ); + const bound = useSettingsView(); + if (bound == null) return null; + return ; } diff --git a/apps/yaak-client/components/Settings/SettingsTheme.tsx b/apps/yaak-client/components/Settings/SettingsTheme.tsx index 268109045..d71e3113e 100644 --- a/apps/yaak-client/components/Settings/SettingsTheme.tsx +++ b/apps/yaak-client/components/Settings/SettingsTheme.tsx @@ -1,173 +1,15 @@ -import { patchModel, settingsAtom } from "@yaakapp-internal/models"; -import { Heading, HStack, Icon, type IconProps, VStack } from "@yaakapp-internal/ui"; +import { SettingsTheme as SharedSettingsTheme } from "@yaakapp-internal/ui"; import { useAtomValue } from "jotai"; -import { lazy, Suspense } from "react"; import { activeWorkspaceAtom } from "../../hooks/useActiveWorkspace"; -import { useResolvedAppearance } from "../../hooks/useResolvedAppearance"; -import { useResolvedTheme } from "../../hooks/useResolvedTheme"; -import type { ButtonProps } from "../core/Button"; -import { IconButton } from "../core/IconButton"; -import { Link } from "../core/Link"; -import type { SelectProps } from "../core/Select"; -import { - ModelSettingRowSelect, - SettingRowSelect, - SettingsList, - SettingsSection, -} from "../core/SettingRow"; - -const Editor = lazy(() => import("../core/Editor/Editor").then((m) => ({ default: m.Editor }))); - -const buttonColors: ButtonProps["color"][] = [ - "primary", - "info", - "success", - "notice", - "warning", - "danger", - "secondary", - "default", -]; - -const icons: IconProps["icon"][] = [ - "info", - "box", - "update", - "alert_triangle", - "arrow_big_right_dash", - "download", - "copy", - "magic_wand", - "settings", - "trash", - "sparkles", - "pencil", - "paste", - "search", - "send_horizontal", -]; +import { useSettingsView } from "../../hooks/useSettingsView"; export function SettingsTheme() { const workspace = useAtomValue(activeWorkspaceAtom); - const settings = useAtomValue(settingsAtom); - const appearance = useResolvedAppearance(); - const activeTheme = useResolvedTheme(); + const bound = useSettingsView(); - if (settings == null || workspace == null || activeTheme.data == null) { + if (workspace == null || bound == null) { return null; } - const lightThemes: SelectProps["options"] = activeTheme.data.themes - .filter((theme) => !theme.dark) - .map((theme) => ({ - label: theme.label, - value: theme.id, - })); - - const darkThemes: SelectProps["options"] = activeTheme.data.themes - .filter((theme) => theme.dark) - .map((theme) => ({ - label: theme.label, - value: theme.id, - })); - - return ( - -
- Theme -

- Make Yaak your own by selecting a theme, or{" "} - - Create Your Own - -

-
- - - - {(settings.appearance === "system" || settings.appearance === "light") && ( - patchModel(settings, { themeLight })} - /> - )} - {(settings.appearance === "system" || settings.appearance === "dark") && ( - patchModel(settings, { themeDark })} - /> - )} - - - - - - - {activeTheme.data.active.label} - (preview) - - - {buttonColors.map((c, i) => ( - - ))} - {buttonColors.map((c, i) => ( - - ))} - - - - - - - -
- ); + return ; } diff --git a/apps/yaak-client/components/core/Link.tsx b/apps/yaak-client/components/core/Link.tsx index 297e205dd..46a42a1e4 100644 --- a/apps/yaak-client/components/core/Link.tsx +++ b/apps/yaak-client/components/core/Link.tsx @@ -3,16 +3,43 @@ import { platform } from "@yaakapp-internal/platform"; import classNames from "classnames"; import type { HTMLAttributes } from "react"; import { appInfo } from "../../lib/appInfo"; -import { Icon } from "@yaakapp-internal/ui"; +import { ExternalLink, isYaakUrl } from "@yaakapp-internal/ui"; interface Props extends HTMLAttributes { href: string; noUnderline?: boolean; } +/** Tags a link to yaak.app with which app it came from */ +export function withYaakRef(href: string): string { + if (!isYaakUrl(href)) return href; + const url = new URL(href); + url.searchParams.set("ref", appInfo.identifier); + return url.toString(); +} + +/** Opens a link outside the app, tagged as `Link` tags it */ +export function openExternalUrl(href: string) { + void platform.openUrl(withYaakRef(href)); +} + export function Link({ href, children, noUnderline, className, ...other }: Props) { const isExternal = href.match(/^https?:\/\//); + if (isExternal) { + return ( + + {children} + + ); + } + className = classNames( className, "relative", @@ -20,37 +47,6 @@ export function Link({ href, children, noUnderline, className, ...other }: Props !noUnderline && "underline", ); - if (isExternal) { - const isYaakLink = href.startsWith("https://yaak.app"); - let finalHref = href; - if (isYaakLink) { - const url = new URL(href); - url.searchParams.set("ref", appInfo.identifier); - finalHref = url.toString(); - } - return ( - // eslint-disable-next-line react/jsx-no-target-blank - { - e.preventDefault(); - void platform.openUrl(finalHref); - }} - className={className} - {...other} - > - {children} - - - ); - } - return ( {children} diff --git a/apps/yaak-client/components/core/Select.tsx b/apps/yaak-client/components/core/Select.tsx index 6abf68c53..d5a13bdfb 100644 --- a/apps/yaak-client/components/core/Select.tsx +++ b/apps/yaak-client/components/core/Select.tsx @@ -1,148 +1,2 @@ -import { platform } from "@yaakapp-internal/platform"; -import { HStack, Label } from "@yaakapp-internal/ui"; -import classNames from "classnames"; -import type { CSSProperties, ReactNode } from "react"; -import { useState } from "react"; -import type { ButtonProps } from "./Button"; -import { Button } from "./Button"; -import type { RadioDropdownItem } from "./RadioDropdown"; -import { RadioDropdown } from "./RadioDropdown"; - -export interface SelectProps { - name: string; - label: string; - labelPosition?: "top" | "left"; - labelClassName?: string; - hideLabel?: boolean; - value: T; - help?: ReactNode; - leftSlot?: ReactNode; - options: RadioDropdownItem[]; - onChange: (value: T) => void; - defaultValue?: T; - size?: ButtonProps["size"]; - className?: string; - disabled?: boolean; - filterable?: boolean; -} - -export function Select({ - labelPosition = "top", - name, - help, - labelClassName, - disabled, - hideLabel, - label, - value, - options, - leftSlot, - onChange, - className, - defaultValue, - filterable, - size = "md", -}: SelectProps) { - const [focused, setFocused] = useState(false); - const id = `input-${name}`; - const isInvalidSelection = options.find((o) => "value" in o && o.value === value) == null; - - const handleChange = (value: T) => { - onChange?.(value); - }; - - return ( -
- - {platform.osType() === "macos" && !filterable ? ( - - {leftSlot &&
{leftSlot}
} - -
- ) : ( - // Use custom "select" component until Tauri can be configured to have select menus not always appear in - // light mode - - o.type === "separator" || o.value !== defaultValue - ? o - : { ...o, label: <>{o.label} (default) }, - )} - > - - - )} -
- ); -} - -const selectBackgroundStyles: CSSProperties = { - backgroundImage: `url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' fill='none' viewBox='0 0 20 20'%3e%3cpath stroke='%236b7280' stroke-linecap='round' stroke-linejoin='round' stroke-width='1.5' d='M6 8l4 4 4-4'/%3e%3c/svg%3e")`, - backgroundPosition: "right 0.3rem center", - backgroundRepeat: "no-repeat", - backgroundSize: "1.5em 1.5em", - appearance: "none", - printColorAdjust: "exact", -}; +export type { SelectProps } from "@yaakapp-internal/ui"; +export { Select } from "@yaakapp-internal/ui"; diff --git a/apps/yaak-client/components/core/SettingRow.tsx b/apps/yaak-client/components/core/SettingRow.tsx index a3c6f6c63..d912d04d3 100644 --- a/apps/yaak-client/components/core/SettingRow.tsx +++ b/apps/yaak-client/components/core/SettingRow.tsx @@ -1,260 +1,36 @@ import type { AnyModel } from "@yaakapp-internal/models"; import { patchModel } from "@yaakapp-internal/models"; -import { Icon } from "@yaakapp-internal/ui"; -import classNames from "classnames"; -import type { ReactNode } from "react"; -import { CopyIconButton } from "../CopyIconButton"; -import { Checkbox } from "./Checkbox"; -import { IconButton, type IconButtonProps } from "./IconButton"; -import { PlainInput } from "./PlainInput"; -import type { RadioDropdownItem } from "./RadioDropdown"; -import { Select } from "./Select"; +import { + SettingRow, + type SettingRowBaseProps, + SettingRowBoolean, + SettingRowNumber, + SettingRowSelect, + SettingRowText, + SettingSelectControl, +} from "@yaakapp-internal/ui"; import { SelectFile } from "../SelectFile"; +export type { SettingRowBaseProps, SettingSnippetLanguage } from "@yaakapp-internal/ui"; +export { + SettingCommand, + SettingOverrideRow, + SettingRow, + SettingRowBoolean, + SettingRowNumber, + SettingRowSelect, + SettingRowText, + SettingSelectControl, + SettingSnippet, + SettingsList, + SettingsSection, + SettingValue, +} from "@yaakapp-internal/ui"; + type ModelKeyOfValue = { [K in keyof T]-?: T[K] extends V ? K : never; }[keyof T]; -type SettingRowBaseProps = { - className?: string; - controlClassName?: string; - description?: ReactNode; - disabled?: boolean; - title: ReactNode; -}; - -export function SettingsList({ children, className }: { children: ReactNode; className?: string }) { - return
{children}
; -} - -export function SettingsSection({ - children, - className, - description, - title, -}: { - children: ReactNode; - className?: string; - description?: ReactNode; - title: ReactNode | null; -}) { - const showHeader = title != null || description != null; - - return ( -
- {showHeader && ( -
- {title != null &&
{title}
} - {description != null &&

{description}

} -
- )} -
{children}
-
- ); -} - -export function SettingRow({ - children, - className, - controlClassName, - description, - disabled, - title, -}: { - children: ReactNode; -} & SettingRowBaseProps) { - return ( -
-
-
-
{title}
- {description != null && ( -
{description}
- )} -
-
- {children} -
-
-
- ); -} - -export function SettingValue({ - actions, - className, - copyText, - enableCopy = true, - value, -}: { - actions?: SettingValueAction[]; - className?: string; - copyText?: string; - enableCopy?: boolean; - value: ReactNode; -}) { - const textValue = typeof value === "string" || typeof value === "number" ? `${value}` : null; - const textToCopy = copyText ?? textValue; - - return ( - <> - - {value} - - {actions?.map((action) => ( - - ))} - {enableCopy && textToCopy != null && ( - - )} - - ); -} - -/** - * A command or config to copy, set as a block: its title, an optional line - * saying which file it goes in, then the snippet at full width with the copy - * button on it. One-line shell commands and multi-line JSON get the same block. - */ -export function SettingCommand({ - className, - description, - disabled, - done, - language, - snippet, - title, -}: { - description?: ReactNode; - disabled?: boolean; - /** A short word shown with a check beside the title once the step is detected as done */ - done?: ReactNode; - language: SettingSnippetLanguage; - snippet: string; - title: ReactNode; -} & Pick) { - return ( -
-
- {title} - {done != null && done !== false && ( - - - {done} - - )} -
- {description != null && ( -
{description}
- )} - -
- ); -} - -export type SettingSnippetLanguage = "shell" | "json" | "text"; - -/** The snippet half of a `SettingCommand`, for places with their own title, like a banner. */ -export function SettingSnippet({ - className, - language, - snippet, -}: { - className?: string; - language: SettingSnippetLanguage; - snippet: string; -}) { - const prompt = language === "shell"; - - return ( -
-
- - {prompt && ( - - $ - - )} - {snippet} - -
- -
- ); -} - -type SettingValueAction = { - icon: IconButtonProps["icon"]; - onClick: () => void; - title: string; -}; - -export function SettingRowBoolean({ - checked, - checkboxSize = "md", - onChange, - title, - ...props -}: { - checked: boolean; - checkboxSize?: "sm" | "md"; - onChange: (checked: boolean) => void; -} & SettingRowBaseProps) { - return ( - - - - ); -} - export function ModelSettingRowBoolean>({ model, modelKey, @@ -272,50 +48,6 @@ export function ModelSettingRowBoolean void; - placeholder?: string; - required?: boolean; - type?: "number"; - validate?: (value: string) => boolean; - value: number; -} & SettingRowBaseProps) { - return ( - - onChange(Number.parseInt(value, 10) || 0)} - type={type} - className={inputClassName} - containerClassName={inputWidthClassName} - disabled={props.disabled} - /> - - ); -} - export function ModelSettingRowNumber>({ model, modelKey, @@ -334,47 +66,6 @@ export function ModelSettingRowNumber void; - placeholder?: string; - required?: boolean; - type?: "text" | "password"; - value: string; -} & SettingRowBaseProps) { - return ( - - - - ); -} - export function ModelSettingRowText>({ model, modelKey, @@ -438,74 +129,6 @@ export function SettingRowDirectory({ return ; } -export function SettingRowSelect({ - defaultValue, - name, - onChange, - options, - selectClassName = "w-48!", - title, - value, - ...props -}: { - defaultValue?: T; - name: string; - onChange: (value: T) => void; - options: RadioDropdownItem[]; - selectClassName?: string; - value: T; -} & SettingRowBaseProps) { - return ( - - - - ); -} - -export function SettingSelectControl({ - defaultValue, - disabled, - label, - name, - onChange, - options, - selectClassName = "w-48!", - value, -}: { - defaultValue?: T; - disabled?: boolean; - label: string; - name: string; - onChange: (value: T) => void; - options: RadioDropdownItem[]; - selectClassName?: string; - value: T; -}) { - return ( - handleChange(e.target.value as T)} + onFocus={() => setFocused(true)} + onBlur={() => setFocused(false)} + disabled={disabled} + className={classNames( + "pr-7 w-full outline-hidden bg-transparent disabled:opacity-disabled", + "leading-none rounded-none", // Center the text better vertically + )} + > + {isInvalidSelection && } + {options.map((o) => { + if (o.type === "separator") return null; + return ( + + ); + })} + + + ) : ( + // Use custom "select" component until Tauri can be configured to have select menus not always appear in + // light mode + + o.type === "separator" || o.value !== defaultValue + ? o + : { ...o, label: <>{o.label} (default) }, + )} + > + + + )} + + ); +} + +const selectBackgroundStyles: CSSProperties = { + backgroundImage: `url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' fill='none' viewBox='0 0 20 20'%3e%3cpath stroke='%236b7280' stroke-linecap='round' stroke-linejoin='round' stroke-width='1.5' d='M6 8l4 4 4-4'/%3e%3c/svg%3e")`, + backgroundPosition: "right 0.3rem center", + backgroundRepeat: "no-repeat", + backgroundSize: "1.5em 1.5em", + appearance: "none", + printColorAdjust: "exact", +}; diff --git a/packages/ui/src/components/settings/SettingRow.tsx b/packages/ui/src/components/settings/SettingRow.tsx new file mode 100644 index 000000000..b2712c2dd --- /dev/null +++ b/packages/ui/src/components/settings/SettingRow.tsx @@ -0,0 +1,457 @@ +import classNames from "classnames"; +import type { ReactNode } from "react"; +import { Checkbox } from "../Checkbox"; +import { CopyIconButton as BaseCopyIconButton, type CopyIconButtonProps } from "../CopyIconButton"; +import { useEditorHost } from "../EditorHost"; +import { Icon } from "../Icon"; +import { IconButton, type IconButtonProps } from "../IconButton"; +import { PlainInput } from "../PlainInput"; +import type { RadioDropdownItem } from "../RadioDropdown"; +import { Select } from "../Select"; + +/** Copies through the host, quietly, as the rows' values do */ +function CopyIconButton(props: Omit) { + const host = useEditorHost(); + return ; +} + +export type SettingRowBaseProps = { + className?: string; + controlClassName?: string; + description?: ReactNode; + disabled?: boolean; + title: ReactNode; +}; + +export function SettingsList({ children, className }: { children: ReactNode; className?: string }) { + return
{children}
; +} + +export function SettingsSection({ + children, + className, + description, + title, +}: { + children: ReactNode; + className?: string; + description?: ReactNode; + title: ReactNode | null; +}) { + const showHeader = title != null || description != null; + + return ( +
+ {showHeader && ( +
+ {title != null &&
{title}
} + {description != null &&

{description}

} +
+ )} +
{children}
+
+ ); +} + +export function SettingRow({ + children, + className, + controlClassName, + description, + disabled, + title, +}: { + children: ReactNode; +} & SettingRowBaseProps) { + return ( +
+
+
+
{title}
+ {description != null && ( +
{description}
+ )} +
+
+ {children} +
+
+
+ ); +} + +export function SettingValue({ + actions, + className, + copyText, + enableCopy = true, + value, +}: { + actions?: SettingValueAction[]; + className?: string; + copyText?: string; + enableCopy?: boolean; + value: ReactNode; +}) { + const textValue = typeof value === "string" || typeof value === "number" ? `${value}` : null; + const textToCopy = copyText ?? textValue; + + return ( + <> + + {value} + + {actions?.map((action) => ( + + ))} + {enableCopy && textToCopy != null && ( + + )} + + ); +} + +/** + * A command or config to copy, set as a block: its title, an optional line + * saying which file it goes in, then the snippet at full width with the copy + * button on it. One-line shell commands and multi-line JSON get the same block. + */ +export function SettingCommand({ + className, + description, + disabled, + done, + language, + snippet, + title, +}: { + description?: ReactNode; + disabled?: boolean; + /** A short word shown with a check beside the title once the step is detected as done */ + done?: ReactNode; + language: SettingSnippetLanguage; + snippet: string; + title: ReactNode; +} & Pick) { + return ( +
+
+ {title} + {done != null && done !== false && ( + + + {done} + + )} +
+ {description != null && ( +
{description}
+ )} + +
+ ); +} + +export type SettingSnippetLanguage = "shell" | "json" | "text"; + +/** The snippet half of a `SettingCommand`, for places with their own title, like a banner. */ +export function SettingSnippet({ + className, + language, + snippet, +}: { + className?: string; + language: SettingSnippetLanguage; + snippet: string; +}) { + const prompt = language === "shell"; + + return ( +
+
+ + {prompt && ( + + $ + + )} + {snippet} + +
+ +
+ ); +} + +type SettingValueAction = { + icon: IconButtonProps["icon"]; + onClick: () => void; + title: string; +}; + +export function SettingRowBoolean({ + checked, + checkboxSize = "md", + onChange, + title, + ...props +}: { + checked: boolean; + checkboxSize?: "sm" | "md"; + onChange: (checked: boolean) => void; +} & SettingRowBaseProps) { + return ( + + + + ); +} + +export function SettingRowNumber({ + inputClassName, + inputWidthClassName = "w-48!", + name, + onChange, + placeholder, + required, + title, + type = "number", + validate, + value, + ...props +}: { + inputClassName?: string; + inputWidthClassName?: string; + name: string; + onChange: (value: number) => void; + placeholder?: string; + required?: boolean; + type?: "number"; + validate?: (value: string) => boolean; + value: number; +} & SettingRowBaseProps) { + return ( + + onChange(Number.parseInt(value, 10) || 0)} + type={type} + className={inputClassName} + containerClassName={inputWidthClassName} + disabled={props.disabled} + /> + + ); +} + +export function SettingRowText({ + inputClassName, + inputWidthClassName = "w-80!", + name, + onChange, + placeholder, + required, + title, + type = "text", + value, + ...props +}: { + inputClassName?: string; + inputWidthClassName?: string; + name: string; + onChange: (value: string) => void; + placeholder?: string; + required?: boolean; + type?: "text" | "password"; + value: string; +} & SettingRowBaseProps) { + return ( + + + + ); +} + +export function SettingRowSelect({ + defaultValue, + name, + onChange, + options, + selectClassName = "w-48!", + title, + value, + ...props +}: { + defaultValue?: T; + name: string; + onChange: (value: T) => void; + options: RadioDropdownItem[]; + selectClassName?: string; + value: T; +} & SettingRowBaseProps) { + return ( + + + + ); +} + +export function SettingSelectControl({ + defaultValue, + disabled, + label, + name, + onChange, + options, + selectClassName = "w-48!", + value, +}: { + defaultValue?: T; + disabled?: boolean; + label: string; + name: string; + onChange: (value: T) => void; + options: RadioDropdownItem[]; + selectClassName?: string; + value: T; +}) { + return ( + as the ref const inputRef = useRef(null); - // Bumped to empty the field, which only takes a value from outside when this changes - const [resetKey, setResetKey] = useState(0); useEffect(() => { - const timer = setTimeout(() => setTerms(searchTerms(text)), SEARCH_DEBOUNCE_MS); + const timer = setTimeout(() => setTerms(text), SEARCH_DEBOUNCE_MS); return () => clearTimeout(timer); }, [text, setTerms]); @@ -40,72 +46,84 @@ export const SearchBar = memo(function SearchBar() { return () => window.removeEventListener("keydown", onKeyDown); }, []); - // The field is replaced to empty it, so focus moves to the new one. + // The field is replaced to take a value from outside, so focus moves to the new one. const clear = () => { - setText(""); - setTerms(null); - setResetKey((k) => k + 1); + setSearchText(""); requestAnimationFrame(() => inputRef.current?.focus()); }; return ( -
-
- { - if (e.key === "Escape" && text !== "") { - e.stopPropagation(); - clear(); - } - }} - leftSlot={ - - } - rightSlot={ - text === "" ? ( - - ⌘F - - ) : ( - - ) - } - /> -
- { - try { - const written = await exportHar(query == null ? {} : { filter: query }, "flows.har"); - if (written != null) { - showToast({ - message: - written === 1 ? "Exported 1 flow as HAR" : `Exported ${written} flows as HAR`, - }); - } - } catch (err) { - showToast({ message: `Failed to export HAR: ${String(err)}`, color: "danger" }); - } - }} - /> -
+ { + if (e.key === "Escape" && text !== "") { + e.stopPropagation(); + clear(); + } + }} + leftSlot={ + <> + + + + } + rightSlot={ + text === "" ? ( + + ⌘F + + ) : ( +
+ + +
+ ) + } + /> ); }); + +/** + * Keeps the placeholder's count current as flows arrive, alone, so new traffic re-renders this + * and not the whole field + */ +function PlaceholderCount({ inputRef }: { inputRef: RefObject }) { + const total = useAtomValue(allTrafficTotalAtom); + useEffect(() => { + if (inputRef.current != null) inputRef.current.placeholder = placeholderFor(total); + }, [inputRef, total]); + return null; +} + +/** "Search 4,102 flows", the title and count that sat in the header's middle */ +function placeholderFor(total: number): string { + if (total === 0) return "Search flows"; + return `Search ${total.toLocaleString()} ${total === 1 ? "flow" : "flows"}`; +} + +/** A search's matches of all traffic, "12 of 4,102", inside the field before its x */ +function MatchCount() { + const matches = useAtomValue(shownTotalAtom); + const total = useAtomValue(allTrafficTotalAtom); + const search = useAtomValue(searchWindowAtom); + if (search == null || search.loading) return null; + return ( + + {matches.toLocaleString()} of {total.toLocaleString()} + + ); +} diff --git a/apps/yaak-proxy/components/Sidebar.tsx b/apps/yaak-proxy/components/Sidebar.tsx index 91c3ff932..a7d1b1d18 100644 --- a/apps/yaak-proxy/components/Sidebar.tsx +++ b/apps/yaak-proxy/components/Sidebar.tsx @@ -1,31 +1,15 @@ -import type { Flow } from "@yaakapp-internal/proxy-lib"; -import type { TreeNode } from "@yaakapp-internal/ui"; -import { selectedIdsFamily, Tree } from "@yaakapp-internal/ui"; -import { atom, useAtomValue } from "jotai"; +import { Icon, IconButton, Tree } from "@yaakapp-internal/ui"; +import classNames from "classnames"; +import { atom, useAtom, useAtomValue } from "jotai"; import { atomWithStorage } from "jotai/utils"; import { useCallback } from "react"; import { SIDEBAR_TREE_ID } from "../lib/atoms"; -import { appKeyOf, appNodeId, sidebarPath } from "../lib/search"; import { applyCollapsed, type CollapsedMap, collapsedMap, folderIds } from "../lib/sidebarCollapse"; -import { flowsAtom, flowTotalAtom } from "../lib/store"; +import { type SidebarItem, SYSTEM_ID } from "../lib/sidebarTree"; +import { hideSystemAtom, noteSidebarInput, sidebarTreeAtom } from "../lib/store"; import { AppIcon } from "./AppIcon"; -/** A node in the sidebar tree: a domain, a path segment, or an app. */ -export type SidebarItem = { - id: string; - label: string; - /** Flows held under this node; for the Domains folder, every flow in the database */ - count: number; - /** An app's path, for its icon */ - appPath?: string; -}; - -const sidebarTreeAtom = atom>((get) => { - const exchanges = get(flowsAtom); - return buildTree(exchanges, get(flowTotalAtom)); -}); - -/** The domains and paths the user expanded, by node id, kept across launches */ +/** The apps, sites, hosts and paths the user expanded, by node id, kept across launches */ const expandedAtom = atomWithStorage>("sidebar_expanded", {}, undefined, { getOnInit: true, }); @@ -33,7 +17,7 @@ const expandedAtom = atomWithStorage>("sidebar_expanded" /** The other nodes' collapsed flags, for this launch, as before */ const otherCollapsedAtom = atom({}); -/** Domains and paths collapsed until the user expands them, for the tree */ +/** Every app's tree, site, host and path, and System, collapsed until the user expands them */ const collapsedAtom = atom( (get) => collapsedMap(folderIds(get(sidebarTreeAtom)), get(expandedAtom), get(otherCollapsedAtom)), @@ -46,196 +30,90 @@ const collapsedAtom = atom( }, ); -/** The selected sidebar node's name, for the header */ -export const selectedSidebarLabelAtom = atom((get) => { - const selectedId = get(selectedIdsFamily(SIDEBAR_TREE_ID))[0]; - if (selectedId?.startsWith("domain:")) return selectedId.slice("domain:".length); - if (selectedId?.startsWith("path:")) return selectedId.slice("path:".length); - if (selectedId?.startsWith("app:")) { - const apps = get(sidebarTreeAtom).children?.find((c) => c.item.id === "apps"); - return apps?.children?.find((c) => c.item.id === selectedId)?.item.label ?? "All traffic"; - } - return "All traffic"; -}); - -const nodeIdsByFlow = new WeakMap(); - -/** - * The ids of the nodes a flow sits under, its domain first and then each path segment's, parsed - * once per flow rather than on every change to the list. - */ -function nodeIdsOf(ex: Flow): string[] { - let ids = nodeIdsByFlow.get(ex); - if (ids != null) return ids; - // By the rules the database filters by, so a node shows the flows it counts. - const path = sidebarPath(ex.url); - const schemeless = !ex.url.includes("://"); - const [hostname = "", ...segments] = schemeless ? [path] : path.split("/"); - ids = [`domain:${hostname}`]; - let node = `path:${hostname}`; - for (const seg of segments) { - node += `/${seg}`; - ids.push(node); - } - nodeIdsByFlow.set(ex, ids); - return ids; -} - -const collator = new Intl.Collator(); - -/** - * Build a domain → path-segment trie from a flat list of exchanges. - * - * Example: Given URLs - * GET https://api.example.com/v1/users - * GET https://api.example.com/v1/users/123 - * POST https://api.example.com/v1/orders - * - * Produces: - * api.example.com - * /v1 - * /users - * /123 - * /orders - */ -function buildTree(exchanges: Flow[], total: number): TreeNode { - const root: SidebarItem = { id: "root", label: "All Traffic", count: 0 }; - const rootNode: TreeNode = { - item: root, - parent: null, - depth: 0, - children: [], - draggable: false, - }; - - // Intermediate trie structure for building - type TrieNode = { - id: string; - label: string; - count: number; - children: Map; - }; - - const domainMap = new Map(); - - for (const ex of exchanges) { - const ids = nodeIdsOf(ex); - let current: TrieNode | undefined; - for (const id of ids) { - const siblings = current == null ? domainMap : current.children; - let node = siblings.get(id); - if (node == null) { - // The domain, or the path's last segment - const label = current == null ? id.slice("domain:".length) : id.slice(id.lastIndexOf("/")); - node = { id, label, count: 0, children: new Map() }; - siblings.set(id, node); - } - node.count++; - current = node; - } - } - - // Convert trie to TreeNode structure - function toTreeNode( - trie: TrieNode, - parent: TreeNode, - depth: number, - ): TreeNode { - const node: TreeNode = { - item: { - id: trie.id, - label: trie.label, - count: trie.count, - }, - parent, - depth, - children: [], - draggable: false, - }; - const sortedChildren = [...trie.children.values()].sort((a, b) => - collator.compare(a.label, b.label), - ); - for (const child of sortedChildren) { - node.children?.push(toTreeNode(child, node, depth + 1)); - } - return node; - } - - // Add a "Domains" folder between root and domain nodes - const domainsFolder: TreeNode = { - item: { id: "domains", label: "Domains", count: total }, - parent: rootNode, - depth: 1, - children: [], - draggable: false, - }; - - const sortedDomains = [...domainMap.values()].sort((a, b) => collator.compare(a.label, b.label)); - for (const domain of sortedDomains) { - domainsFolder.children?.push(toTreeNode(domain, domainsFolder, 2)); - } - - // The apps flows came from, where the platform can tell; none on others, so no group. - const apps = new Map(); - for (const ex of exchanges) { - const key = appKeyOf(ex); - if (key == null) continue; - const id = appNodeId(key); - const app = apps.get(id); - if (app == null) { - apps.set(id, { id, label: ex.clientName ?? key, count: 1, appPath: ex.clientPath ?? key }); - } else { - app.count++; - } - } - if (apps.size > 0) { - const appsFolder: TreeNode = { - item: { id: "apps", label: "Apps", count: 0 }, - parent: rootNode, - depth: 1, - children: [], - draggable: false, - }; - const sortedApps = [...apps.values()].sort((a, b) => collator.compare(a.label, b.label)); - for (const item of sortedApps) { - appsFolder.children?.push({ item, parent: appsFolder, depth: 2, draggable: false }); - } - rootNode.children?.push(appsFolder); - } - - rootNode.children?.push(domainsFolder); - - return rootNode; +function Count({ item }: { item: SidebarItem }) { + const text = item.countLabel ?? (item.count > 0 ? item.count.toLocaleString() : null); + if (text == null && item.kind !== "all") return null; + return ( + + {text ?? "0"} + + ); } function ItemInner({ item }: { item: SidebarItem }) { - const count = item.count; + if (item.kind === "group") { + return ( +
+ {item.label} + +
+ ); + } return (
+ {item.kind === "all" && } {item.appPath != null && } {item.label} - {count > 0 && {count}} + {item.sublabel != null && ( + {item.sublabel} + )} +
); } +/** Beside the row's button rather than in it, since it is a button of its own */ +function ItemRightSlot({ item }: { item: SidebarItem }) { + return item.id === SYSTEM_ID ? : ; +} + +/** The eye on System's row, which leaves its daemons out of All traffic */ +function HideSystemToggle() { + const [hidden, setHidden] = useAtom(hideSystemAtom); + return ( + { + e.stopPropagation(); + setHidden(!hidden); + }} + /> + ); +} + export function Sidebar() { const tree = useAtomValue(sidebarTreeAtom); - - const getItemKey = useCallback((item: SidebarItem) => `${item.id}:${item.count}`, []); - const treeId = SIDEBAR_TREE_ID; + const getItemKey = useCallback( + (item: SidebarItem) => `${item.id}:${item.count}:${item.countLabel ?? ""}`, + [], + ); return ( // The tree scrolls itself, in a row of bounded height, so its virtualizer mounts only the rows - // in view rather than every domain and path. - ); diff --git a/apps/yaak-proxy/harness/fixtures.ts b/apps/yaak-proxy/harness/fixtures.ts index 4c12f1c24..b58b87f09 100644 --- a/apps/yaak-proxy/harness/fixtures.ts +++ b/apps/yaak-proxy/harness/fixtures.ts @@ -50,6 +50,7 @@ const base: Flow = { grpcMethod: null, grpcStatus: null, grpcMessage: null, + tunnelled: false, }; const jsonRequest = JSON.stringify({ amount: 4200, currency: "usd", capture: true }); diff --git a/apps/yaak-proxy/harness/streamFixtures.ts b/apps/yaak-proxy/harness/streamFixtures.ts index 020ac2aca..835b5d3ed 100644 --- a/apps/yaak-proxy/harness/streamFixtures.ts +++ b/apps/yaak-proxy/harness/streamFixtures.ts @@ -49,6 +49,7 @@ const flowBase: Flow = { grpcMethod: null, grpcStatus: null, grpcMessage: null, + tunnelled: false, }; /** Seconds past 21:41:12 as a stored timestamp */ diff --git a/apps/yaak-proxy/hooks/useDecryptList.ts b/apps/yaak-proxy/hooks/useDecryptList.ts new file mode 100644 index 000000000..c8d67bfbe --- /dev/null +++ b/apps/yaak-proxy/hooks/useDecryptList.ts @@ -0,0 +1,45 @@ +import { useQueryClient } from "@tanstack/react-query"; +import type { + DecryptList, + DecryptListChangedPayload, + DecryptListEntry, +} from "@yaakapp-internal/proxy-lib"; +import { useCallback } from "react"; +import { useShowToast } from "../lib/toast"; +import { useRpcEvent } from "./useRpcEvent"; +import { useRpcMutation } from "./useRpcMutation"; +import { useRpcQuery } from "./useRpcQuery"; + +const KEY = ["get_decrypt_list", {}]; +const EMPTY: DecryptListEntry[] = []; + +/** + * The decrypt list's entries, empty until read. Kept current as this process changes it, from + * the app or through its control API. + */ +export function useDecryptList(): DecryptListEntry[] { + const queryClient = useQueryClient(); + const onChanged = useCallback( + (payload: DecryptListChangedPayload) => + queryClient.setQueryData(KEY, payload.list), + [queryClient], + ); + useRpcEvent("decrypt_list_changed", onChanged); + return useRpcQuery("get_decrypt_list", {}).data?.entries ?? EMPTY; +} + +/** + * Adds and removes entries, `edit_decrypt_list` applying them to the list as it is when they + * land, so two changes made in quick succession both stick. Everything that reads the list + * then reads it again. A failure, such as an entry that does not parse, is a toast. + */ +export function useEditDecryptList() { + const queryClient = useQueryClient(); + const showToast = useShowToast(); + return useRpcMutation("edit_decrypt_list", { + // Read again rather than taken from the answer, which can land after a newer change's + // event and would put the older list back. + onSuccess: () => queryClient.invalidateQueries({ queryKey: KEY }), + onError: (e) => showToast({ id: "decrypt-list", color: "danger", message: e.message }), + }); +} diff --git a/apps/yaak-proxy/lib/decrypt.test.ts b/apps/yaak-proxy/lib/decrypt.test.ts new file mode 100644 index 000000000..b14d48ac5 --- /dev/null +++ b/apps/yaak-proxy/lib/decrypt.test.ts @@ -0,0 +1,305 @@ +import { CompletionContext } from "@codemirror/autocomplete"; +import { EditorState } from "@codemirror/state"; +import type { DecryptListEntry } from "@yaakapp-internal/proxy-lib"; +import { filterCompletionSource } from "@yaakapp-internal/ui/editor"; +import { describe, expect, test } from "vite-plus/test"; +import { + blockerOf, + blockingExclusion, + decryptFilterOptions, + detailActions, + decryptMenuAction, + decrypts, + decryptsEverything, + entryLabel, + everythingEdit, + isDecrypted, + listEdit, + normalizeEntry, + setDecrypted, + subjectQuery, + toggleDecrypt, +} from "./decrypt"; + +const user = (...entries: string[]): DecryptListEntry[] => + entries.map((entry) => ({ entry, builtIn: false })); +const builtIn = (...entries: string[]): DecryptListEntry[] => + entries.map((entry) => ({ entry, builtIn: true })); +const BUILT_INS = builtIn("!host:*.apple.com", "!host:*.icloud.com"); + +describe("normalizeEntry", () => { + test("a bare hostname is a host entry", () => { + expect(normalizeEntry("api.example.com")).toBe("host:api.example.com"); + expect(normalizeEntry(" API.Example.com. ")).toBe("host:api.example.com"); + expect(normalizeEntry("https://api.example.com:8443/v1")).toBe("host:api.example.com:8443"); + }); + + test("prefixes and exclusions", () => { + expect(normalizeEntry("HOST:*.Example.com")).toBe("host:*.example.com"); + expect(normalizeEntry("!x.com")).toBe("!host:x.com"); + expect(normalizeEntry("! host:x.com")).toBe("!host:x.com"); + expect(normalizeEntry('app:"Google Chrome"')).toBe("app:Google Chrome"); + expect(normalizeEntry("App: com.google.Chrome")).toBe("app:com.google.Chrome"); + expect(normalizeEntry("host:[::1]:8443")).toBe("host:[::1]:8443"); + expect(normalizeEntry(" * ")).toBe("*"); + }); + + test("what is not an entry", () => { + for (const bad of ["", "!", "app:", "host:a b", "host:a.com:http", "host:a.com:99999"]) { + expect(normalizeEntry(bad)).toBeNull(); + } + }); + + test("labels", () => { + expect(entryLabel("*")).toBe("Everything"); + expect(entryLabel("host:a.com")).toBe("host:a.com"); + }); +}); + +describe("decrypts", () => { + test("an empty list decrypts nothing, with or without the built-ins", () => { + expect(decrypts([], { host: "a.com" })).toBe(false); + expect(decrypts(BUILT_INS, { host: "a.com" })).toBe(false); + }); + + test("hosts, sites and ports", () => { + const list = user("host:*.example.com", "host:other.com:8443"); + expect(decrypts(list, { host: "api.example.com" })).toBe(true); + expect(decrypts(list, { host: "example.com" })).toBe(true); + expect(decrypts(list, { host: "notexample.com" })).toBe(false); + expect(decrypts(list, { host: "other.com", port: 8443 })).toBe(true); + expect(decrypts(list, { host: "other.com" })).toBe(false); + }); + + test("apps by name or bundle id", () => { + const chrome = { name: "Google Chrome", bundleId: "com.google.Chrome" }; + expect(decrypts(user("app:google chrome"), { host: "a.com", app: chrome })).toBe(true); + expect(decrypts(user("app:com.google.*"), { host: "a.com", app: chrome })).toBe(true); + expect(decrypts(user("app:Safari"), { host: "a.com", app: chrome })).toBe(false); + }); + + test("everything keeps the built-in exclusions, which a specific entry beats", () => { + const list = [...user("*"), ...BUILT_INS]; + expect(decrypts(list, { host: "api.example.com" })).toBe(true); + expect(decrypts(list, { host: "gateway.icloud.com" })).toBe(false); + const specific = [...user("*", "host:developer.apple.com"), ...BUILT_INS]; + expect(decrypts(specific, { host: "developer.apple.com" })).toBe(true); + expect(decrypts(specific, { host: "www.apple.com" })).toBe(false); + }); + + test("the user's exclusions win wherever they sit", () => { + for (const list of [user("!host:a.example.com", "*"), user("*", "!host:a.example.com")]) { + expect(decrypts(list, { host: "a.example.com" })).toBe(false); + expect(decrypts(list, { host: "b.example.com" })).toBe(true); + } + }); +}); + +describe("the sidebar's Decrypt and Stop Decrypting", () => { + test("Decrypt adds the host's entry before the built-ins", () => { + const list = BUILT_INS; + const subject = { kind: "host", host: "api.example.com" } as const; + expect(isDecrypted(list, subject)).toBe(false); + expect(toggleDecrypt(list, subject)).toEqual([ + "host:api.example.com", + "!host:*.apple.com", + "!host:*.icloud.com", + ]); + }); + + test("Stop Decrypting removes the entry that decrypts it", () => { + const list = [...user("host:api.example.com", "host:b.com"), ...BUILT_INS]; + const subject = { kind: "host", host: "api.example.com" } as const; + expect(isDecrypted(list, subject)).toBe(true); + expect(toggleDecrypt(list, subject)).toEqual([ + "host:b.com", + "!host:*.apple.com", + "!host:*.icloud.com", + ]); + }); + + test("Stop Decrypting under everything excludes the host", () => { + const list = user("*"); + const subject = { kind: "site", site: "example.com" } as const; + expect(isDecrypted(list, subject)).toBe(true); + expect(toggleDecrypt(list, subject)).toEqual(["*", "!host:*.example.com"]); + }); + + test("Decrypt adds nothing while a broader exclusion of the user's would still win", () => { + const list = user("*", "!host:*.example.com"); + const subject = { kind: "host", host: "api.example.com" } as const; + expect(blockingExclusion(list, subjectQuery(subject))).toBe("!host:*.example.com"); + expect(toggleDecrypt(list, subject)).toEqual(["*", "!host:*.example.com"]); + // The built-ins are not the user's to block with: a specific entry beats them. + expect(blockingExclusion(BUILT_INS, { host: "www.apple.com" })).toBeNull(); + }); + + test("Decrypt undoes the sidebar's own Stop Decrypting", () => { + const subject = { kind: "site", site: "example.com" } as const; + const stopped = toggleDecrypt(user("*"), subject); + expect(stopped).toEqual(["*", "!host:*.example.com"]); + const list = user(...stopped); + expect(blockerOf(list, [subject])).toBeNull(); + expect(toggleDecrypt(list, subject)).toEqual(["*"]); + const host = { kind: "host", host: "api.example.com" } as const; + expect(blockerOf(list, [host])).toBe("!host:*.example.com"); + }); + + test("an exact site entry beats the built-in exclusion of the same site", () => { + expect(decrypts([...user("host:apple.com"), ...BUILT_INS], { host: "apple.com" })).toBe(true); + }); + + test("Decrypt drops the user's own exclusion first", () => { + const list = user("*", "!host:*.example.com"); + const subject = { kind: "site", site: "example.com" } as const; + expect(toggleDecrypt(list, subject)).toEqual(["*"]); + }); + + test("apps by name, and a host with a port", () => { + const slack = { kind: "app", name: "Slack", bundleId: "com.tinyspeck.slackmacgap" } as const; + expect(toggleDecrypt([], slack)).toEqual(["app:Slack"]); + expect(isDecrypted(user("app:com.tinyspeck.slackmacgap"), slack)).toBe(true); + expect(isDecrypted(user("*"), slack)).toBe(true); + const port = { kind: "host", host: "localhost:8443" } as const; + expect(toggleDecrypt([], port)).toEqual(["host:localhost:8443"]); + expect(isDecrypted(user("host:localhost:8443"), port)).toBe(true); + }); +}); + +describe("the sidebar menu", () => { + const a = { kind: "host", host: "a.com" } as const; + const b = { kind: "host", host: "b.com" } as const; + + test("offers Decrypt unless every selected row is decrypted", () => { + expect(decryptMenuAction(user("host:a.com"), [a])).toEqual({ + label: "Stop Decrypting", + on: false, + }); + expect(decryptMenuAction(user("host:a.com"), [a, b])).toEqual({ label: "Decrypt", on: true }); + expect(decryptMenuAction([], [])).toBeNull(); + }); + + test("applies to every selected row, leaving those already so", () => { + const next = setDecrypted(user("host:a.com"), [a, b], true).map((e) => e.entry); + expect(next).toEqual(["host:a.com", "host:b.com"]); + expect(setDecrypted(user("host:a.com", "host:b.com"), [a, b], false)).toEqual([]); + }); +}); + +describe("the Not decrypted detail's buttons", () => { + const slack = { name: "Slack", bundleId: "com.tinyspeck.slackmacgap" }; + + test("each button is offered only when its own change would decrypt the connection", () => { + // `!app:Slack` still excludes Slack's connections whatever host is added. + const list = user("*", "!app:Slack"); + expect(detailActions(list, "edgeapi.slack.com", slack)).toEqual({ + listed: false, + host: false, + app: true, + blocked: null, + }); + // And `!host:` the other way round. + expect(detailActions(user("*", "!host:edgeapi.slack.com"), "edgeapi.slack.com", slack)).toEqual( + { listed: false, host: true, app: false, blocked: null }, + ); + }); + + test("names the exclusion when neither would", () => { + const list = user("*", "!host:*.slack.com", "!app:Slack"); + expect(detailActions(list, "edgeapi.slack.com", slack)).toEqual({ + listed: false, + host: false, + app: false, + blocked: "!host:*.slack.com", + }); + }); + + test("both on an empty list, and none once it is listed", () => { + expect(detailActions(BUILT_INS, "a.com", slack)).toEqual({ + listed: false, + host: true, + app: true, + blocked: null, + }); + expect(detailActions(user("host:a.com"), "a.com", null).listed).toBe(true); + }); +}); + +describe("the first-run step", () => { + test("Everything adds *, and Pick as they appear takes it out", () => { + expect(decryptsEverything(BUILT_INS)).toBe(false); + expect(decryptsEverything([...user("*"), ...BUILT_INS])).toBe(true); + expect(everythingEdit(true)).toEqual({ add: ["*"], remove: [] }); + expect(everythingEdit(false)).toEqual({ add: [], remove: ["*"] }); + }); +}); + +describe("edits sent as additions and removals", () => { + test("a toggle becomes what it adds and removes, not the whole list", () => { + const list = [...user("*", "!host:*.example.com"), ...BUILT_INS]; + const site = { kind: "site", site: "example.com" } as const; + expect(listEdit(list, setDecrypted(list, [site], true))).toEqual({ + add: [], + remove: ["!host:*.example.com"], + }); + const host = { kind: "host", host: "a.com" } as const; + expect(listEdit(BUILT_INS, setDecrypted(BUILT_INS, [host], true))).toEqual({ + add: ["host:a.com"], + remove: [], + }); + }); + + test("two edits from the same stale list both survive, applied in turn", () => { + // What the lib does with each edit: remove, then add after the user's own entries. + const apply = (entries: string[], e: { add: string[]; remove: string[] }) => [ + ...entries.filter((x) => !e.remove.includes(x)), + ...e.add, + ]; + const stale = BUILT_INS; + const first = listEdit(stale, setDecrypted(stale, [{ kind: "host", host: "a.com" }], true)); + const second = listEdit(stale, setDecrypted(stale, [{ kind: "host", host: "b.com" }], true)); + const saved = apply(apply([], first), second); + expect(saved).toEqual(["host:a.com", "host:b.com"]); + }); +}); + +describe("the list's autocomplete", () => { + const source = filterCompletionSource( + decryptFilterOptions({ + hosts: ["api.example.com", "www.example.org"], + apps: [ + { name: "Google Chrome", bundleId: "com.google.Chrome" }, + { name: "curl", bundleId: null }, + ], + }), + ); + + function complete(doc: string) { + const state = EditorState.create({ doc }); + const result = source(new CompletionContext(state, doc.length, true)); + if (result == null || result instanceof Promise) return null; + return { from: result.from, labels: result.options.map((o) => o.label) }; + } + + test("an empty input offers host:, app: and !", () => { + expect(complete("")).toEqual({ from: 0, labels: ["host", "app", "!"] }); + expect(complete("ho")).toEqual({ from: 0, labels: ["host", "app", "!"] }); + }); + + test("after ! it offers the fields negated", () => { + expect(complete("!")).toEqual({ from: 0, labels: ["!host", "!app"] }); + expect(complete("!ap")).toEqual({ from: 0, labels: ["!host", "!app"] }); + }); + + test("after host: the hosts seen, replacing only the value", () => { + expect(complete("host:")).toEqual({ from: 5, labels: ["api.example.com", "www.example.org"] }); + expect(complete("!host:api")?.from).toBe(6); + }); + + test("after app: the apps seen, by name and bundle id", () => { + expect(complete("app:")).toEqual({ + from: 4, + labels: ['"Google Chrome"', "com.google.Chrome", "curl"], + }); + }); +}); diff --git a/apps/yaak-proxy/lib/decrypt.ts b/apps/yaak-proxy/lib/decrypt.ts new file mode 100644 index 000000000..d8a554222 --- /dev/null +++ b/apps/yaak-proxy/lib/decrypt.ts @@ -0,0 +1,394 @@ +import type { DecryptListEntry, SeenTargets } from "@yaakapp-internal/proxy-lib"; +import type { FilterOptions } from "@yaakapp-internal/ui/editor"; + +/** + * The decrypt list as the engine reads it: `host:` and `app:` entries, `*` for everything, any + * of them after `!` to exclude. `DecryptEntry` and `DecryptPolicy` in `crates/yaak-proxy`, which + * decide each connection, follow the same rules; this copy marks the sidebar and the detail. + */ +export const EVERYTHING = "*"; + +export interface ParsedEntry { + negated: boolean; + kind: "everything" | "host" | "app"; + /** A host pattern, lowercased, or an app's name or bundle id as written */ + pattern: string; + /** The port a host entry is limited to */ + port: number | null; +} + +/** + * An entry as typed: with its prefix, or a bare hostname, which is a `host:` entry. A URL is + * taken as its host. Null when it is not one. + */ +export function parseEntry(input: string): ParsedEntry | null { + let rest = input.trim(); + const negated = rest.startsWith("!"); + if (negated) rest = rest.slice(1).trimStart(); + if (rest === "") return null; + if (rest === EVERYTHING) return { negated, kind: "everything", pattern: "", port: null }; + if (/^app:/i.test(rest)) { + const app = unquote(rest.slice(4).trim()); + return app === "" ? null : { negated, kind: "app", pattern: app, port: null }; + } + const host = unquote((/^host:/i.test(rest) ? rest.slice(5) : rest).trim()); + const parsed = parseHost(host); + return parsed == null ? null : { negated, kind: "host", ...parsed }; +} + +function unquote(s: string): string { + return s.length >= 2 && s.startsWith('"') && s.endsWith('"') ? s.slice(1, -1).trim() : s; +} + +function parseHost(input: string): { pattern: string; port: number | null } | null { + let host = input; + const scheme = host.indexOf("://"); + if (scheme !== -1) { + host = host.slice(scheme + 3).split(/[/?#]/)[0] ?? ""; + host = host.slice(host.lastIndexOf("@") + 1); + } + let pattern: string; + let port: number | null = null; + if (host.startsWith("[")) { + const end = host.indexOf("]"); + if (end === -1) return null; + pattern = host.slice(1, end); + const after = host.slice(end + 1); + if (after !== "") { + if (!after.startsWith(":")) return null; + port = parsePort(after.slice(1)); + if (port == null) return null; + } + } else { + const colon = host.lastIndexOf(":"); + const name = colon === -1 ? host : host.slice(0, colon); + // One colon is a port; more, a bare IPv6 address. + if (colon !== -1 && !name.includes(":")) { + port = parsePort(host.slice(colon + 1)); + if (port == null) return null; + pattern = name; + } else { + pattern = host; + } + } + pattern = pattern.replace(/\.+$/, "").toLowerCase(); + if (pattern === "" || !/^[a-z0-9\-._*:]+$/.test(pattern)) return null; + return { pattern, port }; +} + +function parsePort(s: string): number | null { + if (!/^\d+$/.test(s)) return null; + const n = Number(s); + return n <= 65535 ? n : null; +} + +/** The entry as stored: `host:api.example.com`, `!app:Slack`, `*` */ +export function formatEntry(e: ParsedEntry): string { + const not = e.negated ? "!" : ""; + if (e.kind === "everything") return `${not}${EVERYTHING}`; + if (e.kind === "app") return `${not}app:${e.pattern}`; + if (e.port == null) return `${not}host:${e.pattern}`; + const host = e.pattern.includes(":") ? `[${e.pattern}]` : e.pattern; + return `${not}host:${host}:${e.port}`; +} + +/** An entry as typed, normalized as the engine stores it, or null when it is not one */ +export function normalizeEntry(input: string): string | null { + const parsed = parseEntry(input); + return parsed == null ? null : formatEntry(parsed); +} + +/** How the list names an entry: `*` is Everything, anything else as written */ +export function entryLabel(entry: string): string { + if (entry === EVERYTHING) return "Everything"; + if (entry === `!${EVERYTHING}`) return "Nothing"; + return entry; +} + +/** What a connection is: where it goes and, when known, the app that sent it */ +export interface DecryptQuery { + host: string; + port?: number | null; + app?: { name: string; bundleId: string | null } | null; +} + +function glob(pattern: string, text: string): boolean { + let pi = 0; + let ti = 0; + let star: [number, number] | null = null; + while (ti < text.length) { + if (pi < pattern.length && pattern[pi] === "*") { + star = [pi, ti]; + pi++; + } else if (pi < pattern.length && pattern[pi] === text[ti]) { + pi++; + ti++; + } else if (star != null) { + pi = star[0] + 1; + ti = star[1] + 1; + star = [star[0], star[1] + 1]; + } else { + return false; + } + } + return pattern + .slice(pi) + .split("") + .every((c) => c === "*"); +} + +/** `*.example.com` matches `example.com` too, as the search's `@host` does */ +function hostMatches(pattern: string, host: string): boolean { + const h = host + .replace(/^\[|\]$/g, "") + .replace(/\.+$/, "") + .toLowerCase(); + return glob(pattern, h) || (pattern.startsWith("*.") && glob(pattern.slice(2), h)); +} + +function matches(e: ParsedEntry, q: DecryptQuery): boolean { + switch (e.kind) { + case "everything": + return true; + case "host": + return (e.port == null || e.port === (q.port ?? 443)) && hostMatches(e.pattern, q.host); + case "app": { + if (q.app == null) return false; + const p = e.pattern.toLowerCase(); + return ( + glob(p, q.app.name.toLowerCase()) || + (q.app.bundleId != null && glob(p, q.app.bundleId.toLowerCase())) + ); + } + } +} + +function specificity(e: ParsedEntry): number { + if (e.kind === "everything") return 0; + if (e.kind === "app") return Number.POSITIVE_INFINITY; + // A site's `*.` counts for nothing, as `*.apple.com` matches `apple.com` itself. + const pattern = e.pattern.startsWith("*.") ? e.pattern.slice(2) : e.pattern; + return pattern.replaceAll("*", "").length + (e.port == null ? 0 : 1); +} + +/** + * Whether the list decrypts `q`: a positive entry matches and no negative one does, wherever + * each sits. A built-in exclusion yields to a positive entry at least as specific as itself. + */ +export function decrypts(list: DecryptListEntry[], q: DecryptQuery): boolean { + const parsed = list.flatMap((e) => { + const p = parseEntry(e.entry); + return p == null ? [] : [{ ...p, builtIn: e.builtIn }]; + }); + let best = -1; + for (const e of parsed) { + if (!e.negated && matches(e, q)) best = Math.max(best, specificity(e)); + } + if (best < 0) return false; + return !parsed.some((e) => e.negated && matches(e, q) && (!e.builtIn || specificity(e) > best)); +} + +/** Some entry decrypts something: the list is not only exclusions */ +export function decryptsAnything(list: DecryptListEntry[]): boolean { + return list.some((e) => !e.entry.trim().startsWith("!")); +} + +/** What a sidebar row or the detail's buttons decrypt: a host, a site, or an app */ +export type DecryptSubject = + | { kind: "host"; host: string } + | { kind: "site"; site: string } + | { kind: "app"; name: string; bundleId: string | null }; + +/** The entry that names `subject` alone: `host:api.example.com`, `host:*.example.com`, `app:Slack` */ +export function subjectEntry(subject: DecryptSubject): string { + switch (subject.kind) { + case "host": + return normalizeEntry(`host:${subject.host}`) ?? `host:${subject.host}`; + case "site": + return `host:*.${subject.site.toLowerCase()}`; + case "app": + return `app:${subject.name}`; + } +} + +/** The connection a subject stands for, as the list judges it */ +export function subjectQuery(subject: DecryptSubject): DecryptQuery { + switch (subject.kind) { + case "host": { + const parsed = parseHost(subject.host); + return { host: parsed?.pattern ?? subject.host, port: parsed?.port ?? 443 }; + } + case "site": + return { host: subject.site }; + case "app": + // No host matches an empty one, so only `*` and app entries decide. + return { host: "", app: { name: subject.name, bundleId: subject.bundleId } }; + } +} + +/** + * An exclusion of the user's that matches `q`, which no entry added beats: Decrypt cannot help + * until it is edited. Null when there is none. + */ +export function blockingExclusion( + list: DecryptListEntry[], + q: DecryptQuery, + builtIns = false, +): string | null { + for (const e of list) { + if (e.builtIn && !builtIns) continue; + const parsed = parseEntry(e.entry); + if (parsed?.negated && matches(parsed, q)) return e.entry; + } + return null; +} + +/** + * What keeps Decrypt on `subject` from working: an exclusion of the user's that matches it, + * other than its own exact one, which Decrypt removes. Null when Decrypt works. + */ +export function blockerOf(list: DecryptListEntry[], subjects: DecryptSubject[]): string | null { + const own = new Set(subjects.map((s) => `!${subjectEntry(s)}`)); + const rest = list.filter((e) => !own.has(e.entry)); + for (const subject of subjects) { + const blocker = blockingExclusion(rest, subjectQuery(subject)); + if (blocker != null) return blocker; + } + return null; +} + +/** What the Not decrypted detail offers for one tunnelled connection */ +export interface DetailActions { + /** The connection is decrypted from the next one on already */ + listed: boolean; + /** Decrypt Host would decrypt it */ + host: boolean; + /** Decrypt App would decrypt it */ + app: boolean; + /** Neither would: the exclusion that keeps it out */ + blocked: string | null; +} + +/** + * Each button judged on its own: whether decrypting the host, or the app, would decrypt this + * connection, with its host, port and app together. A button whose change something else + * still excludes is not offered; with neither, the exclusion is named. + */ +export function detailActions( + list: DecryptListEntry[], + host: string, + app: { name: string; bundleId: string | null } | null, +): DetailActions { + const hostSubject: DecryptSubject = { kind: "host", host }; + const q = { ...subjectQuery(hostSubject), app }; + if (decrypts(list, q)) return { listed: true, host: false, app: false, blocked: null }; + const works = (subject: DecryptSubject) => decrypts(setDecrypted(list, [subject], true), q); + const hostWorks = works(hostSubject); + const appWorks = app != null && works({ kind: "app", ...app }); + const blocked = + hostWorks || appWorks ? null : (blockingExclusion(list, q) ?? blockingExclusion(list, q, true)); + return { listed: false, host: hostWorks, app: appWorks, blocked }; +} + +/** The list decrypts `subject`'s connections, as its sidebar row marks it */ +export function isDecrypted(list: DecryptListEntry[], subject: DecryptSubject): boolean { + return decrypts(list, subjectQuery(subject)); +} + +/** + * The entries after Decrypt or Stop Decrypting on `subject`, to save whole. Decrypt drops an + * exclusion of the subject the user added, then adds its entry if it is still not decrypted. + * Stop drops its entry, then excludes it if something broader, `*` say, still decrypts it. + */ +export function toggleDecrypt(list: DecryptListEntry[], subject: DecryptSubject): string[] { + return setDecrypted(list, [subject], !isDecrypted(list, subject)).map((e) => e.entry); +} + +/** + * The list with every one of `subjects` decrypted (`on`) or not, each as [`toggleDecrypt`] + * changes it, those already so left alone: several sidebar rows at once. + */ +export function setDecrypted( + list: DecryptListEntry[], + subjects: DecryptSubject[], + on: boolean, +): DecryptListEntry[] { + let next = list; + for (const subject of subjects) { + if (isDecrypted(next, subject) === on) continue; + const entry = subjectEntry(subject); + const drop = on ? `!${entry}` : entry; + next = next.filter((e) => e.builtIn || e.entry !== drop); + if (isDecrypted(next, subject) === on) continue; + // Something the user excluded more broadly still keeps it out; adding would change nothing. + if (on && blockingExclusion(next, subjectQuery(subject)) != null) continue; + const add = on ? entry : `!${entry}`; + next = [...next.filter((e) => !e.builtIn), { entry: add, builtIn: false }, ...builtIns(next)]; + } + return next; +} + +/** + * The sidebar menu's row for the selected rows that can be decrypted: Stop Decrypting when every + * one of them is, else Decrypt. Null when none can. + */ +export function decryptMenuAction( + list: DecryptListEntry[], + subjects: DecryptSubject[], +): { label: "Decrypt" | "Stop Decrypting"; on: boolean } | null { + if (subjects.length === 0) return null; + const on = !subjects.every((s) => isDecrypted(list, s)); + return { label: on ? "Decrypt" : "Stop Decrypting", on }; +} + +function builtIns(list: DecryptListEntry[]): DecryptListEntry[] { + return list.filter((e) => e.builtIn); +} + +/** + * What turns `before` into `after`, as `edit_decrypt_list` takes it: the entries to add and to + * remove, applied to the list as it is when they land, so edits made together each keep the + * other's. + */ +export function listEdit( + before: DecryptListEntry[], + after: DecryptListEntry[], +): { add: string[]; remove: string[] } { + const was = new Set(before.map((e) => e.entry)); + const now = new Set(after.map((e) => e.entry)); + return { + add: after.filter((e) => !was.has(e.entry)).map((e) => e.entry), + remove: before.filter((e) => !now.has(e.entry)).map((e) => e.entry), + }; +} + +/** The edit that adds `*`, or takes it out, as the first-run card's choice makes it */ +export function everythingEdit(on: boolean): { add: string[]; remove: string[] } { + return on ? { add: [EVERYTHING], remove: [] } : { add: [], remove: [EVERYTHING] }; +} + +/** Whether the list decrypts everything, as the first-run card's choice reads it */ +export function decryptsEverything(list: DecryptListEntry[]): boolean { + return list.some((e) => e.entry === EVERYTHING); +} + +/** + * What the list's input suggests: `host:`, `app:` and `!` to start an entry; after `host:` the + * hosts seen in the capture, after `app:` the apps seen, by name and bundle id. + */ +export function decryptFilterOptions(seen: SeenTargets | null | undefined): FilterOptions { + const apps = new Set(); + for (const app of seen?.apps ?? []) { + apps.add(app.name); + if (app.bundleId != null) apps.add(app.bundleId); + } + return { + fieldPrefix: "", + replaceValueOnly: true, + negation: { prefix: "!", info: "Exclude" }, + fields: [ + { name: "host", values: seen?.hosts ?? [], info: "Host or *.site" }, + { name: "app", values: [...apps], info: "App name or bundle id" }, + ], + }; +} diff --git a/apps/yaak-proxy/lib/firstRun.test.ts b/apps/yaak-proxy/lib/firstRun.test.ts index 096e5168d..8fb674103 100644 --- a/apps/yaak-proxy/lib/firstRun.test.ts +++ b/apps/yaak-proxy/lib/firstRun.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "vite-plus/test"; -import { setupIncomplete } from "./firstRun"; +import { decryptStep, setupIncomplete } from "./firstRun"; describe("setupIncomplete", () => { const done = { trusted: true, systemProxySupported: true, systemProxyEverOn: true }; @@ -25,3 +25,21 @@ describe("setupIncomplete", () => { expect(setupIncomplete({ ...done, systemProxySupported: null, trusted: false })).toBe(false); }); }); + +describe("decryptStep", () => { + const builtIns = [{ entry: "!host:*.apple.com", builtIn: true }]; + + test("picks hosts as they appear by default, after the certificate", () => { + expect(decryptStep(builtIns, false)).toEqual({ choice: "pick", state: "todo" }); + expect(decryptStep(builtIns, true)).toEqual({ choice: "pick", state: "next" }); + }); + + test("is done once everything, or a host, is on the list", () => { + const everything = [{ entry: "*", builtIn: false }, ...builtIns]; + expect(decryptStep(everything, true)).toEqual({ choice: "everything", state: "done" }); + const host = [{ entry: "host:api.example.com", builtIn: false }]; + expect(decryptStep(host, true)).toEqual({ choice: "pick", state: "done" }); + // An exclusion alone decrypts nothing. + expect(decryptStep([{ entry: "!host:a.com", builtIn: false }], true).state).toBe("next"); + }); +}); diff --git a/apps/yaak-proxy/lib/firstRun.ts b/apps/yaak-proxy/lib/firstRun.ts index b5f15ef2e..8c5dd28c9 100644 --- a/apps/yaak-proxy/lib/firstRun.ts +++ b/apps/yaak-proxy/lib/firstRun.ts @@ -1,3 +1,6 @@ +import type { DecryptListEntry } from "@yaakapp-internal/proxy-lib"; +import { decryptsEverything } from "./decrypt"; + /** What decides whether setup is done, as the app last read it */ export interface SetupState { /** The certificate is trusted, or null until read */ @@ -19,3 +22,21 @@ export function setupIncomplete(s: SetupState): boolean { if (!s.trusted) return true; return s.systemProxySupported && !s.systemProxyEverOn; } + +/** The first-run card's "Choose what to decrypt" step, as the decrypt list has it */ +export interface DecryptStep { + /** Everything is decrypted, `*` on the list; else hosts are picked as they appear */ + choice: "everything" | "pick"; + state: "todo" | "next" | "done"; +} + +/** + * Picking hosts as they appear is the default and needs nothing done, so the step waits on the + * certificate like the others and is done once a choice is on the list: everything, or any host + * or app picked already. + */ +export function decryptStep(list: DecryptListEntry[], trusted: boolean): DecryptStep { + const choice = decryptsEverything(list) ? "everything" : "pick"; + const picked = list.some((e) => !e.builtIn && !e.entry.startsWith("!")); + return { choice, state: picked ? "done" : trusted ? "next" : "todo" }; +} diff --git a/apps/yaak-proxy/lib/flowViews.test.ts b/apps/yaak-proxy/lib/flowViews.test.ts index afb557db5..7e8c35c65 100644 --- a/apps/yaak-proxy/lib/flowViews.test.ts +++ b/apps/yaak-proxy/lib/flowViews.test.ts @@ -58,6 +58,7 @@ const flow: Flow = { grpcMethod: null, grpcStatus: null, grpcMessage: null, + tunnelled: false, }; function bytes(text: string): FlowBodyData { diff --git a/apps/yaak-proxy/lib/search.test.ts b/apps/yaak-proxy/lib/search.test.ts index 54da3fc48..55c838f53 100644 --- a/apps/yaak-proxy/lib/search.test.ts +++ b/apps/yaak-proxy/lib/search.test.ts @@ -229,6 +229,16 @@ describe("parseSearch", () => { expect(parseSearch("@is:system", true)).toEqual({ system: true }); }); + test("keeps or drops HTTPS passed through undecrypted", () => { + expect(parseSearch("@is:tunnelled")).toEqual({ tunnelled: true }); + expect(parseSearch("-@is:tunnelled stripe")).toEqual({ q: "stripe", tunnelled: false }); + const tunnel = { url: "https://a.com", method: "CONNECT", tunnelled: true } as Flow; + const decrypted = { url: "https://a.com/x", method: "GET", tunnelled: false } as Flow; + expect(matchesQuery(tunnel, { tunnelled: true })).toBe(true); + expect(matchesQuery(decrypted, { tunnelled: true })).toBe(false); + expect(matchesQuery(decrypted, { tunnelled: false })).toBe(true); + }); + test("round-trips what the sidebar writes", () => { const text = [ formatFieldFilter("app", "Google Chrome"), diff --git a/apps/yaak-proxy/lib/search.ts b/apps/yaak-proxy/lib/search.ts index a172bc1a1..39207dde6 100644 --- a/apps/yaak-proxy/lib/search.ts +++ b/apps/yaak-proxy/lib/search.ts @@ -5,7 +5,7 @@ import { getDomain } from "tldts"; * A search over every flow in the database: the search box's terms, its field terms, and * whether System's flows are hidden, as `list_flows` takes them. */ -export type FlowQuery = Pick; +export type FlowQuery = Pick; /** A word of the search box: a free term, or `@field:value`, maybe negated with `-` */ export interface SearchToken { @@ -56,7 +56,8 @@ export function formatFieldFilter(field: string, value: string, negated = false) * not every pairing of their apps and hosts. Within a clause, `@app` terms match any of their * apps; `@host` terms any of their hosts, each with the `@path` terms after it; `@is:system` * keeps only macOS's daemons and `-@is:system` drops them, as hiding System does when the - * clause says neither. Other words match the URL, the method, the status or the type. + * clause says neither. `@is:tunnelled` keeps only HTTPS passed through undecrypted, and + * `-@is:tunnelled` drops it. Other words match the URL, the method, the status or the type. */ export function parseSearch(text: string, hideSystem = false): FlowQuery | null { const clauses = clausesOf(text); @@ -77,6 +78,7 @@ function parseClause(tokens: SearchToken[], hideSystem: boolean): FlowQuery | nu // Each host with the paths written after it. const hosts: { host: string; paths: string[] }[] = []; let system: boolean | undefined = hideSystem ? false : undefined; + let tunnelled: boolean | undefined; for (const t of tokens) { if (t.field === "app" && !t.negated && t.value !== "") apps.push(t.value); else if (t.field === "host" && !t.negated && t.value !== "") { @@ -91,6 +93,7 @@ function parseClause(tokens: SearchToken[], hideSystem: boolean): FlowQuery | nu host.paths.push(path); } } else if (t.field === "is" && t.value.toLowerCase() === "system") system = !t.negated; + else if (t.field === "is" && t.value.toLowerCase() === "tunnelled") tunnelled = !t.negated; else if (t.field == null && !t.negated) words.push(t.value); // Fields and exclusions the query language will add are ignored until it lands. } @@ -105,6 +108,7 @@ function parseClause(tokens: SearchToken[], hideSystem: boolean): FlowQuery | nu ...(under.length > 0 ? { under } : {}), ...(apps.length > 0 ? { app: apps } : {}), ...(system == null ? {} : { system }), + ...(tunnelled == null ? {} : { tunnelled }), }; return Object.keys(query).length === 0 ? null : query; } @@ -190,6 +194,7 @@ export function matchesQuery(flow: Flow, query: FlowQuery): boolean { if (query.under != null && !query.under.some((node) => isUnder(flow.url, node))) return false; if (query.app != null && !query.app.some((app) => matchesApp(flow, app))) return false; if (query.system != null && isSystemFlow(flow) !== query.system) return false; + if (query.tunnelled != null && flow.tunnelled !== query.tunnelled) return false; // One of the groups, each judged whole, as the database judges them. if (query.any != null && query.any.length > 0) { return query.any.some((group) => matchesQuery(flow, group)); diff --git a/apps/yaak-proxy/lib/sidebarTree.test.ts b/apps/yaak-proxy/lib/sidebarTree.test.ts index dd755b31e..986a089a6 100644 --- a/apps/yaak-proxy/lib/sidebarTree.test.ts +++ b/apps/yaak-proxy/lib/sidebarTree.test.ts @@ -103,6 +103,44 @@ describe("buildSidebarTree", () => { }); }); +describe("decrypting from the sidebar", () => { + const tunnel = flow("https://bank.example.com", { + ...slack, + method: "CONNECT", + tunnelled: true, + }); + const root = buildSidebarTree([...flows, tunnel], 5); + const items = itemsById(root); + + test("Not decrypted follows All traffic while there are tunnelled flows", () => { + expect((root.children ?? []).slice(0, 2).map((c) => shape(c)[0])).toEqual([ + "All traffic 5", + "Not decrypted 1", + ]); + expect(items.get("tunnelled")?.query).toBe("@is:tunnelled"); + expect(itemsById(buildSidebarTree(flows, 4)).has("tunnelled")).toBe(false); + }); + + test("hosts, sites and apps name what their menu's Decrypt covers", () => { + expect(items.get("host:bank.example.com")?.decrypt).toEqual({ + kind: "host", + host: "bank.example.com", + }); + expect(items.get("site:example.com")?.decrypt).toEqual({ kind: "site", site: "example.com" }); + expect(items.get("app:Slack|host:edgeapi.slack.com")?.decrypt).toEqual({ + kind: "host", + host: "edgeapi.slack.com", + }); + expect(items.get("app:Slack")?.decrypt).toEqual({ + kind: "app", + name: "Slack", + bundleId: "com.slack", + }); + expect(items.get("path:api.github.com/repos")?.decrypt).toBeUndefined(); + expect(items.get("domains")?.decrypt).toBeUndefined(); + }); +}); + describe("selection and the search text", () => { test("selects rows under different apps as groups of their own", () => { const items = itemsById(buildSidebarTree(flows, 4)); diff --git a/apps/yaak-proxy/lib/sidebarTree.ts b/apps/yaak-proxy/lib/sidebarTree.ts index 866ac65a6..c7a99798d 100644 --- a/apps/yaak-proxy/lib/sidebarTree.ts +++ b/apps/yaak-proxy/lib/sidebarTree.ts @@ -1,5 +1,6 @@ import type { Flow } from "@yaakapp-internal/proxy-lib"; import type { TreeNode } from "@yaakapp-internal/ui"; +import type { DecryptSubject } from "./decrypt"; import { appKeyOf, folderOf, @@ -10,7 +11,7 @@ import { tokenize, } from "./search"; -export type SidebarKind = "all" | "group" | "app" | "site" | "host" | "path"; +export type SidebarKind = "all" | "tunnelled" | "group" | "app" | "site" | "host" | "path"; /** A row of the sidebar: All traffic, a group, an app, or a site, host or path under one */ export interface SidebarItem { @@ -27,6 +28,8 @@ export interface SidebarItem { appPath?: string; /** What selecting the row writes into the search box; empty for all traffic */ query: string; + /** What Decrypt in the row's menu decrypts: its host, site or app */ + decrypt?: DecryptSubject; } /** The ids of All traffic and the groups */ @@ -34,6 +37,7 @@ export const ALL_ID = "all"; export const APPS_ID = "apps"; export const DOMAINS_ID = "domains"; export const SYSTEM_ID = "system"; +export const TUNNELLED_ID = "tunnelled"; /** A row a flow sits under, with what makes it, so a rebuild only looks it up */ interface PlaceRow { @@ -41,6 +45,7 @@ interface PlaceRow { kind: "site" | "host" | "path"; label: string; query: string; + decrypt?: DecryptSubject; } /** @@ -72,8 +77,15 @@ function placeOf(flow: Flow): Place { kind: "site", label: site, query: formatFieldFilter("host", `*.${site}`), + decrypt: { kind: "site", site }, + }, + { + id: `host:${host}`, + kind: "host", + label: host, + query: hostQuery, + decrypt: { kind: "host", host }, }, - { id: `host:${host}`, kind: "host", label: host, query: hostQuery }, ]; let p = ""; for (const segment of segments) { @@ -121,6 +133,7 @@ function child(parent: Trie, row: PlaceRow): Trie { item: { id: row.id, kind: row.kind, label: row.label, count: 0, query: row.query }, children: new Map(), }; + if (row.decrypt != null) node.item.decrypt = row.decrypt; parent.children.set(row.id, node); } node.item.count++; @@ -162,7 +175,8 @@ function group(id: string, label: string): Trie { /** * The sidebar: All traffic, then Apps, each app opening to its own sites, hosts and paths, then * Domains by site, host and path, then System, macOS's daemons, apart from the apps. A group with - * nothing in it is left out. `total` is All traffic's count: every flow it covers in the + * nothing in it is left out. Not decrypted, after All traffic, gathers the HTTPS passed through + * undecrypted, when there is any. `total` is All traffic's count: every flow it covers in the * database. */ export function buildSidebarTree(flows: Flow[], total: number): TreeNode { @@ -177,9 +191,11 @@ export function buildSidebarTree(flows: Flow[], total: number): TreeNode 0) { + root.children?.push({ + item: { + id: TUNNELLED_ID, + kind: "tunnelled", + label: "Not decrypted", + count: tunnelled, + query: formatFieldFilter("is", "tunnelled"), + }, + parent: root, + depth: 1, + draggable: false, + }); + } for (const g of [apps, domains, system]) { if (g.children.size === 0) continue; const node: TreeNode = { diff --git a/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts b/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts index d78e9bc97..70fbac00c 100644 --- a/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts +++ b/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts @@ -104,7 +104,13 @@ grpcStatus: number | null, /** * A gRPC call's `grpc-message`, decoded. */ -grpcMessage: string | null, }; +grpcMessage: string | null, +/** + * An HTTPS connection passed through undecrypted, as the decrypt list leaves it out or its + * client rejected the proxy's certificate: method `CONNECT`, its URL the host, its sizes + * the bytes each way, and no headers or bodies. Open until `elapsedMs` or `error` is set. + */ +tunnelled: boolean, }; export type FlowBody = "request" | "response"; diff --git a/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts b/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts index bd9125d90..302fa98d3 100644 --- a/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts +++ b/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts @@ -87,10 +87,40 @@ schema: GrpcSchema | null, */ note: string | null, }; +/** + * What HTTPS is decrypted: the user's entries in order, then the built-in exclusions not + * removed. A connection is decrypted when a positive entry matches it and no exclusion does. + */ +export type DecryptList = { entries: Array, }; + +/** + * The decrypt list changed, here or through the control API. + */ +export type DecryptListChangedPayload = { list: DecryptList, }; + +/** + * One entry of the decrypt list, as the app shows it. + */ +export type DecryptListEntry = { +/** + * As stored: `host:*.example.com`, `app:Google Chrome`, `!host:*.apple.com`, `*`. + */ +entry: string, +/** + * Shipped with the app: an exclusion of Apple's services or a pinned app's hosts. + */ +builtIn: boolean, }; + export type DropReason = "hop_by_hop" | "set_by_sending" | "body_decoded"; export type DroppedHeader = { name: string, value: string, reason: DropReason, }; +/** + * Entries to add after the user's own and to remove, applied to the list as it is when the + * change lands. Each is normalized first. + */ +export type EditDecryptListRequest = { add: Array, remove: Array, }; + export type ExportHarRequest = { /** * The file to write. @@ -145,6 +175,10 @@ app?: Array, * tells them by their executable. */ system?: boolean, +/** + * Only HTTPS connections tunnelled undecrypted (true), or none of them (false). + */ +tunnelled?: boolean, /** * Only flows one of these groups covers, each judged on its own: several sidebar rows * selected, `@app:Slack @host:a.com OR @app:Chrome @host:b.com`, are two groups and not every @@ -186,6 +220,8 @@ folderId: string | null, }; export type FormatBodyRequest = { text: string, }; +export type GetDecryptListRequest = Record; + export type GetFlowBodyRequest = { flowId: string, which: FlowBody, }; export type GetFlowBodyResponse = { @@ -319,6 +355,8 @@ messages: Array, */ total: number, }; +export type ListSeenRequest = Record; + export type ListWorkspacesRequest = Record; export type ListWorkspacesResponse = { @@ -374,9 +412,9 @@ host: string, }; export type RemoveGrpcSchemaRequest = { id: string, }; -export type RpcEventSchema = { flows_cleared: FlowsClearedPayload, message_write: MessagePayload, model_write: ModelPayload, proxy_state_changed: ProxyStatePayload, }; +export type RpcEventSchema = { decrypt_list_changed: DecryptListChangedPayload, flows_cleared: FlowsClearedPayload, message_write: MessagePayload, model_write: ModelPayload, proxy_state_changed: ProxyStatePayload, }; -export type RpcSchema = { execute_action: [ActionInvocation, boolean], format_body: [FormatBodyRequest, string], get_message_payload: [GetMessagePayloadRequest, GetMessagePayloadResponse], list_actions: [ListActionsRequest, ListActionsResponse], list_flows: [FlowFilter, ListFlowsResponse], list_grpc_schemas: [ListGrpcSchemasRequest, Array], list_messages: [ListMessagesRequest, ListMessagesResponse], remove_grpc_schema: [RemoveGrpcSchemaRequest, boolean], set_recording: [SetRecordingRequest, boolean], set_system_proxy_at_launch: [SetSystemProxyAtLaunchRequest, boolean], set_http_version: [SetHttpVersionRequest, HttpVersion], add_grpc_schema: [AddGrpcSchemaRequest, GrpcSchema], app_icon: [AppIconRequest, string | null], ca_status: [CaStatusRequest, CaStatus], ca_trust: [CaTrustRequest, CaStatus], clear_flows: [ClearFlowsRequest, number], decode_grpc_message: [DecodeGrpcMessageRequest, DecodedGrpcMessage], export_har: [ExportHarRequest, number], filter_body: [FilterBodyRequest, FilterBodyResponse], get_flow_body: [GetFlowBodyRequest, GetFlowBodyResponse], get_proxy_state: [GetProxyStateRequest, GetProxyStateResponse], helper_register: [HelperRegisterRequest, SystemProxyStatus], helper_unregister: [HelperUnregisterRequest, SystemProxyStatus], list_workspaces: [ListWorkspacesRequest, ListWorkspacesResponse], preview_save: [PreviewSaveRequest, SavePreview], reflect_grpc_host: [ReflectGrpcHostRequest, GrpcSchema], save_flow: [SaveFlowToYaakRequest, SavedFlow], save_flow_body: [SaveFlowBodyRequest, SaveFlowBodyResponse], system_proxy_set: [SystemProxySetRequest, SystemProxyStatus], system_proxy_status: [SystemProxyStatusRequest, SystemProxyStatus], take_over: [TakeOverRequest, boolean], }; +export type RpcSchema = { execute_action: [ActionInvocation, boolean], format_body: [FormatBodyRequest, string], get_decrypt_list: [GetDecryptListRequest, DecryptList], get_message_payload: [GetMessagePayloadRequest, GetMessagePayloadResponse], list_actions: [ListActionsRequest, ListActionsResponse], list_flows: [FlowFilter, ListFlowsResponse], list_grpc_schemas: [ListGrpcSchemasRequest, Array], list_messages: [ListMessagesRequest, ListMessagesResponse], remove_grpc_schema: [RemoveGrpcSchemaRequest, boolean], set_recording: [SetRecordingRequest, boolean], set_system_proxy_at_launch: [SetSystemProxyAtLaunchRequest, boolean], set_http_version: [SetHttpVersionRequest, HttpVersion], add_grpc_schema: [AddGrpcSchemaRequest, GrpcSchema], app_icon: [AppIconRequest, string | null], ca_status: [CaStatusRequest, CaStatus], ca_trust: [CaTrustRequest, CaStatus], clear_flows: [ClearFlowsRequest, number], decode_grpc_message: [DecodeGrpcMessageRequest, DecodedGrpcMessage], export_har: [ExportHarRequest, number], filter_body: [FilterBodyRequest, FilterBodyResponse], get_flow_body: [GetFlowBodyRequest, GetFlowBodyResponse], get_proxy_state: [GetProxyStateRequest, GetProxyStateResponse], helper_register: [HelperRegisterRequest, SystemProxyStatus], helper_unregister: [HelperUnregisterRequest, SystemProxyStatus], edit_decrypt_list: [EditDecryptListRequest, DecryptList], list_seen: [ListSeenRequest, SeenTargets], set_decrypt_list: [SetDecryptListRequest, DecryptList], list_workspaces: [ListWorkspacesRequest, ListWorkspacesResponse], preview_save: [PreviewSaveRequest, SavePreview], reflect_grpc_host: [ReflectGrpcHostRequest, GrpcSchema], save_flow: [SaveFlowToYaakRequest, SavedFlow], save_flow_body: [SaveFlowBodyRequest, SaveFlowBodyResponse], system_proxy_set: [SystemProxySetRequest, SystemProxyStatus], system_proxy_status: [SystemProxyStatusRequest, SystemProxyStatus], take_over: [TakeOverRequest, boolean], }; export type SaveFlowBodyRequest = { flowId: string, which: FlowBody, /** @@ -440,6 +478,21 @@ authorization: boolean, }; export type SavedFlow = { requestId: string, name: string, }; +export type SeenApp = { name: string, bundleId: string | null, }; + +/** + * The hosts and apps flows have gone to and come from, for the decrypt list's suggestions. + */ +export type SeenTargets = { +/** + * Hosts as the sidebar names them, with a port other than the scheme's: most recent first. + */ +hosts: Array, +/** + * Apps by name and bundle id, most recent first. + */ +apps: Array, }; + export type ServiceProxy = { /** * The network service, as System Settings names it: "Wi-Fi", "USB 10/100/1000 LAN". @@ -451,6 +504,13 @@ name: string, */ id?: string, http: ProxySetting, https: ProxySetting, }; +export type SetDecryptListRequest = { +/** + * Every entry, in order, as typed: a bare hostname is a `host:` entry. A built-in + * exclusion left out is removed. + */ +entries: Array, }; + export type SetHttpVersionRequest = { httpVersion: HttpVersion, }; export type SetRecordingRequest = { enabled: boolean, }; diff --git a/crates-proxy/yaak-proxy-lib/migrations/00000009_flow_tunnelled.sql b/crates-proxy/yaak-proxy-lib/migrations/00000009_flow_tunnelled.sql new file mode 100644 index 000000000..beea72933 --- /dev/null +++ b/crates-proxy/yaak-proxy-lib/migrations/00000009_flow_tunnelled.sql @@ -0,0 +1,2 @@ +-- An HTTPS connection passed through undecrypted, recorded with only its host and sizes. +ALTER TABLE flows ADD COLUMN tunnelled BOOLEAN NOT NULL DEFAULT FALSE; diff --git a/crates-proxy/yaak-proxy-lib/src/client.rs b/crates-proxy/yaak-proxy-lib/src/client.rs index f1601fb0f..c5458b82c 100644 --- a/crates-proxy/yaak-proxy-lib/src/client.rs +++ b/crates-proxy/yaak-proxy-lib/src/client.rs @@ -5,8 +5,8 @@ use crate::bodies::{self, DecodedBody}; use crate::ca::CaStatus; use crate::control::{ - self, ControlInfo, ControlStatus, ErrorBody, NOTE_HEADER, RecordingRequest, StartRequest, - SystemProxyRequest, TRUNCATED_HEADER, + self, ControlInfo, ControlStatus, DecryptRequest, ErrorBody, NOTE_HEADER, RecordingRequest, + StartRequest, SystemProxyRequest, TRUNCATED_HEADER, }; use crate::db::ProxyQueryManager; use crate::db::Stores; @@ -15,7 +15,7 @@ use crate::har::{self, Har}; use crate::macos::SystemProxyStatus; use crate::models::{Flow, FlowBody}; use crate::save::{SaveFlowRequest, SaveFlowResponse, WorkspaceSummary}; -use crate::settings::ProxySettings; +use crate::settings::{DecryptList, ProxySettings}; use serde::Serialize; use serde::de::DeserializeOwned; use std::path::Path; @@ -146,6 +146,38 @@ impl ControlClient { self.post("/system-proxy", &SystemProxyRequest { enabled }).await } + pub async fn decrypt_list(&self) -> Result { + self.get("/decrypt").await + } + + /// Adds and removes entries in the owner's decrypt list as it is there. + pub async fn edit_decrypt_list( + &self, + edit: &crate::EditDecryptListRequest, + ) -> Result { + let body = serde_json::to_vec(edit).map_err(|e| e.to_string())?; + let req = self + .http + .patch(self.url("/decrypt")) + .header("content-type", "application/json") + .body(body); + let resp = self.send(req).await?; + resp.json().await.map_err(|e| format!("Invalid response from the proxy: {e}")) + } + + /// Replaces the decrypt list in the owner, which saves it and applies it to its engine. + pub async fn set_decrypt_list(&self, entries: &[String]) -> Result { + let body = serde_json::to_vec(&DecryptRequest { entries: entries.to_vec() }) + .map_err(|e| e.to_string())?; + let req = self + .http + .put(self.url("/decrypt")) + .header("content-type", "application/json") + .body(body); + let resp = self.send(req).await?; + resp.json().await.map_err(|e| format!("Invalid response from the proxy: {e}")) + } + pub async fn workspaces(&self) -> Result, String> { self.get("/workspaces").await } diff --git a/crates-proxy/yaak-proxy-lib/src/control.rs b/crates-proxy/yaak-proxy-lib/src/control.rs index 1a2e67948..b6bdb2e23 100644 --- a/crates-proxy/yaak-proxy-lib/src/control.rs +++ b/crates-proxy/yaak-proxy-lib/src/control.rs @@ -12,8 +12,8 @@ use crate::messages::{self, FlowMessage, MessageKind}; use crate::models::{FlowBody, find_header}; use crate::save::SaveFlowRequest; use crate::{ - AddGrpcSchemaRequest, ListMessagesResponse, ProxyCtx, ProxyState, ReflectGrpcHostRequest, - bodies, + AddGrpcSchemaRequest, EditDecryptListRequest, ListMessagesResponse, ProxyCtx, ProxyState, + ReflectGrpcHostRequest, bodies, }; use bytes::Bytes; use chrono::{DateTime, Utc}; @@ -237,6 +237,13 @@ pub struct RecordingRequest { pub enabled: bool, } +/// `PUT /decrypt`: every entry of the decrypt list, in order. A built-in exclusion left out is +/// removed. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DecryptRequest { + pub entries: Vec, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ClearResponse { /// How many flows were deleted. @@ -527,6 +534,15 @@ async fn route(state: &ServerState, req: Request) -> Result json(&ctx.decrypt_list()?), + (&Method::PATCH, ["decrypt"]) => { + let body: EditDecryptListRequest = read_json(req).await?; + json(&ctx.edit_decrypt_list(&body.add, &body.remove).map_err(bad_request)?) + } + (&Method::PUT, ["decrypt"]) => { + let body: DecryptRequest = read_json(req).await?; + json(&ctx.set_decrypt_list(&body.entries).map_err(bad_request)?) + } (&Method::GET, ["flows"]) => { let filter = FlowFilter::from_query(&query).map_err(bad_request)?; json(&flows::list(&ctx.db()?, &filter)?) diff --git a/crates-proxy/yaak-proxy-lib/src/db.rs b/crates-proxy/yaak-proxy-lib/src/db.rs index 13b556277..5cf01fcb7 100644 --- a/crates-proxy/yaak-proxy-lib/src/db.rs +++ b/crates-proxy/yaak-proxy-lib/src/db.rs @@ -25,6 +25,23 @@ pub const BLOBS_DB: &str = "blobs.sqlite"; /// The key both are encrypted with, wrapped by the app's master key in the keychain. pub const DATA_KEY: &str = "data.key"; +/// A pre-release build added `flows.tunnelled` in migration 8. A database it made has the +/// column without migration 9 recorded, which would fail adding it again, so it is recorded +/// as applied first. +fn record_early_tunnelled_column(pool: &SqlitePool) -> Result<()> { + let conn = pool.get()?; + let has_column = conn.prepare("SELECT tunnelled FROM flows LIMIT 0").is_ok(); + let has_tracking = conn.prepare("SELECT version FROM _sqlx_migrations LIMIT 0").is_ok(); + if has_column && has_tracking { + conn.execute( + "INSERT OR IGNORE INTO _sqlx_migrations (version, description) \ + VALUES ('00000009', '00000009_flow_tunnelled.sql')", + [], + )?; + } + Ok(()) +} + /// Reads come from a pool; writes go through one connection, as in yaak-models' /// `QueryManager`. WAL lets the CLI read `proxy.db` while the app writes it. #[derive(Clone)] @@ -42,6 +59,7 @@ impl ProxyQueryManager { } encryption::encrypt_in_place(db_path, key, &|_| {})?; let writer = file_pool(db_path, 1, key)?; + record_early_tunnelled_column(&writer)?; run_migrations(&writer, &MIGRATIONS)?; Ok(Self { readers: file_pool(db_path, 5, key)?, writer }) } @@ -512,7 +530,7 @@ mod tests { .unwrap() .map(|v| v.unwrap()) .collect(); - assert_eq!(&applied[applied.len() - 2..], ["00000007", "00000008"]); + assert_eq!(&applied[applied.len() - 3..], ["00000007", "00000008", "00000009"]); let mark_columns = columns(&conn, "flow_clear_mark"); assert_eq!(mark_columns, ["id", "max_rowid", "cleaner", "cleaner_at"]); drop(conn); @@ -532,4 +550,109 @@ mod tests { ); assert_eq!(crate::messages::count(&db, "fl_ws").unwrap(), 0); } + + /// A database a pre-release build made, with `tunnelled` added by its migration 8, opens + /// without adding the column twice. + #[test] + fn opens_a_database_whose_migration_8_added_tunnelled() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("proxy.db"); + let conn = Connection::open(&path).unwrap(); + conn.execute_batch( + "CREATE TABLE _sqlx_migrations ( + version TEXT PRIMARY KEY, + description TEXT NOT NULL, + applied_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL + );", + ) + .unwrap(); + let mut files: Vec<_> = MIGRATIONS + .files() + .map(|f| (f.path().to_string_lossy().into_owned(), f.contents_utf8().unwrap())) + .filter(|(name, _)| name.as_str() < "00000009") + .collect(); + files.sort(); + for (name, sql) in files { + conn.execute_batch(sql).unwrap(); + conn.execute( + "INSERT INTO _sqlx_migrations (version, description) VALUES (?, ?)", + [&name[..8], &name], + ) + .unwrap(); + } + conn.execute_batch( + "ALTER TABLE flows ADD COLUMN tunnelled BOOLEAN NOT NULL DEFAULT FALSE;", + ) + .unwrap(); + drop(conn); + + let db = ProxyQueryManager::new(&path, &crate::db::test_key()).unwrap(); + let tunnel = crate::models::Flow { + id: "fl_tunnel".into(), + url: "https://a.com".into(), + tunnelled: true, + ..Default::default() + }; + db.with_tx(|c| c.upsert(&tunnel, &yaak_database::UpdateSource::Background).map(|_| ())) + .unwrap(); + assert!(crate::flows::find(&db, "fl_tunnel").unwrap().tunnelled); + drop(db); + // And again, with everything recorded. + drop(ProxyQueryManager::new(&path, &crate::db::test_key()).unwrap()); + } + + /// A database that ran every migration up to the clear cleaner's, as the build before + /// tunnelled flows left it, takes the `tunnelled` column and stores tunnelled flows. + #[test] + fn adds_tunnelled_to_a_database_at_migration_8() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("proxy.db"); + let conn = Connection::open(&path).unwrap(); + conn.execute_batch( + "CREATE TABLE _sqlx_migrations ( + version TEXT PRIMARY KEY, + description TEXT NOT NULL, + applied_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL + );", + ) + .unwrap(); + let mut files: Vec<_> = MIGRATIONS + .files() + .map(|f| (f.path().to_string_lossy().into_owned(), f.contents_utf8().unwrap())) + .filter(|(name, _)| name.as_str() < "00000009") + .collect(); + files.sort(); + assert_eq!(files.len(), 8); + for (name, sql) in files { + conn.execute_batch(sql).unwrap(); + conn.execute( + "INSERT INTO _sqlx_migrations (version, description) VALUES (?, ?)", + [&name[..8], &name], + ) + .unwrap(); + } + conn.execute( + "INSERT INTO flows (id, url, method) VALUES ('fl_old', 'https://a.com/', 'GET')", + [], + ) + .unwrap(); + assert!(!columns(&conn, "flows").contains(&"tunnelled".to_string())); + drop(conn); + + let db = ProxyQueryManager::new(&path, &crate::db::test_key()).unwrap(); + let tunnel = crate::models::Flow { + id: "fl_tunnel".into(), + method: "CONNECT".into(), + url: "https://bank.example.com".into(), + tunnelled: true, + ..Default::default() + }; + db.with_tx(|c| c.upsert(&tunnel, &yaak_database::UpdateSource::Background).map(|_| ())) + .unwrap(); + let filter = crate::flows::FlowFilter { tunnelled: Some(true), ..Default::default() }; + let listed = crate::flows::list(&db, &filter).unwrap(); + assert_eq!(listed.iter().map(|f| f.id.as_str()).collect::>(), ["fl_tunnel"]); + let old = crate::flows::find(&db, "fl_old").unwrap(); + assert!(!old.tunnelled); + } } diff --git a/crates-proxy/yaak-proxy-lib/src/flows.rs b/crates-proxy/yaak-proxy-lib/src/flows.rs index 565dad4aa..11bae5f43 100644 --- a/crates-proxy/yaak-proxy-lib/src/flows.rs +++ b/crates-proxy/yaak-proxy-lib/src/flows.rs @@ -35,6 +35,9 @@ pub struct FlowFilter { /// tells them by their executable. #[ts(optional)] pub system: Option, + /// Only HTTPS connections tunnelled undecrypted (true), or none of them (false). + #[ts(optional)] + pub tunnelled: Option, /// Only flows one of these groups covers, each judged on its own: several sidebar rows /// selected, `@app:Slack @host:a.com OR @app:Chrome @host:b.com`, are two groups and not every /// pairing of their apps and hosts. A group's `after`, `before` and `limit` do not apply. Not @@ -78,6 +81,11 @@ impl FlowFilter { .map(|v| v.parse().map_err(|_| format!("Invalid system {v:?}"))) .transpose()? } + "tunnelled" => { + filter.tunnelled = value + .map(|v| v.parse().map_err(|_| format!("Invalid tunnelled {v:?}"))) + .transpose()? + } "method" => filter.method = value, "status" => filter.status = value, "after" => filter.after = value, @@ -97,7 +105,13 @@ impl FlowFilter { let mut query = url::form_urlencoded::Serializer::new(String::new()); let limit = self.limit.map(|l| l.to_string()); let system = self.system.map(|s| s.to_string()); - for (key, value) in [("q", &self.q), ("host", &self.host), ("system", &system)] { + let tunnelled = self.tunnelled.map(|t| t.to_string()); + for (key, value) in [ + ("q", &self.q), + ("host", &self.host), + ("system", &system), + ("tunnelled", &tunnelled), + ] { if let Some(value) = value { query.append_pair(key, value); } @@ -131,6 +145,7 @@ impl FlowFilter { || self.under.is_some() || self.app.is_some() || self.system.is_some() + || self.tunnelled.is_some() || self.any.as_ref().is_some_and(|groups| groups.iter().any(FlowFilter::narrows)) } @@ -138,12 +153,17 @@ impl FlowFilter { /// for the database `conn` reads, which may be from before the folder or gRPC columns, when /// read without migrating. fn condition_in(&self, conn: &rusqlite::Connection) -> (String, Vec) { - self.condition(has_column(conn, "client_cwd"), has_column(conn, "grpc_status")) + let columns = Columns { + cwd: has_column(conn, "client_cwd"), + grpc: has_column(conn, "grpc_status"), + tunnelled: has_column(conn, "tunnelled"), + }; + self.condition(columns) } - /// [`Self::condition_in`], told which of the later columns the database has: `has_cwd`, - /// the folder a program ran in, and `grpc`, gRPC statuses. - fn condition(&self, has_cwd: bool, grpc: bool) -> (String, Vec) { + /// [`Self::condition_in`], told which of the later columns the database has. + fn condition(&self, columns: Columns) -> (String, Vec) { + let Columns { cwd: has_cwd, grpc, tunnelled } = columns; let mut sql = String::from("1"); let mut params: Vec = Vec::new(); for term in self.q.iter().flat_map(|q| q.split_whitespace()) { @@ -209,11 +229,18 @@ impl FlowFilter { Some(false) => sql.push_str(" AND NOT is_system(client_bundle_id, client_path)"), None => {} } + match (self.tunnelled, tunnelled) { + (Some(true), true) => sql.push_str(" AND tunnelled"), + (Some(false), true) => sql.push_str(" AND NOT tunnelled"), + // A database from before tunnelled flows has none. + (Some(true), false) => sql.push_str(" AND 0"), + _ => {} + } // Each group whole, then any of them. if let Some(groups) = self.any.as_ref().filter(|g| !g.is_empty()) { let mut any = Vec::new(); for group in groups { - let (condition, group_params) = group.condition(has_cwd, grpc); + let (condition, group_params) = group.condition(columns); any.push(format!("({condition})")); params.extend(group_params); } @@ -245,6 +272,17 @@ impl FlowFilter { } } +/// Which of the columns later migrations added a database has, as one opened read-only before +/// they ran may not. +#[derive(Clone, Copy)] +struct Columns { + /// The folder a program ran in. + cwd: bool, + /// gRPC statuses. + grpc: bool, + tunnelled: bool, +} + /// The host and port of `url`, in SQL: what follows `://` up to the first `/`, `?` or `#`. const HOST_SQL: &str = "(SELECT CASE WHEN instr(r, '/') > 0 THEN substr(r, 1, instr(r, '/') - 1) \ ELSE r END FROM (SELECT replace(replace(CASE WHEN instr(url, '://') > 0 \ @@ -647,6 +685,73 @@ pub fn get(db: &ProxyQueryManager, flow_id: &str) -> Result { find(db, flow_id).ok_or_else(|| format!("No flow {flow_id}")) } +/// The hosts and apps flows have gone to and come from, for the decrypt list's suggestions. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct SeenTargets { + /// Hosts as the sidebar names them, with a port other than the scheme's: most recent first. + pub hosts: Vec, + /// Apps by name and bundle id, most recent first. + pub apps: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct SeenApp { + pub name: String, + pub bundle_id: Option, +} + +/// Distinct hosts and apps listed by [`seen`] at most. +const SEEN_LIMIT: i64 = 1000; + +/// The distinct hosts and apps of the flows not cleared, most recent first, up to +/// [`SEEN_LIMIT`] of each. +pub fn seen(db: &ProxyQueryManager) -> Result { + db.with_conn(|c| { + let conn = c.conn().resolve(); + let mut params: Vec = Vec::new(); + let mut cleared = String::new(); + push_clear_mark(conn, &mut cleared, &mut params); + params.push(SEEN_LIMIT.into()); + // The newest flow of each host, whose URL names it as the sidebar does. + let sql = format!( + "SELECT url FROM flows WHERE rowid IN (SELECT max(rowid) FROM flows WHERE 1{cleared} \ + GROUP BY lower({HOST_SQL})) ORDER BY rowid DESC LIMIT ?" + ); + let mut stmt = conn.prepare(&sql).map_err(|e| e.to_string())?; + let urls = stmt + .query_map(rusqlite::params_from_iter(params.clone()), |r| r.get::<_, String>(0)) + .map_err(|e| e.to_string())? + .collect::, _>>() + .map_err(|e| e.to_string())?; + let mut hosts: Vec = Vec::new(); + for url in urls { + let path = sidebar_path(&url); + let host = path.split('/').next().unwrap_or_default().to_string(); + if !host.is_empty() && !hosts.contains(&host) { + hosts.push(host); + } + } + + let sql = format!( + "SELECT client_name, client_bundle_id FROM flows WHERE client_name IS NOT NULL{cleared} \ + GROUP BY client_name, client_bundle_id ORDER BY max(rowid) DESC LIMIT ?" + ); + let mut stmt = conn.prepare(&sql).map_err(|e| e.to_string())?; + let apps = stmt + .query_map(rusqlite::params_from_iter(params), |r| { + Ok(SeenApp { name: r.get(0)?, bundle_id: r.get(1)? }) + }) + .map_err(|e| e.to_string())? + .collect::, _>>() + .map_err(|e| e.to_string())?; + Ok(SeenTargets { hosts, apps }) + }) +} + pub fn count(db: &ProxyQueryManager) -> Result { db.with_conn(|c| count_in(c.conn().resolve())) } @@ -1326,6 +1431,12 @@ mod tests { assert_eq!(FlowFilter::from_query(&filter.to_query()).unwrap(), filter); assert_eq!(FlowFilter::from_query("").unwrap(), FlowFilter::default()); assert!(FlowFilter::from_query("limit=lots").is_err()); + + let tunnelled = FlowFilter { tunnelled: Some(true), ..Default::default() }; + assert_eq!(tunnelled.to_query(), "tunnelled=true"); + assert_eq!(FlowFilter::from_query("tunnelled=true").unwrap(), tunnelled); + assert!(tunnelled.narrows()); + assert!(FlowFilter::from_query("tunnelled=maybe").is_err()); } #[test] diff --git a/crates-proxy/yaak-proxy-lib/src/lib.rs b/crates-proxy/yaak-proxy-lib/src/lib.rs index f2fd75f3a..5411f408b 100644 --- a/crates-proxy/yaak-proxy-lib/src/lib.rs +++ b/crates-proxy/yaak-proxy-lib/src/lib.rs @@ -29,7 +29,7 @@ use crate::macos::{SystemProxyBackend, SystemProxyStatus}; use crate::messages::FlowMessage; use crate::models::{Flow, FlowBody, ModelPayload, ProxyHeader, StreamKind, find_header}; use crate::save::{ClientDb, SaveFlowRequest, SaveFlowResponse, WorkspaceSummary}; -use crate::settings::{HttpVersion, ProxySettings}; +use crate::settings::{DecryptList, HttpVersion, ProxySettings}; use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64; use chrono::{DateTime, Utc}; @@ -80,6 +80,9 @@ pub struct ProxyCtx { /// This context's name as the one deleting cleared flows, which the database leases to one /// process at a time. cleaner_id: String, + /// Held through each change to the decrypt list, so concurrent edits apply in turn, each to + /// the list the last one left. + decrypt_edits: Arc>, /// Where the master key that unwraps the data key is kept: the keychain, normally. keys: MasterKeySource, /// The data key, once read. Held while it is read and the databases converted and opened, @@ -127,6 +130,7 @@ impl ProxyCtx { start_error: Arc::new(Mutex::new(None)), deleting_cleared: Arc::new(Mutex::new(false)), cleaner_id: yaak_database::generate_prefixed_id("cln"), + decrypt_edits: Arc::new(Mutex::new(())), keys, unlocked: Arc::new(Mutex::new(None)), encrypting: Arc::new(Mutex::new(None)), @@ -370,9 +374,15 @@ impl ProxyCtx { let mut config = ProxyConfig::new(port.unwrap_or(settings.port), Arc::new(ca)); config.capture_body_limit = settings.capture_body_limit; config.http2 = settings.http_version == HttpVersion::Auto; + config.decrypt = settings.decrypt_policy(); // Filled once the engine has bound, since the lookup matches sockets by its port. let attribution_slot: Arc> = Arc::new(OnceLock::new()); let slot = attribution_slot.clone(); + config.app_of = Some(Arc::new(move |connection| { + let app = slot.get()?.get(connection, ATTRIBUTION_WAIT)?; + Some(yaak_proxy::DecryptApp { name: app.name, bundle_id: app.bundle_id }) + })); + let slot = attribution_slot.clone(); config.on_accept = Some(Arc::new(move |connection, client_addr| { // A client that is answered quickly may exit soon after, so it is found before // anything is read from it: a few milliseconds once per connection at most. @@ -430,11 +440,17 @@ impl ProxyCtx { return; }; let ended = db.with_tx(|c| { - c.conn().resolve().execute( + let conn = c.conn().resolve(); + conn.execute( "UPDATE flows SET error = ?1 WHERE stream_kind IS NOT NULL \ AND elapsed_ms IS NULL AND error IS NULL AND NOT cancelled", [STOPPED_MID_STREAM], )?; + conn.execute( + "UPDATE flows SET error = ?1 WHERE tunnelled \ + AND elapsed_ms IS NULL AND error IS NULL", + [STOPPED_MID_TUNNEL], + )?; Ok::<_, yaak_database::Error>(()) }); if let Err(e) = ended { @@ -687,6 +703,47 @@ impl ProxyCtx { Ok(()) } + /// What HTTPS is decrypted, as `settings.json` has it. + pub fn decrypt_list(&self) -> Result { + Ok(self.settings()?.decrypt_list()) + } + + /// Replaces the decrypt list, saved in `settings.json` and applied to the running engine's + /// next connections. Connections already open stay as they were. + pub fn set_decrypt_list(&self, entries: &[String]) -> Result { + self.change_decrypt_list(|settings| settings.set_decrypt_list(entries)) + } + + /// Adds and removes entries in the list as it is now, so edits made together each keep + /// the other's, unlike replacing the whole list from a copy read earlier. + pub fn edit_decrypt_list( + &self, + add: &[String], + remove: &[String], + ) -> Result { + self.change_decrypt_list(|settings| settings.edit_decrypt_list(add, remove)) + } + + /// Changes the list in `settings.json`, one change at a time, applies it to the running + /// engine, and tells the app. + fn change_decrypt_list( + &self, + change: impl FnOnce(&mut ProxySettings) -> Result<(), String>, + ) -> Result { + let _turn = self.decrypt_edits.lock().map_err(|_| "lock poisoned".to_string())?; + let mut settings = self.settings()?; + change(&mut settings)?; + settings.save(&self.settings_path())?; + if let Ok(handle) = self.handle.lock() + && let Some(handle) = handle.as_ref() + { + handle.set_decrypt_policy(settings.decrypt_policy()); + } + let list = settings.decrypt_list(); + self.events.emit("decrypt_list_changed", &DecryptListChangedPayload { list: list.clone() }); + Ok(list) + } + /// Intercepts again the hosts passed through after clients rejected the CA, as they no /// longer will once it is trusted. pub fn forget_passthrough(&self) { @@ -947,6 +1004,13 @@ pub enum ProxyState { Stopped, } +/// The decrypt list changed, here or through the control API. +#[derive(Serialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct DecryptListChangedPayload { + pub list: DecryptList, +} + #[derive(Serialize, TS)] #[ts(export, export_to = "gen_rpc.ts")] pub struct ProxyStatePayload { @@ -1090,6 +1154,32 @@ pub struct SetHttpVersionRequest { pub http_version: HttpVersion, } +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct GetDecryptListRequest {} + +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct SetDecryptListRequest { + /// Every entry, in order, as typed: a bare hostname is a `host:` entry. A built-in + /// exclusion left out is removed. + pub entries: Vec, +} + +/// Entries to add after the user's own and to remove, applied to the list as it is when the +/// change lands. Each is normalized first. +#[derive(Debug, Clone, Default, Serialize, Deserialize, TS)] +#[serde(default)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct EditDecryptListRequest { + pub add: Vec, + pub remove: Vec, +} + +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct ListSeenRequest {} + #[derive(Deserialize, TS)] #[ts(export, export_to = "gen_rpc.ts")] pub struct AppIconRequest { @@ -1498,6 +1588,42 @@ fn set_http_version(ctx: &ProxyCtx, req: SetHttpVersionRequest) -> Result Result { + ctx.decrypt_list().map_err(rpc_error) +} + +/// Saved and applied here, or, while another process such as the CLI daemon owns the engine, +/// there, as its engine is the one that reads it. +async fn set_decrypt_list( + ctx: ProxyCtx, + req: SetDecryptListRequest, +) -> Result { + if ctx.port().is_none() + && let Some(owner) = ctx.other_owner() + { + return ControlClient::new(owner).set_decrypt_list(&req.entries).await.map_err(rpc_error); + } + blocking(move || ctx.set_decrypt_list(&req.entries)).await +} + +/// [`set_decrypt_list`], adding and removing entries rather than replacing them all. +async fn edit_decrypt_list( + ctx: ProxyCtx, + req: EditDecryptListRequest, +) -> Result { + if ctx.port().is_none() + && let Some(owner) = ctx.other_owner() + { + return ControlClient::new(owner).edit_decrypt_list(&req).await.map_err(rpc_error); + } + blocking(move || ctx.edit_decrypt_list(&req.add, &req.remove)).await +} + +async fn list_seen(ctx: ProxyCtx, _req: ListSeenRequest) -> Result { + let db = ctx.db()?; + blocking(move || flows::seen(&db)).await +} + async fn take_over(ctx: ProxyCtx, _req: TakeOverRequest) -> Result { ctx.take_over().await.map_err(rpc_error)?; Ok(true) @@ -1802,6 +1928,9 @@ const DUE_CHECK_INTERVAL: Duration = Duration::from_millis(50); /// Why a stream still open when the engine stopped ended. const STOPPED_MID_STREAM: &str = "Yaak Proxy stopped before the stream ended"; +/// Why a tunnel still open when the engine stopped ended. +const STOPPED_MID_TUNNEL: &str = "Yaak Proxy stopped before the connection closed"; + /// A flow between its `RequestStart` and its end, with its bodies being written as they /// stream. struct InFlight { @@ -1956,7 +2085,8 @@ fn run_event_loop(rx: std::sync::mpsc::Receiver, sink: FlowSink) { // as they always have been. Every flow that ended is written. for (_, mut f) in in_flight.drain() { if f.persisted { - f.flow.error = Some(STOPPED_MID_STREAM.to_string()); + let why = if f.flow.tunnelled { STOPPED_MID_TUNNEL } else { STOPPED_MID_STREAM }; + f.flow.error = Some(why.to_string()); sink.finish_flow(f); } } @@ -2061,6 +2191,44 @@ impl FlowSink { }, ); } + ProxyEvent::TunnelStart { id, started_at, authority, connection, client_addr } => { + if !recording.lock().map(|r| r.was_on_at(started_at)).unwrap_or(true) { + return; + } + attribution.begin(connection, client_addr); + let flow = Flow { + id: Flow::generate_id(), + started_at: DateTime::::from(started_at).naive_utc(), + method: "CONNECT".to_string(), + url: format!("https://{authority}"), + client_http_version: "HTTP/1.1".to_string(), + tunnelled: true, + ..Default::default() + }; + let mut f = InFlight { + flow, + connection, + request_body: None, + response_body: None, + persisted: false, + cleared: false, + next_seq: 0, + dirty: false, + last_write: None, + }; + // Shown while it is open, as tunnels often stay open for minutes. + self.write_stream_flow(&mut f); + in_flight.insert(id, f); + } + ProxyEvent::TunnelEnd { id, sent, received, elapsed_ms, error } => { + if let Some(mut f) = in_flight.remove(&id) { + f.flow.request_size = sent as i64; + f.flow.response_size = received as i64; + f.flow.elapsed_ms = Some(clamp_ms(elapsed_ms)); + f.flow.error = error; + self.finish_flow(f); + } + } ProxyEvent::StreamStart { id, kind } => { if let Some(f) = in_flight.get_mut(&id) { let kind = StreamKind::from(kind); @@ -2479,6 +2647,7 @@ define_rpc! { commands { execute_action(ActionInvocation) -> bool, format_body(FormatBodyRequest) -> String, + get_decrypt_list(GetDecryptListRequest) -> DecryptList, get_message_payload(GetMessagePayloadRequest) -> GetMessagePayloadResponse, list_actions(ListActionsRequest) -> ListActionsResponse, list_flows(flows::FlowFilter) -> ListFlowsResponse, @@ -2502,6 +2671,9 @@ define_rpc! { get_proxy_state(GetProxyStateRequest) -> GetProxyStateResponse, helper_register(HelperRegisterRequest) -> SystemProxyStatus, helper_unregister(HelperUnregisterRequest) -> SystemProxyStatus, + edit_decrypt_list(EditDecryptListRequest) -> DecryptList, + list_seen(ListSeenRequest) -> flows::SeenTargets, + set_decrypt_list(SetDecryptListRequest) -> DecryptList, list_workspaces(ListWorkspacesRequest) -> ListWorkspacesResponse, preview_save(PreviewSaveRequest) -> SavePreview, reflect_grpc_host(ReflectGrpcHostRequest) -> GrpcSchema, @@ -2512,6 +2684,7 @@ define_rpc! { take_over(TakeOverRequest) -> bool, } events { + decrypt_list_changed(DecryptListChangedPayload), flows_cleared(FlowsClearedPayload), message_write(MessagePayload), model_write(ModelPayload), diff --git a/crates-proxy/yaak-proxy-lib/src/models.rs b/crates-proxy/yaak-proxy-lib/src/models.rs index 90a9c2d99..ceedbb3be 100644 --- a/crates-proxy/yaak-proxy-lib/src/models.rs +++ b/crates-proxy/yaak-proxy-lib/src/models.rs @@ -91,6 +91,10 @@ pub struct Flow { pub grpc_status: Option, /// A gRPC call's `grpc-message`, decoded. pub grpc_message: Option, + /// An HTTPS connection passed through undecrypted, as the decrypt list leaves it out or its + /// client rejected the proxy's certificate: method `CONNECT`, its URL the host, its sizes + /// the bytes each way, and no headers or bodies. Open until `elapsedMs` or `error` is set. + pub tunnelled: bool, } /// The streaming protocol a flow carries. @@ -239,6 +243,7 @@ impl UpsertModelInfo for Flow { (GrpcMethod, self.grpc_method.into()), (GrpcStatus, self.grpc_status.into()), (GrpcMessage, self.grpc_message.into()), + (Tunnelled, self.tunnelled.into()), ]) } @@ -275,6 +280,7 @@ impl UpsertModelInfo for Flow { FlowIden::GrpcMethod, FlowIden::GrpcStatus, FlowIden::GrpcMessage, + FlowIden::Tunnelled, ] } @@ -320,6 +326,7 @@ impl UpsertModelInfo for Flow { grpc_method: added_column(r, "grpc_method")?, grpc_status: added_column(r, "grpc_status")?, grpc_message: added_column(r, "grpc_message")?, + tunnelled: added_column(r, "tunnelled")?.unwrap_or_default(), }) } } diff --git a/crates-proxy/yaak-proxy-lib/src/settings.rs b/crates-proxy/yaak-proxy-lib/src/settings.rs index 59159ed3f..28f7a1363 100644 --- a/crates-proxy/yaak-proxy-lib/src/settings.rs +++ b/crates-proxy/yaak-proxy-lib/src/settings.rs @@ -1,7 +1,7 @@ use serde::{Deserialize, Serialize}; use std::path::Path; use ts_rs::TS; -use yaak_proxy::DEFAULT_CAPTURE_BODY_LIMIT; +use yaak_proxy::{BUILT_IN_EXCLUSIONS, DEFAULT_CAPTURE_BODY_LIMIT, DecryptEntry, DecryptPolicy}; /// `settings.json` in the proxy data directory. Missing keys take their defaults. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -17,6 +17,12 @@ pub struct ProxySettings { pub capture_body_limit: usize, /// The HTTP versions offered to clients in intercepted HTTPS. pub http_version: HttpVersion, + /// The user's decrypt entries, in order: `host:api.example.com`, `app:Google Chrome`, `*`, + /// `!host:*.bank.com`. Empty, so HTTPS is tunnelled, until hosts or apps are added. The + /// built-in exclusions are not kept here; see [`Self::decrypt_list`]. + pub decrypt: Vec, + /// The built-in exclusions the user removed. + pub decrypt_built_ins_removed: Vec, } impl Default for ProxySettings { @@ -26,10 +32,32 @@ impl Default for ProxySettings { system_proxy_at_launch: true, capture_body_limit: DEFAULT_CAPTURE_BODY_LIMIT, http_version: HttpVersion::Auto, + decrypt: Vec::new(), + decrypt_built_ins_removed: Vec::new(), } } } +/// One entry of the decrypt list, as the app shows it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct DecryptListEntry { + /// As stored: `host:*.example.com`, `app:Google Chrome`, `!host:*.apple.com`, `*`. + pub entry: String, + /// Shipped with the app: an exclusion of Apple's services or a pinned app's hosts. + pub built_in: bool, +} + +/// What HTTPS is decrypted: the user's entries in order, then the built-in exclusions not +/// removed. A connection is decrypted when a positive entry matches it and no exclusion does. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct DecryptList { + pub entries: Vec, +} + /// The HTTP versions offered to clients in intercepted HTTPS. Upstream, the engine speaks /// whichever the server picks either way. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, TS)] @@ -44,6 +72,84 @@ pub enum HttpVersion { } impl ProxySettings { + /// The user's entries, then the built-in exclusions they kept. + pub fn decrypt_list(&self) -> DecryptList { + let user = + self.decrypt.iter().map(|e| DecryptListEntry { entry: e.clone(), built_in: false }); + let built_in = BUILT_IN_EXCLUSIONS + .iter() + .filter(|e| !self.decrypt_built_ins_removed.iter().any(|r| r == *e)) + .map(|e| DecryptListEntry { entry: e.to_string(), built_in: true }); + DecryptList { entries: user.chain(built_in).collect() } + } + + /// Replaces the list with `entries`, as the user may type them: each is normalized, a bare + /// hostname becoming `host:`, and repeats are dropped. A built-in exclusion left out is + /// removed, and one written again is restored. Fails, changing nothing, on an entry that + /// does not parse. + pub fn set_decrypt_list>(&mut self, entries: &[S]) -> Result<(), String> { + let mut user: Vec = Vec::new(); + let mut kept_built_ins: Vec = Vec::new(); + for entry in entries { + let parsed = DecryptEntry::parse(entry.as_ref())?; + let normalized = parsed.to_string(); + let list = if parsed.built_in { &mut kept_built_ins } else { &mut user }; + if !list.contains(&normalized) { + list.push(normalized); + } + } + self.decrypt = user; + self.decrypt_built_ins_removed = BUILT_IN_EXCLUSIONS + .iter() + .filter(|e| !kept_built_ins.iter().any(|k| k == *e)) + .map(|e| e.to_string()) + .collect(); + Ok(()) + } + + /// Removes `remove` from the list as it is and adds `add` after the user's own entries, + /// each normalized, as [`Self::set_decrypt_list`] would take the result. A built-in removed + /// is remembered as removed, and one added is restored. Fails, changing nothing, on an + /// entry that does not parse. + pub fn edit_decrypt_list>( + &mut self, + add: &[S], + remove: &[S], + ) -> Result<(), String> { + let normalize = |e: &str| DecryptEntry::parse(e).ok().map(|p| p.to_string()); + let add = add + .iter() + .map(|e| DecryptEntry::parse(e.as_ref()).map(|p| p.to_string())) + .collect::, _>>()?; + // An entry goes as stored, so one edited by hand, odd case or malformed, can go too. + let removed = |stored: &str| { + remove.iter().any(|r| { + let r = r.as_ref(); + r == stored || normalize(r).is_some_and(|r| Some(r) == normalize(stored)) + }) + }; + let mut entries: Vec = self + .decrypt_list() + .entries + .into_iter() + .map(|e| e.entry) + .filter(|e| !removed(e)) + .collect(); + entries.extend(add); + self.set_decrypt_list(&entries) + } + + /// The engine's policy from the list. An entry saved by hand that does not parse is left + /// out, with a warning, rather than stopping the engine. + pub fn decrypt_policy(&self) -> DecryptPolicy { + let entries = self.decrypt_list().entries.into_iter().filter_map(|e| { + DecryptEntry::parse(&e.entry) + .inspect_err(|err| log::warn!("Ignoring decrypt entry: {err}")) + .ok() + }); + DecryptPolicy { entries: entries.collect() } + } + /// The settings at `path`, or the defaults when there is no file. pub fn load(path: &Path) -> Result { match std::fs::read_to_string(path) { @@ -92,6 +198,8 @@ mod tests { system_proxy_at_launch: false, capture_body_limit: 1024, http_version: HttpVersion::Http1Only, + decrypt: vec!["host:api.example.com".into()], + decrypt_built_ins_removed: vec!["!host:*.zoom.us".into()], }; settings.save(&path).unwrap(); assert_eq!(ProxySettings::load(&path).unwrap(), settings); @@ -105,6 +213,8 @@ mod tests { "systemProxyAtLaunch": false, "captureBodyLimit": 1024, "httpVersion": "http1Only", + "decrypt": ["host:api.example.com"], + "decryptBuiltInsRemoved": ["!host:*.zoom.us"], }) ); } @@ -137,6 +247,90 @@ mod tests { assert!(json.get("systemProxy").is_none()); } + #[test] + fn the_decrypt_list_starts_with_only_the_built_in_exclusions() { + let settings = ProxySettings::default(); + let list = settings.decrypt_list(); + assert!(!list.entries.is_empty()); + assert!(list.entries.iter().all(|e| e.built_in && e.entry.starts_with('!'))); + let policy = settings.decrypt_policy(); + assert!(!policy.decrypts_anything(), "nothing is decrypted by default"); + } + + #[test] + fn setting_the_decrypt_list() { + let mut settings = ProxySettings::default(); + let mut entries: Vec = vec![ + "api.example.com".into(), + "*".into(), + "host:API.example.com".into(), + "app:Google Chrome".into(), + ]; + // Every built-in exclusion but zoom.us, kept. + entries.extend( + BUILT_IN_EXCLUSIONS.iter().filter(|e| **e != "!host:*.zoom.us").map(|e| e.to_string()), + ); + settings.set_decrypt_list(&entries).unwrap(); + assert_eq!(settings.decrypt, ["host:api.example.com", "*", "app:Google Chrome"]); + assert_eq!(settings.decrypt_built_ins_removed, ["!host:*.zoom.us"]); + let list = settings.decrypt_list(); + assert_eq!( + list.entries[0], + DecryptListEntry { entry: "host:api.example.com".into(), built_in: false } + ); + assert!(list.entries[3].built_in); + assert!(!list.entries.iter().any(|e| e.entry == "!host:*.zoom.us")); + + // Written again, a built-in exclusion is restored. + let mut again: Vec = list.entries.iter().map(|e| e.entry.clone()).collect(); + again.push("!host:*.zoom.us".into()); + settings.set_decrypt_list(&again).unwrap(); + assert!(settings.decrypt_built_ins_removed.is_empty()); + + // An invalid entry changes nothing. + let before = settings.clone(); + assert!(settings.set_decrypt_list(&["host:a b"]).is_err()); + assert_eq!(settings, before); + } + + #[test] + fn editing_applies_to_the_list_as_it_is() { + let mut settings = + ProxySettings { decrypt: vec!["host:a.com".into()], ..Default::default() }; + // Two edits made from the same earlier copy both land. + settings.edit_decrypt_list(&["b.com"], &[]).unwrap(); + settings.edit_decrypt_list(&["app:Slack"], &["host:a.com"]).unwrap(); + assert_eq!(settings.decrypt, ["host:b.com", "app:Slack"]); + // A built-in removed and restored. + settings.edit_decrypt_list(&[], &["!host:*.zoom.us"]).unwrap(); + assert_eq!(settings.decrypt_built_ins_removed, ["!host:*.zoom.us"]); + settings.edit_decrypt_list(&["!host:*.zoom.us"], &[]).unwrap(); + assert!(settings.decrypt_built_ins_removed.is_empty()); + let before = settings.clone(); + assert!(settings.edit_decrypt_list(&["host:a b"], &[]).is_err()); + assert_eq!(settings, before); + } + + #[test] + fn entries_edited_by_hand_can_be_removed() { + let mut settings = ProxySettings { + decrypt: vec!["API.example.com".into(), "host:a b".into(), "b.com".into()], + ..Default::default() + }; + settings.edit_decrypt_list(&[], &["host:api.example.com", "host:a b"]).unwrap(); + assert_eq!(settings.decrypt, ["host:b.com"]); + } + + #[test] + fn a_hand_edited_entry_that_does_not_parse_is_skipped() { + let settings = ProxySettings { + decrypt: vec!["host:a b".into(), "x.com".into()], + ..Default::default() + }; + let policy = settings.decrypt_policy(); + assert_eq!(policy.entries.iter().filter(|e| !e.negated).count(), 1); + } + #[test] fn invalid_json_is_an_error() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates-proxy/yaak-proxy-lib/tests/control.rs b/crates-proxy/yaak-proxy-lib/tests/control.rs index ba22e7fca..65185eaff 100644 --- a/crates-proxy/yaak-proxy-lib/tests/control.rs +++ b/crates-proxy/yaak-proxy-lib/tests/control.rs @@ -772,3 +772,112 @@ async fn another_app_owning_the_engine_is_not_taken_over() { assert_eq!(app.port(), None); other.stop().unwrap(); } + +#[tokio::test(flavor = "multi_thread")] +async fn the_decrypt_list_is_read_and_replaced_over_the_control_api() { + let dir = tempfile::tempdir().unwrap(); + let client_dir = tempfile::tempdir().unwrap(); + let ctx = ctx(dir.path(), client_dir.path()); + let _server = ControlServer::start(ctx.clone()).unwrap(); + let info = control::read(dir.path()).unwrap(); + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let url = format!("{}/decrypt", info.base_url()); + + assert_eq!(http.get(&url).send().await.unwrap().status(), 401); + let list: Value = + http.get(&url).bearer_auth(&info.token).send().await.unwrap().json().await.unwrap(); + assert!(list["entries"].as_array().unwrap().iter().all(|e| e["builtIn"] == true)); + + let put = http + .put(&url) + .bearer_auth(&info.token) + .json(&json!({"entries": ["*", "!host:*.bank.example"]})) + .send() + .await + .unwrap(); + assert_eq!(put.status(), 200); + let list: Value = put.json().await.unwrap(); + assert_eq!( + list["entries"], + json!([ + {"entry": "*", "builtIn": false}, + {"entry": "!host:*.bank.example", "builtIn": false}, + ]), + "every built-in exclusion left out is removed" + ); + let read: Value = + http.get(&url).bearer_auth(&info.token).send().await.unwrap().json().await.unwrap(); + assert_eq!(read, list); + + let bad = http + .put(&url) + .bearer_auth(&info.token) + .json(&json!({"entries": ["app:"]})) + .send() + .await + .unwrap(); + assert_eq!(bad.status(), 400); +} + +/// While the CLI owns the engine, the app's change to the decrypt list goes to the CLI, whose +/// engine decrypts the next connection by it. +#[tokio::test(flavor = "multi_thread")] +async fn the_decrypt_list_reaches_the_cli_that_owns_the_engine() { + let dir = tempfile::tempdir().unwrap(); + let client_dir = tempfile::tempdir().unwrap(); + let cli = ctx(dir.path(), client_dir.path()).with_owner(Owner::Cli); + let _server = ControlServer::start(cli.clone()).unwrap(); + cli.start().unwrap(); + let app = ctx(dir.path(), client_dir.path()); + + let list = rpc(&app, "set_decrypt_list", json!({"entries": ["*"]})).await.unwrap(); + assert_eq!(list["entries"][0], json!({"entry": "*", "builtIn": false})); + + // Decrypted, the request is a flow of its own: a 502, as nothing listens there. + let ca = std::fs::read(dir.path().join("ca/ca.pem")).unwrap(); + let closed = TcpListener::bind("127.0.0.1:0").await.unwrap().local_addr().unwrap().port(); + let client = reqwest::Client::builder() + .proxy(reqwest::Proxy::all(format!("http://127.0.0.1:{}", cli.port().unwrap())).unwrap()) + .add_root_certificate(reqwest::Certificate::from_pem(&ca).unwrap()) + .build() + .unwrap(); + let url = format!("https://localhost:{closed}/forwarded"); + assert_eq!(client.get(&url).send().await.unwrap().status(), 502); + let deadline = Instant::now() + Duration::from_secs(5); + loop { + let flows = cli.db().unwrap().with_conn(|db| db.find_all::().unwrap()); + if let Some(flow) = flows.iter().find(|f| f.url == url) { + assert!(!flow.tunnelled); + break; + } + assert!(Instant::now() < deadline, "no decrypted flow for {url}"); + tokio::time::sleep(Duration::from_millis(20)).await; + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn the_decrypt_list_is_edited_over_the_control_api() { + let dir = tempfile::tempdir().unwrap(); + let client_dir = tempfile::tempdir().unwrap(); + let ctx = ctx(dir.path(), client_dir.path()); + let _server = ControlServer::start(ctx.clone()).unwrap(); + let info = control::read(dir.path()).unwrap(); + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let url = format!("{}/decrypt", info.base_url()); + + for add in ["a.com", "b.com"] { + let resp = + http.patch(&url).bearer_auth(&info.token).json(&json!({"add": [add]})).send().await; + assert_eq!(resp.unwrap().status(), 200); + } + let resp = http + .patch(&url) + .bearer_auth(&info.token) + .json(&json!({"remove": ["host:a.com"]})) + .send() + .await + .unwrap(); + let list: Value = resp.json().await.unwrap(); + assert_eq!(list["entries"][0], json!({"entry": "host:b.com", "builtIn": false})); + assert_eq!(list["entries"][1]["builtIn"], true); +} diff --git a/crates-proxy/yaak-proxy-lib/tests/flow.rs b/crates-proxy/yaak-proxy-lib/tests/flow.rs index 303c7904c..ceb04d904 100644 --- a/crates-proxy/yaak-proxy-lib/tests/flow.rs +++ b/crates-proxy/yaak-proxy-lib/tests/flow.rs @@ -267,7 +267,13 @@ async fn cancelled_flow_is_not_an_error() { #[tokio::test(flavor = "multi_thread")] async fn http_version_setting_applies_to_new_connections() { let dir = tempfile::tempdir().unwrap(); - let settings = ProxySettings { port: 0, system_proxy_at_launch: false, ..Default::default() }; + // Decrypted, as only intercepted HTTPS offers the client a version. + let settings = ProxySettings { + port: 0, + system_proxy_at_launch: false, + decrypt: vec!["*".into()], + ..Default::default() + }; settings.save(&dir.path().join("settings.json")).unwrap(); let (events, _event_rx) = RpcEventEmitter::new(); @@ -416,6 +422,213 @@ async fn body_rpcs() { assert_eq!(formatted, json!("{\n \"a\": [\n 1,\n 2\n ]\n}")); } +/// A TCP server that echoes what it reads, then closes once the client does. +async fn spawn_echo() -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + loop { + let (mut tcp, _) = listener.accept().await.unwrap(); + tokio::spawn(async move { + let (mut read, mut write) = tcp.split(); + let _ = tokio::io::copy(&mut read, &mut write).await; + }); + } + }); + addr +} + +/// Opens a CONNECT tunnel to `target` through the engine on `port`. +async fn connect_through(port: u16, target: &str) -> tokio::net::TcpStream { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let mut tcp = tokio::net::TcpStream::connect(("127.0.0.1", port)).await.unwrap(); + let connect = format!("CONNECT {target} HTTP/1.1\r\nHost: {target}\r\n\r\n"); + tcp.write_all(connect.as_bytes()).await.unwrap(); + let mut head = Vec::new(); + while !head.ends_with(b"\r\n\r\n") { + head.push(tcp.read_u8().await.unwrap()); + } + assert!(head.starts_with(b"HTTP/1.1 200"), "{}", String::from_utf8_lossy(&head)); + tcp +} + +async fn wait_for(ctx: &ProxyCtx, check: impl Fn(&Flow) -> bool) -> Flow { + let deadline = Instant::now() + Duration::from_secs(5); + loop { + let flows = ctx.db().unwrap().with_conn(|db| db.find_all::().unwrap()); + if let Some(flow) = flows.into_iter().find(|f| check(f)) { + return flow; + } + assert!(Instant::now() < deadline, "no such flow"); + tokio::time::sleep(Duration::from_millis(20)).await; + } +} + +/// With the default, empty decrypt list, a CONNECT is tunnelled untouched and recorded as a flow +/// with its host and the bytes each way, shown while open and finished once closed. +#[tokio::test(flavor = "multi_thread")] +async fn a_tunnelled_connect_is_recorded_with_its_sizes() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let dir = tempfile::tempdir().unwrap(); + let settings = ProxySettings { port: 0, system_proxy_at_launch: false, ..Default::default() }; + settings.save(&dir.path().join("settings.json")).unwrap(); + let (events, _event_rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + ctx.start().unwrap(); + let port = ctx.port().unwrap(); + let echo = spawn_echo().await; + let target = format!("localhost:{}", echo.port()); + let url = format!("https://{target}"); + + let mut tcp = connect_through(port, &target).await; + // TLS-looking bytes, which a decrypting engine would have taken for a ClientHello. + let sent = b"\x16\x03\x01 not decrypted"; + tcp.write_all(sent).await.unwrap(); + let mut echoed = vec![0u8; sent.len()]; + tcp.read_exact(&mut echoed).await.unwrap(); + assert_eq!(&echoed, sent, "bytes pass untouched"); + + let open = wait_for(&ctx, |f| f.url == url).await; + assert!(open.tunnelled); + assert_eq!(open.method, "CONNECT"); + assert_eq!(open.elapsed_ms, None, "open while the tunnel is"); + + drop(tcp); + let done = wait_for(&ctx, |f| f.url == url && f.elapsed_ms.is_some()).await; + assert_eq!(done.id, open.id); + assert_eq!((done.request_size, done.response_size), (sent.len() as i64, sent.len() as i64)); + assert_eq!(done.error, None); + assert!(done.request_headers.is_empty() && done.response_headers.is_empty()); + + let tunnelled = rpc(&ctx, "list_flows", json!({"tunnelled": true})).await.unwrap(); + assert_eq!(tunnelled["flows"].as_array().unwrap().len(), 1); + assert_eq!(tunnelled["flows"][0]["tunnelled"], true); + let decrypted = rpc(&ctx, "list_flows", json!({"tunnelled": false})).await.unwrap(); + assert_eq!(decrypted["flows"].as_array().unwrap().len(), 0); +} + +#[tokio::test(flavor = "multi_thread")] +async fn the_decrypt_list_rpcs_save_and_apply() { + let dir = tempfile::tempdir().unwrap(); + let settings = ProxySettings { port: 0, system_proxy_at_launch: false, ..Default::default() }; + settings.save(&dir.path().join("settings.json")).unwrap(); + let (events, _event_rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + + let list = rpc(&ctx, "get_decrypt_list", json!({})).await.unwrap(); + let entries = list["entries"].as_array().unwrap(); + assert!(entries.iter().all(|e| e["builtIn"] == true), "only the built-ins: {list}"); + assert!(entries.iter().any(|e| e["entry"] == "!host:*.apple.com")); + + let set = json!({"entries": ["API.example.com", "app:Google Chrome", "!host:*.apple.com"]}); + let list = rpc(&ctx, "set_decrypt_list", set).await.unwrap(); + assert_eq!( + list["entries"], + json!([ + {"entry": "host:api.example.com", "builtIn": false}, + {"entry": "app:Google Chrome", "builtIn": false}, + {"entry": "!host:*.apple.com", "builtIn": true}, + ]) + ); + let saved = ProxySettings::load(&dir.path().join("settings.json")).unwrap(); + assert_eq!(saved.decrypt, ["host:api.example.com", "app:Google Chrome"]); + assert!(saved.decrypt_built_ins_removed.contains(&"!host:*.icloud.com".to_string())); + + let invalid = rpc(&ctx, "set_decrypt_list", json!({"entries": ["host:a b"]})).await; + assert!(invalid.unwrap_err().contains("Invalid host")); + assert_eq!(rpc(&ctx, "get_decrypt_list", json!({})).await.unwrap(), list); + + // Applied to the running engine's next connection: decrypting everything intercepts a + // CONNECT, which a client that does not trust the CA then rejects. + ctx.start().unwrap(); + rpc(&ctx, "set_decrypt_list", json!({"entries": ["*"]})).await.unwrap(); + let echo = spawn_echo().await; + let client = reqwest::Client::builder() + .proxy(reqwest::Proxy::all(format!("http://127.0.0.1:{}", ctx.port().unwrap())).unwrap()) + .build() + .unwrap(); + let err = client.get(format!("https://localhost:{}/", echo.port())).send().await.unwrap_err(); + assert!(err.is_connect() || err.is_request(), "{err:?}"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn seen_lists_hosts_and_apps_most_recent_first() { + let dir = tempfile::tempdir().unwrap(); + let settings = ProxySettings { port: 0, system_proxy_at_launch: false, ..Default::default() }; + settings.save(&dir.path().join("settings.json")).unwrap(); + let (events, _event_rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + let flow = |url: &str, app: Option<(&str, Option<&str>)>, at: i64| Flow { + id: format!("fl_{at}"), + url: url.into(), + method: "GET".into(), + started_at: chrono::DateTime::from_timestamp(at, 0).unwrap().naive_utc(), + client_name: app.map(|(n, _)| n.to_string()), + client_bundle_id: app.and_then(|(_, b)| b.map(String::from)), + ..Default::default() + }; + let chrome = Some(("Google Chrome", Some("com.google.Chrome"))); + ctx.db() + .unwrap() + .with_tx(|c| { + for f in [ + flow("https://api.example.com/v1", chrome, 1), + flow("https://API.example.com:443/v2", None, 2), + flow("https://example.com:8443/", Some(("curl", None)), 3), + flow("http://localhost:80/x", chrome, 4), + ] { + c.upsert(&f, &yaak_database::UpdateSource::Background)?; + } + Ok::<_, yaak_database::Error>(()) + }) + .unwrap(); + + let seen = rpc(&ctx, "list_seen", json!({})).await.unwrap(); + assert_eq!(seen["hosts"], json!(["localhost", "example.com:8443", "api.example.com"])); + assert_eq!( + seen["apps"], + json!([ + {"name": "Google Chrome", "bundleId": "com.google.Chrome"}, + {"name": "curl", "bundleId": null}, + ]) + ); +} + +/// Edits made together each land on the list the other left, and every change is announced to +/// the app, so a list changed elsewhere shows at once. +#[tokio::test(flavor = "multi_thread")] +async fn decrypt_list_edits_apply_to_the_latest_list_and_are_announced() { + let dir = tempfile::tempdir().unwrap(); + let settings = ProxySettings { port: 0, system_proxy_at_launch: false, ..Default::default() }; + settings.save(&dir.path().join("settings.json")).unwrap(); + let (events, event_rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + + let (a, b) = tokio::join!( + rpc(&ctx, "edit_decrypt_list", json!({"add": ["a.com"]})), + rpc(&ctx, "edit_decrypt_list", json!({"add": ["app:Slack"]})), + ); + a.unwrap(); + b.unwrap(); + let list = + rpc(&ctx, "edit_decrypt_list", json!({"remove": ["!host:*.zoom.us"]})).await.unwrap(); + let user: Vec<&str> = list["entries"] + .as_array() + .unwrap() + .iter() + .filter(|e| e["builtIn"] == false) + .map(|e| e["entry"].as_str().unwrap()) + .collect(); + assert_eq!(user.len(), 2); + assert!(user.contains(&"host:a.com") && user.contains(&"app:Slack"), "{user:?}"); + assert!(!list["entries"].as_array().unwrap().iter().any(|e| e["entry"] == "!host:*.zoom.us")); + + let changes: Vec<_> = + event_rx.try_iter().filter(|e| e.event == "decrypt_list_changed").collect(); + assert_eq!(changes.len(), 3); + assert_eq!(changes[2].payload["list"], list); +} + /// Capture throughput through the engine, end to end: 3,000 requests, 16 at a time over /// kept-alive connections, each with a 4 KB response, timed until every flow is stored, and how /// long opening one then takes. Run in release: `cargo test --release -p yaak-proxy-lib --test diff --git a/crates/yaak-proxy/src/decrypt.rs b/crates/yaak-proxy/src/decrypt.rs new file mode 100644 index 000000000..2fbc6a14e --- /dev/null +++ b/crates/yaak-proxy/src/decrypt.rs @@ -0,0 +1,445 @@ +//! Which HTTPS connections the engine decrypts. The policy is one list of entries: +//! +//! - `host:api.example.com`, `host:*.example.com` (the host itself and every host below it), +//! `host:example.com:8443` (that port only), with `*` matching any run of characters; +//! - `app:Google Chrome` (by the app's name) or `app:com.google.Chrome` (by its bundle id); +//! - `*` alone, everything; +//! - any of these after `!`, which excludes what it matches. +//! +//! A connection is decrypted when a positive entry matches it and no negative one does, wherever +//! each sits in the list. Nothing matching tunnels it, so an empty list decrypts nothing. A +//! built-in exclusion (Apple's services, apps that pin their certificates) applies only to +//! positive entries broader than itself, such as `*` or `host:*.com`: an app entry, or a host +//! entry at least as specific as the exclusion, still decrypts what it names. + +use std::fmt; + +/// Hosts excluded by default: Apple's own services, which macOS daemons reach with pinned +/// certificates, and the sync hosts of common apps that pin theirs. Each is removable. +pub const BUILT_IN_EXCLUSIONS: &[&str] = &[ + "!host:*.apple.com", + "!host:*.icloud.com", + "!host:*.mzstatic.com", + "!host:*.apple-cloudkit.com", + "!host:*.icloud-content.com", + "!host:*.cdn-apple.com", + // Push and FaceTime, kept apart so they stay excluded without the rest of apple.com. + "!host:*.push.apple.com", + "!host:*.ess.apple.com", + "!host:*.1password.com", + "!host:*.1password.ca", + "!host:*.1password.eu", + "!host:*.signal.org", + "!host:*.whispersystems.org", + "!host:*.dropboxapi.com", + "!host:client*.dropbox.com", + "!host:bolt.dropbox.com", + "!host:*.zoom.us", +]; + +/// What an entry matches. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DecryptTarget { + /// `*`: every connection. + Everything, + /// A host pattern, lowercased, and the port it is limited to, if any. + Host { pattern: String, port: Option }, + /// An app's name or bundle id, as a pattern, matched in any case. + App { pattern: String }, +} + +/// One entry of the list. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DecryptEntry { + /// Written with `!`: excludes what it matches. + pub negated: bool, + pub target: DecryptTarget, + /// Shipped with the engine rather than added by the user. + pub built_in: bool, +} + +impl DecryptEntry { + /// Parses an entry as the user may type it: with its prefix, or as a bare hostname, which is + /// a `host:` entry. A URL is taken as its host. Prefixes are any case. + pub fn parse(input: &str) -> Result { + let trimmed = input.trim(); + let (negated, rest) = match trimmed.strip_prefix('!') { + Some(rest) => (true, rest.trim_start()), + None => (false, trimmed), + }; + if rest.is_empty() { + return Err(format!("Empty decrypt entry {input:?}")); + } + let target = if rest == "*" { + DecryptTarget::Everything + } else if let Some(app) = strip_prefix_ci(rest, "app:") { + let app = unquote(app.trim()); + if app.is_empty() { + return Err(format!("No app in decrypt entry {input:?}")); + } + DecryptTarget::App { pattern: app.to_string() } + } else { + let host = unquote(strip_prefix_ci(rest, "host:").unwrap_or(rest).trim()); + parse_host(host).ok_or_else(|| format!("Invalid host in decrypt entry {input:?}"))? + }; + let mut entry = Self { negated, target, built_in: false }; + entry.built_in = BUILT_IN_EXCLUSIONS.contains(&entry.to_string().as_str()); + Ok(entry) + } + + /// The literal characters of the pattern: how specific it is. A site's `*.` counts for + /// nothing, as `*.apple.com` matches `apple.com` itself, so `host:apple.com` ranks with it. + fn specificity(&self) -> usize { + match &self.target { + DecryptTarget::Everything => 0, + DecryptTarget::Host { pattern, port } => { + let pattern = pattern.strip_prefix("*.").unwrap_or(pattern); + pattern.chars().filter(|c| *c != '*').count() + usize::from(port.is_some()) + } + DecryptTarget::App { .. } => usize::MAX, + } + } + + fn matches(&self, conn: &DecryptQuery) -> bool { + match &self.target { + DecryptTarget::Everything => true, + DecryptTarget::Host { pattern, port } => { + port.is_none_or(|p| p == conn.port) && host_matches(pattern, &conn.host) + } + DecryptTarget::App { pattern } => conn.app.as_ref().is_some_and(|app| { + let pattern = pattern.to_lowercase(); + glob(&pattern, &app.name.to_lowercase()) + || app.bundle_id.as_ref().is_some_and(|id| glob(&pattern, &id.to_lowercase())) + }), + } + } +} + +/// The entry as stored and shown: `!host:*.apple.com`, `app:Google Chrome`, `*`. +impl fmt::Display for DecryptEntry { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.negated { + f.write_str("!")?; + } + match &self.target { + DecryptTarget::Everything => f.write_str("*"), + DecryptTarget::Host { pattern, port: None } => write!(f, "host:{pattern}"), + DecryptTarget::Host { pattern, port: Some(port) } if pattern.contains(':') => { + write!(f, "host:[{pattern}]:{port}") + } + DecryptTarget::Host { pattern, port: Some(port) } => write!(f, "host:{pattern}:{port}"), + DecryptTarget::App { pattern } => write!(f, "app:{pattern}"), + } + } +} + +/// A value as the search writes one with spaces in it: `"Google Chrome"`. +fn unquote(s: &str) -> &str { + match s.strip_prefix('"').and_then(|s| s.strip_suffix('"')) { + Some(inner) => inner.trim(), + None => s, + } +} + +fn strip_prefix_ci<'a>(s: &'a str, prefix: &str) -> Option<&'a str> { + let head = s.get(..prefix.len())?; + head.eq_ignore_ascii_case(prefix).then(|| &s[prefix.len()..]) +} + +/// A host pattern with an optional port: `api.example.com`, `*.example.com:8443`, +/// `[::1]:8443`, or a URL, whose host and port it takes. +fn parse_host(input: &str) -> Option { + let mut host = input; + if let Some((_, rest)) = host.split_once("://") { + host = rest.split(['/', '?', '#']).next().unwrap_or_default(); + host = host.rsplit_once('@').map_or(host, |(_, h)| h); + } + let (pattern, port) = if let Some(rest) = host.strip_prefix('[') { + let (addr, after) = rest.split_once(']')?; + let port = match after { + "" => None, + p => Some(p.strip_prefix(':')?.parse::().ok()?), + }; + (addr, port) + } else { + match host.rsplit_once(':') { + // One colon: a port. More: a bare IPv6 address. + Some((name, port)) if !name.contains(':') => (name, Some(port.parse::().ok()?)), + _ => (host, None), + } + }; + let pattern = pattern.trim_end_matches('.').to_lowercase(); + let valid = !pattern.is_empty() + && pattern.chars().all(|c| c.is_ascii_alphanumeric() || "-._*:".contains(c)); + valid.then_some(DecryptTarget::Host { pattern, port }) +} + +/// `*.example.com` matches `example.com` too, as cookie domains do and as the search's `@host` +/// does. +fn host_matches(pattern: &str, host: &str) -> bool { + let host = host.trim_start_matches('[').trim_end_matches(']').trim_end_matches('.'); + let host = host.to_lowercase(); + glob(pattern, &host) || pattern.strip_prefix("*.").is_some_and(|apex| glob(apex, &host)) +} + +/// `*` matches any run of characters, none included; everything else matches itself. +fn glob(pattern: &str, text: &str) -> bool { + let (p, t) = (pattern.as_bytes(), text.as_bytes()); + let (mut pi, mut ti) = (0, 0); + let mut star: Option<(usize, usize)> = None; + while ti < t.len() { + if pi < p.len() && p[pi] == b'*' { + star = Some((pi, ti)); + pi += 1; + } else if pi < p.len() && p[pi] == t[ti] { + pi += 1; + ti += 1; + } else if let Some((sp, st)) = star { + pi = sp + 1; + ti = st + 1; + star = Some((sp, st + 1)); + } else { + return false; + } + } + p[pi..].iter().all(|c| *c == b'*') +} + +/// The app that opened a connection, as the policy's `app:` entries match it. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct DecryptApp { + pub name: String, + pub bundle_id: Option, +} + +/// A CONNECT to judge: where it goes and, when known, the app that sent it. +#[derive(Debug, Clone, Default)] +pub struct DecryptQuery { + pub host: String, + pub port: u16, + pub app: Option, +} + +/// The list of entries, in order. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct DecryptPolicy { + pub entries: Vec, +} + +impl DecryptPolicy { + /// Decrypts everything, without exclusions: the engine's default, for callers that keep no + /// list. + pub fn everything() -> Self { + Self { entries: vec![DecryptEntry::parse("*").unwrap()] } + } + + /// The policy from entries as stored. Fails on the first one that does not parse. + pub fn parse>(entries: &[S]) -> Result { + let entries = entries.iter().map(|e| DecryptEntry::parse(e.as_ref())); + Ok(Self { entries: entries.collect::>()? }) + } + + /// Some entry names an app, so judging a connection needs the app that sent it. + pub fn needs_app(&self) -> bool { + self.entries.iter().any(|e| matches!(e.target, DecryptTarget::App { .. })) + } + + /// Some positive entry could match, so a connection may be decrypted at all. + pub fn decrypts_anything(&self) -> bool { + self.entries.iter().any(|e| !e.negated) + } + + /// Whether to decrypt `conn`: a positive entry matches and no negative one excludes it. A + /// built-in exclusion yields to a positive entry at least as specific as itself. + pub fn decrypts(&self, conn: &DecryptQuery) -> bool { + let Some(best) = self + .entries + .iter() + .filter(|e| !e.negated && e.matches(conn)) + .map(DecryptEntry::specificity) + .max() + else { + return false; + }; + !self + .entries + .iter() + .any(|e| e.negated && e.matches(conn) && (!e.built_in || e.specificity() > best)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn policy(entries: &[&str]) -> DecryptPolicy { + DecryptPolicy::parse(entries).unwrap() + } + + fn to(host: &str, port: u16) -> DecryptQuery { + DecryptQuery { host: host.into(), port, app: None } + } + + fn from_app(host: &str, name: &str, bundle_id: Option<&str>) -> DecryptQuery { + let app = DecryptApp { name: name.into(), bundle_id: bundle_id.map(String::from) }; + DecryptQuery { host: host.into(), port: 443, app: Some(app) } + } + + fn with_built_ins(entries: &[&str]) -> DecryptPolicy { + let mut all: Vec<&str> = entries.to_vec(); + all.extend(BUILT_IN_EXCLUSIONS); + policy(&all) + } + + #[test] + fn an_empty_list_tunnels_everything() { + let empty = policy(&[]); + assert!(!empty.decrypts(&to("api.example.com", 443))); + assert!(!empty.decrypts_anything()); + // The built-in exclusions alone decrypt nothing either. + let built_in = with_built_ins(&[]); + assert!(!built_in.decrypts(&to("api.example.com", 443))); + assert!(!built_in.decrypts_anything()); + } + + #[test] + fn host_patterns() { + let p = policy(&["host:api.example.com"]); + assert!(p.decrypts(&to("api.example.com", 443))); + assert!(p.decrypts(&to("API.Example.com", 8443)), "any case, any port"); + assert!(!p.decrypts(&to("www.example.com", 443))); + assert!(!p.decrypts(&to("example.com", 443))); + + let p = policy(&["host:*.example.com"]); + assert!(p.decrypts(&to("api.example.com", 443))); + assert!(p.decrypts(&to("a.b.example.com", 443))); + assert!(p.decrypts(&to("example.com", 443)), "the site itself too"); + assert!(!p.decrypts(&to("notexample.com", 443))); + assert!(!p.decrypts(&to("example.com.evil.net", 443))); + + let p = policy(&["host:api-*.example.com"]); + assert!(p.decrypts(&to("api-eu.example.com", 443))); + assert!(!p.decrypts(&to("web.example.com", 443))); + } + + #[test] + fn host_patterns_with_a_port() { + let p = policy(&["host:example.com:8443"]); + assert!(p.decrypts(&to("example.com", 8443))); + assert!(!p.decrypts(&to("example.com", 443))); + + let p = policy(&["host:[::1]:8443", "host:fe80::1"]); + assert!(p.decrypts(&to("[::1]", 8443))); + assert!(!p.decrypts(&to("::1", 443))); + assert!(p.decrypts(&to("fe80::1", 443))); + assert_eq!(p.entries[0].to_string(), "host:[::1]:8443"); + assert_eq!(p.entries[1].to_string(), "host:fe80::1"); + } + + #[test] + fn apps_by_name_and_bundle_id() { + let by_name = policy(&["app:Google Chrome"]); + assert!(by_name.decrypts(&from_app("a.com", "Google Chrome", Some("com.google.Chrome")))); + assert!(by_name.decrypts(&from_app("a.com", "google chrome", None))); + assert!(!by_name.decrypts(&from_app("a.com", "Safari", Some("com.apple.Safari")))); + assert!(!by_name.decrypts(&to("a.com", 443)), "an unknown app matches no app entry"); + assert!(by_name.needs_app()); + + let by_id = policy(&["app:com.google.Chrome"]); + assert!(by_id.decrypts(&from_app("a.com", "Google Chrome", Some("com.google.Chrome")))); + assert!(!by_id.decrypts(&from_app("a.com", "Chromium", Some("org.chromium.Chromium")))); + + let pattern = policy(&["app:com.google.*"]); + assert!(pattern.decrypts(&from_app("a.com", "Google Chrome", Some("com.google.Chrome")))); + assert!(!policy(&["host:a.com"]).needs_app()); + } + + #[test] + fn everything_keeps_the_built_in_exclusions() { + let p = with_built_ins(&["*"]); + assert!(p.decrypts(&to("api.example.com", 443))); + assert!(!p.decrypts(&to("gateway.icloud.com", 443))); + assert!(!p.decrypts(&to("apple.com", 443))); + assert!(!p.decrypts(&to("courier.push.apple.com", 5223))); + assert!(!p.decrypts(&to("my.1password.com", 443))); + assert!(!p.decrypts(&to("client-cf.dropbox.com", 443))); + assert!(p.decrypts(&to("www.dropbox.com", 443))); + // A broad wildcard yields to them too. + assert!(!with_built_ins(&["host:*.com"]).decrypts(&to("p01.icloud.com", 443))); + } + + #[test] + fn a_specific_entry_beats_a_built_in_exclusion() { + let p = with_built_ins(&["*", "host:developer.apple.com"]); + assert!(p.decrypts(&to("developer.apple.com", 443))); + assert!(!p.decrypts(&to("www.apple.com", 443))); + assert!(with_built_ins(&["host:*.apple.com"]).decrypts(&to("www.apple.com", 443))); + // The site itself, which the exclusion's `*.apple.com` matches too. + assert!(with_built_ins(&["host:apple.com"]).decrypts(&to("apple.com", 443))); + // A narrower exclusion still wins over a broader entry. + assert!( + !with_built_ins(&["host:*.apple.com"]).decrypts(&to("courier.push.apple.com", 443)) + ); + let safari = from_app("www.icloud.com", "Safari", Some("com.apple.Safari")); + assert!(with_built_ins(&["app:Safari"]).decrypts(&safari)); + } + + #[test] + fn a_removed_exclusion_no_longer_applies() { + let kept: Vec<&str> = + BUILT_IN_EXCLUSIONS.iter().copied().filter(|e| *e != "!host:*.zoom.us").collect(); + let mut entries = vec!["*"]; + entries.extend(kept); + let p = policy(&entries); + assert!(p.decrypts(&to("us02web.zoom.us", 443))); + assert!(!p.decrypts(&to("gateway.icloud.com", 443))); + } + + #[test] + fn negative_entries_win_wherever_they_sit() { + for entries in [ + ["!host:secret.example.com", "host:*.example.com"], + ["host:*.example.com", "!host:secret.example.com"], + ] { + let p = policy(&entries); + assert!(p.decrypts(&to("api.example.com", 443))); + assert!(!p.decrypts(&to("secret.example.com", 443))); + } + // The user's own exclusion beats even an exact entry. + let p = policy(&["host:a.example.com", "!host:*.example.com"]); + assert!(!p.decrypts(&to("a.example.com", 443))); + let p = policy(&["*", "!app:Slack"]); + assert!(!p.decrypts(&from_app("slack.com", "Slack", Some("com.tinyspeck.slackmacgap")))); + assert!(p.decrypts(&from_app("slack.com", "Safari", None))); + } + + #[test] + fn entries_are_normalized() { + let parse = |s: &str| DecryptEntry::parse(s).unwrap().to_string(); + assert_eq!(parse("api.example.com"), "host:api.example.com"); + assert_eq!(parse(" API.Example.com. "), "host:api.example.com"); + assert_eq!(parse("HOST:*.Example.com"), "host:*.example.com"); + assert_eq!(parse("! host:x.com"), "!host:x.com"); + assert_eq!(parse("!x.com"), "!host:x.com"); + assert_eq!(parse("https://api.example.com:8443/v1?q"), "host:api.example.com:8443"); + assert_eq!(parse("App: Google Chrome"), "app:Google Chrome"); + assert_eq!(parse(" * "), "*"); + assert_eq!(parse(r#"app:"Google Chrome""#), "app:Google Chrome"); + assert!(DecryptEntry::parse("").is_err()); + assert!(DecryptEntry::parse("!").is_err()); + assert!(DecryptEntry::parse("app:").is_err()); + assert!(DecryptEntry::parse("host:a b").is_err()); + assert!(DecryptEntry::parse("host:a.com:http").is_err()); + assert!(DecryptEntry::parse("!host:*.apple.com").unwrap().built_in); + assert!(!DecryptEntry::parse("host:*.apple.com").unwrap().built_in); + } + + #[test] + fn globs() { + assert!(glob("*", "")); + assert!(glob("a*c", "abbbc")); + assert!(glob("*b*", "abc")); + assert!(!glob("a*c", "abd")); + assert!(glob("client*.dropbox.com", "client-lb.dropbox.com")); + } +} diff --git a/crates/yaak-proxy/src/lib.rs b/crates/yaak-proxy/src/lib.rs index 3c973569d..54d7447b3 100644 --- a/crates/yaak-proxy/src/lib.rs +++ b/crates/yaak-proxy/src/lib.rs @@ -2,6 +2,7 @@ mod body; mod cert; mod connection; mod decode; +mod decrypt; mod grpc; mod messages; #[cfg(target_os = "macos")] @@ -17,7 +18,7 @@ use std::hash::{BuildHasher, Hasher}; use std::net::SocketAddr; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::mpsc as std_mpsc; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, RwLock}; use std::time::SystemTime; use bytes::Bytes; @@ -32,6 +33,9 @@ use tokio::net::TcpListener; pub use cert::{CaStore, KeyCipher}; use connection::handle_connection; +pub use decrypt::{ + BUILT_IN_EXCLUSIONS, DecryptApp, DecryptEntry, DecryptPolicy, DecryptQuery, DecryptTarget, +}; pub use grpc::{detect as detect_grpc, service_and_method}; pub use messages::{MessageDirection, MessageKind, RAW_LIMIT, StreamKind, StreamMessage}; use request::BoxBody; @@ -62,11 +66,21 @@ pub struct ProxyConfig { /// Record WebSocket frames as messages. Off, upgraded connections are spliced blind, as /// other upgrades are. pub websocket_capture: bool, + /// Which HTTPS connections are decrypted; the rest are tunnelled, recorded as a flow each + /// without their contents. Everything by default. See [`ProxyHandle::set_decrypt_policy`]. + pub decrypt: DecryptPolicy, + /// Finds the app that opened a client connection, by its number, for the policy's `app:` + /// entries. Runs on a blocking thread as each CONNECT is judged, only while the policy has + /// such an entry, and may wait briefly for a lookup in progress. + pub app_of: Option, } /// See [`ProxyConfig::on_accept`]. pub type AcceptHook = Arc; +/// See [`ProxyConfig::app_of`]. +pub type AppLookup = Arc Option + Send + Sync>; + impl ProxyConfig { pub fn new(port: u16, ca: Arc) -> Self { Self { @@ -77,6 +91,8 @@ impl ProxyConfig { on_accept: None, http2: true, websocket_capture: true, + decrypt: DecryptPolicy::everything(), + app_of: None, } } } @@ -85,8 +101,10 @@ impl ProxyConfig { /// `RequestStart` and ends with exactly one `ResponseBodyComplete`, `StreamEnd`, `Error` or /// `Cancelled`. Each HTTP/2 stream is a flow of its own. A streaming flow (WebSocket, /// server-sent events, gRPC) has a `StreamStart` and then a `Message` for each message as it -/// passes. Traffic the engine tunnels without decrypting (pinned clients, non-TLS CONNECT -/// payloads, upgrades to protocols other than WebSocket) produces no events. +/// passes. An HTTPS connection the engine tunnels without decrypting, because the decrypt policy +/// leaves it out or its client rejected the engine's certificate, is a flow of its own, from +/// `TunnelStart` to `TunnelEnd`. Other traffic tunnelled blind (non-TLS CONNECT payloads, +/// upgrades to protocols other than WebSocket) produces no events. #[derive(Debug, Clone)] pub enum ProxyEvent { /// Request headers arrived from the client. @@ -181,6 +199,26 @@ pub enum ProxyEvent { elapsed_ms: u64, error: Option, }, + /// A CONNECT is tunnelled without being decrypted: its bytes pass untouched, and only their + /// count is recorded. + TunnelStart { + id: u64, + started_at: SystemTime, + /// `host[:port]`, the port left out when it is 443. + authority: String, + connection: u64, + client_addr: SocketAddr, + }, + /// The tunnel closed, with `error` when connecting upstream or relaying failed. + TunnelEnd { + id: u64, + /// Bytes from the client to the server. + sent: u64, + /// Bytes from the server to the client. + received: u64, + elapsed_ms: u64, + error: Option, + }, } /// A running engine. Dropping it stops the listener and joins its thread. @@ -212,6 +250,12 @@ impl ProxyHandle { self.engine.http2.store(enabled, Ordering::Relaxed); } + /// Decides which HTTPS connections are decrypted from the next CONNECT on. Connections + /// already open stay decrypted or tunnelled as they were. + pub fn set_decrypt_policy(&self, policy: DecryptPolicy) { + *self.engine.decrypt.write().unwrap() = Arc::new(policy); + } + /// Handshake failures counted toward passing `addr` (`host:port`) through, and whether it /// already is. For tests that must wait for the engine to record a failure. #[doc(hidden)] @@ -253,6 +297,8 @@ pub(crate) struct Engine { /// Offer HTTP/2 to clients in intercepted TLS. http2: AtomicBool, websocket_capture: bool, + decrypt: RwLock>, + app_of: Option, /// `Via` value marking the plain-HTTP requests the engine forwards, unique to this run, so one /// that comes back to the engine is refused instead of forwarded again. via: HeaderValue, @@ -271,6 +317,27 @@ impl Engine { self.http2.load(Ordering::Relaxed) } + fn decrypt_policy(&self) -> Arc { + self.decrypt.read().unwrap().clone() + } + + /// Whether the policy decrypts a CONNECT to `host:port` on `connection`, looking up its app + /// only when an entry names one. + async fn decrypts(&self, host: &str, port: u16, connection: u64) -> bool { + let policy = self.decrypt_policy(); + if !policy.decrypts_anything() { + return false; + } + let app = match &self.app_of { + Some(app_of) if policy.needs_app() => { + let app_of = app_of.clone(); + tokio::task::spawn_blocking(move || app_of(connection)).await.ok().flatten() + } + _ => None, + }; + policy.decrypts(&DecryptQuery { host: host.to_string(), port, app }) + } + fn is_passthrough(&self, addr: &str) -> bool { self.passthrough.lock().unwrap().contains(addr) } @@ -352,6 +419,8 @@ pub fn start_proxy(config: ProxyConfig) -> Result { handshake_failures: Mutex::new(HashMap::new()), http2: AtomicBool::new(config.http2), websocket_capture: config.websocket_capture, + decrypt: RwLock::new(Arc::new(config.decrypt)), + app_of: config.app_of, via: via_marker(), }); diff --git a/crates/yaak-proxy/src/request.rs b/crates/yaak-proxy/src/request.rs index 5bf8db271..f2883e2e1 100644 --- a/crates/yaak-proxy/src/request.rs +++ b/crates/yaak-proxy/src/request.rs @@ -31,7 +31,7 @@ use crate::connection::serve_tls; use crate::grpc::{self, GrpcParser}; use crate::messages::{MessageDirection, StreamKind}; use crate::sse::SseParser; -use crate::tunnel::{Rewind, splice, tunnel}; +use crate::tunnel::{Counted, Rewind, is_own_connection, splice, tunnel}; use crate::ws::{self, WsFlow}; use crate::{ClientConn, Engine, ProxyEvent}; @@ -376,6 +376,7 @@ struct Target { authority: String, /// `host:port` to dial. addr: String, + port: u16, } fn handle_connect( @@ -401,6 +402,7 @@ fn handle_connect( server_name: host.trim_start_matches('[').trim_end_matches(']').to_string(), authority: if port == 443 { host.clone() } else { format!("{host}:{port}") }, addr: format!("{host}:{port}"), + port, }; if let Err(e) = intercept(upgraded, target, client, engine).await { debug!("Tunnel to {host}:{port} ended: {}", error_chain(e.as_ref())); @@ -419,8 +421,13 @@ async fn intercept( where S: AsyncRead + AsyncWrite + Unpin + Send + 'static, { + if !engine.decrypts(&target.server_name, target.port, conn.connection).await { + debug!("Tunneling {} undecrypted, as the decrypt list leaves it out", target.addr); + return Ok(tunnel_flow(client, &target, conn, &engine).await?); + } + // The fallback for clients that rejected the engine's certificate. if engine.is_passthrough(&target.addr) { - return Ok(tunnel(client, &target.addr, &[]).await?); + return Ok(tunnel_flow(client, &target, conn, &engine).await?); } let mut buf = vec![0u8; 4096]; @@ -480,6 +487,49 @@ where .await } +/// Tunnels a CONNECT undecrypted, recorded as a flow of its own with the bytes each way. Not +/// one of the engine's own connections coming back, which [`tunnel`] refuses. +async fn tunnel_flow( + client: S, + target: &Target, + conn: ClientConn, + engine: &Engine, +) -> std::io::Result<()> +where + S: AsyncRead + AsyncWrite + Unpin, +{ + if is_own_connection(&target.addr).await { + return tunnel(client, &target.addr, &[]).await; + } + let id = engine.next_flow_id(); + let start = Instant::now(); + engine.emit(ProxyEvent::TunnelStart { + id, + started_at: SystemTime::now(), + authority: target.authority.clone(), + connection: conn.connection, + client_addr: conn.addr, + }); + let client = Counted::new(client); + let (sent, received) = client.counters(); + let result = tunnel(client, &target.addr, &[]).await; + let (sent, received) = (sent.load(Ordering::Relaxed), received.load(Ordering::Relaxed)); + // A tunnel that carried the server's bytes and then broke ended as connections do; one that + // never did failed to reach the server. + let error = match &result { + Err(e) if received == 0 => Some(error_chain(e)), + _ => None, + }; + engine.emit(ProxyEvent::TunnelEnd { + id, + sent, + received, + elapsed_ms: start.elapsed().as_millis() as u64, + error, + }); + result +} + fn record_handshake_failure( engine: &Engine, addr: &str, diff --git a/crates/yaak-proxy/src/tunnel.rs b/crates/yaak-proxy/src/tunnel.rs index c27577ac4..ad77a2ff0 100644 --- a/crates/yaak-proxy/src/tunnel.rs +++ b/crates/yaak-proxy/src/tunnel.rs @@ -1,5 +1,7 @@ use std::io; use std::pin::Pin; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; use std::task::{Context, Poll}; use bytes::Bytes; @@ -23,6 +25,72 @@ where relay(client, TcpStream::connect(target).await?, prefix).await } +/// Whether a tunnel to `target` (`host:port`) would be one of the engine's own connections that +/// came back through the system proxy, which [`tunnel`] refuses. Only port 80 on macOS can be. +pub(crate) async fn is_own_connection(target: &str) -> bool { + #[cfg(target_os = "macos")] + if let Some((host, "80")) = target.rsplit_once(':') { + let host = host.trim_start_matches('[').trim_end_matches(']'); + return crate::nw::is_own_fallback(host, 80).await; + } + let _ = target; + false +} + +/// A stream that counts the bytes read from it and written to it, for a tunnel's sizes. +pub(crate) struct Counted { + inner: S, + read: Arc, + written: Arc, +} + +impl Counted { + pub(crate) fn new(inner: S) -> Self { + Self { inner, read: Arc::default(), written: Arc::default() } + } + + /// The counts of bytes read and written, which keep counting after the stream moves on. + pub(crate) fn counters(&self) -> (Arc, Arc) { + (self.read.clone(), self.written.clone()) + } +} + +impl AsyncRead for Counted { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + let before = buf.filled().len(); + let polled = Pin::new(&mut self.inner).poll_read(cx, buf); + let n = buf.filled().len() - before; + self.read.fetch_add(n as u64, Ordering::Relaxed); + polled + } +} + +impl AsyncWrite for Counted { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + let polled = Pin::new(&mut self.inner).poll_write(cx, buf); + if let Poll::Ready(Ok(n)) = &polled { + self.written.fetch_add(*n as u64, Ordering::Relaxed); + } + polled + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_shutdown(cx) + } +} + /// [`tunnel`] through Network.framework, which can skip the system proxy. When it can't, it falls /// back to the system proxy with a `CONNECT` that arrives here, and tunneling that out the same /// way would repeat it. diff --git a/crates/yaak-proxy/tests/proxy.rs b/crates/yaak-proxy/tests/proxy.rs index 7e045d70e..f3b32bc1d 100644 --- a/crates/yaak-proxy/tests/proxy.rs +++ b/crates/yaak-proxy/tests/proxy.rs @@ -33,8 +33,8 @@ use tokio_tungstenite::tungstenite::protocol::Role; use tokio_tungstenite::tungstenite::protocol::frame::CloseFrame; use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode; use yaak_proxy::{ - CaStore, MessageDirection, MessageKind, ProxyConfig, ProxyEvent, ProxyHandle, StreamKind, - StreamMessage, start_proxy, + CaStore, DecryptApp, DecryptPolicy, MessageDirection, MessageKind, ProxyConfig, ProxyEvent, + ProxyHandle, StreamKind, StreamMessage, start_proxy, }; const BIG_BODY: usize = 64 * 1024; @@ -405,6 +405,7 @@ fn is_terminal(event: &ProxyEvent) -> bool { | ProxyEvent::Error { .. } | ProxyEvent::Cancelled { .. } | ProxyEvent::StreamEnd { .. } + | ProxyEvent::TunnelEnd { .. } ) } @@ -422,7 +423,9 @@ fn flow_id(event: &ProxyEvent) -> u64 { | ProxyEvent::Cancelled { id, .. } | ProxyEvent::StreamStart { id, .. } | ProxyEvent::Message { id, .. } - | ProxyEvent::StreamEnd { id, .. } => *id, + | ProxyEvent::StreamEnd { id, .. } + | ProxyEvent::TunnelStart { id, .. } + | ProxyEvent::TunnelEnd { id, .. } => *id, } } @@ -1252,7 +1255,171 @@ async fn pinned_client_falls_back_to_passthrough() { tokio::time::sleep(Duration::from_millis(50)).await; } assert_eq!(body.as_deref(), Some("hello")); - assert!(h.events.try_recv().is_err(), "passthrough traffic must not produce flows"); + // Passed through, it is a tunnelled flow, recorded without its contents. + drop(pinned); + let events = h.flow_events(); + let authority = find!(events, ProxyEvent::TunnelStart { authority, .. } => authority.clone()); + assert_eq!(authority, format!("localhost:{}", upstream.port())); + let (sent, received) = + find!(events, ProxyEvent::TunnelEnd { sent, received, .. } => (*sent, *received)); + assert!(sent > 0 && received > 0, "sent {sent}, received {received}"); + assert!(!events.iter().any(|e| matches!(e, ProxyEvent::RequestStart { .. }))); +} + +/// TLS straight to the upstream server through a CONNECT tunnel, trusting only the server's own +/// CA, as a client that never trusted the proxy's does. +async fn upstream_tls( + h: &Harness, + port: u16, + alpn: &[&[u8]], +) -> tokio_rustls::client::TlsStream { + let mut roots = RootCertStore::empty(); + roots.add(h.pki.ca_der.clone()).unwrap(); + let mut config = ClientConfig::builder_with_provider(ring()) + .with_safe_default_protocol_versions() + .unwrap() + .with_root_certificates(roots) + .with_no_client_auth(); + config.alpn_protocols = alpn.iter().map(|p| p.to_vec()).collect(); + let name = ServerName::try_from("localhost").unwrap(); + let tcp = h.connect_tunnel(port).await; + TlsConnector::from(Arc::new(config)).connect(name, tcp).await.expect("TLS to the server") +} + +/// `GET /hello` over HTTP/1.1 on `tls`, closing the connection after. +async fn get_hello_and_close(mut tls: tokio_rustls::client::TlsStream) -> String { + tls.write_all(b"GET /hello HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n") + .await + .unwrap(); + let mut response = Vec::new(); + let _ = tls.read_to_end(&mut response).await; + String::from_utf8_lossy(&response).into_owned() +} + +fn with_policy(entries: &[&str]) -> impl FnOnce(&mut ProxyConfig) { + let policy = DecryptPolicy::parse(entries).unwrap(); + move |config| config.decrypt = policy +} + +#[tokio::test(flavor = "multi_thread")] +async fn an_empty_decrypt_list_tunnels_https_untouched() { + within(async { + let h = Harness::start_with(1024, with_policy(&[])); + let upstream = spawn_upstream(Some(h.pki.server.clone())).await; + + // The server's own certificate arrives, so no CA of the proxy's is involved. + let tls = upstream_tls(&h, upstream.port(), &[b"http/1.1"]).await; + let response = get_hello_and_close(tls).await; + assert!(response.starts_with("HTTP/1.1 200"), "{response}"); + assert!(response.ends_with("hello"), "{response}"); + + let events = h.flow_events(); + let (connection, authority) = find!(events, ProxyEvent::TunnelStart { connection, authority, .. } => (*connection, authority.clone())); + assert_eq!(connection, 1); + assert_eq!(authority, format!("localhost:{}", upstream.port())); + let (sent, received, error) = find!(events, ProxyEvent::TunnelEnd { sent, received, error, .. } => (*sent, *received, error.clone())); + assert_eq!(error, None); + // The ClientHello and request one way, the certificate and response the other. + assert!(sent > 100, "sent {sent}"); + assert!(received as usize > response.len(), "received {received}"); + assert!(!events.iter().any(|e| matches!(e, ProxyEvent::RequestStart { .. }))); + }) + .await +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_tunnelled_host_passes_http2_through() { + within(async { + let h = Harness::start_with(1024, with_policy(&["host:example.com"])); + let upstream = spawn_upstream(Some(h.pki.server.clone())).await; + let tls = upstream_tls(&h, upstream.port(), &[b"h2", b"http/1.1"]).await; + assert_eq!(tls.get_ref().1.alpn_protocol(), Some(&b"h2"[..]), "the server's ALPN"); + let (mut sender, conn) = + http2::handshake(TokioExecutor::new(), TokioIo::new(tls)).await.unwrap(); + let conn = tokio::spawn(conn); + let req = Request::get(format!("https://localhost:{}/hello", upstream.port())) + .body(full(Bytes::new())) + .unwrap(); + let resp = sender.send_request(req).await.unwrap(); + assert_eq!(resp.version(), Version::HTTP_2); + assert_eq!(resp.into_body().collect().await.unwrap().to_bytes(), "hello"); + drop(sender); + let _ = conn.await; + + let events = h.flow_events(); + find!(events, ProxyEvent::TunnelStart { .. } => ()); + let error = find!(events, ProxyEvent::TunnelEnd { error, .. } => error.clone()); + assert_eq!(error, None); + }) + .await +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_listed_host_is_decrypted_and_others_tunnelled() { + within(async { + let h = Harness::start_with(1024, with_policy(&["host:localhost"])); + let upstream = spawn_upstream(Some(h.pki.server.clone())).await; + let url = format!("https://localhost:{}/hello", upstream.port()); + assert_eq!(h.client().get(&url).send().await.unwrap().text().await.unwrap(), "hello"); + let events = h.flow_events(); + let flow_url = find!(events, ProxyEvent::RequestStart { url, .. } => url.clone()); + assert_eq!(flow_url, url); + + // Only from the next connection on. + h.proxy.set_decrypt_policy(DecryptPolicy::parse(&["host:*.example.com"]).unwrap()); + let tls = upstream_tls(&h, upstream.port(), &[b"http/1.1"]).await; + assert!(get_hello_and_close(tls).await.ends_with("hello")); + let events = h.flow_events(); + find!(events, ProxyEvent::TunnelEnd { .. } => ()); + }) + .await +} + +#[tokio::test(flavor = "multi_thread")] +async fn an_app_entry_decrypts_that_app_alone() { + within(async { + let h = Harness::start_with(1024, |config| { + config.decrypt = DecryptPolicy::parse(&["app:com.example.curl"]).unwrap(); + // The first connection is from the listed app, the rest from another. + config.app_of = Some(Arc::new(|connection| { + let bundle_id = if connection == 1 { "com.example.curl" } else { "com.other" }; + Some(DecryptApp { name: "curl".into(), bundle_id: Some(bundle_id.into()) }) + })); + }); + let upstream = spawn_upstream(Some(h.pki.server.clone())).await; + let url = format!("https://localhost:{}/hello", upstream.port()); + let client = h.client(); + assert_eq!(client.get(&url).send().await.unwrap().text().await.unwrap(), "hello"); + let events = h.flow_events(); + find!(events, ProxyEvent::RequestStart { .. } => ()); + drop(client); + + let tls = upstream_tls(&h, upstream.port(), &[b"http/1.1"]).await; + assert!(get_hello_and_close(tls).await.ends_with("hello")); + let events = h.flow_events(); + let connection = find!(events, ProxyEvent::TunnelStart { connection, .. } => *connection); + assert_eq!(connection, 2); + }) + .await +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_tunnel_to_nowhere_is_a_failed_flow() { + within(async { + let h = Harness::start_with(1024, with_policy(&[])); + // A port nothing listens on. + let closed = TcpListener::bind("127.0.0.1:0").await.unwrap().local_addr().unwrap().port(); + let mut tcp = h.connect_tunnel(closed).await; + let _ = tcp.write_all(b"\x16\x03\x01").await; + let mut rest = Vec::new(); + let _ = tcp.read_to_end(&mut rest).await; + let events = h.flow_events(); + let (received, error) = + find!(events, ProxyEvent::TunnelEnd { received, error, .. } => (*received, error.clone())); + assert_eq!(received, 0); + assert!(error.is_some(), "{events:#?}"); + }) + .await } #[tokio::test(flavor = "multi_thread")] diff --git a/apps/yaak-client/components/core/Editor/filter/extension.ts b/packages/ui/src/components/editor/filter/extension.ts similarity index 66% rename from apps/yaak-client/components/core/Editor/filter/extension.ts rename to packages/ui/src/components/editor/filter/extension.ts index f98c608c3..869a50853 100644 --- a/apps/yaak-client/components/core/Editor/filter/extension.ts +++ b/packages/ui/src/components/editor/filter/extension.ts @@ -13,6 +13,18 @@ export interface FieldDef { export interface FilterOptions { fields: FieldDef[] | null; // e.g., ['method','status','path'] or [{name:'tag', values:()=>cachedTags}] + /** Written before a field name, `@` by default: `@method:`. Empty for `host:` */ + fieldPrefix?: string; + /** + * A prefix that negates a term, such as `!`: offered where a field name is, and a term may + * start with it before its field name + */ + negation?: { prefix: string; info?: string }; + /** + * Value suggestions replace only what follows the field's colon, so `host:api` offers hosts + * matching `api`. Off, they replace the whole term + */ + replaceValueOnly?: boolean; } const FIELD_IDENT = /[A-Za-z0-9_/]+$/; @@ -44,19 +56,31 @@ function wordBefore( return { from, to: pos, text: m[0] }; } -function fieldCompletionFrom(doc: string, pos: number): { from: number; includeAt: boolean } | null { +function fieldCompletionFrom( + doc: string, + pos: number, + fieldPrefix = "@", + negation?: string, +): { from: number; includeAt: boolean; negated: boolean } | null { const w = wordBefore(doc, pos, FIELD_IDENT); - const from = w?.from ?? pos; - const beforeToken = doc[from - 1]; - - if (from === 0 || (beforeToken != null && /\s/.test(beforeToken))) { - return { from, includeAt: true }; + let from = w?.from ?? pos; + const atTermStart = (at: number) => { + const before = doc[at - 1]; + return at === 0 || (before != null && /\s/.test(before)); + }; + let negated = false; + if (negation != null && doc.slice(0, from).endsWith(negation)) { + from -= negation.length; + negated = true; } - if (beforeToken === "@") { - const beforeAt = doc[from - 2]; - if (from === 1 || (beforeAt != null && /\s/.test(beforeAt))) { - return { from, includeAt: false }; + if (atTermStart(from)) { + return { from, includeAt: true, negated }; + } + + if (fieldPrefix !== "" && doc.slice(0, from).endsWith(fieldPrefix)) { + if (atTermStart(from - fieldPrefix.length)) { + return { from, includeAt: false, negated }; } } @@ -117,13 +141,20 @@ function contextInfo(stateDoc: string, pos: number) { } /** Build a completion list for field names */ -function fieldNameCompletions(fieldNames: string[], includeAt: boolean): Completion[] { +function fieldNameCompletions( + fieldNames: string[], + includeAt: boolean, + fieldPrefix = "@", + fieldMap: Record = {}, + negated = "", +): Completion[] { return fieldNames.map((name) => ({ - label: name, + label: `${negated}${name}`, type: "property", + info: fieldMap[name]?.info, apply: (view, _completion, from, to) => { // Leave cursor right after the field filter colon. - const insert = `${includeAt ? "@" : ""}${name}:`; + const insert = `${negated}${includeAt ? fieldPrefix : ""}${name}:`; view.dispatch({ changes: { from, to, insert }, selection: { anchor: from + insert.length }, @@ -146,8 +177,24 @@ function fieldValueCompletions( })); } +/** Offers the negation prefix itself, reopening the list for a field name after it */ +function negationCompletion(negation: { prefix: string; info?: string }): Completion { + return { + label: negation.prefix, + type: "keyword", + info: negation.info, + apply: (view, _completion, from, to) => { + view.dispatch({ + changes: { from, to, insert: negation.prefix }, + selection: { anchor: from + negation.prefix.length }, + }); + startCompletion(view); + }, + }; +} + /** The main completion source */ -function makeCompletionSource(opts: FilterOptions) { +export function makeCompletionSource(opts: FilterOptions) { const { fieldNames, fieldMap } = normalizeFields(opts.fields ?? []); return (ctx: CompletionContext): CompletionResult | null => { const { state, pos } = ctx; @@ -157,12 +204,12 @@ function makeCompletionSource(opts: FilterOptions) { return null; } - const { inValue, fieldName, emptyAfterColon } = contextInfo(doc, pos); + const { inValue, fieldName, lastColon, emptyAfterColon } = contextInfo(doc, pos); // In field value position if (inValue && fieldName) { const w = wordBefore(doc, pos, VALUE_IDENT); - const from = w?.from ?? pos; + const from = opts.replaceValueOnly ? lastColon + 1 : (w?.from ?? pos); const to = pos; const valDefs = fieldMap[fieldName]; const vals = fieldValueCompletions(valDefs); @@ -186,11 +233,18 @@ function makeCompletionSource(opts: FilterOptions) { } // Not in a value: suggest field names (and maybe boolean ops) - const completion = fieldCompletionFrom(doc, pos); + const completion = fieldCompletionFrom(doc, pos, opts.fieldPrefix, opts.negation?.prefix); if (completion == null) return null; - const { from, includeAt } = completion; + const { from, includeAt, negated } = completion; const to = pos; - const options: Completion[] = fieldNameCompletions(fieldNames, includeAt); + const options: Completion[] = fieldNameCompletions( + fieldNames, + includeAt, + opts.fieldPrefix, + fieldMap, + negated ? (opts.negation?.prefix ?? "") : "", + ); + if (opts.negation != null && !negated) options.push(negationCompletion(opts.negation)); return { from, to, options, filter: true }; }; @@ -204,6 +258,14 @@ const language = LRLanguage.define({ }, }); +/** + * Field and value suggestions, as the sidebar's filter offers them, for a field of another + * language: a plain input of `host:` and `app:` entries, say + */ +export function filterAutocomplete(options: FilterOptions) { + return autocompletion({ override: [makeCompletionSource(options)] }); +} + /** Public extension */ export function filter(options: FilterOptions) { const source = makeCompletionSource(options); diff --git a/apps/yaak-client/components/core/Editor/filter/filter.grammar b/packages/ui/src/components/editor/filter/filter.grammar similarity index 100% rename from apps/yaak-client/components/core/Editor/filter/filter.grammar rename to packages/ui/src/components/editor/filter/filter.grammar diff --git a/apps/yaak-client/components/core/Editor/filter/filter.test.ts b/packages/ui/src/components/editor/filter/filter.test.ts similarity index 100% rename from apps/yaak-client/components/core/Editor/filter/filter.test.ts rename to packages/ui/src/components/editor/filter/filter.test.ts diff --git a/apps/yaak-client/components/core/Editor/filter/filter.ts b/packages/ui/src/components/editor/filter/filter.ts similarity index 100% rename from apps/yaak-client/components/core/Editor/filter/filter.ts rename to packages/ui/src/components/editor/filter/filter.ts diff --git a/apps/yaak-client/components/core/Editor/filter/highlight.ts b/packages/ui/src/components/editor/filter/highlight.ts similarity index 100% rename from apps/yaak-client/components/core/Editor/filter/highlight.ts rename to packages/ui/src/components/editor/filter/highlight.ts diff --git a/packages/ui/src/editor.ts b/packages/ui/src/editor.ts index d49dc84a9..347d2139b 100644 --- a/packages/ui/src/editor.ts +++ b/packages/ui/src/editor.ts @@ -1,6 +1,12 @@ export type { CodeEditorProps } from "./components/editor/CodeEditor"; export { CodeEditor } from "./components/editor/CodeEditor"; export type { LanguageSupports } from "./components/editor/extensions"; +export type { FieldDef, FilterOptions } from "./components/editor/filter/extension"; +export { + filter, + filterAutocomplete, + makeCompletionSource as filterCompletionSource, +} from "./components/editor/filter/extension"; export { closeBracketsExtensions, getLanguageExtension, From 2ca2bc5f008e74b9b058351365ad07132ce878d7 Mon Sep 17 00:00:00 2001 From: Gregory Schier Date: Sat, 7 Nov 2026 13:45:39 -0700 Subject: [PATCH 09/45] Detail card, settings window and small fixes for Yaak Proxy (#824) --- .gitignore | 3 + .../components/DecryptListEditor.tsx | 25 +- apps/yaak-proxy/components/FlowCard.tsx | 307 ++++++ apps/yaak-proxy/components/FlowDetail.tsx | 307 +----- apps/yaak-proxy/components/FlowHeaders.tsx | 109 -- apps/yaak-proxy/components/FlowInfo.tsx | 125 +++ apps/yaak-proxy/components/FlowList.tsx | 2 +- apps/yaak-proxy/components/FlowListHeader.tsx | 7 +- .../components/FlowRequestBody.test.tsx | 61 ++ .../yaak-proxy/components/FlowRequestBody.tsx | 51 + apps/yaak-proxy/components/FlowTimeline.tsx | 61 ++ apps/yaak-proxy/components/LockedState.tsx | 16 +- apps/yaak-proxy/components/ProxyLayout.tsx | 30 +- apps/yaak-proxy/components/ProxyStatus.tsx | 159 +-- apps/yaak-proxy/components/SearchBar.tsx | 41 +- apps/yaak-proxy/components/Settings.tsx | 181 ++++ apps/yaak-proxy/components/SettingsProxy.tsx | 366 +++++++ apps/yaak-proxy/components/Sidebar.tsx | 1 + apps/yaak-proxy/font-size.ts | 9 +- apps/yaak-proxy/harness-settings.html | 212 ---- apps/yaak-proxy/harness/DragRegionCheck.tsx | 57 ++ apps/yaak-proxy/harness/FlowPanesHarness.tsx | 151 ++- apps/yaak-proxy/harness/LockedHarness.tsx | 14 +- apps/yaak-proxy/harness/SettingsHarness.tsx | 26 + apps/yaak-proxy/harness/backend.ts | 97 ++ apps/yaak-proxy/harness/fixtures.ts | 28 + apps/yaak-proxy/harness/main.tsx | 7 +- apps/yaak-proxy/harness/streamFixtures.ts | 6 + apps/yaak-proxy/lib/atoms.ts | 3 - apps/yaak-proxy/lib/body.ts | 3 + apps/yaak-proxy/lib/defaultTab.test.ts | 106 ++ apps/yaak-proxy/lib/defaultTab.ts | 97 ++ apps/yaak-proxy/lib/env.test.ts | 68 +- apps/yaak-proxy/lib/env.ts | 85 +- apps/yaak-proxy/lib/flowViews.test.ts | 6 + apps/yaak-proxy/lib/flowViews.ts | 21 +- apps/yaak-proxy/lib/interfaceSettings.ts | 32 + apps/yaak-proxy/lib/opener.test.ts | 42 + apps/yaak-proxy/lib/opener.ts | 22 + apps/yaak-proxy/lib/phases.test.ts | 59 ++ apps/yaak-proxy/lib/phases.ts | 66 ++ apps/yaak-proxy/lib/search.ts | 57 +- apps/yaak-proxy/lib/settings.test.ts | 118 +++ apps/yaak-proxy/lib/settings.ts | 168 +++ apps/yaak-proxy/lib/sidebarTree.test.ts | 65 ++ apps/yaak-proxy/lib/sidebarTree.ts | 55 +- apps/yaak-proxy/lib/store.ts | 95 +- apps/yaak-proxy/lib/theme.ts | 17 +- .../yaak-proxy-lib/bindings/gen_models.ts | 23 +- .../yaak-proxy-lib/bindings/gen_rpc.ts | 49 +- .../migrations/00000010_flow_connection.sql | 8 + .../00000011_flow_sweep_pending.sql | 7 + crates-proxy/yaak-proxy-lib/src/db.rs | 77 +- crates-proxy/yaak-proxy-lib/src/exit_gate.rs | 137 +++ crates-proxy/yaak-proxy-lib/src/flows.rs | 253 ++++- crates-proxy/yaak-proxy-lib/src/lib.rs | 959 +++++++++++++++++- crates-proxy/yaak-proxy-lib/src/models.rs | 30 + crates-proxy/yaak-proxy-lib/src/settings.rs | 7 + crates-proxy/yaak-proxy-lib/src/terminal.rs | 86 ++ crates-proxy/yaak-proxy-lib/tests/flow.rs | 2 + .../yaak-app-proxy/capabilities/default.json | 1 + crates-tauri/yaak-app-proxy/src/lib.rs | 17 + crates/yaak-proxy/src/body.rs | 17 +- crates/yaak-proxy/src/lib.rs | 31 +- crates/yaak-proxy/src/request.rs | 62 +- crates/yaak-proxy/src/ws.rs | 2 +- crates/yaak-proxy/tests/proxy.rs | 24 + .../ui/src/components/events/AutoScroller.tsx | 2 +- .../components/http/HttpRequestPaneView.tsx | 34 +- .../components/http/HttpResponsePaneView.tsx | 86 +- .../src/components/http/ResponseCookies.tsx | 8 +- .../src/components/http/ResponseHeaders.tsx | 23 +- .../components/settings/SettingsInterface.tsx | 92 +- packages/ui/src/components/settings/types.ts | 9 + packages/ui/src/components/tabs/Tabs.tsx | 31 +- 75 files changed, 4574 insertions(+), 1147 deletions(-) create mode 100644 apps/yaak-proxy/components/FlowCard.tsx delete mode 100644 apps/yaak-proxy/components/FlowHeaders.tsx create mode 100644 apps/yaak-proxy/components/FlowInfo.tsx create mode 100644 apps/yaak-proxy/components/FlowRequestBody.test.tsx create mode 100644 apps/yaak-proxy/components/FlowRequestBody.tsx create mode 100644 apps/yaak-proxy/components/FlowTimeline.tsx create mode 100644 apps/yaak-proxy/components/Settings.tsx create mode 100644 apps/yaak-proxy/components/SettingsProxy.tsx delete mode 100644 apps/yaak-proxy/harness-settings.html create mode 100644 apps/yaak-proxy/harness/DragRegionCheck.tsx create mode 100644 apps/yaak-proxy/harness/SettingsHarness.tsx create mode 100644 apps/yaak-proxy/harness/backend.ts create mode 100644 apps/yaak-proxy/lib/defaultTab.test.ts create mode 100644 apps/yaak-proxy/lib/defaultTab.ts create mode 100644 apps/yaak-proxy/lib/interfaceSettings.ts create mode 100644 apps/yaak-proxy/lib/opener.test.ts create mode 100644 apps/yaak-proxy/lib/opener.ts create mode 100644 apps/yaak-proxy/lib/phases.test.ts create mode 100644 apps/yaak-proxy/lib/phases.ts create mode 100644 apps/yaak-proxy/lib/settings.test.ts create mode 100644 apps/yaak-proxy/lib/settings.ts create mode 100644 crates-proxy/yaak-proxy-lib/migrations/00000010_flow_connection.sql create mode 100644 crates-proxy/yaak-proxy-lib/migrations/00000011_flow_sweep_pending.sql create mode 100644 crates-proxy/yaak-proxy-lib/src/exit_gate.rs create mode 100644 crates-proxy/yaak-proxy-lib/src/terminal.rs diff --git a/.gitignore b/.gitignore index 1beb88f89..f4d96a8bc 100644 --- a/.gitignore +++ b/.gitignore @@ -59,3 +59,6 @@ flatpak/node-sources.json # Claude Code local settings .claude/settings.local.json .claude/worktrees/ + +# Headless screenshots taken while working +.shots/ diff --git a/apps/yaak-proxy/components/DecryptListEditor.tsx b/apps/yaak-proxy/components/DecryptListEditor.tsx index 74f956d0a..fa788ee95 100644 --- a/apps/yaak-proxy/components/DecryptListEditor.tsx +++ b/apps/yaak-proxy/components/DecryptListEditor.tsx @@ -1,10 +1,9 @@ import { Prec } from "@codemirror/state"; import { type EditorView, keymap } from "@codemirror/view"; import type { DecryptListEntry, SeenTargets } from "@yaakapp-internal/proxy-lib"; -import { Dialog, Icon, IconButton, Input } from "@yaakapp-internal/ui"; +import { Icon, IconButton, Input } from "@yaakapp-internal/ui"; import { filterAutocomplete } from "@yaakapp-internal/ui/editor"; import classNames from "classnames"; -import { atom, useAtom } from "jotai"; import { useMemo, useRef, useState } from "react"; import { useDecryptList, useEditDecryptList } from "../hooks/useDecryptList"; import { useRpcQuery } from "../hooks/useRpcQuery"; @@ -13,8 +12,8 @@ import { decryptFilterOptions, entryLabel, normalizeEntry } from "../lib/decrypt /** * What HTTPS is decrypted: one row per entry, the user's in order and then the built-in * exclusions, each removable, and an input that adds one, suggesting `host:`, `app:` and `!`, - * then the hosts and apps seen in the capture. Saves as it changes. Placed by Settings; until - * that exists, by the dialog the state pill's More… opens. + * then the hosts and apps seen in the capture. Saves as it changes. Placed by Settings' Proxy + * tab. */ export function DecryptListEditor({ className }: { className?: string }) { const list = useDecryptList(); @@ -167,21 +166,3 @@ function EntryInput({ ); } - -/** Opens the decrypt list from the state pill's More…, until Settings holds it */ -export const decryptListDialogOpenAtom = atom(false); - -export function DecryptListDialog() { - const [open, setOpen] = useAtom(decryptListDialogOpenAtom); - return ( - setOpen(false)} - size="dynamic" - title="Decrypt List" - className="w-[36rem]" - > -
{open && }
-
- ); -} diff --git a/apps/yaak-proxy/components/FlowCard.tsx b/apps/yaak-proxy/components/FlowCard.tsx new file mode 100644 index 000000000..977b04326 --- /dev/null +++ b/apps/yaak-proxy/components/FlowCard.tsx @@ -0,0 +1,307 @@ +import { formatSize } from "@yaakapp-internal/lib/formatSize"; +import type { Flow, FlowBody } from "@yaakapp-internal/proxy-lib"; +import { + CountBadge, + Dropdown, + EmptyStateText, + type ExtraPaneTab, + HttpRequestPaneView, + HttpResponsePaneView, + type HttpBodyResult, + Icon, + IconButton, + parseCookieHeaders, + ResponseCookies, + type ResponseViewMode, + SplitLayout, +} from "@yaakapp-internal/ui"; +import classNames from "classnames"; +import { useSetAtom } from "jotai"; +import { useCallback, useMemo, useState } from "react"; +import { BODY_TYPE_BINARY } from "@yaakapp-internal/ui/httpTypes"; +import { findHeader } from "../lib/body"; +import { + browserTabMemory, + defaultTabFor, + type FlowPane, + learnTab, + REQUEST_TAB_AUTH, + REQUEST_TAB_BODY, + REQUEST_TAB_COOKIES, + REQUEST_TAB_HEADERS, + REQUEST_TAB_PARAMS, + RESPONSE_TAB_BODY, + RESPONSE_TAB_COOKIES, + RESPONSE_TAB_HEADERS, + RESPONSE_TAB_INFO, + RESPONSE_TAB_MESSAGES, + RESPONSE_TAB_TIMELINE, +} from "../lib/defaultTab"; +import { saveDialogFlowAtom, useFlowActions } from "../lib/flowActions"; +import { + type FlowBodyData, + flowBodyLoader, + flowCookiesView, + flowRequestToView, + flowResponseToView, +} from "../lib/flowViews"; +import { messagesTabLabel } from "../lib/messages"; +import { FlowBodyViewer, useFlowBody } from "./FlowBodyViewer"; +import { FlowInfo } from "./FlowInfo"; +import { FlowRequestBody } from "./FlowRequestBody"; +import { FlowMessages } from "./FlowMessages"; +import { FlowTimeline } from "./FlowTimeline"; +import { ProxyEditorHost } from "./ProxyEditorHost"; + +/** + * One card for a flow: the request half (its URL line and tabs), a hairline, and the response + * half (its status line and tabs). The halves sit perpendicular to the list: stacked when the + * list is beside the card, side by side when it is above. + */ +export function FlowCard({ flow, listBeside }: { flow: Flow; listBeside: boolean }) { + return ( + + ( +
+ +
+ )} + secondSlot={({ style, orientation }) => ( +
+ +
+ )} + /> +
+ ); +} + +/** The tab a half opens on, and learning from the ones chosen by hand */ +function useShapeTab(pane: FlowPane, flow: Flow, available: string[]) { + const memory = browserTabMemory(); + // Read again for every flow, so a tab learned on one opens the next of its shape + const defaultTab = defaultTabFor(pane, flow, available, memory); + const onTabChange = useCallback( + (tab: string) => learnTab(pane, flow, tab, memory), + [pane, flow, memory], + ); + return { defaultTab, onTabChange }; +} + +/** A body read through the engine's body RPC, as the shared panes' loaders ask for it */ +function useLoadedBody( + flow: Flow, + which: FlowBody, + enabled = true, +): HttpBodyResult { + const size = which === "request" ? flow.requestSize : flow.responseSize; + const load = enabled && size > 0; + const body = useFlowBody(flow, which, load); + if (!load) return { data: null, isLoading: false, error: null }; + return { data: body.data ?? undefined, isLoading: body.isLoading, error: body.error }; +} + +function FlowRequestHalf({ flow }: { flow: Flow }) { + const body = useLoadedBody(flow, "request"); + const request = useMemo(() => flowRequestToView(flow, body.data ?? null), [flow, body.data]); + const loader = useMemo( + () => + flowBodyLoader({ + useBody: () => useLoadedBody(flow, "request"), + contentType: findHeader(flow.requestHeaders, "content-type"), + }), + [flow], + ); + const cookies = useMemo(() => flowCookiesView(flow), [flow]); + const sentCount = useMemo( + () => parseCookieHeaders(cookies.cookieHeaders).sentCookies.length, + [cookies], + ); + const extraTabs = useMemo( + () => [ + { + tab: { + value: REQUEST_TAB_COOKIES, + label: "Cookies", + empty: sentCount === 0, + rightSlot: , + }, + content: , + }, + ], + [cookies, sentCount], + ); + const available = [ + REQUEST_TAB_BODY, + REQUEST_TAB_PARAMS, + REQUEST_TAB_HEADERS, + ...(request.auth == null ? [] : [REQUEST_TAB_AUTH]), + REQUEST_TAB_COOKIES, + ]; + const { defaultTab, onTabChange } = useShapeTab("request", flow, available); + + return ( + } + renderBody={(editor) => ( + + ) : null + } + requestSize={flow.requestSize} + /> + )} + /> + ); +} + +/** The method and URL, at the status line's height so the two header lines align */ +function UrlLine({ flow }: { flow: Flow }) { + return ( +
+ {flow.method} + + {flow.url} + +
+ ); +} + +function FlowResponseHalf({ flow, className }: { flow: Flow; className: string }) { + const [viewMode, setViewMode] = useState("pretty"); + // A stream's body is its messages, read by the Messages tab; reading it whole again on every + // message would only hold copies of it + const loadBody = flow.streamKind == null; + const body = useLoadedBody(flow, "response", loadBody); + const response = useMemo(() => flowResponseToView(flow, body.data ?? null), [flow, body.data]); + const loader = useMemo( + () => + flowBodyLoader({ + useBody: () => useLoadedBody(flow, "response", loadBody), + contentType: findHeader(flow.responseHeaders, "content-type"), + }), + [flow, loadBody], + ); + const openSave = useSetAtom(saveDialogFlowAtom); + const flowActions = useFlowActions(); + const messagesLabel = messagesTabLabel(flow); + + const extraTabs = useMemo( + () => [ + { + tab: { value: RESPONSE_TAB_TIMELINE, label: "Timeline" }, + content: , + }, + { tab: { value: RESPONSE_TAB_INFO, label: "Info" }, content: }, + ...(messagesLabel == null + ? [] + : [ + { + tab: { + value: RESPONSE_TAB_MESSAGES, + label: messagesLabel, + empty: flow.messageCount === 0, + rightSlot: ( + + ), + }, + content: , + }, + ]), + ], + [flow, messagesLabel], + ); + const available = [ + RESPONSE_TAB_BODY, + RESPONSE_TAB_HEADERS, + RESPONSE_TAB_COOKIES, + RESPONSE_TAB_TIMELINE, + RESPONSE_TAB_INFO, + ...(messagesLabel == null ? [] : [RESPONSE_TAB_MESSAGES]), + ]; + const { defaultTab, onTabChange } = useShapeTab("response", flow, available); + const saved = flow.savedRequestId != null; + + return ( + Waiting for a response} + viewMode={viewMode} + onViewModeChange={setViewMode} + tabsStorageKey="proxy_response_tabs" + activeTabKey={flow.id} + defaultTab={defaultTab} + onTabChange={onTabChange} + extraTabs={extraTabs} + renderBodyViewer={() => ( + + )} + statusLineCenter={ + flow.responseTruncated ? ( + + + {body.data == null ? "Body cut" : `First ${formatSize(body.data.storedSize)} kept`} + + ) : undefined + } + statusLineRight={ +
+ openSave(flow)} + /> + + + +
+ } + /> + ); +} diff --git a/apps/yaak-proxy/components/FlowDetail.tsx b/apps/yaak-proxy/components/FlowDetail.tsx index a17875b2a..5c31ac522 100644 --- a/apps/yaak-proxy/components/FlowDetail.tsx +++ b/apps/yaak-proxy/components/FlowDetail.tsx @@ -1,50 +1,22 @@ -import { formatSize } from "@yaakapp-internal/lib/formatSize"; -import type { Flow } from "@yaakapp-internal/proxy-lib"; -import { - Banner, - Button, - CountBadge, - Dropdown, - type DropdownItem, - formatMillis, - HStack, - HttpStatusTagRaw, - Icon, - IconButton, - LoadingIcon, - SizeTag, - type TabButtonRenderProps, - type TabItem, - TabContent, - Tabs, -} from "@yaakapp-internal/ui"; import classNames from "classnames"; -import { useAtomValue, useSetAtom } from "jotai"; +import { useAtomValue } from "jotai"; import type { CSSProperties } from "react"; -import { useMemo, useState } from "react"; import { selectedFlowAtom } from "../lib/atoms"; -import { findHeader, mimeEssence } from "../lib/body"; -import { saveDialogFlowAtom, useFlowActions } from "../lib/flowActions"; -import { statusReason } from "../lib/statusReason"; -import { messagesTabLabel, streamIsOpen } from "../lib/messages"; -import { EmptyState, FlowBodyViewer, useFlowBody } from "./FlowBodyViewer"; -import { FlowHeaders, FlowQuery, queryParams } from "./FlowHeaders"; -import { FlowMessages } from "./FlowMessages"; +import { EmptyState } from "./FlowBodyViewer"; +import { FlowCard } from "./FlowCard"; import { NotDecrypted } from "./NotDecrypted"; -const TAB_RESPONSE = "response"; -const TAB_REQUEST = "request"; -const TAB_HEADERS = "headers"; -const TAB_QUERY = "query"; -const TAB_MESSAGES = "messages"; - +/** + * The detail pane: the selected flow's card, or what the pane holds before one is selected. Side + * by side, a wider gutter separates the card from the list and it sits a little under the header. + */ export function FlowDetail({ style, sideBySide }: { style?: CSSProperties; sideBySide: boolean }) { const flow = useAtomValue(selectedFlowAtom); return (
) : ( - + )}
); } -function FlowPane({ flow }: { flow: Flow }) { - const [pretty, setPretty] = useState(true); - const responseBody = useFlowBody(flow, "response", flow.responseTruncated); - // Failed or cancelled before any response; after the headers, the body was cut short mid-way. - const noResponse = (flow.error != null || flow.cancelled) && flow.status == null; - const mime = mimeEssence(findHeader(flow.responseHeaders, "content-type")); - const queryCount = useMemo(() => queryParams(flow.url).length, [flow.url]); - const openSave = useSetAtom(saveDialogFlowAtom); - const flowActions = useFlowActions(); - const messagesLabel = messagesTabLabel(flow); - - const tabs = useMemo( - () => [ - ...(messagesLabel == null - ? [] - : [ - { - value: TAB_MESSAGES, - label: messagesLabel, - rightSlot: , - }, - ]), - { - value: TAB_RESPONSE, - renderButton: ({ buttonProps, isActive }) => ( - - ), - }, - { - value: TAB_REQUEST, - label: "Request", - rightSlot: flow.requestSize > 0 ? : null, - }, - { - value: TAB_HEADERS, - label: "Headers", - rightSlot: ( - - ), - }, - { - value: TAB_QUERY, - label: "Query", - rightSlot: , - }, - ], - [ - messagesLabel, - flow.messageCount, - pretty, - mime, - flow.requestSize, - flow.requestHeaders.length, - flow.responseHeaders.length, - queryCount, - ], - ); - - return ( -
-
- - {streamIsOpen(flow) && } - {flow.cancelled && flow.status == null ? ( - Cancelled - ) : ( - - )} - {flow.elapsedMs != null && ( - <> - • - - {formatMillis(flow.elapsedMs)} - - - )} - {!noResponse && ( - <> - • - - - )} - - {flow.responseTruncated ? ( - - - {responseBody.data == null - ? "Body cut" - : `First ${formatSize(responseBody.data.storedSize)} kept`} - - ) : ( - - )} - - - - - - -
- -
- {flow.error != null && ( - - {flow.error} - - )} - {flow.cancelled && ( - - {flow.status == null - ? "The app cancelled the request before a response came back." - : "The app cancelled the request before the response finished."} - - )} - - - {noResponse ? ( - -
- {flow.cancelled ? "Cancelled" : "No response"} -
-
- {flow.cancelled - ? "The app cancelled the request before a response came back." - : "The request never got a response."}{" "} - The Request tab has what the app sent. -
-
- ) : ( - - )} -
- - - - - - - - - - {messagesLabel != null && ( - - - - )} -
-
-
- ); -} - -interface ViewModeButtonProps { - buttonProps: TabButtonRenderProps["buttonProps"]; - isActive: boolean; - pretty: boolean; - onChangePretty: (pretty: boolean) => void; - prettyLabel: string; - allowRaw: boolean; -} - -/** The client's Response tab: a click on the active tab opens its view menu */ -function ViewModeButton({ - buttonProps, - isActive, - pretty, - onChangePretty, - prettyLabel, - allowRaw, -}: ViewModeButtonProps) { - const showPretty = pretty || !allowRaw; - const items: DropdownItem[] = [ - { - label: prettyLabel, - leftSlot: , - onSelect: () => onChangePretty(true), - }, - { - label: `${prettyLabel} (Raw)`, - hidden: !allowRaw, - leftSlot: , - onSelect: () => onChangePretty(false), - }, - ]; - - return ( - - - - ); -} - -function CheckMark({ visible }: { visible: boolean }) { - return ; -} - function Keys({ keys }: { keys: string[] }) { return ( diff --git a/apps/yaak-proxy/components/FlowHeaders.tsx b/apps/yaak-proxy/components/FlowHeaders.tsx deleted file mode 100644 index 01d8c7496..000000000 --- a/apps/yaak-proxy/components/FlowHeaders.tsx +++ /dev/null @@ -1,109 +0,0 @@ -import { formatSize } from "@yaakapp-internal/lib/formatSize"; -import type { Flow } from "@yaakapp-internal/proxy-lib"; -import { - DetailsBanner, - formatMillis, - HeadersTable, - KeyValueRow, - KeyValueRows, -} from "@yaakapp-internal/ui"; -import { useMemo } from "react"; -import { EmptyState } from "./FlowBodyViewer"; - -const startedFormat = new Intl.DateTimeFormat("en-US", { - month: "short", - day: "numeric", - year: "numeric", - hour: "numeric", - minute: "2-digit", - second: "2-digit", - fractionalSecondDigits: 3, - timeZoneName: "short", -}); - -const bytesFormat = new Intl.NumberFormat("en-US"); - -export function FlowHeaders({ flow }: { flow: Flow }) { - const versions = - flow.upstreamHttpVersion == null - ? `${flow.clientHttpVersion} from the app` - : `${flow.clientHttpVersion} from the app, ${flow.upstreamHttpVersion} to the server`; - - return ( -
- Info}> - - - {startedFormat.format(new Date(`${flow.startedAt}Z`))} - - {flow.url} - {flow.remoteAddr ?? } - {versions} - - {flow.elapsedMs == null ? ( - - ) : flow.elapsedHeadersMs == null ? ( - `${formatMillis(flow.elapsedMs)} in all` - ) : ( - `${formatMillis(flow.elapsedMs)} in all, headers after ${formatMillis(flow.elapsedHeadersMs)}` - )} - - - - {formatSize(flow.requestSize)} sent, {formatSize(flow.responseSize)} received{" "} - - ({bytesFormat.format(flow.responseSize)} B) - - - - - - - -
- ); -} - -export function FlowQuery({ flow }: { flow: Flow }) { - const params = useMemo(() => queryParams(flow.url), [flow.url]); - - if (params.length === 0) { - return No query parameters; - } - - return ( -
- - {params.map(([name, value], i) => ( - // oxlint-disable-next-line react/no-array-index-key -- a name can repeat - - {value} - - ))} - -
- ); -} - -export function queryParams(url: string): [string, string][] { - try { - return [...new URL(url).searchParams.entries()]; - } catch { - return []; - } -} - -function Missing() { - return --; -} diff --git a/apps/yaak-proxy/components/FlowInfo.tsx b/apps/yaak-proxy/components/FlowInfo.tsx new file mode 100644 index 000000000..601f997fd --- /dev/null +++ b/apps/yaak-proxy/components/FlowInfo.tsx @@ -0,0 +1,125 @@ +import { formatSize } from "@yaakapp-internal/lib/formatSize"; +import type { Flow } from "@yaakapp-internal/proxy-lib"; +import { DetailsBanner, formatMillis, KeyValueRow, KeyValueRows } from "@yaakapp-internal/ui"; +import { format } from "date-fns"; +import type { HTMLAttributes, ReactElement, ReactNode } from "react"; +import { utcTimestamp } from "../lib/flowViews"; +import { folderOf } from "../lib/search"; +import { AppIcon } from "./AppIcon"; + +/** + * Where a flow came from and how it travelled: in a proxy you didn't send the request, so the + * connection on each leg, the app and its process, timing, sizes and the flow's record are part + * of the answer. + */ +export function FlowInfo({ flow }: { flow: Flow }) { + const folder = folderOf(flow); + const clientPort = portOf(flow.clientAddr); + + return ( +
+
+ {[ + row("remote", "Remote Address", flow.remoteAddr, { copy: true }), + row("client_version", "Client Version", flow.clientHttpVersion), + row("server_version", "Server Version", flow.upstreamHttpVersion), + row("tls", "TLS", flow.upstreamTlsVersion), + row("cipher", "Cipher", flow.upstreamTlsCipher), + row("client_port", "Client Port", clientPort), + ]} +
+
+ {[ + row( + "app", + "Name", + flow.clientName == null ? null : ( + + {flow.clientPath != null && } + {flow.clientName} + {folder != null && {folder}} + + ), + ), + flow.clientBundleId != null + ? row("bundle", "Bundle ID", flow.clientBundleId, { copy: true }) + : row("path", "Path", flow.clientPath, { copy: true }), + row("pid", "PID", flow.clientPid == null ? null : `${flow.clientPid}`), + row("cwd", "Directory", flow.clientCwd, { copy: true }), + ]} +
+
+ {[ + row( + "started", + "Started", + format(utcTimestamp(flow.startedAt), "MMM d, yyyy, h:mm:ss.SSS a"), + ), + row("headers", "Headers", ms(flow.elapsedHeadersMs)), + row("total", "Total", ms(flow.elapsedMs)), + ]} +
+
+ {[ + row("request_size", "Request", formatSize(flow.requestSize)), + row("response_size", "Response", formatSize(flow.responseSize)), + ]} +
+
+ {[ + row("number", "Number", `#${flow.number}`, { copy: true }), + row("id", "ID", flow.id, { copy: true }), + row("saved", "Saved As", flow.savedRequestId, { copy: true }), + ]} +
+
+ ); +} + +/** A row as `KeyValueRows` takes it */ +type Row = ReactElement>; + +function Section({ title, children }: { title: string; children: (Row | null)[] }) { + const rows = children.filter((r): r is Row => r != null); + if (rows.length === 0) return null; + return ( + {title}}> + {rows} + + ); +} + +/** A row, or nothing when there's no value for it */ +function row( + key: string, + label: string, + value: ReactNode | null | undefined, + opts: { copy?: boolean } = {}, +): Row | null { + if (value == null || value === "") return null; + return ( + + {value} + + ); +} + +function ms(value: number | null): string | null { + return value == null ? null : formatMillis(value); +} + +/** The port of `127.0.0.1:54321` or `[::1]:54321` */ +export function portOf(addr: string | null | undefined): string | null { + const port = addr?.match(/:(\d+)$/)?.[1]; + return port ?? null; +} + +function copyText(text: string) { + navigator.clipboard.writeText(text).catch(console.error); +} diff --git a/apps/yaak-proxy/components/FlowList.tsx b/apps/yaak-proxy/components/FlowList.tsx index 36811a46c..451acae63 100644 --- a/apps/yaak-proxy/components/FlowList.tsx +++ b/apps/yaak-proxy/components/FlowList.tsx @@ -329,7 +329,7 @@ export function FlowList({ style, className }: Props) { className={classNames( "absolute bottom-3 left-1/2 -translate-x-1/2", "flex items-center gap-1.5 h-6 px-3 rounded-full", - "bg-surface border border-border shadow-lg text-text text-xs", + "bg-surface border border-border shadow-lg text-text text-xs tabular-nums", "hover:bg-surface-highlight", )} > diff --git a/apps/yaak-proxy/components/FlowListHeader.tsx b/apps/yaak-proxy/components/FlowListHeader.tsx index b65c56fd0..45971fe0c 100644 --- a/apps/yaak-proxy/components/FlowListHeader.tsx +++ b/apps/yaak-proxy/components/FlowListHeader.tsx @@ -107,11 +107,8 @@ export const FlowListHeader = memo(function FlowListHeader({ - ) - } - /> - ); -} - -/** Saves whether the system proxy turns on as the app launches */ -function useSetSystemProxyAtLaunch() { - const store = useStore(); - return useRpcMutation("set_system_proxy_at_launch", { - onSettled: () => void refreshProxyState(store), - }); -} - -/** Saves the HTTP versions offered to clients, which the engine applies to new connections */ -function useSetHttpVersion() { - const store = useStore(); - return useRpcMutation("set_http_version", { - onSettled: () => void refreshProxyState(store), - }); -} - export function useCopyEnvironment() { const proxyState = useAtomValue(proxyStateAtom); const caStatus = useRpcQuery("ca_status", {}); diff --git a/apps/yaak-proxy/components/SearchBar.tsx b/apps/yaak-proxy/components/SearchBar.tsx index 91954ac83..44d1cc045 100644 --- a/apps/yaak-proxy/components/SearchBar.tsx +++ b/apps/yaak-proxy/components/SearchBar.tsx @@ -7,9 +7,7 @@ import { searchTermsAtom, searchTextAtom, searchTextVersionAtom, - searchWindowAtom, setSearchTextAtom, - shownTotalAtom, } from "../lib/store"; /** How long typing settles before the search runs */ @@ -17,8 +15,8 @@ const SEARCH_DEBOUNCE_MS = 200; /** * The header's search over every flow in the database, by URL, method, status and content type, - * and the `@app`, `@host` and `@path` terms the sidebar writes. Its placeholder counts the flows; - * a search's matches show inside it. ⌘F focuses it and Escape clears it. + * and the `@app`, `@host` and `@path` terms the sidebar writes. Its placeholder counts the flows. + * ⌘F focuses it and Escape clears it. */ export const SearchBar = memo(function SearchBar() { const [text, setText] = useAtom(searchTextAtom); @@ -75,26 +73,24 @@ export const SearchBar = memo(function SearchBar() { }} leftSlot={ <> - + } rightSlot={ text === "" ? ( - + ⌘F ) : ( -
- - -
+ ) } /> @@ -118,16 +114,3 @@ function placeholderFor(total: number): string { if (total === 0) return "Search flows"; return `Search ${total.toLocaleString()} ${total === 1 ? "flow" : "flows"}`; } - -/** A search's matches of all traffic, "12 of 4,102", inside the field before its x */ -function MatchCount() { - const matches = useAtomValue(shownTotalAtom); - const total = useAtomValue(allTrafficTotalAtom); - const search = useAtomValue(searchWindowAtom); - if (search == null || search.loading) return null; - return ( - - {matches.toLocaleString()} of {total.toLocaleString()} - - ); -} diff --git a/apps/yaak-proxy/components/Settings.tsx b/apps/yaak-proxy/components/Settings.tsx new file mode 100644 index 000000000..169b5359c --- /dev/null +++ b/apps/yaak-proxy/components/Settings.tsx @@ -0,0 +1,181 @@ +import { useQuery } from "@tanstack/react-query"; +import { getVersion } from "@tauri-apps/api/app"; +import { appDataDir, appLogDir } from "@tauri-apps/api/path"; +import { + Dialog, + EmptyStateText, + SETTINGS_TAB_GENERAL, + SETTINGS_TAB_INTERFACE, + SETTINGS_TAB_LICENSE, + SETTINGS_TAB_THEME, + type SettingsCallbacks, + SettingsDialog, + SettingsGeneral, + SettingsInterface, + SettingsTheme, +} from "@yaakapp-internal/ui"; +import { atom, useAtom, useSetAtom } from "jotai"; +import { useEffect, useMemo, useState } from "react"; +import { settingsValuesAtom } from "../lib/interfaceSettings"; +import { openLink, revealPath } from "../lib/opener"; +import { proxySettingsView } from "../lib/settings"; +import { getOsType } from "../lib/tauri"; +import { useShowToast } from "../lib/toast"; +import { SettingsProxy } from "./SettingsProxy"; + +export const SETTINGS_TAB_PROXY = "proxy"; + +/** The settings window is open, on a tab, or closed */ +export const settingsOpenAtom = atom(null); + +/** + * Yaak's settings, shared: its General, Theme, Interface and License tabs, plus the proxy's own + * Proxy tab. The header's gear and `⌘,` open it. + */ +export function SettingsWindow() { + const [tab, setTab] = useAtom(settingsOpenAtom); + useSettingsHotkey(); + return ( + setTab(null)} + size="lg" + noPadding + noScroll + className="h-[min(44rem,90vh)]" + > + {tab != null && } + + ); +} + +/** The settings themselves, as the window holds them */ +export function SettingsContent({ defaultTab }: { defaultTab: string }) { + const [values, change] = useAtom(settingsValuesAtom); + const app = useAppInfo(); + const systemAppearance = useSystemAppearance(); + const view = useMemo( + () => proxySettingsView(values, app, systemAppearance), + [values, app, systemAppearance], + ); + const showToast = useShowToast(); + const callbacks = useMemo(() => { + const fail = (message: string) => showToast({ message, color: "danger" }); + return { + onChange: (patch) => change(patch), + // The proxy draws its own title bar always; the row isn't shown + onApplyNativeTitlebar: (useNativeTitlebar) => change({ useNativeTitlebar }), + onCheckForUpdates: () => undefined, + onActivateLicense: async () => undefined, + onDeactivateLicense: () => undefined, + onOpenUrl: (url) => void openLink(url, fail), + onRevealPath: (path) => void revealPath(path, fail), + pricingUrl: (intent) => `https://yaak.app/pricing?intent=${encodeURIComponent(intent)}`, + }; + }, [change, showToast]); + const props = { view, callbacks }; + + return ( + , + }, + { + value: SETTINGS_TAB_THEME, + label: "Theme", + icon: "palette", + content: , + }, + { + value: SETTINGS_TAB_INTERFACE, + label: "Interface", + icon: "columns_2", + content: , + }, + { + value: SETTINGS_TAB_PROXY, + label: "Proxy", + icon: "wifi", + content: , + }, + { + value: SETTINGS_TAB_LICENSE, + label: "License", + icon: "key_round", + content: Coming soon, + }, + ]} + /> + ); +} + +/** `⌘,`, or `Ctrl+,` off macOS, opens settings, as in Yaak */ +function useSettingsHotkey() { + const setOpen = useSetAtom(settingsOpenAtom); + useEffect(() => { + const mac = getOsType() === "macos"; + function onKeyDown(e: KeyboardEvent) { + if (e.key !== "," || e.shiftKey || e.altKey) return; + if (mac ? !e.metaKey || e.ctrlKey : !e.ctrlKey || e.metaKey) return; + e.preventDefault(); + setOpen((tab) => tab ?? SETTINGS_TAB_GENERAL); + } + window.addEventListener("keydown", onKeyDown); + return () => window.removeEventListener("keydown", onKeyDown); + }, [setOpen]); +} + +function useAppInfo() { + const os = getOsType(); + const { data } = useQuery({ + queryKey: ["settings_app_info"], + queryFn: async () => { + const [version, dataDir, logDir] = await Promise.all([ + getVersion(), + appDataDir(), + appLogDir(), + ]); + return { version, dataDir, logDir }; + }, + retry: false, + }); + return useMemo( + () => ({ + name: "Yaak Proxy", + version: data?.version ?? "", + dataDir: data?.dataDir ?? "", + logDir: data?.logDir ?? "", + revealPathLabel: + os === "macos" + ? "Reveal in Finder" + : os === "windows" + ? "Show in Explorer" + : "Show in Folder", + }), + [data, os], + ); +} + +function useSystemAppearance(): "light" | "dark" { + const query = "(prefers-color-scheme: dark)"; + const [dark, setDark] = useState(() => window.matchMedia(query).matches); + useEffect(() => { + const media = window.matchMedia(query); + const onChange = () => setDark(media.matches); + media.addEventListener("change", onChange); + return () => media.removeEventListener("change", onChange); + }, []); + return dark ? "dark" : "light"; +} + +/** Opens settings on a tab */ +export function useOpenSettings() { + const setOpen = useSetAtom(settingsOpenAtom); + return (tab: string = SETTINGS_TAB_GENERAL) => setOpen(tab); +} diff --git a/apps/yaak-proxy/components/SettingsProxy.tsx b/apps/yaak-proxy/components/SettingsProxy.tsx new file mode 100644 index 000000000..5ed520827 --- /dev/null +++ b/apps/yaak-proxy/components/SettingsProxy.tsx @@ -0,0 +1,366 @@ +import { useQueryClient } from "@tanstack/react-query"; +import { formatSize } from "@yaakapp-internal/lib/formatSize"; +import type { HttpVersion, SystemProxyStatus } from "@yaakapp-internal/proxy-lib"; +import { + Button, + Dialog, + DialogFooter, + Heading, + Icon, + RadioDropdown, + SettingRow, + SettingRowBoolean, + PlainInput, + SettingRowSelect, + SettingSelectControl, + SettingSnippet, + SettingsList, + SettingsSection, + VStack, +} from "@yaakapp-internal/ui"; +import { useAtomValue, useSetAtom, useStore } from "jotai"; +import { useState } from "react"; +import { useRpcMutation } from "../hooks/useRpcMutation"; +import { useRpcQuery } from "../hooks/useRpcQuery"; +import { helperDialogOpenAtom } from "../lib/atoms"; +import { + defaultShell, + environmentBlock, + environmentVars, + TERMINAL_SHELLS, + type TerminalShell, +} from "../lib/env"; +import { + CAPTURE_LIMIT_OPTIONS, + KEEP_TRAFFIC_OPTIONS, + sizeFromOptionValue, + sizeOptionsFor, + sizeOptionValue, +} from "../lib/settings"; +import { helperRow } from "../lib/systemProxy"; +import { proxyStateAtom, refreshProxyState } from "../lib/store"; +import { getOsType } from "../lib/tauri"; +import { useShowToast } from "../lib/toast"; +import { DecryptListEditor } from "./DecryptListEditor"; + +/** + * The proxy's own settings tab: how it listens, what it decrypts, the system proxy and its + * helper, how much traffic it keeps, and the environment for one terminal. Everything set once + * and left alone, which is what left the state pill's menu. + */ +export function SettingsProxy() { + const proxyState = useAtomValue(proxyStateAtom); + const store = useStore(); + const refresh = { onSettled: () => void refreshProxyState(store) }; + const setHttpVersion = useRpcMutation("set_http_version", refresh); + const setCaptureLimit = useRpcMutation("set_capture_body_limit", refresh); + const setAtLaunch = useRpcMutation("set_system_proxy_at_launch", refresh); + const systemProxy = useRpcQuery("system_proxy_status", {}); + // Another process owning the engine wouldn't see changes until it restarts + const elsewhere = proxyState?.otherOwner != null; + + return ( + +
+ Proxy +

Listening, decryption, storage and terminals

+
+ + + + + name="httpVersion" + title="HTTP version" + description="For new connections" + value={proxyState?.httpVersion ?? "auto"} + disabled={elsewhere} + onChange={(httpVersion) => setHttpVersion.mutate({ httpVersion })} + options={[ + { label: "Auto", value: "auto" }, + { label: "HTTP/1.1 only", value: "http1Only" }, + ]} + /> + { + const bytes = sizeFromOptionValue(v); + if (bytes != null) setCaptureLimit.mutate({ bytes }); + }} + options={sizeOptionsFor(CAPTURE_LIMIT_OPTIONS, proxyState?.captureBodyLimit ?? null) + .filter((o) => o.bytes != null) + .map(({ label, value }) => ({ label, value }))} + /> + + + + + + + {systemProxy.data?.supported === true && ( + + setAtLaunch.mutate({ enabled })} + /> + + + )} + + + + + + + + + + +
+ ); +} + +/** A port from 1 to 65535, or null */ +function parsePort(text: string): number | null { + if (!/^\d+$/.test(text.trim())) return null; + const port = Number(text.trim()); + return port > 0 && port < 65536 ? port : null; +} + +/** + * The port, saved when the field is left or Enter is pressed rather than on each keystroke: + * every save restarts a running proxy, so typing 8080 must not restart it on 8, 80 and 808. + */ +function PortRow({ disabled }: { disabled: boolean }) { + const proxyState = useAtomValue(proxyStateAtom); + const store = useStore(); + const showToast = useShowToast(); + const saved = proxyState?.configuredPort ?? 9090; + const [draft, setDraft] = useState(null); + // Puts the saved port back in the field after a rejected or failed change + const [resets, setResets] = useState(0); + const reset = () => { + setDraft(null); + setResets((n) => n + 1); + }; + const setPort = useRpcMutation("set_port", { + onError: (err) => showToast({ message: err.message, color: "danger" }), + onSettled: () => { + reset(); + void refreshProxyState(store); + }, + }); + const commit = () => { + if (draft == null || setPort.isPending) return; + const port = parsePort(draft); + if (port == null || port === saved) { + reset(); + return; + } + setPort.mutate({ port }); + }; + return ( + + parsePort(v) != null} + onChange={setDraft} + onBlur={commit} + onKeyDownCapture={(e) => { + if (e.key === "Enter") commit(); + }} + /> + + ); +} + +function HelperRow({ status }: { status: SystemProxyStatus }) { + const queryClient = useQueryClient(); + const openDialog = useSetAtom(helperDialogOpenAtom); + const unregister = useRpcMutation("helper_unregister", { + onSuccess: (status) => + queryClient.setQueryData(["system_proxy_status", {}], status), + }); + const row = helperRow(status.helper, status.backend); + return ( + {unregister.error.message}
+ ) : ( + `${capitalize(row.title.replace(/^Helper: /, ""))}. ${row.description}` + ) + } + > + {row.action != null && ( + + )} + + ); +} + +function KeepTrafficRow() { + const proxyState = useAtomValue(proxyStateAtom); + const store = useStore(); + const size = useRpcQuery("get_storage_size", {}, { refetchInterval: 10_000 }); + const setKeep = useRpcMutation("set_keep_traffic", { + onSettled: () => void refreshProxyState(store), + }); + const keep = proxyState?.keepTrafficBytes ?? null; + return ( + +
+ {size.data != null && ( + + {formatSize(size.data.bytes)} used + + )} + setKeep.mutate({ bytes: sizeFromOptionValue(v) })} + options={sizeOptionsFor(KEEP_TRAFFIC_OPTIONS, keep).map(({ label, value }) => ({ + label, + value, + }))} + /> +
+
+ ); +} + +function ResetRow() { + const [confirming, setConfirming] = useState(false); + const store = useStore(); + const queryClient = useQueryClient(); + const showToast = useShowToast(); + const reset = useRpcMutation("reset_captured_data", { + onSuccess: () => { + setConfirming(false); + void queryClient.invalidateQueries(); + showToast({ message: "Captured data reset" }); + }, + onSettled: () => void refreshProxyState(store), + }); + return ( + + + setConfirming(false)} + size="sm" + title="Reset captured data?" + description="Every captured flow and the keys that encrypt them are deleted." + > + {reset.error != null &&

{reset.error.message}

} + setConfirming(false) }, + { + label: "Reset", + color: "danger", + isLoading: reset.isPending, + onClick: () => reset.mutate({}), + }, + ]} + /> +
+
+ ); +} + +function TerminalRow() { + const proxyState = useAtomValue(proxyStateAtom); + const caStatus = useRpcQuery("ca_status", {}); + const showToast = useShowToast(); + const os = getOsType(); + const [shell, setShell] = useState(() => defaultShell(os)); + const openTerminal = useRpcMutation("open_terminal", { + onError: (err) => showToast({ message: err.message, color: "danger" }), + }); + const port = + proxyState?.port ?? proxyState?.otherOwner?.proxyPort ?? proxyState?.configuredPort ?? 9090; + const caPath = caStatus.data?.certPath; + const shellLabel = TERMINAL_SHELLS.find((s) => s.value === shell)?.label ?? shell; + + return ( +
+
+ Environment + setShell(v as TerminalShell)} + items={TERMINAL_SHELLS.map((s) => ({ label: s.label, value: s.value }))} + > + + + +
+

Proxies one terminal instead of the system

+ {caPath != null && ( + + )} +
+ ); +} + +function capitalize(s: string): string { + return s.charAt(0).toUpperCase() + s.slice(1); +} diff --git a/apps/yaak-proxy/components/Sidebar.tsx b/apps/yaak-proxy/components/Sidebar.tsx index d519b3ab7..5bea89e8a 100644 --- a/apps/yaak-proxy/components/Sidebar.tsx +++ b/apps/yaak-proxy/components/Sidebar.tsx @@ -63,6 +63,7 @@ function ItemInner({ item }: { item: SidebarItem }) {
{item.kind === "all" && } {item.kind === "tunnelled" && } + {item.kind === "folder" && } {item.appPath != null && } {item.label} {item.sublabel != null && ( diff --git a/apps/yaak-proxy/font-size.ts b/apps/yaak-proxy/font-size.ts index ffe5fe96f..37af96871 100644 --- a/apps/yaak-proxy/font-size.ts +++ b/apps/yaak-proxy/font-size.ts @@ -1,3 +1,6 @@ -// Yaak's default interface size, so the two apps read as one family. Later from the shared -// Interface settings. -document.documentElement.style.fontSize = "15px"; +import { applyFontSizes } from "./lib/interfaceSettings"; +import { readSettingsValues, SETTINGS_VALUES_KEY } from "./lib/settings"; + +// Yaak's default sizes unless changed in Settings' Interface tab, so the two apps read as one +// family. +applyFontSizes(readSettingsValues(localStorage.getItem(SETTINGS_VALUES_KEY))); diff --git a/apps/yaak-proxy/harness-settings.html b/apps/yaak-proxy/harness-settings.html deleted file mode 100644 index 14957782e..000000000 --- a/apps/yaak-proxy/harness-settings.html +++ /dev/null @@ -1,212 +0,0 @@ - - - - - - Yaak Proxy, shared settings harness - - - - -
- - - - - diff --git a/apps/yaak-proxy/harness/DragRegionCheck.tsx b/apps/yaak-proxy/harness/DragRegionCheck.tsx new file mode 100644 index 000000000..0d7a99deb --- /dev/null +++ b/apps/yaak-proxy/harness/DragRegionCheck.tsx @@ -0,0 +1,57 @@ +import { useEffect, useState } from "react"; + +interface Probe { + x: number; + y: number; + kind: "drag" | "control" | "dead"; +} + +/** + * Dev only, with `?window&dragcheck`: probes the header across its width and marks what a press + * at each point lands on. Green drags the window (the element hit carries + * `data-tauri-drag-region`, which Tauri reads off the target alone), blue is a control, and red + * is dead space that neither drags nor does anything. + */ +export function DragRegionCheck() { + const [probes, setProbes] = useState([]); + useEffect(() => { + const timer = setTimeout(() => { + const header = document.querySelector(".x-theme-appHeader"); + if (header == null) return; + const rect = header.getBoundingClientRect(); + const out: Probe[] = []; + for (const y of [rect.top + 6, rect.top + rect.height / 2]) { + for (let x = rect.left + 4; x < rect.right; x += 10) { + const el = document.elementFromPoint(x, y); + const kind = + el == null + ? "dead" + : el.hasAttribute("data-tauri-drag-region") + ? "drag" + : el.closest("button, input, [role=button], .cm-editor, [role=combobox]") != null + ? "control" + : "dead"; + out.push({ x, y, kind }); + } + } + setProbes(out); + }, 800); + return () => clearTimeout(timer); + }, []); + return ( +
+ {probes.map((p) => ( + + ))} +
+ ); +} diff --git a/apps/yaak-proxy/harness/FlowPanesHarness.tsx b/apps/yaak-proxy/harness/FlowPanesHarness.tsx index ff15d61ba..dc01632b0 100644 --- a/apps/yaak-proxy/harness/FlowPanesHarness.tsx +++ b/apps/yaak-proxy/harness/FlowPanesHarness.tsx @@ -1,24 +1,17 @@ -import { - HttpRequestPaneView, - HttpResponsePaneView, - type HttpBodyResult, - type ResponseViewMode, -} from "@yaakapp-internal/ui"; -import { useMemo, useState } from "react"; -import { ProxyEditorHost } from "../components/ProxyEditorHost"; -import { findHeader } from "../lib/body"; -import { - type FlowBodyData, - flowBodyFromRpc, - flowBodyLoader, - flowRequestToView, - flowResponseToView, -} from "../lib/flowViews"; +import { useSetAtom } from "jotai"; +import { useLayoutEffect, useState } from "react"; +import { FlowDetail } from "../components/FlowDetail"; +import { FlowList } from "../components/FlowList"; +import { selectedFlowAtom } from "../lib/atoms"; +import { browserTabMemory, learnedTabKey, flowShape } from "../lib/defaultTab"; +import { flowWindowAtom } from "../lib/store"; import { type FlowFixture, fixtures } from "./fixtures"; /** - * Dev only: renders captured flows through the shared request and response panes, read-only, - * side by side, the way the proxy's detail view will. Open `/harness.html?flow=0&theme=dark`. + * Dev only: captured flows in the detail card, over fixtures. Open + * `/harness.html?flow=0&theme=dark`; `&list` puts the flow list beside the card (its halves + * stacked), `&below` puts it above (side by side), and `&tab=info` opens the response half on a + * tab, as though it were chosen by hand for flows of that shape. */ export function FlowPanesHarness() { const params = new URLSearchParams(location.search); @@ -27,87 +20,57 @@ export function FlowPanesHarness() { if (fixture == null) return null; return ( - -
-
- {fixtures.map((f, i) => ( - - ))} -
- +
+
+ {fixtures.map((f, i) => ( + + ))}
- - ); -} - -function FlowPanes({ fixture }: { fixture: FlowFixture }) { - const { flow } = fixture; - const [viewMode, setViewMode] = useState("pretty"); - const requestBody = useMemo(() => flowBodyFromRpc(fixture.requestBody), [fixture]); - const responseBody = useMemo(() => flowBodyFromRpc(fixture.responseBody), [fixture]); - const request = useMemo(() => flowRequestToView(flow, requestBody), [flow, requestBody]); - const response = useMemo(() => flowResponseToView(flow, responseBody), [flow, responseBody]); - - const requestLoader = useMemo( - () => - flowBodyLoader({ - useBody: () => loaded(requestBody), - contentType: findHeader(flow.requestHeaders, "content-type"), - }), - [requestBody, flow], - ); - const responseLoader = useMemo( - () => - flowBodyLoader({ - useBody: () => loaded(responseBody), - contentType: findHeader(flow.responseHeaders, "content-type"), - }), - [responseBody, flow], - ); - - return ( -
-
- -
- Waiting for a response
} - viewMode={viewMode} - onViewModeChange={setViewMode} - tabsStorageKey="flow_response_tabs" - activeTabKey={flow.id} - onCopyAsCurl={() => undefined} - /> +
); } -function loaded(data: FlowBodyData): HttpBodyResult { - return { data, isLoading: false, error: null }; -} +function FixtureCard({ fixture }: { fixture: FlowFixture }) { + const params = new URLSearchParams(location.search); + const setSelected = useSetAtom(selectedFlowAtom); + const setWindow = useSetAtom(flowWindowAtom); + const [ready, setReady] = useState(false); + useLayoutEffect(() => { + const flows = fixtures.map((f, i) => ({ ...f.flow, number: 1401 + i })); + const tab = params.get("tab"); + if (tab != null) { + browserTabMemory()?.setItem(learnedTabKey("response", flowShape(fixture.flow)), tab); + } + setWindow({ flows, hasOlder: false, total: flows.length }); + setSelected(flows.find((f) => f.id === fixture.flow.id) ?? fixture.flow); + setReady(true); + // oxlint-disable-next-line react-hooks/exhaustive-deps -- once per fixture + }, [fixture]); + if (!ready) return null; -/** The engine formats JSON over RPC, which a page outside the app can't reach */ -async function formatLocally(text: string, language: unknown): Promise { - if (language !== "json") return text; - try { - return JSON.stringify(JSON.parse(text), null, 2); - } catch { - return text; + if (params.has("list")) { + return ( +
+ + +
+ ); } + if (params.has("below")) { + return ( +
+ + +
+ ); + } + return ; } diff --git a/apps/yaak-proxy/harness/LockedHarness.tsx b/apps/yaak-proxy/harness/LockedHarness.tsx index c3b2a80be..f8c77a071 100644 --- a/apps/yaak-proxy/harness/LockedHarness.tsx +++ b/apps/yaak-proxy/harness/LockedHarness.tsx @@ -3,6 +3,7 @@ import type { GetProxyStateResponse } from "@yaakapp-internal/proxy-lib"; import { useStore } from "jotai"; import { type ReactNode, useEffect, useState } from "react"; import { ProxyLayout } from "../components/ProxyLayout"; +import { DragRegionCheck } from "./DragRegionCheck"; import { useHotkeyContextValue } from "../lib/hotkeys"; import { syncFlows, syncProxyState, syncSidebarSelection } from "../lib/store"; @@ -21,6 +22,8 @@ const STATE: GetProxyStateResponse = { encrypting: null, locked: { message: LOCKED, keychain: true }, otherOwner: null, + captureBodyLimit: 10 * 1024 * 1024, + keepTrafficBytes: null, }; /** What the backend answers while the keychain refuses: the state, and every read refused */ @@ -55,8 +58,9 @@ export function installLockedBackend() { if (cmd.startsWith("plugin:event|")) return 0; throw new Error(`Not in the harness: ${cmd}`); }; - (window as unknown as { __TAURI_OS_PLUGIN_INTERNALS__: Record }) - .__TAURI_OS_PLUGIN_INTERNALS__ = { + ( + window as unknown as { __TAURI_OS_PLUGIN_INTERNALS__: Record } + ).__TAURI_OS_PLUGIN_INTERNALS__ = { os_type: "macos", platform: "macos", family: "unix", @@ -67,7 +71,10 @@ export function installLockedBackend() { localStorage.setItem("system_proxy_ever_on", "true"); } -/** The whole window, as the app renders it, while captured traffic is locked */ +/** + * The whole window, as the app renders it: while captured traffic is locked, or over the + * fixture backend with `?window` + */ export function LockedHarness() { const store = useStore(); const [synced, setSynced] = useState(false); @@ -81,6 +88,7 @@ export function LockedHarness() { return ( + {new URLSearchParams(location.search).has("dragcheck") && } ); } diff --git a/apps/yaak-proxy/harness/SettingsHarness.tsx b/apps/yaak-proxy/harness/SettingsHarness.tsx new file mode 100644 index 000000000..8b3be070e --- /dev/null +++ b/apps/yaak-proxy/harness/SettingsHarness.tsx @@ -0,0 +1,26 @@ +import { useStore } from "jotai"; +import { useEffect, useState } from "react"; +import { SETTINGS_TAB_PROXY, SettingsContent } from "../components/Settings"; +import { syncProxyState } from "../lib/store"; + +/** + * Dev only: the settings window over the fixture backend, at the dialog's size. Open + * `/harness.html?settings&theme=dark`; `&tab=general` opens another tab than Proxy. + */ +export function SettingsHarness() { + const store = useStore(); + const [synced, setSynced] = useState(false); + useEffect(() => { + syncProxyState(store); + setSynced(true); + }, [store]); + const tab = new URLSearchParams(location.search).get("tab") ?? SETTINGS_TAB_PROXY; + if (!synced) return null; + return ( +
+
+ +
+
+ ); +} diff --git a/apps/yaak-proxy/harness/backend.ts b/apps/yaak-proxy/harness/backend.ts new file mode 100644 index 000000000..14a7c5f05 --- /dev/null +++ b/apps/yaak-proxy/harness/backend.ts @@ -0,0 +1,97 @@ +import type { GetProxyStateResponse } from "@yaakapp-internal/proxy-lib"; +import { fixtures } from "./fixtures"; + +/** + * Dev only: answers the engine's RPCs the harness pages need, from fixtures, so the card and + * settings render outside the app as they do inside it. + */ +export function installFixtureBackend() { + const internals = (window as unknown as { __TAURI_INTERNALS__: Record }) + .__TAURI_INTERNALS__; + internals.invoke = async (cmd: string, args?: { cmd?: string; payload?: unknown }) => { + if (cmd === "rpc") return answer(args?.cmd ?? "", args?.payload); + if (cmd.startsWith("plugin:event|")) return 0; + throw new Error(`Not in the harness: ${cmd}`); + }; + (window as unknown as { __TAURI_OS_PLUGIN_INTERNALS__: Record }) + .__TAURI_OS_PLUGIN_INTERNALS__ = { + os_type: "macos", + platform: "macos", + family: "unix", + version: "15.0", + eol: "\n", + }; +} + +export const HARNESS_STATE: GetProxyStateResponse = { + state: "running", + port: 9090, + configuredPort: 9090, + recording: true, + systemProxyAtLaunch: true, + httpVersion: "auto", + captureBodyLimit: 10 * 1024 * 1024, + keepTrafficBytes: 2 * 1024 * 1024 * 1024, + startError: null, + encrypting: null, + locked: null, + otherOwner: null, +}; + +function answer(cmd: string, payload: unknown): unknown { + const req = (payload ?? {}) as Record; + switch (cmd) { + case "get_flow_body": { + const fixture = fixtures.find((f) => f.flow.id === req.flowId); + if (fixture == null) throw new Error(`No fixture for ${String(req.flowId)}`); + return req.which === "request" ? fixture.requestBody : fixture.responseBody; + } + case "format_body": { + const text = typeof req.text === "string" ? req.text : ""; + try { + return JSON.stringify(JSON.parse(text), null, 2); + } catch { + return text; + } + } + case "app_icon": + return null; + case "get_proxy_state": + return HARNESS_STATE; + case "get_storage_size": + return { bytes: 412 * 1024 * 1024 }; + case "ca_status": + return { + trusted: true, + installed: true, + certPath: "/Users/you/Library/Application Support/app.yaak.proxy/ca.pem", + }; + case "system_proxy_status": + return { + enabled: true, + port: 9090, + services: [], + backend: "helper", + helper: "enabled", + supported: true, + }; + case "get_decrypt_list": + return { + entries: [ + { entry: "host:api.stripe.com", builtIn: false }, + { entry: "app:Google Chrome", builtIn: false }, + { entry: "!host:*.apple.com", builtIn: true }, + ], + }; + case "list_actions": + return { actions: [] }; + case "list_flows": { + const flows = fixtures.map((f, i) => ({ ...f.flow, number: 1401 + i })).reverse(); + return { flows, total: flows.length, matching: null }; + } + case "list_seen": + return { hosts: [], apps: [] }; + default: + throw new Error(`Not in the harness: ${cmd}`); + } +} diff --git a/apps/yaak-proxy/harness/fixtures.ts b/apps/yaak-proxy/harness/fixtures.ts index b58b87f09..2f052dd74 100644 --- a/apps/yaak-proxy/harness/fixtures.ts +++ b/apps/yaak-proxy/harness/fixtures.ts @@ -51,6 +51,12 @@ const base: Flow = { grpcStatus: null, grpcMessage: null, tunnelled: false, + clientAddr: "127.0.0.1:54321", + elapsedConnectMs: 18, + elapsedTlsMs: 41, + elapsedRequestMs: 43, + upstreamTlsVersion: "TLSv1.3", + upstreamTlsCipher: "TLS13_AES_128_GCM_SHA256", }; const jsonRequest = JSON.stringify({ amount: 4200, currency: "usd", capture: true }); @@ -147,4 +153,26 @@ export const fixtures: FlowFixture[] = [ requestBody: body(""), responseBody: body(""), }, + { + label: "Tunnelled", + flow: { + ...base, + id: "fl_tunnel", + method: "CONNECT", + url: "bank.example.com:443", + status: null, + elapsedHeadersMs: null, + elapsedMs: 4_210, + clientHttpVersion: "HTTP/1.1", + upstreamHttpVersion: null, + clientName: "Safari", + clientBundleId: "com.apple.Safari", + clientPath: "/Applications/Safari.app", + requestSize: 3_412, + responseSize: 48_210, + tunnelled: true, + }, + requestBody: body(""), + responseBody: body(""), + }, ]; diff --git a/apps/yaak-proxy/harness/main.tsx b/apps/yaak-proxy/harness/main.tsx index 85a58af3a..8fa7d599a 100644 --- a/apps/yaak-proxy/harness/main.tsx +++ b/apps/yaak-proxy/harness/main.tsx @@ -3,22 +3,27 @@ import { applyThemeToDocument, defaultDarkTheme, defaultLightTheme } from "@yaak import { createStore, Provider } from "jotai"; import { StrictMode } from "react"; import { createRoot } from "react-dom/client"; +import { installFixtureBackend } from "./backend"; import { FlowPanesHarness } from "./FlowPanesHarness"; import { installLockedBackend, LockedHarness } from "./LockedHarness"; import { MessagesHarness } from "./MessagesHarness"; +import { SettingsHarness } from "./SettingsHarness"; import "../main.css"; const params = new URLSearchParams(location.search); const dark = params.get("theme") !== "light"; applyThemeToDocument(dark ? defaultDarkTheme : defaultLightTheme); if (params.has("locked")) installLockedBackend(); +else installFixtureBackend(); createRoot(document.getElementById("root") as HTMLElement).render( - {params.has("locked") ? ( + {params.has("locked") || params.has("window") ? ( + ) : params.has("settings") ? ( + ) : params.has("stream") ? ( ) : ( diff --git a/apps/yaak-proxy/harness/streamFixtures.ts b/apps/yaak-proxy/harness/streamFixtures.ts index 835b5d3ed..9fcbe3c9a 100644 --- a/apps/yaak-proxy/harness/streamFixtures.ts +++ b/apps/yaak-proxy/harness/streamFixtures.ts @@ -50,6 +50,12 @@ const flowBase: Flow = { grpcStatus: null, grpcMessage: null, tunnelled: false, + clientAddr: null, + elapsedConnectMs: null, + elapsedTlsMs: null, + elapsedRequestMs: null, + upstreamTlsVersion: null, + upstreamTlsCipher: null, }; /** Seconds past 21:41:12 as a stored timestamp */ diff --git a/apps/yaak-proxy/lib/atoms.ts b/apps/yaak-proxy/lib/atoms.ts index 45a2bfc2e..291ddef95 100644 --- a/apps/yaak-proxy/lib/atoms.ts +++ b/apps/yaak-proxy/lib/atoms.ts @@ -17,9 +17,6 @@ export const followingAtom = atom(true); /** When the helper was last registered, so its state is read until approval shows */ export const helperRegisteredAtAtom = atom(null); -/** Settings was asked for, by the header's gear; the settings window opens from it once it exists */ -export const settingsRequestedAtom = atom(0); - /** The dialog that walks through allowing the helper in System Settings */ export const helperDialogOpenAtom = atom(false); diff --git a/apps/yaak-proxy/lib/body.ts b/apps/yaak-proxy/lib/body.ts index 08bca1075..3fdb468bf 100644 --- a/apps/yaak-proxy/lib/body.ts +++ b/apps/yaak-proxy/lib/body.ts @@ -21,6 +21,9 @@ export function flowBodyQueryOptions(flow: Flow, which: FlowBody) { queryFn: async () => viewerBody(await rpc("get_flow_body", { flowId: flow.id, which })), // Reading a stored body fails the same way every time, so the error shows at once. retry: false, + // A stream's body changes with each message, each a new key: dropped once unwatched, so a + // long stream left selected doesn't hold a copy per message + ...(flow.streamKind == null ? {} : { gcTime: 0 }), }); } diff --git a/apps/yaak-proxy/lib/defaultTab.test.ts b/apps/yaak-proxy/lib/defaultTab.test.ts new file mode 100644 index 000000000..b0773c294 --- /dev/null +++ b/apps/yaak-proxy/lib/defaultTab.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, test } from "vite-plus/test"; +import { + defaultTabFor, + flowShape, + learnTab, + REQUEST_TAB_BODY, + REQUEST_TAB_HEADERS, + RESPONSE_TAB_BODY, + RESPONSE_TAB_HEADERS, + RESPONSE_TAB_INFO, + RESPONSE_TAB_MESSAGES, + RESPONSE_TAB_TIMELINE, + type TabMemory, +} from "./defaultTab"; + +type ShapeFlow = Parameters[0]; + +const ok: ShapeFlow = { + streamKind: null, + status: 200, + error: null, + cancelled: false, + requestSize: 0, + responseSize: 120, +}; + +function memory(): TabMemory & { items: Map } { + const items = new Map(); + return { + items, + getItem: (k) => items.get(k) ?? null, + setItem: (k, v) => void items.set(k, v), + }; +} + +const RESPONSE_TABS = [ + RESPONSE_TAB_BODY, + RESPONSE_TAB_HEADERS, + "cookies", + RESPONSE_TAB_TIMELINE, + RESPONSE_TAB_INFO, +]; + +describe("flowShape", () => { + test("names what the flow has", () => { + expect(flowShape(ok)).toBe("response_body"); + expect(flowShape({ ...ok, responseSize: 0, requestSize: 20 })).toBe("request_body"); + expect(flowShape({ ...ok, responseSize: 0 })).toBe("neither"); + expect(flowShape({ ...ok, streamKind: "websocket" })).toBe("stream"); + expect(flowShape({ ...ok, status: null, error: "refused" })).toBe("no_response"); + expect(flowShape({ ...ok, status: null, cancelled: true })).toBe("no_response"); + }); + + test("a body cut after the headers still has its response", () => { + expect(flowShape({ ...ok, error: "reset mid-body" })).toBe("response_body"); + }); +}); + +describe("defaultTabFor", () => { + test("follows the table", () => { + const m = memory(); + // A response body + expect(defaultTabFor("request", ok, [], m)).toBe(REQUEST_TAB_HEADERS); + expect(defaultTabFor("request", { ...ok, requestSize: 9 }, [], m)).toBe(REQUEST_TAB_BODY); + expect(defaultTabFor("response", ok, RESPONSE_TABS, m)).toBe(RESPONSE_TAB_BODY); + // A request body, no response body + const post = { ...ok, requestSize: 30, responseSize: 0 }; + expect(defaultTabFor("request", post, [], m)).toBe(REQUEST_TAB_BODY); + expect(defaultTabFor("response", post, RESPONSE_TABS, m)).toBe(RESPONSE_TAB_HEADERS); + // Neither + const empty = { ...ok, responseSize: 0 }; + expect(defaultTabFor("request", empty, [], m)).toBe(REQUEST_TAB_HEADERS); + expect(defaultTabFor("response", empty, RESPONSE_TABS, m)).toBe(RESPONSE_TAB_HEADERS); + // A stream + const ws = { ...ok, streamKind: "websocket" as const }; + expect(defaultTabFor("response", ws, [...RESPONSE_TABS, RESPONSE_TAB_MESSAGES], m)).toBe( + RESPONSE_TAB_MESSAGES, + ); + // No response at all + const failed = { ...ok, status: null, error: "refused", responseSize: 0 }; + expect(defaultTabFor("response", failed, RESPONSE_TABS, m)).toBe(RESPONSE_TAB_TIMELINE); + expect(defaultTabFor("request", { ...failed, requestSize: 4 }, [], m)).toBe(REQUEST_TAB_BODY); + }); + + test("a tab chosen by hand becomes the default for the same shape and pane", () => { + const m = memory(); + learnTab("response", ok, RESPONSE_TAB_INFO, m); + expect(defaultTabFor("response", ok, RESPONSE_TABS, m)).toBe(RESPONSE_TAB_INFO); + // Not for other shapes, nor the other pane + const post = { ...ok, requestSize: 30, responseSize: 0 }; + expect(defaultTabFor("response", post, RESPONSE_TABS, m)).toBe(RESPONSE_TAB_HEADERS); + expect(defaultTabFor("request", ok, ["body", "headers", "info"], m)).toBe(REQUEST_TAB_HEADERS); + }); + + test("a learned tab the flow doesn't have falls back to the table", () => { + const m = memory(); + const ws = { ...ok, streamKind: "websocket" as const }; + learnTab("response", ws, RESPONSE_TAB_MESSAGES, m); + learnTab("response", ok, RESPONSE_TAB_MESSAGES, m); + expect(defaultTabFor("response", ok, RESPONSE_TABS, m)).toBe(RESPONSE_TAB_BODY); + }); + + test("works without storage", () => { + expect(defaultTabFor("response", ok, RESPONSE_TABS, null)).toBe(RESPONSE_TAB_BODY); + }); +}); diff --git a/apps/yaak-proxy/lib/defaultTab.ts b/apps/yaak-proxy/lib/defaultTab.ts new file mode 100644 index 000000000..14c02d83f --- /dev/null +++ b/apps/yaak-proxy/lib/defaultTab.ts @@ -0,0 +1,97 @@ +import type { Flow } from "@yaakapp-internal/proxy-lib"; + +/** The request half's tabs, named as the shared request pane names them */ +export const REQUEST_TAB_BODY = "body"; +export const REQUEST_TAB_PARAMS = "params"; +export const REQUEST_TAB_HEADERS = "headers"; +export const REQUEST_TAB_AUTH = "auth"; +export const REQUEST_TAB_COOKIES = "cookies"; + +/** The response half's tabs, named as the shared response pane names them */ +export const RESPONSE_TAB_BODY = "body"; +export const RESPONSE_TAB_HEADERS = "headers"; +export const RESPONSE_TAB_COOKIES = "cookies"; +export const RESPONSE_TAB_TIMELINE = "timeline"; +export const RESPONSE_TAB_INFO = "info"; +export const RESPONSE_TAB_MESSAGES = "messages"; + +export type FlowPane = "request" | "response"; + +/** What a flow has, which picks the tab each half opens on */ +export type FlowShape = "response_body" | "request_body" | "neither" | "stream" | "no_response"; + +export function flowShape( + flow: Pick< + Flow, + "streamKind" | "status" | "error" | "cancelled" | "requestSize" | "responseSize" + >, +): FlowShape { + if (flow.streamKind != null) return "stream"; + if (flow.status == null && (flow.error != null || flow.cancelled)) return "no_response"; + if (flow.responseSize > 0) return "response_body"; + if (flow.requestSize > 0) return "request_body"; + return "neither"; +} + +/** The tab a half opens on for a shape, before anything was learned */ +export function shapeDefaultTab(pane: FlowPane, shape: FlowShape, hasRequestBody: boolean): string { + if (pane === "request") { + return hasRequestBody ? REQUEST_TAB_BODY : REQUEST_TAB_HEADERS; + } + switch (shape) { + case "response_body": + return RESPONSE_TAB_BODY; + case "stream": + return RESPONSE_TAB_MESSAGES; + case "no_response": + return RESPONSE_TAB_TIMELINE; + case "request_body": + case "neither": + return RESPONSE_TAB_HEADERS; + } +} + +/** Where the tab last chosen by hand for a shape is kept */ +export function learnedTabKey(pane: FlowPane, shape: FlowShape): string { + return `proxy_learned_tab.${pane}.${shape}`; +} + +/** Where learned tabs are read from and written to: local storage in the app */ +export interface TabMemory { + getItem: (key: string) => string | null; + setItem: (key: string, value: string) => void; +} + +/** + * The tab a half opens on for a flow with no tab of its own: the one last chosen by hand for + * flows of its shape, when this flow has that tab, else the shape's default. + */ +export function defaultTabFor( + pane: FlowPane, + flow: Parameters[0], + available: string[], + memory: TabMemory | null, +): string { + const shape = flowShape(flow); + const learned = memory?.getItem(learnedTabKey(pane, shape)) ?? null; + if (learned != null && available.includes(learned)) return learned; + return shapeDefaultTab(pane, shape, flow.requestSize > 0); +} + +/** A tab chosen by hand becomes the default for flows of the same shape */ +export function learnTab( + pane: FlowPane, + flow: Parameters[0], + tab: string, + memory: TabMemory | null, +) { + memory?.setItem(learnedTabKey(pane, flowShape(flow)), tab); +} + +export function browserTabMemory(): TabMemory | null { + try { + return window.localStorage; + } catch { + return null; + } +} diff --git a/apps/yaak-proxy/lib/env.test.ts b/apps/yaak-proxy/lib/env.test.ts index 0bd24dd08..b7708d7f0 100644 --- a/apps/yaak-proxy/lib/env.test.ts +++ b/apps/yaak-proxy/lib/env.test.ts @@ -1,19 +1,75 @@ import { describe, expect, test } from "vite-plus/test"; -import { environmentExports } from "./env"; +import { defaultShell, environmentBlock, environmentExports } from "./env"; + +const MAC_PATH = "/Users/me/Library/Application Support/app.yaak.proxy/ca/ca.pem"; describe("environmentExports", () => { test("points the proxy variables at the port and the CA variables at the certificate", () => { - const path = "/Users/me/Library/Application Support/app.yaak.proxy/ca/ca.pem"; - expect(environmentExports(9090, path)).toBe( + expect(environmentExports(9090, MAC_PATH)).toBe( [ "export HTTP_PROXY=http://127.0.0.1:9090", "export HTTPS_PROXY=http://127.0.0.1:9090", "export http_proxy=http://127.0.0.1:9090", "export https_proxy=http://127.0.0.1:9090", - `export SSL_CERT_FILE='${path}'`, - `export NODE_EXTRA_CA_CERTS='${path}'`, - `export REQUESTS_CA_BUNDLE='${path}'`, + `export SSL_CERT_FILE='${MAC_PATH}'`, + `export NODE_EXTRA_CA_CERTS='${MAC_PATH}'`, + `export REQUESTS_CA_BUNDLE='${MAC_PATH}'`, ].join("\n"), ); }); }); + +describe("environmentBlock", () => { + test("bash and zsh quote only what needs it", () => { + const lines = environmentBlock("posix", 9090, "/tmp/it's.pem").split("\n"); + expect(lines[0]).toBe("export HTTP_PROXY=http://127.0.0.1:9090"); + expect(lines[4]).toBe(`export SSL_CERT_FILE='/tmp/it'\\''s.pem'`); + }); + + test("fish", () => { + const lines = environmentBlock("fish", 8080, MAC_PATH).split("\n"); + expect(lines[0]).toBe("set -gx HTTP_PROXY http://127.0.0.1:8080"); + expect(lines[6]).toBe(`set -gx REQUESTS_CA_BUNDLE '${MAC_PATH}'`); + expect(environmentBlock("fish", 1, "/a b/it's\\x").split("\n")[4]).toBe( + `set -gx SSL_CERT_FILE '/a b/it\\'s\\\\x'`, + ); + }); + + test("PowerShell", () => { + const path = "C:\\Users\\O'Neil\\AppData\\Roaming\\app.yaak.proxy\\ca.pem"; + const lines = environmentBlock("powershell", 9090, path).split("\n"); + expect(lines[1]).toBe("$env:HTTPS_PROXY = 'http://127.0.0.1:9090'"); + expect(lines[5]).toBe( + "$env:NODE_EXTRA_CA_CERTS = 'C:\\Users\\O''Neil\\AppData\\Roaming\\app.yaak.proxy\\ca.pem'", + ); + }); + + test("cmd", () => { + const path = "C:\\Program Files\\Yaak\\ca.pem"; + const lines = environmentBlock("cmd", 9090, path).split("\n"); + expect(lines[0]).toBe(`set "HTTP_PROXY=http://127.0.0.1:9090"`); + expect(lines[4]).toBe(`set "SSL_CERT_FILE=${path}"`); + }); + + test("names every variable a tool reads", () => { + const block = environmentBlock("posix", 9090, "/ca.pem"); + for (const name of [ + "HTTP_PROXY", + "HTTPS_PROXY", + "SSL_CERT_FILE", + "NODE_EXTRA_CA_CERTS", + "REQUESTS_CA_BUNDLE", + ]) { + expect(block).toContain(`export ${name}=`); + } + }); +}); + +describe("defaultShell", () => { + test("by the platform", () => { + expect(defaultShell("macos")).toBe("posix"); + expect(defaultShell("linux")).toBe("posix"); + expect(defaultShell("windows")).toBe("powershell"); + expect(defaultShell(null)).toBe("posix"); + }); +}); diff --git a/apps/yaak-proxy/lib/env.ts b/apps/yaak-proxy/lib/env.ts index 1b7511fa7..042deb724 100644 --- a/apps/yaak-proxy/lib/env.ts +++ b/apps/yaak-proxy/lib/env.ts @@ -1,21 +1,70 @@ import { shellQuote } from "./curl"; -/** - * Shell exports that send one terminal's traffic through the proxy on `port` and trust its CA at - * `caPath`, for tools that read neither the system proxy nor the login keychain: Node, Python, - * and anything built on OpenSSL. - */ -export function environmentExports(port: number, caPath: string): string { - const proxy = `http://127.0.0.1:${port}`; - const ca = shellQuote(caPath); - return [ - `export HTTP_PROXY=${proxy}`, - `export HTTPS_PROXY=${proxy}`, - // curl reads only the lowercase name for plain HTTP. - `export http_proxy=${proxy}`, - `export https_proxy=${proxy}`, - `export SSL_CERT_FILE=${ca}`, - `export NODE_EXTRA_CA_CERTS=${ca}`, - `export REQUESTS_CA_BUNDLE=${ca}`, - ].join("\n"); +/** The shells the terminal environment is written for */ +export type TerminalShell = "posix" | "fish" | "powershell" | "cmd"; + +export const TERMINAL_SHELLS: { value: TerminalShell; label: string }[] = [ + { value: "posix", label: "bash / zsh" }, + { value: "fish", label: "fish" }, + { value: "powershell", label: "PowerShell" }, + { value: "cmd", label: "cmd" }, +]; + +/** The shell a platform's terminal opens by default */ +export function defaultShell(osType: string | null | undefined): TerminalShell { + return osType === "windows" ? "powershell" : "posix"; +} + +/** + * The variables that send one terminal's traffic through the proxy on `port` and trust its CA + * at `caPath`, for tools that read neither the system proxy nor the login keychain: Node, + * Python, and anything built on OpenSSL. + */ +export function environmentVars(port: number, caPath: string): [string, string][] { + const proxy = `http://127.0.0.1:${port}`; + return [ + ["HTTP_PROXY", proxy], + ["HTTPS_PROXY", proxy], + // curl reads only the lowercase name for plain HTTP. + ["http_proxy", proxy], + ["https_proxy", proxy], + ["SSL_CERT_FILE", caPath], + ["NODE_EXTRA_CA_CERTS", caPath], + ["REQUESTS_CA_BUNDLE", caPath], + ]; +} + +/** The variables as `shell` sets them, one a line */ +export function environmentBlock(shell: TerminalShell, port: number, caPath: string): string { + return environmentVars(port, caPath) + .map(([name, value]) => setLine(shell, name, value)) + .join("\n"); +} + +function setLine(shell: TerminalShell, name: string, value: string): string { + switch (shell) { + case "posix": + return `export ${name}=${SAFE.test(value) ? value : shellQuote(value)}`; + case "fish": + return `set -gx ${name} ${fishQuote(value)}`; + case "powershell": + return `$env:${name} = '${value.replaceAll("'", "''")}'`; + case "cmd": + // The quotes around the pair keep spaces and stop at the closing one, unlike `set A="b"` + return `set "${name}=${value}"`; + } +} + +/** Needs no quotes in any POSIX shell or fish */ +const SAFE = /^[\w@%+=:,./-]+$/; + +/** Fish quotes like the POSIX shells, except a backslash and a quote escape inside single quotes */ +function fishQuote(value: string): string { + if (SAFE.test(value)) return value; + return `'${value.replaceAll("\\", "\\\\").replaceAll("'", "\\'")}'`; +} + +/** Shell exports for bash and zsh, as the palette and Settings copy them */ +export function environmentExports(port: number, caPath: string): string { + return environmentBlock("posix", port, caPath); } diff --git a/apps/yaak-proxy/lib/flowViews.test.ts b/apps/yaak-proxy/lib/flowViews.test.ts index 7e8c35c65..5c7daff3c 100644 --- a/apps/yaak-proxy/lib/flowViews.test.ts +++ b/apps/yaak-proxy/lib/flowViews.test.ts @@ -59,6 +59,12 @@ const flow: Flow = { grpcStatus: null, grpcMessage: null, tunnelled: false, + clientAddr: null, + elapsedConnectMs: null, + elapsedTlsMs: null, + elapsedRequestMs: null, + upstreamTlsVersion: null, + upstreamTlsCipher: null, }; function bytes(text: string): FlowBodyData { diff --git a/apps/yaak-proxy/lib/flowViews.ts b/apps/yaak-proxy/lib/flowViews.ts index f4945e04a..3b0110e96 100644 --- a/apps/yaak-proxy/lib/flowViews.ts +++ b/apps/yaak-proxy/lib/flowViews.ts @@ -15,7 +15,7 @@ import { type HttpResponseView, } from "@yaakapp-internal/ui/httpTypes"; import { useEffect, useState } from "react"; -import { base64ToBytes, bytesToText, findHeader, mimeEssence } from "./body"; +import { base64ToBytes, bodyAsText, findHeader, mimeEssence } from "./body"; import { statusReason } from "./statusReason"; /** A captured body as the engine's body RPC returns it, decoded */ @@ -32,7 +32,8 @@ export function flowBodyFromRpc(res: GetFlowBodyResponse): FlowBodyData { /** A captured request as the shared request pane shows it, read-only */ export function flowRequestToView(flow: Flow, body: FlowBodyData | null): HttpRequestView { const contentType = findHeader(flow.requestHeaders, "content-type"); - const text = body == null ? null : bytesToText(body.bytes, body.truncated); + // In its declared charset, as the response viewers read text + const text = body == null ? null : bodyAsText(body.bytes, body.truncated, contentType); const { bodyType, body: bodyView } = requestBodyView(contentType, body, text); return { @@ -84,7 +85,18 @@ export function flowResponseToView( ): HttpResponseView | null { const ended = flow.elapsedMs != null || flow.error != null || flow.cancelled; if (flow.status == null && !ended) return null; + return responseView(flow, body, ended); +} +/** + * A flow as the shared cookie lists read it, whether or not a response came: the request + * half's Cookies tab shows what was sent before anything was received. + */ +export function flowCookiesView(flow: Flow): HttpResponseView { + return responseView(flow, null, flow.elapsedMs != null || flow.error != null || flow.cancelled); +} + +function responseView(flow: Flow, body: FlowBodyData | null, ended: boolean): HttpResponseView { const encoding = findHeader(flow.responseHeaders, "content-encoding")?.trim().toLowerCase(); const encoded = encoding != null && encoding !== "" && encoding !== "identity"; @@ -133,7 +145,10 @@ export function flowBodyLoader({ return { useText: () => { const body = useBody(); - return { ...body, data: mapData(body, (b) => bytesToText(b.bytes, b.truncated)) }; + return { + ...body, + data: mapData(body, (b) => bodyAsText(b.bytes, b.truncated, contentType)), + }; }, useBytes: () => { const body = useBody(); diff --git a/apps/yaak-proxy/lib/interfaceSettings.ts b/apps/yaak-proxy/lib/interfaceSettings.ts new file mode 100644 index 000000000..0bf77df03 --- /dev/null +++ b/apps/yaak-proxy/lib/interfaceSettings.ts @@ -0,0 +1,32 @@ +import type { SettingsPatch, SettingsValues } from "@yaakapp-internal/ui"; +import { atom } from "jotai"; +import { readSettingsValues, SETTINGS_VALUES_KEY } from "./settings"; + +/** The shared settings tabs' values, kept in local storage */ +const valuesAtom = atom( + readSettingsValues(localStorage.getItem(SETTINGS_VALUES_KEY)), +); + +/** Reads the values, and writes a patch of them through to storage and the window */ +export const settingsValuesAtom = atom( + (get) => get(valuesAtom), + (get, set, patch: SettingsPatch) => { + const next = { ...get(valuesAtom), ...patch }; + set(valuesAtom, next); + localStorage.setItem(SETTINGS_VALUES_KEY, JSON.stringify(next)); + applyFontSizes(next); + if (patch.appearance != null) { + // Loaded on demand: the theme module shows the window as it loads + import("./theme").then((m) => m.setChosenAppearance(next.appearance)).catch(console.error); + } + }, +); + +/** The interface and editor sizes, the editor's through the variable the editors read */ +export function applyFontSizes( + values: Pick, +) { + const root = document.documentElement; + root.style.fontSize = `${values.interfaceFontSize}px`; + root.style.setProperty("--editor-font-size", `${values.editorFontSize}px`); +} diff --git a/apps/yaak-proxy/lib/opener.test.ts b/apps/yaak-proxy/lib/opener.test.ts new file mode 100644 index 000000000..a59946036 --- /dev/null +++ b/apps/yaak-proxy/lib/opener.test.ts @@ -0,0 +1,42 @@ +import { beforeEach, describe, expect, test, vi } from "vite-plus/test"; + +const opener = vi.hoisted(() => ({ + openUrl: vi.fn<(url: string) => Promise>(), + revealItemInDir: vi.fn<(path: string) => Promise>(), +})); +vi.mock("@tauri-apps/plugin-opener", () => opener); + +const { openLink, revealPath } = await import("./opener"); + +describe("opener", () => { + beforeEach(() => { + opener.openUrl.mockReset(); + opener.revealItemInDir.mockReset(); + }); + + test("a folder that can't be shown says so", async () => { + opener.revealItemInDir.mockRejectedValue("not allowed: opener.reveal_item_in_dir"); + const report = vi.fn(); + await revealPath("/data/app.yaak.proxy", report); + expect(opener.revealItemInDir).toHaveBeenCalledWith("/data/app.yaak.proxy"); + expect(report).toHaveBeenCalledWith( + "Failed to show /data/app.yaak.proxy: not allowed: opener.reveal_item_in_dir", + ); + }); + + test("nothing to report when it works", async () => { + opener.revealItemInDir.mockResolvedValue(); + opener.openUrl.mockResolvedValue(); + const report = vi.fn(); + await revealPath("/logs", report); + await openLink("https://yaak.app", report); + expect(report).not.toHaveBeenCalled(); + }); + + test("a link that can't be opened says so", async () => { + opener.openUrl.mockRejectedValue(new Error("denied")); + const report = vi.fn(); + await openLink("https://yaak.app", report); + expect(report).toHaveBeenCalledWith("Failed to open https://yaak.app: Error: denied"); + }); +}); diff --git a/apps/yaak-proxy/lib/opener.ts b/apps/yaak-proxy/lib/opener.ts new file mode 100644 index 000000000..7d653095c --- /dev/null +++ b/apps/yaak-proxy/lib/opener.ts @@ -0,0 +1,22 @@ +import { openUrl, revealItemInDir } from "@tauri-apps/plugin-opener"; + +/** Says what failed, as a toast, rather than leaving a button that does nothing */ +export type ReportFailure = (message: string) => void; + +/** Shows a file or folder in the platform's file manager */ +export async function revealPath(path: string, report: ReportFailure): Promise { + try { + await revealItemInDir(path); + } catch (err) { + report(`Failed to show ${path}: ${String(err)}`); + } +} + +/** Opens a link in the default browser */ +export async function openLink(url: string, report: ReportFailure): Promise { + try { + await openUrl(url); + } catch (err) { + report(`Failed to open ${url}: ${String(err)}`); + } +} diff --git a/apps/yaak-proxy/lib/phases.test.ts b/apps/yaak-proxy/lib/phases.test.ts new file mode 100644 index 000000000..edff97e7f --- /dev/null +++ b/apps/yaak-proxy/lib/phases.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, test } from "vite-plus/test"; +import { flowPhases, type PhaseFlow } from "./phases"; + +const done: PhaseFlow = { + elapsedConnectMs: 12, + elapsedTlsMs: 40, + elapsedRequestMs: 41, + elapsedHeadersMs: 90, + elapsedMs: 120, + status: 200, + error: null, + cancelled: false, + streamKind: null, +}; + +describe("flowPhases", () => { + test("lists each phase with how long it took", () => { + expect(flowPhases(done).map((p) => [p.key, p.atMs, p.tookMs])).toEqual([ + ["start", 0, 0], + ["connect", 12, 12], + ["tls", 40, 28], + ["request", 41, 1], + ["headers", 90, 49], + ["end", 120, 30], + ]); + }); + + test("leaves out phases the engine has no time for", () => { + const phases = flowPhases({ + ...done, + elapsedConnectMs: null, + elapsedTlsMs: null, + elapsedRequestMs: null, + }); + expect(phases.map((p) => p.key)).toEqual(["start", "headers", "end"]); + }); + + test("an open flow has no end yet", () => { + const phases = flowPhases({ ...done, elapsedMs: null }); + expect(phases.at(-1)?.key).toBe("headers"); + }); + + test("a failure ends with the engine's error", () => { + const phases = flowPhases({ + ...done, + elapsedTlsMs: null, + elapsedRequestMs: null, + elapsedHeadersMs: null, + elapsedMs: 31, + status: null, + error: "Connection refused", + }); + expect(phases.at(-1)).toMatchObject({ kind: "error", atMs: 31, detail: "Connection refused" }); + }); + + test("a stream closes", () => { + expect(flowPhases({ ...done, streamKind: "websocket" }).at(-1)?.label).toBe("Closed"); + }); +}); diff --git a/apps/yaak-proxy/lib/phases.ts b/apps/yaak-proxy/lib/phases.ts new file mode 100644 index 000000000..051ed78b7 --- /dev/null +++ b/apps/yaak-proxy/lib/phases.ts @@ -0,0 +1,66 @@ +import type { Flow } from "@yaakapp-internal/proxy-lib"; + +export type PhaseKind = "start" | "step" | "end" | "error" | "cancelled"; + +/** One row of a flow's Timeline: a moment in the exchange, and how long it took to reach it */ +export interface FlowPhase { + key: string; + label: string; + kind: PhaseKind; + /** Milliseconds from the start */ + atMs: number; + /** Milliseconds since the phase before */ + tookMs: number; + detail?: string; +} + +export type PhaseFlow = Pick< + Flow, + | "elapsedHeadersMs" + | "elapsedMs" + | "status" + | "error" + | "cancelled" + | "streamKind" + | "elapsedConnectMs" + | "elapsedTlsMs" + | "elapsedRequestMs" +>; + +/** + * The phases the engine recorded for a flow, in order: started, connected, TLS, request sent, + * headers, and how it ended. A phase the engine has no time for is left out; an open flow has + * no end yet. + */ +export function flowPhases(flow: PhaseFlow): FlowPhase[] { + const steps: Omit[] = [ + { key: "start", label: "Started", kind: "start", atMs: 0 }, + ]; + const add = (key: string, label: string, atMs: number | null | undefined) => { + if (atMs != null) steps.push({ key, label, kind: "step", atMs }); + }; + add("connect", "Connected", flow.elapsedConnectMs); + add("tls", "TLS handshake", flow.elapsedTlsMs); + add("request", "Request sent", flow.elapsedRequestMs); + add( + "headers", + flow.status == null ? "Headers received" : `Headers received, ${flow.status}`, + flow.elapsedHeadersMs, + ); + + if (flow.elapsedMs != null || flow.error != null || flow.cancelled) { + const atMs = flow.elapsedMs ?? steps[steps.length - 1]?.atMs ?? 0; + if (flow.error != null) { + steps.push({ key: "end", label: "Failed", kind: "error", atMs, detail: flow.error }); + } else if (flow.cancelled) { + steps.push({ key: "end", label: "Cancelled", kind: "cancelled", atMs }); + } else { + const label = flow.streamKind == null ? "Body received" : "Closed"; + steps.push({ key: "end", label, kind: "end", atMs }); + } + } + + // Times come from different clocks in the engine; keep them in order regardless + steps.sort((a, b) => a.atMs - b.atMs); + return steps.map((s, i) => ({ ...s, tookMs: i === 0 ? 0 : s.atMs - (steps[i - 1]?.atMs ?? 0) })); +} diff --git a/apps/yaak-proxy/lib/search.ts b/apps/yaak-proxy/lib/search.ts index 39207dde6..8cd2f4046 100644 --- a/apps/yaak-proxy/lib/search.ts +++ b/apps/yaak-proxy/lib/search.ts @@ -5,7 +5,10 @@ import { getDomain } from "tldts"; * A search over every flow in the database: the search box's terms, its field terms, and * whether System's flows are hidden, as `list_flows` takes them. */ -export type FlowQuery = Pick; +export type FlowQuery = Pick< + FlowFilter, + "q" | "under" | "app" | "cwd" | "system" | "tunnelled" | "any" +>; /** A word of the search box: a free term, or `@field:value`, maybe negated with `-` */ export interface SearchToken { @@ -75,12 +78,14 @@ export function parseSearch(text: string, hideSystem = false): FlowQuery | null function parseClause(tokens: SearchToken[], hideSystem: boolean): FlowQuery | null { const words: string[] = []; const apps: string[] = []; + const cwds: string[] = []; // Each host with the paths written after it. const hosts: { host: string; paths: string[] }[] = []; let system: boolean | undefined = hideSystem ? false : undefined; let tunnelled: boolean | undefined; for (const t of tokens) { if (t.field === "app" && !t.negated && t.value !== "") apps.push(t.value); + else if (t.field === "cwd" && !t.negated && t.value !== "") cwds.push(t.value); else if (t.field === "host" && !t.negated && t.value !== "") { hosts.push({ host: t.value.replace(/\/+$/, ""), paths: [] }); } else if (t.field === "path" && !t.negated && t.value !== "") { @@ -107,6 +112,7 @@ function parseClause(tokens: SearchToken[], hideSystem: boolean): FlowQuery | nu ...(q === "" ? {} : { q }), ...(under.length > 0 ? { under } : {}), ...(apps.length > 0 ? { app: apps } : {}), + ...(cwds.length > 0 ? { cwd: cwds } : {}), ...(system == null ? {} : { system }), ...(tunnelled == null ? {} : { tunnelled }), }; @@ -136,19 +142,49 @@ export function widerQueries(text: string): string[] { return [...out]; } -/** The folder a program outside an app ran in, which names it better than its executable */ +/** The folder a program outside an app ran in */ export function folderOf(flow: Pick): string | null { - if (flow.clientBundleId != null || flow.clientCwd == null) return null; - const name = flow.clientCwd.replace(/\/+$/, "").split("/").pop() ?? ""; + if (flow.clientBundleId != null) return null; + return folderName(flow.clientCwd); +} + +/** The last part of a path: `fieldwork` of `/src/fieldwork/` */ +function folderName(path: string | null): string | null { + if (path == null) return null; + const name = + path + .replace(/[/\\]+$/, "") + .split(/[/\\]/) + .pop() ?? ""; return name === "" ? null : name; } +/** Runtimes that run anyone's code, so six rows of `node` say nothing without their folder */ +const GENERIC_RUNTIMES = new Set(["node", "python", "bun", "deno", "ruby", "java", "cargo"]); + +/** A program outside an app that runs other code: `node`, `python3.12`, `java` */ +export function isGenericRuntime(flow: Pick): boolean { + if (flow.clientBundleId != null || flow.clientName == null) return false; + const name = flow.clientName.toLowerCase().replace(/\.exe$/, ""); + return GENERIC_RUNTIMES.has(name.replace(/[\d.]+$/, "")); +} + /** - * What `@app` names a flow's app by, and what the sidebar groups its apps by: an app's name, or - * for a program outside an app the folder it ran in, else its name. Null when unknown. + * The folder that names a generic runtime's row, `node` in `fieldwork`; null for anything + * else, whose folders are rows under it + */ +export function runtimeFolderOf( + flow: Pick, +): string | null { + return isGenericRuntime(flow) ? folderOf(flow) : null; +} + +/** + * What `@app` names a flow's app by, and what the sidebar groups its apps by: an app's name, a + * program's name, or for a generic runtime the folder it ran in. Null when unknown. */ export function appKeyOf(flow: Flow): string | null { - return folderOf(flow) ?? flow.clientName ?? flow.clientBundleId ?? flow.clientPath; + return runtimeFolderOf(flow) ?? flow.clientName ?? flow.clientBundleId ?? flow.clientPath; } const SYSTEM_DIRS = ["/System/", "/usr/libexec/", "/usr/sbin/", "/sbin/", "/Library/Apple/"]; @@ -193,6 +229,7 @@ export function matchesQuery(flow: Flow, query: FlowQuery): boolean { } if (query.under != null && !query.under.some((node) => isUnder(flow.url, node))) return false; if (query.app != null && !query.app.some((app) => matchesApp(flow, app))) return false; + if (query.cwd != null && !query.cwd.some((cwd) => matchesCwd(flow, cwd))) return false; if (query.system != null && isSystemFlow(flow) !== query.system) return false; if (query.tunnelled != null && flow.tunnelled !== query.tunnelled) return false; // One of the groups, each judged whole, as the database judges them. @@ -209,6 +246,12 @@ function matchesApp(flow: Flow, app: string): boolean { return flow.clientName?.toLowerCase() === lower || folderOf(flow)?.toLowerCase() === lower; } +/** By the whole working directory exactly, or its folder's name in any case */ +function matchesCwd(flow: Flow, cwd: string): boolean { + if (flow.clientCwd == null) return false; + return flow.clientCwd === cwd || folderName(flow.clientCwd)?.toLowerCase() === cwd.toLowerCase(); +} + /** `404` exactly, `40` as the start of one, or a class such as `4xx` */ function matchesStatus(status: number | null, term: string): boolean { if (status == null) return false; diff --git a/apps/yaak-proxy/lib/settings.test.ts b/apps/yaak-proxy/lib/settings.test.ts new file mode 100644 index 000000000..158c7dea7 --- /dev/null +++ b/apps/yaak-proxy/lib/settings.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, test } from "vite-plus/test"; +import { + CAPTURE_LIMIT_OPTIONS, + DEFAULT_SETTINGS_VALUES, + KEEP_TRAFFIC_OPTIONS, + proxySettingsView, + readSettingsValues, + sizeFromOptionValue, + sizeOptionsFor, + sizeOptionValue, +} from "./settings"; + +const APP = { + name: "Yaak Proxy", + version: "2026.11.0", + dataDir: "/data", + logDir: "/logs", + revealPathLabel: "Reveal in Finder", +}; + +describe("readSettingsValues", () => { + test("Yaak's defaults when nothing is stored", () => { + expect(readSettingsValues(null)).toEqual(DEFAULT_SETTINGS_VALUES); + expect(readSettingsValues("not json")).toEqual(DEFAULT_SETTINGS_VALUES); + expect(DEFAULT_SETTINGS_VALUES.interfaceFontSize).toBe(15); + expect(DEFAULT_SETTINGS_VALUES.editorFontSize).toBe(13); + }); + + test("stored values over the defaults, the wrong types ignored", () => { + const values = readSettingsValues( + JSON.stringify({ appearance: "dark", interfaceFontSize: "big", editorFont: "Menlo", x: 1 }), + ); + expect(values.appearance).toBe("dark"); + expect(values.interfaceFontSize).toBe(15); + expect(values.editorFont).toBe("Menlo"); + expect("x" in values).toBe(false); + }); +}); + +describe("proxySettingsView", () => { + test("resolves the system appearance and the themes", () => { + const view = proxySettingsView(DEFAULT_SETTINGS_VALUES, APP, "dark"); + expect(view.themes?.resolvedAppearance).toBe("dark"); + expect(view.themes?.active.dark).toBe(true); + const light = proxySettingsView( + { ...DEFAULT_SETTINGS_VALUES, appearance: "light" }, + APP, + "dark", + ); + expect(light.themes?.resolvedAppearance).toBe("light"); + expect(light.themes?.active.dark).toBe(false); + }); + + test("a missing theme falls back to Yaak's", () => { + const view = proxySettingsView({ ...DEFAULT_SETTINGS_VALUES, themeDark: "gone" }, APP, "dark"); + expect(view.themes?.dark.id).toBe(DEFAULT_SETTINGS_VALUES.themeDark); + }); + + test("leaves out what the proxy doesn't have yet", () => { + const view = proxySettingsView(DEFAULT_SETTINGS_VALUES, APP, "light"); + expect(view.updates).toBeNull(); + expect(view.license).toBeNull(); + expect(view.hiddenInterfaceRows).toEqual([ + "editorKeymap", + "editorSoftWrap", + "coloredMethods", + "nativeTitlebar", + ]); + expect(view.app.name).toBe("Yaak Proxy"); + }); +}); + +describe("size options", () => { + test("keep traffic is unlimited by default", () => { + expect(KEEP_TRAFFIC_OPTIONS[0]).toMatchObject({ label: "Unlimited", bytes: null }); + expect(sizeOptionValue(null)).toBe("none"); + expect(sizeFromOptionValue("none")).toBeNull(); + }); + + test("labels in the units the limits are set in", () => { + expect(CAPTURE_LIMIT_OPTIONS.map((o) => o.label)).toEqual([ + "1 MB", + "5 MB", + "10 MB", + "50 MB", + "100 MB", + ]); + expect(KEEP_TRAFFIC_OPTIONS.map((o) => o.label)).toEqual([ + "Unlimited", + "500 MB", + "1 GB", + "2 GB", + "5 GB", + "10 GB", + "20 GB", + ]); + }); + + test("values round trip", () => { + for (const o of [...KEEP_TRAFFIC_OPTIONS, ...CAPTURE_LIMIT_OPTIONS]) { + expect(sizeFromOptionValue(sizeOptionValue(o.bytes))).toBe(o.bytes); + } + expect(sizeFromOptionValue("abc")).toBeNull(); + }); + + test("a size set elsewhere gets its own option, in order", () => { + const options = sizeOptionsFor(CAPTURE_LIMIT_OPTIONS, 3 * 1024 * 1024); + expect(options.map((o) => o.bytes)).toEqual([ + 1024 * 1024, + 3 * 1024 * 1024, + 5 * 1024 * 1024, + 10 * 1024 * 1024, + 50 * 1024 * 1024, + 100 * 1024 * 1024, + ]); + expect(sizeOptionsFor(CAPTURE_LIMIT_OPTIONS, 10 * 1024 * 1024)).toBe(CAPTURE_LIMIT_OPTIONS); + }); +}); diff --git a/apps/yaak-proxy/lib/settings.ts b/apps/yaak-proxy/lib/settings.ts new file mode 100644 index 000000000..7c22db897 --- /dev/null +++ b/apps/yaak-proxy/lib/settings.ts @@ -0,0 +1,168 @@ +import { formatSize } from "@yaakapp-internal/lib/formatSize"; +// The themes alone, not the package index, which reaches for the window as it loads +import { defaultDarkTheme, defaultLightTheme } from "@yaakapp-internal/theme/src/defaultThemes"; +import type { + SettingsAppInfo, + SettingsThemeOption, + SettingsValues, + SettingsView, +} from "@yaakapp-internal/ui"; + +/** Yaak's own defaults for the shared tabs, so the two apps start the same */ +export const DEFAULT_SETTINGS_VALUES: SettingsValues = { + appearance: "system", + themeLight: defaultLightTheme.id, + themeDark: defaultDarkTheme.id, + interfaceFont: null, + interfaceFontSize: 15, + editorFont: null, + editorFontSize: 13, + editorKeymap: "default", + editorSoftWrap: true, + coloredMethods: false, + useNativeTitlebar: false, + hideWindowControls: false, + updateChannel: "stable", + autoupdate: true, + autoDownloadUpdates: true, + checkNotifications: true, + promptFeedback: true, +}; + +/** Where the shared tabs' values are kept in the proxy */ +export const SETTINGS_VALUES_KEY = "proxy_settings_values"; + +/** The stored values over the defaults; anything unreadable or of the wrong type is ignored */ +export function readSettingsValues(raw: string | null): SettingsValues { + if (raw == null) return DEFAULT_SETTINGS_VALUES; + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return DEFAULT_SETTINGS_VALUES; + } + if (parsed == null || typeof parsed !== "object") return DEFAULT_SETTINGS_VALUES; + const out: Record = { ...DEFAULT_SETTINGS_VALUES }; + for (const [key, fallback] of Object.entries(DEFAULT_SETTINGS_VALUES)) { + const value = (parsed as Record)[key]; + const nullable = fallback === null; + if (value === undefined) continue; + if ( + (nullable && (value === null || typeof value === "string")) || + typeof value === typeof fallback + ) { + out[key] = value; + } + } + return out as unknown as SettingsValues; +} + +const THEMES: SettingsThemeOption[] = [defaultDarkTheme, defaultLightTheme].map((t) => ({ + id: t.id, + label: t.label, + dark: t.dark, +})); + +/** The appearance showing, with `system` resolved by the OS's */ +export function resolveAppearance(appearance: string, system: "light" | "dark"): "light" | "dark" { + return appearance === "light" || appearance === "dark" ? appearance : system; +} + +/** + * The proxy's settings as the shared tabs render them. The proxy has Yaak's two themes, no + * installed-font list, updater or feedback prompts yet, and its license comes with the license + * work, so those parts are left out, as are the Interface rows it doesn't act on. + */ +export function proxySettingsView( + values: SettingsValues, + app: SettingsAppInfo, + systemAppearance: "light" | "dark", +): SettingsView { + const light = THEMES.find((t) => t.id === values.themeLight && !t.dark) ?? lightOption(); + const dark = THEMES.find((t) => t.id === values.themeDark && t.dark) ?? darkOption(); + const resolvedAppearance = resolveAppearance(values.appearance, systemAppearance); + return { + app, + values, + themes: { + themes: THEMES, + light, + dark, + active: resolvedAppearance === "dark" ? dark : light, + resolvedAppearance, + }, + fonts: null, + windowControlsSetting: false, + updates: null, + feedbackPrompts: false, + license: null, + // Not acted on in the proxy yet: its editors take no keymap or wrap setting, its list draws + // methods uncoloured, and it always draws its own title bar. + hiddenInterfaceRows: ["editorKeymap", "editorSoftWrap", "coloredMethods", "nativeTitlebar"], + }; +} + +function lightOption(): SettingsThemeOption { + return THEMES.find((t) => !t.dark) as SettingsThemeOption; +} + +function darkOption(): SettingsThemeOption { + return THEMES.find((t) => t.dark) as SettingsThemeOption; +} + +/** A size choice in a select, `null` for no limit */ +export interface SizeOption { + label: string; + value: string; + bytes: number | null; +} + +const MB = 1024 * 1024; +const GB = 1024 * MB; + +function sizeOptions(sizes: (number | null)[], none: string): SizeOption[] { + return sizes.map((bytes) => ({ + label: bytes == null ? none : sizeLabel(bytes), + value: bytes == null ? "none" : `${bytes}`, + bytes, + })); +} + +/** A size in the binary units the limits are set in, so 10 MiB reads "10 MB", not "10.5 MB" */ +export function sizeLabel(bytes: number): string { + if (bytes >= GB && bytes % (GB / 2) === 0) return `${bytes / GB} GB`; + if (bytes >= MB && bytes % MB === 0) return `${bytes / MB} MB`; + return formatSize(bytes); +} + +/** How much captured traffic is kept before the oldest flows are dropped */ +export const KEEP_TRAFFIC_OPTIONS = sizeOptions( + [null, 500 * MB, GB, 2 * GB, 5 * GB, 10 * GB, 20 * GB], + "Unlimited", +); + +/** How much of each body is stored */ +export const CAPTURE_LIMIT_OPTIONS = sizeOptions( + [MB, 5 * MB, 10 * MB, 50 * MB, 100 * MB], + "Unlimited", +); + +/** + * The option a stored size selects. A size set elsewhere (the CLI or `settings.json`) that no + * option has is added as its own, so the select still shows it. + */ +export function sizeOptionsFor(options: SizeOption[], bytes: number | null): SizeOption[] { + if (options.some((o) => o.bytes === bytes)) return options; + const extra = sizeOptions([bytes], "Unlimited"); + return [...options, ...extra].sort((a, b) => (a.bytes ?? Infinity) - (b.bytes ?? Infinity)); +} + +export function sizeOptionValue(bytes: number | null): string { + return bytes == null ? "none" : `${bytes}`; +} + +export function sizeFromOptionValue(value: string): number | null { + if (value === "none") return null; + const n = Number(value); + return Number.isFinite(n) && n > 0 ? n : null; +} diff --git a/apps/yaak-proxy/lib/sidebarTree.test.ts b/apps/yaak-proxy/lib/sidebarTree.test.ts index 986a089a6..002a7bb15 100644 --- a/apps/yaak-proxy/lib/sidebarTree.test.ts +++ b/apps/yaak-proxy/lib/sidebarTree.test.ts @@ -103,6 +103,71 @@ describe("buildSidebarTree", () => { }); }); +describe("programs outside an app", () => { + const claude = (cwd: string) => ({ + clientName: "claude", + clientPath: "/usr/local/bin/claude", + clientCwd: cwd, + }); + const python = { + clientName: "python3.12", + clientPath: "/usr/bin/python3.12", + clientCwd: "/src/ml", + }; + const curl = { clientName: "curl", clientPath: "/usr/bin/curl", clientCwd: null }; + const root = buildSidebarTree( + [ + flow("https://api.anthropic.com/v1/messages", claude("/src/yaak")), + flow("https://api.anthropic.com/v1/models", claude("/src/fieldwork")), + flow("https://statsig.anthropic.com/v1", claude("/src/yaak")), + flow("https://pypi.org/simple", python), + flow("https://example.com/", curl), + flow("https://api.github.com/repos", node), + ], + 6, + ); + const items = itemsById(root); + const apps = (root.children ?? []).find((c) => c.item.id === "apps"); + + test("one row per program, its working directories one level down above the hosts", () => { + expect(shape(apps as TreeNode).slice(0, 12)).toEqual([ + "Apps 6", + " claude 3", + " fieldwork 1", + " anthropic.com 1", + " api.anthropic.com 1", + " /v1 1", + " /models 1", + " yaak 2", + " anthropic.com 2", + " api.anthropic.com 1", + " /v1 1", + " /messages 1", + ]); + }); + + test("generic runtimes keep their folder in the row's name", () => { + const labels = (apps?.children ?? []).map((c) => [c.item.label, c.item.sublabel ?? null]); + expect(labels).toEqual([ + ["claude", null], + ["curl", null], + ["node", "fieldwork"], + ["python3.12", "ml"], + ]); + // A program with no working directory has its hosts straight under it. + expect(items.get("app:curl|site:example.com")?.kind).toBe("site"); + }); + + test("a folder row selects that program in that folder", () => { + const query = items.get("app:claude|cwd:yaak")?.query ?? ""; + expect(query).toBe("@app:claude @cwd:yaak"); + expect(items.get("app:claude|cwd:yaak|host:statsig.anthropic.com")?.query).toBe( + "@app:claude @cwd:yaak @host:statsig.anthropic.com", + ); + expect(parseSearch(query)).toEqual({ app: ["claude"], cwd: ["yaak"] }); + }); +}); + describe("decrypting from the sidebar", () => { const tunnel = flow("https://bank.example.com", { ...slack, diff --git a/apps/yaak-proxy/lib/sidebarTree.ts b/apps/yaak-proxy/lib/sidebarTree.ts index c7a99798d..3e1066689 100644 --- a/apps/yaak-proxy/lib/sidebarTree.ts +++ b/apps/yaak-proxy/lib/sidebarTree.ts @@ -6,19 +6,31 @@ import { folderOf, formatFieldFilter, isSystemFlow, + runtimeFolderOf, sidebarPath, siteOf, tokenize, } from "./search"; -export type SidebarKind = "all" | "tunnelled" | "group" | "app" | "site" | "host" | "path"; +export type SidebarKind = + | "all" + | "tunnelled" + | "group" + | "app" + | "folder" + | "site" + | "host" + | "path"; -/** A row of the sidebar: All traffic, a group, an app, or a site, host or path under one */ +/** + * A row of the sidebar: All traffic, a group, an app, a program's working directory, or a site, + * host or path under one + */ export interface SidebarItem { id: string; kind: SidebarKind; label: string; - /** Dim beside the label: a program's folder */ + /** Dim beside the label: a generic runtime's folder */ sublabel?: string; /** Flows under the row, of those loaded */ count: number; @@ -42,7 +54,7 @@ export const TUNNELLED_ID = "tunnelled"; /** A row a flow sits under, with what makes it, so a rebuild only looks it up */ interface PlaceRow { id: string; - kind: "site" | "host" | "path"; + kind: "folder" | "site" | "host" | "path"; label: string; query: string; decrypt?: DecryptSubject; @@ -50,7 +62,8 @@ interface PlaceRow { /** * Where a flow sits, parsed once per flow rather than on every rebuild: its site, host and path - * rows in Domains, and the same rows under its app. + * rows in Domains, and the same rows under its app, below its working directory for a program + * outside an app that isn't a generic runtime (those are named by their folder instead). */ interface Place { /** What `@app` names its app by, or null when unknown */ @@ -99,10 +112,28 @@ function placeOf(flow: Flow): Place { } const system = isSystemFlow(flow); const appQuery = app == null ? "" : appQueryFor(app, system); - const appRows = - app == null - ? [] - : rows.map((r) => ({ ...r, id: `app:${app}|${r.id}`, query: `${appQuery} ${r.query}` })); + let appRows: PlaceRow[] = []; + if (app != null) { + const folder = runtimeFolderOf(flow) == null ? folderOf(flow) : null; + if (folder == null) { + appRows = rows.map((r) => ({ + ...r, + id: `app:${app}|${r.id}`, + query: `${appQuery} ${r.query}`, + })); + } else { + const folderId = `app:${app}|cwd:${folder}`; + const folderQuery = `${appQuery} ${formatFieldFilter("cwd", folder)}`; + appRows = [ + { id: folderId, kind: "folder", label: folder, query: folderQuery }, + ...rows.map((r) => ({ + ...r, + id: `${folderId}|${r.id}`, + query: `${folderQuery} ${r.query}`, + })), + ]; + } + } place = { app, system, rows, appRows }; places.set(flow, place); return place; @@ -164,8 +195,10 @@ function toNodes( }); } +/** By label, a program's working directories before the sites it reached directly */ function byLabel(a: Trie, b: Trie): number { - return collator.compare(a.item.label, b.item.label); + const folders = Number(b.item.kind === "folder") - Number(a.item.kind === "folder"); + return folders || collator.compare(a.item.label, b.item.label); } function group(id: string, label: string): Trie { @@ -203,7 +236,7 @@ export function buildSidebarTree(flows: Flow[], total: number): TreeNode(null); */ let changesDuringPage: ModelPayload[] | null = null; +/** + * Flows the keep-traffic sweep deleted while a page was being read: the page may be a snapshot + * from before, so it is filtered by these as it lands. Forgotten once no page is being read. + */ +const removedWhileLoading = new Set(); +let pagesLoading = 0; + +function pageStarted() { + pagesLoading++; +} + +function pageSettled() { + pagesLoading = Math.max(0, pagesLoading - 1); + if (pagesLoading === 0) removedWhileLoading.clear(); +} + +/** A page's flows less any the sweep deleted since it was asked for */ +function withoutRemoved(flows: Flow[]): Flow[] { + return removedWhileLoading.size === 0 + ? flows + : flows.filter((f) => !removedWhileLoading.has(f.id)); +} + +/** Changes still to apply, less the ones that would bring back a deleted flow */ +function dropRemovedChanges(changes: ModelPayload[], removed: Set) { + for (let i = changes.length - 1; i >= 0; i--) { + const c = changes[i]; + if (c != null && c.change.type !== "delete" && removed.has(c.model.id)) changes.splice(i, 1); + } +} + /** Loads the page of flows before the oldest one shown, from the search's flows when searching */ export const loadOlderFlowsAtom = atom(null, async (get, set) => { const search = get(searchWindowAtom); @@ -206,12 +237,13 @@ export const loadOlderFlowsAtom = atom(null, async (get, set) => { if (changesDuringPage != null || !hasOlder || first == null) return; const changes: ModelPayload[] = []; changesDuringPage = changes; + pageStarted(); try { const res = await rpc("list_flows", { ...search?.query, before: first.id, limit: PAGE_SIZE }); // Dropped when the window moved on meanwhile, trimmed, cleared or searched again. const merge = (win: FlowWindow) => { if (win.flows[0]?.id !== first.id) return win; - const merged = prependOlder(win, res.flows, PAGE_SIZE); + const merged = prependOlder(win, withoutRemoved(res.flows), PAGE_SIZE); const kept = search == null ? changes : changesFor(search.query, changes); // Counted when they first applied. return { ...applyChanges(merged, kept), total: merged.total }; @@ -225,6 +257,7 @@ export const loadOlderFlowsAtom = atom(null, async (get, set) => { console.warn("Failed to load older flows", err); } finally { changesDuringPage = null; + pageSettled(); } }); @@ -319,21 +352,49 @@ export function syncFlows(store: Store) { load(); }); + // The oldest flows, swept to keep the capture under its size limit: dropped wherever they + // are held, and the totals counted again from the database. + listen("flows_removed", ({ ids }) => { + const removed = new Set(ids); + const drop = (win: FlowWindow) => + win.flows.some((f) => removed.has(f.id)) + ? { ...win, flows: win.flows.filter((f) => !removed.has(f.id)) } + : win; + if (pagesLoading > 0) for (const id of ids) removedWhileLoading.add(id); + dropRemovedChanges(pending, removed); + if (changesDuringPage != null) dropRemovedChanges(changesDuringPage, removed); + store.set(flowWindowAtom, drop); + store.set(searchWindowAtom, (s) => { + if (s == null) return s; + const buffered = [...s.buffered]; + dropRemovedChanges(buffered, removed); + return { ...s, win: drop(s.win), buffered }; + }); + if (removed.has(store.get(selectedFlowAtom)?.id ?? "")) store.set(selectedFlowAtom, null); + if (recountTimer == null) recountTimer = setTimeout(recount, RECOUNT_MS); + }); + function load() { loaded = false; const thisLoad = ++loads; - void rpc("list_flows", { limit: WINDOW_SIZE }).then((res) => { - // A clear since asked for a newer page. - if (thisLoad !== loads) return; - store.set(flowWindowAtom, windowFromPage(res.flows, WINDOW_SIZE, res.total)); - seenId = res.flows[0]?.id ?? null; - loaded = true; - store.set(flowsLoadedAtom, true); - // Flows recorded before this install remembered it mean setup was done. - if (res.total > 0) store.set(systemProxyEverOnAtom, true); - flush(); - recount(); - }); + pageStarted(); + void rpc("list_flows", { limit: WINDOW_SIZE }) + .then((res) => { + // A clear since asked for a newer page. + if (thisLoad !== loads) return; + store.set( + flowWindowAtom, + windowFromPage(withoutRemoved(res.flows), WINDOW_SIZE, res.total), + ); + seenId = res.flows[0]?.id ?? null; + loaded = true; + store.set(flowsLoadedAtom, true); + // Flows recorded before this install remembered it mean setup was done. + if (res.total > 0) store.set(systemProxyEverOnAtom, true); + flush(); + recount(); + }) + .finally(pageSettled); loadSearch(store.get(activeQueryAtom)); } reloadFlows = load; @@ -360,12 +421,16 @@ export function syncFlows(store: Store) { buffered: [], }; }); + pageStarted(); void rpc("list_flows", { ...query, limit: WINDOW_SIZE }) - .then((res) => settle(windowFromPage(res.flows, WINDOW_SIZE, res.matching ?? res.total))) + .then((res) => + settle(windowFromPage(withoutRemoved(res.flows), WINDOW_SIZE, res.matching ?? res.total)), + ) .catch((err) => { console.warn("Failed to search flows", err); settle(null); - }); + }) + .finally(pageSettled); } // The sidebar takes the window's flows at once when it empties or first fills, and at most diff --git a/apps/yaak-proxy/lib/theme.ts b/apps/yaak-proxy/lib/theme.ts index ef1e877bd..a4b7d5535 100644 --- a/apps/yaak-proxy/lib/theme.ts +++ b/apps/yaak-proxy/lib/theme.ts @@ -9,24 +9,35 @@ import { subscribeToPreferredAppearance, type Appearance, } from "@yaakapp-internal/theme"; +import { readSettingsValues, resolveAppearance, SETTINGS_VALUES_KEY } from "./settings"; import { showWindow } from "./tauri"; setPlatformOnDocument(platformFromUserAgent(navigator.userAgent)); +// The appearance chosen in Settings, `system` following the OS's +let chosenAppearance = readSettingsValues(localStorage.getItem(SETTINGS_VALUES_KEY)).appearance; + // Apply a quick initial theme based on CSS media query let preferredAppearance: Appearance = getCSSAppearance(); -applyTheme(preferredAppearance); +applyTheme(); // Then subscribe to accurate OS appearance detection and changes subscribeToPreferredAppearance((a) => { preferredAppearance = a; - applyTheme(preferredAppearance); + applyTheme(); }); // Show window after initial theme is applied (window starts hidden to prevent flash) showWindow().catch(console.error); -function applyTheme(appearance: Appearance) { +/** Applies the appearance chosen in Settings: `light`, `dark` or `system` */ +export function setChosenAppearance(appearance: string) { + chosenAppearance = appearance; + applyTheme(); +} + +function applyTheme() { + const appearance = resolveAppearance(chosenAppearance, preferredAppearance); const theme = appearance === "dark" ? defaultDarkTheme : defaultLightTheme; applyThemeToDocument(theme); if (theme.base.surface != null) { diff --git a/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts b/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts index 70fbac00c..7e5b70058 100644 --- a/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts +++ b/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts @@ -48,7 +48,28 @@ elapsedHeadersMs: number | null, /** * Milliseconds from `started_at` to the end of the response. */ -elapsedMs: number | null, error: string | null, +elapsedMs: number | null, +/** + * Milliseconds from `started_at` to the TCP connection to the server, and to the end of + * its TLS handshake. Only for intercepted HTTPS on a connection opened for this flow. + */ +elapsedConnectMs: number | null, elapsedTlsMs: number | null, +/** + * Milliseconds from `started_at` to the request body being sent whole. None without one. + */ +elapsedRequestMs: number | null, +/** + * The client's end of its connection: `127.0.0.1:54321`. + */ +clientAddr: string | null, +/** + * The TLS negotiated with the server for intercepted HTTPS: `TLSv1.3`. + */ +upstreamTlsVersion: string | null, +/** + * `TLS13_AES_128_GCM_SHA256`. + */ +upstreamTlsCipher: string | null, error: string | null, /** * The client gave up before the response ended, by resetting its HTTP/2 stream or closing * the connection. Not an error. diff --git a/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts b/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts index 302fa98d3..2e70e0589 100644 --- a/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts +++ b/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts @@ -170,6 +170,11 @@ under?: Array, * it ran in (`fieldwork`). */ app?: Array, +/** + * Only flows from a process that ran in one of these working directories: the whole path, + * or its folder's name in any case (`fieldwork`). With `app`, a program in a folder. + */ +cwd?: Array, /** * Only flows from macOS's own daemons (true), or none of them (false), as [`is_system`] * tells them by their executable. @@ -212,6 +217,11 @@ before?: string, }; export type FlowsClearedPayload = Record; +/** + * Flows deleted to keep captured traffic under the keep-traffic cap, the oldest first. + */ +export type FlowsRemovedPayload = { ids: Array, }; + export type FolderSummary = { id: string, name: string, /** * The folder this one is in, if any. @@ -282,6 +292,14 @@ systemProxyAtLaunch: boolean, * The HTTP versions offered to clients. */ httpVersion: HttpVersion, +/** + * Bytes of each body stored per flow. + */ +captureBodyLimit: number, +/** + * The most captured traffic kept, in bytes, or null to keep everything. + */ +keepTrafficBytes: number | null, /** * Why the engine last failed to start, until it starts. */ @@ -301,6 +319,15 @@ locked: ProxyLock | null, */ otherOwner: OtherOwner | null, }; +export type GetStorageSizeRequest = Record; + +export type GetStorageSizeResponse = { +/** + * The databases' used pages, as the keep-traffic cap counts them; locked, their files' + * sizes on disk. + */ +bytes: number, }; + export type GlobalAction = "recording_toggle" | "flows_clear"; export type HelperRegisterRequest = Record; @@ -370,6 +397,12 @@ yaakFound: boolean, workspaces: Array, }; */ export type MessagePayload = { message: FlowMessage, }; +export type OpenTerminalRequest = { +/** + * Environment variables the shell starts with, by name: `HTTPS_PROXY`. + */ +vars: Array<[string, string]>, }; + export type OtherOwner = { owner: Owner, pid: number, startedAt: string, /** * The engine's port while it runs. @@ -412,9 +445,11 @@ host: string, }; export type RemoveGrpcSchemaRequest = { id: string, }; -export type RpcEventSchema = { decrypt_list_changed: DecryptListChangedPayload, flows_cleared: FlowsClearedPayload, message_write: MessagePayload, model_write: ModelPayload, proxy_state_changed: ProxyStatePayload, }; +export type ResetCapturedDataRequest = Record; -export type RpcSchema = { execute_action: [ActionInvocation, boolean], format_body: [FormatBodyRequest, string], get_decrypt_list: [GetDecryptListRequest, DecryptList], get_message_payload: [GetMessagePayloadRequest, GetMessagePayloadResponse], list_actions: [ListActionsRequest, ListActionsResponse], list_flows: [FlowFilter, ListFlowsResponse], list_grpc_schemas: [ListGrpcSchemasRequest, Array], list_messages: [ListMessagesRequest, ListMessagesResponse], remove_grpc_schema: [RemoveGrpcSchemaRequest, boolean], set_recording: [SetRecordingRequest, boolean], set_system_proxy_at_launch: [SetSystemProxyAtLaunchRequest, boolean], set_http_version: [SetHttpVersionRequest, HttpVersion], add_grpc_schema: [AddGrpcSchemaRequest, GrpcSchema], app_icon: [AppIconRequest, string | null], ca_status: [CaStatusRequest, CaStatus], ca_trust: [CaTrustRequest, CaStatus], clear_flows: [ClearFlowsRequest, number], decode_grpc_message: [DecodeGrpcMessageRequest, DecodedGrpcMessage], export_har: [ExportHarRequest, number], filter_body: [FilterBodyRequest, FilterBodyResponse], get_flow_body: [GetFlowBodyRequest, GetFlowBodyResponse], get_proxy_state: [GetProxyStateRequest, GetProxyStateResponse], helper_register: [HelperRegisterRequest, SystemProxyStatus], helper_unregister: [HelperUnregisterRequest, SystemProxyStatus], edit_decrypt_list: [EditDecryptListRequest, DecryptList], list_seen: [ListSeenRequest, SeenTargets], set_decrypt_list: [SetDecryptListRequest, DecryptList], list_workspaces: [ListWorkspacesRequest, ListWorkspacesResponse], preview_save: [PreviewSaveRequest, SavePreview], reflect_grpc_host: [ReflectGrpcHostRequest, GrpcSchema], save_flow: [SaveFlowToYaakRequest, SavedFlow], save_flow_body: [SaveFlowBodyRequest, SaveFlowBodyResponse], system_proxy_set: [SystemProxySetRequest, SystemProxyStatus], system_proxy_status: [SystemProxyStatusRequest, SystemProxyStatus], take_over: [TakeOverRequest, boolean], }; +export type RpcEventSchema = { decrypt_list_changed: DecryptListChangedPayload, flows_cleared: FlowsClearedPayload, flows_removed: FlowsRemovedPayload, message_write: MessagePayload, model_write: ModelPayload, proxy_state_changed: ProxyStatePayload, }; + +export type RpcSchema = { execute_action: [ActionInvocation, boolean], format_body: [FormatBodyRequest, string], get_decrypt_list: [GetDecryptListRequest, DecryptList], get_message_payload: [GetMessagePayloadRequest, GetMessagePayloadResponse], list_actions: [ListActionsRequest, ListActionsResponse], list_flows: [FlowFilter, ListFlowsResponse], list_grpc_schemas: [ListGrpcSchemasRequest, Array], list_messages: [ListMessagesRequest, ListMessagesResponse], remove_grpc_schema: [RemoveGrpcSchemaRequest, boolean], set_recording: [SetRecordingRequest, boolean], set_system_proxy_at_launch: [SetSystemProxyAtLaunchRequest, boolean], set_http_version: [SetHttpVersionRequest, HttpVersion], set_capture_body_limit: [SetCaptureBodyLimitRequest, number], set_keep_traffic: [SetKeepTrafficRequest, number | null], add_grpc_schema: [AddGrpcSchemaRequest, GrpcSchema], app_icon: [AppIconRequest, string | null], ca_status: [CaStatusRequest, CaStatus], ca_trust: [CaTrustRequest, CaStatus], clear_flows: [ClearFlowsRequest, number], decode_grpc_message: [DecodeGrpcMessageRequest, DecodedGrpcMessage], export_har: [ExportHarRequest, number], filter_body: [FilterBodyRequest, FilterBodyResponse], get_flow_body: [GetFlowBodyRequest, GetFlowBodyResponse], get_proxy_state: [GetProxyStateRequest, GetProxyStateResponse], helper_register: [HelperRegisterRequest, SystemProxyStatus], helper_unregister: [HelperUnregisterRequest, SystemProxyStatus], edit_decrypt_list: [EditDecryptListRequest, DecryptList], list_seen: [ListSeenRequest, SeenTargets], set_decrypt_list: [SetDecryptListRequest, DecryptList], list_workspaces: [ListWorkspacesRequest, ListWorkspacesResponse], preview_save: [PreviewSaveRequest, SavePreview], reflect_grpc_host: [ReflectGrpcHostRequest, GrpcSchema], save_flow: [SaveFlowToYaakRequest, SavedFlow], save_flow_body: [SaveFlowBodyRequest, SaveFlowBodyResponse], system_proxy_set: [SystemProxySetRequest, SystemProxyStatus], system_proxy_status: [SystemProxyStatusRequest, SystemProxyStatus], take_over: [TakeOverRequest, boolean], set_port: [SetPortRequest, number], get_storage_size: [GetStorageSizeRequest, GetStorageSizeResponse], reset_captured_data: [ResetCapturedDataRequest, boolean], open_terminal: [OpenTerminalRequest, boolean], }; export type SaveFlowBodyRequest = { flowId: string, which: FlowBody, /** @@ -504,6 +539,8 @@ name: string, */ id?: string, http: ProxySetting, https: ProxySetting, }; +export type SetCaptureBodyLimitRequest = { bytes: number, }; + export type SetDecryptListRequest = { /** * Every entry, in order, as typed: a bare hostname is a `host:` entry. A built-in @@ -513,6 +550,14 @@ entries: Array, }; export type SetHttpVersionRequest = { httpVersion: HttpVersion, }; +export type SetKeepTrafficRequest = { +/** + * Null keeps everything. + */ +bytes: number | null, }; + +export type SetPortRequest = { port: number, }; + export type SetRecordingRequest = { enabled: boolean, }; export type SetSystemProxyAtLaunchRequest = { enabled: boolean, }; diff --git a/crates-proxy/yaak-proxy-lib/migrations/00000010_flow_connection.sql b/crates-proxy/yaak-proxy-lib/migrations/00000010_flow_connection.sql new file mode 100644 index 000000000..d530ba0db --- /dev/null +++ b/crates-proxy/yaak-proxy-lib/migrations/00000010_flow_connection.sql @@ -0,0 +1,8 @@ +-- The client's address, when the request was sent and the upstream connection made, and the +-- TLS it negotiated, for the flow's timeline and details. +ALTER TABLE flows ADD COLUMN client_addr TEXT; +ALTER TABLE flows ADD COLUMN elapsed_connect_ms INTEGER; +ALTER TABLE flows ADD COLUMN elapsed_tls_ms INTEGER; +ALTER TABLE flows ADD COLUMN elapsed_request_ms INTEGER; +ALTER TABLE flows ADD COLUMN upstream_tls_version TEXT; +ALTER TABLE flows ADD COLUMN upstream_tls_cipher TEXT; diff --git a/crates-proxy/yaak-proxy-lib/migrations/00000011_flow_sweep_pending.sql b/crates-proxy/yaak-proxy-lib/migrations/00000011_flow_sweep_pending.sql new file mode 100644 index 000000000..cd64ce42e --- /dev/null +++ b/crates-proxy/yaak-proxy-lib/migrations/00000011_flow_sweep_pending.sql @@ -0,0 +1,7 @@ +-- Flows the keep-traffic sweep deleted whose bodies in `blobs.sqlite` may not be deleted yet: +-- written with the rows' deletion and removed once their bodies are, so a sweep that fails +-- between the two finishes the bodies next time. +CREATE TABLE flow_sweep_pending +( + flow_id TEXT NOT NULL PRIMARY KEY +); diff --git a/crates-proxy/yaak-proxy-lib/src/db.rs b/crates-proxy/yaak-proxy-lib/src/db.rs index 5cf01fcb7..e6835aebf 100644 --- a/crates-proxy/yaak-proxy-lib/src/db.rs +++ b/crates-proxy/yaak-proxy-lib/src/db.rs @@ -165,6 +165,9 @@ fn missing_data_key(data_dir: &Path) -> String { pub struct Stores { pub db: ProxyQueryManager, pub blobs: BlobManager, + /// `blobs.sqlite` and its key, for measuring it: its pools are its manager's own. + blobs_path: PathBuf, + key: Option, } impl Stores { @@ -173,9 +176,26 @@ impl Stores { pub fn open(data_dir: &Path, key: &DatabaseKey) -> std::result::Result { let db = ProxyQueryManager::new(&data_dir.join(PROXY_DB), key) .map_err(|e| format!("Failed to open proxy database: {e}"))?; - let blobs = open_blobs(&data_dir.join(BLOBS_DB), key) + let blobs_path = data_dir.join(BLOBS_DB); + let blobs = open_blobs(&blobs_path, key) .map_err(|e| format!("Failed to open proxy blob store: {e}"))?; - Ok(Self { db, blobs }) + Ok(Self { db, blobs, blobs_path, key: Some(key.clone()) }) + } + + /// Bytes the captured traffic takes, as [`used_bytes`] counts them, in both databases. + pub fn used_bytes(&self) -> std::result::Result { + let fail = |e: rusqlite::Error| format!("Failed to measure captured traffic: {e}"); + let flows = self.db.with_conn(|c| used_bytes(c.conn().resolve())).map_err(fail)?; + let conn = rusqlite::Connection::open_with_flags( + &self.blobs_path, + OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX, + ) + .map_err(fail)?; + if let Some(key) = &self.key { + encryption::apply_key(&conn, key).map_err(fail)?; + } + conn.busy_timeout(Duration::from_millis(5000)).map_err(fail)?; + Ok(flows + used_bytes(&conn).map_err(fail)?) } /// The databases in `data_dir` for reading, with the data key the master key in `keys` @@ -198,10 +218,58 @@ impl Stores { .map_err(|e| format!("Failed to open {}: {e}", db_path.display()))?; let blobs = open_blobs_read_only(&blobs_path, key.as_ref()) .map_err(|e| format!("Failed to open the proxy's blob store: {e}"))?; - Ok(Self { db, blobs }) + Ok(Self { db, blobs, blobs_path, key }) } } +/// Bytes the data in a database takes: its pages less the free ones. SQLite reuses free pages +/// but gives none back to the file without a vacuum, so this shrinks as rows are deleted where +/// the file does not. The WAL is left out, as its pages are copies of ones counted here. +pub fn used_bytes(conn: &rusqlite::Connection) -> rusqlite::Result { + use rusqlite::types::Value; + // SQLCipher answers `page_size` with its own as text. + let pragma = |name: &str| { + let value = conn.query_row(&format!("PRAGMA {name}"), [], |r| r.get::<_, Value>(0))?; + let n = match &value { + Value::Integer(n) => Some(*n), + Value::Text(s) => s.trim().parse().ok(), + _ => None, + }; + n.ok_or_else(|| rusqlite::Error::InvalidColumnType(0, name.into(), value.data_type())) + }; + let used = (pragma("page_count")? - pragma("freelist_count")?) * pragma("page_size")?; + Ok(used.max(0) as u64) +} + +/// The databases' files and what goes with them: their WAL and shared memory. +fn database_files(data_dir: &Path) -> impl Iterator + '_ { + [PROXY_DB, BLOBS_DB] + .into_iter() + .flat_map(|name| ["", "-wal", "-shm"].map(|suffix| format!("{name}{suffix}"))) + .map(move |name| data_dir.join(name)) +} + +/// Bytes the databases' files take on disk, free pages and WAL included, for when they are +/// locked and cannot be measured as [`Stores::used_bytes`] does. +pub fn file_bytes(data_dir: &Path) -> u64 { + database_files(data_dir).filter_map(|p| std::fs::metadata(p).ok()).map(|m| m.len()).sum() +} + +/// Deletes the databases and the data key that encrypts them, for good. Nothing may write to +/// them meanwhile. +pub fn remove_captured(data_dir: &Path) -> std::result::Result<(), String> { + let key = data_key_path(data_dir); + for path in database_files(data_dir).chain([key.with_extension("tmp"), key]) { + match std::fs::remove_file(&path) { + Err(e) if e.kind() != std::io::ErrorKind::NotFound => { + return Err(format!("Failed to delete {}: {e}", path.display())); + } + _ => {} + } + } + Ok(()) +} + /// Either database in `data_dir` is plain, left by a build from before encryption. pub fn needs_encrypting(data_dir: &Path) -> bool { [PROXY_DB, BLOBS_DB] @@ -443,6 +511,7 @@ mod tests { "_sqlx_migrations", "flow_clear_mark", "flow_messages", + "flow_sweep_pending", "flows", "grpc_schemas" ] @@ -530,7 +599,7 @@ mod tests { .unwrap() .map(|v| v.unwrap()) .collect(); - assert_eq!(&applied[applied.len() - 3..], ["00000007", "00000008", "00000009"]); + assert_eq!(&applied[6..], ["00000007", "00000008", "00000009", "00000010", "00000011"]); let mark_columns = columns(&conn, "flow_clear_mark"); assert_eq!(mark_columns, ["id", "max_rowid", "cleaner", "cleaner_at"]); drop(conn); diff --git a/crates-proxy/yaak-proxy-lib/src/exit_gate.rs b/crates-proxy/yaak-proxy-lib/src/exit_gate.rs new file mode 100644 index 000000000..254ded054 --- /dev/null +++ b/crates-proxy/yaak-proxy-lib/src/exit_gate.rs @@ -0,0 +1,137 @@ +//! Keeps background database work from running into SQLCipher's teardown as the process exits. +//! +//! SQLCipher frees its global state (the key shield, its private heap) in an `atexit` handler it +//! registers the first time a database is keyed. A thread still reading or writing a database, +//! or closing its last connection, then crashes inside SQLCipher. The threads this crate starts +//! (the event loop writing flows, deleting cleared flows, keeping traffic under its cap) hold +//! [`hold`] around each unit of work. [`install`], called once a database is open and so after +//! SQLCipher's handler, registers one that runs before it, as handlers run last registered +//! first: it stops any more work starting, waits for work in flight to finish, for a few seconds +//! at most, and lets the process exit with that work parked. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Once, RwLock, RwLockReadGuard, TryLockError}; +use std::time::{Duration, Instant}; + +static GATE: RwLock<()> = RwLock::new(()); +/// Set as the process exits: no more work starts. +static EXITING: AtomicBool = AtomicBool::new(false); +static INSTALL: Once = Once::new(); + +/// The longest the exit handler waits for work in flight before letting the process go on +/// exiting regardless, so a stuck thread never hangs a quit. +const EXIT_WAIT: Duration = Duration::from_secs(5); + +/// Held around a unit of background database work, a batch, closing connections included. Once +/// the process is exiting, this parks the thread for good instead: the work never starts. +pub(crate) fn hold() -> RwLockReadGuard<'static, ()> { + if EXITING.load(Ordering::SeqCst) { + loop { + std::thread::park(); + } + } + GATE.read().unwrap_or_else(std::sync::PoisonError::into_inner) +} + +/// Registers the exit handler, once per process. Call after a database is open, so SQLCipher's +/// own handler is registered first and runs after this one. +pub(crate) fn install() { + INSTALL.call_once(|| { + // SAFETY: `close_gate` is an `extern "C" fn()` that never unwinds. + let failed = unsafe { atexit(close_gate) } != 0; + if failed { + log::warn!("Failed to register the database exit handler"); + } + }); +} + +unsafe extern "C" { + fn atexit(callback: extern "C" fn()) -> std::ffi::c_int; +} + +/// Stops more work starting, then waits for the work in flight to finish its batch and keeps +/// the gate shut for the rest of the process's life. +extern "C" fn close_gate() { + EXITING.store(true, Ordering::SeqCst); + wait_for_work(EXIT_WAIT); +} + +/// Takes the gate for good once no work holds it, or gives up after `wait`. Returns whether it +/// was taken. Polled, as a blocking write can't be given up on; with no new work starting, the +/// work in flight can't starve it. +fn wait_for_work(wait: Duration) -> bool { + let deadline = Instant::now() + wait; + loop { + match GATE.try_write() { + Ok(guard) => { + std::mem::forget(guard); + return true; + } + Err(TryLockError::Poisoned(e)) => { + std::mem::forget(e.into_inner()); + return true; + } + Err(TryLockError::WouldBlock) if Instant::now() < deadline => { + std::thread::sleep(Duration::from_millis(1)); + } + Err(TryLockError::WouldBlock) => return false, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + use std::sync::mpsc::channel; + + const SELF: &str = "exit_gate::tests::exiting_waits_for_work_in_flight_and_starts_no_more"; + + /// A batch in flight finishes before the gate shuts, however busy other work keeps it, and + /// no work starts after. Run in a process of its own, as shutting the gate parks every + /// thread that would hold it, which the other tests' threads would be. + #[test] + fn exiting_waits_for_work_in_flight_and_starts_no_more() { + if std::env::var("YAAK_EXIT_GATE_CHILD").is_err() { + let status = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", SELF, "--test-threads", "1"]) + .env("YAAK_EXIT_GATE_CHILD", "1") + .status() + .unwrap(); + assert!(status.success()); + return; + } + + // Busy work, taking the gate again as soon as it lets go, as the event loop does. + let busy = Arc::new(AtomicBool::new(false)); + for _ in 0..4 { + let busy = busy.clone(); + std::thread::spawn(move || { + loop { + let _held = hold(); + busy.store(true, Ordering::SeqCst); + } + }); + } + // A batch in flight, a slow one. + let (holding_tx, holding_rx) = channel(); + let (finished_tx, finished_rx) = channel(); + std::thread::spawn(move || { + let _batch = hold(); + holding_tx.send(()).unwrap(); + std::thread::sleep(Duration::from_millis(200)); + finished_tx.send(Instant::now()).unwrap(); + }); + holding_rx.recv().unwrap(); + while !busy.load(Ordering::SeqCst) { + std::hint::spin_loop(); + } + + EXITING.store(true, Ordering::SeqCst); + assert!(wait_for_work(Duration::from_secs(5)), "not starved by the busy work"); + let finished = finished_rx.try_recv().expect("the batch finished before the gate shut"); + assert!(finished <= Instant::now()); + // Nothing holds the gate after, the busy work included: it is all parked. + assert!(GATE.try_read().is_err(), "shut for good"); + } +} diff --git a/crates-proxy/yaak-proxy-lib/src/flows.rs b/crates-proxy/yaak-proxy-lib/src/flows.rs index 11bae5f43..619fddb66 100644 --- a/crates-proxy/yaak-proxy-lib/src/flows.rs +++ b/crates-proxy/yaak-proxy-lib/src/flows.rs @@ -1,5 +1,5 @@ use crate::bodies; -use crate::db::ProxyQueryManager; +use crate::db::{ProxyQueryManager, Stores}; use crate::models::{Flow, FlowBody}; use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64; @@ -31,6 +31,10 @@ pub struct FlowFilter { /// it ran in (`fieldwork`). #[ts(optional)] pub app: Option>, + /// Only flows from a process that ran in one of these working directories: the whole path, + /// or its folder's name in any case (`fieldwork`). With `app`, a program in a folder. + #[ts(optional)] + pub cwd: Option>, /// Only flows from macOS's own daemons (true), or none of them (false), as [`is_system`] /// tells them by their executable. #[ts(optional)] @@ -65,8 +69,8 @@ pub struct FlowFilter { } impl FlowFilter { - /// The filter from a query string's `q`, `host`, `under`, `app`, `system`, `method`, - /// `status`, `limit`, `after` and `before`. `under` and `app` may repeat. + /// The filter from a query string's `q`, `host`, `under`, `app`, `cwd`, `system`, `method`, + /// `status`, `limit`, `after` and `before`. `under`, `app` and `cwd` may repeat. pub fn from_query(query: &str) -> Result { let mut filter = Self::default(); for (key, value) in url::form_urlencoded::parse(query.as_bytes()) { @@ -76,6 +80,7 @@ impl FlowFilter { "host" => filter.host = value, "under" => filter.under.get_or_insert_default().extend(value), "app" => filter.app.get_or_insert_default().extend(value), + "cwd" => filter.cwd.get_or_insert_default().extend(value), "system" => { filter.system = value .map(|v| v.parse().map_err(|_| format!("Invalid system {v:?}"))) @@ -116,7 +121,11 @@ impl FlowFilter { query.append_pair(key, value); } } - for (key, values) in [("under", &self.under), ("app", &self.app)] { + for (key, values) in [ + ("under", &self.under), + ("app", &self.app), + ("cwd", &self.cwd), + ] { for value in values.iter().flatten() { query.append_pair(key, value); } @@ -144,6 +153,7 @@ impl FlowFilter { || given(&self.status) || self.under.is_some() || self.app.is_some() + || self.cwd.is_some() || self.system.is_some() || self.tunnelled.is_some() || self.any.as_ref().is_some_and(|groups| groups.iter().any(FlowFilter::narrows)) @@ -224,6 +234,19 @@ impl FlowFilter { } sql.push(')'); } + if let Some(cwds) = &self.cwd { + if has_cwd && !cwds.is_empty() { + let marks = vec!["?"; cwds.len()].join(", "); + sql.push_str(&format!( + " AND (client_cwd IN ({marks}) OR lower(folder_name(client_cwd)) IN ({marks}))" + )); + params.extend(cwds.iter().map(|c| c.clone().into())); + params.extend(cwds.iter().map(|c| c.to_lowercase().into())); + } else { + // None, or a database from before the column, which has no folders. + sql.push_str(" AND 0"); + } + } match self.system { Some(true) => sql.push_str(" AND is_system(client_bundle_id, client_path)"), Some(false) => sql.push_str(" AND NOT is_system(client_bundle_id, client_path)"), @@ -665,6 +688,142 @@ pub fn clear_mark_in(db: &ProxyQueryManager) -> i64 { db.with_conn(|c| clear_mark(c.conn().resolve())) } +/// When captured traffic takes more than `cap` bytes, as [`Stores::used_bytes`] counts them, +/// deletes the flows written first, with their bodies and messages: as many as the average flow +/// says would bring it under, from one to [`DELETE_BATCH`]. Returns the ids deleted, none once it +/// is under or no flow is left, so a caller sweeps a batch per call until then. Cleared flows are +/// left to [`delete_cleared`], and a stream still open is never swept: its writer would go on +/// adding messages to it. +pub fn sweep_oldest(stores: &Stores, cap: u64) -> Result, String> { + // Bodies a sweep that failed partway left behind go first: they count against the cap. + let pending = pending_sweeps(stores)?; + delete_swept_bodies(stores, &pending)?; + let used = stores.used_bytes()?; + if used <= cap { + return Ok(Vec::new()); + } + let count = count(&stores.db)? as u64; + if count == 0 { + return Ok(Vec::new()); + } + let average = (used / count).max(1); + let batch = (used - cap).div_ceil(average).clamp(1, DELETE_BATCH as u64); + let rows = sweepable(stores, batch as usize)?; + let deleted = delete_swept_rows(stores, rows)?; + delete_swept_bodies(stores, &deleted)?; + Ok(deleted) +} + +/// Flows whose rows a sweep deleted and whose bodies it may not have, at most a batch of them. +pub(crate) fn pending_sweeps(stores: &Stores) -> Result, String> { + stores + .db + .with_conn(|c| { + let conn = c.conn().resolve(); + let mut stmt = conn.prepare("SELECT flow_id FROM flow_sweep_pending LIMIT ?")?; + stmt.query_map([DELETE_BATCH as i64], |r| r.get(0))?.collect::>() + }) + .map_err(sweep_failed) +} + +/// A stream still open: it has no end yet, and its writer goes on adding messages to it. +const OPEN_STREAM: &str = + "stream_kind IS NOT NULL AND elapsed_ms IS NULL AND error IS NULL AND NOT cancelled"; + +/// Fail messages for sweeping, all of a kind. +fn sweep_failed(e: impl std::fmt::Display) -> String { + format!("Failed to delete the oldest flows: {e}") +} + +/// Up to `limit` of the oldest flows that are neither cleared nor an open stream, by rowid. +pub(crate) fn sweepable(stores: &Stores, limit: usize) -> Result, String> { + stores + .db + .with_conn(|c| { + let conn = c.conn().resolve(); + let mut stmt = conn.prepare(&format!( + "SELECT rowid, id FROM flows WHERE rowid > ? + AND NOT ({OPEN_STREAM}) + ORDER BY rowid LIMIT ?" + ))?; + stmt.query_map(rusqlite::params![clear_mark(conn), limit as i64], |r| { + Ok((r.get(0)?, r.get(1)?)) + })? + .collect::>() + }) + .map_err(sweep_failed) +} + +/// Deletes the flows' rows and their messages' rows in one transaction, first, so nothing +/// written for them after this has an owner left to count under, and records them as pending +/// until their bodies go, so a failure before then is finished by the next sweep. A flow that +/// changed or was cleared since it was chosen is kept. Returns the ids deleted. +pub(crate) fn delete_swept_rows( + stores: &Stores, + rows: Vec<(i64, String)>, +) -> Result, String> { + stores + .db + .with_tx(|c| { + let conn = c.conn().resolve(); + let mut delete = conn.prepare(&format!( + "DELETE FROM flows WHERE rowid = ? AND id = ? + AND NOT ({OPEN_STREAM})" + ))?; + let mut delete_messages = + conn.prepare("DELETE FROM flow_messages WHERE flow_id = ?")?; + let mut pending = + conn.prepare("INSERT OR IGNORE INTO flow_sweep_pending (flow_id) VALUES (?)")?; + let mut deleted = Vec::with_capacity(rows.len()); + for (rowid, id) in rows { + if delete.execute(rusqlite::params![rowid, id])? > 0 { + delete_messages.execute([&id])?; + pending.execute([&id])?; + deleted.push(id); + } + } + Ok::<_, yaak_database::Error>(deleted) + }) + .map_err(sweep_failed) +} + +/// Deletes the bodies and message payloads of flows whose rows are gone, then once more the +/// message rows: anything a writer added for them while the rows went is an orphan, which +/// would otherwise go on counting against the cap with nothing to sweep it by. The flows stop +/// being pending once this is done. +pub(crate) fn delete_swept_bodies(stores: &Stores, ids: &[String]) -> Result<(), String> { + if ids.is_empty() { + return Ok(()); + } + stores + .blobs + .with_tx(|b| { + for id in ids { + b.delete_chunks(&Flow::body_id(id, FlowBody::Request))?; + b.delete_chunks(&Flow::body_id(id, FlowBody::Response))?; + b.delete_chunks_like(&format!("{id}.msg.%"))?; + } + Ok::<_, yaak_models::error::Error>(()) + }) + .map_err(|e| sweep_failed(format!("flow bodies: {e}")))?; + stores + .db + .with_tx(|c| { + let conn = c.conn().resolve(); + let mut orphans = conn.prepare( + "DELETE FROM flow_messages WHERE flow_id = ? + AND NOT EXISTS (SELECT 1 FROM flows WHERE flows.id = flow_messages.flow_id)", + )?; + let mut done = conn.prepare("DELETE FROM flow_sweep_pending WHERE flow_id = ?")?; + for id in ids { + orphans.execute([id])?; + done.execute([id])?; + } + Ok::<_, yaak_database::Error>(()) + }) + .map_err(sweep_failed) +} + /// A flow by id, with its number. None once it is cleared, though its row may not be deleted /// yet. pub fn find(db: &ProxyQueryManager, flow_id: &str) -> Option { @@ -1014,6 +1173,42 @@ mod tests { assert_eq!(app(&["com.google.Chrome", "/usr/bin/curl"]), 3); } + /// A program in a folder: its working directory by the whole path, or its folder's name in + /// any case, along with whatever else the filter asks. + #[test] + fn cwd_matches_a_path_or_a_folder_name() { + let node_in = |cwd: Option<&str>| Flow { + client_name: Some("node".into()), + client_path: Some("/usr/local/bin/node".into()), + client_cwd: cwd.map(String::from), + ..flow("https://example.com/", "GET", Some(200), None) + }; + let (_dir, db) = db_of(vec![ + node_in(Some("/Users/ada/Workspace/fieldwork")), + node_in(Some("/Users/ada/Workspace/fieldwork")), + node_in(Some("/Users/ada/Workspace/Ledger")), + node_in(None), + Flow { client_cwd: Some("/tmp/fieldwork".into()), ..node_in(None) }, + Flow { client_path: Some("/usr/bin/curl".into()), ..node_in(Some("/tmp/ledger")) }, + ]); + let count = |app: Option<&str>, cwds: &[&str]| { + let cwd = Some(cwds.iter().map(|c| c.to_string()).collect()); + let app = app.map(|a| vec![a.to_string()]); + urls(&db, FlowFilter { app, cwd, ..Default::default() }).len() + }; + assert_eq!(count(None, &["/Users/ada/Workspace/fieldwork"]), 2, "the whole path"); + assert_eq!(count(None, &["/users/ada/workspace/fieldwork"]), 0, "a path exactly"); + assert_eq!(count(None, &["FieldWork"]), 3, "a folder name, any case"); + assert_eq!(count(None, &["fieldwork", "ledger"]), 5, "any of them"); + assert_eq!(count(Some("/usr/local/bin/node"), &["ledger"]), 1, "and the program"); + assert_eq!(count(Some("/usr/bin/curl"), &["ledger"]), 1); + assert_eq!(count(None, &[]), 0); + let query = FlowFilter { cwd: Some(vec!["a".into(), "/b".into()]), ..Default::default() }; + assert_eq!(query.to_query(), "cwd=a&cwd=%2Fb"); + assert_eq!(FlowFilter::from_query(&query.to_query()).unwrap(), query); + assert!(query.narrows()); + } + #[test] fn a_search_pages_back_and_counts_what_it_matches() { // Every third flow is a POST, newest last. @@ -1460,4 +1655,54 @@ mod tests { ("/wD+", BodyEncoding::Base64, 3) ); } + + /// Sweeping takes the flows written first, with their bodies, until the traffic is under the + /// cap, and no more; under it, nothing goes. + #[test] + fn sweeping_deletes_the_oldest_until_under_the_cap() { + use yaak_database::UpdateSource; + use yaak_models::blob_manager::BodyChunkWriter; + const BODY: usize = 64 * 1024; + let dir = tempfile::tempdir().unwrap(); + let stores = Stores::open(dir.path(), &crate::db::test_key()).unwrap(); + let empty = stores.used_bytes().unwrap(); + let ids: Vec = (0..20).map(|i| format!("fl_{i:02}")).collect(); + for id in &ids { + let flow = Flow { id: id.clone(), ..Default::default() }; + stores.db.with_tx(|tx| tx.upsert(&flow, &UpdateSource::Background)).unwrap(); + let body_id = Flow::body_id(id, FlowBody::Response); + let mut writer = BodyChunkWriter::new(stores.blobs.clone(), body_id); + writer.write(&[7; BODY]).unwrap(); + writer.finish().unwrap(); + } + let full = stores.used_bytes().unwrap(); + assert!(full > empty + (20 * BODY) as u64, "{empty} {full}"); + + assert_eq!(sweep_oldest(&stores, full).unwrap(), Vec::::new(), "under the cap"); + + let cap = (empty + full) / 2; + let mut removed = Vec::new(); + loop { + let swept = sweep_oldest(&stores, cap).unwrap(); + if swept.is_empty() { + break; + } + removed.extend(swept); + } + assert!(stores.used_bytes().unwrap() <= cap); + // About half went, oldest first, and no more than had to. + assert_eq!(removed, ids[..removed.len()]); + assert!((8..=11).contains(&removed.len()), "{removed:?}"); + let kept: Vec = list(&stores.db, &FlowFilter::default()) + .unwrap() + .into_iter() + .rev() + .map(|f| f.id) + .collect(); + assert_eq!(kept, ids[removed.len()..]); + let bodies = stores.blobs.connect(); + let has_body = |id: &str| bodies.body_exists(&Flow::body_id(id, FlowBody::Response)); + assert!(removed.iter().all(|id| !has_body(id).unwrap())); + assert!(kept.iter().all(|id| has_body(id).unwrap())); + } } diff --git a/crates-proxy/yaak-proxy-lib/src/lib.rs b/crates-proxy/yaak-proxy-lib/src/lib.rs index 5411f408b..80e16fb68 100644 --- a/crates-proxy/yaak-proxy-lib/src/lib.rs +++ b/crates-proxy/yaak-proxy-lib/src/lib.rs @@ -6,6 +6,7 @@ pub mod client; pub mod control; pub mod convert; pub mod db; +mod exit_gate; pub mod flows; pub mod grpc; pub mod har; @@ -15,6 +16,7 @@ pub mod messages; pub mod models; pub mod save; pub mod settings; +pub mod terminal; use crate::actions::{ActionInvocation, ActionMetadata, GlobalAction}; use crate::attribution::Attribution; @@ -40,7 +42,7 @@ use std::io::Write; use std::net::{SocketAddr, TcpStream}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::{Arc, Mutex, OnceLock, PoisonError, RwLock}; +use std::sync::{Arc, Condvar, Mutex, OnceLock, PoisonError, RwLock, Weak}; use std::time::{Duration, SystemTime}; use ts_rs::TS; pub use yaak_crypto::MasterKeySource; @@ -97,6 +99,11 @@ pub struct ProxyCtx { stores: Arc>>, /// Why they could not be unlocked, the last time it was tried. lock_reason: Arc>>, + /// Resets of the captured data so far, so a thread deleting from databases since deleted + /// stops. + resets: Arc, + /// The thread keeping captured traffic under the keep-traffic cap, while there is one. + sweeper: Arc<(Mutex, Condvar)>, pub events: RpcEventEmitter, /// gRPC schemas as decoded, and the protoc that compiles attached protos. pub grpc: GrpcState, @@ -137,6 +144,8 @@ impl ProxyCtx { stops: Arc::new(AtomicU64::new(0)), stores: Arc::new(RwLock::new(None)), lock_reason: Arc::new(Mutex::new(None)), + resets: Arc::new(AtomicU64::new(0)), + sweeper: Default::default(), events, grpc: GrpcState::default(), }; @@ -243,6 +252,8 @@ impl ProxyCtx { converted?; } let stores = Stores::open(&self.data_dir, &db_key)?; + // SQLCipher's exit handler is registered by now, so the gate's runs before it. + exit_gate::install(); *self.stores.write().unwrap_or_else(PoisonError::into_inner) = Some(stores); *unlocked = Some(key.clone()); drop(unlocked); @@ -253,6 +264,7 @@ impl ProxyCtx { if self.db().is_ok_and(|db| db.with_conn(|c| flows::clear_mark(c.conn().resolve())) > 0) { self.delete_cleared(); } + self.sweep_soon(); Ok(key) } @@ -336,7 +348,13 @@ impl ProxyCtx { /// another process owns the engine, so there are never two. A failure is kept until the /// next start, for the app to show. pub fn start_on(&self, port: Option) -> Result<(), String> { - let started = self.start_engine(port); + self.start_routing(port, None) + } + + /// [`Self::start_on`], the system proxy pointed at the engine when `system_proxy` says so, + /// or as the settings ask at launch when it is None. + fn start_routing(&self, port: Option, system_proxy: Option) -> Result<(), String> { + let started = self.start_engine(port, system_proxy); if let Ok(mut start_error) = self.start_error.lock() { *start_error = started.as_ref().err().cloned(); } @@ -352,7 +370,7 @@ impl ProxyCtx { self.start_error.lock().ok()?.clone() } - fn start_engine(&self, port: Option) -> Result<(), String> { + fn start_engine(&self, port: Option, system_proxy: Option) -> Result<(), String> { if let Some(owner) = control::live_owner(&self.data_dir) && owner.pid != std::process::id() { @@ -403,6 +421,7 @@ impl ProxyCtx { attribution, recording: self.recording.clone(), finished: Default::default(), + clock: BatchClock::Real, }; std::thread::spawn(move || run_event_loop(event_rx, sink)); } @@ -414,10 +433,11 @@ impl ProxyCtx { if let Ok(mut last_port) = self.last_port.lock() { *last_port = Some(bound); } - self.system_proxy_on.store(settings.system_proxy_at_launch, Ordering::SeqCst); - if settings.system_proxy_at_launch { + let route = system_proxy.unwrap_or(settings.system_proxy_at_launch); + self.system_proxy_on.store(route, Ordering::SeqCst); + if route { // The password dialog can sit open for a while; the engine is up regardless. - let ctx = self.clone(); + let ctx = self.detached(); std::thread::spawn(move || { if let Err(e) = ctx.change_system_proxy(true, bound, true) { warn!("Failed to turn on the system proxy: {e}"); @@ -463,7 +483,7 @@ impl ProxyCtx { self.system_proxy_on.store(false, Ordering::SeqCst); let port = self.take_handle()?; if let Some(port) = port { - let ctx = self.clone(); + let ctx = self.detached(); std::thread::spawn(move || { if let Err(e) = ctx.change_system_proxy(false, port, true) { warn!("Failed to turn off the system proxy: {e}"); @@ -533,7 +553,7 @@ impl ProxyCtx { let mut ports: Vec = previous.and_then(|p| p.proxy_port).into_iter().collect(); ports.extend(self.settings().ok().map(|s| s.port)); - let ctx = self.clone(); + let ctx = self.detached(); std::thread::spawn(move || { if let Err(e) = ctx.reset_stale_system_proxy(&ports) { warn!("Failed to reset a stale system proxy: {e}"); @@ -686,7 +706,9 @@ impl ProxyCtx { pub fn set_system_proxy_at_launch(&self, enabled: bool) -> Result<(), String> { let mut settings = self.settings()?; settings.system_proxy_at_launch = enabled; - settings.save(&self.settings_path()) + settings.save(&self.settings_path())?; + self.emit_state(); + Ok(()) } /// The HTTP versions offered to clients, saved in `settings.json` and applied to the running @@ -700,9 +722,257 @@ impl ProxyCtx { { handle.set_http2(version == HttpVersion::Auto); } + self.emit_state(); Ok(()) } + /// The port the engine starts on, saved in `settings.json`. A running engine is restarted on + /// it at once, the system proxy following. When it cannot bind there, the port before is put + /// back and the engine restarted on it, and why is returned. Another process's engine takes + /// it at its next start. + pub fn set_port(&self, port: u16) -> Result { + if port == 0 { + return Err("The port must be from 1 to 65535".to_string()); + } + let mut settings = self.settings()?; + let previous = settings.port; + settings.port = port; + settings.save(&self.settings_path())?; + if self.port().is_none_or(|running| running == port) { + self.emit_state(); + return Ok(port); + } + // Read once: stopping clears it, and a failed start would leave the rollback reading + // the cleared value, so the old port would come back without the system proxy. + let proxy_on = self.system_proxy_on.load(Ordering::SeqCst); + if let Err(e) = self.restart(proxy_on) { + settings.port = previous; + settings.save(&self.settings_path())?; + if let Err(e) = self.restart(proxy_on) { + warn!("Failed to restart the proxy on port {previous}: {e}"); + } + return Err(e); + } + Ok(port) + } + + /// Stops the engine and starts it again as `settings.json` has it, routing the system proxy + /// to it only when `proxy_on`, as it was before the first stop: a restart keeps the routing + /// as the user left it rather than applying the launch setting again. + fn restart(&self, proxy_on: bool) -> Result<(), String> { + self.stop()?; + self.start_routing(None, Some(proxy_on)) + } + + /// Bytes of each body stored per flow, saved in `settings.json` and applied to the running + /// engine's bodies from now on. + pub fn set_capture_body_limit(&self, bytes: usize) -> Result { + let mut settings = self.settings()?; + settings.capture_body_limit = bytes; + settings.save(&self.settings_path())?; + if let Ok(handle) = self.handle.lock() + && let Some(handle) = handle.as_ref() + { + handle.set_capture_body_limit(bytes); + } + self.emit_state(); + Ok(bytes) + } + + /// The most captured traffic kept, saved in `settings.json`, or None to keep everything. + /// Swept down to it soon after. + pub fn set_keep_traffic(&self, bytes: Option) -> Result, String> { + if bytes == Some(0) { + return Err("Keep more than 0 bytes of traffic, or all of it".to_string()); + } + let mut settings = self.settings()?; + settings.keep_traffic_bytes = bytes; + settings.save(&self.settings_path())?; + self.sweep_soon(); + self.emit_state(); + Ok(bytes) + } + + /// Bytes the captured traffic takes, as the keep-traffic cap counts them: the databases' + /// used pages, so it shrinks as flows are deleted though the files do not. Locked, the + /// files' sizes instead, as nothing in them can be read. + pub fn storage_size(&self) -> Result { + match self.stores() { + Ok(stores) => stores.used_bytes(), + Err(_) => Ok(db::file_bytes(&self.data_dir)), + } + } + + /// Deletes every flow, body and message for good, with the data key that encrypts them, and + /// opens new databases under a new one, so the app goes on, empty. Works while locked, as + /// the way out of a key the keychain lost. The master key in the keychain stays: it wraps + /// only the data key, and wraps the new one, or a new one is made when it is gone. The CA is + /// kept, its key encrypted again, while the old key can read it; locked, it goes too, and a + /// new one has to be trusted. Refused while another process owns the engine, as it has the + /// databases open. A running engine is restarted on the new ones. + pub fn reset_captured_data(&self) -> Result<(), String> { + if let Some(owner) = control::live_owner(&self.data_dir) + && owner.pid != std::process::id() + { + return Err(format!("{}. Quit it first.", control::already_owned(&owner))); + } + let running = self.port().is_some(); + let proxy_on = self.system_proxy_on.load(Ordering::SeqCst); + if running { + self.stop()?; + } + { + // Held so nothing unlocks the old databases meanwhile. + let mut unlocked = self.unlocked.lock().unwrap_or_else(PoisonError::into_inner); + let old_key = unlocked.take(); + *self.stores.write().unwrap_or_else(PoisonError::into_inner) = None; + self.resets.fetch_add(1, Ordering::SeqCst); + *self.deleting_cleared.lock().unwrap_or_else(PoisonError::into_inner) = false; + + let ca_key = self.ca_dir().join("ca.key"); + let ca_pem = old_key.and_then(|key| match std::fs::read(&ca_key) { + Ok(stored) => key.decrypt(&stored).ok(), + Err(_) => None, + }); + // The event loop and a clear's batch let go of the databases moments after the + // engine stops; until then Windows refuses to delete files still open. + if let Err(e) = remove_captured_when_closed(&self.data_dir) { + drop(unlocked); + // What is left opens again, so the app is not left without its databases. + if let Err(e) = self.unlock() { + warn!("Failed to reopen the proxy's databases: {e}"); + } + if running { + self.start_routing(None, Some(proxy_on))?; + } + return Err(e); + } + let new_key = db::load_or_create_data_key(&self.keys, &self.data_dir); + match (ca_pem, &new_key) { + (Some(pem), Ok(new_key)) => { + let sealed = new_key.encrypt(&pem).map_err(|e| e.to_string())?; + write_private(&ca_key, &sealed)?; + } + // Unreadable without the old key: a new CA is made as the databases open. + _ => { + for name in ["ca.key", "ca.pem"] { + let _ = std::fs::remove_file(self.ca_dir().join(name)); + } + } + } + *self.lock_reason.lock().unwrap_or_else(PoisonError::into_inner) = None; + } + log::info!("Reset the proxy's captured traffic in {}", self.data_dir.display()); + let unlocked = self.unlock(); + // Whatever the app had is gone, opened again or not. + self.events.emit("flows_cleared", &FlowsClearedPayload {}); + self.emit_state(); + unlocked?; + if running { + self.start_routing(None, Some(proxy_on))?; + } + Ok(()) + } + + /// A clone sharing everything but the databases, for a thread that may outlive every + /// context that owns them, like one turning the system proxy off as the engine stops. Were + /// it the last to hold them, it would close them itself, and at a process's exit that races + /// SQLCipher's own teardown and crashes; this way they close where their owner lets go. + fn detached(&self) -> Self { + Self { stores: Arc::new(RwLock::new(None)), ..self.clone() } + } + + /// Has the sweeper look at the captured traffic's size now, started when the keep-traffic + /// cap is set and it is not running. Without a cap nothing runs, and a sweeper running + /// stops. + fn sweep_soon(&self) { + let (state, wake) = &*self.sweeper; + let mut sweeper = state.lock().unwrap_or_else(PoisonError::into_inner); + if sweeper.running { + sweeper.poked = true; + wake.notify_all(); + return; + } + if self.keep_traffic().is_none() { + return; + } + sweeper.running = true; + drop(sweeper); + let ctx = self.detached(); + let stores = Arc::downgrade(&self.stores); + let spawned = std::thread::Builder::new().name("proxy-sweep".into()).spawn(move || { + lower_thread_priority(); + ctx.run_sweeper(&stores); + }); + if let Err(e) = spawned { + warn!("Failed to start keeping captured traffic under its cap: {e}"); + state.lock().unwrap_or_else(PoisonError::into_inner).running = false; + } + } + + /// The keep-traffic cap, as `settings.json` has it. + fn keep_traffic(&self) -> Option { + self.settings().ok().and_then(|s| s.keep_traffic_bytes) + } + + /// Sweeps the oldest flows while the captured traffic is over the cap, then waits for + /// [`SWEEP_INTERVAL`] or a nudge, until the cap is taken away. The cap is read under the + /// sweeper's lock, so one set as this stops finds it stopped and starts another. + fn run_sweeper(&self, stores: &Weak>>) { + let (state, wake) = &*self.sweeper; + loop { + let cap = { + let mut sweeper = state.lock().unwrap_or_else(PoisonError::into_inner); + let Some(cap) = self.keep_traffic() else { + sweeper.running = false; + return; + }; + sweeper.poked = false; + cap + }; + self.sweep_under(cap, stores); + let sweeper = state.lock().unwrap_or_else(PoisonError::into_inner); + if !sweeper.poked { + drop(wake.wait_timeout(sweeper, SWEEP_INTERVAL)); + } + } + } + + /// Deletes the oldest flows a batch at a time, yielding the writer between, until the + /// captured traffic is under `cap`, telling the app of each batch. Stops early when the + /// sweeper is nudged, to read the cap again. `stores` is the context's databases, held + /// weakly so the sweeper never keeps them open past the context that owns them. + fn sweep_under(&self, cap: u64, stores: &Weak>>) { + loop { + // Held through the batch, letting go of the databases included. + let gate = exit_gate::hold(); + // Each batch from the databases open now, which a reset replaces. + let Some(stores) = stores + .upgrade() + .and_then(|s| s.read().unwrap_or_else(PoisonError::into_inner).clone()) + else { + return; + }; + match flows::sweep_oldest(&stores, cap) { + Ok(ids) if ids.is_empty() => return, + Ok(ids) => { + // After the commit, as a clear's is. + self.events.emit("flows_removed", &FlowsRemovedPayload { ids }); + } + Err(e) => { + warn!("{e}"); + return; + } + } + drop(stores); + drop(gate); + if self.sweeper.0.lock().unwrap_or_else(PoisonError::into_inner).poked { + return; + } + std::thread::sleep(DELETE_PAUSE); + } + } + /// What HTTPS is decrypted, as `settings.json` has it. pub fn decrypt_list(&self) -> Result { Ok(self.settings()?.decrypt_list()) @@ -873,18 +1143,46 @@ impl ProxyCtx { /// already doing so. Yields the writer between batches, so new flows are written meanwhile. fn delete_cleared(&self) { // Only ever after a clear or unlocking, so open. - let Ok(Stores { db, blobs }) = self.stores() else { + let Ok(Stores { db, blobs, .. }) = self.stores() else { return; }; if !claim(&self.deleting_cleared) { return; } let (running, cleaner) = (self.deleting_cleared.clone(), self.cleaner_id.clone()); + let (resets, reset) = (self.resets.clone(), self.resets.load(Ordering::SeqCst)); let spawned = std::thread::Builder::new().name("proxy-clear".into()).spawn(move || { lower_thread_priority(); + clean(&db, &blobs, &running, &cleaner, &resets, reset); + // Its last hold on the databases, which may close them, is let go of in a batch too. + let _gate = exit_gate::hold(); + drop((db, blobs)); + }); + if let Err(e) = spawned { + warn!("Failed to start deleting cleared flows: {e}"); + *self.deleting_cleared.lock().unwrap_or_else(PoisonError::into_inner) = false; + } + + /// Deletes cleared flows a batch at a time until none are left, another process takes + /// over, or a reset replaces the databases. + fn clean( + db: &ProxyQueryManager, + blobs: &BlobManager, + running: &Mutex, + cleaner: &str, + resets: &AtomicU64, + reset: u64, + ) { let mut retry = DELETE_PAUSE; loop { - match flows::delete_cleared(&db, &blobs, flows::DELETE_BATCH, &cleaner) { + let gate = exit_gate::hold(); + // Its databases are gone, and the reset has let another start. + if resets.load(Ordering::SeqCst) != reset { + return; + } + let cleaned = flows::delete_cleared(db, blobs, flows::DELETE_BATCH, cleaner); + drop(gate); + match cleaned { Ok(flows::Cleaned::More) => { retry = DELETE_PAUSE; std::thread::sleep(DELETE_PAUSE); @@ -896,7 +1194,11 @@ impl ProxyCtx { *running.lock().unwrap_or_else(PoisonError::into_inner) = false; // A clear since the last batch saw this running and left its flows to // it, so they are deleted here unless another thread has taken over. - if flows::clear_mark_in(&db) == 0 || !claim(&running) { + let more = { + let _gate = exit_gate::hold(); + flows::clear_mark_in(db) != 0 + }; + if !more || !claim(running) { return; } } @@ -909,10 +1211,6 @@ impl ProxyCtx { } } } - }); - if let Err(e) = spawned { - warn!("Failed to start deleting cleared flows: {e}"); - *self.deleting_cleared.lock().unwrap_or_else(PoisonError::into_inner) = false; } } } @@ -923,6 +1221,50 @@ fn claim(flag: &Mutex) -> bool { !std::mem::replace(&mut *set, true) } +/// Whether the sweeper runs, and whether it has been asked to look again since it last read the +/// cap. +#[derive(Debug, Default)] +struct Sweeper { + running: bool, + poked: bool, +} + +/// Deletes the captured databases, trying again for a few seconds while something still has +/// them open, as Windows refuses to delete an open file. +fn remove_captured_when_closed(data_dir: &Path) -> Result<(), String> { + let mut tries = 0; + loop { + match db::remove_captured(data_dir) { + Ok(()) => return Ok(()), + Err(_) if tries < REMOVE_TRIES => { + tries += 1; + std::thread::sleep(Duration::from_millis(100)); + } + Err(e) => return Err(e), + } + } +} + +/// Tries at deleting the captured databases, 100 ms apart, before a reset gives up. +const REMOVE_TRIES: u32 = 50; + +/// How often the sweeper measures the captured traffic against the keep-traffic cap. +const SWEEP_INTERVAL: Duration = Duration::from_secs(30); + +/// Writes `contents` to `path` through a file beside it and a rename, readable by the user +/// alone where the platform has such modes. +fn write_private(path: &Path, contents: &[u8]) -> Result<(), String> { + let fail = |e: std::io::Error| format!("Failed to write {}: {e}", path.display()); + let tmp = path.with_extension("tmp"); + let mut options = std::fs::OpenOptions::new(); + options.write(true).create(true).truncate(true); + #[cfg(unix)] + std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600); + let mut file = options.open(&tmp).map_err(fail)?; + file.write_all(contents).and_then(|_| file.sync_all()).map_err(fail)?; + std::fs::rename(&tmp, path).map_err(fail) +} + /// How long deleting cleared flows rests between batches, leaving the database to new flows. const DELETE_PAUSE: Duration = Duration::from_millis(5); @@ -1051,6 +1393,13 @@ pub struct ClearFlowsRequest {} #[ts(export, export_to = "gen_rpc.ts")] pub struct FlowsClearedPayload {} +/// Flows deleted to keep captured traffic under the keep-traffic cap, the oldest first. +#[derive(Serialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct FlowsRemovedPayload { + pub ids: Vec, +} + #[derive(Deserialize, TS)] #[ts(export, export_to = "gen_rpc.ts")] pub struct GetProxyStateRequest {} @@ -1070,6 +1419,12 @@ pub struct GetProxyStateResponse { pub system_proxy_at_launch: bool, /// The HTTP versions offered to clients. pub http_version: HttpVersion, + /// Bytes of each body stored per flow. + #[ts(type = "number")] + pub capture_body_limit: usize, + /// The most captured traffic kept, in bytes, or null to keep everything. + #[ts(type = "number | null")] + pub keep_traffic_bytes: Option, /// Why the engine last failed to start, until it starts. pub start_error: Option, /// How far encrypting traffic stored before encryption has got, from 0 to 1, while it @@ -1154,6 +1509,51 @@ pub struct SetHttpVersionRequest { pub http_version: HttpVersion, } +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct SetPortRequest { + pub port: u16, +} + +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct SetCaptureBodyLimitRequest { + #[ts(type = "number")] + pub bytes: u64, +} + +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct SetKeepTrafficRequest { + /// Null keeps everything. + #[ts(type = "number | null")] + pub bytes: Option, +} + +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct GetStorageSizeRequest {} + +#[derive(Serialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct GetStorageSizeResponse { + /// The databases' used pages, as the keep-traffic cap counts them; locked, their files' + /// sizes on disk. + #[ts(type = "number")] + pub bytes: u64, +} + +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct ResetCapturedDataRequest {} + +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct OpenTerminalRequest { + /// Environment variables the shell starts with, by name: `HTTPS_PROXY`. + pub vars: Vec<(String, String)>, +} + #[derive(Deserialize, TS)] #[ts(export, export_to = "gen_rpc.ts")] pub struct GetDecryptListRequest {} @@ -1561,6 +1961,8 @@ async fn get_proxy_state( recording: ctx.recording(), system_proxy_at_launch: settings.system_proxy_at_launch, http_version: settings.http_version, + capture_body_limit: settings.capture_body_limit, + keep_traffic_bytes: settings.keep_traffic_bytes, start_error: ctx.start_error(), encrypting: ctx.encrypting(), locked: ctx.locked(), @@ -1588,6 +1990,46 @@ fn set_http_version(ctx: &ProxyCtx, req: SetHttpVersionRequest) -> Result Result { + blocking(move || ctx.set_port(req.port)).await +} + +fn set_capture_body_limit( + ctx: &ProxyCtx, + req: SetCaptureBodyLimitRequest, +) -> Result { + let bytes = usize::try_from(req.bytes).map_err(rpc_error)?; + ctx.set_capture_body_limit(bytes).map_err(rpc_error) +} + +/// The cap as saved, in bytes, or null to keep everything. +fn set_keep_traffic(ctx: &ProxyCtx, req: SetKeepTrafficRequest) -> Result, RpcError> { + let bytes = ctx.set_keep_traffic(req.bytes).map_err(rpc_error)?; + Ok(bytes.map(|b| usize::try_from(b).unwrap_or(usize::MAX))) +} + +async fn get_storage_size( + ctx: ProxyCtx, + _req: GetStorageSizeRequest, +) -> Result { + let bytes = blocking(move || ctx.storage_size()).await?; + Ok(GetStorageSizeResponse { bytes }) +} + +async fn reset_captured_data( + ctx: ProxyCtx, + _req: ResetCapturedDataRequest, +) -> Result { + blocking(move || ctx.reset_captured_data()).await?; + Ok(true) +} + +async fn open_terminal(_ctx: ProxyCtx, req: OpenTerminalRequest) -> Result { + blocking(move || terminal::open(&req.vars)).await?; + Ok(true) +} + fn get_decrypt_list(ctx: &ProxyCtx, _req: GetDecryptListRequest) -> Result { ctx.decrypt_list().map_err(rpc_error) } @@ -1991,6 +2433,28 @@ struct FlowSink { recording: Arc>, /// Flows that have ended, waiting to be written together. finished: std::cell::RefCell, + /// What a batch's wait is timed by. + clock: BatchClock, +} + +/// What the event loop times a batch's wait by: the time, or in tests a time that stands still, +/// so a batch is written only once no events wait or it is full. +#[derive(Debug, Clone, Copy, Default)] +enum BatchClock { + #[default] + Real, + #[cfg(test)] + Stopped(std::time::Instant), +} + +impl BatchClock { + fn now(self) -> std::time::Instant { + match self { + Self::Real => std::time::Instant::now(), + #[cfg(test)] + Self::Stopped(at) => at, + } + } } /// Flows that have ended and are yet to be written, with when the first of them ended. Each @@ -2033,6 +2497,7 @@ fn run_event_loop(rx: std::sync::mpsc::Receiver, sink: FlowSink) { match rx.try_recv() { Ok(event) => event, Err(TryRecvError::Empty) => { + let _gate = exit_gate::hold(); sink.write_messages(&mut in_flight, &mut pending); sink.write_finished(); sink.write_due_flows(&mut in_flight); @@ -2045,6 +2510,7 @@ fn run_event_loop(rx: std::sync::mpsc::Receiver, sink: FlowSink) { match rx.recv_timeout(wait) { Ok(event) => event, Err(RecvTimeoutError::Timeout) => { + let _gate = exit_gate::hold(); sink.write_due_flows(&mut in_flight); continue; } @@ -2056,6 +2522,8 @@ fn run_event_loop(rx: std::sync::mpsc::Receiver, sink: FlowSink) { Err(_) => break, } }; + // Only while writing, not while waiting for an event above. + let _gate = exit_gate::hold(); if let ProxyEvent::Message { id, message } = event { record_message(&mut in_flight, &mut pending, id, &message); if pending.len() >= MESSAGE_BATCH { @@ -2080,6 +2548,7 @@ fn run_event_loop(rx: std::sync::mpsc::Receiver, sink: FlowSink) { } } + let _gate = exit_gate::hold(); sink.write_messages(&mut in_flight, &mut pending); // The engine stopped: a stream still open ends with it. Other flows in flight are dropped, // as they always have been. Every flow that ended is written. @@ -2091,6 +2560,8 @@ fn run_event_loop(rx: std::sync::mpsc::Receiver, sink: FlowSink) { } } sink.write_finished(); + // Its hold on the databases goes before the gate opens again. + drop(sink); } /// When the next changed streaming flow is due to be written. @@ -2174,6 +2645,7 @@ impl FlowSink { method, url, client_http_version: http_version, + client_addr: Some(client_addr.to_string()), ..Default::default() }; in_flight.insert( @@ -2202,6 +2674,7 @@ impl FlowSink { method: "CONNECT".to_string(), url: format!("https://{authority}"), client_http_version: "HTTP/1.1".to_string(), + client_addr: Some(client_addr.to_string()), tunnelled: true, ..Default::default() }; @@ -2274,9 +2747,10 @@ impl FlowSink { f.changed(); } } - ProxyEvent::RequestBodyComplete { id, size, truncated } => { + ProxyEvent::RequestBodyComplete { id, size, truncated, elapsed_ms } => { if let Some(f) = in_flight.get_mut(&id) { f.flow.request_size = size as i64; + f.flow.elapsed_request_ms = Some(clamp_ms(elapsed_ms)); if f.keeps_bytes(FlowBody::Request) { f.flow.request_truncated = truncated; finish_body(f.request_body.take()); @@ -2290,6 +2764,21 @@ impl FlowSink { f.changed(); } } + ProxyEvent::UpstreamConnection { + id, + elapsed_connect_ms, + elapsed_tls_ms, + tls_version, + tls_cipher, + } => { + if let Some(f) = in_flight.get_mut(&id) { + f.flow.elapsed_connect_ms = elapsed_connect_ms.map(clamp_ms); + f.flow.elapsed_tls_ms = elapsed_tls_ms.map(clamp_ms); + f.flow.upstream_tls_version = tls_version; + f.flow.upstream_tls_cipher = tls_cipher; + f.changed(); + } + } ProxyEvent::ResponseStart { id, status, http_version, remote_addr, elapsed_ms } => { if let Some(f) = in_flight.get_mut(&id) { f.flow.status = Some(status as i32); @@ -2496,7 +2985,7 @@ impl FlowSink { flow.client_cwd = app.cwd; } let mut finished = self.finished.borrow_mut(); - finished.since.get_or_insert_with(std::time::Instant::now); + finished.since.get_or_insert_with(|| self.clock.now()); finished.flows.push(FinishedFlow { flow, persisted: f.persisted, chunks }); } @@ -2508,7 +2997,9 @@ impl FlowSink { fn finished_due(&self) -> bool { let finished = self.finished.borrow(); finished.flows.len() >= FLOW_BATCH - || finished.since.is_some_and(|t| t.elapsed() >= FLOW_BATCH_WAIT) + || finished + .since + .is_some_and(|t| self.clock.now().saturating_duration_since(t) >= FLOW_BATCH_WAIT) } /// Writes the flows that have ended: their bodies' last chunks in one transaction, then @@ -2657,6 +3148,8 @@ define_rpc! { set_recording(SetRecordingRequest) -> bool, set_system_proxy_at_launch(SetSystemProxyAtLaunchRequest) -> bool, set_http_version(SetHttpVersionRequest) -> HttpVersion, + set_capture_body_limit(SetCaptureBodyLimitRequest) -> usize, + set_keep_traffic(SetKeepTrafficRequest) -> Option, } async_commands { add_grpc_schema(AddGrpcSchemaRequest) -> GrpcSchema, @@ -2682,10 +3175,15 @@ define_rpc! { system_proxy_set(SystemProxySetRequest) -> SystemProxyStatus, system_proxy_status(SystemProxyStatusRequest) -> SystemProxyStatus, take_over(TakeOverRequest) -> bool, + set_port(SetPortRequest) -> u16, + get_storage_size(GetStorageSizeRequest) -> GetStorageSizeResponse, + reset_captured_data(ResetCapturedDataRequest) -> bool, + open_terminal(OpenTerminalRequest) -> bool, } events { decrypt_list_changed(DecryptListChangedPayload), flows_cleared(FlowsClearedPayload), + flows_removed(FlowsRemovedPayload), message_write(MessagePayload), model_write(ModelPayload), proxy_state_changed(ProxyStatePayload), @@ -2738,17 +3236,31 @@ mod tests { /// An event loop writing to `ctx`'s databases, fed by the returned sender. fn event_loop( ctx: &ProxyCtx, + ) -> (std::sync::mpsc::Sender, std::thread::JoinHandle<()>) { + event_loop_timed(ctx, BatchClock::Real) + } + + /// [`event_loop`], its batches timed by `clock`. + fn event_loop_timed( + ctx: &ProxyCtx, + clock: BatchClock, ) -> (std::sync::mpsc::Sender, std::thread::JoinHandle<()>) { let (tx, rx) = std::sync::mpsc::channel(); - let sink = FlowSink { + let sink = sink(ctx); + let sink = FlowSink { clock, ..sink }; + (tx, std::thread::spawn(move || run_event_loop(rx, sink))) + } + + fn sink(ctx: &ProxyCtx) -> FlowSink { + FlowSink { db: ctx.db().unwrap(), blobs: ctx.blobs().unwrap(), events: ctx.events.clone(), attribution: Attribution::new(0), recording: ctx.recording.clone(), finished: Default::default(), - }; - (tx, std::thread::spawn(move || run_event_loop(rx, sink))) + clock: BatchClock::Real, + } } fn open_websocket(tx: &std::sync::mpsc::Sender, id: u64, url: &str) { @@ -2842,6 +3354,7 @@ mod tests { id: 1, size: request.len() as u64, truncated: false, + elapsed_ms: 1, }); send(ProxyEvent::ResponseStart { id: 1, @@ -3020,7 +3533,7 @@ mod tests { len: 30, data: bytes::Bytes::from_static(b"junk"), }); - send(ProxyEvent::RequestBodyComplete { id: 1, size: 30, truncated: true }); + send(ProxyEvent::RequestBodyComplete { id: 1, size: 30, truncated: true, elapsed_ms: 1 }); send(ProxyEvent::ResponseStart { id: 1, status: 200, @@ -3548,22 +4061,403 @@ mod tests { } } - /// A flow that ends alone is written at once, not after the batch's wait. + /// A flow that ends alone is written at once, not after the batch's wait: with the batch's + /// clock stopped, the wait never ends, so only the queue running dry writes it. #[test] fn a_lone_flow_shows_at_once() { let dir = tempfile::tempdir().unwrap(); let (events, _rx) = RpcEventEmitter::new(); let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); - let (tx, event_loop) = event_loop(&ctx); - let sent = std::time::Instant::now(); + let stopped = BatchClock::Stopped(std::time::Instant::now()); + let (tx, event_loop) = event_loop_timed(&ctx, stopped); send_flow(&tx, 1, "https://example.com/alone"); + // The engine still runs, so it is not the stop that writes it. wait_for_flow(&ctx, |f| f.url == "https://example.com/alone"); - let shown = sent.elapsed(); - assert!(shown < Duration::from_millis(50), "shown after {shown:?}"); drop(tx); event_loop.join().unwrap(); } + /// The batch's wait is what writes a batch while events keep coming. + #[test] + fn a_batch_is_due_once_its_first_flow_has_waited() { + let dir = tempfile::tempdir().unwrap(); + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + let started = std::time::Instant::now(); + let mut sink = FlowSink { clock: BatchClock::Stopped(started), ..sink(&ctx) }; + let f = InFlight { + flow: Flow { id: Flow::generate_id(), ..Default::default() }, + connection: 1, + request_body: None, + response_body: None, + persisted: false, + cleared: false, + next_seq: 0, + dirty: false, + last_write: None, + }; + sink.finish_flow(f); + assert!(!sink.finished_due(), "the clock stands still"); + sink.clock = BatchClock::Stopped(started + FLOW_BATCH_WAIT); + assert!(sink.finished_due()); + sink.write_finished(); + assert_eq!(flows::count(&ctx.db().unwrap()).unwrap(), 1); + } + + /// A flow keeps the client's address, its connection's timings and TLS, and when its + /// request was sent whole, for its timeline and details. + #[test] + fn a_flow_keeps_its_connection_and_timings() { + let dir = tempfile::tempdir().unwrap(); + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + let (tx, event_loop) = event_loop(&ctx); + let send = |event| tx.send(event).unwrap(); + send(ProxyEvent::RequestStart { + id: 1, + started_at: SystemTime::now(), + method: "POST".into(), + url: "https://api.example.com/upload".into(), + http_version: "HTTP/2".into(), + connection: 1, + client_addr: "127.0.0.1:54321".parse().unwrap(), + }); + send(ProxyEvent::UpstreamConnection { + id: 1, + elapsed_connect_ms: Some(3), + elapsed_tls_ms: Some(9), + tls_version: Some("TLSv1.3".into()), + tls_cipher: Some("TLS13_AES_128_GCM_SHA256".into()), + }); + let data = bytes::Bytes::from_static(b"payload"); + send(ProxyEvent::RequestBodyChunk { id: 1, len: data.len(), data }); + send(ProxyEvent::RequestBodyComplete { id: 1, size: 7, truncated: false, elapsed_ms: 12 }); + send(ProxyEvent::ResponseStart { + id: 1, + status: 201, + http_version: "HTTP/2".into(), + remote_addr: Some("93.184.216.34:443".parse().unwrap()), + elapsed_ms: 20, + }); + send(ProxyEvent::ResponseBodyComplete { + id: 1, + size: 0, + truncated: false, + elapsed_ms: 25, + error: None, + cancelled: false, + }); + drop(tx); + event_loop.join().unwrap(); + let flow = flows::list(&ctx.db().unwrap(), &Default::default()).unwrap().remove(0); + assert_eq!(flow.client_addr.as_deref(), Some("127.0.0.1:54321")); + assert_eq!((flow.elapsed_connect_ms, flow.elapsed_tls_ms), (Some(3), Some(9))); + assert_eq!((flow.elapsed_request_ms, flow.elapsed_headers_ms), (Some(12), Some(20))); + assert_eq!(flow.upstream_tls_version.as_deref(), Some("TLSv1.3")); + assert_eq!(flow.upstream_tls_cipher.as_deref(), Some("TLS13_AES_128_GCM_SHA256")); + } + + /// Waits up to 5 seconds for `check`. + fn wait_until(what: &str, check: impl Fn() -> bool) { + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while !check() { + assert!(std::time::Instant::now() < deadline, "{what}"); + std::thread::sleep(Duration::from_millis(10)); + } + } + + fn sweeper_running(ctx: &ProxyCtx) -> bool { + ctx.sweeper.0.lock().unwrap().running + } + + /// Without a cap nothing sweeps. With one, the oldest flows go until the traffic is under + /// it, and the app hears which; taken away, the sweeper stops. + #[test] + fn keeping_traffic_under_a_cap_sweeps_the_oldest() { + let dir = tempfile::tempdir().unwrap(); + let (events, rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + let empty = ctx.storage_size().unwrap(); + let ids: Vec = (0..20).map(|i| format!("fl_{i:02}")).collect(); + for id in &ids { + flow_with_response(&ctx, id, "application/octet-stream", &[3; 64 * 1024]); + } + let full = ctx.storage_size().unwrap(); + + ctx.sweep_soon(); + assert!(!sweeper_running(&ctx), "no thread without a cap"); + assert_eq!(flows::count(&ctx.db().unwrap()).unwrap(), 20); + + let cap = (empty + full) / 2; + // Swept here rather than waited on from the sweeper's thread, which a busy machine may + // not run in time: what goes is the same either way. + ctx.sweep_under(cap, &Arc::downgrade(&ctx.stores)); + let removed: Vec = rx + .try_iter() + .filter(|e| e.event == "flows_removed") + .flat_map(|e| serde_json::from_value::>(e.payload["ids"].clone()).unwrap()) + .collect(); + let kept = flows::count(&ctx.db().unwrap()).unwrap(); + let after = ctx.storage_size().unwrap(); + let sizes = format!("empty {empty}, full {full}, cap {cap}, after {after}"); + assert!(after <= cap, "under the cap: {sizes}"); + assert!(!removed.is_empty() && removed.len() < 20, "some, not all: {removed:?}, {sizes}"); + assert_eq!(removed.len(), 20 - kept, "the app hears of each"); + assert_eq!(removed, ids[..removed.len()], "the oldest, in order"); + + // Set, the cap starts the sweeper, which finds nothing more to do; taken away, it stops. + assert_eq!(ctx.set_keep_traffic(Some(cap)).unwrap(), Some(cap)); + assert_eq!(ctx.settings().unwrap().keep_traffic_bytes, Some(cap)); + assert!(sweeper_running(&ctx)); + assert_eq!(ctx.set_keep_traffic(None).unwrap(), None); + wait_until("the sweeper stops", || !sweeper_running(&ctx)); + assert_eq!(flows::count(&ctx.db().unwrap()).unwrap(), kept, "nothing more went"); + assert!(ctx.set_keep_traffic(Some(0)).is_err()); + } + + /// Writes a body for `body_id` straight to the blob store, as the event loop does. + fn write_blob(ctx: &ProxyCtx, body_id: &str, bytes: &[u8]) { + let mut writer = BodyChunkWriter::new(ctx.blobs().unwrap(), body_id.to_string()); + writer.write(bytes).unwrap(); + writer.finish().unwrap(); + } + + /// The bodies stored under a flow's id: its request and response, and its messages' payloads + /// numbered below `messages`. + fn blobs_of(ctx: &ProxyCtx, id: &str, messages: usize) -> usize { + let blobs = ctx.blobs().unwrap(); + let conn = blobs.connect(); + let bodies = [ + Flow::body_id(id, FlowBody::Request), + Flow::body_id(id, FlowBody::Response), + ]; + let payloads = (0..messages).map(|n| format!("{id}.msg.{n}")); + bodies.into_iter().chain(payloads).filter(|b| conn.body_exists(b).unwrap()).count() + } + + /// A thread turning the system proxy on or off holds none of the databases, so it never + /// closes them itself, as it would at a process's exit, crashing in SQLCipher's teardown. + #[test] + fn threads_that_outlive_the_engine_hold_no_databases() { + let dir = tempfile::tempdir().unwrap(); + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + assert!(ctx.stores().is_ok()); + let detached = ctx.detached(); + assert!(detached.stores().is_err(), "none of its own"); + assert_eq!(Arc::strong_count(&ctx.stores), 1, "the owner's alone"); + // Everything else is shared, as the system proxy's state is. + ctx.system_proxy_on.store(true, Ordering::SeqCst); + assert!(detached.system_proxy_on.load(Ordering::SeqCst)); + } + + /// A message written for a flow while the sweep deletes it, before its row goes or between + /// its row and its bodies, is deleted with it rather than left counting against the cap. + #[test] + fn a_message_written_while_its_flow_is_swept_goes_too() { + let dir = tempfile::tempdir().unwrap(); + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + for id in ["fl_a", "fl_b"] { + flow_with_response(&ctx, id, "text/plain", &[1; 4096]); + } + let stores = ctx.stores().unwrap(); + + let rows = flows::sweepable(&stores, 1).unwrap(); + assert_eq!(rows.iter().map(|(_, id)| id.as_str()).collect::>(), ["fl_a"]); + // Written after it was chosen, before its row is deleted... + write_blob(&ctx, "fl_a.msg.7", b"late"); + let deleted = flows::delete_swept_rows(&stores, rows).unwrap(); + // ...and after its row is gone, before its bodies are. + write_blob(&ctx, "fl_a.msg.8", b"later"); + flows::delete_swept_bodies(&stores, &deleted).unwrap(); + + assert_eq!(deleted, ["fl_a"]); + assert_eq!(blobs_of(&ctx, "fl_a", 10), 0, "no orphans"); + assert_eq!(blobs_of(&ctx, "fl_b", 10), 1, "the other flow kept"); + } + + /// A sweep that deletes the rows and then fails before the bodies, as on a full disk, leaves + /// them pending, and the next sweep deletes them, even with nothing more to sweep. + #[test] + fn bodies_a_failed_sweep_left_go_on_the_next() { + let dir = tempfile::tempdir().unwrap(); + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + for id in ["fl_a", "fl_b"] { + flow_with_response(&ctx, id, "text/plain", &[1; 4096]); + } + let stores = ctx.stores().unwrap(); + + let rows = flows::sweepable(&stores, 1).unwrap(); + // The rows go, and the bodies don't. + assert_eq!(flows::delete_swept_rows(&stores, rows).unwrap(), ["fl_a"]); + assert_eq!(flows::pending_sweeps(&stores).unwrap(), ["fl_a"]); + assert_eq!(blobs_of(&ctx, "fl_a", 0), 1, "left behind"); + + // Under the cap, so only what was left goes. + assert!(flows::sweep_oldest(&stores, u64::MAX).unwrap().is_empty()); + assert_eq!(blobs_of(&ctx, "fl_a", 0), 0); + assert!(flows::pending_sweeps(&stores).unwrap().is_empty()); + assert_eq!(blobs_of(&ctx, "fl_b", 0), 1, "the other flow kept"); + } + + /// A stream still open is never swept, even while its writer adds messages as the sweep + /// runs and it is the oldest flow there is. + #[test] + fn an_open_stream_is_not_swept_while_it_is_written() { + let dir = tempfile::tempdir().unwrap(); + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + let open = Flow { + id: "fl_open".into(), + stream_kind: Some(StreamKind::Websocket), + status: Some(101), + ..Default::default() + }; + ctx.db().unwrap().with_tx(|tx| tx.upsert(&open, &UpdateSource::Background)).unwrap(); + write_blob(&ctx, "fl_open.msg.0", &[2; 16 * 1024]); + let ended: Vec = (0..10).map(|i| format!("fl_{i:02}")).collect(); + for id in &ended { + flow_with_response(&ctx, id, "application/octet-stream", &[3; 32 * 1024]); + } + + let done = Arc::new(AtomicBool::new(false)); + let writer = { + let (ctx, done) = (ctx.clone(), done.clone()); + std::thread::spawn(move || { + let mut seq = 1; + while !done.load(Ordering::SeqCst) { + write_blob(&ctx, &format!("fl_open.msg.{seq}"), &[4; 1024]); + seq += 1; + } + seq + }) + }; + // A cap below anything reachable, so the sweep takes every flow it may. + let stores = ctx.stores().unwrap(); + while !flows::sweep_oldest(&stores, 1).unwrap().is_empty() {} + done.store(true, Ordering::SeqCst); + let written = writer.join().unwrap(); + + let db = ctx.db().unwrap(); + assert!(flows::find(&db, "fl_open").is_some(), "the open stream stays"); + assert_eq!(flows::count(&db).unwrap(), 1, "every ended flow swept"); + assert_eq!(blobs_of(&ctx, "fl_open", written), written, "its messages all kept"); + for id in &ended { + assert_eq!(blobs_of(&ctx, id, 0), 0, "{id} left nothing behind"); + } + } + + /// Resetting deletes the traffic and its data key and opens empty databases under a new + /// one; the CA, readable with the old key, is kept, so it stays trusted. + #[test] + fn resetting_captured_data_starts_empty_and_keeps_the_ca() { + let dir = tempfile::tempdir().unwrap(); + let (events, rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + flow_with_response(&ctx, "fl_old", "text/plain", b"secret"); + let pem = ctx.load_ca().unwrap().pem().to_string(); + let key = std::fs::read(db::data_key_path(dir.path())).unwrap(); + rx.try_iter().count(); + + ctx.reset_captured_data().unwrap(); + assert!(ctx.locked().is_none()); + assert_eq!(flows::count(&ctx.db().unwrap()).unwrap(), 0); + assert!(!ctx.blobs().unwrap().connect().body_exists("fl_old.response").unwrap()); + assert_ne!(std::fs::read(db::data_key_path(dir.path())).unwrap(), key, "a new key"); + assert_eq!(ctx.load_ca().unwrap().pem(), pem, "the same CA"); + let heard: Vec<&str> = rx.try_iter().map(|e| e.event).collect(); + assert!(heard.contains(&"flows_cleared") && heard.contains(&"proxy_state_changed")); + // Capturing goes on into the new databases. + flow_with_response(&ctx, "fl_new", "text/plain", b"hello"); + assert_eq!(flows::count(&ctx.db().unwrap()).unwrap(), 1); + } + + /// With the key lost from the keychain, nothing can be read, and resetting is the way out: + /// the traffic and the CA, which only the lost key could read, make way for new ones. + #[test] + fn resetting_while_locked_is_the_way_out_of_a_lost_key() { + let dir = tempfile::tempdir().unwrap(); + let pem = { + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + flow_with_response(&ctx, "fl_old", "text/plain", b"secret"); + ctx.load_ca().unwrap().pem().to_string() + }; + // Another keychain, without the master key that wrapped the data key. + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + assert!(ctx.locked().is_some()); + assert!(ctx.db().is_err()); + assert!(ctx.storage_size().unwrap() > 0, "the files, measured on disk"); + + ctx.reset_captured_data().unwrap(); + assert!(ctx.locked().is_none()); + assert_eq!(flows::count(&ctx.db().unwrap()).unwrap(), 0); + assert_ne!(ctx.load_ca().unwrap().pem(), pem, "a new CA"); + flow_with_response(&ctx, "fl_new", "text/plain", b"hello"); + assert_eq!(flows::count(&ctx.db().unwrap()).unwrap(), 1); + } + + /// A port that is free and unused now. + fn free_port() -> u16 { + std::net::TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port() + } + + /// The running engine moves to a new port at once; one it cannot bind leaves it, and the + /// setting, where they were. + #[test] + fn setting_the_port_restarts_the_engine_on_it() { + let dir = tempfile::tempdir().unwrap(); + let first = free_port(); + let settings = + ProxySettings { port: first, system_proxy_at_launch: false, ..Default::default() }; + settings.save(&dir.path().join("settings.json")).unwrap(); + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + assert!(ctx.set_port(0).is_err()); + ctx.start().unwrap(); + assert_eq!(ctx.port(), Some(first)); + + let second = free_port(); + assert_eq!(ctx.set_port(second).unwrap(), second); + assert_eq!((ctx.port(), ctx.settings().unwrap().port), (Some(second), second)); + assert!(is_listening(second)); + + let taken = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let taken_port = taken.local_addr().unwrap().port(); + assert!(ctx.set_port(taken_port).is_err()); + assert_eq!((ctx.port(), ctx.settings().unwrap().port), (Some(second), second)); + assert_eq!(ctx.start_error(), None); + ctx.stop().unwrap(); + + // Stopped, it is only saved, for the next start. + let third = free_port(); + assert_eq!(ctx.set_port(third).unwrap(), third); + assert_eq!(ctx.port(), None); + ctx.start().unwrap(); + assert_eq!(ctx.port(), Some(third)); + ctx.stop().unwrap(); + } + + /// The capture limit is saved and shows in the proxy's state, as the keep-traffic cap does. + #[tokio::test(flavor = "multi_thread")] + async fn the_state_has_the_capture_limit_and_the_traffic_cap() { + let dir = tempfile::tempdir().unwrap(); + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + let req = SetCaptureBodyLimitRequest { bytes: 4096 }; + assert_eq!(set_capture_body_limit(&ctx, req).unwrap(), 4096); + let req = SetKeepTrafficRequest { bytes: Some(1 << 30) }; + assert_eq!(set_keep_traffic(&ctx, req).unwrap(), Some(1 << 30)); + let state = get_proxy_state(ctx.clone(), GetProxyStateRequest {}).await.unwrap(); + assert_eq!((state.capture_body_limit, state.keep_traffic_bytes), (4096, Some(1 << 30))); + let size = get_storage_size(ctx.clone(), GetStorageSizeRequest {}).await.unwrap(); + assert!(size.bytes > 0); + ctx.set_keep_traffic(None).unwrap(); + } + /// A flow that fails to write is rolled back alone: the others in its batch are kept. #[test] fn one_bad_flow_loses_no_others_in_its_batch() { @@ -3581,14 +4475,7 @@ mod tests { Ok::<_, yaak_database::Error>(()) }) .unwrap(); - let sink = FlowSink { - db: ctx.db().unwrap(), - blobs: ctx.blobs().unwrap(), - events: ctx.events.clone(), - attribution: Attribution::new(0), - recording: ctx.recording.clone(), - finished: Default::default(), - }; + let sink = sink(&ctx); // One batch of three, the middle one refused. for url in ["before", "bad", "after"] { let flow = Flow { diff --git a/crates-proxy/yaak-proxy-lib/src/models.rs b/crates-proxy/yaak-proxy-lib/src/models.rs index ceedbb3be..a11671ef3 100644 --- a/crates-proxy/yaak-proxy-lib/src/models.rs +++ b/crates-proxy/yaak-proxy-lib/src/models.rs @@ -59,6 +59,18 @@ pub struct Flow { pub elapsed_headers_ms: Option, /// Milliseconds from `started_at` to the end of the response. pub elapsed_ms: Option, + /// Milliseconds from `started_at` to the TCP connection to the server, and to the end of + /// its TLS handshake. Only for intercepted HTTPS on a connection opened for this flow. + pub elapsed_connect_ms: Option, + pub elapsed_tls_ms: Option, + /// Milliseconds from `started_at` to the request body being sent whole. None without one. + pub elapsed_request_ms: Option, + /// The client's end of its connection: `127.0.0.1:54321`. + pub client_addr: Option, + /// The TLS negotiated with the server for intercepted HTTPS: `TLSv1.3`. + pub upstream_tls_version: Option, + /// `TLS13_AES_128_GCM_SHA256`. + pub upstream_tls_cipher: Option, pub error: Option, /// The client gave up before the response ended, by resetting its HTTP/2 stream or closing /// the connection. Not an error. @@ -244,6 +256,12 @@ impl UpsertModelInfo for Flow { (GrpcStatus, self.grpc_status.into()), (GrpcMessage, self.grpc_message.into()), (Tunnelled, self.tunnelled.into()), + (ElapsedConnectMs, self.elapsed_connect_ms.into()), + (ElapsedTlsMs, self.elapsed_tls_ms.into()), + (ElapsedRequestMs, self.elapsed_request_ms.into()), + (ClientAddr, self.client_addr.into()), + (UpstreamTlsVersion, self.upstream_tls_version.into()), + (UpstreamTlsCipher, self.upstream_tls_cipher.into()), ]) } @@ -281,6 +299,12 @@ impl UpsertModelInfo for Flow { FlowIden::GrpcStatus, FlowIden::GrpcMessage, FlowIden::Tunnelled, + FlowIden::ElapsedConnectMs, + FlowIden::ElapsedTlsMs, + FlowIden::ElapsedRequestMs, + FlowIden::ClientAddr, + FlowIden::UpstreamTlsVersion, + FlowIden::UpstreamTlsCipher, ] } @@ -311,6 +335,12 @@ impl UpsertModelInfo for Flow { response_truncated: r.get("response_truncated")?, elapsed_headers_ms: r.get("elapsed_headers_ms")?, elapsed_ms: r.get("elapsed_ms")?, + elapsed_connect_ms: added_column(r, "elapsed_connect_ms")?, + elapsed_tls_ms: added_column(r, "elapsed_tls_ms")?, + elapsed_request_ms: added_column(r, "elapsed_request_ms")?, + client_addr: added_column(r, "client_addr")?, + upstream_tls_version: added_column(r, "upstream_tls_version")?, + upstream_tls_cipher: added_column(r, "upstream_tls_cipher")?, error: r.get("error")?, cancelled: added_column(r, "cancelled")?.unwrap_or_default(), saved_request_id: added_column(r, "saved_request_id")?, diff --git a/crates-proxy/yaak-proxy-lib/src/settings.rs b/crates-proxy/yaak-proxy-lib/src/settings.rs index 28f7a1363..969d04523 100644 --- a/crates-proxy/yaak-proxy-lib/src/settings.rs +++ b/crates-proxy/yaak-proxy-lib/src/settings.rs @@ -15,6 +15,9 @@ pub struct ProxySettings { pub system_proxy_at_launch: bool, /// Bytes of each request and response body stored per flow. pub capture_body_limit: usize, + /// The most captured traffic kept, in bytes of the databases' used pages; the oldest flows + /// are deleted past it. None keeps everything. + pub keep_traffic_bytes: Option, /// The HTTP versions offered to clients in intercepted HTTPS. pub http_version: HttpVersion, /// The user's decrypt entries, in order: `host:api.example.com`, `app:Google Chrome`, `*`, @@ -31,6 +34,7 @@ impl Default for ProxySettings { port: 9090, system_proxy_at_launch: true, capture_body_limit: DEFAULT_CAPTURE_BODY_LIMIT, + keep_traffic_bytes: None, http_version: HttpVersion::Auto, decrypt: Vec::new(), decrypt_built_ins_removed: Vec::new(), @@ -178,6 +182,7 @@ mod tests { assert_eq!(settings.port, 9090); assert!(settings.system_proxy_at_launch); assert_eq!(settings.capture_body_limit, 10 * 1024 * 1024); + assert_eq!(settings.keep_traffic_bytes, None, "everything is kept"); } #[test] @@ -197,6 +202,7 @@ mod tests { port: 8080, system_proxy_at_launch: false, capture_body_limit: 1024, + keep_traffic_bytes: Some(1 << 30), http_version: HttpVersion::Http1Only, decrypt: vec!["host:api.example.com".into()], decrypt_built_ins_removed: vec!["!host:*.zoom.us".into()], @@ -212,6 +218,7 @@ mod tests { "port": 8080, "systemProxyAtLaunch": false, "captureBodyLimit": 1024, + "keepTrafficBytes": 1073741824, "httpVersion": "http1Only", "decrypt": ["host:api.example.com"], "decryptBuiltInsRemoved": ["!host:*.zoom.us"], diff --git a/crates-proxy/yaak-proxy-lib/src/terminal.rs b/crates-proxy/yaak-proxy-lib/src/terminal.rs new file mode 100644 index 000000000..1b153449e --- /dev/null +++ b/crates-proxy/yaak-proxy-lib/src/terminal.rs @@ -0,0 +1,86 @@ +//! Opens a terminal whose shell has the proxy's environment set, so what runs in it goes through +//! the proxy. macOS's Terminal only for now. + +/// The script Terminal runs: it deletes itself, exports `vars`, then becomes a fresh interactive +/// login shell, the user's own. Fails on a name a shell could not export. +#[cfg_attr(not(target_os = "macos"), allow(dead_code))] +fn script(vars: &[(String, String)]) -> Result { + let mut script = String::from("#!/bin/sh\nrm -f \"$0\"\n"); + for (name, value) in vars { + let valid = name.chars().next().is_some_and(|c| c.is_ascii_alphabetic() || c == '_') + && name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_'); + if !valid { + return Err(format!("{name:?} is not a name an environment variable can have")); + } + script.push_str(&format!("export {name}='{}'\n", value.replace('\'', r"'\''"))); + } + script.push_str("exec \"${SHELL:-/bin/zsh}\" -l\n"); + Ok(script) +} + +/// Opens a Terminal window whose shell starts with `vars` set, through a `.command` script in +/// the temporary directory that deletes itself as it runs. +#[cfg(target_os = "macos")] +pub fn open(vars: &[(String, String)]) -> Result<(), String> { + use std::os::unix::fs::PermissionsExt; + let script = script(vars)?; + let name = format!("yaak-proxy-{}.command", yaak_database::generate_prefixed_id("sh")); + let path = std::env::temp_dir().join(name); + let fail = |e: std::io::Error| format!("Failed to write {}: {e}", path.display()); + std::fs::write(&path, script).map_err(fail)?; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o700)).map_err(fail)?; + let output = std::process::Command::new("/usr/bin/open") + .args(["-a", "Terminal"]) + .arg(&path) + .output() + .map_err(|e| format!("Failed to run open: {e}"))?; + if output.status.success() { + return Ok(()); + } + let _ = std::fs::remove_file(&path); + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + Err(if stderr.is_empty() { format!("open exited with {}", output.status) } else { stderr }) +} + +#[cfg(not(target_os = "macos"))] +pub fn open(_vars: &[(String, String)]) -> Result<(), String> { + Err("Opening a terminal with the proxy set is only supported on macOS so far".to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_script_exports_each_variable_quoted_then_starts_a_login_shell() { + let vars = vec![ + ("HTTPS_PROXY".to_string(), "http://127.0.0.1:9090".to_string()), + ("NOTE".to_string(), "it's $HOME".to_string()), + ]; + assert_eq!( + script(&vars).unwrap(), + "#!/bin/sh\nrm -f \"$0\"\nexport HTTPS_PROXY='http://127.0.0.1:9090'\n\ + export NOTE='it'\\''s $HOME'\nexec \"${SHELL:-/bin/zsh}\" -l\n" + ); + for name in ["", "1A", "A B", "A;rm"] { + assert!(script(&[(name.to_string(), "x".to_string())]).is_err(), "{name:?}"); + } + } + + #[test] + fn the_script_runs_as_a_shell_sets_them() { + let vars = vec![("YAAK_TEST".to_string(), "a 'b' $c".to_string())]; + // Without deleting itself, as `$0` is no script here, or starting a shell. + let script = script(&vars) + .unwrap() + .replace("rm -f \"$0\"\n", "") + .replace("exec \"${SHELL:-/bin/zsh}\" -l\n", ""); + let output = std::process::Command::new("sh") + .args(["-c", &format!("{script}printf %s \"$YAAK_TEST\"")]) + .output(); + // Only where there is a POSIX shell. + if let Ok(output) = output { + assert_eq!(String::from_utf8_lossy(&output.stdout), "a 'b' $c"); + } + } +} diff --git a/crates-proxy/yaak-proxy-lib/tests/flow.rs b/crates-proxy/yaak-proxy-lib/tests/flow.rs index ceb04d904..612183589 100644 --- a/crates-proxy/yaak-proxy-lib/tests/flow.rs +++ b/crates-proxy/yaak-proxy-lib/tests/flow.rs @@ -336,6 +336,8 @@ async fn body_rpcs() { "recording": true, "systemProxyAtLaunch": false, "httpVersion": "auto", + "captureBodyLimit": CAPTURE_LIMIT, + "keepTrafficBytes": null, "startError": null, "encrypting": null, "locked": null, diff --git a/crates-tauri/yaak-app-proxy/capabilities/default.json b/crates-tauri/yaak-app-proxy/capabilities/default.json index f6b586f42..9e821f2ae 100644 --- a/crates-tauri/yaak-app-proxy/capabilities/default.json +++ b/crates-tauri/yaak-app-proxy/capabilities/default.json @@ -7,6 +7,7 @@ "dialog:allow-save", "opener:allow-default-urls", "opener:allow-open-url", + "opener:allow-reveal-item-in-dir", "os:allow-os-type", "core:window:allow-close", "core:window:allow-is-fullscreen", diff --git a/crates-tauri/yaak-app-proxy/src/lib.rs b/crates-tauri/yaak-app-proxy/src/lib.rs index b5e7d52df..dc5360ce1 100644 --- a/crates-tauri/yaak-app-proxy/src/lib.rs +++ b/crates-tauri/yaak-app-proxy/src/lib.rs @@ -177,4 +177,21 @@ mod tests { ); assert_eq!(target("../yaak-app-client/vendored/protoc/include"), Some(PROTOC_INCLUDE_DIR)); } + + /// The window may call each opener command the app's settings use: without its permission, + /// Tauri refuses the call and the button does nothing. + #[test] + fn the_window_may_open_links_and_reveal_folders() { + let capability: serde_json::Value = + serde_json::from_str(include_str!("../capabilities/default.json")).unwrap(); + let permissions: Vec<&str> = capability["permissions"] + .as_array() + .unwrap() + .iter() + .filter_map(|p| p.as_str()) + .collect(); + for needed in ["opener:allow-open-url", "opener:allow-reveal-item-in-dir"] { + assert!(permissions.contains(&needed), "{needed} missing from {permissions:?}"); + } + } } diff --git a/crates/yaak-proxy/src/body.rs b/crates/yaak-proxy/src/body.rs index 2630cf5d6..0c490fd43 100644 --- a/crates/yaak-proxy/src/body.rs +++ b/crates/yaak-proxy/src/body.rs @@ -56,6 +56,8 @@ pub(crate) struct CaptureBody { id: u64, start: Instant, engine: Arc, + /// The capture limit as the body started, so a change applies to bodies after it. + limit: usize, captured: usize, size: u64, finished: bool, @@ -77,6 +79,7 @@ impl CaptureBody { direction, id, start, + limit: engine.capture_body_limit(), engine, captured: 0, size: 0, @@ -122,16 +125,20 @@ impl CaptureBody { self.emit_messages(messages); } let truncated = self.size > self.captured as u64; + let elapsed_ms = self.start.elapsed().as_millis() as u64; let event = match self.direction { Direction::Request if self.size == 0 => return, - Direction::Request => { - ProxyEvent::RequestBodyComplete { id: self.id, size: self.size, truncated } - } + Direction::Request => ProxyEvent::RequestBodyComplete { + id: self.id, + size: self.size, + truncated, + elapsed_ms, + }, Direction::Response => ProxyEvent::ResponseBodyComplete { id: self.id, size: self.size, truncated, - elapsed_ms: self.start.elapsed().as_millis() as u64, + elapsed_ms, error, cancelled, }, @@ -153,7 +160,7 @@ impl Body for CaptureBody { Poll::Ready(Some(Ok(frame))) => { if let Some(frame_data) = frame.data_ref() { let len = frame_data.len(); - let room = self.engine.capture_body_limit.saturating_sub(self.captured); + let room = self.limit.saturating_sub(self.captured); let data = frame_data.slice(..len.min(room)); self.captured += data.len(); self.size += len as u64; diff --git a/crates/yaak-proxy/src/lib.rs b/crates/yaak-proxy/src/lib.rs index 54d7447b3..763f8391d 100644 --- a/crates/yaak-proxy/src/lib.rs +++ b/crates/yaak-proxy/src/lib.rs @@ -16,7 +16,7 @@ use std::collections::hash_map::RandomState; use std::collections::{HashMap, HashSet}; use std::hash::{BuildHasher, Hasher}; use std::net::SocketAddr; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::mpsc as std_mpsc; use std::sync::{Arc, Mutex, RwLock}; use std::time::SystemTime; @@ -139,6 +139,21 @@ pub enum ProxyEvent { size: u64, /// Whether `size` exceeded `capture_body_limit`. truncated: bool, + /// Milliseconds from `started_at` to the body's end. + elapsed_ms: u64, + }, + /// The connection to the server an intercepted HTTPS request goes on, before it is sent. + /// Plain HTTP shares pooled connections the engine does not see opened, so it has none. + UpstreamConnection { + id: u64, + /// Milliseconds from `started_at` to the TCP connection, and to the TLS handshake's + /// end. None when the connection was opened for an earlier request and reused. + elapsed_connect_ms: Option, + elapsed_tls_ms: Option, + /// `TLSv1.3`, `TLSv1.2`. + tls_version: Option, + /// `TLS13_AES_128_GCM_SHA256`. + tls_cipher: Option, }, /// Response headers arrived from upstream. ResponseStart { @@ -250,6 +265,11 @@ impl ProxyHandle { self.engine.http2.store(enabled, Ordering::Relaxed); } + /// Bytes of each body captured, for bodies that start from now on. + pub fn set_capture_body_limit(&self, limit: usize) { + self.engine.capture_body_limit.store(limit, Ordering::Relaxed); + } + /// Decides which HTTPS connections are decrypted from the next CONNECT on. Connections /// already open stay decrypted or tunnelled as they were. pub fn set_decrypt_policy(&self, policy: DecryptPolicy) { @@ -285,7 +305,7 @@ pub(crate) struct ClientConn { pub(crate) struct Engine { ca: Arc, - capture_body_limit: usize, + capture_body_limit: AtomicUsize, events: std_mpsc::Sender, next_id: AtomicU64, next_connection: AtomicU64, @@ -309,6 +329,11 @@ impl Engine { let _ = self.events.send(event); } + /// Bytes of each body captured, as it is when the body starts. + fn capture_body_limit(&self) -> usize { + self.capture_body_limit.load(Ordering::Relaxed) + } + fn next_flow_id(&self) -> u64 { self.next_id.fetch_add(1, Ordering::Relaxed) } @@ -408,7 +433,7 @@ pub fn start_proxy(config: ProxyConfig) -> Result { let (event_tx, event_rx) = std_mpsc::channel(); let engine = Arc::new(Engine { ca: config.ca, - capture_body_limit: config.capture_body_limit, + capture_body_limit: AtomicUsize::new(config.capture_body_limit), events: event_tx, next_id: AtomicU64::new(1), next_connection: AtomicU64::new(1), diff --git a/crates/yaak-proxy/src/request.rs b/crates/yaak-proxy/src/request.rs index f2883e2e1..a09f83c11 100644 --- a/crates/yaak-proxy/src/request.rs +++ b/crates/yaak-proxy/src/request.rs @@ -150,7 +150,7 @@ impl Flow { fn capture_request(&self, body: Incoming) -> BoxBody { let parser = self.grpc.as_ref().map(|call| { - let limit = self.engine.capture_body_limit; + let limit = self.engine.capture_body_limit(); let encoding = call.encoding.as_deref(); let parser = GrpcParser::new(MessageDirection::Client, encoding, call.text, limit) .with_content_encoding(call.content_encoding.as_deref()); @@ -165,7 +165,7 @@ impl Flow { /// How the response's body is read for messages, if it streams them, after recording that /// it does. fn response_parser(&self, resp: &Response) -> Option { - let limit = self.engine.capture_body_limit; + let limit = self.engine.capture_body_limit(); let headers = resp.headers(); if self.grpc.is_some() { // A call that fails at once answers with its status in the headers, and no body. @@ -189,6 +189,21 @@ impl Flow { Some(BodyParser::Sse(Box::new(SseParser::new(encoding, limit)))) } + /// Records the connection the request goes on: its timings when it was opened for this + /// request, and its TLS either way. + fn goes_on(&self, conn: &UpstreamConn, reused: bool) { + let since = |at: Instant| { + (!reused).then(|| at.saturating_duration_since(self.start).as_millis() as u64) + }; + self.engine.emit(ProxyEvent::UpstreamConnection { + id: self.id, + elapsed_connect_ms: since(conn.tls.connected), + elapsed_tls_ms: since(conn.tls.handshaken), + tls_version: conn.tls.version.clone(), + tls_cipher: conn.tls.cipher.clone(), + }); + } + fn respond_head(&mut self, resp: &Response, remote_addr: Option) { self.answered = true; self.engine.emit(ProxyEvent::ResponseStart { @@ -576,9 +591,36 @@ enum Upstream { struct UpstreamConn { sender: Upstream, remote_addr: Option, + tls: TlsInfo, +} + +/// When a connection to the server was made, and the TLS it negotiated. +#[derive(Debug, Clone)] +struct TlsInfo { + connected: Instant, + handshaken: Instant, + version: Option, + cipher: Option, +} + +impl TlsInfo { + fn of(conn: &rustls::ClientConnection, connected: Instant) -> Self { + let version = conn.protocol_version().map(|v| match v { + rustls::ProtocolVersion::TLSv1_3 => "TLSv1.3".to_string(), + rustls::ProtocolVersion::TLSv1_2 => "TLSv1.2".to_string(), + other => format!("{other:?}"), + }); + let cipher = conn.negotiated_cipher_suite().map(|s| format!("{:?}", s.suite())); + Self { connected, handshaken: Instant::now(), version, cipher } + } } impl UpstreamConn { + /// Shared by another request, as an HTTP/2 connection is. + fn share(&self, sender: Upstream) -> Self { + Self { sender, remote_addr: self.remote_addr, tls: self.tls.clone() } + } + fn is_h2(&self) -> bool { matches!(self.sender, Upstream::H2(_)) } @@ -636,10 +678,7 @@ impl Upstreams { conns.retain(|c| !c.is_closed()); if let Some(i) = conns.iter().position(UpstreamConn::is_ready) { let conn = match &conns[i].sender { - Upstream::H2(s) => UpstreamConn { - sender: Upstream::H2(s.clone()), - remote_addr: conns[i].remote_addr, - }, + Upstream::H2(s) => conns[i].share(Upstream::H2(s.clone())), Upstream::H1(_) => conns.swap_remove(i), }; return Ok((conn, true)); @@ -650,10 +689,7 @@ impl Upstreams { } let conn = connect_upstream(engine, &self.target, false).await?; match &conn.sender { - Upstream::H2(s) => conns.push(UpstreamConn { - sender: Upstream::H2(s.clone()), - remote_addr: conn.remote_addr, - }), + Upstream::H2(s) => conns.push(conn.share(Upstream::H2(s.clone()))), Upstream::H1(_) => self.h1.store(true, Ordering::Relaxed), } Ok((conn, false)) @@ -682,10 +718,12 @@ async fn connect_upstream( h1_only: bool, ) -> Result { let tcp = TcpStream::connect(&target.addr).await?; + let connected = Instant::now(); let remote_addr = tcp.peer_addr().ok(); let config = if h1_only { &engine.upstream_tls_h1 } else { &engine.upstream_tls }; let server_name = ServerName::try_from(target.server_name.clone())?; let tls = TlsConnector::from(config.clone()).connect(server_name, tcp).await?; + let tls_info = TlsInfo::of(tls.get_ref().1, connected); let is_h2 = tls.get_ref().1.alpn_protocol() == Some(b"h2"); let io = TokioIo::new(tls); @@ -711,7 +749,7 @@ async fn connect_upstream( }); Upstream::H1(sender) }; - Ok(UpstreamConn { sender: upstream, remote_addr }) + Ok(UpstreamConn { sender: upstream, remote_addr, tls: tls_info }) } /// Addresses a request to the server in the protocol of `conn`. It may be addressed for the @@ -790,6 +828,7 @@ async fn forward_https( Ok(taken) => taken, Err(e) => return Ok(flow.fail(error_chain(e.as_ref()))), }; + flow.goes_on(&conn, reused); let (mut parts, body) = req.into_parts(); if let Err(e) = address(&mut parts, &conn, target, &path, !body.is_end_stream()) { @@ -813,6 +852,7 @@ async fn forward_https( Err(e) => return Ok(flow.fail(error_chain(e.as_ref()))), }; reused = false; + flow.goes_on(&conn, reused); let (mut parts, body) = unsent.into_parts(); if let Err(e) = address(&mut parts, &conn, target, &path, !body.is_end_stream()) { return Ok(flow.fail(e)); diff --git a/crates/yaak-proxy/src/ws.rs b/crates/yaak-proxy/src/ws.rs index dc5753908..c9b7c327a 100644 --- a/crates/yaak-proxy/src/ws.rs +++ b/crates/yaak-proxy/src/ws.rs @@ -120,7 +120,7 @@ pub(crate) async fn relay( }; let (client_read, client_write) = tokio::io::split(TokioIo::new(client)); let (upstream_read, upstream_write) = tokio::io::split(TokioIo::new(upstream)); - let limit = flow.engine.capture_body_limit; + let limit = flow.engine.capture_body_limit(); let outbound = FrameParser::new( MessageDirection::Client, flow.clone(), diff --git a/crates/yaak-proxy/tests/proxy.rs b/crates/yaak-proxy/tests/proxy.rs index f3b32bc1d..ff6566a20 100644 --- a/crates/yaak-proxy/tests/proxy.rs +++ b/crates/yaak-proxy/tests/proxy.rs @@ -415,6 +415,7 @@ fn flow_id(event: &ProxyEvent) -> u64 { | ProxyEvent::RequestHeader { id, .. } | ProxyEvent::RequestBodyChunk { id, .. } | ProxyEvent::RequestBodyComplete { id, .. } + | ProxyEvent::UpstreamConnection { id, .. } | ProxyEvent::ResponseStart { id, .. } | ProxyEvent::ResponseHeader { id, .. } | ProxyEvent::ResponseBodyChunk { id, .. } @@ -1710,6 +1711,29 @@ async fn h2_client_to_an_h2_upstream() { (*size, error.clone(), *cancelled)); assert_eq!((size, error, cancelled), (text.len() as u64, None, false)); assert_eq!(body(&events, false).0, text); + + // The connection opened for it, timed from the flow's start, and its TLS. + let (connect, tls, version, cipher) = find!(events, + ProxyEvent::UpstreamConnection { elapsed_connect_ms, elapsed_tls_ms, tls_version, tls_cipher, .. } => + (*elapsed_connect_ms, *elapsed_tls_ms, tls_version.clone(), tls_cipher.clone())); + let headers_at = find!(events, ProxyEvent::ResponseStart { elapsed_ms, .. } => *elapsed_ms); + assert!(connect.unwrap() <= tls.unwrap() && tls.unwrap() <= headers_at); + assert_eq!(version.as_deref(), Some("TLSv1.3")); + assert!(cipher.unwrap().starts_with("TLS13_")); + + // The next request shares it: no connect or handshake of its own, the same TLS. + let resp = client.send_request(h2_request("POST", port, "/echo", full("again"))).await; + resp.unwrap().into_body().collect().await.unwrap(); + let events = h.flow_events(); + let (connect, tls, version) = find!(events, + ProxyEvent::UpstreamConnection { elapsed_connect_ms, elapsed_tls_ms, tls_version, .. } => + (*elapsed_connect_ms, *elapsed_tls_ms, tls_version.clone())); + assert_eq!((connect, tls, version.as_deref()), (None, None, Some("TLSv1.3"))); + let (sent_at, end_at) = ( + find!(events, ProxyEvent::RequestBodyComplete { elapsed_ms, .. } => *elapsed_ms), + find!(events, ProxyEvent::ResponseBodyComplete { elapsed_ms, .. } => *elapsed_ms), + ); + assert!(sent_at <= end_at, "{sent_at} {end_at}"); } #[tokio::test(flavor = "multi_thread")] diff --git a/packages/ui/src/components/events/AutoScroller.tsx b/packages/ui/src/components/events/AutoScroller.tsx index 7cf61d18e..8f335142d 100644 --- a/packages/ui/src/components/events/AutoScroller.tsx +++ b/packages/ui/src/components/events/AutoScroller.tsx @@ -116,7 +116,7 @@ export function AutoScroller({ className={classNames( "absolute bottom-3 left-1/2 -translate-x-1/2 z-10", "flex items-center gap-1.5 h-6 px-3 rounded-full", - "bg-surface border border-border shadow-lg text-text text-xs", + "bg-surface border border-border shadow-lg text-text text-xs tabular-nums", "hover:bg-surface-highlight", )} > diff --git a/packages/ui/src/components/http/HttpRequestPaneView.tsx b/packages/ui/src/components/http/HttpRequestPaneView.tsx index 3402eea16..56a6fd619 100644 --- a/packages/ui/src/components/http/HttpRequestPaneView.tsx +++ b/packages/ui/src/components/http/HttpRequestPaneView.tsx @@ -106,6 +106,18 @@ export interface HttpRequestPaneViewProps { /** Tabs only the app has, after the shared ones */ extraTabs?: ExtraPaneTab[]; + + /** Replaces the URL bar, as the proxy's card shows a captured request's method and URL */ + urlLine?: ReactNode; + /** Tabs with nothing in them read in the subtlest colour */ + dimEmptyTabs?: boolean; + /** + * The tab to open on for a request with no tab of its own; with it, a tab chosen for one + * request doesn't carry to the next + */ + defaultTab?: string; + /** A tab was chosen */ + onTabChange?: (tab: string) => void; } type HeadersEditorInherited = { name: string; value: string; enabled?: boolean; id?: string }[]; @@ -153,6 +165,10 @@ export function HttpRequestPaneView({ authTabs, authContent, extraTabs, + urlLine, + dimEmptyTabs = false, + defaultTab, + onTabChange, }: HttpRequestPaneViewProps) { const { ErrorBoundary } = useEditorHost(); const readOnly = !editable; @@ -168,27 +184,27 @@ export function HttpRequestPaneView({ : 0; } + const headerCount = request.headers.filter((h) => h.name && h.enabled !== false).length; const tabs = useMemo( () => [ bodyTab ?? { value: REQUEST_TAB_BODY, label: bodyTypeLabel(request.bodyType, request.contentType), + empty: dimEmptyTabs && request.bodyType == null, rightSlot: numParams > 0 ? : null, }, { value: REQUEST_TAB_PARAMS, rightSlot: , label: "Params", + empty: dimEmptyTabs && paramPairs.length === 0, }, ...(headersTabs ?? [ { value: REQUEST_TAB_HEADERS, label: "Headers", - rightSlot: ( - h.name && h.enabled !== false).length} - /> - ), + empty: dimEmptyTabs && headerCount === 0, + rightSlot: , }, ]), ...(authTabs ?? @@ -205,14 +221,15 @@ export function HttpRequestPaneView({ [ authTabs, bodyTab, + dimEmptyTabs, extraTabs, + headerCount, headersTabs, numParams, paramPairs.length, request.auth, request.bodyType, request.contentType, - request.headers, ], ); @@ -235,6 +252,7 @@ export function HttpRequestPaneView({ style={style} className={classNames(className, "h-full grid grid-rows-[auto_minmax(0,1fr)] grid-cols-1")} > + {urlLine ?? ( + )} {authContent ?? (request.auth != null && )} diff --git a/packages/ui/src/components/http/HttpResponsePaneView.tsx b/packages/ui/src/components/http/HttpResponsePaneView.tsx index 2549182df..7d0d10537 100644 --- a/packages/ui/src/components/http/HttpResponsePaneView.tsx +++ b/packages/ui/src/components/http/HttpResponsePaneView.tsx @@ -69,6 +69,22 @@ export interface HttpResponsePaneViewProps { infoRows?: ReactElement[]; /** The copy button for a body held back for its size */ renderCopyButton?: (getText: () => Promise) => ReactNode; + /** + * The request is shown apart, as the proxy's card shows it beside the response: no Request + * tab, and Headers and Cookies hold the response's alone + */ + requestShownApart?: boolean; + /** Drawn inside a card the app draws, without a card of its own */ + bare?: boolean; + /** Tabs with nothing in them read in the subtlest colour */ + dimEmptyTabs?: boolean; + /** + * The tab to open on for a key with no tab of its own; with it, a tab chosen under one key + * doesn't carry to the next + */ + defaultTab?: string; + /** A tab was chosen */ + onTabChange?: (tab: string) => void; } /** Yaak's response pane: a status line over Response, Request, Headers and Cookies tabs */ @@ -95,6 +111,11 @@ export function HttpResponsePaneView({ renderBodyViewer, infoRows, renderCopyButton, + requestShownApart = false, + bare = false, + dimEmptyTabs = false, + defaultTab, + onTabChange, }: HttpResponsePaneViewProps) { const { ErrorBoundary } = useEditorHost(); const mimeType = @@ -138,9 +159,15 @@ export function HttpResponsePaneView({ : []), ]; + const noBody = response == null || (response.contentLength ?? 0) === 0; + const headerCount = response?.headers.length ?? 0; + const cookieCount = requestShownApart + ? cookieCounts.received + : cookieCounts.sent + cookieCounts.received; return [ { value: RESPONSE_TAB_BODY, + empty: dimEmptyTabs && noBody, options: { value: viewMode, onChange: (v) => onViewModeChange(v === "raw" ? "raw" : "pretty"), @@ -153,40 +180,48 @@ export function HttpResponsePaneView({ itemsAfter: itemsAfter.length > 0 ? itemsAfter : undefined, }, }, - { - value: RESPONSE_TAB_REQUEST, - label: "Request", - rightSlot: (response?.requestContentLength ?? 0) > 0 ? : null, - }, + ...(requestShownApart + ? [] + : [ + { + value: RESPONSE_TAB_REQUEST, + label: "Request", + rightSlot: + (response?.requestContentLength ?? 0) > 0 ? : null, + }, + ]), { value: RESPONSE_TAB_HEADERS, label: "Headers", - rightSlot: ( - + empty: dimEmptyTabs && headerCount === 0, + rightSlot: requestShownApart ? ( + + ) : ( + ), }, { value: RESPONSE_TAB_COOKIES, label: "Cookies", - rightSlot: - cookieCounts.sent > 0 || cookieCounts.received > 0 ? ( - - ) : null, + empty: dimEmptyTabs && cookieCount === 0, + rightSlot: requestShownApart ? ( + + ) : cookieCounts.sent > 0 || cookieCounts.received > 0 ? ( + + ) : null, }, ...(extraTabs?.map((t) => t.tab) ?? []), ]; }, [ cookieCounts, + dimEmptyTabs, extraTabs, mimeType, onCopyAsCurl, onCopyBody, onSaveBody, onViewModeChange, + requestShownApart, response, viewMode, ]); @@ -197,9 +232,8 @@ export function HttpResponsePaneView({ className={classNames( className, "x-theme-responsePane", - "max-h-full h-full", - "bg-surface rounded-md border border-border-subtle overflow-hidden", - "relative", + "max-h-full h-full overflow-hidden relative", + !bare && "bg-surface rounded-md border border-border-subtle", )} > {response == null ? ( @@ -234,6 +268,9 @@ export function HttpResponsePaneView({ tabListClassName="mt-0.5 -mb-1.5" storageKey={tabsStorageKey} activeTabKey={activeTabKey} + defaultValue={defaultTab} + keyedDefault={defaultTab != null} + onChangeValue={onTabChange} > @@ -259,6 +296,7 @@ export function HttpResponsePaneView({ + {!requestShownApart && ( + )} - + - + {extraTabs?.map(({ tab, content }) => ( diff --git a/packages/ui/src/components/http/ResponseCookies.tsx b/packages/ui/src/components/http/ResponseCookies.tsx index 61649b941..29801ae9e 100644 --- a/packages/ui/src/components/http/ResponseCookies.tsx +++ b/packages/ui/src/components/http/ResponseCookies.tsx @@ -10,6 +10,8 @@ import type { HttpResponseView } from "./types"; interface Props { response: HttpResponseView; + /** Only the cookies sent or only those received, for a pane that shows one side */ + side?: "sent" | "received"; } interface ParsedCookie { @@ -102,7 +104,7 @@ export function parseCookieHeaders({ sent, received }: HttpResponseView["cookieH }; } -export function ResponseCookies({ response }: Props) { +export function ResponseCookies({ response, side }: Props) { const { useDetailsBannerStoredValue } = useEditorHost(); const { sentCookies, receivedCookies } = useMemo( () => parseCookieHeaders(response.cookieHeaders), @@ -111,6 +113,7 @@ export function ResponseCookies({ response }: Props) { return (
+ {side !== "received" && ( )} + )} + {side !== "sent" && ( )} + )}
); } diff --git a/packages/ui/src/components/http/ResponseHeaders.tsx b/packages/ui/src/components/http/ResponseHeaders.tsx index 3206152de..6f7c050c6 100644 --- a/packages/ui/src/components/http/ResponseHeaders.tsx +++ b/packages/ui/src/components/http/ResponseHeaders.tsx @@ -2,6 +2,7 @@ import { format, formatDistanceToNowStrict } from "date-fns"; import type { ReactElement } from "react"; import { DetailsBanner } from "../DetailsBanner"; import { useEditorHost } from "../EditorHost"; +import { EmptyStateText } from "../EmptyStateText"; import { HeadersTable } from "../HeadersTable"; import { IconButton } from "../IconButton"; import type { KeyValueRowProps } from "../KeyValueRow"; @@ -12,10 +13,30 @@ interface Props { response: HttpResponseView; /** More rows for the Info section, after the standard ones, each keyed */ infoRows?: ReactElement[]; + /** Only the response's headers, for a pane whose request and info are shown elsewhere */ + responseOnly?: boolean; } -export function ResponseHeaders({ response, infoRows }: Props) { +export function ResponseHeaders({ response, infoRows, responseOnly }: Props) { const { openUrl, useDetailsBannerStoredValue } = useEditorHost(); + if (responseOnly) { + return ( +
+ {response.headers.length === 0 ? ( + No Headers + ) : ( + + {response.headers.map((h, i) => ( + // oxlint-disable-next-line react/no-array-index-key + + {h.value} + + ))} + + )} +
+ ); + } return (
!view.hiddenInterfaceRows?.includes(row); return ( @@ -125,43 +131,55 @@ export function SettingsInterface({ - - callbacks.onChange({ editorKeymap })} - options={keymaps} - /> - callbacks.onChange({ editorSoftWrap })} - /> - callbacks.onChange({ coloredMethods })} - /> - + {(shown("editorKeymap") || shown("editorSoftWrap") || shown("coloredMethods")) && ( + + {shown("editorKeymap") && ( + callbacks.onChange({ editorKeymap })} + options={keymaps} + /> + )} + {shown("editorSoftWrap") && ( + callbacks.onChange({ editorSoftWrap })} + /> + )} + {shown("coloredMethods") && ( + callbacks.onChange({ coloredMethods })} + /> + )} + + )} - - - {view.windowControlsSetting && ( - callbacks.onChange({ hideWindowControls })} - /> - )} - + {(shown("nativeTitlebar") || view.windowControlsSetting) && ( + + {shown("nativeTitlebar") && ( + + )} + {view.windowControlsSetting && ( + callbacks.onChange({ hideWindowControls })} + /> + )} + + )} {trailingSections} diff --git a/packages/ui/src/components/settings/types.ts b/packages/ui/src/components/settings/types.ts index ea0355b39..72e71aa6a 100644 --- a/packages/ui/src/components/settings/types.ts +++ b/packages/ui/src/components/settings/types.ts @@ -106,8 +106,17 @@ export interface SettingsView { feedbackPrompts: boolean; /** Null when this build has no license */ license: SettingsLicenseView | null; + /** Interface rows this app doesn't act on yet, left out of the tab */ + hiddenInterfaceRows?: SettingsInterfaceRow[]; } +/** The Interface tab's rows an app may leave out */ +export type SettingsInterfaceRow = + | "editorKeymap" + | "editorSoftWrap" + | "coloredMethods" + | "nativeTitlebar"; + export interface SettingsCallbacks { onChange: (patch: SettingsPatch) => void | Promise; /** Save the native title bar choice and restart, since it only applies at launch */ diff --git a/packages/ui/src/components/tabs/Tabs.tsx b/packages/ui/src/components/tabs/Tabs.tsx index 8dd9a24bf..541117c8b 100644 --- a/packages/ui/src/components/tabs/Tabs.tsx +++ b/packages/ui/src/components/tabs/Tabs.tsx @@ -41,6 +41,8 @@ export type TabItem = value: string; label: string; hidden?: boolean; + /** Nothing in it: the label reads in the subtlest colour */ + empty?: boolean; leftSlot?: ReactNode; rightSlot?: ReactNode; } @@ -57,6 +59,8 @@ export type TabItem = export interface TabOptionsItem { value: string; options: Omit; + /** Nothing in it: the label reads in the subtlest colour */ + empty?: boolean; leftSlot?: ReactNode; rightSlot?: ReactNode; } @@ -89,6 +93,11 @@ export interface TabsProps { activeTabKey?: string; /** Where the order and active tab under `storageKey` are kept */ useStoredValue?: UseStoredValue; + /** + * A key with no stored tab opens on `defaultValue`, not on the tab last chosen under another + * key, and a stored tab no longer offered falls back the same way + */ + keyedDefault?: boolean; } export const Tabs = forwardRef(function Tabs( @@ -105,6 +114,7 @@ export const Tabs = forwardRef(function Tabs( storageKey, activeTabKey, useStoredValue: useStoredValueProp, + keyedDefault = false, }: TabsProps, forwardedRef: Ref, ) { @@ -131,8 +141,12 @@ export const Tabs = forwardRef(function Tabs( // Get the active tab value - prefer storage (if activeTabKey), then defaultValue, then first tab const storedActiveTab = activeTabKey ? storage?.activeTabs?.[activeTabKey] : undefined; - const [internalValue, setInternalValue] = useState(undefined); - const value = storedActiveTab ?? internalValue ?? defaultValue ?? originalTabs[0]?.value; + const [internal, setInternal] = useState<{ key?: string; value: string } | undefined>(undefined); + const internalValue = + keyedDefault && internal?.key !== activeTabKey ? undefined : internal?.value; + const offered = (v: string | undefined) => + v != null && (!keyedDefault || originalTabs.some((t) => t.value === v)); + const value = [storedActiveTab, internalValue, defaultValue].find(offered) ?? originalTabs[0]?.value; // Helper to normalize storage (handle migration from old format) const normalizeStorage = useCallback( @@ -144,7 +158,7 @@ export const Tabs = forwardRef(function Tabs( // Handle tab change - update internal state, storage if we have a key, and call prop callback const onChangeValue = useCallback( async (newValue: string) => { - setInternalValue(newValue); + setInternal({ key: activeTabKey, value: newValue }); if (storageKey && activeTabKey) { await setStorage((s) => { const normalized = normalizeStorage(s); @@ -501,7 +515,15 @@ function TabButton({ ) : "options" in tab ? ( ) : ( - ); @@ -574,6 +596,7 @@ function TabOptionsButton({
} {...buttonProps} + className={classNames(buttonProps.className, !isActive && tab.empty && "text-text-subtlest!")} > {option && "shortLabel" in option && option.shortLabel ? option.shortLabel From 7453869ac61491506f9fe0e8613bf4d917197ef7 Mon Sep 17 00:00:00 2001 From: Gregory Schier Date: Sat, 7 Nov 2026 23:30:56 -0700 Subject: [PATCH 10/45] Move Yaak's sidebar filter onto a shared Rust query engine (#825) Co-authored-by: Claude Opus 5.5 --- Cargo.lock | 11 + Cargo.toml | 2 + apps/yaak-client/components/Sidebar.tsx | 101 +- .../core/Editor/filter/extension.ts | 88 ++ .../core/Editor/filter/format.test.ts | 43 - .../components/core/Editor/filter/format.ts | 7 - .../components/core/Editor/filter/query.ts | 308 ----- crates/yaak-query/Cargo.toml | 37 + crates/yaak-query/bindings/query.ts | 128 ++ crates/yaak-query/build-wasm.cjs | 40 + crates/yaak-query/index.ts | 85 ++ crates/yaak-query/package.json | 15 + crates/yaak-query/pkg/package.json | 17 + crates/yaak-query/pkg/yaak_query.d.ts | 49 + crates/yaak-query/pkg/yaak_query.js | 6 + crates/yaak-query/pkg/yaak_query_bg.js | 609 ++++++++++ crates/yaak-query/pkg/yaak_query_bg.wasm | Bin 0 -> 340382 bytes crates/yaak-query/pkg/yaak_query_bg.wasm.d.ts | 21 + crates/yaak-query/src/ast.rs | 195 +++ crates/yaak-query/src/complete.rs | 378 ++++++ crates/yaak-query/src/eval.rs | 1055 +++++++++++++++++ crates/yaak-query/src/format.rs | 89 ++ crates/yaak-query/src/fuzzy.rs | 206 ++++ crates/yaak-query/src/lexer.rs | 320 +++++ crates/yaak-query/src/lib.rs | 40 + crates/yaak-query/src/parser.rs | 393 ++++++ crates/yaak-query/src/schema.rs | 307 +++++ crates/yaak-query/src/tokens.rs | 100 ++ crates/yaak-query/src/wasm.rs | 161 +++ package-lock.json | 12 + package.json | 1 + vite.config.ts | 2 + 32 files changed, 4430 insertions(+), 396 deletions(-) create mode 100644 apps/yaak-client/components/core/Editor/filter/extension.ts delete mode 100644 apps/yaak-client/components/core/Editor/filter/format.test.ts delete mode 100644 apps/yaak-client/components/core/Editor/filter/format.ts delete mode 100644 apps/yaak-client/components/core/Editor/filter/query.ts create mode 100644 crates/yaak-query/Cargo.toml create mode 100644 crates/yaak-query/bindings/query.ts create mode 100644 crates/yaak-query/build-wasm.cjs create mode 100644 crates/yaak-query/index.ts create mode 100644 crates/yaak-query/package.json create mode 100644 crates/yaak-query/pkg/package.json create mode 100644 crates/yaak-query/pkg/yaak_query.d.ts create mode 100644 crates/yaak-query/pkg/yaak_query.js create mode 100644 crates/yaak-query/pkg/yaak_query_bg.js create mode 100644 crates/yaak-query/pkg/yaak_query_bg.wasm create mode 100644 crates/yaak-query/pkg/yaak_query_bg.wasm.d.ts create mode 100644 crates/yaak-query/src/ast.rs create mode 100644 crates/yaak-query/src/complete.rs create mode 100644 crates/yaak-query/src/eval.rs create mode 100644 crates/yaak-query/src/format.rs create mode 100644 crates/yaak-query/src/fuzzy.rs create mode 100644 crates/yaak-query/src/lexer.rs create mode 100644 crates/yaak-query/src/lib.rs create mode 100644 crates/yaak-query/src/parser.rs create mode 100644 crates/yaak-query/src/schema.rs create mode 100644 crates/yaak-query/src/tokens.rs create mode 100644 crates/yaak-query/src/wasm.rs diff --git a/Cargo.lock b/Cargo.lock index d18d8bebb..90c0664ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11969,6 +11969,17 @@ dependencies = [ "yaak-templates", ] +[[package]] +name = "yaak-query" +version = "0.1.0" +dependencies = [ + "serde", + "serde-wasm-bindgen", + "serde_json", + "ts-rs", + "wasm-bindgen", +] + [[package]] name = "yaak-rpc" version = "0.0.0" diff --git a/Cargo.toml b/Cargo.toml index 40557a8ea..5e29d0da6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ members = [ "crates/yaak-mcp", "crates/yaak-models", "crates/yaak-plugins", + "crates/yaak-query", "crates/yaak-sse", "crates/yaak-sync", "crates/yaak-templates", @@ -96,6 +97,7 @@ yaak-lifecycle = { path = "crates/yaak-lifecycle" } yaak-mcp = { path = "crates/yaak-mcp" } yaak-models = { path = "crates/yaak-models" } yaak-plugins = { path = "crates/yaak-plugins" } +yaak-query = { path = "crates/yaak-query" } yaak-sse = { path = "crates/yaak-sse" } yaak-sync = { path = "crates/yaak-sync" } yaak-templates = { path = "crates/yaak-templates" } diff --git a/apps/yaak-client/components/Sidebar.tsx b/apps/yaak-client/components/Sidebar.tsx index 80d474656..cb3044736 100644 --- a/apps/yaak-client/components/Sidebar.tsx +++ b/apps/yaak-client/components/Sidebar.tsx @@ -63,10 +63,9 @@ import { isSidebarFocused } from "../lib/scopes"; import { navigateToRequestOrFolderOrWorkspace } from "../lib/setWorkspaceSearchParams"; import type { ContextMenuProps, DropdownItem } from "./core/Dropdown"; import { ContextMenu, Dropdown } from "./core/Dropdown"; -import { type FieldDef, filter } from "@yaakapp-internal/ui/editor"; -import type { Ast } from "./core/Editor/filter/query"; -import { evaluate, parseQuery } from "./core/Editor/filter/query"; -import { formatFieldFilter } from "./core/Editor/filter/format"; +import type { Node as QueryNode, Schema } from "@yaakapp-internal/query"; +import { formatField, parseQuery, QueryMatcher } from "@yaakapp-internal/query"; +import { queryFilter } from "./core/Editor/filter/extension"; import { HttpMethodTag } from "./core/HttpMethodTag"; import { Icon, @@ -111,7 +110,7 @@ function Sidebar({ className }: { className?: string }) { const activeWorkspaceId = useAtomValue(activeWorkspaceAtom)?.id; const treeId = `tree.${activeWorkspaceId ?? "unknown"}`; const filterText = useAtomValue(sidebarFilterAtom); - const [tree, allFields, emptyFilterSuggestions] = useAtomValue(sidebarTreeAtom) ?? []; + const [tree, fieldValues, emptyFilterSuggestions] = useAtomValue(sidebarTreeAtom) ?? []; const wrapperRef = useRef(null); const treeRef = useRef(null); @@ -552,18 +551,18 @@ function Sidebar({ className }: { className?: string }) { const filterCompartmentMountExtRef = useRef(null); if (filterCompartmentMountExtRef.current == null) { filterCompartmentMountExtRef.current = filterLanguageCompartmentRef.current.of( - filter({ fields: allFields ?? [] }), + queryFilter({ schema: sidebarFilterSchema, values: fieldValues }), ); } useEffect(() => { const view = filterRef.current; if (!view) return; - const ext = filter({ fields: allFields ?? [] }); + const ext = queryFilter({ schema: sidebarFilterSchema, values: fieldValues }); view.dispatch({ effects: filterLanguageCompartmentRef.current.reconfigure(ext), }); - }, [allFields]); + }, [fieldValues]); if (tree == null || hidden) { return null; @@ -851,16 +850,35 @@ function setSidebarFilterText(text: string) { jotaiStore.set(sidebarFilterAtom, { text, key: `${Math.random()}` }); } -function getSidebarSuggestionValue(ast: Ast | null) { +/** The fields `getItemFields` fills in. Bare words match `getItemText` by word prefix */ +const sidebarFilterColumns = [ + "method", + "url", + "name", + "type", + "folder", + "grpc_service", + "grpc_method", +]; +const sidebarFilterSchema: Schema = { + fields: sidebarFilterColumns.map((name) => ({ name, kind: { type: "text" } })), + freeText: "fuzzy", +}; + +function getSidebarSuggestionValue(ast: QueryNode | null, query: string) { if (ast == null) return null; if (ast.type === "Term" || ast.type === "Phrase") { - const value = ast.value.trim(); + const value = ast.text.trim(); return value.length > 0 ? value : null; } if (ast.type === "Field") { - const value = ast.value.trim(); + const raw = + ast.value.type === "text" + ? ast.value.text + : query.slice(ast.valueSpan.start, ast.valueSpan.end); + const value = raw.trim(); return value.length > 0 ? value : null; } @@ -882,7 +900,7 @@ const sidebarSuggestionFieldOrder = [ ]; const sidebarTreeAtom = atom< - [TreeNode, FieldDef[], SidebarFilterSuggestion[]] | null + [TreeNode, Record, SidebarFilterSuggestion[]] | null >((get) => { const allModels = get(memoAllPotentialChildrenAtom); const activeWorkspace = get(activeWorkspaceAtom); @@ -903,15 +921,18 @@ const sidebarTreeAtom = atom< return null; } - const queryAst = parseQuery(filter.text); - const suggestionValue = getSidebarSuggestionValue(queryAst); + const suggestionValue = getSidebarSuggestionValue(parseQuery(filter.text), filter.text); + const matcher = new QueryMatcher(filter.text, sidebarFilterSchema); + + // Every request goes into one flat table (its text, then a cell per column), matched in a + // single call after the tree is built + const leaves: TreeNode[] = []; + const cells: (string | null)[] = []; - // returns true if this node OR any child matches the filter const allFields: Record> = {}; const suggestionFields = new Set(); - const build = (node: TreeNode, depth: number): boolean => { + const build = (node: TreeNode, depth: number) => { const childItems = childrenMap[node.item.id] ?? []; - let matchesSelf = true; const fields = getItemFields(node); const model = node.item.model; const isLeafNode = !(model === "folder" || model === "workspace"); @@ -929,12 +950,12 @@ const sidebarTreeAtom = atom< } } - if (queryAst != null) { - matchesSelf = isLeafNode && evaluate(queryAst, { text: getItemText(node.item), fields }); + if (isLeafNode && !matcher.isEmpty) { + leaves.push(node); + cells.push(getItemText(node.item)); + for (const column of sidebarFilterColumns) cells.push(fields[column] ?? null); } - let matchesChild = false; - // Recurse to children node.children = !isLeafNode ? [] : undefined; @@ -952,19 +973,21 @@ const sidebarTreeAtom = atom< for (const item of childItems) { const childNode = { item, parent: node, depth }; - const childMatches = build(childNode, depth + 1); - if (childMatches) { - matchesChild = true; - } + build(childNode, depth + 1); node.children.push(childNode); } } + }; - // hide node IFF nothing in its subtree matches - const anyMatch = matchesSelf || matchesChild; - node.hidden = !anyMatch; - - return anyMatch; + // Hide a node IFF nothing in its subtree matches; with no query, nothing is hidden + let matched: Set> | null = null; + const markHidden = (node: TreeNode): boolean => { + let visible = matched == null || matched.has(node); + for (const child of node.children ?? []) { + if (markHidden(child)) visible = true; + } + node.hidden = !visible; + return visible; }; const root: TreeNode = { @@ -974,15 +997,17 @@ const sidebarTreeAtom = atom< depth: 0, }; - // Build tree and mark visibility in one pass build(root, 1); + if (!matcher.isEmpty) { + const rows = matcher.matchingRows(sidebarFilterColumns, cells); + matched = new Set(Array.from(rows, (i) => leaves[i]!)); + } + matcher.free(); + markHidden(root); - const fields: FieldDef[] = []; + const fieldValues: Record = {}; for (const [name, values] of Object.entries(allFields)) { - fields.push({ - name, - values: Array.from(values).filter((v) => v.length < 20), - }); + fieldValues[name] = Array.from(values).filter((v) => v.length < 20); } const suggestions = Array.from(suggestionFields) .sort((a, b) => { @@ -993,9 +1018,9 @@ const sidebarTreeAtom = atom< }) .map((field) => ({ field, - filterText: formatFieldFilter(field, suggestionValue ?? ""), + filterText: formatField(field, suggestionValue ?? ""), })); - return [root, fields, suggestions] as const; + return [root, fieldValues, suggestions] as const; }); const sidebarGitStatusByModelIdAtom = atom>((get) => { diff --git a/apps/yaak-client/components/core/Editor/filter/extension.ts b/apps/yaak-client/components/core/Editor/filter/extension.ts new file mode 100644 index 000000000..003729fe5 --- /dev/null +++ b/apps/yaak-client/components/core/Editor/filter/extension.ts @@ -0,0 +1,88 @@ +import type { Completion, CompletionContext, CompletionResult } from "@codemirror/autocomplete"; +import { autocompletion, startCompletion } from "@codemirror/autocomplete"; +import { highlightingFor } from "@codemirror/language"; +import type { EditorState, Extension } from "@codemirror/state"; +import { RangeSetBuilder } from "@codemirror/state"; +import type { DecorationSet, ViewUpdate } from "@codemirror/view"; +import { Decoration, EditorView, ViewPlugin } from "@codemirror/view"; +import type { Tag } from "@lezer/highlight"; +import { tags as t } from "@lezer/highlight"; +import type { Schema, TokenKind } from "@yaakapp-internal/query"; +import { queryCompletions, queryTokens } from "@yaakapp-internal/query"; + +export interface QueryFilterOptions { + schema: Schema; + /** The values seen for each field path, offered after `@field:` */ + values?: Record; +} + +const tokenTags: Record = { + field: t.attributeName, + value: t.attributeValue, + operator: t.operatorKeyword, + quote: t.string, + paren: t.paren, + negation: t.operator, + text: null, +}; + +function buildDecorations(state: EditorState): DecorationSet { + const builder = new RangeSetBuilder(); + for (const token of queryTokens(state.doc.toString())) { + const tag = tokenTags[token.kind]; + const className = tag == null ? null : highlightingFor(state, [tag]); + if (className == null || token.span.start === token.span.end) continue; + builder.add(token.span.start, token.span.end, Decoration.mark({ class: className })); + } + return builder.finish(); +} + +const highlight = ViewPlugin.fromClass( + class { + decorations: DecorationSet; + + constructor(view: EditorView) { + this.decorations = buildDecorations(view.state); + } + + update(update: ViewUpdate) { + if (update.docChanged || update.viewportChanged) { + this.decorations = buildDecorations(update.state); + } + } + }, + { decorations: (v) => v.decorations }, +); + +function completionSource({ schema, values }: QueryFilterOptions) { + return (ctx: CompletionContext): CompletionResult | null => { + const result = queryCompletions(ctx.state.doc.toString(), ctx.pos, schema, values); + if (result == null) return null; + + const { start, end } = result.span; + // Field names pop up for something typed; on an empty spot, only when asked for + const onlyFields = result.items.every((item) => item.kind === "field"); + if (onlyFields && start === end && !ctx.explicit) return null; + + const options: Completion[] = result.items.map((item) => ({ + label: item.label, + type: item.kind === "field" ? "property" : "constant", + info: item.info ?? undefined, + apply: (view, _completion, from, to) => { + view.dispatch({ + changes: { from, to, insert: item.insert }, + selection: { anchor: from + item.insert.length }, + }); + if (item.reopen) startCompletion(view); + }, + })); + + // The engine has already filtered and ranked the options + return { from: start, to: end, options, filter: false }; + }; +} + +/** Highlighting and completion for a search box written in Yaak's query language */ +export function queryFilter(options: QueryFilterOptions): Extension { + return [highlight, autocompletion({ override: [completionSource(options)] })]; +} diff --git a/apps/yaak-client/components/core/Editor/filter/format.test.ts b/apps/yaak-client/components/core/Editor/filter/format.test.ts deleted file mode 100644 index 07ba7afe6..000000000 --- a/apps/yaak-client/components/core/Editor/filter/format.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { describe, expect, test } from "vite-plus/test"; -import { formatFieldFilter } from "./format"; -import { evaluate, parseQuery } from "./query"; - -function matchesFormattedUrl(value: string) { - return evaluate(parseQuery(formatFieldFilter("url", value)), { - fields: { url: value }, - }); -} - -describe("formatFieldFilter", () => { - test("keeps URL-like values bare", () => { - expect(formatFieldFilter("url", "yaak.app/foo-bar")).toBe("@url:yaak.app/foo-bar"); - expect(matchesFormattedUrl("yaak.app/foo-bar")).toBe(true); - }); - - test("keeps non-syntax punctuation bare", () => { - expect(formatFieldFilter("url", "yaa$&#*@tsrna(*)")).toBe("@url:yaa$&#*@tsrna(*)"); - expect(matchesFormattedUrl("yaa$&#*@tsrna(*)")).toBe(true); - }); - - test("keeps values that start with an operator token bare", () => { - expect(formatFieldFilter("url", "-foo")).toBe("@url:-foo"); - expect(matchesFormattedUrl("-foo")).toBe(true); - }); - - test("keeps boolean operator words bare", () => { - expect(formatFieldFilter("url", "AND")).toBe("@url:AND"); - expect(formatFieldFilter("url", "or")).toBe("@url:or"); - expect(formatFieldFilter("url", "Not")).toBe("@url:Not"); - expect(matchesFormattedUrl("AND")).toBe(true); - }); - - test("escapes quoted values", () => { - expect(formatFieldFilter("url", 'say "hi"')).toBe('@url:"say \\"hi\\""'); - expect(matchesFormattedUrl('say "hi"')).toBe(true); - }); - - test("quotes values that start with a quote", () => { - expect(formatFieldFilter("url", '"hi"')).toBe('@url:"\\"hi\\""'); - expect(matchesFormattedUrl('"hi"')).toBe(true); - }); -}); diff --git a/apps/yaak-client/components/core/Editor/filter/format.ts b/apps/yaak-client/components/core/Editor/filter/format.ts deleted file mode 100644 index f1fc7215e..000000000 --- a/apps/yaak-client/components/core/Editor/filter/format.ts +++ /dev/null @@ -1,7 +0,0 @@ -const bareFieldValue = /^[^\s"]\S*$/; - -export function formatFieldFilter(field: string, value: string) { - const escapedValue = value.replace(/\\/g, "\\\\").replace(/"/g, '\\"'); - const filterValue = bareFieldValue.test(value) ? value : `"${escapedValue}"`; - return `@${field}:${filterValue}`; -} diff --git a/apps/yaak-client/components/core/Editor/filter/query.ts b/apps/yaak-client/components/core/Editor/filter/query.ts deleted file mode 100644 index 12bf4a4bb..000000000 --- a/apps/yaak-client/components/core/Editor/filter/query.ts +++ /dev/null @@ -1,308 +0,0 @@ -// query.ts -// A tiny query language parser with NOT/AND/OR, parentheses, phrases, negation, and field:value. - -import { fuzzyMatch } from "fuzzbunny"; -///////////////////////// -// AST -///////////////////////// - -export type Ast = - | { type: "Term"; value: string } // foo - | { type: "Phrase"; value: string } // "hi there" - | { type: "Field"; field: string; value: string } // method:POST or title:"exact phrase" - | { type: "Not"; node: Ast } // -foo or NOT foo - | { type: "And"; left: Ast; right: Ast } // a AND b - | { type: "Or"; left: Ast; right: Ast }; // a OR b - -///////////////////////// -// Tokenizer -///////////////////////// -type Tok = - | { kind: "LPAREN" } - | { kind: "RPAREN" } - | { kind: "AND" } - | { kind: "OR" } - | { kind: "NOT" } // explicit NOT - | { kind: "MINUS" } // unary minus before term/phrase/paren group - | { kind: "COLON" } - | { kind: "WORD"; text: string } // bareword (unquoted) - | { kind: "PHRASE"; text: string } // "quoted phrase" - | { kind: "EOF" }; - -const isSpace = (c: string) => /\s/.test(c); -const isWordStart = (c: string) => c !== "" && !isSpace(c) && !/[():"@-]/.test(c); -const isWordChar = (c: string) => c !== "" && !isSpace(c) && !/[():"@]/.test(c); - -export function tokenize(input: string): Tok[] { - const toks: Tok[] = []; - let i = 0; - const n = input.length; - - const peek = () => input[i] ?? ""; - const advance = () => input[i++]; - - const readWord = () => { - let s = ""; - while (i < n && isWordChar(peek())) s += advance(); - return s; - }; - - const readFieldValue = () => { - let s = ""; - while (i < n && !isSpace(peek())) s += advance(); - return s; - }; - - const readPhrase = () => { - // assumes current char is opening quote - advance(); // consume opening " - let s = ""; - while (i < n) { - const c = advance(); - if (c === `"`) break; - if (c === "\\" && i < n) { - // escape \" and \\ (simple) - const next = advance(); - s += next; - } else { - s += c; - } - } - return s; - }; - - while (i < n) { - const c = peek(); - - if (isSpace(c)) { - i++; - continue; - } - - if (c === "(") { - toks.push({ kind: "LPAREN" }); - i++; - continue; - } - if (c === ")") { - toks.push({ kind: "RPAREN" }); - i++; - continue; - } - if (c === ":") { - toks.push({ kind: "COLON" }); - i++; - if (peek() && !isSpace(peek()) && peek() !== `"`) { - toks.push({ kind: "WORD", text: readFieldValue() }); - } - continue; - } - if (c === `"`) { - const text = readPhrase(); - toks.push({ kind: "PHRASE", text }); - continue; - } - if (c === "-") { - toks.push({ kind: "MINUS" }); - i++; - continue; - } - - // WORD / AND / OR / NOT - if (isWordStart(c)) { - const w = readWord(); - const upper = w.toUpperCase(); - if (upper === "AND") toks.push({ kind: "AND" }); - else if (upper === "OR") toks.push({ kind: "OR" }); - else if (upper === "NOT") toks.push({ kind: "NOT" }); - else toks.push({ kind: "WORD", text: w }); - continue; - } - - // Unknown char—skip to be forgiving - i++; - } - - toks.push({ kind: "EOF" }); - return toks; -} - -class Parser { - private i = 0; - constructor(private toks: Tok[]) {} - - private peek(): Tok { - return this.toks[this.i] ?? { kind: "EOF" }; - } - private advance(): Tok { - return this.toks[this.i++] ?? { kind: "EOF" }; - } - private at(kind: Tok["kind"]) { - return this.peek().kind === kind; - } - - // Top-level: parse OR-precedence chain, allowing implicit AND. - parse(): Ast | null { - if (this.at("EOF")) return null; - const expr = this.parseOr(); - if (!this.at("EOF")) { - // Optionally, consume remaining tokens or throw - } - return expr; - } - - // Precedence: NOT (highest), AND, OR (lowest) - private parseOr(): Ast { - let node = this.parseAnd(); - while (this.at("OR")) { - this.advance(); - const rhs = this.parseAnd(); - node = { type: "Or", left: node, right: rhs }; - } - return node; - } - - private parseAnd(): Ast { - let node = this.parseUnary(); - // Implicit AND: if next token starts a primary, treat as AND. - while (this.at("AND") || this.startsPrimary()) { - if (this.at("AND")) this.advance(); - const rhs = this.parseUnary(); - node = { type: "And", left: node, right: rhs }; - } - return node; - } - - private parseUnary(): Ast { - if (this.at("NOT") || this.at("MINUS")) { - this.advance(); - const node = this.parseUnary(); - return { type: "Not", node }; - } - return this.parsePrimaryOrField(); - } - - private startsPrimary(): boolean { - const k = this.peek().kind; - return k === "WORD" || k === "PHRASE" || k === "LPAREN" || k === "MINUS" || k === "NOT"; - } - - private parsePrimaryOrField(): Ast { - // Parenthesized group - if (this.at("LPAREN")) { - this.advance(); - const inside = this.parseOr(); - // if (!this.at('RPAREN')) throw new Error("Missing closing ')'"); - this.advance(); - return inside; - } - - // Phrase - if (this.at("PHRASE")) { - const t = this.advance() as Extract; - return { type: "Phrase", value: t.text }; - } - - // Field or bare word - if (this.at("WORD")) { - const wordTok = this.advance() as Extract; - - if (this.at("COLON")) { - // field:value or field:"phrase" - this.advance(); // : - let value: string; - if (this.at("PHRASE")) { - const p = this.advance() as Extract; - value = p.text; - } else if (this.at("WORD")) { - const w = this.advance() as Extract; - value = w.text; - } else { - // Anything else after colon is treated literally as a single Term token. - const t = this.advance(); - value = tokText(t); - } - return { type: "Field", field: wordTok.text, value }; - } - - // plain term - return { type: "Term", value: wordTok.text }; - } - - const w = this.advance() as Extract; - return { type: "Phrase", value: "text" in w ? w.text : "" }; - } -} - -function tokText(t: Tok): string { - if ("text" in t) return t.text; - - switch (t.kind) { - case "COLON": - return ":"; - case "LPAREN": - return "("; - case "RPAREN": - return ")"; - default: - return ""; - } -} - -export function parseQuery(q: string): Ast | null { - if (q.trim() === "") return null; - const toks = tokenize(q); - const parser = new Parser(toks); - return parser.parse(); -} - -export type Doc = { - text?: string; - fields?: Record; -}; - -type Technique = "substring" | "fuzzy" | "strict"; - -function includes(hay: string | undefined, needle: string, technique: Technique): boolean { - if (!hay || !needle) return false; - if (technique === "strict") return hay === needle; - if (technique === "fuzzy") return !!fuzzyMatch(hay, needle); - return hay.indexOf(needle) !== -1; -} - -export function evaluate(ast: Ast | null, doc: Doc): boolean { - if (!ast) return true; // Match everything if no query is provided - - const text = (doc.text ?? "").toLowerCase(); - const fieldsNorm: Record = {}; - - for (const [k, v] of Object.entries(doc.fields ?? {})) { - if (!(typeof v === "string" || Array.isArray(v))) continue; - fieldsNorm[k.toLowerCase()] = Array.isArray(v) - ? v.filter((v) => typeof v === "string").map((s) => s.toLowerCase()) - : [String(v ?? "").toLowerCase()]; - } - - const evalNode = (node: Ast): boolean => { - switch (node.type) { - case "Term": - return includes(text, node.value.toLowerCase(), "fuzzy"); - case "Phrase": - // Quoted phrases match exactly - return includes(text, node.value.toLowerCase(), "substring"); - case "Field": { - const vals = fieldsNorm[node.field.toLowerCase()] ?? []; - if (vals.length === 0) return false; - return vals.some((v) => includes(v, node.value.toLowerCase(), "substring")); - } - case "Not": - return !evalNode(node.node); - case "And": - return evalNode(node.left) && evalNode(node.right); - case "Or": - return evalNode(node.left) || evalNode(node.right); - } - }; - - return evalNode(ast); -} diff --git a/crates/yaak-query/Cargo.toml b/crates/yaak-query/Cargo.toml new file mode 100644 index 000000000..cd1813563 --- /dev/null +++ b/crates/yaak-query/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "yaak-query" +version = "0.1.0" +edition = "2024" +publish = false + +# The search query language shared by Yaak's sidebar and Yaak Proxy: grammar, +# parser, schema, evaluator, completion and highlighting. Pure Rust, with a +# WebAssembly build (`@yaakapp-internal/query`) for the frontends. + +[package.metadata.wasm-pack.profile.release] +wasm-opt = false # Matches yaak-templates; wasm-opt has caused errors in CI + +[lib] +crate-type = ["cdylib", "rlib"] + +[features] +default = ["wasm"] +# The `#[wasm_bindgen]` exports that make up the @yaakapp-internal/query +# package. They only exist when compiling for wasm32, so native builds and +# `cargo test` never see them. Off for crates that link this one into their own +# wasm module and do not want these re-exported from theirs. +wasm = [] + +[dependencies] +serde = { workspace = true, features = ["derive"] } + +# Everything that only exists for the WebAssembly package is target-scoped, so +# a native consumer (the proxy, the CLI) links none of it. +[target.'cfg(target_arch = "wasm32")'.dependencies] +serde-wasm-bindgen = "0.6.5" +serde_json = { workspace = true } +wasm-bindgen = { version = "0.2.100", features = ["serde-serialize"] } + +[dev-dependencies] +serde_json = { workspace = true } +ts-rs = { workspace = true } diff --git a/crates/yaak-query/bindings/query.ts b/crates/yaak-query/bindings/query.ts new file mode 100644 index 000000000..a9efad919 --- /dev/null +++ b/crates/yaak-query/bindings/query.ts @@ -0,0 +1,128 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type CompareOp = ">" | ">=" | "<" | "<="; + +export type Completion = { +/** + * What to show: a field's path, or a value as it is. + */ +label: string, +/** + * What replaces the span: `@method:`, `@header.`, or a value quoted when it needs to be. + */ +insert: string, kind: CompletionKind, info: string | null, +/** + * The term is not finished after inserting (a field still needs its value, a map field its + * key), so completion should open again. + */ +reopen: boolean, }; + +export type CompletionKind = "field" | "value"; + +/** + * Completions for the term at the cursor, all replacing `span`. + */ +export type Completions = { span: Span, items: Array, }; + +/** + * Something in a query the schema does not accept. The query still parses and runs; the + * offending term matches nothing. + */ +export type Diagnostic = { "type": "unknownField", path: string, span: Span, } | { "type": "invalidValue", path: string, span: Span, }; + +export type DocValue = boolean | number | string; + +/** + * One value or several. + */ +export type DocValues = DocValue | Array; + +/** + * What a query is matched against: free text for bare words and phrases, and field values by + * lowercase dotted path. A map field's values sit under `name.key` (`header.x-cache`). + */ +export type Document = { text?: string, fields?: { [key in string]?: DocValues }, }; + +/** + * One field: `@name:value`. + */ +export type FieldDef = { +/** + * The field's path, dotted when nested: `method`, `req.header`. + */ +name: string, kind: FieldKind, +/** + * A map of named values, such as headers: `@header.:value` is valid for any key, and + * the key is matched against the document's keys (case-insensitively). + */ +map?: boolean, +/** + * A short note shown with the field in completions. + */ +info?: string, }; + +export type FieldKind = { "type": "text" } | { "type": "number", units: Array, } | { "type": "flag" }; + +/** + * What a field term's value asks for. The shape is decided by the parser alone; the schema + * decides what it means for a given field. + */ +export type FieldValue = { "type": "text", text: string, quoted: boolean, } | { "type": "wildcard", pattern: string, } | { "type": "exists" } | { "type": "compare", op: CompareOp, number: NumberLiteral, } | { "type": "range", low: NumberLiteral, high: NumberLiteral, }; + +/** + * How a bare word matches the document's free text. A quoted phrase is always a substring. + */ +export type FreeText = "fuzzy" | "substring"; + +/** + * A node of the parsed query. + */ +export type Node = { "type": "And", children: Array, span: Span, } | { "type": "Or", children: Array, span: Span, } | { "type": "Not", node: Node, span: Span, } | { "type": "Term", text: string, span: Span, } | { "type": "Phrase", text: string, span: Span, } | { "type": "Field", +/** + * The dotted path, one entry per identifier: `@req.header.accept` is + * `["req", "header", "accept"]`. + */ +path: Array, value: FieldValue, span: Span, +/** + * `@path:`, from the `@` through the colon. + */ +pathSpan: Span, +/** + * The value as written, quotes included. Empty (and at the end of the field) when + * nothing follows the colon. + */ +valueSpan: Span, }; + +/** + * A number as written in a comparison or range, with its unit when it has one (`500ms`, + * `2.5mb`). The schema says which units a field accepts and what they are worth. + */ +export type NumberLiteral = { value: number, unit: string | null, +/** + * The number as it appears in the source, unit included. + */ +raw: string, }; + +/** + * A consumer's searchable fields. + */ +export type Schema = { fields: Array, +/** + * Substring when left out. + */ +freeText?: FreeText, }; + +/** + * A half-open byte range `start..end` in the query source. + */ +export type Span = { start: number, end: number, }; + +export type Token = { kind: TokenKind, span: Span, }; + +export type TokenKind = "field" | "value" | "operator" | "quote" | "paren" | "negation" | "text"; + +/** + * A unit a number may be written in, and how many of the field's base unit it is worth: + * `{ name: "s", factor: 1000 }` for a field counted in milliseconds. + */ +export type Unit = { name: string, factor: number, }; diff --git a/crates/yaak-query/build-wasm.cjs b/crates/yaak-query/build-wasm.cjs new file mode 100644 index 000000000..22a6b65b3 --- /dev/null +++ b/crates/yaak-query/build-wasm.cjs @@ -0,0 +1,40 @@ +const { execSync } = require("node:child_process"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); + +if (process.env.SKIP_WASM_BUILD === "1") { + console.log("Skipping wasm-pack build (SKIP_WASM_BUILD=1)"); + return; +} + +// Remap machine-specific paths that rustc embeds into the binary (panic +// location strings), so builds are reproducible across machines +const sysroot = execSync("rustc --print sysroot").toString().trim(); +const cargoHome = process.env.CARGO_HOME ?? path.join(os.homedir(), ".cargo"); + +execSync("wasm-pack build --target bundler", { + stdio: "inherit", + env: { + ...process.env, + CARGO_BUILD_TARGET: "wasm32-unknown-unknown", + RUSTFLAGS: `--remap-path-prefix=${cargoHome}=/cargo --remap-path-prefix=${sysroot}=/rustc`, + }, +}); + +// Rewrite the generated entry to use Vite's ?init import style instead of +// the ES Module Integration style that wasm-pack generates, which Vite/rolldown +// does not support in production builds. +const entry = path.join(__dirname, "pkg", "yaak_query.js"); +fs.writeFileSync( + entry, + [ + 'import init from "./yaak_query_bg.wasm?init";', + 'export * from "./yaak_query_bg.js";', + 'import * as bg from "./yaak_query_bg.js";', + 'const instance = await init({ "./yaak_query_bg.js": bg });', + "bg.__wbg_set_wasm(instance.exports);", + "instance.exports.__wbindgen_start();", + "", + ].join("\n"), +); diff --git a/crates/yaak-query/index.ts b/crates/yaak-query/index.ts new file mode 100644 index 000000000..14ca5657a --- /dev/null +++ b/crates/yaak-query/index.ts @@ -0,0 +1,85 @@ +export * from "./bindings/query"; +import type { Completions, Diagnostic, Document, Node, Schema, Token } from "./bindings/query"; +import { + QueryMatcher as WasmQueryMatcher, + parse_query, + query_completions, + query_diagnostics, + query_format_field, + query_tokens, +} from "./pkg"; + +// Every span and offset here counts UTF-16 code units, like JavaScript strings do. + +/** The parsed tree, or `null` when the query has no terms */ +export function parseQuery(input: string): Node | null { + return parse_query(input) as Node | null; +} + +/** Highlighting spans, in order */ +export function queryTokens(input: string): Token[] { + return query_tokens(input) as Token[]; +} + +/** + * What can complete the term at `cursor`: field names, or after `@field:` the values given for + * that field in `values` + */ +export function queryCompletions( + input: string, + cursor: number, + schema: Schema, + values?: Record, +): Completions | null { + return query_completions(input, cursor, schema, values ?? {}) as Completions | null; +} + +/** The terms the schema cannot run: unknown fields and values that cannot apply */ +export function queryDiagnostics(input: string, schema: Schema): Diagnostic[] { + return query_diagnostics(input, schema) as Diagnostic[]; +} + +/** `@field:value` (or `-@field:value`), with the value quoted when it needs to be */ +export function formatField(field: string, value: string, negated = false): string { + return query_format_field(field, value, negated); +} + +/** The length that marks a cell with no value (`Table::NO_VALUE` in the crate) */ +const NO_VALUE = 0xffffffff; + +/** A query parsed once, to match against many documents */ +export class QueryMatcher { + readonly #inner: WasmQueryMatcher; + + constructor(input: string, schema: Schema) { + this.#inner = new WasmQueryMatcher(input, schema); + } + + /** The query has no terms, so it matches everything */ + get isEmpty(): boolean { + return this.#inner.isEmpty; + } + + matches(doc: Document): boolean { + return this.#inner.matches(JSON.stringify(doc)); + } + + /** + * The indexes of the matching rows, for many documents in one call. `cells` is flat: for + * each row, its free text, then a value (or `null` when it has none) for each of `columns` + */ + matchingRows(columns: string[], cells: (string | null)[]): Uint32Array { + // Packed as one string plus each cell's length, which crosses into wasm far faster than + // JSON: nothing to escape or parse, and the cells are read in place + const lengths = new Uint32Array(cells.length); + for (let i = 0; i < cells.length; i++) { + lengths[i] = cells[i]?.length ?? NO_VALUE; + } + return this.#inner.matchingRows(columns, cells.join(""), lengths); + } + + /** Frees the parsed query now rather than when it is garbage collected */ + free() { + this.#inner.free(); + } +} diff --git a/crates/yaak-query/package.json b/crates/yaak-query/package.json new file mode 100644 index 000000000..da9e0b86a --- /dev/null +++ b/crates/yaak-query/package.json @@ -0,0 +1,15 @@ +{ + "name": "@yaakapp-internal/query", + "version": "1.0.0", + "private": true, + "main": "index.ts", + "scripts": { + "bootstrap": "npm run build", + "build": "run-s build:*", + "build:pack": "node build-wasm.cjs", + "build:clean": "rimraf ./pkg/.gitignore" + }, + "devDependencies": { + "rimraf": "^6.1.2" + } +} diff --git a/crates/yaak-query/pkg/package.json b/crates/yaak-query/pkg/package.json new file mode 100644 index 000000000..a51f42d53 --- /dev/null +++ b/crates/yaak-query/pkg/package.json @@ -0,0 +1,17 @@ +{ + "name": "yaak-query", + "type": "module", + "version": "0.1.0", + "files": [ + "yaak_query_bg.wasm", + "yaak_query.js", + "yaak_query_bg.js", + "yaak_query.d.ts" + ], + "main": "yaak_query.js", + "types": "yaak_query.d.ts", + "sideEffects": [ + "./yaak_query.js", + "./snippets/*" + ] +} \ No newline at end of file diff --git a/crates/yaak-query/pkg/yaak_query.d.ts b/crates/yaak-query/pkg/yaak_query.d.ts new file mode 100644 index 000000000..d326c7931 --- /dev/null +++ b/crates/yaak-query/pkg/yaak_query.d.ts @@ -0,0 +1,49 @@ +/* tslint:disable */ +/* eslint-disable */ + +/** + * A query parsed and resolved once, then matched against many documents. + */ +export class QueryMatcher { + free(): void; + [Symbol.dispose](): void; + /** + * Whether a `Document` (`{ text, fields }`), as JSON, matches. JSON crosses into wasm far + * faster than walking a JS object property by property. + */ + matches(doc_json: string): boolean; + /** + * The indexes of the matching rows, for many documents in one call. The cells (per row, + * its free text, then one per column) are packed as in [`Table::from_packed`]: joined into + * one string, with each one's UTF-16 length, or `Table::NO_VALUE` for no value. + */ + matchingRows(columns: string[], joined: string, lengths: Uint32Array): Uint32Array; + constructor(input: string, schema: any); + /** + * Whether the query has no terms, so it matches everything. + */ + readonly isEmpty: boolean; +} + +/** + * The parsed tree (`Node | null`), with UTF-16 spans. + */ +export function parse_query(input: string): any; + +/** + * `Completions | null` at UTF-16 offset `cursor`. `values` maps field paths to the values seen + * for them (`Record`). + */ +export function query_completions(input: string, cursor: number, schema: any, values: any): any; + +/** + * The terms the schema cannot run (`Diagnostic[]`), with UTF-16 spans. + */ +export function query_diagnostics(input: string, schema: any): any; + +export function query_format_field(field: string, value: string, negated: boolean): string; + +/** + * Highlighting spans (`Token[]`), in UTF-16 offsets. + */ +export function query_tokens(input: string): any; diff --git a/crates/yaak-query/pkg/yaak_query.js b/crates/yaak-query/pkg/yaak_query.js new file mode 100644 index 000000000..d3d8a23f4 --- /dev/null +++ b/crates/yaak-query/pkg/yaak_query.js @@ -0,0 +1,6 @@ +import init from "./yaak_query_bg.wasm?init"; +export * from "./yaak_query_bg.js"; +import * as bg from "./yaak_query_bg.js"; +const instance = await init({ "./yaak_query_bg.js": bg }); +bg.__wbg_set_wasm(instance.exports); +instance.exports.__wbindgen_start(); diff --git a/crates/yaak-query/pkg/yaak_query_bg.js b/crates/yaak-query/pkg/yaak_query_bg.js new file mode 100644 index 000000000..8b94229f4 --- /dev/null +++ b/crates/yaak-query/pkg/yaak_query_bg.js @@ -0,0 +1,609 @@ +/** + * A query parsed and resolved once, then matched against many documents. + */ +export class QueryMatcher { + __destroy_into_raw() { + const ptr = this.__wbg_ptr; + this.__wbg_ptr = 0; + QueryMatcherFinalization.unregister(this); + return ptr; + } + free() { + const ptr = this.__destroy_into_raw(); + wasm.__wbg_querymatcher_free(ptr, 0); + } + /** + * Whether the query has no terms, so it matches everything. + * @returns {boolean} + */ + get isEmpty() { + const ret = wasm.querymatcher_isEmpty(this.__wbg_ptr); + return ret !== 0; + } + /** + * Whether a `Document` (`{ text, fields }`), as JSON, matches. JSON crosses into wasm far + * faster than walking a JS object property by property. + * @param {string} doc_json + * @returns {boolean} + */ + matches(doc_json) { + const ptr0 = passStringToWasm0(doc_json, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.querymatcher_matches(this.__wbg_ptr, ptr0, len0); + if (ret[2]) { + throw takeFromExternrefTable0(ret[1]); + } + return ret[0] !== 0; + } + /** + * The indexes of the matching rows, for many documents in one call. The cells (per row, + * its free text, then one per column) are packed as in [`Table::from_packed`]: joined into + * one string, with each one's UTF-16 length, or `Table::NO_VALUE` for no value. + * @param {string[]} columns + * @param {string} joined + * @param {Uint32Array} lengths + * @returns {Uint32Array} + */ + matchingRows(columns, joined, lengths) { + const ptr0 = passArrayJsValueToWasm0(columns, wasm.__wbindgen_malloc); + const len0 = WASM_VECTOR_LEN; + const ptr1 = passStringToWasm0(joined, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len1 = WASM_VECTOR_LEN; + const ptr2 = passArray32ToWasm0(lengths, wasm.__wbindgen_malloc); + const len2 = WASM_VECTOR_LEN; + const ret = wasm.querymatcher_matchingRows(this.__wbg_ptr, ptr0, len0, ptr1, len1, ptr2, len2); + if (ret[3]) { + throw takeFromExternrefTable0(ret[2]); + } + var v4 = getArrayU32FromWasm0(ret[0], ret[1]).slice(); + wasm.__wbindgen_free(ret[0], ret[1] * 4, 4); + return v4; + } + /** + * @param {string} input + * @param {any} schema + */ + constructor(input, schema) { + const ptr0 = passStringToWasm0(input, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.querymatcher_new(ptr0, len0, schema); + if (ret[2]) { + throw takeFromExternrefTable0(ret[1]); + } + this.__wbg_ptr = ret[0]; + QueryMatcherFinalization.register(this, this.__wbg_ptr, this); + return this; + } +} +if (Symbol.dispose) QueryMatcher.prototype[Symbol.dispose] = QueryMatcher.prototype.free; + +/** + * The parsed tree (`Node | null`), with UTF-16 spans. + * @param {string} input + * @returns {any} + */ +export function parse_query(input) { + const ptr0 = passStringToWasm0(input, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_query(ptr0, len0); + if (ret[2]) { + throw takeFromExternrefTable0(ret[1]); + } + return takeFromExternrefTable0(ret[0]); +} + +/** + * `Completions | null` at UTF-16 offset `cursor`. `values` maps field paths to the values seen + * for them (`Record`). + * @param {string} input + * @param {number} cursor + * @param {any} schema + * @param {any} values + * @returns {any} + */ +export function query_completions(input, cursor, schema, values) { + const ptr0 = passStringToWasm0(input, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.query_completions(ptr0, len0, cursor, schema, values); + if (ret[2]) { + throw takeFromExternrefTable0(ret[1]); + } + return takeFromExternrefTable0(ret[0]); +} + +/** + * The terms the schema cannot run (`Diagnostic[]`), with UTF-16 spans. + * @param {string} input + * @param {any} schema + * @returns {any} + */ +export function query_diagnostics(input, schema) { + const ptr0 = passStringToWasm0(input, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.query_diagnostics(ptr0, len0, schema); + if (ret[2]) { + throw takeFromExternrefTable0(ret[1]); + } + return takeFromExternrefTable0(ret[0]); +} + +/** + * @param {string} field + * @param {string} value + * @param {boolean} negated + * @returns {string} + */ +export function query_format_field(field, value, negated) { + let deferred3_0; + let deferred3_1; + try { + const ptr0 = passStringToWasm0(field, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ptr1 = passStringToWasm0(value, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len1 = WASM_VECTOR_LEN; + const ret = wasm.query_format_field(ptr0, len0, ptr1, len1, negated); + deferred3_0 = ret[0]; + deferred3_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred3_0, deferred3_1, 1); + } +} + +/** + * Highlighting spans (`Token[]`), in UTF-16 offsets. + * @param {string} input + * @returns {any} + */ +export function query_tokens(input) { + const ptr0 = passStringToWasm0(input, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.query_tokens(ptr0, len0); + if (ret[2]) { + throw takeFromExternrefTable0(ret[1]); + } + return takeFromExternrefTable0(ret[0]); +} +export function __wbg_Error_bce6d499ff0a4aff(arg0, arg1) { + const ret = Error(getStringFromWasm0(arg0, arg1)); + return ret; +} +export function __wbg_String_8564e559799eccda(arg0, arg1) { + const ret = String(arg1); + const ptr1 = passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len1 = WASM_VECTOR_LEN; + getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true); + getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true); +} +export function __wbg___wbindgen_bigint_get_as_i64_410e28c7b761ad83(arg0, arg1) { + const v = arg1; + const ret = typeof(v) === 'bigint' ? v : undefined; + getDataViewMemory0().setBigInt64(arg0 + 8 * 1, isLikeNone(ret) ? BigInt(0) : ret, true); + getDataViewMemory0().setInt32(arg0 + 4 * 0, !isLikeNone(ret), true); +} +export function __wbg___wbindgen_boolean_get_2304fb8c853028c8(arg0) { + const v = arg0; + const ret = typeof(v) === 'boolean' ? v : undefined; + return isLikeNone(ret) ? 0xFFFFFF : ret ? 1 : 0; +} +export function __wbg___wbindgen_debug_string_edece8177ad01481(arg0, arg1) { + const ret = debugString(arg1); + const ptr1 = passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len1 = WASM_VECTOR_LEN; + getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true); + getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true); +} +export function __wbg___wbindgen_in_07056af4f902c445(arg0, arg1) { + const ret = arg0 in arg1; + return ret; +} +export function __wbg___wbindgen_is_bigint_aeae3893f30ed54e(arg0) { + const ret = typeof(arg0) === 'bigint'; + return ret; +} +export function __wbg___wbindgen_is_function_5cd60d5cf78b4eef(arg0) { + const ret = typeof(arg0) === 'function'; + return ret; +} +export function __wbg___wbindgen_is_null_2042690d351e14f0(arg0) { + const ret = arg0 === null; + return ret; +} +export function __wbg___wbindgen_is_object_b4593df85baada48(arg0) { + const val = arg0; + const ret = typeof(val) === 'object' && val !== null; + return ret; +} +export function __wbg___wbindgen_is_string_dde0fd9020db4434(arg0) { + const ret = typeof(arg0) === 'string'; + return ret; +} +export function __wbg___wbindgen_is_undefined_35bb9f4c7fd651d5(arg0) { + const ret = arg0 === undefined; + return ret; +} +export function __wbg___wbindgen_jsval_eq_c0ed08b3e0f393b9(arg0, arg1) { + const ret = arg0 === arg1; + return ret; +} +export function __wbg___wbindgen_jsval_loose_eq_0ad77b7717db155c(arg0, arg1) { + const ret = arg0 == arg1; + return ret; +} +export function __wbg___wbindgen_number_get_f73a1244370fcc2c(arg0, arg1) { + const obj = arg1; + const ret = typeof(obj) === 'number' ? obj : undefined; + getDataViewMemory0().setFloat64(arg0 + 8 * 1, isLikeNone(ret) ? 0 : ret, true); + getDataViewMemory0().setInt32(arg0 + 4 * 0, !isLikeNone(ret), true); +} +export function __wbg___wbindgen_string_get_d109740c0d18f4d7(arg0, arg1) { + const obj = arg1; + const ret = typeof(obj) === 'string' ? obj : undefined; + var ptr1 = isLikeNone(ret) ? 0 : passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + var len1 = WASM_VECTOR_LEN; + getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true); + getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true); +} +export function __wbg___wbindgen_throw_9c31b086c2b26051(arg0, arg1) { + throw new Error(getStringFromWasm0(arg0, arg1)); +} +export function __wbg_call_13665d9f14390edc() { return handleError(function (arg0, arg1) { + const ret = arg0.call(arg1); + return ret; +}, arguments); } +export function __wbg_done_54b8da57023b7ed2(arg0) { + const ret = arg0.done; + return ret; +} +export function __wbg_entries_564a7e8b1e54ede5(arg0) { + const ret = Object.entries(arg0); + return ret; +} +export function __wbg_get_3e9a707ab7d352eb() { return handleError(function (arg0, arg1) { + const ret = Reflect.get(arg0, arg1); + return ret; +}, arguments); } +export function __wbg_get_98fdf51d029a75eb(arg0, arg1) { + const ret = arg0[arg1 >>> 0]; + return ret; +} +export function __wbg_get_unchecked_1dfe6d05ad91d9b7(arg0, arg1) { + const ret = arg0[arg1 >>> 0]; + return ret; +} +export function __wbg_get_with_ref_key_6412cf3094599694(arg0, arg1) { + const ret = arg0[arg1]; + return ret; +} +export function __wbg_instanceof_ArrayBuffer_53db37b06f6b9afe(arg0) { + let result; + try { + result = arg0 instanceof ArrayBuffer; + } catch (_) { + result = false; + } + const ret = result; + return ret; +} +export function __wbg_instanceof_Map_16f217b9a2a08d8c(arg0) { + let result; + try { + result = arg0 instanceof Map; + } catch (_) { + result = false; + } + const ret = result; + return ret; +} +export function __wbg_instanceof_Uint8Array_abd07d4bd221d50b(arg0) { + let result; + try { + result = arg0 instanceof Uint8Array; + } catch (_) { + result = false; + } + const ret = result; + return ret; +} +export function __wbg_isArray_94898ed3aad6947b(arg0) { + const ret = Array.isArray(arg0); + return ret; +} +export function __wbg_isSafeInteger_01e964d144ad3a55(arg0) { + const ret = Number.isSafeInteger(arg0); + return ret; +} +export function __wbg_iterator_1441b47f341dc34f() { + const ret = Symbol.iterator; + return ret; +} +export function __wbg_length_2591a0f4f659a55c(arg0) { + const ret = arg0.length; + return ret; +} +export function __wbg_length_56fcd3e2b7e0299d(arg0) { + const ret = arg0.length; + return ret; +} +export function __wbg_new_02d162bc6cf02f60() { + const ret = new Object(); + return ret; +} +export function __wbg_new_310879b66b6e95e1() { + const ret = new Array(); + return ret; +} +export function __wbg_new_7ddec6de44ff8f5d(arg0) { + const ret = new Uint8Array(arg0); + return ret; +} +export function __wbg_next_2a4e19f4f5083b0f(arg0) { + const ret = arg0.next; + return ret; +} +export function __wbg_next_6429a146bf756f93() { return handleError(function (arg0) { + const ret = arg0.next(); + return ret; +}, arguments); } +export function __wbg_prototypesetcall_5f9bdc8d75e07276(arg0, arg1, arg2) { + Uint8Array.prototype.set.call(getArrayU8FromWasm0(arg0, arg1), arg2); +} +export function __wbg_set_6be42768c690e380(arg0, arg1, arg2) { + arg0[arg1] = arg2; +} +export function __wbg_set_78ea6a19f4818587(arg0, arg1, arg2) { + arg0[arg1 >>> 0] = arg2; +} +export function __wbg_value_9cc0518af87a489c(arg0) { + const ret = arg0.value; + return ret; +} +export function __wbindgen_cast_0000000000000001(arg0) { + // Cast intrinsic for `F64 -> Externref`. + const ret = arg0; + return ret; +} +export function __wbindgen_cast_0000000000000002(arg0) { + // Cast intrinsic for `I64 -> Externref`. + const ret = arg0; + return ret; +} +export function __wbindgen_cast_0000000000000003(arg0, arg1) { + // Cast intrinsic for `Ref(String) -> Externref`. + const ret = getStringFromWasm0(arg0, arg1); + return ret; +} +export function __wbindgen_cast_0000000000000004(arg0) { + // Cast intrinsic for `U64 -> Externref`. + const ret = BigInt.asUintN(64, arg0); + return ret; +} +export function __wbindgen_init_externref_table() { + const table = wasm.__wbindgen_externrefs; + const offset = table.grow(4); + table.set(0, undefined); + table.set(offset + 0, undefined); + table.set(offset + 1, null); + table.set(offset + 2, true); + table.set(offset + 3, false); +} +const QueryMatcherFinalization = (typeof FinalizationRegistry === 'undefined') + ? { register: () => {}, unregister: () => {} } + : new FinalizationRegistry(ptr => wasm.__wbg_querymatcher_free(ptr, 1)); + +function addToExternrefTable0(obj) { + const idx = wasm.__externref_table_alloc(); + wasm.__wbindgen_externrefs.set(idx, obj); + return idx; +} + +function debugString(val) { + // primitive types + const type = typeof val; + if (type == 'number' || type == 'boolean' || val == null) { + return `${val}`; + } + if (type == 'string') { + return `"${val}"`; + } + if (type == 'symbol') { + const description = val.description; + if (description == null) { + return 'Symbol'; + } else { + return `Symbol(${description})`; + } + } + if (type == 'function') { + const name = val.name; + if (typeof name == 'string' && name.length > 0) { + return `Function(${name})`; + } else { + return 'Function'; + } + } + // objects + if (Array.isArray(val)) { + const length = val.length; + let debug = '['; + if (length > 0) { + debug += debugString(val[0]); + } + for(let i = 1; i < length; i++) { + debug += ', ' + debugString(val[i]); + } + debug += ']'; + return debug; + } + // Test for built-in + const builtInMatches = /\[object ([^\]]+)\]/.exec(toString.call(val)); + let className; + if (builtInMatches && builtInMatches.length > 1) { + className = builtInMatches[1]; + } else { + // Failed to match the standard '[object ClassName]' + return toString.call(val); + } + if (className == 'Object') { + // we're a user defined class or Object + // JSON.stringify avoids problems with cycles, and is generally much + // easier than looping through ownProperties of `val`. + try { + return 'Object(' + JSON.stringify(val) + ')'; + } catch (_) { + return 'Object'; + } + } + // errors + if (val instanceof Error) { + return `${val.name}: ${val.message}\n${val.stack}`; + } + // TODO we could test for more things here, like `Set`s and `Map`s. + return className; +} + +function getArrayU32FromWasm0(ptr, len) { + ptr = ptr >>> 0; + return getUint32ArrayMemory0().subarray(ptr / 4, ptr / 4 + len); +} + +function getArrayU8FromWasm0(ptr, len) { + ptr = ptr >>> 0; + return getUint8ArrayMemory0().subarray(ptr / 1, ptr / 1 + len); +} + +let cachedDataViewMemory0 = null; +function getDataViewMemory0() { + if (cachedDataViewMemory0 === null || cachedDataViewMemory0.buffer.detached === true || (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer)) { + cachedDataViewMemory0 = new DataView(wasm.memory.buffer); + } + return cachedDataViewMemory0; +} + +function getStringFromWasm0(ptr, len) { + return decodeText(ptr >>> 0, len); +} + +let cachedUint32ArrayMemory0 = null; +function getUint32ArrayMemory0() { + if (cachedUint32ArrayMemory0 === null || cachedUint32ArrayMemory0.byteLength === 0) { + cachedUint32ArrayMemory0 = new Uint32Array(wasm.memory.buffer); + } + return cachedUint32ArrayMemory0; +} + +let cachedUint8ArrayMemory0 = null; +function getUint8ArrayMemory0() { + if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) { + cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer); + } + return cachedUint8ArrayMemory0; +} + +function handleError(f, args) { + try { + return f.apply(this, args); + } catch (e) { + const idx = addToExternrefTable0(e); + wasm.__wbindgen_exn_store(idx); + } +} + +function isLikeNone(x) { + return x === undefined || x === null; +} + +function passArray32ToWasm0(arg, malloc) { + const ptr = malloc(arg.length * 4, 4) >>> 0; + getUint32ArrayMemory0().set(arg, ptr / 4); + WASM_VECTOR_LEN = arg.length; + return ptr; +} + +function passArrayJsValueToWasm0(array, malloc) { + const ptr = malloc(array.length * 4, 4) >>> 0; + for (let i = 0; i < array.length; i++) { + const add = addToExternrefTable0(array[i]); + getDataViewMemory0().setUint32(ptr + 4 * i, add, true); + } + WASM_VECTOR_LEN = array.length; + return ptr; +} + +function passStringToWasm0(arg, malloc, realloc) { + if (realloc === undefined) { + const buf = cachedTextEncoder.encode(arg); + const ptr = malloc(buf.length, 1) >>> 0; + getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf); + WASM_VECTOR_LEN = buf.length; + return ptr; + } + + let len = arg.length; + let ptr = malloc(len, 1) >>> 0; + + const mem = getUint8ArrayMemory0(); + + let offset = 0; + + for (; offset < len; offset++) { + const code = arg.charCodeAt(offset); + if (code > 0x7F) break; + mem[ptr + offset] = code; + } + if (offset !== len) { + if (offset !== 0) { + arg = arg.slice(offset); + } + ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0; + const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len); + const ret = cachedTextEncoder.encodeInto(arg, view); + + offset += ret.written; + ptr = realloc(ptr, len, offset, 1) >>> 0; + } + + WASM_VECTOR_LEN = offset; + return ptr; +} + +function takeFromExternrefTable0(idx) { + const value = wasm.__wbindgen_externrefs.get(idx); + wasm.__externref_table_dealloc(idx); + return value; +} + +let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }); +cachedTextDecoder.decode(); +const MAX_SAFARI_DECODE_BYTES = 2146435072; +let numBytesDecoded = 0; +function decodeText(ptr, len) { + numBytesDecoded += len; + if (numBytesDecoded >= MAX_SAFARI_DECODE_BYTES) { + cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }); + cachedTextDecoder.decode(); + numBytesDecoded = len; + } + return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len)); +} + +const cachedTextEncoder = new TextEncoder(); + +if (!('encodeInto' in cachedTextEncoder)) { + cachedTextEncoder.encodeInto = function (arg, view) { + const buf = cachedTextEncoder.encode(arg); + view.set(buf); + return { + read: arg.length, + written: buf.length + }; + }; +} + +let WASM_VECTOR_LEN = 0; + + +let wasm; +export function __wbg_set_wasm(val) { + wasm = val; +} diff --git a/crates/yaak-query/pkg/yaak_query_bg.wasm b/crates/yaak-query/pkg/yaak_query_bg.wasm new file mode 100644 index 0000000000000000000000000000000000000000..c71e7d627d96d264cd5b1696e568d644dd54f9ad GIT binary patch literal 340382 zcmd?S4VYcmS>LC>>4g6b#v1^x#(+V zV$Ud&B~2XXR+c8Q;tDWeNLmbai3kPqEXXe!b>s@OHhfcjW41ys1srVlq41$9}c$WW; z9}Le1{KpG_&K!)+^2#6H`5T`-Yrigo!b%A2F9q~sCG>|P2|wi857`5H<=y>u5@_~d z$+F1v4`h~I5^4F9wR5oI)5)m1$1`Wn_=?AmyWV(LCue+vXVj}&zB7L~ox?>1l{|jj z;O5cF@@qv&JZ%N6JZ}`&Un`urvhi7UKfqu8Caz*~*7v{{$s}#l=JvRP^(Ko#D1BVVRA2{*8g_9pRaQHoM zSUPp}fdlV9{GJ2vSy(ynzGFvjSva!1Fn?g@{NlpQ{M4R9^ZRzr?>jsj?Cn!?($ze+ zaL3Alndx2o4(*(tIlMT#XJ&EV?qK$%E6g81ws2t2^x?hphxW`)?b>~Kc42;3Fw>{P z7T3e=hmO8);lRElN2d1d+`Bqp8i!fyDGxn zzP*d{i+gs?Pl1QoJqw5XV$7F6Mfusi3x{S7?cBFGy?5u{J$q+^eSOCNwcYmLe=9(p zTv$A?yzl|Ay>r)*#obf;ruXdIH?wa#`1(GT3!TjzUYOoBJG1x5%)Y6G-Fv5kf7qwI zAOAmDIeF``_Z--}XJ&d~&z^m=`}QpyIWm7J=tn!gi(?D#Kd^h})ZW>Bhi7IE&n)cQ zv#>LW`gBn!KecOq=gh9dM`n&JPVHKpnR=!2v-9%{M`q?1rl%Jd_b%?4?>qawQwSj* zncuyz3;JSK_RaU5b&ulPP99%5zVd^;15VbAo!{KB5T({h!EaAo1-p_St& z575KT!_%{iyQg=~AK5*<*mv%W(q-47=>@UM;-0C!yAMw-_U*+JmzZ()0!%nHd+6}& z{O&!wV8(v@;XAO|-nn~bX3zY-#huf;;gIE3oYDw?7-CQ)Sj6`3~t}lt|QaadxAIhp)^mXg_082X;c1|x&^_}w9 zudBXv3XyzZ;lzO>AZu#x;oS>Ui@W#jKD@6l>|Tn$_2fzB{_qhL%k;i|i;Gi-rVlMH z27`S7{qP;%{LdqU#HC-v*8!URpS^a^NsRbNBq>-aUt5 z5j4r=H*m^{MkLYH;ykl8HGh~1pYGehdSdalMJ)3>r}oWGPaT<>-??{jdVaPqv3Su? zn6<;lj~`t)bj;{&*Y2t5#lw4#?A^0_Y8O9!fmq)Jcw9uyuiSe4*nvGq=4Ynn_Z(TA z-FtX?VPUav6L07@74JJXzp!}gF${&>dk!ClF^|kH&d;Fz`pb-|b!|8TI(q#0sRfbI z)S>y=*~7E5J7?z)@07Cb#{#?8wc;s47(I0Az^yaW2c~yUEr1Q$n%Q}1e(&zSh+RLV z`Gv#pd(QzsL5Q&<3ww9Y&K{bd+KG1V+lGlWF9mKrcI(OkjG%>+$E2WF4jn$a5Cnb5 zIxWoQkwd3e4ovm_+Zjy1hFZIVsn<|zcVEVPB?za3srdQiE6MOkt(J_Bm&QxUo{8GX z@W{w;sXjh3vY|GTIvo z_^D5jJ{*_IwN0g3DK3?gfl}#;n8)GaBq3`wu9Zt6og}nhx?;FCTnp%HOKqI)%B7v9 za%q^{*!=^3nkr9Fyf&PSlms7Ru8oY)G`$U%c}_~>64}NZI#LACE_Bq=HIoc$gyf@$vDiwq6N1WeN@_afw0K<9c1y zX`>tqv;0fw2`I{fm+s0j;Kb#PWS2+B$Kz77+@cY>uYYahYsZpOT&dJc8#cFJH!?Ev zHKU{PaFgz60$7F<3R0i)CAzJZ6a6cf)3iKXyMlSE?SA7MuclAN7lxsFFPEDdhkN92 zshre7m{qhtKwGZ|jMRYblyeWIAg~APPD8KtkSk|08uy}G|A^3PyyY0|P zq>g0;m*Nq(6gqPJy|*1*kXSqwJdr2P-+Jgh$Bv&`x%J4Y;J4#ZS90Ho((L&fUENP za6?ZOFVaqh;imOf=6ye(kA@4C49nk_Gz&>A#ZImS+p8ao!XOK4-yME$e8+FZzaRat zzY=|S{3p>b#6KSY=lHkc|1Z(?~Fe<@Vm)xC%>P3HTit9I`ChUKT7@}c{=&=6TSZ5CD(l2j%UN4O8#Z^$>cvI_a#4* z{8{`+{2!xxqn}RxL;OS0U&Ozfd?^08CU3hD@UzfkVE^W=7{qQ@iTq|wgY!7kP{H8{b zhSvm}TLWqMhH#=4rb*iRhA2prEX*P*t@5vVbGtM-6I~!P3rI`PKbR^ZiS}prg>P+_ zXe7xhOIg6{3$>s|Bkga99#DC;(#~5cQ|YnDT4`G=O87*`NL&*SgiC9hdEGZekE?>3 zd0nxYXJTvSb)9BD8+Dp_JnCr%DDMy7YA`+*3&{IDkXbkrJ*#&3y$1w5l(y2>ot`^; z`U7{)edY%~{Y-LZZuN(L^};8k)9R&N`gE%j?HAg!;CeNehRyE`H&Y+DJI{3Pp6ela z$evY?X}J7p6|GtIN_O@i8#F;$STgau}9a;O`AKdq$P@mCn!iO^ymik zLS@PNe_=2PtabGOh{?9-nYdlm@BM@AFs-KHjg#%cY}<`C^Mj2b3tEHqq?Qh5;fa?0 zysg2&T6O+w)T0_uQ#^a}+-HL3b0Kp&n0>>E){sijBmIh2GQez=R3ip0X=z&&w1?A> zpQUy=t<&^UyEy|ng;}!Ht}}*YNo7XTx|OM?<#hOlw3(J~I7tnatfwP4kXpatBpsC? zhgON&*4emRsV4+jjfrIQi#sDDY8};Xgo$#b(MVkg7`9Hc=yu?4m$LXotKsTrb)cLJTCFs(o=BZpux-6cL^D8fQHcovXy{q_Fz@QErm%Qli01Q z)2b-2u?~Ut^$3*E5WTOXLK7$9-z@?=#PBL9u@`|!TAe`x*W(&^fh?tEB+?r>600K3 zhLG5>f?%Vfq>7jB?GThE*}Vgj?@}TUgeK!%S|-iY8lz8F?31PS#YoE{6&vpqd$PQy znC9kk|3}Z%&tjTvZ!k%8g@oY+BvN`sT1ObxOucvrPcOAlKbfH55xHr#c^pc|RI9Zy z9k<^JxNsA5zNxi2t%-0=jtVLff@J9s;@q@=6R8E!-Y%IOMb6*Q2vJ&A9@W*FgoJC5 z7(A!O`2|!OoXCdsLBsYZHv;Z2-493noqt{Xwqxh*;D;>nlu>-&8s#m_UhEy zmNjZ^QrrYxh@VWQnOZHS8=O|N9!#T|zC@5pU-oFUvgoQNk(X6kleKgrwf~r+Ev7uu zEd@O1SRT9#EPpezQEP4Sm`%-&o?nib zksyDYcwS4)FNNo^%f@r9L(J(U=AZC93O(~*PS1mV={dK+3f32LwAcFjpGeQN{kMtd z%cW;i9+!=4Q%E^YL#E(*a~faQXr@uRc?tYaExjo~IGG)FCDhJ;%z|0L^>nio*rt~; zh6_`HMbSGIxgmTN0wp(uuky;!RAJ9jM|M#ES844MrM85LO$RU!%z-q9@ok?~ z%X*u!zrENB1!VenwrcsGV=HcH*$Qrprfd|Q(pkQsK8wp2kanezcBKr1%USCE{a&fO z)+HZkD6CYj#90B0Un%5`LGN0rbP#x@5SSa;TB%TXtI36}sTkb6OZQABbkR!$pZ_lyFZk5t@Itz{B|^xP zJW!@7M!b}v6zc&AUoCKWL~@IvW%Dn5d3O}3h zq4E>SSCQb?UAj6Lu4O-u%PrgNJa6ZdKsZJ=bUktX3iM1uzXGq=LfF$qfYK;-xh;SD zzSscjDV@e?oPG8yI0`Mdx9rQk1*#OsW+A9^3i#=EY5Qk`>sAp0b2&~dKQ_ zochq&Ao7eX3S@W4bwPT(KONW>Jz8>73kW5eY@$qnk&HtESGubv$r8>5t(HKrMP4Xh zvV5`GR}0QoS~a(7O~}1Ft>Ife$0p9|N5A+~yxL~pgpq2RT*{d{RY(@j+CqgQtrkrJ zwZz;R=Ts%ZeCFz-)spuxGJbdEbG6T%fgj5 zBCxau6T*clQs>8exYhFWC!dATQdx_Zw$LJa0Ka8c8GH9wA0pV+xdah zwurfim2RfbYPDYy8)~!FmRG3KgAZy5A9Q25uvss-;WhY&xJ+-C(wa;F-27n!%9RFh5uQ-XyvThPVeRC&7s@Ym61>>H z=P&aTUa-H=N${fo*-nC3s4$t?1u5&v9%bEIU|U|ipXP)R{T>cj3uYiPafxk342p|D zNp~+N-+b)sYZ?dbRxx;r|c1kq=^&NN^<5RvgmI=HlETW3JqIuGG42wJ0A*g`|4x+@*ob6Y1img{Qu5B#Y5n7=$8MRHag>|8y)}_u zVGi?6=|=nA%&;b%Xwr(>tli1h7I22|5!a*`0YA8=Ko|(22Y5t#5uUfrr*8Xd;&0F; zZx3!9bLMYVD+9Jha!lE^&3iq?YOEk8ykU;afGg?t%G`-*&rDXxT(Gb>(Ylfpc>;Km zmw2nh;#EqsB94X~9&!?r2xyovGADR0857An)2S7chwY(UKw2>g$Hoovh!=$8&`Sx& ztAsHY2f1*xCAT5W*s|!5--yZwB=3+%Hk#mtzSOoH@*UY&PQj#THLJ9)Ot)%Jaub;5 zKg8cv!gM*ksw4ionC|rzzZ|C7@3<<*v^|?!{wnc+5Pnt11qQ--5yGz$!sS9A44n-M-*K-Qj#*?fqtSlK@7!XB-C+kJZ z&{~ml)jE-q+`z_UQKVe;QX*w5mJuSQ6SvYuc-P76_xKF9%ck-nA>DK!NM}qSeT&@n zVH%}dmPm8%dP6^G5n@ayb|XgC@7MqT!^&7^V+AoIQE_?=A0r4gX7W;%{h zF!PLoA-|S|Kyf zT;aWsZ|JI?3VjOT+k5#Idb{k?LT`Z{h#@#eo;@Wz$V($CbN@>_p(yRDiPNGjODq+r z3cGPhjw$;avtn0SFZeWML=LQpPx6+jlKWe;Yje8RY$xZHtl2@tj4h6GxzPpWGObyK z)eN^rW#12Lo#f?ETFtH6VNu+uIC|7PCZiY#{Fkqf7`zz>MyI)rD94B_M`2S4(xS=R z3}&QZhA5v+dznsd9oIF`Zp2I}y-e&Gm5DeUih{6g24vdAOd)zv9#XuVAyOt>+7Tds zPv9ZI8WRgBmBr>DoWi4_tKzKAPy>{Uf(A-jO4d}@IbWGWYsg%#xNPa-tyrH}i#hAm z2N(>Lc7M)s@CF@b59O4X?UZf-#vpb6*=&m53|t%-O0>4*!_8NT=@>gxH1f^-fUmB{ z7Ci&gc(&=caC2j&GG2nT!_&CBa0Jkmw8ubpvE&@O4$wz)K!4%&(FCWOiR8UEUT()% z2aJNXA8cgnp4D&`mmIiTY9-`MEM@5u;kovNB~#+*UPeMjO=-|b zdnh~mjkG=_GHMNp^cy>ZFncrZ4-&6BVN155>;OKew3(Gw(&3Y?*sxwG8&URTix@nL z-=;vzVanB4)|84>ij2_}V*Z~CsDWG!%@7#EtCbCCh(lQ_u{@NmhBq{9E2uj%QY*I; zySB76G#OX(e?k7>e?y`%Y$!W|XZ|$HB8a{s#}7?j3f|_P*^3G5umWj2u4_c@k~B4{867dP9UaLIz;p1OFO{UkqlWbC5}ENONHH zI0O{!QAd=cjwtyNQGyC~k8+|McPdyZ?J?Cy>@{2Cm}1Fw?F|hifQUuJH=Gk6QU_0y z(crLXaLCi35V%OKoNt zSeLOgup=6{3x_l41lO58{(jJXzG=z1q$GI6p!-WKPUoj&Mjc5C^)(m*Elxzc)XI zV;?+WST~Pl(@--*Y%gMK0&UrRJvy zk|gPT2r}wN2^g1~u!1x53T=4LPm^=06Su+h%N;O0(9i4Btl^hWluhEF=&2V zG+!9T81rt!vymvHJuE6=vrr-SAyLkQLog*Ub4=7b;3$%qEi$maz1b?KJa9yf9~ zVG7azV}>){8H16yd{>^tH~29$MGOWVWgTs644$@|=>{7^PboKs4TUiPN7D;fJt1HV zDuV=xTE;@VO$pc`Ct%0&e?k7>e?x?6QxlO3*v3d5gNSx6!5B>;p($xsiJJCr7{VcQ z$J-jiSkfZqx_(Xlfte_6G~Yq^tqD(Q#E$etMQ)FtjGV+m?SBHnga}?;y<;Z2kLPXC z(-De@wwl@RAlmDUK**jbYjqG^y0$fYYRf$wyTxeA;W*SYl}?`&|gZB$N~^`z*C| zL}(pS?npYioNkbJbwiqHv8EXi+oNcaBX4cQ!06{}qwg`Fv0s91VnCq82$SpNQESMl zCUZL~RU(`h7&dHPG|i8)9!f`Kua3~6s+(|Y+m?zGYC{OMEV1%EIv4?fR&>NS7tY2P zAUxBKmDv~~6ahj~rZ|f#**du_x!qv|BKJYUa`RgR+@;7-`gMe-I_nU-z*&qFXkbp1 z&Bl_GgS)C9LqUVlDEA%=giG(II|#p ztwT6-^JQ@6W;nATf;*g9NR!=boN03TZLkgaZFV`+Nn|*46G|3*k;{(7ayq8iP(rjk zXEyyf1YJ7hjF#Tnn}BA7zg&xVtd_pw*Cx3`JXFiozz|1QFuB%JzgAR_K;sboSlT6f zB|+`%x#U!BrC6jZGqnV}nzHDEmZS zzz9&J=?PQIVv~9r5zsY1BZNC*m$}f3vzMZH&Y#RgpNlY~giZ4=C}`&i+WT?87?N;s zOL-H2N%Pkt%&WXl9`1FWquE$jZjT=^{W+**#h7q~TpX0d*w}eNNNn;i8(7LWyNegC zTFI^PXM5xYj?$)bfiu-xc3YmlZK*RYDxc=*X>U3(b*EGXfvYp2Y0dh4(wt5I9~lb7 zu^qX~ejl@VUxer~bLKi6HZpeNCvAC#_eMi!rmDgS9Ac>PYs%bGB8_QrGg{U(Rs1wj zudI3%{`hvvn*IuZ{5z7~nNaB@dLXQ`1LjyD%087-cqiqVJOvp5*bhY5LLO{nw^zfT zfET>FbweWqzXzH#JIySkPb0akC&AAb8lMi9Up)|TiH&wc(xi{yU zm>cE+p)g9Lr?68RBL$_QCQ%x>u%$YAotaR4%& z>>(G{R0VQzjt_Yj8q3X(LoH|wcnnKTzP-I-ANk?-=8|W?NiU^Mb<<38DXk;91|+r0 zUW^The_TWGkh8r89B--;e3qxdhpwFCO@L5P4{0o+dGI`JzL;~)7n8Dybq5lkpJ{$r zR8P5|YxrT;Xd9I|LNYN}DiXZPmx|H8D1^Z8H_~JB3eAG#KATRT8 z({STioTsVzPO;j!hQOFKGS7s2fd(Sm>XMtH1S30#MM6X7u)rf6r=^xo0S{mf5@Bp% zfKkR;A)wH6E?|$K9Q>)|<7F!0s#jk~mZ~D$gY5ppH7Bz2?P)~`9A3#2BpD+s zRqsuVEfLp4v1DS2tOkpT1D43DzL;3DM6AlG?MDq`~ujoxQ*F<|`Z=$hWd!jdS*xKAwNR)`oWWlG^)_}~#BpXJxXT<-S$4l9p zZN7b{tIUy}%n_f-$ysGKdor6oli(m_4)~;4)tUX z`Ai~}RI|~O+3=ZIzsjulWY&Eqf<~ExJ(+_(Q#rMsoSNlS5n}LAIlE2K^}`A;g`=wo zeq$tdE*xXACzEniQAe~C1X&^6t6g;rg~O3MKuECe`*l(VC~RSiXiV!Zwu%UHnn7m4 z6^_>wc2rsFQe}5;C=GSG zEylmA=~**5a14{{91}Y)bJ|Ic956G&iCPATH=51_+)RuKmQ!INAvTJG` zzb&e3D;T>>7u1v_l$>NK+ho7_M7DbNq{tyF-Ol!x>x=^lY57foej3@uQ1V$L<8lT) zndYtEjzd#iW%U~dCs!0{ht}$9qrBmo7|JfWCa7q6vCC7DA>V>4nPj84XSZ2*62Tam z?j|KKElJp-a}*jQ1VfdWTwA6LzSh#wjKMyr8eEGiE=J(G9qpVq z#Qga%iAFFq3FEFSR64+YPgCW^ia|AVwa~PMf517qJtjK z5Dp1i0)vF|S|li;2MG+TX7)P~0qf|cycP*Ko=aM0mC~Ui5*ok(t!U38M*_t(77FG_ zfLaU*TG@IUaTPogtVX07u3|H%5h_|9`d4`>(k#%3W}t%v=n15Fk|c*GDeiJwgXhrL zUa^R!v<3u4{;H2IODUm;rBGxQqZ6Y);)J^_1%G)IHCtd34AzNH5tUd+9PxmCa)xjW zr7`=?+@y+9I!MYHX$_JPAUUOgRwDyPDIFx)z{%5C4N2u5B#E0lNP>}g_o!iTSS&ee z2Kz`G($-2Fl5%O2v?P(dv`M(posX62Z7~LE$UxlgogicIQFh0PR-6UfqNkpH zOFO|f$gXM)fPo^aAi6B24c2|oJiy4Rz!K06f4=q zKl$ZAj+%DKx+z;%pf*4s(Mh(6V&Uz}{JrYB$(5E4kOf*+q@_0D!3nT%qhBoZm7isv z@ke4RQ@h$?No9>9SX-7RVyD(P#Z+DbcI)jNdINxFyY#k6rA|^5M+RM`I7Fhl?O=+p z-TyT1CmDAsYZ^K@`J&;~I6D((@$K#C_IAJ}bBJJ4@jEbh=C+c>E95`%1qRy|c|55n z4Ld?NwOM{hSP~9Y2L$|26&W53e_M3v*|&TXyOKd8$}x_ciT7K2k>^lhnAPA7L!1n6 zwgUj+VcZ@V`-G-S{}UqWz*3S^tCWp19z#tqX>gV8po$qlBm-g(bO>6iEb#}|cQ9$k zLb3GIGJ_{Muw?aGCXZLOUd|g1WH~Eh3-=*0k`5Skt{a#fL9}(Moi!O?a?B9Ov_2aU z2;)LU9Yf#Vm;)9Ln1%)(D7TnPisHg^Lu{IbnHpAsK~3P?vymd;N-ksEXBdrlP|293 zkRpuKNoWZyF$OFINK;gi32kT6%KJ~uh$2|Q!q8x{bD`1X}{l~p1gRnjVrkVWzx zCp?_aPPddCcILYadkZo~gDP|hNSxOJ*mLKDB#LmRad4T!vv1)6_@H@a0_F!6WDH0a zacGdd83vhK@0L{PryDU(x{NIHgXM|kB7@C`%Hg{(A1WB=4wZEa9Iv4cxS?|B%n!BF z9V+CF{X{_|PQ?u$g0FO@Ne7z{D;g>k?>L$J?!?f3wxA!Q2;_rJ2b3gXz#wz-z&5F&5e+T4jJjbh-tT$>d@!}#){OAq4Q(Xe z0HDP;h*P00-mBmju#kxm9S`Fzgg>376)}Rg&v?LoZ!vGCq+}~}i3-{?k1@@~%WMz9 zo<+gKiv&+uf~V|6g9#`nA4*-pL%|BbdSpT&wnnE4RJ=y;l-3HKQdjVl@`k&Dhw&kL zj0$+cbOMD)|j39(l7Ix{p2dotE;C^H2+dI1;@h^Z%PEV(F(1#1{_|W zLw1p0X|q)UxV*w`hVd`cwgh~1BR4?t;@}Vz&nibU@*)bm6~y~BV(D!(iMpyR5w@ep ziWDr4gAS%&5WJdwDu@T@i^(Qcv-~2*@K(R{7SbS=&wfD<$co48fl?RsVAFuevuD{0 zGttw9^e654lBGXeD09i4pRx2W7s@R zr|kKnrGK_i=Au16Zt0&ZlzCjw=sLE!P*DL_ceIfr@E;g)rJrymu$QnO-w6+EA*2hk z5e>+h+{%I9*jdKfq9TzKMBG1*J%M&+8KAWRGD$)No~_F=LIkSn?dbq9d0qk=?}~9b z3WP-9l^1q8%U%pP`4I1Sq5!K*_IpPNL|V1=X&o8hq=*ZTaFsnY(yG*9aahK+YV|9u zxpFhJfE0`gLyExbK?JT2AwoW9XYTbygdysna4Zai1_XL@iPcLbW2E40yQGgC_ay7> z@h`kZi%-S7P<)VQrs<`nmJ|V1D8B{6PAJlTI{hHCsn6)p!YB+y;b1Ar_Bn%LCu5n^ zWhUMMZ#ER7pNdYEA!Vh?DjflfDXRISfK*w@sWOZbugU`HEw9RCu}d|ZDszHW{?4GQ z$^eR)knv{9t1dLjc%xUD9k{?ud@~N4CWt{WdukOXyK1@ONZ1INMx~>dTQO*6ewjH% ze%doep4%g035d>O(WqNe=$>X}q&0tCUdDJyb<+~uKrv^BvFtFEN&n@ReN5Pc)BSNr z_s2q5sRv;g3sfo~jHW%p;sV0rmPHv_Qoe4*^6Rc!TbMl#H~2(AkyrZqk+siW4phSP zvX~gMgcsfLp#hocM#MsScL3f-a4Q0{|J||#wI1zvZMnTtwdGd}dcZ~&D84#RM%E%M z!;P6$epLZXW{;p^!9Li{Wrk|m%1HUHC0RaNn`m3gRzI5Y!Z5s`Vlk8t$3`42YoKC8 ztPFOJ;Tn3=JEYkeEE(%0l!T8^=|rAy1YChnRqsr;8&DSm<|7bNDJWcY3hK#Gg)_4l zi_DOgO$AM2^2Fxc(V$=JL=nP*py0e-Q2U#9H0rPKf{{zPS4lP(jsX2M+Z9P*j<{<3 zn}oXUYqL?Zw8O$g2*j!@5DT(M>nqtua4_OZQA8;-Wq2M(OlajGmurR?dIBcN&g)wf zip%*|xYDyHc*BH0PmT`PG(Q;|uUiz|Oq7BaNhENQo16W9@yY#;76JVJHNPPsl(Ju9 z-p)Tr)atv^uz6qqX4zb(@Q#2G4+it506#>upWp!^He^TY(rq^yv-~qWfcL8t z&0MRKx1nm=KpyC}9GaKQskA40GSq!gY_Wp0nl9y74^t5?p~wS3cyC@9>_^R;eHS$? zi#+V~TLudr8u{aJXILm>Crt|~i?nBF&-&)B3POrHK2P@b6j-T<`qK9JQb+_fzt4nR zm2UO?23Z#D2=2qg7GARREEck$`Cp>VFaTODKL4e+v^UL#r)Tw~gnPgAme%I%i66Zf zfZ(U}qb)=n^kQM$=TT?eOi-dM^QXqJgDuMRUCcJm4}UT5%~G)3Uq#C%z?B}KH< zt;_|%>?bG`0RgKfT`Qr92mM?NdI-JwE1^7jz3h^+3IfBJz7Lx~;@o&6*)JL>u*^qc z8ByaC>{#bSHl*F$oS=wjagQ7tWXt|+rzh}!{-D@L;>tp%bvaeVvv=Z7h)s~`*v~m z*pFJm!!i4J$hg|JK;k-h=KlRMPig;?QM-l%IETc^R={bt2l)hw0INg6L2E4Dk3Jb( zCU&Qs5kypWF!cKy?^kMymLRGm`{yQ$b@koQ*MwoE!vEL)3oE5=QdI_B_9KLSwDdVT z|Buh2*}usC`VXM-ojBw7(}p+!CZWz@WQm^*3|GQ{&=&Y)9B>#AAH5-oU>M9^ojTO5 z38;$fFFAoGtxO^P$Athj9o|u4VrvFfze+U~eb$OnEH7}@daqK@WphGL6v6`Bl*uJb zfzjsA6*S2~&E&lJsirfPmks2fUp|m-JTD34AEAOw9_2tjw+_hL1mubXa{N^Q`Cl&| z$Ud0NQSET~7i&=Mrs1sNSwn!?yLeuOYkKj#YOHh4GwghKrSi`ls?k-SqSPE6xe3bZDve~*8T}$@x%C;LP9@y41y3@8c71pvdoHH(S zzmjp-SW%YTfeBa{BxTg`gdx6YXWS&j-|P_2;m;1tJT`wUY}0$JQ}W?j)+RP?ATYRWk;`uw`0cc|!TSCs7v_t6t!OZeDe5jGx)x>iDPW3f1d zIAp;d_U)CT%uo$kbNnE2@Q159`D1Z=f%}`C;=St1Q-rjR4PWLoCU+nV+2V#&%Atm@7-()z$e` z3DZJNKG&m!s>SPxD8?}0g#ZpMNQ9;uD_A7%*@UGvgg$UaVjUxqsQL_(keL=$8L>RR zj##eOoP4936RtIW{zs7Z+p-G?eY=j)gUy;5>#NuW^z7`=yvie;DmgkZzaVx1g%t`C zT`~@{x{__W5uF&O6>nBU)uc&^Iy4lxtm$W~XR5_uqnHaWsxzNSX^CAy5Xby~?Y}-r zzDTkL`8I9Nq-POhsN*cbXpd2|SD5>U4s#rUCLYh1oLXCQN+qXVNsdPsW?AG?%0_&} zbZx~{6|c@Gr&tj!(^+)n5N=yreVeLZv{??c<;L-ZH;%PRy+RwI+9BMi2d=f@fCCTZIy$Ojp*X{=?v?>sw`jbqI52Z6 z+=-AM92~~tB!IPZs$*E@lPVU zPR|3rKG(-WLJJDF=wR`2N$-%0jxByp&$j5u#XZu|(V%1m;^OaY1%dCD{%se_EhOMZ zfBelRN$Uu+jepPLl-)vCOAm#6be2hwS?@B51pvBCGi1SoHdY?xeRTsbI173kh6I_f z&YDaj)cgTu)I-BQ_2s^NYo>X)CdPm?MJpmgD$3L)wy2~DNa}6-jRz(Au2FT(29+T;{8h2 z={;P67C~*tZ_`*Rkb+u7s3bp9YJxvxJJF9r79T{*8no^K-4gdPZ2KeL|kpEATOW|b%2C3x2KcWBcUB!hs>=-3jz04{t|5a><}9x& zLUe!$*V&^vU|q&Ew`>mhL!Eqqm!Ewz)K`~ffCGfgpf(6HegW-}wXGC>5zQi$BHlo_ zHs$T%=p`4mK`&Sa-LJp1WDF8%Zy`2Mh zVw4Qmv~CFUpUV02)BpJ=N;84o9|U2`j;;UtvHe=d znEp8rz;e2u<9F^H5oN85I#=WWf8moyZSGVj7rY&Dzq#1t-jypzIroMPgWr$p zN^64Ca?Jj*z-|1|Pq$pX04v3#r+)kd%E<*^hvG zZz#>ik847jv$*mE&nH4QaQJ=5L5gd`v@nzNoXUZC!7G;s{!)R;dQv*P3o%VUCs)(E z$tbZ&!{a;c(NZ-|pQ^F)g$EUo#?PKM`voD>yhS-|OWt+8;^eJlyp3w%y zC-uUw5JqwOyWUOLqw3-b16VCT8M>D5V^&2(5~_}f3=kw{TeLw1FFa`zl9px^{&15R zUbZu^S-%)Ur4^~zk-&d;eC!wwXs)H7ZSHCs$ugSU9`AR-x7ZJ^@6MeK7LZ$$or)fI z@Dk-IaA}EC#vMN3b=41qM;Nsy>(w*PK$mgYXd@iPq!kYXPHcINA|1nwc%7oJiaQKF zAR)&V3~;3NWu0ySc%2$Z%Mz0pSy{Wy5o}M3=M7oP57}0Y9XflHVqSFGhh)zB;%}3W z!>!-gG=3AU4>kXs&hQb~mmg$9MA#eb_i;SPb(dnK3}zLnE(N$ztDX+}T6G$xa>Wiw zQ&;h5oQ2^@7M58LEY;1eq+7x;bnF&n*$PrxG8I%d9n|476d4{5Gy|RE-UMpco(8LF zLu*>1*<3@W95@2c?h~D0YG5vYmq{@8?Y=&ocVu-CXjw7Ai`u$@53@Ho&V}TlJmrb< zcoR25j_|RH?B}P;kKZk7htAVP4F z8z0ukK}^0qe9ns`WNG%yhb{`m;;PShCIjE3C=))aqJACol(t7Jv#~^h!RDC#yWmm| z>RT1dS7WTsd#ZgBF59LdJR(xDkv4zS3x}uKWD;!+G83I!pZst{NV&mB%t3Aow(My7 zHt1Tq5Wh;2ZEJp6x7X!Ot1lYH1PaYW^bLz=F)uV6z}_BjlNFALO{44H6nqVA$K4iJ z^W$~dL<(36&BH9SJ?sntsSp693D&#bWPg}l?(Q$JcME*pE3<`stvui-&p3)L;}Qid z9Q@i)U_-PE^yTPIClFy9oj~j`5uQfdx6&b3y9|^AV>*E-F*xW>Aci_^7@Y)7 zb}rVUQ6oE*xFc{U5as_FbSDs{baetzg=6?sh{+4P6Nq8q1fmqPPKS!CIe`e%3})Z5 zl--HPh$nXg<6w5(QuaQdVmB}1^3N7XKuqQcHEi^nd<_^WpI5V*i|Z>baSbVwnm~_4 z(u&}$>{sX?Gmuo$YtC8U9=wRl&CbgjcH9vs%Qs)Y@}oST!?Mb!uG=zjM`azRVDFkx zSLc|{Nu5lwx2nA{`YLWMcSCB`-H_@|tlIp!In>pCHFtI2K>jbxAN;QtXEp9+smfnj zxa?FEQ%KX@QFdPxo#;+Kv*CGmSGn~dv#|D%a_V4Re6eMz`CFnE7-Nf5LIhsxc8aXx z1;nZ?6Tf3?(qx$!-Wi_e!ieVrHvX@cWc5tZwVpN5%ln^p@`Lt(1Lst>fkjeq{p8RU zK^Vo!K&f1*4%X_8q2cDp=-BuMumZOvtq_8Q9VcWUzA4zm8!oa)W=8hFBP^fpiYXFg zcgbWF9|qbZPwIGP!ktvtjTqEZ_0iclz1-^D#~JPpUvqU`yt)TR_iOB(jTwE8?6p>B zOC+4~V~<|_*m0YNyZqQ+Lv`J^Iqk>(8mhBu?~MI5ROgUW_b}eeCD=UH-%? zF5~M!{Wz53>yN*R>g*0c<(_JFmkS{koRZ>(?EIu3F&sMSSB3}4)4@VWlLNDd_#tpm z$AS-L5i3TvZ4l14vn(j*bzLQY6w+!PRxMXK&g`_`L3S6cvgIFf`6?!Pu9L~GG;-aY zD?XS3t@$VfH_+LJ808grBL|X&xNQL)1BTAaVn3aUl4*qXgH*BBw|kOst-y?t+Po}Y z#^RY{pVnsdH5 z)$v9PJl#Gd%RyzDKOF--{&-;nmqnB@X_ws*4wcL9=w}KYvAWVEvZ<4-U~l=E6|C+M zj}zNz1rjVn1Lz8!mq47BC8$f;zm!Zh=Qm+4PHgAF{tO#)r(B$nvL;06Rsg%khQrPc z{uhzq?2NY<#7#NCIAqG6eoN<+EKc13ZfBw|N$jKm~6_~li{OB5_8 z%t{N$NVT_=FjR1^^#kS=l)RiJx3_G?#Vz`64YTIHLD2jzCddudU8ygpAq;7YGKFX+ zD_$K|P|Vm^8W7fw8&CH9$I&}qfk)Tt9gS!|ln~(6a_}`j&Wz~ZHFB<{*9oU-<#g`R zT3D6|Ah2DcSg9&^z6kmf!wLI>D`0ilxdW{11rFcns;Wrqi@p5V4BBis62-}X4TbpG zrXM2-pyhZFF&%LGv?B>m7$JaEj4+hoz7N%*3bHaT($7Q2h>`R<3v3t8H(!hvn!{ zuH7)(X0mSP6`HuIIGWz)oX%v1uo}C55@T{`xLoN(tjH*FLBykXHWdp#elW* zg%~k82=!9&Z7>A1%gT1JyVxcn_gJN+MpXrd)rO4qVjCRe!|C$vz{(ByRqgGSKy%)m zmI^N3!>-l-Zx%GcW0d_#{ur}3Gte-UGNLLJ_X{YcV)tcvOs#v2^lqCxhM8-V$GlA* z)uvnHdg!ztPP%kM7hv_qRi=_m+v9hd$n|3xy~$m1+9}KNbfV|&9uSm}(Is=0;xB>} z)dzpY#}|JHuKF*+Z4X~CkM4F4b8uHR8Xbx*@&*GsNsXIgvTe(1cDk0cf!kXq_9j|`&^1Zd5u3wGrB|vs>86fh zD7-`^FN}5a4)6w$PEHuphN>HI2U)Hs9&On(xjdE(A9T|oA!?fU+gV7~bb@#75&@z> zN{izx>Tnq0(h6ep`@o&*8vQmzc=K9OvaeiWQ#!I|P(5W7`7a$TFt?jmeHw_)GP z;$7cpREzGUms-SY;MZHUSYjue7Duc_j>>b%6zKMvLhadf0t?q8(mij6TvHAx3d7)K zqfv@zB1e*x6y^zhfIjL@J9NOzXZQW^#UQ&TdyJpvr(>w)a!Rn;I*2GUGGZx~(u248 zjOS)9+ajYo7M|L({E3NV^lX$bMpwf^*tlsX%~9K{6-jii2SB19T^fo)t*%8_myj@z zcnO2i==j}vanOd&JtJEQ&&CQIV@8shPnC9k?JlM!UHco@O2iVX??#SKk&3g86zNHp zEED!vylrHA4TN3@94pzBXI-{YLsTN`s1tJ_8ZoKlpk$N6cC)H75ESPF$5`*p_Hf=i zvP}jROsT`Kb+?7sfnZ!8$AMUBuU8ow4u@T5Lf>$%5C?}kYjbp1>5XCiQ>{shQUg_b zR1_ND%KkYX6=il!Q2!YoVT;lmLVHv-$|_Y+eU$xK9wozH53)D^{I!Aod+yEo!!Mp^BBvG}k>1)8Y6aw) z%(829b+xH+Ci8)37^ih=XX^PebJ8x@6R+vu6BFPZ+s3#G;iFD1w;m0i9f1(#0F$bR zwVfxC`KsXe$)1zZq>cv@>TxzK8Go4}+oMu_-)mfI9 z=-Pb4z|l*@}xgRzSS!C)YPlkmDp6zw39QZR_^JHyXd zS2}_4E!^RGlXBeE5XP|^P%3$d_;B_Z&M+S$4nA;_^Fm7xpyZlOhUH2NbGpmTj+wZ_(1sUAZlY|EgEr~n5E-6>utdl3>=-}#I7Gu+zzPFo z`?zBP8wDS9{G~q4Vi|SxDw(qu_UwWLZz2m=lm0*4?PED2yz;xgbp?Wf!Cc$+P)IQJ zK%uz)YtkN(%@>XWQ#uNG*(kUsU=wC;9aeJG!H9+^jge0)dF+h!!_IAao$ha6jvlP7 zQIIRriqd}s+3QjIZ+m=)6r}XuHNlEhq4eJyLVJ{sqpVU{ys4}dq_F&Y?tOU$1qsQb zAn%}pRNoqsTW`IsEddE2R~;c_xw>>yydx0!G^!F1$z0f)usFF_-N`G1kv$;#Dj9vP z(aVTh?`NoR4X{bnPFP5Z_)@<7<2`Cz=qbI2b;SK@&N||PNt8>kz9C{(SAoiQl8#XU z-N-WWpq=CD3{KS68z8(9@3wH6MCu90OiABQ5Ps}V5J+KvWHmfyp0x={fG<- zR!H_y?g?{Kk#t>Khp-yGTHk4Zv-i5@20+=mg^=3f3dzoJX&eW&bDd9azCw~m1DVoc z18GxcSAmn5CJw<4Mj6HW5?arIa!JsC{Ayc6U;z2xdaR|F9R^IWtuPYf;y!S)@?x`( zf@e--AB7%>c?)omX$W`@LoF75uo+ccfTQ2F>hkcbd6M-rPOK0=&31yD9^ITQa|aIL zOgd~WXA}!JvN-~&WuqoU`;rxZG)lXJIvO=H!vEL)3rB{!Nm!2%t{sgUa-ZkY-vR#^ z+i&9DhP6Ec2<~WfBPg{F%ctX~MB8ybYj)!n(3p^(#HcswkQl zaL1}DZl~6ry2`#+Gl~0&tOK_=wqCLv%dLIXnxH0 z;9UGItZ4@yB-)IR!92KM&S@>r2}xpc&K>)jL@j+-9d+2ywSz3ssZPqho3#@^?`+2z zi!u69)v`$l$i~ih<19BXg}>LXY|%-Ii*+AcP^LwA;smq9X(sB~`AEwaG@9MxI3_K- z8<4-V^UB0<1?`y=EPJ&VsXcT%Vs0B-c7-)9u7h2nmw!lMSKJ*?hWR<7EC#zppu_%t zOdah}CI|tm#*bKUSTlx<=685+sD=rMh^8`!A)L^G(J_nqgXWmnNw<7@!sgHR0qy%E z588ULZtFyW1&9oYWtjkQR=YS zOCgxWmb?aFM9o5`vanEpa;I3Brz}TS)^;JG#^{}iFEpZtvk#siswKP{`$6iGn=}&L z?uF|EzJ2Ud7lW+Yyw*`Z;K>Yd*DIJ07`d%=2Le|PcQXU7MM&_r-#JdEv$L{CwJC*? zgm1CWiFaw0EjzQyF>(a$K^ZET^*wPW7e)yGr?tC1iJIrBwep0*6kbMcw5JX9ge?CK! z;nohhYQ&u-vJ$eVif4!u;gBX{U*6r6t8<0+=>wuM^1G=my4{D96!82FAQLlzrWXf6d`=&z=vIk_x zXc25~CSdby^=JJ|068HCYTgkci-g`Lp%2%jHs}VA>Z&O3eqQ~! ziQoqp6|I5_`_7QDtudHwnh^!Bs13{QAd0q@+XF*Use`CNLsTg&<%mK(f|?vrC68og ztVB&AOQWeU40?%-tSSYB!H7kK@#;n=-b7IHVFM6`!~BLuNz}=323&&0kLS{2L&2 z?#<^PWUcFP;4x91n?67H;_zMV+FT&PF?bWp#7G}-8=M=s8~Y}h`^=M<{_OYe`i{Ro zJr^{eQYLFTrT>SY{Of0b;o^fo`zt}7f!@U`+QudeKU1gM7(h2atmnO_+x59O-JRCv zO6Sj6qclKc*`s$V*qO$HIa=OMgcX090CP7cu>CuGI<4LO@M(I?Q!WO$&#QCcU31^_ zuRitEUtRixAAIwNIkPtS4pJ$0_QUUJCzENNuF~Y;(};l6hKle1(J%buABOK>lGSOv zA51~XLJLBoO6xyh1^$U>QS<~zKk!qZ|Lphk-zGmmf;a*G_A_Z!O$Kj=SgH@Y7^)6< zm2*hN+Jo%%|0)OY2eJ=;eC(*k2-PuWq1Y^!>~ZV>bmbjO+4%bilF?1+Tm_wQwCE$5 zFod)Kz*E^5A7~uSnp{+<#@O6{^NEM=R_AUiE@8+~=kbqZ)aazjdIXPLFRh>3G!a!d z5qI=VgvRJ50-&@))X5I4u4VOd9w@@|?b@B)Ld2CDdKos8qK`EhmIod_4S~@U#`X62 zCd0O)ycdJh>iP78ZP;pijFGuLX13q16bz`ebR$=58=qwtI2TcuxVZQ`3Y4&60-M8A z5k&(IMfDs7_fk>#BHhs75QFnbopF-LAiEG2sPPlL40g#8%FnS-&+vb^(c4fVGG-dG z6;au!-=R?tineUj?Dx>i`A~mXIhoKyw>EMr7YSD}J&2T3`Z5u-@W?i>T@5w zSiWoSE1}QXdpGc}el@;pZrk1G^tk#0*%Ow!y81k=RPH{fjLLa|M*`0idZwJx0YTMy z+I#-oxpOUK%K7_?04Aw3iqou97-hXUN*0x}^bpw6 ziq+Xb7lPLcb!Kg#TzODL<0Q*4WGwy7*>*3Un*!qvGciGM`~nKY7JorYQ;#8S#E zdYrw@8__;(S!@yd`>hnvP2q`oDoC-zQA}YPzEH!{>0y$oZ*!Q}(MOfx!EQAs85T+i z_E3`oo;FIZjWk@MutujG9I_5dKnjTxFJw=h$I=Lj>SNpjkK5-FCA6{S# z71FdZj20c>Bq_RcCb~MU7(-qpoNSANAjfeX@xww*14b`syEuCTB5t7h+o7^pU+65E zVU5?ihM_=jN|;Uj2JC3v4xaEGHK-VP1aZWXLc}74Gzh+af^@@ZV}~Hju4Y#wo3Nu> zA(zSV1za=GZI9EIC6!D!`_$MI--ba^%jWgl+zE04V3kFY9OXRTp|4&Wkf#e9{$!ly zfYT+ZIJVPB~ z&VK11!)=D+#$ZNYR;^^yOZ)-k=KmJ$f62qOl?=Z_*5`2TXyGzw z!hX?ge3M4JSJ2=WL=0DS)b}=i2z5Owkl+TLs%1%dj|Wj8Yl#S`f*4R~_Ss-*h7-f~ z%yBp_7;(>*ey)&i1Fd8q(`i_~PH161ZkV?ShJ%FW@H5{YHg@Il7Fmh#3t)iLVL&Cz zrnX}GmCst3={eZ^(U5Xz60;X|{EF4zfPM#<5nYBRGsI2zwZ2hnmT$4>o>=~}WVO!d zmZiipxpw5o)tT<(?RC}cbS`;GIt=M}+OrWse`DDuqeaalucHi=Z|TApj6e*b&vC#y z(>yT^hLw@oSo4m&$kd9`52b?!n`=Bi2v`mSR|^3A)>@zI zTH@xHAa)W`XnfmI9VD)>JL^J9SEH{FuclKeYEB&y94;T>kUQhc3lhY2=W@k|#5RR&!{T7zQ!}SG{#TOuM6f~$qv)Kdb7Mg0^>41C4{P>Y$r?dH#Af&^oxf#!F`4(k?E3Yqz+9}{nm zW!RXblqly~QKmqb<5vkB^wi|irXz)%_B=(Hr?=vcu@;CVDiR5htR)inJw~Oc%)Mv@ ziQ{R7<8DSP1Mb8N>tK-!tk)$M`)Kt#a)F{nF35hE8;Unl8J4in=cJ3Ik~&h^B5jnD zGSVRRTUFvCEk)tvf!;&w_)K-Cl?dxPKQNqJr;u905403`anf`XZhFb|Kt;AUJFOob zjuV{`%ufi$Xis!4%?Xfx|3k$yIZv^0j`q9f{~lZP6LMHHh?QUmIy4cA&xqAvWKVp6 zTjv1ZNdn*8`e(d7iNdoA_kNtk?OidX*YP=PBt*dJb5srvO49S=_!fDBJ?-`1+gEqs zKWxjdizo2hTJodsqBjnQuMMw!Rck#(Ma+N&7U$5-YMUruGio*=?B~8 z=KYCu5*=t=ZKazR%-TRvMri&5(p!XLQ-fjvxm(~4NYV; zB|T&is6Zc*erFoEeAi6##Ym3*4vkEyd$}dU=Dh>(IP@7~2L3Wetgugfq znqLG3>~!W{Mxkv-@NOIrX9&KI5cc6MVdP;(w zmz*#I#h3Zyk0Ew`D(okASp3rLH#Y{g^ZD{*tCN72OQ_mG>ySrI*@zIKomp@j6b#l0 zk$-dfLPX;DRS6Nh)Z*0%kwM$r{&l8L*EO)>iK6}4uiJ@;DEp%yzZmf1)@u<@&2JOQ z>&ck1FBxg#>_AYL&AS9yB*Y}fdGbmV%Xvmu-#BYmv+_}1D_jPL-Fmz3T$BAO{~&8TK7iDYvSB_!0^LM>e)Po` z^*zh%EH8K!%Exe&zK4$-W?y~r#W&wbWN$=Wz7=6h7Q#i6=3m13m{IT!KiIlHuG#OH zSo}9Iv2&LNlwrNCPnZX9(+($BV8}4xK^8+*eY_Z-Ao`n<2waFpU{>YT7{d)WS3W=Y z!4I7~fA58htKnVg*xZX>{hPo3t3UnyUwSb($Gz@qfmOk)sSsg508||rr)WJh%q$S! zx;Fpb<#lbQ2}5%Von0+Fvq6C0M`Di+DT!t!rWMB*dczx#DDZuV;Z|*-$mR!MkZ9L; z1Q>kg3m5|511rniVlEECB|NaQq%&n;hrPmdxcPASV7tZ_FS35`*Gm^}lBSh(neW@JuXmHjN0N^H?+-#q8kCjP_8rHq7!ngx?8}PT z{lPR^T^4;UpQ26|EA1*n6C@bdo48LyJ|BH?`3^OW&qMD(ln=BDlrP#DndY{DuVttF z@L5^R=IZvKIU6+7Cc|aPVhKVwEsyDk^h%yykwebX^;4TswQT`$nySXbOfW%uWhUST zab-^^dy9Hg_LiAoi_6}^T(TSGK(9K`^)V0wUGJJ}efFU@%O`NGYzg-Z#}4#%lIF?590y0++-}lI?uSfu#nJLa^M9^xurv$6W_3 z4&4T?2;28vcUkPG%cTqH<4K6v{S*i229&gJPCrOlt8hfj?HSUEB{z_bNOlG0vNAWJ zxS0j(ka&bGM-dw`pg`OBE+kS)aL15$wK!^r$~8vv#An)A>7i42qT9;1K<0>&Zo-FK zV8iEnM zF`Q^~518aCMel4kX5+m)H`3t`w1@6)Lz?H?ctEaa$c?uXiboY++7+y)lp0YfYFBMa$Eu%9c;RDoT>5`t|57g* zf`YQolzqvk1D^9@8P*J$)f@HE-T|!i?Ba8u;qu+=xj*}i4*^*Ha}u^@7x-~80gotQ zOZM4L!UJC>VJ3TuA6Me3zaU{Y`)DWO<4=>YBYUb-> zt$A1!j4Xj=-r+Bscac--G@J`t!!*}GNDoVT>)V}}qqGM_vat^RpMOxz+Lo2}bD*yg zauC=}6@VjvwOxOwWH2vxwl}1sH{%`{W}I8@Bz`9yJ`J_fP&#~Pd+c3Y2SAB;5ix@Y z#y2>?2OAg-KgaMd=sa&Ndq|V=IT`kXMtl9$bTr*UpXp})+?kG?0e!Gg{Z4M!6)frK zovleT!jRgM5Mwt(-DBy{>2wR@e-}ikKoS=9>Pm4d60%Gue{jr`wG?ndr^w_-V~BM>|j7 zITxL75#*3=I1Lq~o9;{}D0;euIB3;v2*?WRsrSI32dJhWf`vT~qH(2w-4N(QodOe8 z-i1I!u+f^ZekY@7Fk1$cMWR79T+b`?8ac{s^4ZSYFacT!f~{$T8BGUlMIwlygsp;w zj5}#|>zy}itpi5U7_1~joM0eQkp={E`t&<{@>3kWUmsTZ(UhYhWLwl*!|H219adi@ z>uXq3S5L>)5E<%gyr-|o^))g#ekO&YB~{hoJJ0m=NbB7m>CP)As40>A0-f~Bjr9w& zKFcxSyQ~FSmpl6|%>t|$zc9malGW5%n2D$4>FI0cJv=vY23toVaA7*eI>D?WXYvt^ zhQ`iKzOZ2O>HN^knS2B>49JAYO}@g)PiCl0hCHia4T3~$AZHEQddRH7O4k}B4%=IU zT18E2t8~3J2w`WpyplBt`ToCP4dPH{au5m5K+`g9Ih#1TWCr1gcI95yAgswCeA{g(EKjvod5jy zQ&5SRV#f2IRkQl1Lzzl92BP#>Uj?~H(h`lc(8+4)Sy?lgDs)j$gV-Yc1QJQ46>YjD zPBfTcgE1&7&PvBy5tE2YEA1rRQ8V2{V;VJSQ8UwM$1uC6eaAf=AhJbs&LP>Kd*ibXa0MUQ zsb+jptNq{l9ZoivT^9I1XQsaxk?2=b^Xc|qqJ~M*bllFt? zl^2iq-_8bsCOB-#6?j%+c3?*#jD9}uCHqqW}4fO_+ zCxYT0Y>iCRXK!a~IG(n8jbLjDZ2}>7*MpxaI|EqT_2BzYs|P=$53(No^z7_RYEndI z(j#w+nj=*N5M@Nd1Ym%XKyj9jTV1#-0K`TKK1Kl^bPjX~Ryv~#N4xFBsqF(#VQyfj zYHz;6U!#Bw@PnJ(0d~MBs+S}88uxn(W~*Zg@i_@L)HHT%ZRGbslV_1`vOn-OdX`rS z#5+oMQW@HkT@ZET@*3%=MU=;!N_>R)a0vE9PiBO*vrq(Q4idr|+qW1wo8*?Kfw}Y; zsV<{1%4{*2+aAyHKjY#R1s-2M=VudMm&>E+Hme@bO`ITU3o$WvgJwC-Eqm{DwHoc| ztIWGXIFwg)U6}H!7mP{$^R;?NzOYKilqYcOJgxo@ELbBb#)LfK@xprBj{*Cm@A>Ew zX0O=5YvN%t*qdS!i5N5rjW*l>@c_*TzDd`j6AkS*DoUc8e464}BF9F_B-SG;p`>bU zqhm1FahK3Jk{$6)>0vM^Jf?v_?08i#;$`;gak%9S`N4CibyQGjCf z?dh9z7!!_YD|qwrbCd2AwF$uspx8UFx=x3%E;S$!wwlyjn=ePJWq=oxwVI3kakN!Y zEe-^Eel%>vw+x8akw9xDoz>t}Fd1~=x@3RRcBxKww!Q4}L|Bj=t79mKCv}?t8`!Pa zW|ErJcvjS`v7&iP+whE+6OI=(*j(6^H>LNR3?WrQ;1k)cFG6%E4;5E?!i61ub$Xe) z6qC|Cj>E-w<5j^?4hR)?dm{#Skve8vUgG6=MGZCGdPQ{!KWne75A%b6(N(ma6S8j# zl*n6%+Scx*E^I=(({1j<#XI|O@s2K_W9fk-h9c8BMsj&LB{GE9@BIAZGecn8(G5Xu zYXnxs7|=9QCyNd;Aj^(njQK(Pela%MnCE3@hrQ5JLoo^ftxCpdd>NP zeisSPi$%87ArtMg%RN*a`SRi-3F9sWZ_BgP4o5qRxK7Cypj!hvs~rXbY;cTCR$%GXH+i8cx^ zKa@jahCg%pHLLJ5UdD^mGhGr!YmTH_=i!$bL=j0bPyeJmp*IaG+6${o_+g@sMA3R| zD}`P_^aT{6*LlYCk{|mhcdtlY=26#UJfQuOM;N}+APVl%qw)07W`49eeH0kRhli(+ zSVmFTMeQSwj8w#7#X-m!PB@~ma>&({d`)H+``pqG#7+?pe0mZH`l+b&gg%-C4(j8Y zPemCIQ)NU2#>MYxCF_D83*nsCYHbdNZD0v4Zf- z3e8SA<6_Vh*1{#o8wu&jo>>Pi9({M;KvEQs-$hDV zrLAqwmEgDgBdE>)3EIO%Sc$lI7yyp1As*C~kpcXR@B2vqMXKN}N2pDy5zedEv7KYi z2;Sn)b+@PT{7aNT^V#G>26`{qfkAfU9O% SS0;OZw6SktlsZc9hy3j@~Qrw8hk( z#kjj}8i%v=fZ!1!9DVwf@HYn)bxc=*ZY$Nb7uG~J2wCVGcJP!-JdIJZSxGG`&zT(E z(vB4~9e*MZQ?$L9UYtx(JBxY432x#Go~TJAQ$%KJ5p~6g>}l%XWcx)L>5c)4nS0Lp zr3%eV@S9LwoL66-mOSgWia3&)g$qSW&t}$lz=Tmw$w19~*D)h;@Ji)+k2RqyVmsU8 z?2Z_Sf%B@V!vG!bh+ZF|?mZqP=O6McYfxViiYf^ z)k~N-k)Gu+=@v#_1sj1Oly-S+K=v_Y29tgd$CdRO{zW4`h<^5T*3v&~u+0o$%rb%n z1UtMv@MPy3$w!z0RLh@Tz+!WGvv$>tq{yybn91mVa`HgFb`Y{yA7XuVrA-yjL;Ob}Hc~Gs4Oou@a7*@T)>&Tc zxaIyq+0K5q)^T@5>K*5Rg?->4of2=>lNuR|L5Am)f0u^s2M$c5Z{Q23>4~{b8>HcG* z2Vv4F2AS;`0a)^aVXBY^tPesjiOzmm`P474o(~m8y7|D7>mO}~Cw)@!KBG2_$FYrUfs1yzMaXO zO^Vz%9I)p5Cb`_RZviG~Kw9^EbP5QF5i`JduekLjgbes!BVyF%OJIhQhG(D~&)noC ztc6mwQ==z|a2+!3VrK}^qd!yNa7iHb5}?5=7?W-!X+9CNu$h4ePyI46AWxT|te$+c z_XK1>kfonGB#w*apI9kZ0`3UFxd-NSlu4g zkM)p97g9EvIMWgpqlr(EWA^KRWjr5yyBvT=(>aUlHsfvxDzaCkxvwOw&rgD)u-)GUdc^!;{*j zN^JfqXO)zWA`A4%G}$vnFo;i!_%b#DT@B>NoO!Jri`QOknEEk36%$J#`vN`!y2Ij$ zfN390(ND;L;Q>pGY6-2ma*oNC5|^xjy}RfF(TbdH_7Jar{`II^e^SyVelA-LbqR;R z<`i77=Gb@YT^pMZm;%W&4zs{|JdHHIPV9IcUH=5{2a_kPYpO7)9%i?pAvond|H>11 z!;L)56FUeO-=q|BmU-6pU9S?J)wHp0>3u$nbAU{^MkW9XUng(eM{y7Hv@nC8E1VAV zJZpNGXOx0Ga$X*8y%1MO4JZL(jM&iOUAgaoSEDQ9r{re|7a9wVd9$YV5j%qWEq!pY z?(>!zpf~If*ebh>21d~R5i5xYSVxXEx4BbP%O;tEyCdR}X;v|t)gyMng7^@N1(iYG z=@|*$**cF`uUPF-_byYCvcb$uNhFfv6eQ;kEfv`64;kkL9-@Dnxh&Ot+~pTN9!K`F z?R{>K6|vXmNON_0MWzh#U}iA9z(qEX0wqJZ@UdKz%Ej?U3YXBEL88*OnePP|y$vAE z%I*tE*`;*g@{g+Jo@NG38lKoVPxE2rainc_fLsy-wXifgK8OcJxEQ7sgW!MQJ`p&@ zB$`m1oo#~e%`XspvR-oAu`Jy!newi70ZX-^Uy zBUxXNen{(~ddqqCisA1|a%(-+PVZo9o#r>|TsXFKipcA89nyLEcFd#87HFAZS?Ft_bE;SS(Ik}K*X^{#-DIZX=nwie8&32yH zyWKj^E{V>swVe;`TugB+sqLLt1lX(dYj$2)g3hmf0i6e~uvh=T@&F4j@Clnxi#Lob z4Bbl&*e1{d&f>%r-6BKmlIS+bpDku|V}`}tWy7yx1%ZI-Pwqo@V}ro}v6n&C`lCxhErJ)r zc?~gwSj|0ZA&Wz*$K#ovqaLIHj14N0$$v`NqI=BJeKo_4^URM-hgGIw;rLSbkc{M#mj2pP zdfv{PVG4To^szJcs_+f*3?k(fFpci{&+EdmD|+2LxP@-=2OKr}{auQD50=!Q>&h@P zzTcl(Bh$Wj7fkOMaMCW8U|?uEc5U;MU404C`^~JRTjS|EdQbNex?7q(*fwh#Q4eRZ zJ+Dl3wAW~#=qf?6LnW6BUbENLB1;R?v>IL*r~P8pt(BdmGu?~$d_Acps$U~XG~}G#@#yb6b6w;x(+0W z?w_!5lZ;<-APhL4x}jcfIC?F2WD|mc4M@i|DY748_@rv?$2(!N#M+Yp(T($}b4{&D zZ_jD5Gt$jJuW9xuSoT%0OTDS6cq!+MvO*m$D6kY|$VOjB!j zEW;eS$4@i&My?X#0Udtl#IJ2b7qRFGX_ zA0{3Y{Y{X4Y4(l|NDuP2zzHp4V>&H)NHLOPu z@y*E{u2KI*3+BLRj3h2EDW(u*Gyo@60``4%MNhjv_gXZD6-cq4_ zac+Eufdd!FSo&hy_#%Da8BCl3hTF%-T4j8X*o8r!xSLaJH`$s=0rwu?)4PuEWcvQP z^!+a5lg!MG@7nam8{03ojSrR386hSXq8;Ny?cHncad<6W8X65S&#^HSS?%z8Gk94g zyp;09>+kn_L^ZhV=s|q~`JSZBDNPcDS(t4^K0d!{Lz)N)gIcHcZ^b)qZ> zkhb64IrGh3-8Tq5c}P$A4RSwo%{O*)D}534{0_cgIGYfDczK%C@C1wbL=M{Pl|EB` z<dK`w#_<2X>?ssw%L!%7dVyaiM>b8HeY<~xClx0yn5B<>I zG{t5*w!H3c@BGOlh?`7mc)O08yM7X^sM(3}(4%>nKJ`9+`8^ipB~KAcNTBPYJ}c5D zVp~rOeyvB~*P!yvdd8Y{-$coTMY$x?#@;#X(`y^+ZmROcr0)n4;X7n5zMLkdZhrGY zx)+CM?=tJ|N4wbtCFZ~*xo*C&gCV^AzwA$y=glMT4 zzbNcCV-D4xvEQ5>ChsxMsvw33JV;qr;dftN9Bd~JTXj2EoVB$=fpm<)OCGx8LbplA z(g$@gM`61@+7-OV5E#EU?FhBgS0n7CFX2gS9H+nlJAg}efqIcd@x|ei5E?q|1X9#M zHoXq?kI7yhL`3z*FVfugqR=gd*TRdbi_9qP0S;#YER+32lGNbvE%?NPIc|sna((JK z9i$G_IjqCMcdxWsMbAR{o1j&^;sw$w&ea3#ch@S)?OIh6TcgC-N@F2etDxUsg~}0v ziTph-v&!+w!i5aQ`mUg%*RY}fo2YUe)5-m*r^xm{UeV29hw;n^4GzHb;h88EsS6BM zsd#Xv)?TV$7cJs!+x=8(PqiU6QnzZu*{=ras~1L3I9qWB1*6X35flsENJ62;R3rB% zO(2uMO%uqbJ!=BTohI;&uk^gIP$+G_(f}=`Nu+NNsqqq_+7mx=lPb3|@b=dI+3-~N z$9^|-e;1hbw8}x7?42hCm0pbW2SJja(%5;``9{#lpm){%%|c^3IcV_48|tfCWnVu6 z2vVI$v71zlvX75d_7T;2J`KC`4?dm1<@$q`(K12Yg(Y8FZ9Fzd^ z(FMEd_lVZbWVY4soj<4Fi}~51-(zO7evj!;N1HJ08vA6gP12vc!E#1z5li7r6nZmQ zv)F!>CW_hGKgt?B#9SRsw-p|jFM!I9?c+h*B(lox`&N2Op6~G;al(g*R8ib954~1m zKeJy)0BcM9^o9nxF6kW4R15d0kUruQlJuW@9!TEj>sZ)1+vBJso<=-vOwDj4O{xXbT zXI3{-fazn;u=GAkAGY80nSlAd`y8Y6&v;_ZFtNxxTVEPb^uwqdyGjqa(^q#AfzCPfr$ zQKK0ujlb+(?&l@Ur#vqYnH-x5oYf^%w_C45Uu0Nls_OGyRYbvug|Mw^@SIhtR%nJ% zY}cmXG0_Ky#WLtcXnc+zX9>p@y=|a0hyhkH`fzAA4zd6+Foc8`IG~oovmz|L+n_=_ z&Wq=22|Z8S(l19#=WlNbX`Zf<=p=gVYWHHrBO9O%lqUEF)j&D=ZUIOxe0z#`I#4{N zaP0WQq5EVfpvJc>Xd+_U=9_0+$^TEafv{g`6%GgO08&_;hfr*CjpO#7t#^i_U`P!o z#~~0iRc%V*Ge~F0yT43<1;B`dgnFDuPSw|N)Irm~QH!ga2%Xs&6C+@NoRi04*zmlH zp$WPr{>kiaI&sZ)l;TBZj!c|hw|F(g17`%s<+rt9<$~b?CUcM*iA%$wWj7R9em_|X zko{A2HLaugGDQ|+Yz&H1Rn=S2*Zgek7d$=AI3_Fe;7`*aIOeAz_7Zw&lR~@QCs2j^ zkg$vBj)Z$e8s$^dxv=pRM8hbHF(7qI6qWPyG#( zHz+@`QN>&;8y!b!&$T}PgHwh?lt2m)hMw?_jTUzVEVebh0qV0IY!cf zLp+>ZdccZU8u;9$%GZP6H&1%*JoSP-$?sol5%Brr^2!4|N(v@?aABl0G@u0-6h}e3 z@w(=UmmsUKFHipR5k!N$LVG!YhEE!rYv>07H2raLoc0R>X!=Yq0W<^J2EC|DRBFHK zQ#JM_EGl>jg=5EHCV+-mU=<8hz)DJ|tFu=mC|OMMF(ER$Jtj0}+hgky6=k3we1IkP zuAX^~D&eUI^;|3vieZM*#scB}=TM9V(r`2#3jLTJiVDctpn-BqU_+bIvEk@n zK+)+a=Sb*dq}x9O;RcUif(KBz$_F=`WtI#+`3TsRQ&wd%e44sqVduJVcpcla^ajbH z5Z`Pz9NE`E<26RQdMP3=$7Y7N6F_0YzXlN$4Gr#%d@Y%!lk`1KS9AcTzT|(-YhG_y z&QyeQ7T0c^mT|TT_@XzFFyQ;{pxVkMnd3s|X)uwWG#T*$YsY0x7WcLzC4K{OhNn{% z8EGJ}&{Uu@b|aODuV!p5nU6fcatI&Y^jz>uZ+uHrx#1MKLkeoVe%G!%fD-Bc>33si z$3rR>=}D82vQN?MxZDSX1SusD+9x2zF*&YC($aU;tWxPm+B-8mG@otn-o0FVAZ>`; z4-Yz9`Iu>iMNm_Y7VjYJi zd>wFufSi13{luqUHC}%gs&QqJIA36uGhj|OPkiD7amw(~ID?#)hUR@l_(1&Ohy_*K z{E3#<-havWee^Yh-*|fK5-U>soBq-S#(_au^9KU~;vOeyaEMPF=JQRrAN`egy!}@{ z`J>XIV88ZIsTud*&#x;n#7AE?vf1h1fN9d_1`zb! zqwKIb>&l2^97d?csxCU3-b1&+MH2f;W@gO}51aq?M2gSi$toRO<R zv8s-KO%b}SjJHUei9Ta8Ts~BQBnco6kyvoqLbyaW#^cxo4oS{&vblmk14^KZF+%Cq zknrLhZ3k55fhAh8HHyOk961Uc5sja<2`bLlCxKW-b{pWs?QO98`PG%%WK&Jd|TReCX)2y++H#i7lPZ0K65N3P!1bLK~(y#FlAPT}X#zj(yvS%F# zkbvYu2T9ii1D^)h0ta_8Fw15v~ zyb!G2oWy*v2nrQV`_lf%K8~+%e!pM}Xf`6fq9a7p%3-2sphG!X+DLncr5PE+oNopU zE7P!Yu;d>qDub|=6_GRMh!^YjL%QBUrPsY4uZdXXnG&&ZDp=#e)>iN9CcOg;%B-_o ztOwmu=G3vD*x;ML~kVBq*_IrNz{r~!*r&m2(gtLcfVD6!7j?4rV?e%Dnb;1>kDO?R= z+7J$NXGDW&PKuM_yB*_9hr~!XdQ_CqEmqs<`;-7c|1Wd48RRf}iF{x+$OB~vu|M~{ z#K0LLM`zfn9kYu}Z1256yz8d;YO;tj{sp?wq9YebQpVhoTobuU-7Nw(Y!z%>et4-` z$$<_@7G{EhVDr|TH>^kYKyEw{q$Rt~H9KATp(+VCNytEIf_4(=zyWfIFs~v4#~Def z7KPSQi>Xj|3TzmK03{ONsckX7Ty&N9jnX8p6WtLKp~C}Rtp?k;K}UQjNwfZyxr&`g zEJabyEuApquoJ5nkC(KbMU*U4nF>V`e_r)`MR=r&8CY|o6pK(>&A^GWni$3AtxR@K z4ZQbTQJfFy=VFlHD50q+^>b#RVTU*JR_eAAEm>z02H0>C1{v5$RT}WqCfbJ)SSz85 z!GvTNV3-6d9>$Y^t)cy!v?H>gk@TMDsh=8d{L3N_|GY)#GgU>(m)L#=GhLr!rt7oJ zbbXGQ#-4-v7NFo%;H-%pPTV?YB>+#tL(MT{I{==a9*slpmzsswAh9}I`D4Cs9(!{} z_IRkv^Aa9N3}Z~91(h(C4f_*~wrkKc0AQ@HK}*T;1nDXb5i!C^RMDrvU6Y#+lTgg# zAsJB6q$O-a?!uiaV;iU^wnYWxO|d%CaZn;J``jeF+!5G8*=*pXi`nw@Jz?mFiR>35682&@5d`%sGtSGil&cmrs^BqK>7G2I9a* znMF;ORO$rtA2sq1r>#+FMOI~!Ge^CEjW8BBn*oS9XY3tM2{-~FgLLuTLZWKTT|w(I zQM+r95{B2!-AbYnoUHccV=Xq`zg#Zqi}++R*R^1n{WY^u$^p`l1B5|`wB^&0>44Pr zbFgqE9HJNFp^^4r8HfIA#Pgsh{$*n{P(ROlF=MxSN(LusIhQfwMs^NAOx^xuML_e7 zIz;@5>QiIrP|zJjfSEt8oiG5Qe?n>+y`^X3Rklwn5^_xv-Sp49v_$SkapV*xLcojL z<^SpjN7%*vGelyv?0OsHx2`}6PL#=DAnsD^5=dHTBgh!K=>+MD0NK}TpmIdQ9%ttM z=thelZa*exg!^jFd3X6?JP4~pmqj?b!{do?VWTpA5&Lycho>jhcdp7-?7JW`1<5e@ z$I2n70su58_p;HWks~%@uH~`ETBI8-(+(L8LiY@_R(jTDJ53Th)6&!_^FlWyq17MV zi=cj}IAW;4p`zIc6Hcs|66}WXjM3puRMfFDkcFI@aym82?o^;@OV`2FxEydg2g7y_ zLilIQ!xn&ySLnN|V2By|xNZr3+yYJqeZA2l15_e!sQyINfsTO{B1PC%l1CU0yG2uO z7?L=h&AFj#X25?vMlVS|vstCrr=L}-+;vtx5*9JNo7FhyWhKvG9HO>uR^x=39kc2j zF{{u}@B$L2?f?$e!Gl@hr1C%?kP6IU=8UW+P!-M*Ag}gcSX{<~`n=-Eg+&S6I1wc^ z1hS^XO(GX|nQjm+8>Tj8G-}s++DDfcZeQtX9pW`-y)*NnZ#0d7=O2p2y+38I4umf| zV^z=tU0@^+&>cN-LoYE`m*_)vPTKb87o=uT3%o6edlU+E+XFwOExaesdPtC};wn(S z8ogEhCa&%Ai=9oYCse)^-07SBlh|Rz5HK}ndmDi5Pd)AW;ci?*msKFVWvLpV<(;F- ztZvvIm2)}>_4W0Uxv!km&3S0SZHSkJ^&_ZhS})f^W&ygl6WbTl6A1;}4q8g~MQM*yE{#9amq*>X039bX-S^=BE6s}`UAZWf$ewpwK@dzLn9&!l4Z$5TMy9L~A@4lb?TtmtBJ~oJRZ__dW@Wb$3oyU7-2+T)Hsa@t%tOv~l z>tjOL7*MX85cZJ?%jYH}o_5G}^b>#dMQFBeo(#9)d43U-lHYZhzRSA~*Jt0w+F2Ca zVUK5s?Olhug$KUpI;Qx}6SaIhuzk22wo9LI(MV8jKKLI$z^;3AZ8(RQ%{V`P+{bnI z_*9Xff6V9o?)g><9V(hn`+RBoyvpV8d4}g_`y9L|NB_8d$#dhyXBKPs_hIcCb^c{c zb{G~U#Gy_(bg6a)l+V%?2nXV_yPSdgL?*Nsdd9^`c&US?v6T#r=xNzUnT!=6?X*0? z61hI-pfYbSi5GoH;zhJLTtwo<6_YG3UXNgib&(B{sJG}dHgXl8TptSi<&=04LuVvj zv~azxa7w(mGLv}GQjjwDi^%=%-r2;v-C$o893`!W=>AfWCg+>9iEbCuQ&Sg1Ox^ee zDG^lqW4%L!0U zRx@+9xx2pO{_Ih~{mOAPm88W7DO)&>hpllt0LON)!#wTnVqV}9XcU+yR)Kj-LO)^N zl7L$VzY^xf)|+$BXE4u2y_hf$916f$xCW6#)3$KJyrn7NCI*pw5S6b!Nnjwlxy9z+ zJ^O5NqWKYg5dLQb1Xk`8Y{CdV4Qo!1F0Ro-iFf>ncd;OQE#6U?in8A5)f7aOE?m2F zm5pmX0V7~d*ny9!DJ)$oG+6~`Jg^L!=T?IW3T!u)?9pY?Yu0B*$+fiqU5j&J7CAU0 z{6gqcd>pe{QMG9=N+QluT~$|C=pRE`Lu?*bsWQ~b7uF^LTXVmP&|{hjnoBVv+`&1< zqgSeztRd!ZyMe-5l7neFT~Hltj%eNvMOvi!?7K74ymP5qNa{?8UH>3JJ>6f;D^UbB zId!jGi-gC{frP2w@(>55!(AQ#C@T*@2^lWS7)61}z0q7P!EVVUWmYv)2ZdM0WGQJd zmYiq~ynp0W(R^9+3_qhsgGEz3%#;Y5)e%&@^HB6-XxaG$8`heCg_0v`fUJ9w!LIuUXY-AJH{%Q+KcQ8!g*rWQL-IPEM+{NIex~ znZAaC$!t+e2|aS|P>M~v04Z=P2qih2O;xHqJZa9FKtnL$3!j-1tYgh-}oN6Oc%N*(s>$UsF{KmfSVpD&={eQm>rPqmdD5_b~uww8ZXv@RXP2hg zQ@XnR%n~P3mA9b3%pqtD8od(ff6Pz=dwC~tn6zkc}f>JA~DR6l;A1V1Hl^sF3Jp+CFT)GUGi6~J@5z)vwR#o zPl;8C)n|d$t8iRj0Xwy2F@lz%{E#j`;eYa?=hW34m3BcwCvh>K0H)AAXl{Gc7Rd~6 zYHq`MolZxBV>81jN{lw7}&zHx2COSxyF1h-iIqoZbfikW!Bv_`H_2&bM!=+Wr@ zz$Z;}REoY_J`D*1L6|vxRhN>nKWZX;>Y3N0Ap5hg&q=S7 zMVgrbm2Lr4YR(%{A+j=~W@KQ0MhMj^u$+>>ju?7no~U+hx0ST)Jf z6|`#ZZtqUFw0^y#yblxn%UOV84c)+OvwwnUuW`uSzi>sk7j6!UY{r4Waus3_OTSge zp)bPF*kC$1+58jCWT*yEmm=t}_d@@MleXFOTw~}-!@XX1>w`FdOh-m;+xH%d>|#FQ0FNe`L6t=);uLKQuD5_NE4XLy!C8qWnN z&r?)vdLBLZd5{pNndL3vmx-rt8;Semx`OY{G!7(+%D zcGAc}TLZ+ym)f3{lG&RD^{N>kOK`^PDcYEXPFZtmueEc`qh~@)YIKpeGvMe(AC_h7 zpOhAEdCX;7j^18{k~A;`V07|Hpa%~ESvoX9;mm4LnZKDpc{xAKCK{i;V@vH zr)Gn4BoS#i`nO=`z;kohvF@?TnbD{aa+|PswhLhpo9QMH3t}7*8qzhAS|Q0kvNV4N zd7UWg7`ln=-5<64q~ z1B$T7XkAK-ic)Ok(-}o2K>c>BAHSGLB@cVdqB$6a8W)v|_Ppe9V6g*nVgu zt%k11U^Dhm2Dugk z+3f2keZ{Zu8h#kS>-pi7@XPsOR7dy$-6CoKdG#sC=V1nakx1}ZnNdJ+ASVPLNwYj3Wk}2|x2eF#6Hj^8D zkYaHms)RRtw+sg6pllf$F!N;d5~L6NG3NXZ`yI=g?N@+4_v-5d zT6{K$ASL6h!ubPRvv2kzxbx$Y-~1q7RYzd@i8_;zfvU#Gc#zZxb_Lm)pKCT2TzOBi zrFPk2z*ZL<`)1lznThT}O7(QVs4xp~a)~HE>ZC)EOYx9LE@u|1BdLfhfS#xjRp*A09Pq&1 zpi2k^8{vH@XQXsJSw0_W7LQvs_v{^Hg8k6k8-6lg&YF84W}6Z5e;@%8JX-qtdM4EK zRa$eU+WbRBvKe7E=BqfYtIjqf3)kBU@3a|N3b9V*_55F|`?yZr{7zcS1MXSn6%&uj zVra?K!RswfP-GtCK?C#l(UzzE5~Dwir4F1~D&ygC6}9fIm~M6@hyy5Ud9|`Z90+wXmkuzh~Zz#!~BbEOixtWFI*+Da2X@FRLqWgK>wC)n*k znqOykW`6lD&o5CJ9}m5>Zttu%n5-s2szLknT-5|3Aq`+(GQwOqK`FX=*nU9$TnqIK zlw8!TnaXWEG1%@jW2ttJ(y~d5g`Oy`rZq+w`xpbgU3BJ`kFz0_j*bTmR9)(6X56>X zHdXc6u2VwCoav9WGyT*soge%(4wY<+^&TlEQ0L(WJ!HYi-_zf)Rnu=dP8hJcAkH{N zUH3P^rnvxe92brmKn}tj7vBhFP`bh(VODzBBm5~?RsM=w#!MN{v}0T_j@hW|2p~t; zq9$rVPAUU|HK`e)pB|&@-|aEBhi#9Y?AW@MqyGR%WK_vZv_pByo}EY@2Mg?KjI{+Ipzj&#BAGD5bCQN4PNI|# z0b=cDUDDrd`FPvGz=9 zbSTdh5(wBkdij6t7{^v-7UQ6OGZ?2l*CLtdiNpn6sh7Whk=z=XKE{dA1ofyBUb=Cq z-MCDtjkr1gH0f-s(W#YDR0;6!QujBo0bpe z^#B5p8Pd7p;jX7cFfjOfR6q0GDyn5Zf>?&9Im3l0;`B;hjcXOdD0;qPn5is>fpXY@ zIV4ox6xLvbNWO>2#>+Y{3Ka1!cql;W&mt=UWCVDkFMSNuM7b|=F3#d*U?r{lqPOFP z2$@EyuPxn_bucPglhCujl%z*@GYmr53LTjA3z0+hmqS}}Oy_O{5YO2loF&HvqBZ8E zJcq~C3SC~NzbmmNMqdFj$>Z@yFflBrJcgAA*^l1$cl3VG@lJ=O3R=6Q!y4q-2ZHC& zFxYkkakXkd5<(`YBY4(O~* zr&hrQw4eEOQruMsDjxlpd+It6 zJX9R*+y`0{pKiH}GFr1e8>P@^u2x`Jed^hc*2u+wWn)c**6~8+oqrGPb+%!dSljXp z@_;k~ImKL4T^9V2-DVJ`KU_>A5c#cMJ8E)^Fk+xzl@UfReutDv^KcTK^?otk7pRcRI-j$ltO_~@e;N^;*@C_bY5CXIi z(qNSkSp6b`EWGKVWaBbvEvQ*}8!DXi2HYvne0STpGMEl|>QM9O?lskZ1F>knH^anf zS7`*@-@O{1o~9bCjp=OhT#XbPiT};&c!5Q7oEbKw&^MuZN*9dT)W>@L=Q96CrgdBC zoEjz#b7!~3OO#HYwjU{hpHXD{N{@H`>rNTVC>jCSI+Kl`B`54HcYhGX;tzqD{9&`= zpFTYvHviz*+REZm_h%^Thp_HOcR#8bLSRKOf#GYDHByy~Osx+%={4SAqn& zi#UUh{poXcEcJZPj+wAvwqu*yJGN=Z&QBc+@^^+E`^0l~Y!`?bvn@yy%>OKEz)0DG zw4XtmAilll7j5tPMfUvS)N^Qa-4IQFU4}S#KYFradyG-KpN?`NhHLk?(C6Abk-NWz zy0W9-lo``{z41_hrP_8;!lNQP@Tn)IpBJ+a^y9s@>9SG_C`WX4~{xq5F z>GSq-z(FLAOMmkd@Z$pr{BdUF{6IbYrhCEgb26e&?Ppk zVC~MwqQCej8Zu7N5PncK|HO*h!@|x_XuC;2bqGlAlkBfOswLLX9ER)$Mv!+K7^QH% z=Iu7KkWu84)XWjVVc=~B<1PnVLHN__`}`i6M$PEEQ7Aga7ezMkI`~W?@A-SOA7UYz?C;_Q~}X&pVFRIb$2M}dXO&~YPcYkt&bvY~QTt#|BQ(5VI_e3;FTgBkPV!u3EdGFrqy z7Xmh|A!&j72MHVm-~>1<%rwju-f4cc6xNmn*Wo}nzZKp5*z}E8PB!nvj0ns{k$7&6o!?IJeQow|Lxv7sY8L55(*c5s?1hT7-efOxf4bwL2OoN-_1NZ(*{J&r; zsS1rS3&)ALiWoi0k8+L~%SM_X>qzcoi%~9^n8}YOOX0Fd0FFI33zOU89!Q8(<^NY6 zaMxtsTZ*<0Qy5AWd6}8yt3d`ni7k`rOhsejc_c=pGWItcWuhGkrjDQaOaL?zHjp$F zvMt;Y-*$tWqBFFNJ>qy-Av{9IRO7EOElfC5>ZFjRToCa;m`ao#Al?3miik^;a;48$ zt{oUnziy=vWG2d`nZZUata3cZ6f?Bw0vxh-_w+uXZME<;Csy`JwUr>GzJC&c0kF8e z?u?x|r9&svxf`eiSO+ffV~^oWr_*(3Z@2ItbY`?Z8ezXM;C$6|4ML7pmo3D?@LvKlNC{J)&V;=FLz4 z6cnf&eJxLv)wP-;c&Pw*IRNCM&3_@5>N_f61Gu2;&|DT&CM94QvVg#w3h#M>W|N+0 zvLFVh=F~>AppkfDVcS4zf}}zsUo?i!9T`O}L{iqv_`St11(E=z+>r4{(HsFrWJmb4 z@}(32IT0zRYOYS%vYC-sLaZ9lX$gDvp@IQxp9mPlV;ykp*>c;nz+s#9d`{X+Aek)6 z1#}@;lw?Co6UXPCrEa|uiOo|(u=Uab*_mk~iI*ZFm?V4JbsWIkt_OdMzPG7fisrUf zpD3m0!vRAIt2*6ejM}ox<2-9Nza;h+JDh|%DL zpbUwa)76OoU3=&bQ$^kZ$y&H_GB_|;1o;-xkK!vPuX%3?qX!F!D+o8^BxELXH7pQ^f? z3asPsrK_XcTll^QUxtIoRL~%&0v|yv?@Bz*NI@apAQ>O4-}GqwYl=c-aIOhl6Brbu z5`>D*y%I%duUGU+C{5jbO!rcRiST~>SgG@)H5-OoJs#_;z2IkMk#QSA2`BoNrAE<-Q6!8AD2I zN@m42@;o}AFD$aG|C1WuAW_kC$dom&5#ugNb%hy#dp$|xz(x=v9D!LCi!vX_ z$6%<@rWVyhM|*8k(p<3sC4jD93cTC#kZ2=TU?9gNY%#G6wqdzA4%t>?Bz#!{!k7J8 z`xn=Us&JE%d{+&aP!bIkEypa-jF(~}gATUqij56&;ZSNMg$zfEw@?^lbQ{c}8x8_J zVguRrgJ-U*gl7q%F;SD2Hc<=;8YWC{hE0m9ABa!GF0?sPO>|<2V+Lo84;Vr!aBqmEKorhxFB|W0GST zew5x+c}o;GNgLoBKyHk}WFcFZ^umnVx{4@aRnTtt#1kxK%od==A&Jg}kZ&&Kl+$Y3 z0mvEzS;>Ko+4)7uwvE<&$TlrMtnx#Ih^D$xsTMIH5@-5u$NAt^&L!&YF?2()8eh)!*1Wd)wz0PH-8&#}FlY*ES?L1oYUe zpuCE#BniWpt0L0@0x&q12LJ_c2N1ZBL9CFx`yid*401(zcEJe})wCjkCgP^9ohxK^ z=@|mhzmFP&6P*IFF*pLQc^jLD2xJkv`8^EE! z3U=Z`EY^jih*ibpA{@4LP|_+H>rBwp!t$FI%5|ABTVoXSO{vn$4TDbRlh`>-bPk#< z>{}?3>Ld*?I&RV*#=ve9NiH@Y3>%tX=se7zI2Quh(NX4GM!0^c5ZP|aUU@%^M#e5p zpzr*ILogi3<&=gbGXiB4I@aw!HPgd&hUIJ~J(LQl$XB~c<>pKpQI!fPJ^>PZ!u`|@ zesp~*JyHRcKe?W7*3~wp!r{+5%6h1%daf`p&N_+6^K?P7r6C`(zr!&>TR5mXx|s&+ zL&fTZqvZ}2D-(dbFB}hD?Dn7caBL@h1rp^QI2{jFQ1K4ABeXeO&+BQFM=8i=+k#{U zI))S{a1j2XVKar1fBl+f7LW5HJi8_<`&JNK@(1h~8OZx?BPrUuutX=oHn&C1VN`%z zI_=RD9*p79`^HuQFOJeUTj^X2+T^oLc#vKBebB&R0m)Xl^0Oq?Qy(Myji?sN6Gk(w zCZ&<`L9QlHY_=L?C`1D#0A|(%-mAwaD-wiwP`Lup1|`BT_1y;95V%|DE^@a6!c``g zQ5Dj-aOTI57~IBPye_Z2tX4rZc~ld{3ROb`bbOwTmr^4B?vUJ(D5ap)NmxEleG3AWBdK29$bX;q4(H~b*M ztTeFY;PvcxDEjnYd;sKddlLmwG=r%0z)!sG(qC8=!Fvc?_KHEZsyg%_iUA z2y7zD$lSx-Fzya>^@`YL_1RVUezW=PZKs4Vb{0b1u@)w<1)SiLX`#h_pLW4oilWYO zm2}g20K%sRGT?8HV9HevT25OEb4s_=SK6aGh4OKl7#nI9qPn>qeLphScSe%>9;55E z<8gFpAdhVuhUcrw{H1xXb4N2N;{T23~NHW?fjmt$lQE3(5{P&dI3pgKs+Je0QjE`;c5$ef3FC zlR>)2k}H3HvOGPBg#=b;FZ?sSW4wH{e9KV-ocXhUJGU3y`2ffEoS5_vaOz2UJ1h>3 z{S3CPT*aEK8gvK;Ab`tlRnI(IaI(`?2RP0k)xE`&*Anj8D~$JqTmnBd=t%cc^nUX% zzg2-~?E$hcYz&SjBHMmC=4h4jE&P+K8oqWiWY2cAItoehaEkm{^K0MQJpQdbLNlo* zxW=hV84&?$_!Z?-9#%(wu_yUG%KqpbFyJ6J_`C_ zn;#F%{B`Ot3;1~R*zdF6|GVaCenxj@mcqvX1M~$whRu)OPKlrAAHY(4-2?!|6GUtd zx{+j*y!gsHAMgeZQE*Mw<3N}$H8_Xd?cUEZurjHUhY+ZH{e2JBzM{Iv%e#k}))5Ag z-~PrYySGsbAG_g;k}UD4%(XoI;y1T_(Ux{{AkHKgwJAJO-r+B#I(qTZ=~t;2@WDLB z7&4QyFari|;b#t+O?-5Rf1iMJgbN(;IX5;hQttWgxus;rACW_6rLrRv>%jBVC=uNW#H)95vyxy0VkKqeA z9!T$6CF?0hyjgv4$QNLeKtvb*(NKJ@I47aBQ39BmHtNa~7pTBzdrlm5=ga|YS(H_e z)2uw`F@Kp1I)`=S6%}2z-@s+`RtOZAPY?AR9xVUmpd7@GbGmftW5mcX043m?PJr8t znYCD-09n}wC=Qg(-%4A%FlP+LLxOvgKvXqZ#afRZD{+Y+o;y+ix=VZf*>Asc75xmC z%g3nNUsNk4EpirvvI;v)>fVCMw9io2)0oHa4p#KmDo@U+hcTD8c8* zP}Zy~6;Kg97%xowz}|hf79|w^4-K1rw)nI#`v)0e>K=F<9_EgPGZO5`EhiWWQjX#c zpyC4Dp+{la&E0s7Y2x;(Ie{YA)#2^7ekK;x7N!H6ekM5AX7#zmgvdYgyJgA< zxI`$*hSGOfH4fS$-d!6jHRM5u4u#gqs^ke5T92quBA%=Tz48HZJD@#U5uQNp*0^Q$ z=PX^MXb{a|zDyiJ7m@C0yyOmsgH7JFMJBNMZZ~XeMt>e%*)}05=c#chs6MYn?MpZB zb@NYTRP6Sd4YH8FCx(qg<9kU8aw#XR!ft+m&{($+YNa{OxF&k|jf^aBJ;r?wYKtvc zO0N$VMvi*Ucgnm$usaurrxEy@JV`n3?%`#delf0d)B59T z%qXgp@MzWb;wLG;9QjIDLbf!MH7xc?ES%i+ppG)q*x#8y0j|XzTRYJSaE1=|O?gRU zq{Y!(wJVNC5g& z8gcCSqnNa=Z61DrybHfR`er1ACkldsSo{p~+BK6Uh{`7zetax_j+?$ZK08nUU@ z9AU85fKOI==ckSyQ_B3x6Aw*B?03M)mz8h()X@#_=V&<|y=AiR=&d(T&Kj@XK3V5@ zQ5J5qq|6(%>s~VZ$BZimyH1A9-M`MRv*6;?9e?kNRTcx69xKX2f@rW@He2`n(Z}m6 zR7OEzd6OQ5Ldv*cyzwmb(C_wg87hj~R3et+u=%8|H16lg3A2z(ev)iGVYnYC@8Jl+ z%f%}viym~`CxP|T>~qg;KF&`P(?0EkgU#K4(mwb%>=T=f=Dzj;(EL|L^RTyZC88bLz?%9_xGFd&$LFZuZvIC~J( z=9E4E2%CP~a>x5dznT)Wr#d`ES5(+I`)IkrS?Ef(Xl65 z5T}mjQp4!uI%!9zos~K8!IRSuGONf5fj>W(jftO`6X@xgusbBV8Fm-nK&ryd`+{@3 zo4*ilM+axP-L?+?wJorj1=D>@+s)ew#`^X5MX-ND*9t^!!t?VK}1sz4y}k^4J&h_`@8K3sqTsrH92f zZ*#|LU8r4ABd95Ft2p=A=%-T^y7F;_ey+ZJG9105XDV7-WfL2vNB8wY4_D{SM;-eh zzceX8Sn(FTC+2;n7U~trpH7_9X!&ZbKjpK0|zG@@x_fM6J=MX09 zUJ*VHX;-Gx8wWx`W&BmOb>JS6#iof}{ypNAK!;^xs8!CW&u}A-~h< zOuIR@qX3jz${5;*QcU*zRu5;l@1gtfkC|dIx-P)mZPhMQoJMmb`)jt7{DWsc$>QrC zlYB=z$w3@BP7isNjes&-b%fg_K>hxEPT|PkI;L?odTSrey2tN@e9IKYDU<>gP&$RF zz1W$yF z=#B&D?l(FIf%Qqfeb59!@kOjU**3x(uy*aa$)*p;xlPYhb~O*YLb`-FLPq7gXmV;L z%Gtj)dpPkOYb<-OJAjXWWwdzAH-Cg(hupPwIk1oHX-$m_I3nq1-tGtK1^TXWE-R`454FPT)#{8%sD4+d0mN-1ZfO!@L;QkKER_ z%O=HkrAX26y{?@4P;n&T+V*xY$(2x|X;r59v&HeCq11N>EUz4LpTLb`i5;cjp1#lv zZrmLYx-vngV%*Y(aHq@1gX+EKL6YG#M>>5dk85}-SHBpF|F8L8ecDQP znx8qMk>>U@7(vJ7N<=t$H4K@y^dv+>($a6_2AC>b(l?QWGG>DvNOvM4@7%-9$CL%o zrSigfQh8!xcuxfS%*KGg+pXBI3V|~m4LFoWqc)3ns#uK3xh_V`6^}?VUOxImR>SP|j}OXzyGna$Um3nY zW#ALwLsn*^lrFUQfvxISB+zB(0aVpz$Z5}pc#K_>kPQl45^vx;HEg_INEj zbqWWJ2tdN%LhlsuZ? z^N1YqU9yi}T1Mk?`9}xU3dg9z*X~Q{I;taFC9dqVbb@wu#J^(>>MIS!S=?~i7>uMC`JKb5{4(=MH_Hh5 zt1K`*%J8uxuHk5b6C6oc$4fkh#Zgn)Yzc+q1)*KfIbvd@Dl_j!n< zaS=2icMs*jVrp(hXQJ@TLbXWm_XRhPqal|(>VzN=xAszc`aK>!Ad4MrpjxBHMySOF ze&`$+53p5_B`eLc{c>axbrB6GjvPZ;pzr#8SqmD533?g4jI4z>!A)p5odg=%c1AdJ z*NgLPnNUWu#t%Sd!UNh#Z?Ys{>_~MafF1{;gg@?5*go*K8?TZFB*DZ;R%&84?4gQ% z2Hs;Xk`HUY1v*xZap+;ArbQuB%>aGjG(cap0W7%!9p6}i-giPPfj$rR6&#%s*jYn! zU|(y&&T)ZLu+vd++(Y|jK}7GS0BeLzzK^3*x7~Lr4;K{WmkPSZ+{D4GKG70iEdTr< z34qxPwaB5@z+ZzGv7|+eg@IAcjBGeEP$YP6ft1N#3z` zU_?T*pj#GDB@fgT1b#iN2cH%goTbUQebP@7C+IP2RpJEbu7eX;xZYNHCquB5JN20O zs06QCC=_0`h@#NzaPC!PNIBmGoJ4#EaGGX>9~^?c8nR3z+GW^?HWp<%fC=|Wus6_# z`dcT^cvx*FPT=hP3NFr7f0NJ6l5B5~zO-T{j-n6Et1 zY>}GT21x0=lscfB84G>Ue9J^uH94g-6IYqi8G4cvGUpF+O6T~(O#%6A{BXI+DV_N+ z-l-Plokf;&4XI=`P3a6|(V2p0suvkkf)N;FtQi83tg)Z=KkU5d?{myR!r(Lm82bW) zL*Fm_C6X+H96RZz=3ue>MC-@`ZyM{5TLfOw6fCUpL75-KGAd#=0GJnwfL!3-RuCxA z{P+(pU&if|aq{Mq|8nUvaO#JiUb^hW37+2l;K^gni7UL^x|_>?du_Vj;`)b9#`VL$ zb;Ggd+tcIEUv{kdi%-2SZl1pVqAPGs)V-|$_ZbgP{;U?3bCTO#46GKH!EZhJstL|& z1NzV1$GC%DiOKwEco-?v6WIbi${!jc;k=KqMW$z$S z_V52vF_K&b5vL77p$H(;m7c<0-DNbX(wXu6$5r!8Ob@B;IIeYwyJ+)?kJCrs>Mk8Il#mmOHlG#l`i80p($NRXF44lh zv80%_BzULR`?N03=hpk@7pH#jRp~3J8m9By`u%zrGPAk$e)H(>wXH`mjj`8rtyB2; zgjGTo{ZwxX8%>)V5T*GQl>R3gMp$kF7TI2 zY1I~uEJY$|K17I;=bxHuOR@xNY4=cJCAL9X&Da%S#zQNUGerk9l*%|f)HVZN-hk88 zhLP&P2k28mNKaPDGX-!!3s|N zMY|EMSxk;nS2Mnz?p}tR6&tmZ)d@WueH)fD>aLI8`_rF#{O$bz#aGizebmdi{aV)O zV$)}-IyOi?RBJy4mTtkga`f5q_M6$z>6Rip5${M1zz5<9&SQR=U>^k?35a z%UUq$-E=N-e*zolp~M$_`>x68?YUD=-}*lVM&w%1$ZV?xpGm}~`C8y_J?a_#03M+p zni5_&OlyK|u@km-^t>9U8bPeM2gFr>afTvu9ZGGO*1wTy<>ggpbporR2;hcvMas(L zb1I`yw&=tvw*HD$a)@3Ms|Jct{p(XP4tto9r?dE97q`1!||2lir(`{ z5&#`mK2rhdCpv6l1i-n%I|M*_v+WG+sECv}gEl1&S!Ys8oWwI}CKLq@C7PST`KqKSGJB(&Y|Mvs(6Cp!|t_3FA8{99oP8qC14wK!AmtbBM5t+h@jv2o^^zJc+q z*;Kwq8_QVpmsN8?gyuEpq*n*0ugc~iULN@%GCWBYpy80=OU2M3Vp4qFbHXLIdzX97 z@EBaxz)1P%7ec%X-zR$zTKecqK9Y`S!;sTm3qS{;w7Vv>?k*!NvZyQ~4~w`H$T|9E z00^g28vI2=U@=$0+wQv>933^;W-Lf94;)dvDn}Y8N$-aMwZ|# z4GopypnHZLMD0S?vnx};=^9ztChH!}#NwbU7atov+S8ThuwV0llx2^{jQb9L^-gYF zV}@~iRxd*sbsbx4{t=r0qB4GZwfWtLPnn(VKGPY&U&Yx~^s!BGUh&k!N65Q~Toked z(uX{79fdF%bjItULG7=Bl_XE%#Y#5Tt@t7eCeOiKP+F1je!NUvy_>BB()1l{nD0Uk zgpmfcK<(y(D4FOT7J^;oDtg9xYUf6a#LDO)q)X5oro+!+1cO|emN=eVl#b9Hd0rh( zh|C$Y6kuFZkZ7vQR3^;V>6iW;-B$|kL&f2wuZ?y*m%?W+C$~?v-2AFN9!8IqEu--n z^vd0lA^?Jz^B@PRjUCCsa8pE^Q^SAIJ0#~uiAy&S*$?)bgW79iljK)f1Med?eld_s z$4l2E*nN3{<_kql8Jc`KeG63-?Iw$z?IU1uD;qOCZ-YZyV+Eo8jJdM6Y4v^QpjuDC zeB)i5AaO8MT*OJv4N7ur(^q<+5Vw%IQd%Uc%%3gsnVOsI={RCJgAY{3E+Fe@#>|y1 zO10=;Te(1%Fp_`^xY(+&T+0vyHtT?IIiNaTavvJFFAO_)LA@!b%aqxwI?%0hl|YAY z-)TT6ux5Z)%mI({l%ErL4q(l$!#v!WwnHOgl7$eoCCoW;8ODDMnuHc;CNmXS9iMXe z5EUgb80)Ok6;jAoenMcj$}Z({Rg`&F^f#OyM`iWh(o(ZxIv4Ckt=A$Pu4o?lLD-3! z)nDuHWiHCwM#h533_!S$ycKK`p&oW9nTt|63#2RwA=|qLDJXDBOqDX|jJYTbM6U-* zJij_wjrKMdnFmSE45=UeWHuLRXr&`1jrMk$i-5f+dfR6%4o;G|N`E6o1wU-{lDQ~F zlupe>@s98KlJRA85g7~S%|&G^Dwgx+qA;WFG#8cg=AyY$JA^}YFRm!5AsRwZdd6Ht z%u#19;<{7YaXo7;qTf44+?k8I5ktunBx0lp$&7Nlxrm-XtfU6OaX<5}AlP6on$Af% zGbdsNTilqIIuaj7&8%adK`y#$MLe7=$Vk|V zms%dm=0!ImCx+P#42XB6r4;4W7Dugh3cj*q*Th^=u#O@zaW%m;YFmBUW|J`0U4FTO zw#vX6e~)JJCGMuc2-j9pS82i|2!l;K86nY(4swdbJkb(fgz&tELTC+uLjo z&$dZc{iqm2(6l<>s^o0B$f;08^)Qy>srsU;o^Yz-3i`#5CbP^5wv8*Do0ME$rC!HlN4;1i*UpNH|5%T z3|2VeTqVzp^Or^X*#PTkVYdM5XklA`b z7bzv8YurB>Zjmv*FVz^*1rb9~4dbS-@J;`D9f@Vjt67jSm^NgXqndqbI7~1C^pXEB zd+!2u*;y87?#ns+u!B=ZtwkjKO4*w+cqdv0ONUW+h(IgXYG0%*EV;a*6wq6Ii-2g z0Zxf|Qu{1^5TnhX=d}CokWH)P_d2^zs=kLGvH`6wju)|qqO1J@;v-?UT$;RVsWxy& zsg`(u#+HTRFgyLS=zmdcbiSw6v~xUVz-7fDI;V0xt)}61TG3_7<8g}D4l4qa#@C@- z!`9Ken=YN`ZW>^CO3AYD;d+}|TwaMM__})EY}|R9HjyPVPp^+FBUu3{ucosp0(4y+ z0PkdCI`2UQAI8~qX>c|T#VY2qTN*2zzqe9sYKg5j`E0dqT-)i>k5ZedWwcj zQKg$G6-~EP6!zfxeLa0eyEavnuA!)!#=6?>c8+KaciRe80aKEFhbct{XMVV*#h~_N z+B_(WKA~ng!5{P_b6`sY$2!bHx`RHFSzuwEY8I|>L|J+ama@{9;|;0_eeMm~x!Soj z7j!_qH|X?DaZxR>>yG=rPIm-L-Y%45Sk+V_h@8UlvqC$8rs`#Slu`6~x@masUb^c6 z;u!`zR-D77OH;v*VYkZ=f~AHSlpG4fyurTj^8~j75Xv*~0p66T6k2)^C%R$lB0Wxg zQ>N$+g#$F>h2%>$oPzHd5nH8dO8^R*OBVz!#Nm3`!)(nHGi)SbwXk&n6-K50BseHB z@bnzKIYb_zrtDULN@|Egp@+bpzzS-L>0eTC4}Wd>$fwVj-=b!onqnKmuE`(x#sqA-7A zQ1gLd7gid&|K9VJ8f%}aQfPSIQ~6HKXlT!*Kb&5FZ-8j6PBwEO!DNhm9}!&vg)&bo zFt#1s-SK=YGGIW0=+gkAPdRWqViL`I2W{4^S?Vh;pE-?mmqCZe;}gmEqu%;3(9q*N zV;D(HU?d_t7VRB=mMkF(0AFff`9bxPC(WszL>sk9CqefvyIsf14FSYSKxBwXz(eE= zCZ0>RauQ&;6tx-5E1%>LP9`5769o}+qRW`E%Bi=Q@z077F+!;R;xy*`=G(C-Wmgcz z<>~+pL$i9137o>*AmLzF!9%ozCD#6 z2>E&Wb`C$Fw*8sqEGD6dV{dH`%d)$W%^cTseF@pDLNA?$Y6BLDU@@zFRkepWVx(F#PTg zepD<#V3q+u5e&rwG|@M};j`c~&7#pb(jPhuK8!k?%JA{U`ZGJbWpK!K+MutHrD!&K z#}wDyX>o@T8J!snnmt@ajM~+O5Jr%S{U?gfVMvEX8b^#l zEE0~(PNZa35|g$Q=uvI1MxXgIovpD-SX4_f|8bZ&Z?gI>{q(D%vWE8hS}Toe}q!>n^7lcGGD#J&8Wtz6ZnI!1%6A%~kWfe~9MV+<@Z z-He~4Efy-Rl*}|ej+NY1y55p5-j1%3J5Bm4?FTW97n(CyTCL=o!4RRj41&Q2l8cRU zXiR9VVWHTgEon-95gNjGM)v&+9xrpE z+Tz>z89n1NLjDA&%mAI3=NUsD(BQ8d8Daj1Pu4kGxyd}~>10l{Wj5nAU5y{*}nmTx13$mF>Opa|pSVeT1M1Q9|rq)SK9 zvV+?n_VgUI2Cq}JT<=8K#xW5%Tua+^IVN>4Seo&1j$n)~kGXu0j9F3QUI)9#gU zpQ<)M$NXJ`=k+l^p#y<@NAk+alb26y9@v+LK!Z!-FE*tKd%~ZOAzNy%2ZrrMweX*o zxRv(Vmv9c9gU8PIO8XkUD0rj@%PsHoUU%~@!$h^u@d}3(kz)AO$5VP?Zf-6byp9IP zWlI299jGO%n%QXV&AC%SUF{rO5Q$bWO8%jef92 z;3!ZqCeH4#uhW{`4aQ#CE=DyJWSy}Yv}eumImP66nD+Q85qYo{AJrqdwA17`Xj73qOhO${<8>d7@C3AncQSbaoMrvL@muA&U^v z3!Pe=I4Tc!g`Y&sLS-67(oaI7*@Y~GyoaBJnQNM`>OOO-!j~8PJIsZnEDH@r;E<8Z97dbbR*Q!e|H)?@hOLo&k16B57i zp;*Wb=wH+5PJC`xr2TipeLG#Z!tn=3HzP=axvq?Z@tr<6~C>!dW4yU)AJH;9x?u1+ri8Aa_&@Y&njs$#TA*)m!fJrJZO>b5= z-ZvC-YW3weUE3^Q1G9>2D}Hd1Z=UIin==V?!9i6xO$Dy@yi0Dn_L|x9xi#)Sf*`(* z9Ed?@*1MYJqcdX8U>Z8JMvBKHBgq{kWm3Yi5u%!-b4n}B$6h3GWu{vOo zCkHN#RX~x(DopHWf~K*r9o4=>Lo)tA*enC^(HW1#Q4>r`a}MD|$kvWhq0ooSFgXhE z(7De90S9JA$R4hCJ;+|J>4DisJOLmklpMfz6W1|Jql3fTRnDE)tAI4|E2v`nh=Mar zU*+^wP9J95%*T)%Jm?HA@?0Fqccj>e=ru!)F}(?gXlg@P z;|c`^6vX8Untn68S$!Pg>~4TZAgIjaj^VRQINc3c;bYfbfBZ=K0=!ueR5c~=Lr`vu zB>mJX8v2YLy~AfO#|)g-OpngCZ(MnlDj)pYFMi>(fA;PNUvT*94LR(n_R6bg+lfe1 zLmA+;41Tu#Kiz;ziSo_TC5MQ&hLyE_u392^DCTOtz1SCIi|g_3VS`fpVFMa%H=q%m z702M_l}C3umnXMDI|R&IsGW%Nw7Q)V<+WFD6zP4o25lsD+-#GPh7o{|V_Xfjg!9b2 z=BQ`3P5DBhMdgpf!Ks-u`Pm*7aNUnS^E1z`n1CLGPKB*0l3Y<@CX_AU7#0{h@;WQ> zy^M*1?IpC@+2{TuZrTTU`;R*PL&sVI>84561 zYSt)gp6Aft5w?Nx#U;l{;w$6@dlg+eHE&>>_6Xt+*Yc6XKqEvhk!vcRWCEE;El&CAy0U2JzR?`Y~8a3>`e z2P91S-GHpcK4#So@Ncch0nx@#9k8D6g15>Gi4ZPGneHhNL0BPEtqWYDy9YSD#fFHy z4Bm;&Ln4OxV6zC@BWRv_G@X3lEoi#*cw!H6oz(F5kLcUW2b7Br?x9ewTc*dlPyNS& zbvI!};x%yL14xCP(xcq}@J|Tk;*mW^xetwE!PpE&`|QN8bz;QCs_}wClVBYMG@m(* z7#l+vkD|RAb#B-T1HV33{0e)rT}(QZ5x<&Bkix*P-3Re2bq0Q&`gBVCs+CVe)5-Dc zi^t`1-uM)|56`P3M~-(_t@ikpc1gvP3hLS5QnYT*6O>7n3ahzULYLFLR7QCjsn7IGqNe-wW z8vgDigBNf`ReM&V)}cajS|uzBVms;~v@UoCkrjOgyh zE4mn&SQxK+-?5ifmmT7!@lH9MKjO!(KxXHHrCUDtiSfyEqW`(XRSe}*=Ul1|!cD!d zyS8GN*KOZcy|pd8Is^&myEabD;Vz5@FnYT5nZIi2xb!lH@q~`+vNvuUjKi!QXSS6= za(HG)U`y@&A^CHMSbcyg&|*iw5ylWk!e~4F=^9$ifa*Y9fnOO_`9*B1<(UlNsuIBJ zhCTD|?C63pK>D5Ue(*~Q2j6#CFE@Ml{vZ39U-?+EnC+U)Lc=OTgL2Pm^7ANGeppa| z!j|v;t)63GcGHYB9|lovUV{}KBnQ)oSZ`7GDOwP>idw9d$>gs_(Cex!u*o=c+es1utvA~^ z1`DK44015LE)VG3xLG#%3|E`s%l8I=S4 zsC~(67;R5`qtQHdAJRs?seNy;bkPGg&Mcez*BQu9=pR_|Wc^cyiC{So-2tbC;pN7z z*>;XDbyDQKQ|joM$K5zCG7u3^5J8Y->@`@7BCR7aGcf#W8vojuC1!PRI>}SEv8+{u zdCLUOR+%tc#>gDRzzVU?QK4%MV_66!AxI&qqcQr^5g~?09kRq@hoxpi*%+E(5LroJ z;3t%KAKgIX7+2-AuOgH$y=x~|NOaN8f)-^Vkmp&6?cGNS@B4%j=ZGDbV9svu`oTRV zi;XpTkmo?MJR0pwH-Iita6!8C7k~i0i3er@29YOEB)E@IoaQOuNunpAY#y7)`fZgV%J(CUAiEF+a z1t7dCj&Y-iaJ{M&nOxhfA7;3>5fqmjhOT(QVOALO=s_CLLXQeS-6<1yEN%corN`x) z3`{mm_pGVe{?vj#Bw1*$bQ`fvsqwagd-yn@sOTG~Bnz{4+3p z3slJk!Ynze1&ahWbty;zd0}RcXh{L~)4k`MG!{Y*0ni-@Bm)6I2uKV+?puzuVDqg# zHgt<^&0dl9(57O70Ld)tv3uN#-(6AX#hZxr4 z>8JA;WK8W8i%F<~y)UK6Z8!S+RY8dd7b}hJq}N#o;V4+XP`G%cFp|;ao-7$V%a&w0 zBeZB#Mf_G=G2)PC1+@at&C0+t><_R^VC|~4P5ulQ%*P~0^e9jgokQ*3-_^{TVU(qG zR2Jdj^yRiNfi?qkjcF*`fbcq=TPjz5KF|p{Z5b77&7-F#)0)9M2@PyV){rGmLfOZsl6oekH1^@ZFKa;X|8%5BOc(>>2qL>@5%R*MMg z4D*wqQLK!rL2T%CHWe!^=2!=!`pXdkngIz8J=PG9LM?xdPBM_fO8cea+hL?%GqlZ1 zI&B0==vSvFLfz{1cf5mBA+-&BheH-Do6v3pa{l7xz=^sT)FoJOZ@t_ODT!g*VVo`y zUmh{*Yo-fm^P$q9cW`8xI3ve7?=wOGiP?eA?&fKgDw#zLvqsTqj7s4kK*ks`y`SE8 zL(f!Yx+Bf^kbG!_dTg=^?U`H#f4`kW2JP$T<}4-BdteU>lPTx%sk3cB#ss?r@Q>!nv%P-002l!-WIOq3TG}@K zWjI`UH(V3Iav%!k%^6zt#+Od~6%?q{SiSC;@$KrQfl23MK8X|61hJzb!9NQK!9V9U zb$CMH=)IqmyzB)q`@3E)Aa{*pj>p66_AL9w`Ec%eiC*o`F+B&e4Q%Im&kMlk?J3w{ z0%5u#Hvs``dIPQdfl|BkN|6Z6Fu!bt57X|h5$24l`pZ@*BzxriXZ20tIdt%-mvex} zb8d)A@7aA8nalRboBO5A^e)^5(_I1+!A9Hr$kg6)6FDEip8bvy^6LmAo*Wef!L<=* zt^GGA02ttvuIfF^j4@({I% z0u{2y?N!gk?6GlWvwAI}TOct>9KeM)tqdkG;B$q*Lu`m8P?T7+KE&RFXH)}2UX&hU z9yVRXuhq&WF6_q*5+2Tk^Hg8v3-rZRvJ!4AvX#I_qZgH!hsAyDkJ7GN24SP{`4&qizDGKUwB% zRqk|+FWO<${vkxilSKil@MzuY4YG$vj>Lh#;4Q*AC|E~js^B-^dz{?V!+<6rfR>pb z$n*9A?0Fu)67QtkpJ)B>Yi^iPW>&W_nx)otRcho5(T4YxuzN%86SByLR8oZ3TOltR z{9w8syRP{*!?UC1761IV@=3CDA6vfWm>Wc>-vG zq>e1b`~ZI^C@}IL}fS_ z`lHH>rdi}1G>D}=8=qYit%p?snGsG6ltD1z<53K~_~kc#xIo*rv{_dLs(Yw$KC+B~n0xpg579yJe&wSDal@0r-I)#NQaVvn^ z-Q~RE3whHGawT$w)*mh_-5}k;eXjI>TwU~klo)qTnC{7-VZ>;Piu8Fba2XPJ%|ik3 z9H9U(7pQ^*ZX86V#gC}l*=a{L#$pzMvOyZ{voHK=$4Jz159!xX@x@Dz&lpfmgG5U*{qI%HY&2ZK}2o?8Y)*nvbbk`Dk*#BrpOs zry8|7&?2n2|SWiJ#+E&DV$Fxz4HlTYcok@GMos=$(SaXgu+FjNlR$r zBDpk_#8iA34rDgzSO~plhT8+UU=)Y0KJa$N?sg2?+2uFeWi}hm zAg{mYy^LA=w)mMo&GF=Tc|qnHR>uQi3Cz zbQlUnFll0@dbq)j1X7KnqGRx$xF1d}w1{pKu#Qv@{YqdlDxZa5N-vs`5zFNn07(N_ z;65-qs{s6bec8B6gACQ}zto|gSigJ|;!wLahTW`jp+<*G8)Gv?VKbzvBQX9-pv`t6 z%rsywzR{+NMJgJ zw!z5@yk0a*mvd&2TJc7CI^deq@h6;+{CIfXi9Z=?vm?3a9X2o6fS$VQ2-pyHksF4< zJJrT|f@)p~CaH297}?0bbh(qoTFsH2ylzyk%`H7H50#}YD?3$uh&(CI;i^dC@>Hj2U1) zBZmMB^$>z1C-;TG)6NP=*o=xsEA_BVRfbpyB>j zFa|%Wg%Dg7>1Y70La=IU4F~bc=V(%;dX{sN7GV$@`*$gw}GEq?C> zKYvEDzfV8d{-C>qlnBNAOUQBm7YkTbNt|+7Vgbd)djmgRPu@Smp`z15d3aPj1A@Lf z1|kLnp(yS+b>Xmub)G_~A_{MO9mi~XF5|>N>2dfbf-8nKI~2p_+Mm~t>g*`rEOiCj zNL}MiAV7|LCCYJmjv=}TFQ&SQyCmbT5h;vfimay0e)qFRIq2?RVl`ON(@Sqi?;g?7lXAa z27r_p#og=!8b;3tcERMh-l^>y=?=pI8cq`78H;3Ql^iln23!q7j#3A{0OQ^%8jqlG z(0^7qiRkyE7hnv0C%W50_3GS>I1l!U5m>k>%VkT4gX;;^SwvTJ(exT8q=OueQM*4L zKsA#q$pBE|ZnY^6PArjd?|3kr7e?&^+TB^De1~W_7(6433TwBB@ENCjqv<`|eyh5p zRz!wpgd3NFsFkv7Yjso~m6UE+NRqL!V7TLWkd?4K-Dcm1);u>N;a=h2-JEfxW(|4A zNmCg^u$dA=CuT5yCvwAmwAbXBo$Dbc?4%2^8Zq|cuuY#ix9()Vb)!M5hwY5!`j)5; z9S!Poa8TDK(pKET;D?B)1*AU1M$g8Co@ucMH9Rnp6v6RSJmYO2)GTJ>x=v_wu1-n0 zhz^&aB;mA6KG7#A&>v#cNXaK)2z2tsVQRQ`mO(~uE`<O}L^*-YSg5GN@wM_vX! z4Ud~VC#I6vxz`#~*@vJ_T~*`bxd0-ttOygXmd>=tyX*46DZ0}HM>FcRzoMRH9Mqwy zl3a$D{SslHs2{rHc_n+pG+>HCd9j554^u`0@*&F0$Q1_?+|D)iG}RM|5|@}RrveN) za~;9uxw8`(j0DcHETnakR$Vi5}QwcY`Tw7!-bDt z1>dYbrPMGz@njOVa*+oD^L)su{8S??KM;rAa%9--xf%*PkEmztYDm*~a&t$lSV8r$ zf&#%M!xV>$0{Ro`uv9hLi! z6eMb(!dx(io9kT$>IH&DKw0?~P`%=b)Z_maLGtXOkqL)Jke)d-vh~P_QS{+Y zWdImV&`Aj9L~c}3*ts_(3UHriW4Caj;)fp{ve+4*eaQW zahZQ5a}FdFVYk$PM%4%(^jpz`ei3RNOUTj1;<{cX**j-;D(Ugu;J}w0fs3rjx*I-DXBq1wOk*70ERAs=_RE4sv*^@mdU-m z{V9k#$P_BaP$w(56I!iRiDn@r2JM%z0~!KwzXEj;ujjDKh%n+mXAKbxn6?$pR!Wi` zD*olZ88Dq-@(q3n9*Znzp;!jxV91;|$#+m8Jjep9rLeBFkNb8-ZlGQ1pp8l%_pNxD zg0N}A);yuVP~Bn=;Op%Hy!g^)@Z-=`UOO<7fC@B~5z{PsG7~FQ?P*bbYgxXqF}kk9 zC?_P^^I_z(K}3Wt4K^0qyEtAYyvftW2~W!P=}$ERROn5-hrr}f9Q65tC7*UaL1pDV9R_op!fuS zmU9HrSJtvPCR1b>!A-CXnH4z`(aJ(oAs*GP<~}@)GFoaqU>drkS-V77Y17DE#4c17 zh02RMD7x!n1_}qKST4Kb)WkX)W4Iw0UMgfpCe+I)w)Jojl!2;pcG!&fB^N<}^g!A$ zJ<075_z{>ayNhX~y`2XeJud8+GMTzOHhE6OGO*&cL8a_>zMomGFGypY_U@r_X?Jp0 zi;_QB@ZTR11uge?|F4wmI}r3@a4&0>a*Ai1zI~}Et9mrn1(+qKlF)~h)DKpBN~eAL z3g)-HlQT~UTZ#JuD3<`zEsG{T=io^_ztWl*n-glHiN5`qoUA_@uA(JMIKd0)<>qv{ z6i#k{by1gDzK_=fQ1wvs@_a!E>pVOo`wss{e`8JQ?`L1NOSn~-jc!e;SlB!$!Dxb~ z`?Z81*xO+Hp&jM`Ml|MNe)9y)@0`^+N8+E*(vrA~agSBJ&=+=XrnOv)VsFmrXBRCH zH%|n&vnxLlTdJ?LZ1>6m<%l49Fjsv=4GnfG$7ooj(>FZ%C$^-1CYSU-FDbV|zdine zm()*F*UEnYN7?KrOI+m>tZb`3nI`<^mPWPyP8mM9e z8@e6;nqx!<)=JfsDjJApwP!~|SZ&=s=d4dxj8pG4!(ZG;LBf}uKHDLZBUYD?;AePn z&uJ|eavb8&8nt+oA%gr*WzUGKp)Dk+r{t8(k7`bn8+|kmiso!i^XIhY(aobsW>>41 zSv}ND^&AqeCGNZd|L#y?;TDKy2><|yzx4c3&8m&jyGJ!Uq%heb-k0J9stbs)x1o)* zv(ig#Ml+na|s+gc^=Kg`GuJR$4$`0%N0xDB9bc6jolpXszxfCQbQ!k=RDs`1I9 zsEFT`%K7ok9UYb{jy{Wz(;ydmHdJ6zQMqDr)zH}K4XBZ^vTuWWon350(bm(%&A(Ks zH}StexcymGvGCt$;rU@7>&P+Y-=!|^GQTUuYRS9F1N3p!40$jnKX9Jlyre70BQ#^Ae%2ddMc85RB>2M5wMFvcSsb0(oHa zp*pd?xj>i!92S9+tRw=q; zL0GKfF^*U<}OeNbF=AuECb{UBYqmjJB~(i%4bvz(TuFH+_q zj(Gewsjz|Wj+J<_44O7~mAHYdKE- zn~nnJ!i6&rRa0n*vnGz&GGkt!6@pgadNt7hdU)2n?TyEa z_KDL^>ZQ;Lf|J+leN?@mVbM1DD$&hiw2$w@_nKA>Tqp&~ZBn zS`DuhJfYtFiP3j6A!{&@D~S7HyUMoOIYg-uUr4aey(jS6#_D&|Z%mCMuQff9hY=dP z7l8x63no3dSGYvRXq?bTMh0p$K%YXic~P6(#3aF)P++kW;cdr+&f#%(m8kpcE^yVDE~6*Q5`l%!MvlS{ z-oTeu#Gth`KVc_D%hnfW-a~&uiOz}(>X#6LMpX7PrJh#yGjjn>@+ojom&MUpIXL}P z6v4zOC8)w(4r}w13wCiHz2MA@8`r5qXN!@`i_%mQP~lAJYIi(&beYu{M!yQer_vQ$ zYCr=lU0XbeI6w{25^7;v!cZ-WAkuHPw9T!E<0N--rFo2nx8m^3v=jt;S>ruF=#H$M z{{bk<%OC$?nbX@vy@@%9IZk{FJaRviSC8^`)kW)?Qw#=SRylk~|H|nL%Xq`gnV7h< zF|$=23daHNUjmE@*)!<{$t!Irb=eGo*j#vCl-Cq=(<6@4H`HNhC9AX`PsN~Q2?uf3 z6(?GM8}TF#WRrX$0TGLkCa8h;Ln#}nYtNzUnu$6i1A|P(Y@4_|9x0M4?aT+TklD2r z7eh~r7+)|Bl{HIwmT?Yu5t0|JitBAF%j>OyNT&7MpLF2`Ofj?Y_04DJ{*EvVP1+!x zVEg`T8N^rG%t|Q$1Nstb&XATi4|GVO1VH$$Uf=_fF&sAzBh2Km91t(_!?FIu&*^#C z6mluy^hB7XWpPo^94uNe34>{j@mmga;H>uNVhvmaLQQ)%+yTZraCqWWw2U4#gY;cK zn+%wAG?1Kpnq$bI0-La)z74Rc3zUN4A8F-`J&$L_V(t$zTED>{Eiixk>(q2%>X@OOeRg#QCnpHnM4KiiMR?hXv62#_z$34qpBVqF0VEYQ#r>w?1nJi5Yg3UFP{F*@5tKJE zP`AA4c+no`pXp!Y`q9BnM}phT5yJmK4mTKht#>_KRN)-NfV9`ZY)=0(Lo}~KYmnsA zdwj4{bzCy`t!^-Zr&*Re+%>LUK%jZfR z6M&qgo9v4VJ&aMo$Kf*FhPNHaC27P7J-AVXMR$n9s*;UTY&+PDvkR8<30{xj=oU!iX4(lbGdyy z!AI07$T&k+qEd(kE8~Lx>S&r{HA-unK-Wqm(`U*O5?--4=7@~o?(owjhN8Q)=ATZu zFY>JNSp%zwEa~3Yf*KKMGny+G&+X61ZQ=6R_c;RaG(gvIFEtzMePLau{J_Yp=k>z@H!ZQeyLQsuR%3QlO?bCBGqtH|S?DVHx4^&-33b{612nV{ z=2Cq#BU#Qg&=gT1%FV9xIdBj4Z-lMb{-`1iuFET{Ym@bD+jmS)Q*!sC!~~fCvbbrZ zxwffpx<&<CcWBoxl+PZ6N`V__F=93G107NFa?&2ObkKr*ysn} znCDryt}F8vTsoocp5-Hc+*4Kud|2=7)P3kOMXs<)D@nPS*udcsp@}3^iZF(3%TT&G zU1|ST@>=hr5n#ivguv?e&+1;Ws&QDw>S$$U6$F{zpNeZ^Y#Kzn+{6?*K!3jQ{pT2Dw;?1^RnzXy}+jq>rEwvx=8$(Z-X-XQJ%>!YK@~G1E^@4Ad zRz>l?e>pm|&l#ooyy&s;mDPmwwop?C6b?H*01NhIQgt;E=&tAKvJmMp01N3kNR1O_ z136)Z%qvU?__cFJ;7LB+&pRfQw{%SQ5f15&_Bzp_(V+qr%47`Yr9Dqo6IkT8SuioH z`0{*DKfaRRa5X*8n+z|`I=HoT!#PD4`CL*mLC^{9KMwfS>0XB*Vqpq4JO+s0*4f(@23nzeRjb)V?pHuStt@m z%vA0=Ho9u3K?W7#X>sZ>{I-WQ-}Llxqph=8WT_IPC3`nw==kmE zYowwdi`1Q)Qg?T$q3u!Nj=Y2piM#_vX3);8o*J$|K7H@G)mbj5xJ%T5Ixo1hC_Mcs zJ%oc$6%G3e2Au(nRYJzIh9wfi{#ya_5%169SaTQQ1BE5p>dg^8zF1f588#iAPMq$s z&+D{Ui4s(R>lhmo90m%MUsK6K0KHRtXFTvF{Q66aTF|@;K^f8$@Ff zM6HXX9&Ma7j(TVzQ7|$wT=MGy?V0#7Hx#=2MQdK3Bh6UqkO=elq0_^hN&-eGdZVtc zi6rKoadYf5HhR4GvC+*>ztxs7gEACJK-N9ij&DB#xms1#%y|CBz2!gCsekWW^f(N5 zBM1OWT=0iX75#C>m|f>@woE(}87!oOhER+-w=wi_0D5WEL^txlE8_#E3^&#&lusm6}vfj9TM3K z;(&$oVk9z}AV7Y^2IfqoTD^y;fBLe|zv4q5d&gZS;Xm1oH$HyXPk;Uo@BO1&?=>l0Q8)g_@BHr7AAR#5zvJ^J zX~SUgvqp*;Qy#pFHay;^&*$5Npy_{`c`9y&JqQ^j*=sW3{JkFobfOs--L956v{Yx- z4H6JPsD>yQUX!T&a0}Wv@7il-OGjtR+>62c>fxD}e&od+i2H|Us3nKx&kK|-zP zMIhs&vr&HBc(6PgrB*G4im6#T3T7I4X$4Fvh;HM8mk^n zhf=3B=#17$Z zA(Ty)KCwfpx95G@ow-$V)7EmkxlisBL!niY#k?}TN`*G=No~ou>eSy{Z*@Km;&i)#?UU?0U`a zhaRUR&BE(7(XEXXKCdahzu5?$K}7n z>sz5`7^91fB(>avX2z_bnP7f>-s{%NffE_BSiY{__f0TVMX+#|kLmw|e{`H)?2(m! zX7A)Hze4uzMWcRCV?!H`k_8U>3I)55ybNl8DTR zonI-=xiVpVkmY13Apv6b5Y1j&2jT$NPUi6|g{sNz_v$d2gFyTq1Qlm}P7xpzohnP1=4pB)3f^iuTPn!v*3g>J}0ZHN>7 zwHYXvfi={s@PBlb;pudxh?2A})c|vvNeHUxEzqaP7{|(Cp>fE8Lr;2~%%po(ng{yO zO%PPKRhnav2Wc*tOa>@C0+oswmENmIVKzK=es|OUmJbZlk0=n5TnKc8zfDPQtRxrD z9PCq)gV1c1U*VxK3sq8eS}E zqqY?g2^pba5Zt19o@qnT6H)T$TVQ9ohpSf_UUX znU(&a%%f|lb8NInOF0ZfyKqcKudJY3@PQDk^orxyI-x)K%V7?VrFh35e>s-2gBl1% zI&HT1u6|bTnH839TTo=_bk4kN({r5k(>PoMfKW$ULFciP#3fVI3hkCek*IMJ$_by^ zA7F`V1y((X<4K_9gEOwEjVP0V4{rewkMb8WG)ONoo09ZS>;i0>CS;^}m2dO=Zf`gEUl^59ZwiUWq&c9&r@MwvK-#`< zcnjsu?q)SlyyP@>Ctl=wu{4H{+CNY0|Busp=k=$Y*87G}PV0w0V)q^VcoM}?^iu z0(p#dvOu?~R8!6REcS*n9o>yjXn;6GK}#WagiqE&^Kwo8p{nE4ZZe3$qkH(Y#iWk zZCheGx05HoQB01FIGfl&wH6en3iPOKY_hvY+$!nG!8FJ>Tx<=Z-&fsz@A<(C!Y&dc zS@R{tPX_SLd(VfE$sKatNsvFtVo8KZxoH(P_MZBu|5eY+ZkEhW%Z7Y{+oSq>?>%4b zWIWT9=RVbOJdc}`C}?>=g1gR&?L#u3bJ*tKgue#|8n!?)6aowx>^Nya;`A>gqH%RV zIPSRj{8`nOe6O##d{zS=y#x~a!zYsO$II!%pL{O{g|hdcBMHbN5tU=v-r;A4aSy^P zT4Se*9)x95Mbuq%7E*l6}YoEx}_jVY6`Y6$5$n$}b>TBUKYjHlZ z7M~Vs>BkIHJ1b#H{pwuA3O`rLa z+PmL^!n2%yC*M?eg>LZQM>R~gg3(oMo9Br`c;Bh!86Hn1qJ9C>?_oPqkxEB?ny`$q z@U5Sp5JT$eXWaV0!P+*Q5c>8LxGp-J2c{ThkAEnjQ9e{oZi%q*chVJM#uFsiUI*Kq z3dcqJyOEaoft9Zg*doH#$-uzE6<~xekUf{oUo`emV9=Ccq^z|Smf(W}mK3>W_${-O zfPoGLh@&p%>^itmS3&^A?4iGzJ|h=1NuYQZwb458QgS>8-DyFkl*z^{U!I)Jt5nUv7WZ#9?Us)BXaxqt>770HO%XT}S z3M@<2c-AG8ORd&`!7}T(D6BB}(W7Z&4AiX|F(?F@>5Em}F`Ht(aD~uducT9e?y3(^Y*BlMF&#A+TXU}7iY~0QA4DTo7zcHwRXN5wy7j(gTaT zOw7=i8R3a&RJ1U}cXAu4CujY!UpGagVg`YiDMg9P#??Ltkb$U92*wm3f`^IcBw?#s zL_RwU&f8KxPEvM(pyoX-v!|3G;F1MdH(Ls#D9I%=A0n4JgJR>(a~FLnw4kpU)fF6F zc^G`2lbq0$)BnSIc1p(?VIGNd+n;-7)1*^x?jN>MxC%jlQDA5l?`#{4fwU@A>P%+f zOWA-TAG^{ri1Qj4iuS(O9xtXJucv>4UhUoKg^{zl_-|j~W$j~l#1#*H!1gs+wgAmScorrHu?*$8T~rH`Y|4V2M^Mz+k5~Ql>Rb9qj&PlekkAd8y6Ju z-+fQ(AKpy^SAN^+zib#r5KHKolTOhy3(1ktonyZ^bSGEG^zLC)_s&Np*r0tbuBsp1 zY1CpJ?B0^};AuOGOU~LA&W>L8Ff$2){e7iMYoAM;i0DKhwAp`R~;|f^QQk# zwWmCBCINeTEY@Z$kpDhdN$%{-sqj`$F0d~+wd)SO0f_5R&KZ(xCrR+19@%Eg1h-0D zS!WK+>O23it-laMV6rG3(KH| zJPkU>CDy}it3RD(vHJLCtU;xT7~(Lz)D(JPI&f!yU>Y_bq{S2^wO5?kxo{qGr!eJs3hkX? z8~shiys=I&5O$@!f?Gs`Sqj}LcqF%Ux$(y5|L=@=%w8swr>4gf!yV zZ+r8t9CD`98$ww9@ND^7oIr}{#{%3ur3Bo&X4S@n_0|0SPG=BA`{9>SuD$#5UDRBe08>7P;yPw6Koy6T~%ESC-*$2<~?%C4mD*Z<7mv#`qpD`d0Q8V+uFIYRO zLaCKw3S{s7p*%tU!slUDzUkPBRqpt1@>z-E0oAdy{gsEz)rM*N_ByZoybf2`l+j_g zOO~;o3+z^`Yp$j^hZ0Zl$$I_A$Gr<%c{i4BeBGo0v3K{mcR@zQ^!pe`s@NBCOL-0k zH==fXJKyw^vD+FDV|4)Tu{+Hme`yT_*}J>Q(+40CNh*4!1X7c5v6DD^f}}w2<$G}iOo2j2ApTy+Yo0Tu zS-0RW<~hz{I1t0}Vovz>XOQ#K8!+DH1IX9u%sTmOol+N6IwzIcsM=3(88cohUHe3kBl_Scz8bxBsl*!eT~88xbm}w&?}?v*a7^wVCnUA0^#4 zD7(pvWzmfTwz~bCrIz|qvH3Uyw6}M=0h^E7pS_+uTwvY4*@yA;6~R;X{xE3)3zRZs zE!rCvv-XZR211V`fwuewTORypvSsjR65G&3W&d~E|E6={ahe$d9qieVS8;D3$KB2@ z3NE{Id+G-n%8upsi<_$VVAI#KfH=g|ces7w3Z4#4dHVD{;Ydn_46EKNvdT*-`|HHw zvm03I!wiBYS*T0{$FXCoJ3pz!3^ z3IQaH-X1>U^#Oq3{4m#C;sG$*9;&$=Av5t|3CI>CCk2=Id*SJe5(#-&HcC)#L^f^# z*q+LlkHV@)v(&XMJ-?D20rh}!Cdd`$al7REuXRp6^~NiE?}G1f&(EC=ye4sn-UWF3^SrSB&(~J_h8e>A36sIpXv`07d2dp%jr+Zx8Plbdc|Q1 zhaWSq%kV%e10T>%;HH#M!rPaL?wJH&DP+fVV^t>T8;e##E8~8&PI_pCp^E{Gr_Xeu zA3p>nz#n=Dd8G~pMjwuXO(NC;6^rqp9{0ZvhML2?C4fcma}_djqRw3hr>~2)ciAn9 zLqq6yAKw(Z>1aRUV0)cn#z*UuRikJYoAAHXeTX8q;owjJs7{-jPG(>sHZ6wzff4R} z#N=ii9axMbN2Kxm8@>dvM<>LlrF=%S3s-`~&aQM_FriS=7bh3jiLK6ZlTGfqvAs5TJY6j7&`yIWY)O_=dCa>Ahlx9;Z;K69dDJ24+^R;;>Gg zKvh&IIN8nN_w|+xA;I<9koCKBiLQEn$f1lo*90c(bD7oxn&w%%G+?3z4@l#{y9onEd+waOd3VM z;N*Qgk+2u$0;fY1jw*QQ4a9YM`q`3Mik-q6hfv@-j*ldO5uMqMlKUp%V%G>r2R2(s z@=0doNU9A~JAJE8k)#0#akB>G-V%3!r3Xn?I~z>7OD5h}RuqU&J7?v$Q`vH8#vqd1 zHiPJG1I8sn+sD#SQBBc!W)L`;wlw{f(QFlVS~0}BBQ^j`#(Kl5PNp`HYj`RMtGIE; zWcn$!$&!eWv}Yk)M>D({iJnMSHVCj4Dbt05(-$Sa3xYXg&I9Lt!0p%Y)*d( zPF`I?H!?=$kuVJ6q`;YTucqM%3hypjaue%8i-~F151|dlb5;5<@_Jc*S1xInx%Dy9 zs!J}tlxtySLCY&x$39f?z&2a{md49)1kmcjf~j&-R>Hwq3aG(ANDxu7u3QvQnxSv%xy(eBDK9tD%Rf0 zYx{?O*m1CZvoA0I>}`2hH4C)6e1^}Rt|U}H^?(j3HkGG*cirDkyhzS+(@1;~_`b}8 zPtr7Pnp#TKP2(}7xX zn85pRQUe1LutO{W;SqTywX$Zg(%{P0lb~Td=1hWkFKtd>#^e#BZz7wy`Kc#olN;*1 zC#X}*kb_`k?ww(Wz40S&{D`-!kjVJa9mPP{Fqdk#1p7;*Sc2d#dKw1eWI(Y-hc3Vw zTX!&Wzk*1J1s_dS*36vM3V{?>BA%^IzZinst{GT3lnT28^Q?Bp@72yVF%1L`YiI#) ziL6uilzR(|9ukmoI1MTWzVRerq@^OuAS9fhVRYDdtSIJYW?&kbx|PJf%5c-U9E|@v zqPg>!7)Ni#0BtG81q!r)6yq7>vwkr>0x8PaI72bbEKWhHJH~=(&u!N}@Q&k{AtMYS zcyKkijoeqLl#S- z+dbiTUrTlA*prK=N{pUVok!K|wf%hy`wvckUgBgM3%ken-LBbr#mV__?a8h|XDpQ9 zj?`6(1w-}fb|M^6R;sA&(qQ*XsemAkH4&L8j_?Z0-UNwry}s=-D@qh;FgK%#Z{gJ6 z1dLhX{w+|Rvjs{AgL44&{sk!a&!GfH#p5E@TiFzzFA#8P_ila=uz?`>Wjps*`o0j% zXS^^F)DQ5CYZtAD2Ujhnj)z+G*XoMfMPC58$X(nUNB8r&i#nh*HWTNvG|_3EaElDr zNRWT6k9=E4mIg~5fU1{1%=Rklj$lp1$2aF^!hgYJ=gx%DDcEtYrUW@*RN6sT@o!>z~>_k+{* zXbg@7Y!p3Jl?DqlNEC1wj4(TvbOrbT2CbWneah`2QFz&BdLmnw1-jU$2;*y4)8mDp z1|=k({aqex&t&kd-$_v5dUlu!Y+`bwj6ZzjeQZTUILf0sllvp98>@mResXaZzr1pn zVjyRZ`-e0h^t5Trg#xh5KhtlZ+qBdKE0JyqENSc2A0hV*B}hWkh8lfSQg|(J%r@*Y znF}$_bLWdmXvZYteUg64h~1nwRkc_8vv(GruNBp^Y)EKCCSYb5sj#ebRn3yu>Ohvb zDz(2>2VE__SL+&h6V{g<>%2n0+`Co#*e6y0fgkmA8a2J~3Sy+J#y@d1r35xPD-e@o z)A6xFzSlzy=JUHbVM3*J_aL5dzqd08wRUNPCAc!^jhvw?`80Egza$XJ34>kz- z>||yTsA499P3CxH?Vf!lzx}2Suro-<^(J4>DqT9adOgo%MwR}gDnNV*K41ysFL9fd z?Ocb5Nf>op%}DwnV`U;m7#W3ZJ2L%5EO-)ehH=63 zv}?1B7_1e;+iil)T!x5GyEve1b$ReMcG~zh6*9mq&LF<5t<-`; zM@dDY9M)MoSF!Ky+|4I+uKRG^{6jN(N~xnh7!yPgXivpO494WTZYRxSnQI6gZVPs! z=Uu^p1^0{;1{+vb2{a5RP(FY#&1J!S1| zuv}pv-TrYsFN5F0xD40xB1X{-?cH$9EIs-7V471l3PT?g0rFwUsRG}!`8Vgcj8j*OyRQ-$Ut5z0A*7s;Iq?(5{E+J-M9*BgDRVw7_yRA*cfl^m(5^OMr=jkouuX3E zdw@L64F;LFGgS`Yb>_^nI}{9Q0k$&@WTvPQrVkf{@fH}A!K>2z;Fq4$7IC7b_)5O$mcw`a7Vr{%%eGV?1t8mj3l8 zPyVWtCr?hEY;eevw8=I&S@J0$JTaM07~LtlCsm1MxE1Z_*ZHHUm)tzBd|(4^;mFPP zW$4nL>u$aB!dayqnr!#5#2a74+kItiL)kKJf&6N&0Fr|##AN@JZczKZk{;d( z%LDbc>!KNWP8?YOA(=1gGP~D#J&G^`X9r?U?k{ek`iBD^|C3SN!(UFBSYWdU0j__`NSezD~kxxmPy}Go{;p7 zAl*9Ly@!Gk@p5z0{=P1J`MT==RbSUU>72fvCCH;HL3BsizqWNqsJl%VhNfW$BYzkM#Jh)w?wwWcrxKN#|g}+vK%#(|kF% zC^>=MR`CZWEhWv+F_N&}>od@G!ofnnf9T-$y+tt^@&DHU7L%4jcUZx9dPxC}kxi&7 zS?q?Q-^))3`RU|bw%C<0Cu5kRNgXUY%i5o~TP%ZEPMMU+``j!G#x-*X(X}<&imk4o z$6oE4r;}_5x>g;i)e&T8mc7vSq*{W7q8;RHO!Ty!3v!t*$PuS2)C4_9^PoJTf$QN; zPUsa%t=ZAeb;yp|J55T+qkmj+g1>sf21clymB7`3O}t{Z_%Z+;i^$+t{NU0dAQ1}%^zZ6d*^eaHtCW_#{TS@m;_)=U{g_njzYI4nNzwux z73y4z^uf~?Fxamyy2xp8Pq7=^P5Tj4)C8h3fZeH{Qg&Hlu<%4!|%#@vX_?yR!kV?(E7*FDTc3S!Gi+WXg- z-S}Z=H-Jb3`r^gXNA6RHFm7+(fMQ>_RL0UxM< zT3jJ&QXuhkfT6T;4T&M3;;tp=d$Byy3?nGi3=T^In*33^(e7#Q=Vy9M?}1}~PQn@O zjr?T75p&GiTl$2b5OdKaa7P(}>XI+#N zD%w-j2eQu2c!8lbj1#@3B*Fs%wfd>CqBFVG^m?q0r9~C2#f3o8-NsQvU-Wxc3>Sf- zN28?-6ivBv!W#^y%O6`=2Mi@+frO=(QqL185>v{H<|pT z(N*Fkc&}RVR0ycrnMU-N1mIB~ik5}@wCd?k4t#UyL}t}+vNPC{Du*rPgEP}QYCgDk zf4tR+x{2RxOc@f0qRRNITOZ3vvIL`bQsshlX@@-6=%&W(LPJH6qF9!53Y1ML8(O(q z5M~@i1rBz`H8=Cp;TbU-Jffg_E1b(a9@+sw6q_K3PBxW{TQzP5H z@D^kh0Gdmm6_Q*8z>EBIf%9aY2&owc8wN1R1)gG}EBqzxWO_S6;xmBUUsr$6cHraO zfC}=)z=vl0?`j8zGH3~|uQ(Lvi^qBQX&Pt8?Zun$+m~m_bqAbizGVk>-*r<>mxC z?y!bWX2;#d%F~X!hxK7YVj)}xfnJUltT>tcX+`YTrdEx*#cKj;*J1}ee2Mgr%U;-4 z{Nbt>?@fd#@xpCfQ137$VIIa$ODnSO^5LzCK_S>&9CZ%5`-z|vj%DV-CIb0y*1#IE z!L5ypR=xH+Y)?uU&AT7dzVkqD^aUjpit+O4}3yLnlzN2hicyvFn%?7RrC z#kM83>HJu9S5m`at?dW;p8WXS-*SX3Ml0b0G$*ZrDfGRWzCX|*2C}gWS7fh`1c)1R!w5-o(MZpG%x0$RafKsVpL4t?|jBu?6;xje?gI$W9MCDRX;j1x@_O|!mE)^y104^eG!IrMTb-3xBWKGHXu?j1@L zD*CR_2${praZ{pRCQPBK?@mPCqy^VI1}*ppJ7YNFR2aeE^b7jfxdc`&jh_nT_w9F_ z-%ju(=AT(*1X3oHSfc(ceD1$gr~HDB;1p#}@nxZUaBKB+N900iE%opT8h860sQs8X zsHZ?a=88Q;t;Sg;7E#@v0p}P;N&Ldn2}!l{B++}fxCF-0S#!x>#-?=2Fvq*xSlz6$ zJq24$^!(ucw-pMf+U(gR!3~GfBUlwOl$UH8%ttX?pa0E%Fjwb;`M!@n%gL(Jo9)2+YwyNs9I-#?5CW=VH zs=g!nju5k^p-o`Y(n=+rmbYU25ev~f>sTM%SrL>TO=;1ck=b+5Vvh6%9k#AmgECvN z2J77@JMe8fe#u!4HZHke_1Clv7ipb7No_?Q?598@9-F1LdU{25!1{;wTz4mF&l;yM ziR~+c5UdIn83&&HrogG#i17rw6!jLSEC<%IVUBLav!jx!Rupv!)x~82aM!`qfd_haOq3gG4y&4_;f2`0eN^B|Eu$<*3Iv`PgDlMtL9Y+z zMg*O)>9fizCx^bkUmp+mqz9_ro_riWBWR@4RZm}2NlkWknnxau&m3c8w~F={&$vo` z2TKWp5QjaWkH|MnKQd%cPNf&c;~7;K&d6olFl@2e>ZC?Ksl$C9{x{+M1D$Hl*EB^*&~TRNq{kTOWyr-(0ak zpt@Ho!LQulGU=-8Djcc`zQWGoM)&>v-~xZylZ$8bM1`2y>x&qJI|^H)&D?m^0Vc15 zoSlTX3_JY|Xi9{aP@D5*gZpI_zUx(}0D4>#{Cr!wmVvW^?LlIJBEUVOI)WT{Si&F9 zLNfVtw#Xnd@PBJ<)9QNiZVaU=Rxy8G>I3V}534792tJIdfE5W8$F?fN6^o6hk6pMZ z0RW7r;wytVd8 zpNEmSH&LrGDm=u1JS6sEvj;z#*hf)HR7psWXf2@%XqY6|o`=Ly^UVn?>(Cm6*+xS< z$J3VJ84G%ntR<#B)H(@JjJL#^WgS5()J|>n9O-YB0*>U8KQ-Xwtws&tVPZJVwcYF1 zQ8Jc6{yzNi+lqFXtL6-hWe${Gc2^zA9#9ww}+ zeYrlTVl3!u!F*9^)YpF0mbidATZ$S&;b2Rf>#<@>gC<;)c%$O=h9D%(|BHm9ASm!P z0&(&x(Dbr^``pN%(Xj{(7@}u1V4%RTxM`zEzO7x5xQOJuG8pwYudr zqsfaINI8$Wi(6y#!>YZ5TP{3;m#wFTpp6H%+9X)kjp|hg( zuCDBATnj?a(iy(5O_sqXPS$g|;*)$6hU5b8lN>S^9JDL;77jzpoL_U#`KkljE1%?J zroH^0^Yw1V!<7du&`qiL0%JznG1AnGNhr$rM^~yn;}8EpGk#!4$rN3Qin*`;mE`t3 z5nB|u+k4J;lHe)9N?!6}U;8>qJ)~3;VgEAT0eVGQ02+PCJ?Arlib?qDji&F2eTayl zGMiCi^q*v&Yz7EB;KO4%!Wj>hAUSCa6k`dMxJx?)Jq^6xiqIiF1Z-~K#FzgB`)0^lLG;uJMx6`wZ?>QQUF%nrjTWLC(_fA0jFjvQDdGfIEaOfe z!hShCfF}U`g&UJnJkH>!f}4$H5V#;Z2KhcX{a`1(jixWeu+`TbWXYOoVxeXb+2&e7W*JJ) z73kqNnTP3ri8>MWrav*<@4-&^AbW-^TLM5U>wxnTN@^*{k|z$?)}1}iEwB`h8*P|X zm%2@{)NKmm2&L;!e&2FpvF_cfa}yhW4)aZFisZR^J>mNo8ENzKFoP6_49j zv3=ysh;Uk_!|*e1LrL3>7$SxzNd>N|8n@ zSjki##R=uUsw%IHK&UEKK}7Sd5)MGyYlfPli^TVdAr&+w?yj@?o4f$kKTZ9!7Zk-g z|KfifT0j5)bi0Gsf8n(x<$vF=Q(h(Gw6Cf9b88JLxBtR%5)q&JQ)>uR0jcM@Ly`(P z>g~0ntMeFgOX#yZ$`tYuMzL_{hLw8&sneRftM2KvQ)Xq9?gkRc>^&X1)6q7UNMC=} z8)KRDlNH7q;>s__q@0x*^RB%a(Azih4_@KSp#4JziugMZ`03oN@Lr$a_G&ol2N=rV z>((|73(GLv%gTxXjYNU6YixvJ%>v3VH>dqqL|8sSQk+Bw<_VLamSk&x37)!h``lzIZg6G`s9Yi z5dAwVxQp94>K8D%bkkxS_$k{bxtn3cNYIlF1mW&u*SPl`d2O1m5)bVAwvyYt{qd`h z!}B`)_4K$XXDAxH|0rw5ihrsKuD<4k3TV`}A7KTzK1>A<0sPjYzu!sEpw%#;#L{cu zhKpglZMXxp3q2_-%o+@FA!5QH94W%FP!(-zhlKI_y1JOY8WDoAr9eh6;h_`#!v?e5 zzV}yXVF;ucD?E7Vm{Qkw?WPXazVoBrd_ib9<(v?ka541ZZT4XZeHgn*0Z5j8x+Tyd z5~?e=eZNKOK29Uzn7ynHuw@@%yuvuO^~-W`_9%MQIb_V3T-h>}TH$*gthHNQ3yRQP z$bk~tQXHXW8E`Z?dr;Zu?U!iLA#=_0)4_STHz?mnct4k4Xv&BV7y_<3{B*)#7xpd} zIR3Di*e0E4MW!!z8=lnVLnz)i`&%e$gcA+{=n(vBy0d7Yz`0}+(H-Uq2VT# z2FWUr9^>N@vqBf}+@}2KPc->#hxI!1T|Z*|Z_=9Oj^SdP#fqrfpME=&gj-hoY9Ho- zGC`Qiw59Q^V<`?<%k~!+v+hJy;d&%1)`Bqg4Oy%9MzgMT;Wt_MvFOPPDz70Mtc4DM zy1jX^*1egvA*-D?1j3MY)V_T&E5S*vbst$F+UjrTS;y_KF9Ja9XS41nE0f#Z(Axp9 z)V{~8oH<|x%rENIO2rH%&S!tRJc0yG{-VOs+PEG5Y2iHHhPH=b;c(u1z{x?I0)ucP z>MgOk+4fa8^QJP}=hyQr(Nr;l0c+#Rql-~G`U4_;XpgGB9v=m|KvvNK1!oF1s0fOPT>(Ks`U4Bn{@?GJxtrY(d_2qh{NDS&KyvTga_-ET)91_#6iJ2QTs{Cf zQYGPp7PiT0O9fm>uL2@4d?$#2eQ4eLQXFQ@6D=$qV(D2`YHZCv%#A}Pzg%E=E?^iM zM^Om*R<&N6a^n=bb*zCZ%OPPco;kV8FRVfQGw^vV>j(_1GfyeUa!Zsk+*8VFq&jkS z#wzYcq8<{nlIPKo5_ro~c0^s{jcIL~8<`E%NJ zryIhBFYz@qF-xnrx<)7}?Qgk!5-2ekEvI2(35s(GeAq_tFVs912wg?w{2o0G1aF81 zoXgS`P_MA&K(;iA5Ph^s&6FL%l@irzaJ(tr8Z@AU(8bhbkWEdi@XQ6rj23C2J$|zl zzzM`12Yx*B{LesQ09A#-Sp5(VOfDc&@j(?MRAg!2U_~mol^a077z%|QZrfm{gsA4R z>#(~>R9Ft@9E5pZV&vNfdpf|QVJjwe28ESqm%r`EjMLlqO?nKY-yxoLn9y|D`^X7` zXu@2I+!))h@5Es=Ufzb~>kP+h)Fh#vGE5^Z+!=-Ti}VLqBc`#=%n$S||8Tk-F@e?r z!iuO8^uPFXa#B1*jR}jdD?_LlKmbW@5I}GW)nZ{VJe(yS#DrMPZ$w8z847awd9iu! zp~$nwIW5S5Q|kh-z%@Z3j<^H`WZdvbjwjQokqXjn;+mjT3pH^KFWA(Ypja3M>wj|F z1h~d{OFE0!@QPX!G*lkWVoDVe&NV^#FqYs5dRVOq8dA6>s5nq>VICpIG(g5sl>h`m zBhh>iyadi*x!^XF#H}OIKCGc=bQq56%Mi`$bhA-W4zz$7=?F1Y37)0GMf1hnv9I)d zAhd46Kuk!@U=k?oI?^N)$&C)Qn!`M^*AzhII78ttRzwlGI}k2N6A$y$mw*E=(GO{g zE0uYLt4e6!qyZqYR8gLY#@Xa)1^5{A3pRr*>D^UVIo>#P;;S-=0XH;vk>}J}F4Tr> zfVvpBi99wK7G#FKbX5VhzN0I@1x0E~hD)!||}hp@z|agZu#7`C?K1TDw|5k5+& zhH0Aw#&Xys=>iTX0VIdt(61g#Bt?U|z}z{HRW5iiQpRVv&LXV>iPZvO)Lby7VW;v# z>>yO=b%~`Qoep1TPoc*_X8I%GT>xqgn|BhHxWsALNHH$}Typhb^>X4dZS4X}HY7KV zHO#UYDH$~SJ=!BMhA$!A5)8pN=t*MU7GErZR3!67xrct`Jw8%Ge_1_g@suCw$&zQO zV3;0WWfU$WlU-FHVn|Se zsRi+<-;~9PcpK&NNXH-d@fk1W#&fd*!+ujm`W+$6G)p<2e*n&39(vqyP+sa)0&k;V&Ol}q8HHRA&!nD3*_f0y3>8AHg_oeIyz3w zbHXF-VJI1v6Iu@xz|ZkdR+B_;xr8+#t;i7p;gj9ul0V2D!~w*j_&_WECNsjZ`@ct& zj>}jke zt@5*x?vB_eAma%}p~MUbn5hVgy29QEG}r_tj4u-!zAuJ5?RBsSDHE0!B^hQKy{$PB z!YRZ*Y5-1*z*pH^a-s~qGcXax2n9#VO9YIR!>-euC}S_eT@fJT#YH$Vl!1#902|7{ zg`|O~XNia4adNakIMMWWDUKzi5yn{~Dv-egmJA?RG zjVK@%REA69pLyd1KcNA19~h)W2dexAExa}rl3jpK3?@6984tT^1wgX%(@e1nK)K;m z77N~dBrIb3L>j9zl> z32~UA1x>*ts~3rI^V?XmY7C+{Fi0>CDC0CzV1N;>f2t=UL%0>^I$Xk_ixCx0)-|UV zgI?*HqbyU4OCYXGks+~cK`EG+oD3|%UI9o7HsTkO?0FTBQLLpIj4ra7h{naY5RvOx ztxToBJx(lP*03;JSV$!AnlD$LhGp;AV=$+#R6J+#t0Ki9zUWeVt?+6tC!O`I1He_ zNHKs=f~q3+^8W8YnCh4!2)pd!{}5(U9u~bRhQcnp_&vpf?Y6JaFe0bq?DH!)2T)EVnR4 z&KZ)$3mtg@z9}cS&>w{vhGBiUhwV_X!BT2$9R!QW=DB6KBb!gqERdlMoY4spn@Zil zX3MmL04$rwntbA_CsMjtwG* z8w=Jmom9GGINYKsQdHKc)384tIN%S;VH2g9=y!8XDOzT*9AOA@T#w)7AR9()h^tVn zl0f@hd^fof0fd_a`!@f;KKolw_2Fy4vkdHW)XsFrIED`Dk-FnWEZ77{j7B-OSb$tW zaGLW-Nk|yU=WTqeHb(Np)JQ%TWhKG+NAhifBl+qfq?kw*`DXDW`J^-PV)Pu&SFPr} z`rz>B|HW7!*%-kd<@Xku?INLE6}pC#bfsx7tJOT0pp<{&U&S0%^udf^qRYZs=@n!+ zTbD71AO~<61ArL1*D(k5do&GyAUG1Sb_yf#F@=#62UAQTi<@^K7I+8FlPRV+{7APJ zyxT=BF`xoi%!lPI#Jd=5K?;>u#d|Oc%yCYOkS_)ZLB-5R;S)nqRfM0w8etC1PvCakx(=}7>X*4eHU({h$2adjC91-2+=*m~R(F@q=*C4~(fDI=j2GqSb_L@y(Rrhw2dX)z;jmEl|J z<*-6ABQMMSvg$q`3kwi4KxUwrk-1*Dro;?*8RshF6tFQ99W+D1UlEvwCPa+dGkFD~ zFw>Nm%rzZ(Lt;};T5X(^F6r%xR)_{c1^UYHOc;(;01U%0qkZQB)(_`Ck?op2zTq zscwCujB1{hs-R(&nARgI1cTll4uRt?5Liu8_?fv3RwP3(0PvkTWk4k?gCa~x==tP9 zK({hEU?J;ogZa9 zpLwTT@F?2)>!G&n47?C+J<4;3HO>s5AS}u~M-eZ?cB7uTl8=v-Pq3$+&T){fwpDU| zxM!Zq*XN#~ycCV+W}BBBa>?m=aVFXnP61v*rDGQZwgEngk6aPPxf#JQ?hlG!O6L-Z zY8E~v6H5NAt1ZANdH_RXS|&P~=Mu0>HKgO47RnMsi8ii^!DymMTev|AC2d(jWspi@ z4Do=473682SET(D&G0wFJt5e#lRc?6f&RrBi7zPdgoPTivm|Eu5V(cn194$|m&OkI zTgPTd0eA|VK<|AN9J9|>92oy7%R|<{NtFKf7SRFLIR4|87}uk$RU3mb#w@`&b+ZcM z%;Se+G6`%-j^JU9m1x62nA2FbHcqmfb#dL=I5q7N{BPldj3|Y8o?BZ0gUFGOuoAU9 z@Dl1-&uccvE>bGwwB#YnsGuyI`}is}5usUdk1QR}m+&C7tmJ|M4wRr4_!NP1S#?Rt z26&raQU|Flpid-=3_&RkWGOUo1QqCdTCh0$Th>$uOV$EoLl7e6RaS?#aDW&zs}six z*Hpg{yrw!SGH4BRa65?b%FC)>&En@2G9p#1cJP&GX_z;nhZMtz%%%u8r_91G!t&dN zq=RxrHCb8EiDDIuZH$AcHjuLi@bgSgk%YvU+ z2oP#H%<~p+TR}2RP)Aaf%yX8Wrg&tn@CS2qAJ=hxcqUjv&>DwEsf1$*wK zGm_&@29$t5Yy-6oBe-(k*h+{}A{YS$tXK56zl{-zM`ijL-l&o$%U6Mq zVNBq;DtfjJ`At?{C+9`5bufU!Wb%MSUb&bKhTBT;z}=f=@gJQmj8QAD(NoIf^!)2f z0g05Mg9gEfGpkbbqJXjqg*vUXK8CgjI z;z@j8C=j)324F`!{SJ#0tx%i&HNeGu=v+8Sh)&?L6P9ru9FbSb1!Nr9=1(9wBQ5+u zttdz-cuSm9hc*@tv72s5a9)V+H)UgRJeh2cy4AK8E+j{d3ry0OQD$@j<070Xu^P?e z49M0+x9vM&QBx+M7(z_tsj4Cy+=r5G!5NtI#4pR_K=u=~0F7h4!95rWZ52y^ID(-> zD-#EHD_&IlC6l{B(@$8?SO`Yq74ktw+ANAP)$a8An_STvK#3VV%Z+;hjl0vM)gc(8 z)3*&lIwB|>Jt0+u4?$#d1QI)#{A{tvjP1S1rS^|&YdnfAYH)!k zkp-NSVU5EKYv~UhX}X-$3-rM((;dJicp%Rq*`jbDMR3%SU2d@To?kh0OR|}<@Vx>B z_QwK8m`8I+Lj$e%HeGs%e0JsznX;u`0CZFeFtgC7<|l-KEo2*34V@3s?>GYP_k2K# zZY?`r<0e)fMEvh=?L2^%DdX8dCKj9jNEn6F)oFqpd=5Z3qwP3PES%2;a>tqpLiOe} z0JOyasKs-Fk6QJkoMl15Zkjx>{QQq%o_ZiuZoY>{vC}viS1luMR>0=@MQU^= z$7uj)0ab(qF9Y;Kh|&v~a|be4;!app3xDW$8^bcdg$g`K4#Kfqt5rA(s1z+YS`nctsz^hQ=OXk}d(mv@Bc<$4TKpbTKzlzY zTCp9@mD?**k&0=AHb-GMB7Vd_qgJ(AUJt?Rs!=d}%*LWG_w$86 zJ=^tIu`Ql0^Bw<4i$#luI8Gxj_Rm^8PfdS>brR<9XBbbKI%w?#Hg{pr)97q-4ukIC zuz(K6c|ZpNDKLq#ltXzr!~-jjK-VyHVQo7G$y+(0~qbJ+~QK?!S5Qs3aLI<0BLMtMd> zMF@rK_`^aGLxZWVk&r3WU!kzI`C=8u-$V%!3SO>~W@{X0)GazuO^aOidzovxE-(I+Z48@M8Q{S`8T2@rvcojCGtjAB^ zFlf@HhpIK!~ffk+Qr!E$|kD1@ns;Yhc}fjfKQ4#y{; z5h%_WwZH^RL*!3j2msDLycEYi8F+$nG5p&NF-!u@A>JB|C4@dH1z?O|SQ>H$!a}H9 zhY55xBDdDy%xhWN0ZEEBW%Y;H4_XSXez) zqvs5qfunxc0+_1Pzy$%xg;8gXyLW}wN2ZH!O;IRO*6?-`JU`5VC*@Wb&XhwL`~(pZl8eskm5V{KK4 z2H$h+M9&!vDwykdTFgKZNCj_S

M+taTgze9zk8{cLUEHqaA|EVQs9rK4C{;8j=w zI{sTY+EO$GECb%^eLPJ$1)|2g5|pCL@&LHU3;>InedfTBI-0@< zHBT`vViieC4Fb1EJic&>BMnwuL3N>pOd{vtggI>-@>X8w0I3tiJbPLJ;W_mratI1@ zcW6)xDy3xwB%KIe5pMc28z$rt!O)KRRsRs;tEwFy1Qd`zgg7!f4-pCO)C!6VNgc7; zX$-!)8znaYG0rkite_Wfhv9IAch|{vTZRb>tefO@?1dHW*3a zp<+}RXUvn)>84>>3m6u-js*(zDpfY(7EI3Z5J(0+B2tQz=NSv+S{^ug8^egaa1bjf zNMzJPg%3=>EK6U)_@KSdlmM8$@TPBkyzX+HWZ9|h&;H#b`B+qI4@6lT)X(w{Pk zh7ADYusPVQ*CI6!+NIa5E;oq~9jAwtKp?1s4B(h+VwlV$;P|KFmDkq*^^*3SMmt0Q zm?x9{OJoQdgnV=;>_3=%{)4N&#-hwo*lD5E7qXi##EKXKD1t5!gLYaG;)CU+E{i=( zm13wtdk8TILkn_Sc*14jVF!YcLCjOaJ5?i1x35MZP77};!beDxF%H}VrSVKJ&_V-5 zinQ3pLcu{Ir&N*%AAdl@%M%f3Lb5JcdP9l791mr+?4)Y5!Iq>UBv^IhEqpW9gp6fTJh3W3fZYjr3t6xxP^1vZdK)wV49v2oY}bHL z1`_c&@Flkc?%7{L2gs6RM2i;lHi~%*#C#~lJkhffPPVcKY!L2bEt>JP!Wnz#y6)^a zepu{u^Cq6*unGs~@C58d5&aPFqF<>74&z)*MR^-|rj-vMg`iQ^RwF*e%LK}52Q0-5 z(Fs9^Pe3jpRN`0|%B7Jc`ooK$9Rw@=cB)NOp|Vp&8)|b@h7tHOqAzlgktmhB1r38~ zt`tkK%!GpWpe{72nbJkVfOG_4p<6mBlvgs)hKD>Ka_O=6FrNN=A-RVs^GGR1uDa=j z1O$Fyqd{PnD`!Z9u&4{OZgGIVme3B}7YRpmKTr@UmwjRDrJ2mZ1rS8n)!N{4jtVgG zOICc*$v6s?=29=J`kuX9D9tT{C8S|#%2SrW#jAiu!85-mW-1$JLDDli0Da64LGw_J zgRkfbNLGYoT1x5$0uXAoC`xBe$wS$Kg>b~xLDlJmp-DKPNyKM}`MT!>mhY*~d^@_v+zwP?gr1gMcaj0A9F0l7diA>+xcmJ?}X z8nJS&5F19sqInXM)`*oc2PH62#bCsSNp)Ysa>)MGtmvYGKe_ZC*%aU6O1+ZDDz#-L zpx~KFL61p@3#5+fZ(QjZhce@1sJ|f4(S>0#2M86J9azd9cx1u{28h(Z%Lcd-;e7(z zpt2;Kp&2+!5Ra5+K9zSUcCKMxsohRs8I|^2i5^(t;%Gqx1$)I$35cym>M&3cITX?X z8--{5hO&eLCB$5S7(_GV_pCHGkwg*}k)?C{}bsv+W}f2`(-KFTJ} zp2>CLIy0zig%yr(;XBBvuWKAS;So#7s?wDG*Q^ZUQCgz+_o^&j!&AFG^_azef1#}s z0$~q(4b~q~D_%dUEP*WeIdE!V6*)u2sp0hyeP+XERMK1o;)PCegioU4^2)jvBMRte zl$E`}@W~WR|wncdncYD2aSBB*HW~yh)y`$W%atHOP{36&AK6rU7$DHsK4}&?W)U3ZcXf+?7de z5QV-F&rLjm#mtz41p_41>y-5vPq8}23&x;>OGnu}Q)K!ca7z8bM>#^lz=3Eu)Qrhv z7}0=iN~J9rQ7%Fcw@7S3KE;WFP>tIBu-WHo*#q=RzmlYpz{v?9hI|`33B4_=L^?wl z&2xB8&n)0I40di(ODN)bpdkVW@q0o%O~fkUKn>sE&gDS>E=&&u0QGDt9MOi^Sa|bXj-dF6xfDe04*bSYq_X<}2(P;(jY;EO}tF_NF%VH?z^%&jI z081q7$Ve;LqqqP!WQdAxtHDu5xA_L7isCW4#i9h%bv7OhBW8j~3xI`303#T9L7Kag zh-f0%38+wGe%kp$1*~x(qauM00>iiPRdg0;SFS9 z?o+|W6)>nG8#645ja66RQH&z8FB})p*%YPKnVTtz9EfhovQm2Xo98QY+MBs+2kA+q zwP)``1;jkoZ-ReiK|KKovX>-^v%(}A-6Ixzh9zvG6ErB^T-j+W|is@G7hDc_EscAo+1{_bB2@y3*a6l zng5!IFYVAUshzG3c5+f@|F>#0i}FIIYbu%`Taa6n+|WS_a=DfIoywVi!mk*$D9%j< z#R7$<)RGcqSg`9kQc-3DJswWMk16ORX)auZ@lAZ5X1~l21?~7}irPH;6lvOUmqH>M z&+j)cnbR1*o>4%dhatD^RtRr3Xbti7H-GkxD|~hmQse}Y7~Cc!McE`9miSdhB&N|& z0pS92iJiD>LziHn2bK(x8D()i3Ee|Vry)iQlmP)|n&5rCa|xQMl(`s*eAt%iv5o{r z?PD}Y@B=X51B%@mV_ieKnPmIl91*=PHp4pWF9;2VM?ot=;b4sc$67jPIzmr`Sd`ep^xNNiBhOzIDs8{~oPudK;lcpoagy>K zkCNZX925}xGPVUhiL6(d^XP%N77=YaS2Aq}I4?A(@S2$Evn?QXf=7p{h3?R%Kwr#S z=!@Z#$`^yRi#EX)jFL;(corjarY~k(=8Hj!7wd}!dtRf0d0*I+T1Uc5Xsn@YS$yy^ z?-@pGJK_3!&yxX1Bt z{91@7`rrDsD*wi>LE8RXzxHR^Hp#}_;>5;)(6>?OYq>(vZw`vUK41!_%_Ffi{#J#xTWyDObgR9CHltg2B-LI;W@jTXoek zlmb)QgcD4FrDa8tMDqb{3JGWv&)HOwDMNKUk@{;S{$K$T*|Q_c8V9?@Gf9ObtPrZ= z2jMB~E$+!eGAVi!=eUEMB4kC16%5T35N4nVv)`$j0s|Dp0jVEk`caV+P^{Sbrj5r8 zNZDZ!N4-C&XSuSqR3wH2d^)%5F9-jg71oI#e5UGh?NYHXA)LjkgBgK^OjI-otG+ac zu|it{e>mYCsijy#1zxscF8OYtJ7_Ohp_IhNqcH?^ERKok(Av|0E%J{z84|wH&WFu(o>)=oHQ>C@>3X?g)Sv&DpcZnI7qk-tWU}e^{7Qt#0F6Th){D;lu!vl zc#h31P`m;Awg9e?5A|2^7LqS5S#&B$08B=R5KW4L^`ch()pJZ&8PPJ;Dy!~jpo?!9 z=E^$4g;)C(wWJ1!X7UO}U>2Yab-B8)q+DfN{9jpMex;C>M5=)DuJB#R1X?P0`oAPy zLcS2!(!l&O57HMUW-Cq+r4K@%W!-U*@hI7t&XgI`MUmHs*SjaLFs%|}&_aJ06FQGn`{*fJvxj5+HV zu;_IPGU^I3Fw99A5+cJj(HRmCk~|s}@zr9sXmHQ(^{PQ_8o;QAxRzH6?cFM|4-exU z5Lsed*-R<^cY(jdFB1N^k!zAYjod)9wg<)~ta=k`Ex{36jIv>II4!|UN`#>ngiCCs;X8scfiEd1R_y& zwX%aU5}_<5>l$yt@)IpoxM&BMp?4&elL{5+g&BfY%IGDLK)wgU;~^x9*}4Kqtw#^z z3vpl=WKmozEd(P0>gd!VDONye5Js;ICfa)jsYa{#S71w_F_lpch0ei87`IMar)o?E zhl!%Zm9m?tki`npL9^}wqex0S;w}Tk;!kL2Ac}^oIn2uF5!aVeM%zo%3W!%G+KJJ2 zyT7y0_rxt596{bP%zQR71>WbXLt@tH^gOKY1w5J2d{cW{>^=OAw$L76hFMwguQd+T zB%?T1vNcX+T^l}8cKB*FmSM$_z5(wRfD4s{#44g*R_O}<0gdP#_`v`2u(<=DgdPi% zgKdnS`6q4RVayKTE_dt~)zT{soEGM1DQGab*m3xp9>l2;yr@3ViNe;vI#61q^U3LK=TO&An4`(Koa@htO(sV{7q;0aE z<6MXkIhl)@bsl(!$Rv@>O|%fCZ#rxgLlVKK@2Hutb8&P7)`&E0!Z~^(zS&T|XVQyI z#1EjC_J-8M;wi~FE@+iwn?+Z{;l|J@P@qPu0TUR2`zP4rAKKK$d87^@1E=fn@`is~ zsjI|)r^}HBb~7AjCH@SADGZ?sj$bhPi#m!JCi$p#YTB^aI~0d06HNfATs$1=j+#*Z^0 zeXTx}laGWlaZ0Sh?4HNfh&2qpP1Z;tQn9w>UjecHuLOaJQa~*zrXdbM?;PM+gE^z% zCx~#If%L4BFyKMExyL-_aDtoBQ8JxVz3~(#dh&T50hs*KDhM|~7ygk=e|JxG6EBAO zx*6m0zYYtDS=o5-cP*n%id)H3%=#E)E)a!k1M`l>I=N3RKSfQ4ZJNcVfG(s)R@2Zi zif->%RFsRbcKqg-ZpavG14$|oVrEQ-t0fU38=?LS6H~4P==>D`A=nHh(MF8lU_c$v zHl^k{zRx)m=nt%8rqnzn2ZMmHbF{UhR&8_AOhLQMP#gqzM;W3L-bQyrI)msTU@&&3 z*GZzpR#6Z!Y>W<=PL+ok9!xTTGdf^ax}}UG!QNg8Gu>BKPk8ZhM+&cLR029iNbkYo zn1}IuvF$GdQ}U02Y3nCEWfig6V$fjVX(w1;HP|RzE&&pU@@$5TbSVjTWf&oU07j5r z#{O9TDA0KTRgl@IDc|45r69L#nj;n3|vSjxZd zq4WsdICjAIu%c=oOaB_{s`d2)rV2^5QF;WyjHf5WLs!UEpT2RTj6hf1T_sUVS8hPs83Om25nAZbRJpDgbO%y-GNBySP97^ z$vv>rCEz5K;1gnbG8!G`*ks|dBe+iV;g76B65%YN*r2Ju(uEbms^T_$k6VR_D3v3& z39J2o$Iy40#HMx4jubP#|(&d0Ym%@G$rE0FpgGDsEHcasnSKCfv+q-v9zZ zJdw*%lB^^ZmJw=>`g!ud-@-E}FFE7ep^gU-6DcIFE55tN|E`=BzDqG7oXK{8yrxph z3VE5{71&Gr>Zq*{6e|xD$c&c8x}VWLw2kP|RwI^-VI~%F<-fv`;>G%W+NLQ|6E%X) zph&cquC~k|lmQf*Ns1l|BTu~V_=%d9AM<>8Xp7+yEHgj>Oo|F-iaQ=+Vx!gWMiteN zM<`G$gs#Zgalj@iO8f&{l7#grVk}-!p9_>)R*Oz(fC7UJM{=M^$B)pY`-3l0%tAZP zLR?Moixc`*{)rDdH%CWYXr0>Y`dg&*)Eu=AzBuW03e{XN3s|iHeGub+TA7;cX;K&M zAfTyVppg3~&yEg!wy+jTrWuY8RglEWp+az|-cgO?1y*}Ke=YRW=w_?qUPwVyM+OD8 zgP`O`@sH<=7je~(FT4`;!m1Y~iQh0r&I&1#P<6T4Uv-6Urd&mk4uL@AT8aG9N}>TC zRV3qh7nAQiNDYH^M^XZ&XJL9N`CbVw33J)&2H1#%&Sn4ku+l3)5%)LYP?F)VAM$0Lxz~yoDyItLKDGQ0sw#-Bh|cEdp%r4v zF$eX$(EtU56@wC`C=EsMkaHd7$}3pJ2X8pOw;+Kw_SaOfekuR_JEVvlE?)SuS0x+0w(Ip z=oDl^Vg0j5U_aa-P~az1SbIL3vYE?MO@S6f>1&okPy$0-fmY;8B@BCYa0T9|*Pdaf zz*UAFkB8W6;wGt|?V|(wI2=&Ja~c<_VQC@59%(k!v-j90>`d=DkApcC07PrpYXss= z+f;?9`ZIU017t^jsANqDf2fQJwuT?RmWqp&cRU!1I>RZ#G%w6vZC}k^MQR-wcB0O8 zxHYm=w+8gC1+S{E{WJBgL0OdPG;XjsEjL&khzODsK_cKpE2?YWfzZLiw^lsH6>uFT z?L}U!9Q~(bTKz|A+(9ou2?Y zfT*B5P=i;vQ~HW`Dqiuw>sS|vsGDwn*WY1ZmpdL}<5QooRrP%Aw zTY+LBQJT5RNdnAYLhO}@DhiJUDrZ5~bIwBix zO2D@ip4GSXWma0Suk1W~1w2DaEhNMe_R6|b95)A}%6~oP3fJofUngu-DySW#4owP% z<5LKm4)&l+pIv_GbGal5!K^%#pCjH7ED0cgYarx^y17k&~GAT(z zxC({Y5Y`nG53w0RLbkJLaXb=wEkrcV)wEsRXt}WHc;UY(E3=YxueSuEv$xJYr`i@0m|-+hzC(*ELh77M>Sn-dqA45F&B zgvIjy;+Mhi7P)(fcb$B=ltTMCxX}9avBJ~Yu+w3o-}?%lt|kldlrcQ3S=wpnBJeSJ zA3CC>|I5#@X!94pN0$fOJ;5Tkk5jci&0@H&2w29k9~xEl3?s`ZpjN#85A>LzdoHLz zp_`8h@xzTL`9Soe=fOqt+%9_kRajK*LN}KxJgr)I8dZ3DQ{ic}y`;j-Yc})j5&zk- z5cAq`|Jf=3*$MyIN&ndbg=w=1bz3d=v% zR-RL;l(*3;t2~Dj^(Ib2%`0Wh6NhVYTs+%V@%@e125^0`XZ=TdLPj0Z6 zU@afD*uxCRW1trs0#~6(dJN&;C*|+T5L`toox>sO=mz%$MQJNlQsc$w+n%EwBtlyW_{?`kW~l ziN4X!f@_>9InKVcu-@us3^53uj6$Z>419 zq$K*B1>w$U!q0IQgw-wtWr-^=O6O(Z(UIQl6mLeZv%uQCd4W-Y8x}d%D=@BMJbr3l4p35WbO|I}9&6)J-B>BlD_WE0L3%&CVz= z%H?IGW@L`a@Z)4?LB!BHPT%M(UqY@g5l@s&$r$NPOG)&9)L#5;Je22a?kottUB-Mn zcTCphcOBlG9A9>BN@j+0Xqqnxc&_ZsPDvh?JJgxvO-b`5mU7~=Nx1-9vXr`mQ{(d2 z9Ko-t=Zkw4U0!^vtDx2BtZbjti{Fe)6CpW9LbeyQRDX=un_727o-cb${haKC`bl|X z$BwC&onzF`&dbS7sPBty(6m`>llY`WcY_Arq*!-id_%W8(VNsD$(xwuYvOY^PHY&T z=&GNV5}ys4t)GyY?Gs2DdFl0&(sJF2zJ&Vd7kn&vO#S5Sl$^Xj1KyjKmI-*QX?pIT z0Tfiv#FQ~ZlF~B0fe_Ik!2yr_9`WICWlnCQT+A7hQ$KfDw$Gb51oiT!)z8QrqRKe| zmHS;#g)IV>0$%!;F@HIr1aEe7X8ml)FMwA4E9_KH^?kjR%(|{-vCR^ankF=kZxr9C zQIn+lAlyXXkl{I*8FgLt8q{+&ij{`?va>V)RPFru7D-e;-Ity)OaK;7hiSgizQ5*s zq(ji48??w{O1d{~NRDqr0H%MpdkX8Fn_WN4o15#)&Iow-U+$&94PLSQmYD5L&irfY zOZf3A$vFHa%?JC|*RYDGfKlEY`BL$BeUQrm_4wT`3}OuE2YK(0@CK_?Hy$jM?8~Uz zpq{&)%k5I!F`8tN?Mou);wzkxnVyyA%k>3R`j68N`@*E@EN^zsUjr3JL_%hwuYMMk zXs$OtO+o!5h!P{Au+Ly`8U>Yt zUW2~M^JOIX42M`Lv~jaC!KC6%81Ov81PYxH!%iQ<3oAEU-7Sgde98V&d}d~v&kMtp z&z4P3$;lzKq!eFTqPkWtF)s_w0+==MD!zf|Ep#d5Z#^Ttb538@HM+o3+bFO*qcd}y zVkJ0v!7w_8JB%7Qdf;@?uQSVWDAQLn%XQ)K)j~PZo9lHZ!XT$7la5}afDq>qI;j21W^fTPTB1(qWN%Aesr%eT@>3%EWE6(w%X_dY-c z3I8m?%>LwrVe`Vg+FE+t`-;O^Ck{VNI-#c zty#d4HneZuYvopMDiocsfSfLEm=D|MnLN>#$ za?;GIUdmzIj^|gPvsh2FyaeSNQ7)8gDBwqN#idITa9 z+GggZC1%`|>rC`P_@*GBH5P%n#<8?1@^DDX&P>NEZBmju!CF9APa=tUpn3DIox0Vl zhi7o+>8}HnF4SRNC5N<+l{0Y}iF1?w5`1ZCInKmP(bE$0%ga(?i$OsZ`=eaCS?hn& zHs22{$DwS(b6yin&*pQKxzvGXneu`%p#gqP9nAW=@mo{}H^jBSN(YNhrQe^P#mHf9 zCJeV!geym7__A|`rDQR1*%PrL)gg!qBf!VNE$l*I6$*KjFR>f=oi?8*BQXK@6SI98 zREOQOyJhBP;9hTEb~^dJ$FOW~j;}q9F@ze1iD>3YFFY&-p72*fstAu#mm;v&J*!o# zmbcu}Qr_%Kt87GGCTcqh--4M=U=&Ke(Fg?PB&c^XvtUKV!w6xFHZ2qNP9D}xu5^K# z&IS-D82Aw;{;$-J^}J5gJoSow3*7UEG)?(KI&X?!v(K90yg7c0_Sr4CHpZuHF{;8s z{b(3*RU8InmeCGJs3N0R!Z2UDH&{Bv>wCj+B;5)w+WC?KC58!SwuUzsM;o4kQ?!tT>aMD)aQz&P#PvE|7JI`J$OkFtB`tDP%(JePhQrr^_k=Tumo!B18Ce%g ziK7y&oOwAAdm_gI;5>+P>f_#-slE&n0kJIw_L5*lSPEt;g43DV!ZBo~j0|70SnI@# zVXTgD82tgK5q>F;Ou6;hwZU==x8#R(-x|Mv5>=PsY6$Al6>!_afaf<7(lT@MV5@6;!YOFC`6nw*x-K|xRjw7_$Qh1!+~mO zmRvPv0to@1Y!o#Py38x@=4Iq~lQ6cBmf{sZ68%JQUPzTO=q$KB@Cu-S1k;q1)Dfey zGeHyNrAMG62BIy~@mmhRX8B%S_Kzx|JQegoTl5z+J~E(u4rR&~^S;#@eBX{Tb+>uH zEXvfyX1OBDEJ-rNBs^#!foat1*@gPHu4f9)4XIS9os2`t&5RoT!6+y7V z_&E|-aRx$5qI~l?qmvPj=;rN4ri!Uw7nW!3G#FsQ0_-&~VRj}22076=3tqwrRlIoPDb*P#$=f$mIs z^miR8ZLH9eA<%?j7 Iw*-Hy6vpy7>!J+eI>C3YuJmW$zq+sd@2NkA0eVDESlw|( z4`Zm2WlS~}6zS!BV~tT@d}!=7P8PXybzl6m?_b?l{#&2gi*&kPkOMoQ2kFzAyLBun+<4HyM3)S z;j>T78`8dVM(9f~t-r7RvUl4@efQ+H_O|h#{rJ#_7u!$m`cAgDO~Ve~ImhfgTYgxF zYAOBNuCM$^hZi2b=H`!g?CMaX^RhFurk3dV#GoywYA3Yp_(RoeoBTF7yQ4RM?^~mP zda~n~ZB>5W^Ws+>yG+$o^IJ=zpKZB+bVQDFtAnPTetNa`tYf)6VBe2zCI-5(XZ-v zX?J?_fdhvVx>Q1^x6w(d>4X8$_>ySL&q zx^_6%tjuc{R&=d8cH2j_H-6f6`I6@CbDu5UEphpQXSU60)2-z@U+>vaZ&bI4X7|rc z`R&+D?blv&`pet{@ctS7yMb=(-+FFv-Di{X2j^Y;Qqz8ucMmS_-5-{oU-Hgp z#&@mNE+p>GJzoy3+b1;l&h6!29la{-={qO&`{9~C9lyQv`3HK`Yc#LIkUL&KFlK62 z$06BA23L7(?!+NCt!;kdVCA(#P7NuU`t8?03_147`?vP{q{h%QJ(thv`$W&7FWfY% z-m~3ifjgp`_lw#%)Vg@U^pTDJF?7rM%o}!ZzR`Q;-23(J${FMxJ)!l+M%xy8_Y9g} zr)~4C-iX+>zkIyyoOhP-;n?YeUGYyF)0ZAx=ZoKM9K3X0)I;&pohOF$`su^?Cl3AC z3Z|ym$dBUa{&!^ruKP$nV{80PRYadUz7}utM-@TtFEFV_;mqv9QiCZ_H z&TZSIUE=7q4SH6*bxdO4y{{(?c;NZO#u1J8{<87A#J2O>ewMeoimzf$?`|J|*3EbH zoimQ<9j5sXopn|6RbB7%j-EL1jhe@OE$#Dv-Scr&QpAoIU*0~uU()<3Lk?UTJ}>F) zn%9nh8^1ZJ>BGZ+Tzu~5q?zqP4^G%nCwX`0>z}{pj5qnN8b=4dIO2iiBehnZx~|Oj zXMaDrHZ*{9$u??@fv5yRFT0DHVs$PaXSl z)#;sv=Y$RD=sY-Sc=+Q@>c*Z}JG|7q8BHxm4h`>e^2wQ-Zm5~spl|eFj)(dIZ=RDNBZ=^f@NcT4HKJ#=^`uE@6``R_Vm!)UzXwm4Nbswci zY`%N*3z12k=NA94?1_wTy}wO}4EZvn&V<*GFWT?O z9Jws@=H}II&0I7s?4|Jdv6(H)*BWu_gf*GluSx&lfn^6WCk@&*`SmSTvvxL{@L`{8 zdt|kpy{z34=k%=JtF~0SvG%K3hZn^@l$moPtK8SdrW>BRenj7j9#{Q)`;SN+u<*i$ zm+u+z-KS%}iFI!o@zuPl*KEG{%ZOg5mOqqzv2OP6pW8lK;?4N%yUr}S>)njS+0VFs zJ2I~G2if)^<8$V9v*vs-@{SK1>~EYirFVA7#s|}KKFob5>gBzU=7jVd^UQbs_vFlc zvfl&MPuO!SZu|M`5qX~6Dr0l|?AVo;`^b`yew@v!9B{9y83BVY559scd6H%C^QFn2}F=ch*wTvmV1$eq!nvZkG@+uJr|)Q#?q%PPj- zH)_qqqg$3X-!^K{BR|B(b-Fm}>}E&gwqx$m>reNt-Tdj~(N8p3wRZM>506g1vvXFX z@w-M_ejc>FO|=qZrZ&u-RAK)uV_f-9)$dX|drZr1_qJHR;K?yv$BruXPV84>vYx#= z>3n{~m;n-0Bu&bYC@wCl$=Zt}v|Nge7`3z>0n?41LCiM;juYsY=L zxOt-^J9~`V)$%Uu_qG}1+Ruxb_HDZj@@l9{uvh|r#Z;gNFOwRhgrO%H4_0)o-XM5D2&|>km1*!KYOeobda!uH% zB@?>LtvzVlq8$^~{Iay!h3dA6%kqrbw;X6P(RQD)>b9@bC+@!^u5W11*S~jg+sH``bDzDX=D24kjEP(9p0D^S|wHI|Jaq4|pNuzx6YJEAan@HvA7kS54%9E2{o;f&Vvf!T;Lu z{{a{L_jIzaZbNr0V!k{!)JL~Iy7aMuI=Y(aV^e)>qK}RBv5`J%crFdkrQx|WJeP*o zK*MXG;Wg0k8fbV8G`t2HUIPuUfri&W!)u`6xtl3??q&*}yP1OLZl>V5n<;qiW(uCW znS$qTrr^1oDR}N?8eUTkuc?ODRKshk;WgFpnre7WHN2)8UQ-RPsfO1?!)v19HPP^z zXn0LDye1l66AiD4hSx;HYog&b*64*@EU7)jWoPQ z8eSs}uaSn=NW*KS;Wg6m8fkcqG`vO{UaW=}tKr3Jc(EE@tcDk>;l*lru^L{ih8L^h zHPrALYIqGbyoMTHLk+K?hSyNTYpCHh)bJW=cy0~Pt>L*fJhz7D*6=icxHW&c-5Q=- z!*gqRnm^nw4bP?FximbNhUe1t)BNGq{NdL8;nw`&*8Jht{NdL8;nw`&*8Jht{NdL8 z;nw`&*8G7PUBZ_x%^xn!A1=)wF3le<%^#R6q@P##b7}r?Y5s6&{%~pjaB2QKcz=4|UY=z#gis;ekC= zS;GT+sIrC!_E2RF59}dju!lNocwi4z*6_d{s;uFGJyco41A9mr?4gbt9@s;bH9W9~ zDrFNL*6<*nR9VA=_)=vJ58_Rg z75-y%|HeQZ%5~lUF%Xxktl>eNsRP-R^|%^xvPCvsi$PYl$JDr?<5a z-{C0sA&z1{;VAY1j$;4eDE1(ZVjtot_9BjAkAg2WJf-hY*6@@*L|MaA`VwUgPw7*X zH9Vzn!CxAl(#I%kcuHTRtl=qrjqd zS;JHIF3K97vWHRD@RYqAqwQxLb$gY4jk0d9vcFN*?N#D^M_mWhxYHFM>IUmA8ySbZp|NV%^z;fAD9D8 zdpE|d`NOUG!>#$lt@*>P`NOUG!>#$lt@*>P`NOUG!>#Q57`NsRx8@JG<`1{#54YwI zx8@JG<`1{#54YwIx8@JG<`1{#54YwIx8@JG<`1{#54YwIx8@JG<`1{#54ZA{V%(ZP z+?qe!nm^o{Kirx>+?qe!nm^o{Kirx>+?qe!nm^o{Kirx>+?qe!nm^o{Kirx>+?qe! z%D;?pYyNO+{%~vlaBKc>YyNO+{%~vlaBKc>YyQyjp%@)6!cpu;QYVbrCuWnZhX?9ibbUMSi4mzFzMEr0fK@44QEFN~@&*Opg$ z+WqfTt~LC`t3N&b$qkw9c76EW*t0cOw@Z!A3B7I9+ry{D?>PF9`)H}QXOq9{_Q}Xw zYn`(k+T42O2lxHrSr&Eo?!L#ySXZ1bTW-_nyq;}S_q6L9JJ)mTlBZXVPQ9x_sc8{= zQavXY_I;z*Cw;f3F4=zl{P(w(OKp7g*i&~OeP!kwZ`Ntny8038inWWL-gDYr@zY<% zmFWK66VvYA`)aT4Yg&K4{iiS9jT$t+*6RL;hFw}VziiyvkKVX{>F15pnk;;@=H^#R zhC8mWd+yTwb{%HCboWYkbVRA$&zwDbX>FyY-|tVmy|sIO#JktDI6rB@>Y5+U`|jT4 zNuRH5`CR97%g-#isoe|9MtQ#7S?+_SHGX@?HOY1V*V8^J_i5X{adAD*SD$pE)r7mV zZ%9r0u~Br#q!tJ6eCD3&t9KrUENJWgBChkpjr+cs@Y&SiCE{A`eq+I~Zc+R5y0@CO za7Dt;GvedbICRLw3 zY1nU*m)akGJvQn4@!89-AHLk$?9s{H-a4}BQqAh!WCKNHtF3wMnoP^N zW2;_x3*?OnJsd1ITQhUd{ebN1FsA5`bHZF|#ujgDK` zY5Q|kGG^!ea`?CY9fv)z;HNt#eeuX$!+YdC-t76hmOICeSvr5V?SVEu-wm6dK6=Ve zpM5%M?W$=lK7Mjs_`z*^_PKX{Fl6W_iKS*Qc(C(z=WhI9(xZD@44N@1tj~k_$Nw=X zv*6ozKkK~q$?H1)SkQ4*{WEjc^gDk~jiDdj_(iQ#Uo2{Is^5;62ahR#%hdkiuj~nL z`S!?TQ)<0bx$^rfGTuv!xNYvp-osX;ulV|_3GbJ@Fk$E5`iq{n+DqCi^&Wh&%lN@< z8?`w5@xCVy-Z-!KZ;w2mer;#V+vDDP(6Kyg&EfT1tCf6ondjvP8x`!{I=6bQU{K; zgGZwiYc=Zo=C6-8{^-u6 zI&IQVpC{FQH2$ggl6Q@}P-{!Vx7i;Kef{y4Pk%FcTKaXr40z=I4sAvsDLt>(HQvX= zzt256(CuzKA2PU-n5(QElB?PTX*{rpQlcKx!0GEeO7&gaeDEai$B;o zGyn1EUE9k%erWQ^3C~|BnD*4R4_*wLT>gV7Z`$B$Enj$i*KbpXggg69P92kQt!wM? zC$}tH(;_staf9#PKRMX8&DF4&U}}>#U}a ze-sn8{rcXcrd?XN^Xr@1KNglSF8s|pwfiJadb!OfFI4Wbt;MzP9Uql5yk*S;_jP`x z#*hu~Ia~d7^X&MU_uaGX_|)ui>n8S|_hO@)&m}i~vF)nsyFciCy4$$k54K0BpBpSX^nbvd=QzVUw3FK?>6V*KzP1Iy&EUf-hN zLfEcT%igO#knm)U^lLiLUr?{r$Vx|-b$;aHEO$t^dj>qX z;e(H_y{X;TZ(pe0dd9v}C*q3U0*4FO-efisOF8A0I>mt7Te)Ima z*Zwm9nd}We&iLVEhx_t#zPxYmnYDe(rudR0j}2HqyT`8eRsONJPOmMkHl++}kkco7 zT2A-HbI(<1{!W!Pp6ho0*e$Wg^&6l6ZAtjqNACUT;It>=C)7%N_RD5(c7CP(FSmZ1 zy>jr52e#aH*R)seanIc_C$)RIpQemo_sBDeK6mT-PiKU!=##zTA@9sR8!uigGp<*u zZ8P#_-D7%zpo~Sdk z(!gKd>iB)j3NubjJs0)EmpNURJowWWLrV8gdgieWNA~XeaA>PdYd?C$alG{d+n>5| zX^GIY^ISEz9$a*I=WCaGW=-nUe$=T8A3s#~!YS`3%Nss<;@F%YU%t3)YOnDb8{T`T zV1DZ1dQT)?9G5VxTB($cXV>IxeQfNG3oYxmioE?v?={Q1g%5f%;?Xz4*3U~hSoXJS zTjEQd-r2oVM#mdZZTaTDU-zH>ZTpEWk9~R9z*j~#PaX7S`jYB<^9HuB{lKrjiX~?@ z?0MH4*Y~e*YVq3Fex5X7PIAGMF~{rF>wj!Q{_xxrm0yW#+VRUpUsdn7uG{0ApE~mO zfgbIzEx)0{0j%E9_4K_-m99JZY5oI;D%aR{c7t=xsG%RYPt2@!=fZmBYkZYkaYw23 zlP|4$YV}JmZRlI8!o5qUl>UC`o1c`gdi{4PX~!1-d}*I!{<6jYkG(sOt7-rL{=e5A z+Pf0TkXZ30Q)bDWq(XyA17(a-hJ;jRLLy~|gvcy&na9jR88Xkq^;o+)$8o-& z&+oc!*B`&%?RT@!xqJ3*@4eRAYp?g(tKO~GI-Xkme*R&%cXhsWn16a_`1i~N@4)mO zQ__<^EilVYe6-(vN|*Du@7!`e|6%8pv`6JX9MEaDZmxGqYr9FW&Uvqj-Dz5Jx5Jhk z$5R)V`W%k5pEz>T757!wcW;`kJa}UK(|Hb4d=xIDq5`^aalclpYef_16(@2&v`)!e zalM_+qor3uEoO#>WX!pA^hoddy0w2goEiDL#~EF#h^$uq1~(13oBLzWi#k8zI&CnT z?eQ|B_mU&+8>b|_ID^}_B z+#Jy(uE&)YDG_ho7D^zs0PfZu)E^+Ib z6g(xj+phb!JXd7ZxZnQ3w#ZX8r7P;+>~DW$tn0GeEyJFEU3a8o>cTNs-W=WA!ERZ= zgD)GZYkFsmJiesH~i?MBXY%Xjd zG|_$-S#H81tGId|+q42Q6F=v!*tB4=zx`g%_QU3vf28_!(^lEEo z?%ms_+uMoR3y*a38)5t=C*5e1LycEgo@czRp>379)uv9%{g+xC9fb6%|)8g61=zuMBqHE-p6#C-DU+~UzE z?$jWMCE&rtMNyk0K5e;HVV~F8+Skhu zE6R#1>c8)7WMRgUHF{kOmu@I*wV`p*^i77Nc6PnCv_XgR?z3akAFa8#rFCL;o$1eG z=CrT!`Q^*?;c13)v$1)O)s16@?%rG!opjAa(bn$hCYQ_>Ej*kWZa>#6RO5`^yIAXr z2ivBoef*M4+h-t~PCuKJeW-E>uY;<56pwrCK59ulAjGF*eY1nW5q4!oBCOT+lAK9X0X% z(Ss&iF1tUq?~&7cM0|$BsGCdLK96W$QM!I1Mx4)dv?$J0o=E}LExm(hoXT`PZ zyroWh-W!J*8(vhqGv?j5PCc6K%$;#9*FP#QuPCg=x5^_vy4bCoFYT2Q8s2bi`!>rh z-+#P#+N@JzrNNJUHFZu)!r$zQ`u@#vb~WEsr{{H>c5l({3qh-=S~%souNyJ(=BmLH z8fxg}teVkO_QSy{zTfeem$pVFe3aLJAD(?XymkLJ!S?4WA6RZ@85)q_6??4l+}L=@ z@^Ob3&(yEbR?{(}zT>RsmDRV-{9x&~BBW-)@k5g(d)YbaXL$F>|p~CgynwI&7 z8FA5XS09duaI925qlaB3)BAhw0}O^ z*6muWDjPf(d^~XFMRx9w$G2+snmx3ns$7n zW9hPrEATkqeFe@g!K zZ+G^$Tj?*M!hZW@RN`;#lfSjU{IhMl@}KR#NB`M2eD{xbSi;hOv<>zj_($8w`{h5{ zpQZ7?wrzKJ%KEjfyW#TFU)xQeL@fHH9rn)P@vdLmi7irG9{$pnA3Rkv=9l(y)BWqx z|8CoIM5le{@AlI<@&+g7{@osxcKlh$mcQCzlk|5tx_14q_QEvZmLZW*f3-6PX?3VN zHs!DO+81YYYaY2&)=rG5Z0p)@h*)|-eLK0K`og^%{5F)eTURi6*LcC1+_H9rX56Qc zRvl)PwrvZv_gY<3E?8UI?t7>A82PzVZ%&oAlM?FB=xjU1?^|iR*Q#Cq#+$YrUscjJ zc%^vQ(@L+p_3@Inkxx#|dq)S}jrdT~Zh2yOMW5CQ+Le~8+qTo&?yRCbH92?5A$41K zO%H=x-S*WulKo2Ez8qQS-E5s*eV0v05Zht)l`FM)*7reDhsyiKcB0b_{qGG6YQ*%Z zE|&R{s8aKs4_N1&-*cfjR%{oXUORM)LZ)b(le(Lx?do<8udAQvl(T(zLILgbigPQ+ z`3*0coZE8R0@~{uNM=5o_x4eZnFpPBGPk+S!q(mPHCw%QRCCLFoUXdX+CckC=R493 ztJ>9z=DgIi_1`t>d^NsPdByxD+o%jO-8kJpq37hw{kpt*dsAXt(PhHza&0S|Gkj}2 z;reXJNw2S-TC+DeN?sOcE-s%cnbdibT&E~zY*=AxlFh{{Vb`xN?YuB4pyR9Y&qlkg zpVhZ(BFy8X) z(~dhEJ)-8^yEt)HIrBC>7q4jkZ1$RWPG{Y&B9rbH zeLJbRbF1E6isub%eKT+T^%qOhcAa_nYF_Y?=O@p1is@aLt-t<2trsu*w`=_M*yR@M z++N)7@zQP6=8Zi+)%t4a(s@LjS8-7Ll+#ZaSUqa>GV^NiUVCG`7{~7qcG^3iOmXjd z=}Fbhgz=m9-x}%Ax^~fl3Xc|^Yw@E?q0^lGw`V6MZLL&$SH}a{OWjgay(_1l>$QBa z`-2N6FQ&CkNXU6UH_?CO?rHPawH=UGP`l2zO!6x>JZxOHWJRgwK;(-7wvoCLfTXoPP93_pcvsMlG21 z zp-r80)m!hExwOu8=pHioYl9OWwu#+R<&z&B$~EbsziPtD4o2=e&qgh;J>KqfZRcZg z;eA?WpWo0%;-Ry6qDRpCQ1^$^p4%@tZQ$~BpVz^$%g)s4omXkzvT08zJn-sVeB|J? zZ#%jO+*dbjjZSNHe|E5DA@e%`@)`L3mRW70-s&*_+L*!4+;Z@#fWxyUp%xPlR2hIbDww12G<)FIWOqtOv%N2q1jR64{a@cJ!Wgg z&FjN7hFpJMIN!g9`%B?weB4=sZhqyT?id`o@P|dm*|+njSDvrXY;ou1MvL|{`)#gr zdCiblU3z3E6kRxc?m@xj8mE;0Y1LNytZvz0*`X~JW^KPM&+*QCAnW|o`=Hq}$K+a7 zB6oKQzZFsSXP4qB)1T(KEg$1i&0X)YuS;~5t$kk4n09DnhXvEl^yxmac=@vRVB^;}`X7!il zxAVV08Dmtt-&(=)YL}cnLX`pE@6Xubu|d=B$h_+x8`;%JnKHC~T^D5+&mn`JEFbK- zy>Na=u(|b|B}bc^tUbInyq0_V)&7zQ@2MG+Lp0wkPim0u96jbxOSjy~c4CaDILuH)C$4x&`a+Mi+*i>YBCKT1dC@ zG55>Sf7Rgrz8jnNM09kv?<&#WQ~SBkji$Pn9v3VMsxst|PI7FeM<;3&bz0c`*4Qo9 zOHW2ktadwe?V|x1Un?G6+WA0MleiHBuAi#yb$`;;w?D3qYx1u4K8wby`#*>-H+ry3 z@}p(d9u*4douAa$zF@lP$2{jlBb?psYU{79mmgU7%ezj&{u8pMbr0AokDs&2u~w)< zcMUJkJGaW;?EED)cC4HJs#{TV+11$CzSllo&9rOl9C0zF^O55#hQ&q>i`E2+Qy|nm3qZ{Qq#a~mltXp@&3olp8@IA_>-LoIt zPk)kpY18q&(xha~Q7e56s=BOQsH$W=R(7sJ^rWLvn{HXT4=wPnU$1`HoF&a}4(Q(f z$oK7cG`v%qCi-OW*4@0~)|+YHAnVp;k|V7ptx0cT_$v2`A~aL;`dq8hQ643 zB6 z<)J^9Z;rB4IlsDS|8=2}apyq?JI1FtIs~ODtG2d$6I|7#dF+=xZRlyl2DcR<|sUtk@90&~sY&-Y=a@iM)c5v1 z_HJ{X@jm28*2{Y}Yv;GDx_V>HthFum13jDG80EhB;EGv`))}4tk)J0EUNCuT$TsN{ z%QHK^wZDASzJK7&;i+3(w+jc0S8a6v;#Fz3s&&=slWHB=>V5qD_dT7fZJw(dwr1)2 zfaN(KI=5>)X!U?CbCH2|2cVbwb0P=}83+?rSQ$`X}u2=(7K1-8(vKoZCeW2`%@n z{?9?riZ(i3(AZTj<@%YZ+rdApE`Brnik;!>jheX!3I{kI%-`Mo*<8nQFDjOsVQSda zsQ6OBmkr}Kog7@c?KF*gtZ;_i|e$r0b(UxaGWf&gax4 zmPzH$FaDJ0^`pP#r)M7rzU|aY@7SIX;fgzs^(Ma^T6L*||GY`&lLC%j-BdVa`|Fwq zhW9iba;RFLpmLSm{cF8k_ag9asNbtQqbfd`*FNXMQebZcP4Tp_; zW3BYB8Mlugw=C=M@u*rApIXarpUs^AB`zChfLRV7}0B2 zkJNJKGY`lbwl=G~B!5@pgW6XsM7VFYH#M9Vcqe6i()^1Bb1wdPUNn4*v+lskF*@m8 zU7AI;-_$UuT5z2Ave$iYSD*ONsJCA83(ZfpefTq>|6IS#%WwF5XV0AcRXhJ++b3z- zPJ(W)93Qu#U&>!9)~?b=JMqH&IisVz)bWdOccXFvqwkyrsNLs@4mK#Uq_F$9iWzO6o96 ze_*dowLk5B_s~w`QSIEA(W_EN>NT`_8gI)jRU# z$atf=F}=5}Q_T%sbnHRnrhBvAB)?ua_{`kr*LJ`r**jPdv?3-hl6jMUK*^W`CUIIZtl&jPTJ1i?$9#6L zqPs2Ip@MzV#-2SV?0ao}pvL7I<5pF2nt#FNu4B=Q6&D(`5Bku|B74W9kh@g}H1sx1>9wfFO54ohnACYO13n(f zoO8a0!O`>UQv(vVrZ?FYXqXnU_UYSW(E}^Ay;&z{-9nRV1*gVW`>LNX$76QvY>!D( zlABMPf9Lb6ug|7!-d*%#Y0ZWEZnqd1GV4onohu$2+E1?^)YR#YrT60DM&`3z8b>XN zpWXV(Kv|bPfp4ZKTfZLrHmm8Jxy>R5CF>T~JwMB&({|5bW0Ms-K8*2tUukCGwhu}EyHVdif!MlEhrH`uG)s&XsWeqEva>V|<)n*F)#HD5Zr4KROwI;%pHme=BKw{#8FpOSLu z>VeBCL#9VMcRPDFHReTk|LB+9E1T8b6g2eb!GUka4v%i`HN5(_Mi(o#`04lDG(2#( z^N|}1oCk-ePWyS>@q3Gt4d1ss|2D`p$!Wy&-J7Pg&+^;R;h33T>vPX6T?#JN%r%;o zw{g;l9`<27XAH}8yAT#S`e?uERwJAe>yB!9bid;MHCv11HAi=Oo!t24$6MpBH|F;} zydcWLZPu55dw;z8u(VJ5#p`=7OGYeSdC7duz~u)o_$vE6)tq6u=jcgE*XLo)BehPB z8rAc@UGlpxhXyxx-PibB_SEdWvxQHVt_)eSr`8plux7W@WVVm$9XvWP`>~r=*LxwA zm&*G(7rhKh^SSr!?BHr=%H5UrpVUA8V#QW_@0*#{XzJ;C!P2ZaC$>}=n;<>6p^DZvx$e>GY^ZcH*yUwW9&$Mf1|H4mO9(Aa6uVCp~ zw}Q{tA0{`Lc;j}3YJ(U2v>(xY<(Qam6N)w#X;v5%*x{;Wn{Z*r%}NFRJkDNh)+eur zOzY8%I~N9zNS-!b`Du}B1-nSoit}cE{L*;aHu={NtrlNwJn707w_TrKFTZs>G)QH0 zBK^9M4z3&%Q zC}@=@(TcM0u0Qr#@WZojnlz7?VLSS=B)rPn$Bwna{NA-LSbKiD=HZ7!yN?{(VC1~? zIU`OVyW1muW%@81yJj{oYd8DwZP&Hcw`HpXx}?;6m=Sxc`sxc^!o7~HQ~L0GnE!ec zFR7s}0NJ9fn!Z?|k=o(^XqS~P9&KeonMmI8dwxKoqUx3tbK zS2dz0>_5N5)iTCX(fHSz_P!oL6a50bfBn`k6LF;yHub2*OM8@l*Tdb_ojTZo>dT@q zgR++R5Z4LR&J+vfmrc~2YTI1WheEyv{+|9GrHk+CBVIQqlHV9e+Kew zS1x5Z&cod^c-$}d`S|%!0X%?Bt7K~8{7Y(f8hMzGF*OJb9Y+;VZv*jhO(=}ZJ6!gn z#HkpJb@g@k`Kx-Wr>}poxcvoC>7MevWpzx&3!4OaGH>cjxCD(NH-is<_NwOrIT z=8_H`9^PV!c#87!mHI*h;ylFqrmlaL5LM4L$j{Hfhl|5;VS+%v-~cxdF6tN<$|cv_ zg4Gv(Qonlj_uN?_;)-+&@Cytyarc-YUPe;=ig?k~N){KCiH#F`oA_-C*_IT773U+~ z%UsmgPT^iQ81;;){`>cvr7ou(5G>YKXSsv@{nZ=Fzl)=a(<{9!CD)D<-%GXy3a^Tr zP|1~Z)Ynr9_7$&OGsZ7~iqP&QmtgpNUV*{t3tx;57Iz*&CH&Qwf;cFeS*!6RTS>zLmX zQ{P#vQ=HDk+2#Dl=X3SZvgf;o{14CH&GU75{;#iZCT(%KD?+dIYq4Fav@JfqJgPuk zC)%Z7i|q;!?-L)}2%O}iuJfo*ijNhSRjjiuPEY)v_*juR&B_qhiC70-{GRxjst~6u z&O`iKe4O~TNSw#Y@w$>UWFrrEQ4DE-uB0;Rp*if)7sKF>S%||r?7}Tn3e=S}LnrjX zV2nZl=3ognU+*#2b=fu-!Pm18iL#r(7vcbpD zme;0MvPXaq{e#-X6!vxFiv>H&+KIHyX;azx zpD&E>wV~ccve5R~1eHCYlturAMwkt(zWLrr0{T+m=Kq}8QznXX)iFEv=g5dtTI z%4$pN@Sm+zP^tu}wu~p4D9nY9()z*#nOv$VXvl6!rOaPYic^)UG_<5bV?kvqEmv7? zCh&0nH_|L8HQ)sbG8-PkBFS2-q*7ar{xU&FP>M5`Nn3W%5$bzLxKCTABmO5Xr%*|? z#8s_mp=tdkpJ2dcq7#BM0vVHC-p`yu6{Y|!W2wyckgbF+(+DbYHFB#fwG?|eo3E0cx`tebPZnj6dXMWNZo2GK zWeTP)-q>C99l1gw7b+Uaq+H&mK!d$(IrW?*rh0WEMzhQ7RhgZ-#8@b=(O04@r;yap zs3GY)(ok*~q|MG-+_RLXMC=&i-qKvh+_|b&Rf(;{)}Xqyy2MH7%I;D8Sd#Nji=QtY z*~QTlg<9fDV2PCMimD6MSsVYPl4uE)H8d!MZZFQ4IZKvt?AC)rfN30%ipRTU?I_yZ zE&1zWkzApIGrG+<3(w5b%*xE#%*L!0r+RZ%HU(2GILX+G!ZS7$$uZ+ZRL&XY^h`@m zfwbl{!&aQa$Jx>5=H?dWmgZLG*5)?mtt`wesNiB@Zed|zX<=nyZDC{4%F@h|DlwMk zmKK(lmR6S5mNu5Htjw%9)7Z+~%EHRh%F4>x%EqddwV5?%3tO98TUc9KTUlFM+gP`< zF|(nfj*YpEg^i_+m5sHHjZLdoEMhBW&sD#fP%9qPs+EbED8{!T}`PWGLvoqhUs4%mU5A z%wlX-+N9d-vpH*f+qT$N+PzBmCf(cjbL{8YFRb6vA=`(X8uHNfyKA*k7G8b5#(B;2 z+8l5ypfEsdvhn0zll{XNhwl#050^$8MfZuG7`GAAV?X3|I1OKB5I>$uu}lRH&3H z^^}wvQcanrTs=9NN|0-4@^LColpe{xK&2rbKXCxdj!VhjRYN^}xl%(#@8kGPJrk*n z<64zM$%nOiK?M$%C7M!|ra~c6FSJ5aLn7CZ%azK~<(F!zBvKBuWGbab$x9LBnjHJF zi_v6aB`-@US4b3m_G@Zr@<8=s$T%jDYD#!I9|oGGFIJ_&9!tRlq%w)7qU3dJ%G5`3 zN*<<=D^(?{K+X%}Geo77Nt8;NX6YJbIaD$UyD$v}?>C1+B`b!7VK2i0m`bXl$%BTSQgh8hr$vW6QUfaQYK?5#DiLK)p0B@9>*x8%tk|lMNvsK zN>-y%s$|6~HQ1f0*zGDfVq||RzTYwpnNm}!Qt>WoiaSe9g}^LJRxaD6rb;~4Qpgo5 zYWXS^%(Y}aYf3cbnsPorlu9YYN*6_Tq)%p(qQ$o_Lz3*W~b0l$=TyGU4AW~l5n)dT2yIj zmTrtv@#CA=*ejQBZ7uBj~DM8)G)iHfCT6^TDcvSBI3qf-@sx>vH@3lfE# z1!PT$`v$f&6`xfcy|cU}eL|t&7?Q-6G8tQfLMCIoEqROOa=An$ZrUnl!bT^q44y63 z&`?Tw$4dGjT~97&b7%A7t(1wYM0~zX#s;A(?LT~)@e{U8A|8^6uTjB@5vRbLEMADG zv|B0E$Jlgcaf?uL)F2+ZtLSwy9?qw4$@U{S+0eyJm(OWd7k!uQhIuGiAL@0) z&kssov64+ny(zQq*ktLBY(wIQd)c0aZC#~ek05R}Y}(?R&L+lQf!D*9&8sfyuWbC9 zyiPVuan~Sj!E6n5T@@Wj$&QoWQM&Q47h(0VnX)yhH)!@!>>$LSQDvI!W#}#?om<6M z8XSO#8lrpS zHl>JL8%a&Bd+WqqTOdA1ybZnDCibf1yO$l>tkr0;xkJTI9v(GEXAF*49yd%Ha{7eB zgoEwZO~1c2tNRBNomcxqogMvh=iXg5v@ky~RJU2P>+j+ehck^6uP^d^+GFkt>X3i# zGCs-bWbe#~2X}mCKIpb&_tN>j>?b{V*l3&Gh0tOE@#2K5;E(JZrvFndw^ zX>Bg&ZZH{PW0}2vb;lyhTxW+lc6ob)BWI@d)abwCe&VFu_&GBcYsOu8R@^DDZjJ8M z?Y=%+`$_xr@yE|9>n4v}u}1I9=+hs1Pn>wKQH{&zZDw{|vPbegqrOx8((MLwX1b+E zj|enaz1`Bu*~K@*Y1IDP&8-%0I$k?`v&F|b^`1>OzOt`VhTg|V4Xy~m&kHx0wDEA- zyxn`buT{f(r_6VB{WRx5(dCsIhqWSnJ@*}2y=+1=--cTD-CpZ7Jl(0tU*pi}TkeA* zo1RNB8Z^QyRzq&HU$6a(#=&9x-#mXWw^k`~yqC;>IzBe>ndw;b9D9ezkL#xmZV@%i zesS%3_r7|)+&QV|>aE|$22|PO-sF0gqG;rhyPa#aljYTToNXS{B+78FU#hp6#;C*R z7Yw}jDfDvYr40#@`=hd6AKU3uZQ{~7N!=E--IUs-iD~A&s5@~#QqR>eZ`fTs>qh?g z=*rr)r_Vd|YK>pb7$e;QG23qRdNg9|tLI9Gb?GfD7mmNx*LBs08_k}c@>OhGlh&f~ zoQuQjw^r3^ao}{$@)P<${8KgFZSJP!mOXB2$c(+K+V#4sO22aR?7n82y`Q~WJhRKI zSsP-Gt46-x*RWor7d8zB`)us}=7ZLrH%+%~_A^p8^X+=!{?-hg(c9kFINdok;#Shx znY~^tJh^86jlOM{Eb-9GoxVQNXjhiw|k z57Fwk`VjKp>O;)>tvx0l50Uj-eTd7y)raW* zTYZRHztx9m@mqa}_P^DK==EED2;JZ6Lk#$>K19lI^&yIWs}GT+t`hNgeTZ|Vbs|cB z%PI*g&9X==`=T^Y{?`8T&-S1C5Pz+YKlLI0)Q9*}AL37ah(Glq{?v!~Qy=0_eTYBx zA^y~d_){O^Pko3#^&$S$hxk(;;!k~uKlLI0e_bDfKW|ju#B+1Sn|nz}p7JNa;!Ril zV|Z?fktL6`wG+?uCreB&V0RQRgG=0sPIaMDlb(H}O1QNm((}#DLz_ z#FKz+C+WT&N`~bmO`9+KAO0?%^V)*~O*t*xsEK$Mw8g*wa(`6t=)dpp)p4+kWB0Bu z_T9U7AK=*AwYR75XfxA)KdGMhEYCm}^$Gk&O`1sb@n!A3|LysIFXcGb5MB|FEqmT1R+4y5bJ^bm9{zgv2Te)Y-{<9jy{vAR zT~F!5*L-!m?D|F(zdcg7+m>E$sM)vY>UP=nf4Y2or*4;BkH_`fH+8%0`Z$xn$;DF> zBxTpDN%&Sy-7dR+O!~LV>b7<1^<2(;tEFz2U0>z#H$!#1?0P4X@6FWhccs@KsrlVP z-7YA-9*E`lw(53F>Ge7Kez#Y*%dVH<{k^BUU3UG7S>Fe!+XH>UP=n0UG{@P`4dR>-Bf|5v^{Q)z2UL zBTn5et7jkdW2L&iptQdHf*(ohc3Hjmlpks8c3x@y^CemR_NCH#+TDNVtJ~#E>s!11e57taE3G#@?dNlK`)q0b z=Vd?NsoTv;>oM>A`Ayvx-*Y3u09U zYB!YDhny%xsM}@r8e@fMb-S#7Vww=AZtpCuXLv$bscx6m7rZYdsoQ1s{=Nxm>h|!` z`g>KS8R~XfJ-imu{p$9E()x5grN`9mvU+i&rDxUcyQTHpW=Joq+hz6CR!H;J?Xvo2 znbJqPC>ULQ@E)Cf?b-S!SmH{VNh~H>ZTCb{&tem=CRzJ!~ zR=MQ<(t1w*vRdkPS$(B>GDCISq_p1A2F^WTcaB7T`S z#i-TGS~fi`*2S-{RfqPkuhq7+{h!_&Q@61mZr&d5E`hqMU)uq+ z4S#w5MB2@NX-Ci&>!p-UZ!~T3v%Rb>ZrSyJX;+~A>-5Y2))s5b{CfYy-`XL+wI~19 zK0;f3Kg#C+m9KwY9!`(?$MR14?fGIY8*%7o-Aurx3D+;M3`c_bXe4-0|9GpT#ssAMK7oC9dCbM zhr9~cby=2$6{D7ZshXS3>>KA|Zcw@BR>Ens~u>2Rk?{bUt zT|PQ0Bx9&$(vM{`g}6?{6ORAiys-%Xmp9H|{x3I93;V~7Q_MKY?C;w!Q&UrCns+(h zwGTO3qhvo;){gtF9sgT9kPXA=m*5CP7HAMH)=9cEJfvNFtRs=(?*t5Mw1-k@%L~T_aSc24{#I@aSV@e90kb1W1PSf zh=c!6aSDYvjUt@EGl-wG;-}>~yg)8qA`itlk5{;W*SLr`xP-S5Kbgf(*DH9BtN4KH z_=FqyjGGYu@Zc82J}7>Ii=V=`@f~-dJ}9T7aRMJ_<8hi9#7|~Ux})d^CpUuA3OQ?& zz6f#OCGM}peU!N05cegzs0ck&g80c@1yw=u07*4eM-9}3_(@(Hbx;=usE7J!0P&N( zAsV4EjL-y4VT@)lfhnk&A~6HUl#-UDImAzXOIX1gt(5Q!Z+ zK|Ctxj4tSkZg4;ki2KoA=#4(;i+<=2M+|@y24WBfV+e*~7>2_c)S;2MU?f~I3T|+R z2S#HI#=;X`@P-e@!54n;hj_lA`1ufsAOvFq#LtKjhzH%Fu5k|pF= zaw)kCE8$0b6&XN^b)JIBL~_#l#-$U-jeq`eqa&WvfPmrf@vNTVVXK|)9&yl&v!+Bi5MO?yVT)|ab z!*$%iP2560?%*!&;XWSVAs(RskMRUgQHUZu!*jgAOBCZZ-r`MZz9Zk`1LE-!2k{A? z@daP;4d3wtKOteAQ16DbZrLwD4o+$2Okws5;Iv*&O=iCU3W7+=fhswY@~8k^RD>QX zp)#tVD)dne)lmaAQ46(E2X$e9)htJSvH=Xy5RK3nMreYjFh(<&z!c44h8AcE@%Xqk zJesi|AlsrHmNWhKO%k+ef6gzU#|_-XE#%`i?%*!&;XWSVAs(RskMRWJv0)*K@C?uK0xwaFcMxCyM|{F(yulZI z#dl1`4@|*NOofC4m%gAZgQOogZ(GtIoUSf$gm_>x0G#bEaRNmUBm+Sa1<4?A(zj$V z#B=nAKnp{mjbSK<;n0CI%3}m7zy-Q6Lq)WJ9$KOj%uyK@r~*q=g%$K+jcTw#b+kea zv_?&|K`pdJZBV6(x_k5m)I|pvz!vq;5%pn*2IvGs*rOpjqY=8GF}lJC-OvQx(G(6a zMh`SYPne(=Owk+7(FY^picxTbJ3KHNV=xw;@PapdFb=-(gFnV20D%ZXFeYFkLJ*2c zn2afyifNdRFwDSQ%tJKhBL)kw5Q`9t#fU>Z60ihIu?)+x0xPi!tFZ=&Sc`R7k0fkB zGB#opQm`4RNW&Iv#Wrk5I(A?uGO!Do*o{5di!AKJeq`eS4&o5ZpO5sD;|7gSs$4J=8}77@{E>p)rim1WjR# zW-x&%nxh3=(GsIz4mVi99hUHb6-L7vV_<`^Xa!HSh8Nnv8*Sl(b{L2D@I?ps!504L zi1DyP06IbZdCVR`=!{@=!31>0M07(4x+4@0n1mjfjGmZ+UYLsBn1()>j=l&(Kg>XX z%!DJtF#r*8!Ym9#BocLb&rRraq(H7CrQ~{2MkbMRas#O#lSvJ7BdH`ekt#BU)Fd~P zT4XAzO{S6M$StG}xs@zWZX+v@+euwAovcXiAoa+dWF<0#tW54AtB{#wRdP3}PwpYB zk$cJNWENS2+(*_V_mj2AY_c|afUH9vB!DKQyf!s(=BsY;EWC|HdZYC#@spMob zjhsSmA*YgC$!X*^ayq%43?tLY8RQOfCb^RgCo{+hau+#^%p@bp-DDKGhn!9BCFhV? zyEB+rqn$Xs$YnMbZ6&y$Jd1#&HUkz7Y!BG;3b$t3a$xq-Y& zCX?65jpTK56M2J7A#akK$y;P9nNOyXx5+K!9davqm)u6)Be#?H$#n7oxr2O2?j#?P z8Ds&ui+oIGl26FpY+Xwzz_}52#sNcCTI#{G=m9D(Hv%IftE0b z1uS6&YuKO_TB8lxq8-|!18mU|cIX6qbVe6+MK^SZ1A3q*dZ9P^pfCENKO8XtP8f(m z7>pqpieVTIXN-UgM#2@N;0AYiU^K>HEIi=_Z}?yweBlRwj7IcP#X00659e_K7jX%faRpa#4cBo4H*pL3xQ#owi+i|_2Y84_D8OSp!BZ5X2+!~w zFYpq@c!k$^gSU8x_xOO1_=L~+g0J|7@A!eAP}Sv4fF=ZJK?-fiP!4kFK!NhmKm{mK z4Z5g~il_lS)I=rJLS@uO6&S$)O;8U_Q6I)=fMzg+2^zu_jnEv8VFNR?LJPD;OSFMG z+QI_uV2SpyLI+r*8*I@X9pL~w^gt){ggttpGlrlahQSfeaDod4VJt?%6Rz;WD0ssS zK5)l4c)%B<;fFDpjWEo?49vw$%tJV$5rO%bg&0I)0iv)J3$YA~upF^ifyG#fIIKcE zRwDsxumqcsh!m{FW~@Uh)*}r`*n$n%iezlVM(n|M>_s}V;E#P6kNwz*lL){m1mZM; za0bCRi=()X^SFTvxQPk4g^75Idnm+x6k!scVKSbh0G|+o&j`gAOu<)7#W%b~qk1e8 zzaMH!deaq*$pW$^*&8F_4_A!GC zMiUG{Qw)VMhM^gT!vxMS#RxQq3r4~fqu|Z&x!uUAwb`Dyd;FNXABNLJBq`6=~Rlt=NX`NXHKBL+M6i(v|&f*+$k%#lRfQz_<%eaE8xQ6Sv zft$F6eB8zz+{HcI#{)dXGd#x&yu=&4g>5qZ0(R&Gdvrz@bVWCGhXZ<`Cwieb`k*iR zp+6ik08SW)K^Tl77>Z#S4rh#j3r4~fqu>U2cwjWfU@Sc01#kFZ9KtXIGZBsm%t9oh zFdK6)7xNH}`G~;+EW{$jVlm8>D(FI-64c*~@9_Wc)=#4(;i+<=2M+|@y z24WBfV+e*~7>2_cBjAFOaK$LN!5tnLjWHMtPk6x_J{SjI_#qUNFd0)Y6`|~RrjgST zh8dWNa717hA`yjoh{k+)us>NyE`7Bx55s zAqAT;n)#%X!^t$#mE1!5kXuPFayz-2Oedqr9pox!$i!~!!CquxANC^~ z2XGLFa2Q8$6vq&UC59A==9IOBN&c>?2s0^@lCc>?2s z0^@lCc>?2s0^@mHcLX-5h*r=;Yg9rTR7P7=K|54Md+4JBxQq?2s0^@lC zc>?2s0^@lCc>?2s0^@lCc>?2s0^@lCc>?2s0^@lC zc>?2s0^@lCc>?2s0^@lCc>?2s0^@lC<5dDpg<5mJ=Ljq%0 z0^>sh<5vP>L;_=20^>vi<5&V?MFL}40^>yj<5>b@Mgn760^>#k<5~h^M*?G80^>&l z<68n_NCIPA0^>*m<6Ht`NdjYC0^>;n<6Qz{N&@3u0^>>oV_pJdO9JCw0^>^pV_yPe zOakLy0^>{qV_*VfO#~rV_^bgP6Fd$0^?2sV`2hhPXgm&0^?5tV`BniPy*v) z0^?8uV`KtjQ37LZ0^?Bv<7EP4QUYUU0^?Ew<7NV5QvzdW0^?HxV=97V3C3e70?4FnTFdRIXKL8PlR6;M=)VlQC#6P54x-m-INcD9AA zk37#|cQbR_d*AoGr~J?VAU7Z%L%xW79JvwcLvBLWA~z#vBDWxCA-5uDBex;vAh#pu zB6lF?A$KC5K<+}$NA5;0K<+^in1o;YbDRLij8FD{zIr3HH3gm0ZmB@q0 zr;vw`zNx%BWG(V2awhUk}LoP+0M=nENKrTmKM6N)-i+mFK9&#n}edJTfOUS2@myypPuOOdA zet>)q`5|%@@+xvQ@+0K)$d8e0ke?veBCjD|Kwd}IA#Wh-kvEa+ke?#gBR@lKK;A;W zi2NM65%~pj6Y@*sX5?4MEy%BtTan)&w;{hpZbyEH+=2WaxfA&Vau@P8ayRl0au4!H zM?S&n=ZIT`sFQbInCoPzWr4zd<;kuwnwIScWTvylKf2U)i@^9WgwJc?Y0d=t4I zc?`J$c^vs7@&s}t@+5K-@)UA2@-%V_@(gk-@-5^x`-Rkmr%R zkr$AAkQb4Ak?$g3LcWK58Tmf)733x4KICQOe&iM8tH=+KuOUA~9zb429z=eGJcRrh z`8x6w|$B_#wPh=7@3z>|}My4Qhkg3RAWD{gBWEvtN(-9fj6j6{Fh>C26XvhZ;9oZZ) zkS!1s*%Gmktq>d88fikdLH0(rMdl&fA^RZPBl{vdARk6{MCKzqAq$X3WFayWS%ln; z?YjH}r{p%<>W#*)@6&9)iRpU-m8&BEk?b)Zy+EaRCR{6$tv-3AM&e`$Z#<}DEbBb~8 zd+UsU=VqF@$&UGo%Kz;~Sevx1F;2xOW@3ocZ1xTaGI_TlGzNbF03o>$hFE_4@7Rx09v({+rw1 z`11N4#=X8^XX%ZZyNvtwo4ZJVoVn`-f9%|K9372lp-2Cg33>Xe>w$xkb8O4Gy5 z8a}Z7=5(LjD!7haPt6wby>(j0->c>Fe%&;M{x0Pncxvv+pr~ ze&NO9=35))F~@%V);sU|y8gyy=beAawZ%=Q&Dd-YS!-InWa)v29_9xZTy*iJ_dWX9 z<4=D3<&*Baf4l7`Oq@Jr^Q{cK>E>I0@WV;wWtZPPVe;PlhAo#~F{501+Y)6bijcJAHMA9V~#uFq?1qaf^b#WX&?Rg1vlJ!*WKTE z;m*6-%TN8}wNIYB^BE0=;+#TQ$n7<^@5~(v^7QSCADXmd!>oq=iqrP&yLCJ*;$%7~&2QYTICYXyXlmGY zLUHPZ!Eu?Kj_Q(xF^zzfHE8FnRm| z6L*``GiBkvdye0`Ve=_!5xF0B3X_^qbTn7;p{?sUbQr=~Igq z55N1w*~R>{NxQ8*ZsHE~dEL9XBd@9NUC>9gAfh=}o)$q9Ce4_<{giE{ZnMdz)22=@ zZd%xEvky$#GPhN6>)bYlZ6|G)+djY3mW_p3`8iYO=A@#WS8})HZ!O+7@t^s38~#&x zZ_*v7^paETdbIq{ow4&ufFD)=bqnW z#vV<3FJ5}=i6@u*;Nr_~;fN=mdHt=Qzq84V#Y_F5@9cXYdiar-UU}z_XPtB54L3gg z$P-UJ|LTtp`0Ur8ee${ImmYf9vB#fWI{%VO@44^(M;?9hsaIxfxz&j${q67nd9SbS z)Yo30wo`j~`yEOj{n(v%z4rPc4?eWzRy*#rWdB1C6N~0!AOFe|-+$@H@BHzvon4o9 z_v|`%ubb|=|B1j z2{R@iK7F$ZN7K4#`^hwfqBj`bunJS@lr?31ZrT_iKRz@+{&{@-bN;3A&rduS|6KWx=x5*Aa~F)`zZW+1 zqMv8K@|oyo@a(hVXJctyejz_$bTGxw>#p3kji2*>{(0#-eqOfY;}5;ePi@z~u9~`N zT;CgJ?_&!Vja&DFbB@(kFBfNYH2Cuw}cFQ?ijGmLS#QA&((xX z5LaDr_OBlsH)U(_tN2fhch!HNx~x!aciMt+n@;a_oK>Y$dxFm1xoe!Rw%pe)EnMF) z?zoi{ul9B|H7)4ubb99Yupv3s(&oWei#_Oq&vPX`?hN~ zH8q=oOnXjKlWkq`>Ft?Ynx1)SchKo{mph}Aw+E+nr{@&Emkdp@9a%Hn(A164RymdD zxOxzpwkNrY>8M)%klIrn%8}9Oe2VYV(}~|py5?HGqnnbVx@O=joX)isLk~?wk$qKF zOucZ}pr>2ywD$C$FZ!codmcSXEhn^0N3(2C+@GTPrX4s|VA_TeSjF>l=^0B+P4h%2 z`$jRuCAX#BUlFu7HF-`KPkmt5B7Qc8<<7?R`-eCk^A4JQ%)ShJOM6$h)9wZ3FnyrP zzm)i1Nj5@7HeK6M9Le%6aVMdoOAMT9+aB#qUAf`F^u(3d346zi?&kEI@n0oHw`E5P zwb0OQN7u!@JtOx;tCiSELWNEc&Zt>4p~QI*o4a!1H`Zj9uvIzGs+@Z8>!4 zwWlnodXG&_?ZKM#Ec{whWr=3Gu3@-FVC#X*DSTS+vb<&4HXX-s-Kh)HQ^Z3);2e_T z)3Vp~3@?P~X>iRY_U_2qqE#y?(Xdfc>NwV!6iMkUEi~OX1+vkVuRDmj%jj5Q+Hk4SDkRv%Qjdu zR7Pny-B9^v$>G^dUvqt->{!U$Qs_9A>g%XK$CG^5o^izn-%|9>7WRZ;&^g>aWxiU{ zRUeJwN=9g)yg~-FnZHHDXxC@PIkqZqR;{;ChFv8`Y)I)uZRv{6Sh7*UR>06YK8x2B z59nx)Y&e05)(=C+l0VSDr5)PS<_4@U!Oy^TOU%1qh4_bfOKa&`#=kC&|FI;=fo*8M ztZ5FX*LBX!B2qO!;7fWaIkvfZ|6QibAZTY92JD{k8IwOtEK0`=ELRS6E${X7R2^CICBsw9 zEmx+~INfZUnp(@{Rb8dlmQ_4>|EUjgdflM%odLVC;;>@T#Owl=>Pq5TE)B8`ofh3H z6Dm-ht@=j+d!-cx9BbdMb5}QIXXeREe+a5LORa)5S<6F+0y`{ZG=XBk0 z7*ktPJbIsVR)w))xAj!Xm1I}7w>x3bvvxOkmizT}=ETvzW1Xn7D_P6|S23_tIJX~S zuy`JegeL2({q0X4^xQ0r_MTSe-oV?AKPmC_3fmelP!41*6pCF^9h$52HXKj(nBF^_ zGU)kNc40%6f>Y~UApNN1@j4|2HjqNqR$ZGbsA1@9^kv7gFyugWcU(5;6`~!ql-bdu zy|mV!dGw-ZM^=5camZ_wBlD%s*l>;bvyzLIqWYnwyC^)&PjL+^@MOo}jo6Nc zX)$xiYg8Kznd!qd;?GKKxl-seH`yJS`kELk$uVWwWhwA1*uFatzO<5))L&Ya4@@t#!)T`NXR zv%HG)z`aGY#MKcWoW?O7S{l zFX#$}T8QZvN|NK7rfZtA>AQ057-)9rsBDV7K;Q6mbGObxuNE(x$XRr{N-YMHX~4q` z_)aO=7rCqserJGx|h%n;pB%sn?0|AZ+>8ZiQQDYGP$L7$ev>ey6jd%+7&VNHnC>)m-lE4urSZ%;GZ~9AnIq zkil^FT0E*V1*bV;Z*DeK*H}ir8O4^(6?kA zTi#^HZfdIk(!-XZ)h|`Qdet>BOnjT6M{gMJ-Ye_+^0o7Cl=Cxw*SOC*#wPog<#_~W z=FM+*x|&@qKo&u#gB^IJv*swrZ4KrfHCwErPBFz-2ZTMu7kSc9Ruo&3eS&kWCP&}r zprMt9MskMbF;55z`$f4v|<~ym&Ge+_DpOrE5JTQBu82>CaEK$1XX@OU77(IP zG(}rzXJ@ns;&N|JCtGh%dyB{Ym4YtM=@7SRTD}~ZA+LyV32%InIqc{2`o5@VJMzu$V-%g_k2hc(dVM^n`y>DwN%|?0pXex&)dH(drTA-?T?ve zH4DG<6sf|GVx|KQJd_b-4e-@?TM%!=w(x|AglfbCjL<)zZ7ddRzjkyqQ#J5IZA-&i z@yw+o7?DtO2%574$=Fz?W*_+9+e&m@!!0t{szXDz4ey|;y)W!J{d6{*=tn8a%n&gV zGmdz!ArTyvSXzp09ehGYrC`k{ST(72keQ2EI4#}molBU$OvmUIqP=-x(S&9UIX7;x zVh}|Oh`6{|DTlD{iWq2QE%l!}T0`klik*3m)m_t-eN(b@Ebc(n4$ay}{m1f43${k7 zOSqx&VyyZnW+;&jkHDZ51{&-6u=3FR>ru{%6@i~@@W-@twZy57dip;3lt*4IzQ>>n zts&#lg~U`GM^lLu*@rL8Jo6A1Vp~sZsiV9`7EZWgd$z)zyQ-q%4$4PVO}xaYi=VbK zUJiK?PqLMepp0bLM@m`iZe^KJ1<48)lgAF&AxsafBv1AU2x$hclkF^<5#^7v8{|nJSEHciveyrZjj?da*4Lx&3m%g^{}d)l2&aqH~WisyJ}*T6CeXCEVF z)knqZB_meujua~kSXD#Ykt9qG-wlqPmpvj@XYs1&Ci~RXbjBH8Yq_hZGx+F7xgMT+ z;1NMbH+q_?O2^6Bqq|VVZ3O4zlu{V2i)P6REuVO~qswxLRy}?noDGM~9aNwU{|^5UgTNa~CasNrbY+Y0%0c7QY;N#9Vaw#IEd( zzdr@NM3_^@h@4uDnsz*&pr%4(lY~>Kt0z@$>Y2B*rraquJ(+lu#_Kd)S63|8SnkbB z=I(0YO&t?>WCO(t%>HXvN8ptrT_fV3>?Zszl#vL3Vm>VuNw>_$%^E6MWSo2wSDN8@ zCmY$3NMGTysb;Wd8ukEuRl#i+Nasqbo??+^IXy=IJ&2( z8TILvdxRt5msxH6R`Qye=NyC^XS)mUa`4mwm-kY(mOBrQ`2jDs}dBb$ch0$&_`GJ@&AhFiDmRzoFCT+Su_p zoI#>z{1#_Ld%3H-#p?>k;*GnmL1aT@hS}KMKv^-2uU>5|N6(p9$~+;~TtMp7S2SI5 zn#baHu^BBxSR8D~A?c2mF)?iuPb46?sA&Pa1yfX#SB~|rNPY@)+L3Gr+X!oF)xq<2 zt(w@8!O(Bu9v)k|%yD9ALNz)fmKGZjqeyMhQ*WM3?7S)nSY zY3!$BYxN2F3VQ`-QMFCU!^!t;GN?}5?5rM$<1|f8(Z0dAZHIRug(2B%ALCPQx8uc? zSp1PT=k9RXqFY6`r+jR7eR_8PV(qUE z=)05bMQ-5yC~Qu!a-Tbw3|3Pq-}IOLERN;G>} z6c_?K#Hetvq`8_wjz!iSji7}Q^76I9gQdfr)g*zn`0G31CnGG!fMA4TU z%_YGggsN%aGZ4Gi9cyiF_ModqyepmjqH5}-%tHkr#hJN%vvVX4qbw6pj3c9nO4q8E z4qcXa#e36b~O5f>q4uP#ZfwOFz{?-dSyT@%DK5JD>L!kg}GenAZO8XT0E&aQUSU~ zst`1ap)w)u^Kw&TFRr`1DroQeL@pa49R684TUInz9)^gpz-e(_Uo*kQou5laKGJkG z3Nqe@P~=3jq7@6afZQ1vDV@YSu_^t6+=3xreMcELnvi^{95OI|l(b-rmI!*1C&4JT z1NFk(DcQH*Z!s*Zgv2V=a(!A98!42gr)(zvo<(GvnK=PK1Hq!&DWi z1~ENX3Vj0DgYbkQnHpnn$=<~|D>HNI8zT7l3WtInk$lq0uFC-1UT{gSI$9HLK9Nlk z`*S>~2mo7-qi8niEfUchY3WOIW4_J>~xc|TjW?&tG&Qb%9 zI3y^^Pv-W@T1b_2k3h@xi0DiRV8zFcC-AKqs_tKzTa|g)b^!>BuC77yQ2n2$ z6A45j0jMaW{^-4SUgAEasX2I)44_QU37>@p5IySU`dt;#d|>= z3*m4=^xR0Z?eVYCbwiGoaIDi`Db(U;}cac>7&+^k9B&}g&k}3l1nh1 zdy&De64)fk9I#;r_UCf)K7gQbVUF3fq>@>AYehgp8dchh!vK?z$|QhUs0Co0EOecyAGTVzI~Y zeX`J=>R|z@09Nsmugxv1T2{#}-`y$p_{wv{YcBp!e8PCeFC==)1h4SlB~Nu_EC-MS zUl{r^RE6WK9wQ*P2P(^w2ttE^QP%aZ_G;BGmSJY4Me&x}k10Nrym%8Ixdo3^-{c!foKdksLjnaFY=d>sMgS0vfF zA(w=VhEWgw7AQjo@(a7uwLq#8&nFc7#ava$H{MB(5H$?_d8Q~N8`u&;_`J90s?5?_Wq=iss-?<6 z`KlS9SVfHiKRxS?k-a4JV@L$iw8$$0nD+1d-<^g)B5c5qBoFY4d@Z#Am5K=5B~UPmZm9Q-KnWx!P$eKE^0guHJc4?jp#=8*xvr{x zOe_!_-yv<}{?A!EkwF}^2Et^jwUd*77SD={X@MbO#(BD81Ygaq&Z?f+wyLXmY8LDO zG(mhQMq8Sz%Iw`-81T6(Ah9Ss;fz>c8!Np-o|3q}Ws%dh2?+ZSq~3IOoPmLA5gWMh zUV4B#CHa_UQUjr5;&GDdA@-;~n5&9hN3RiUA^IsE3qZH$*a{GIf`Xd+Q0}AI@vQhp zb;p!BKNLWta`d7YQis=Gqt0y&V3BIrfLU>KmGJ8$*0M>$iD%#`2`yv7vf&eQavsk8 zH_#NZL!^UKXA>re4|+6^oJan5RW(p$DI|^@5XU9TrhTK@4NmOC&YpIFO5H6Yg3T67 zazVhDfwvpB@@Q_GC~6d&Tb*%egnG0i%WJy6nNt(5GmdX3k%`zF@Anwc_>^Hfc#*_< zLMwP|BBC5=^r=X3v>Jzy!$=?yjNJW?8#5(=2f<}%5PO0TpKuvR;*4jUWB?ePo)5uu#zTQ&*Y}W zJ5e+dzm+>KHX9<{SO+#UXtI_Kw5!E!?JaeZSQqF5!>6K_vONN3uOE#tnsm=14Aclu{{G34Sp4stF(@Rp5c?IwU)E&HUj; z*f{|Cz`+2{42UN&K;zZiUWv~aS6HLDS^f?HeZaY?5fWQSoIRjkT$E}yQ)p;Y*y#Y#48WAS zKS@(;gPcwj(UhV{OAHgKJ{!w~BDpAAA_;fvwOspvC|wdiCht(CvzE@TUvn}bKY@1a z?9WiBj%!2Rpt+Pq=@gN^{=d6Cgn4I(5CpZJ6kTd>XV0dDsA<9Tt zqo8)AH>*=tu|p^_Y8h;RLBS^+Ckmz5fJ9aAr>V|=KM_tM5I-BMy`QwGsN5i;DGB8% zzm;1&VCEdw9vl`{bY|Vqt_3Q~H#9&f)GY%9{(0__mZx#*-Ts?)%lk|(+R;6g+Ma6lg2wGK&{EuJemQ)Og%5PPsM2w3lwK|^9(yfhDn4x2^V`1SC`@bIR6&_N9 zuK-pyJd)Cu4uFQ#-i_tpYPB73f3&SMbz!lj@h zjA=z>*B`1K;&}ZN4HBxIGDoH%2}`QU12!)MV2o|PotvLs1oc!+c@;SRML$e<{VjG6kXZ*WCWGg%Bys$mmZn zVHpY^0SJp?1U*sWG5RVDG!EsQRHw@jvY4{{x7^qvYh&w11&C!k(2T7 zfwNK zAm#PPh>{rh*Q^3!2m3=xQihwt4gZxpWaFI-6X4$)^?b_m|7_IrsmtHXt;w$5jTUO} zs@@NIm(UvE=839*K(AP6efd!>w79-3s(R2BV8@WykmZIkO*hx(6D&myi&#~%LQIA@ z3=l8xst>u@dYK9V7==>+bfzMZaK@SWjX62AOg6g{-`&A{mLaBzziG;nm$}{;Bh(_PAYKD z$&X=BG*k^xB0x+{=wYa3J$Edg2t+@A0#FE`x)6`(=j9hwk9~}hqG+KtW>4y`LF&g( zu@1=LP!CCdlv{;q#{5M7$aqH9aW=9Ow!d~P33~NY5is`6lr?cjE^ttt#}AeB^G9#| zlcLTp7?blV%P$<0^Qy})8k6&C%P$_2^Xkhl8I$uG%P-CEJy6Y8BW*I~tPXz<=myWV z0?BtI`Let=Pwba98)_|3h~h|-m?sbv5WmdD%kyJx0J#vcp~Hx?Lb<~)SB%w8U3Imk_c1m*qo{?^pw9n;7mPKfbl62$(14R$O{gQvDwBxG0H|QNN z#>t2Q74`17@+o(_+QEUVASsBQFhc}Qm8waM(AD`Wj7S=R!6a~FHXlb+|R4<|T8XVu>CL!)=m9I-GL-2woh2S}$&Q zFpzehZHNP^eI48)@jR3Ta%&`C9T4|wa9y5on_z&hx=m3VcKv_lg;GS!CPVd$069@g zaYO!NS)E$JnQTZXp>lKur6PQi{&~fK0Ple08NCkJ0}>nSi}{U6VX}<*RdYQ&X^5HO zMTHuGFqeH}zB~KCY4l@59ydKUik#G3763Hmvg9jN3|N3UH~m-MuBSpJL#o7ZF?ew` zZvL-am112e6yY^xw*mnr-ICwnTq0$Ocz&<|la1DNS-Ul#R2oOCiM&h;dG$D(9{G+D z3s8qxyh2$KQM0Hr6agUs?Cx!2gTTKLp&z)g@26 zV=ONw*&-FsV@a^S6WzNr|KaS4iK7+$W^DXtHG2nnX`R|w-;vp}%)q@XziE}aBWZMZ zzN)Ax-8ExGO4#ba%-xLh4_N|S5LBY!YFhW?odG&LDdvf{o5ZTCx1-nAZMePA0&)!; zSQaUaXuM3pn||*YT*DQlubwVgBgi|;-k0)9{XI0IR&QpX5jSQ+t_FJs7$KIlWq&!J z;9`fc8sgDQ$WC#H1=nA*YqCBuqa7csmcqcV!(|7#FTZc%hQp&)isC46se$QQDk@qcN};j z*K!p9{?VQQT6vIe771V=f{;Xgb;JY0IIJKTkg!#QPZHopk0dn_Eg$@8{vb9e*u;h% z+0+~>m8p>iCMvSi@EGC?K?P=5ssmN+1EYN|AVAO!Krb$^_?+#*(Vl?X1BhIJj*Dtm z@L><-j~MRNBs_QGHA51@)5Cd({gxPUV$ukbQXR5~fU+b~G+)mrTpo4eCE3{&rB0$L zlwyCO&=WN37Uf-V1`|GfIKObX+Y0AR|=f^a?GfnM6PKs2dbS`8n~ zr=>;Jr>V`6#S^JwVn&HLOal+1mNLonW=nXa0cbTiD5>PvzB$SpGhuLs#n=;oeQ<;F zW9f^DV1_B5E>?K^VKl$s!i*yYD_(HM8LOk*?pz9|+aiu7nbIPGpM)<1w4Ulep10!L zAN&zT#8ymtRX{vE&xJ`HNMvX|kw1KZW*(h&-7G6m{hHK=n2L@%_MXhwDFlqn`cgf@ zC|qQC*r4JeAQ$4WJe6NFXe=)NfO+2|nAT8y{H$}a6u{m!JF6bDZ(M|Uxp9!+iYbw0 z5q!+<(;H#=$rg=YV623c%oQ`{nT>Ex`l3mYk4g|~+Xlt=-^w4Fyt7#EWgo2G4mw~e zQ=%->lR|jGREg=zxARF^Pxbr};g+m_ONl0g;K}yu!WM*?Y_Uk`Mc{?sdA~PBiwvmM z;GTn8)+B84Y<|luydkQqjWWW|MFCRrM2f#@oOQV?4Rn+J63x?BNVzVnb zZIBMbt1FUeI3K13myO^HwoqL!|e|)bt8yXY*tVv zz~K%KM{*r%Q04FD6V}ZdCDznLiA+1psSHmP58>hPFbK5~od^3!pt#KMZ-kWrl?T)|D11UY@Zzd3y$}C|z!j+%(ZNPA$Eq)HxOyf6NzWfj!zsXo z99?@Q|Nistf<<3~gcj~H*QQG42R!Lu%1!8icn;KyhCnw@(Gy%mI!Fc-ukypunk}Fp zsLY`ejZ4oZsIgvsze}Ge6|NBvCn`x`SN>6c(Y$I^bYNFxmDE02)p`&F8Y=YpswT>y zCF95WiKP;RR^2V?Px1-ayN(ew*eHq}LUTMq*_d~R?SgQ&UP~v%`#cY+q1Q*ZOTztZ zhaOA;RIUiX@EiG~hJ63=Q`Kk!9w}RU#{;sR(6R@%b7AOFIsIn-==XJ6{Cv?j45lK{ zYRraaQ*(5f?|zy;FPCWG$jnO#5-Q2R7&Vgp&akwF*|L-9tT-_ueOt|D2%?NAjzdC$ z&Sek<|15uC_LbLea+w;|6ttf488pziP@q$~0$FijYj5TEsqYZ>2UV;6YZ9-}2oY5# zGEY}2Ap$ZwO~Ph+?s z)P#a(@R#Yl9=$Efs7e-f5P~GcgTUigqcw{iB7ATnz{?LZ&Q*S$KPc;##6gPKdL-~4 zwK1w1(yBk2VPL=jqp=x54BK&n-{h_O8-GF@9_{)GjVPUopg+VHli_!^)!&Z%P^c5Z z5$X_w!|_Jg<98!JR2S4oHs$Mv;4n4(-;exIE2ND99lZ>dn4xOQe;E0pzGDaYy&89` z*@hXuJ@P{X%E1ju@I<&e6z#@4BR@2x(-r-<1iBRB6wCi3i>JnYKRn{Fje&Rrl$8jN z;%`$mL7dPql|PMgs_6k8_8_!@Bv6H$!Ts~dZ(YPS0d}$InW=Qi`^zXV4aJX3j6nP+ z(Jd(WE8BMkm>U7|mi0pI*CrNv<%0!caV8X8$vkeR?ua-J1aNROkUk0jmO6hm%Dysu zX-~LjHZl5H;JcD-14L>5edLFNVid7rP)Rywr=tBMFLDEfUMg3u$;=8zK{Osh9?}4wOW-N)J z$8|&0eLI3<%p?vb%9j2!n%4kR1+Yz&HDwu;Lb~sb=3t8!PxRQL2N-n%;2Zl2|BWoB zKq}DofR3mH(#dG9EsQQ21l^xVy3+LmUy1*nSvWL)Hx;;QhTbRhQ3BS8OpKNcY85&n zd^9lfaIOn$^Q^)EghJ$QS5{v_FCFN#34@u>V?t7Ar=DF%sM9J7sOG@bmLh!XK~ztN zQs`&tEqTsJFG+Za#12IRY-t3af^$bXc<>pKOyC4;h#^Yzyip!c6Ela}AKF_Hvvt(q z6QewyozbIXBbzzpDewuOKg#1F28UrD<}(xlI?uUaw8sw){6?K46XJSCk#VN+d&G}Gx^dl2opELc0%{$!`els4}-RMs&nzkFR?5* zbwrn-si!Lwdt$S}>h;u13j3!Z%v3GKIUI4ecrqub`qzdeKuH_v{wGIyL$sN7;mIMT0T2fsnkx&{c#4*;#WX1I{Zyd} z!V(jT4azB+7V-(Z1p|N!1cI6)_;lgeDmO7X%}7Yg1oM8%UMWlinf{<~iRf%B=tlK3 zg`}<`!kfe(tLRryCKRbdaSeG`#NR@%QxrQGtw3b|jL#OTdd_5HqAV6@K(iM`8Q{{h za9j@>UHx-~#g)Q{PSPnN;NR2U)zi@-nvjWTAdCS(l}ho4qOr0_L2B5tFxOMh>66I` zm8%L%hg%xavaJUcR%6;z^IuqY8I|(1ikW_Va~Q#(%rU>^#!Balh1VEi`8Y^yOC}y!j4cqx}Oq9knHO={C$g3cA1t$gg{lAN*dP} z<_z0lAo3XFDU$;w_)H$+L@HTxxnX(7N`c-C&*@WFx8 zP!4E$Nivk4e3EukVb8478EdZO)|vx{vS(~YTl5pgT)cUxFT?#|v#k?jq2V|I=v#*R zGTT)^5}oJ*rL$yRZr$*$3j-9ZhUfCJiJ%<2ZBR?Htp$P)GUzEL&=rv@&!dSPRG(}z zx2HODfDP*RbWqnOlDO%y?2-lyz9eV_p2c(S7{#%qq3K-X5s-3p&^_Lr1*?A4Ea~;p z((dtIqK1mtyr5?YE>WdPvr7H04cTY2WoIPV8Z-!GCjb(>7a9ndl!+37 z{z@TX%8u4JNstR;M@epptf1k;$qOHX_U~3dcoD-|)VH?o6%sTW%9EWy0So z$ZA?e+k&K4zrQelm=FXhZ{dB3>SAEBoI(`5geHEZ%W{NZ@T-LphX)BeQ3V7~u1OIg zPU+W1ekgQYFa{BrB*{(J8S8<JMPaC>=wP43U)&UFqS%`{#e6$CYL>4`_5J^qKz1P@11!AEGHaiCrH`B~|@KVNR@W z2UxhVONXH%tHLm#=a_&6Jz7Y@NwlH_KS_0WWY$!|NifNn0Tw@Grlw8--Z#_bStm)A z9mbdwv8Lv^nfC7RQTi|(gsx3l%43BY8Kn|8t9`t%M-?hC3eU6A2oAKyDB~8zE*b@3 z<%td7x>cUW)CK=Im>W=1PmWYmP^l+)=5+}5+06s*snKf+ItxW~l+b*mac%qQ0yTd_ zPxv9>^L5Rfzy#@_g5!ZkR>wYgrjRg+N8gUmhwUpc``NB4aYb(2-2hGC(ZJWC*RT&f zjHR;*HwwWpE)BdKj+*?ZEV9ymc%Y29#AwuZS7YDGdRMqFNWi0EI}QK+hWAeILdmP&zx*Df}Y4ZmZ_^Qe&~&HW&HE z1Xu@dv_;fE5aARcX^{7fKFZ+xg{0+=7qr3{5If~gt1zG;(k^#`pNTwWg40CZWmw5v zI;S}POB;Ukkn4ff#^wk)6}31oZ}_PqtCe&BMl3))Cw!%F)Q}m7B>9kqKh0ZAEUPAe zZbRmdn@yuc61#MZ|3P7|D$R%mGR)NYVc|nnM?|wC7@$U9O($KZ72Ggq{!wA)c%FuF zNA<{(ubr7cPUArX=R@|{d2re_^t85u3ubPR93iO>3WUlSK&JXn3Q10VxU&EtOF+(o zX{jXwax+tWtP8qf1B;|<$7_YzL%6S^abC#{R)8QNuL3#9>xJ#($Bc_QY9H{8!gevZ zI{H%Wr*9UH9mzBpSeQ3pCeRQohS8{htroIPA30XqdcA@rvU2G*Fl9WNmG73%Hwf2LB$Y4!{vp^ zL;KArkH>zZ2>^9}q^*FTsJ|WM@k}Pr5YYM%qPA%f|GQBhPg`r5NV13hY=RMVe_u%Q zv~^6v(OWgBLBRt<)QO;_OX~}a)jt&MT2|F1O#K$Og>F;yp^f8BIG7wOfSK@G(C!B3 z9tN1VNBMM;=wRc}qgE7VQ2z1GD36C9g&vCr1Irq6b@h)0ZJt{$w+<7OB54a*4J|Ts zl~ry!e;Vaw>8Sup(`QL&IPi-9=fZja_0q<)T$&x7;1QK|P*RaN_)9^nudJ$lDc^*M zR#=PU18>--JjVQ6Ves}CiSCT!rjTr>l)?m0 zxf7p`#$*=(uK9m2436&`@kt16|6`+^M_K-Vg%xyKjPMcFJ0kyqz_3 zjorTr#|&fVr=ICyk%yY*8u}r5Jyt}i0u?g=AfZz!z?K2x{Ch)QJ*Yy~uS5)t)-kY+ z|7Sx^f;CT7H$(^orc9K+x&Ja0N=43Oj>cMztg^0Y?y; zo@TGOypHRdHJ4ahJU%VD!IX*FR>Ylm^|rZXjQPs5z>VzSl{x}Fh)e2nZcsEHJb4%$ z&_d&v)SUaw%DI>Hw3BZsx6fN5vc60nZY@4_RLnwti*bM&h#EzSG>kYmE?a_gO`G-cc2`9ChSf(B8}0qqts+$jnigM-LbvH2`*mF@E+4jVX!x!4W~IO#Xh7{t zxTtt^dVy@!T2xaNpP64vG$0OW4;xTlOt3^3vlG%k2pC`ypw%pC5U=LOFRm<}SlJNX zOC6*5yvd&>`kn&zv~c8!zxsjBxdE&~2almCGyx8N&ABfr_NMooj(1GZ+f{09fwlp9 zx1`bMAx^IwRK7Fdg+RAR7$Jm)7?3z<7Ow3Q7bBsD^~Ng(R9JKEON;KH+ftNep!Xl; zXz?R1CU($Sf;P;6SPpz386R96E<%+X*%ex=g{*6|M$JVoEAECi+dVc5_Gp~5ZzHNY ziNf&qGx3w`)cl)t&U3AT97AzJfafiox*8 z;uu&05kCcp2@-LDo8ZKzmc#ti7+i_OnV7>6U_fz73XVp$04_dV?AYkmO$~Tmb#Za= z6Ew1ZQ1|+uDNYx21$Tc%&<alf)_S!8);Oe!p~>Ug zl0ll2L=Pwmnml?})lP=oCLHM8mJ;!aBpZKz56 z`QlDXhyKPj#hs2G`Wx4d!KA=u;_*b&0s5!2=>R3A!}u49a|d1VaUn1qIxy+sRfj;G zyl`Dn8}xvcT`1~OL}sKB{D&*nq)gF+S;$ylq=QpeTPc|fu?|V*MSK{n;n0Z3Oh`}z z&|w(LA5d(drmrg|xs>Ql$NaLfItrbr+lxRs;h~UVC-;D#d3{l;v*xS9!-T>>l`s+t z+`j~EZzwL!?w0*g{F>de8UvL2WftZU#RnvET$#$uFBT6^Rd{4&BvrhX9Y>|*m8d!p z>2Vv>1TO|~i-Cnfb&2UfC*s!3yBmu~q^C@GoQx_KyBNbQ6Za+g1Y#CM8RrCChBd(^ zV1aW0c3gALn~Eo<=UkRpl$FgTzS;Eel1xC?g*hB>2d=Ot)VxXiQ<#iNN&754rPg@8 zxwtrWDR^xWoM&0|vt~qV`PHCQ5hz`f*TkX2z`mtu#>Ql9ZO4j01ilBLDtVtQTXBUahwL!D4B+gzw-8hq|UAfGw1@ za8q?s1%duh5gy*5Q_6#1Axggupku=UMW4#51zii2bL_Xc#`4!kGL(42GFe>0Nh0m|Lbki~gRNH%^xMzBm#Cjl!=ZI(pw*;vH&}*87h*yr)@nlg+zuaGb*H!9* z$|ys?Ll-A}7PaBWfbLNYhdo-XP`9TfxDI1a2i z?Gi}i;QLW^CON%I1y)2nzcuu6>6YFsD%5Dj4%L%qaAE!1#ld5% zJY9vd1+s2YCuWl{!bFl?=Q~Afs3!!-9>B2_rOQPtIT2JMvM#%{^Llo;m8H>!04O9J zF0=wMWc#_|!Snpy_5p0&sZ}$KmBa=Qste8%jGd5)^L%laA@=$xjTz~l7lyk}Lm^l} zp%j|ghm^;CakvAh@KQn2(p1j^(59K+Eo!wvxq1l&i7bYtNzfxwWJxmmd&BQ;)Pqw+ zBIq_C;Pox%`^A(aslx0U;~PlDip8G!qvW@&h6E*oPBYhFwZdD#7t*4Xi2PFelxa-7 zw)l@iA~NS-1r#XS@ZMi8wx;W@qCW%}qoY^!{fc~)G*HsfS?(^2$S2-s6j3D@?)vO2 z#QZ4ZHw|ux5=%n2!ht{=s7|e^{z}msd}abJi>>wSGHx$nHR*;!_(ybzrx=Q>aPKzA zb$VjbG?m7qH4ecKiseDCQt|BuTr2&kM1M2(N>?SnNNZi!=8EWP(RRSa;WQ{_t95IB zSUh8xD^}dXL9UyAp+s9alODq0U42oLBf^6OL>(2fB_O-Lfex&B-dBsa%Fg=T}E1?&DjXI3pW${EL5S`TohQEKP}!d2DckA+3gpE8{vlGf27YMIDOTo^^p_) zta#@b+;IKL5Ix4ISA0N$H&`kMRY^;jKzXZp?-<PQB7*AERNXYuX zD(*If-4uTmQzQPmICq5KOzN|KQ{1z9Z=+QcWM$Z`>`bV1e7!)#)JTn4>Tip3^_RI8 z7EFAIK=5%4W~LZmdNUGQ*BtqGMWx;`F{m?by@j0ti8%;d@I)%?1-}>WtAIERR&bC& zC`UbeWjY}FfUg0->#ky3e<)s^OX?*mF}s94Hm*7uZhMW(PpbCO!#i8rXlu1P81?P3 z`;!4Ss0x}%)sF%o>V{2C7bSYZ(^=j8>-)BZl;wXLfxklO zsu?wZbOKxQmA@BNB1gmZ7lEmrP`yRO0Rk*bQ~ps*LGUqxdtrpxKBT9+t_2uZZ=u`c zcuq{rtpMzdI^Y@qS3Eyml+~&udV|E&MNYTYs;oYz@NSZLer%C1i~cHEq`Cu2;Y}OB zJ^mmHOA1BBoMEyLJBDo5x*Pv2c2(a*#(&Ay&kZpAD<76Z2$n(IQ+>_FJVhA&I)MvE z;2f!mvGGP~jQ)3rxN7`hG4g7_RVyEs1iq6bH|(PTY(OAzRp>InG{O!;eU}NHUCmYh zRa~1J^kM7N{0+GJfbWzN3;`|=Z5xOnaTB2o)iS{}QU?lwiCQo6-^De9-UCGEJu7P7 zZ03DrKP~}efc8~}F`EzsF&nN7e2SJUSOJVYq*%3{)_+F4K}Su=p{Au1i(qVX-y89f zB4>iBM;HRq3lM944I@n)QBDI%BhH4Q&{NY^){gi{hkcbc9lA#wbh2XRnIk?@(D-U_ zQ4nka%;eZ-jrhm_W)<2>^F~}CDu#ddh>!G048p%H`i7!jOzWJ6L*}JXj03ykbzE8e z%y$fsPNzjQI9$?MZsQ9=vG7}}jX}tHWc&k2n-uV2=Qi|5F0(X3!}vHLKu9EY`AYexhN3dDIUph-@ULAPGar>qBr$8gsyK&uCkLbMs60Bkj1haEu48mAj>S~5yf_5|gpEXm< z#hWxW+@m0bi1fk@If(_8T;R!se<%yYn7*jNjP4}a+eG&#crjyhG~V1w33)x8=im@< zNc>3zw3OiD1}!#g>LeT@KH%u(HpB1(q7rUbf_Ol)!%G@YO6Nn0E9wVi`4Qh^ zB`SrWwIXcI&C=&xKu*90vzCaF$pDoVQ?F5imo|Jlm!7}htF6{ZgGTA0)|q$E>|+E? zWHx#p-%sUVN_?**Q(6l}yXcz;%GeT{0)<4do}yN$fT7V;Ew1jehJK`F6myRtXej|( ze2w_8lJIdH8l1pO06kF@W%4NoB6i}co zplB4;!ejuP4gILoL6bD;HI{L= zf;weI&{+ce0u>cWaANV4LyHJ5)|8-O*t2PCfYOtxYWC5{8U^WfuzTv)+RT^rd(?>Ue(Yu=-5`(2n@Jt z=F<|1I8CqzLeeaOI2F%IW7@#NP6H@Ken6@Ppk6)1b*u7c1FoC-w4@8V9>D@h`h3IS)+uMsWZ&vtxc}o4tym!q);yRN$)fvW)#x_RP7O34 zRdMtVsh!2wG*q)Pi{jX*ZskaR5e8B0Xvehu@rU9Qp)^;t0~|@rAwknYOElPuNJ(3s zc5TBjggyd)L%JDLBHE_E(6I0D4mT~oW9L>>>6fB$;S_FbP+G05YuF{-UL#eNc8>s@ zs{_l#ECD1&Qg403uIcwu(wOp<;H;-<&iu=I3}9NbWWO)A2(rcXPHh^POf3D z_%W_y5?M9B#|t{U@)KLB2=Afqd9HXQLpp9MA9(>S9{o9PZ=pdXKfN8~xwbY}ZroiM zheB(Ijnvc_b0l?+qVF+&4HeNnUE`&>iZNH8Dwf@x void; +export const parse_query: (a: number, b: number) => [number, number, number]; +export const query_completions: (a: number, b: number, c: number, d: any, e: any) => [number, number, number]; +export const query_diagnostics: (a: number, b: number, c: any) => [number, number, number]; +export const query_format_field: (a: number, b: number, c: number, d: number, e: number) => [number, number]; +export const query_tokens: (a: number, b: number) => [number, number, number]; +export const querymatcher_isEmpty: (a: number) => number; +export const querymatcher_matches: (a: number, b: number, c: number) => [number, number, number]; +export const querymatcher_matchingRows: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => [number, number, number, number]; +export const querymatcher_new: (a: number, b: number, c: any) => [number, number, number]; +export const __wbindgen_malloc: (a: number, b: number) => number; +export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number; +export const __wbindgen_exn_store: (a: number) => void; +export const __externref_table_alloc: () => number; +export const __wbindgen_externrefs: WebAssembly.Table; +export const __externref_table_dealloc: (a: number) => void; +export const __wbindgen_free: (a: number, b: number, c: number) => void; +export const __wbindgen_start: () => void; diff --git a/crates/yaak-query/src/ast.rs b/crates/yaak-query/src/ast.rs new file mode 100644 index 000000000..0b1164154 --- /dev/null +++ b/crates/yaak-query/src/ast.rs @@ -0,0 +1,195 @@ +//! The parsed query tree. +//! +//! Every node carries the byte span of its source text, so a consumer can highlight it, point +//! at it in a diagnostic, or rewrite it in place. A consumer that compiles queries (to SQL, say) +//! walks [`Node`] directly; nothing here depends on how the query is evaluated. + +use serde::{Deserialize, Serialize}; + +/// A half-open byte range `start..end` in the query source. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub struct Span { + pub start: u32, + pub end: u32, +} + +impl Span { + pub fn new(start: usize, end: usize) -> Self { + Self { start: start as u32, end: end as u32 } + } + + pub fn start(&self) -> usize { + self.start as usize + } + + pub fn end(&self) -> usize { + self.end as usize + } + + /// The source text this span covers. + pub fn slice<'a>(&self, source: &'a str) -> &'a str { + &source[self.start()..self.end()] + } + + pub(crate) fn to(&self, other: Span) -> Span { + Span { start: self.start.min(other.start), end: self.end.max(other.end) } + } +} + +/// A node of the parsed query. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type")] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub enum Node { + /// Every child matches: terms side by side, or joined with `AND`. + And { children: Vec, span: Span }, + /// Any child matches: terms joined with `OR`. + Or { children: Vec, span: Span }, + /// The node does not match: `-term`, `-(group)`, `-@field:value`. + Not { node: Box, span: Span }, + /// A bare word, matched against the document's free text. + Term { text: String, span: Span }, + /// A double-quoted string, matched against the free text with its spaces kept. + Phrase { text: String, span: Span }, + /// `@path:value`. + Field { + /// The dotted path, one entry per identifier: `@req.header.accept` is + /// `["req", "header", "accept"]`. + path: Vec, + value: FieldValue, + span: Span, + /// `@path:`, from the `@` through the colon. + #[serde(rename = "pathSpan")] + path_span: Span, + /// The value as written, quotes included. Empty (and at the end of the field) when + /// nothing follows the colon. + #[serde(rename = "valueSpan")] + value_span: Span, + }, +} + +impl Node { + pub fn span(&self) -> Span { + match self { + Node::And { span, .. } + | Node::Or { span, .. } + | Node::Not { span, .. } + | Node::Term { span, .. } + | Node::Phrase { span, .. } + | Node::Field { span, .. } => *span, + } + } + + /// Calls `f` on this node and every node below it, parents first. + pub fn walk<'a>(&'a self, f: &mut impl FnMut(&'a Node)) { + f(self); + match self { + Node::And { children, .. } | Node::Or { children, .. } => { + for child in children { + child.walk(f); + } + } + Node::Not { node, .. } => node.walk(f), + Node::Term { .. } | Node::Phrase { .. } | Node::Field { .. } => {} + } + } + + /// Rewrites every span in the tree, for a consumer that counts positions differently (the + /// WebAssembly package hands JavaScript UTF-16 offsets). + pub fn map_spans(&mut self, f: &impl Fn(Span) -> Span) { + match self { + Node::And { children, span } | Node::Or { children, span } => { + *span = f(*span); + for child in children { + child.map_spans(f); + } + } + Node::Not { node, span } => { + *span = f(*span); + node.map_spans(f); + } + Node::Term { span, .. } | Node::Phrase { span, .. } => *span = f(*span), + Node::Field { span, path_span, value_span, .. } => { + *span = f(*span); + *path_span = f(*path_span); + *value_span = f(*value_span); + } + } + } +} + +/// What a field term's value asks for. The shape is decided by the parser alone; the schema +/// decides what it means for a given field. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub enum FieldValue { + /// Plain text, matched as a case-insensitive substring. `quoted` when it was written in + /// double quotes (escapes already removed). Empty for a trailing `@field:`. + Text { text: String, quoted: bool }, + /// A bare value containing `*`, which must match the whole field value. `*` matches any + /// run of characters, including none. + Wildcard { pattern: String }, + /// `*` alone: the field has a value. Negated, it has none. + Exists, + /// `>`, `>=`, `<` or `<=` followed by a number. + Compare { + op: CompareOp, + number: NumberLiteral, + }, + /// `[low TO high]`, inclusive. + Range { + low: NumberLiteral, + high: NumberLiteral, + }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub enum CompareOp { + #[serde(rename = ">")] + Gt, + #[serde(rename = ">=")] + Gte, + #[serde(rename = "<")] + Lt, + #[serde(rename = "<=")] + Lte, +} + +impl CompareOp { + pub fn as_str(&self) -> &'static str { + match self { + CompareOp::Gt => ">", + CompareOp::Gte => ">=", + CompareOp::Lt => "<", + CompareOp::Lte => "<=", + } + } + + pub fn test(&self, left: f64, right: f64) -> bool { + match self { + CompareOp::Gt => left > right, + CompareOp::Gte => left >= right, + CompareOp::Lt => left < right, + CompareOp::Lte => left <= right, + } + } +} + +/// A number as written in a comparison or range, with its unit when it has one (`500ms`, +/// `2.5mb`). The schema says which units a field accepts and what they are worth. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub struct NumberLiteral { + pub value: f64, + pub unit: Option, + /// The number as it appears in the source, unit included. + pub raw: String, +} diff --git a/crates/yaak-query/src/complete.rs b/crates/yaak-query/src/complete.rs new file mode 100644 index 000000000..9849c50c2 --- /dev/null +++ b/crates/yaak-query/src/complete.rs @@ -0,0 +1,378 @@ +//! What can complete the term at the cursor. + +use crate::ast::Span; +use crate::format::{format_value, is_valid_path}; +use crate::lexer::{TokKind, is_ident_char, lex, lex_quoted}; +use crate::schema::{FieldKind, Schema}; +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; + +/// Completions for the term at the cursor, all replacing `span`. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub struct Completions { + pub span: Span, + pub items: Vec, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub struct Completion { + /// What to show: a field's path, or a value as it is. + pub label: String, + /// What replaces the span: `@method:`, `@header.`, or a value quoted when it needs to be. + pub insert: String, + pub kind: CompletionKind, + pub info: Option, + /// The term is not finished after inserting (a field still needs its value, a map field its + /// key), so completion should open again. + pub reopen: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub enum CompletionKind { + Field, + Value, +} + +/// Completions at byte offset `cursor` of `input`. +/// +/// At the start of a term, or in a word made of identifier characters (with or without its +/// `@`), these are the schema's fields whose path contains what is typed, those starting with +/// it first. After `@map.` they are the keys given in `values` under the map's name. After +/// `@field:` they are the values given in `values` under the field's path that contain what is +/// typed (quotes and escapes aside); the whole value is replaced. A field written without its +/// `@` (`method:post`) gets the term with the `@`. Inside a phrase, after a `)`, or in other +/// text there are none. +pub fn completions( + input: &str, + cursor: usize, + schema: &Schema, + values: &HashMap>, +) -> Option { + let mut cursor = cursor.min(input.len()); + while !input.is_char_boundary(cursor) { + cursor -= 1; + } + + let toks = lex(input); + let Some(tok) = toks.iter().find(|t| t.span.start() < cursor && cursor <= t.span.end()) else { + // Between terms: anything may start here + let at_term_start = input[..cursor].chars().next_back().is_none_or(char::is_whitespace); + return at_term_start + .then(|| field_completions(schema, values, "", Span::new(cursor, cursor))) + .flatten(); + }; + + match &tok.kind { + TokKind::Minus | TokKind::LParen if cursor == tok.span.end() => { + field_completions(schema, values, "", Span::new(cursor, cursor)) + } + TokKind::Word(word) => { + let typed = &input[tok.span.start()..cursor]; + if let Some(fix) = missing_at(word, typed, schema) { + return Some(Completions { span: tok.span, items: vec![fix] }); + } + let partial = typed.strip_prefix('@').unwrap_or(typed); + if !partial.chars().all(|c| is_ident_char(c) || c == '.') { + return None; + } + field_completions(schema, values, partial, Span::new(tok.span.start(), cursor)) + } + TokKind::Field { path, path_span, value_span, .. } => { + if cursor < value_span.start() { + // In the path: replace it through the colon + let partial = &input[tok.span.start() + 1..cursor]; + return field_completions(schema, values, partial, *path_span); + } + value_completions(input, cursor, path, *value_span, schema, values) + } + _ => None, + } +} + +/// A word written like a field but without its `@` (`method:post`), once the cursor is past the +/// colon: the same term with the `@`, replacing the word. Words like `localhost:3000`, whose +/// part before the colon is no field, get nothing. +fn missing_at(word: &str, typed: &str, schema: &Schema) -> Option { + if word.starts_with('@') || !typed.contains(':') { + return None; + } + let (path, value) = word.split_once(':')?; + if !is_valid_path(path) { + return None; + } + let path: Vec = path.split('.').map(String::from).collect(); + schema.resolve(&path)?; + let term = format!("@{word}"); + Some(Completion { + label: term.clone(), + insert: term, + kind: CompletionKind::Field, + info: None, + // With no value yet, go on to the field's values + reopen: value.is_empty(), + }) +} + +fn field_completions( + schema: &Schema, + values: &HashMap>, + partial: &str, + span: Span, +) -> Option { + let needle = partial.to_lowercase(); + let mut items = Vec::new(); + + // Keys under a map: `@header.x-` offers `@header.x-cache:` + if let Some((map_name, key_partial)) = needle.rsplit_once('.') + && let Some(map) = schema.fields.iter().find(|f| f.map && f.name.to_lowercase() == map_name) + { + let keys = lookup(values, map_name).into_iter().flatten(); + for key in ranked(keys, key_partial) { + let path = format!("{}.{}", map.name, key); + if !is_valid_path(&path) { + continue; + } + items.push(Completion { + insert: format!("@{path}:"), + label: path, + kind: CompletionKind::Field, + info: None, + reopen: true, + }); + } + } + + for field in ranked_by(&schema.fields, &needle, |f| f.name.as_str()) { + let insert = + if field.map { format!("@{}.", field.name) } else { format!("@{}:", field.name) }; + items.push(Completion { + label: field.name.clone(), + insert, + kind: CompletionKind::Field, + info: field.info.clone(), + reopen: true, + }); + } + + (!items.is_empty()).then_some(Completions { span, items }) +} + +fn value_completions( + input: &str, + cursor: usize, + path: &[String], + value_span: Span, + schema: &Schema, + values: &HashMap>, +) -> Option { + let field = schema.resolve(path)?; + let typed = &input[value_span.start()..cursor]; + let needle = if typed.starts_with('"') { lex_quoted(typed, 0).0 } else { typed.to_string() }; + + let joined = path.join("."); + let flags = ["true".to_string(), "false".to_string()]; + let candidates: Vec<&String> = match lookup(values, &joined) { + Some(given) => given.iter().collect(), + None if field.def.kind == FieldKind::Flag => flags.iter().collect(), + None => return None, + }; + + let items: Vec = ranked(candidates, &needle.to_lowercase()) + .into_iter() + .map(|value| Completion { + label: value.clone(), + insert: format_value(value), + kind: CompletionKind::Value, + info: None, + reopen: false, + }) + .collect(); + (!items.is_empty()).then_some(Completions { span: value_span, items }) +} + +/// The values a consumer gave for `path`, matched case-insensitively. +fn lookup<'a>(values: &'a HashMap>, path: &str) -> Option<&'a Vec> { + values + .get(path) + .or_else(|| values.iter().find(|(k, _)| k.eq_ignore_ascii_case(path)).map(|(_, v)| v)) +} + +/// The strings containing `needle` (lowercase), those starting with it first, each once. +fn ranked<'a>(items: impl IntoIterator, needle: &str) -> Vec<&'a String> { + let items: Vec<&String> = items.into_iter().collect(); + let mut seen = HashSet::new(); + ranked_by(&items, needle, |s| s.as_str()) + .into_iter() + .copied() + .filter(|s| seen.insert(*s)) + .collect() +} + +fn ranked_by<'a, T>(items: &'a [T], needle: &str, key: impl Fn(&T) -> &str) -> Vec<&'a T> { + let lower = |item: &T| key(item).to_lowercase(); + let starts = items.iter().filter(|i| lower(i).starts_with(needle)); + let contains = items.iter().filter(|i| { + let l = lower(i); + !l.starts_with(needle) && l.contains(needle) + }); + starts.chain(contains).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::schema::{FieldDef, FreeText}; + + fn schema() -> Schema { + let text = |name: &str| FieldDef { + name: name.into(), + kind: FieldKind::Text, + map: false, + info: None, + }; + Schema::new( + vec![ + text("method"), + text("url"), + text("name"), + FieldDef { name: "header".into(), kind: FieldKind::Text, map: true, info: None }, + FieldDef { name: "error".into(), kind: FieldKind::Flag, map: false, info: None }, + ], + FreeText::Fuzzy, + ) + } + + fn values() -> HashMap> { + HashMap::from([ + ( + "method".to_string(), + vec!["GET".to_string(), "POST".to_string(), "PATCH".to_string()], + ), + ("name".to_string(), vec!["My Request".to_string(), "Other".to_string()]), + ( + "header".to_string(), + vec![ + "x-cache".to_string(), + "Accept".to_string(), + "bad key".to_string(), + ], + ), + ]) + } + + /// The completion at the `|` in `input`: the replaced text and the inserts. + fn at(input: &str) -> Option<(String, Vec)> { + let cursor = input.find('|').unwrap(); + let input = input.replacen('|', "", 1); + let c = completions(&input, cursor, &schema(), &values())?; + Some((c.span.slice(&input).to_string(), c.items.into_iter().map(|i| i.insert).collect())) + } + + fn inserts(list: &[&str]) -> Vec { + list.iter().map(|s| s.to_string()).collect() + } + + #[test] + fn fields_at_a_term_start() { + let all = inserts(&["@method:", "@url:", "@name:", "@header.", "@error:"]); + assert_eq!(at("|"), Some(("".into(), all.clone()))); + assert_eq!(at("a |"), Some(("".into(), all.clone()))); + assert_eq!(at("-|"), Some(("".into(), all.clone()))); + assert_eq!(at("(|"), Some(("".into(), all))); + } + + #[test] + fn fields_for_a_partial_word() { + assert_eq!(at("@met|"), Some(("@met".into(), inserts(&["@method:"])))); + assert_eq!(at("met|"), Some(("met".into(), inserts(&["@method:"])))); + // Starts-with first, then contains + assert_eq!(at("@me|"), Some(("@me".into(), inserts(&["@method:", "@name:"])))); + assert_eq!(at("-@ur|"), Some(("@ur".into(), inserts(&["@url:"])))); + assert_eq!(at("x (@na|"), Some(("@na".into(), inserts(&["@name:"])))); + assert_eq!( + at("@e|"), + Some(("@e".into(), inserts(&["@error:", "@method:", "@name:", "@header."]))) + ); + assert_eq!(at("@zz|"), None); + assert_eq!(at("localhost:30|"), None); + } + + #[test] + fn map_keys() { + assert_eq!( + at("@header.|"), + Some(("@header.".into(), inserts(&["@header.x-cache:", "@header.Accept:"]))) + ); + assert_eq!( + at("@header.ac|"), + Some(("@header.ac".into(), inserts(&["@header.Accept:", "@header.x-cache:"]))) + ); + } + + #[test] + fn values_after_the_colon() { + assert_eq!(at("@method:|"), Some(("".into(), inserts(&["GET", "POST", "PATCH"])))); + assert_eq!(at("@method:p|"), Some(("p".into(), inserts(&["POST", "PATCH"])))); + assert_eq!(at("@method:t|"), Some(("t".into(), inserts(&["GET", "POST", "PATCH"])))); + assert_eq!(at("@name:my|"), Some(("my".into(), inserts(&["\"My Request\""])))); + assert_eq!(at("@name:\"my r|"), Some(("\"my r".into(), inserts(&["\"My Request\""])))); + // The whole value is replaced, even past the cursor + assert_eq!(at("@method:p|os x"), Some(("pos".into(), inserts(&["POST", "PATCH"])))); + assert_eq!(at("@method:zz|"), None); + // Fields with no values given offer none, unknown fields neither + assert_eq!(at("@url:|"), None); + assert_eq!(at("@nope:|"), None); + // Flags offer true and false + assert_eq!(at("@error:|"), Some(("".into(), inserts(&["true", "false"])))); + } + + #[test] + fn editing_a_field_name() { + assert_eq!(at("@met|hod:post"), Some(("@method:".into(), inserts(&["@method:"])))); + } + + #[test] + fn suggests_the_at_for_a_field_written_without_it() { + assert_eq!(at("method:post|"), Some(("method:post".into(), inserts(&["@method:post"])))); + assert_eq!(at("x method:p|"), Some(("method:p".into(), inserts(&["@method:p"])))); + assert_eq!(at("-method:po|st"), Some(("method:post".into(), inserts(&["@method:post"])))); + assert_eq!(at("(METHOD:get|"), Some(("METHOD:get".into(), inserts(&["@METHOD:get"])))); + assert_eq!( + at("header.accept:json|"), + Some(("header.accept:json".into(), inserts(&["@header.accept:json"]))) + ); + // With nothing after the colon, the field's values come next + let c = completions("method:", 7, &schema(), &values()).unwrap(); + assert_eq!(c.items[0].insert, "@method:"); + assert!(c.items[0].reopen); + assert!(!completions("method:x", 8, &schema(), &values()).unwrap().items[0].reopen); + // Not a field: plain text, nothing to suggest + assert_eq!(at("localhost:3000|"), None); + assert_eq!(at("nope:x|"), None); + assert_eq!(at("http://x|"), None); + // Before the colon, the word still completes field names + assert_eq!(at("met|hod:post"), Some(("met".into(), inserts(&["@method:"])))); + } + + #[test] + fn nothing_inside_phrases_or_after_parens() { + assert_eq!(at("\"my |"), None); + assert_eq!(at("(a)|"), None); + // A word is replaced up to the cursor + assert_eq!(at("a|b"), Some(("a".into(), inserts(&["@name:", "@header."])))); + } + + #[test] + fn cursor_is_clamped() { + let c = completions("é", 1, &schema(), &values()); + assert!(c.is_none() || c.unwrap().span.end <= 2); + assert!(completions("", 99, &schema(), &values()).is_some()); + } +} diff --git a/crates/yaak-query/src/eval.rs b/crates/yaak-query/src/eval.rs new file mode 100644 index 000000000..85dd998e8 --- /dev/null +++ b/crates/yaak-query/src/eval.rs @@ -0,0 +1,1055 @@ +//! Runs a parsed query in memory, against one document or a whole table of them. + +use crate::ast::{FieldValue, Node}; +use crate::fuzzy::{FuzzyScratch, fuzzy_match_in}; +use crate::schema::{FieldKind, FreeText, Schema, parse_flag, to_base}; +use serde::{Deserialize, Serialize}; +use std::borrow::Cow; +use std::collections::HashMap; + +/// What a query is matched against: free text for bare words and phrases, and field values by +/// lowercase dotted path. A map field's values sit under `name.key` (`header.x-cache`). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub struct Document { + #[serde(default)] + #[cfg_attr(test, ts(optional, as = "Option"))] + pub text: String, + #[serde(default)] + #[cfg_attr(test, ts(optional, as = "Option>"))] + pub fields: HashMap, +} + +/// One value or several. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub enum DocValues { + One(DocValue), + Many(Vec), +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub enum DocValue { + Flag(bool), + Number(f64), + Text(String), +} + +impl Document { + pub fn new(text: impl Into) -> Self { + Self { text: text.into(), fields: HashMap::new() } + } + + /// Adds a value under `path`, keeping any already there. + pub fn with(mut self, path: &str, value: impl Into) -> Self { + let value = value.into(); + let path = path.to_lowercase(); + match self.fields.remove(&path) { + None => self.fields.insert(path, DocValues::One(value)), + Some(DocValues::One(first)) => { + self.fields.insert(path, DocValues::Many(vec![first, value])) + } + Some(DocValues::Many(mut values)) => { + values.push(value); + self.fields.insert(path, DocValues::Many(values)) + } + }; + self + } +} + +impl From<&str> for DocValue { + fn from(value: &str) -> Self { + DocValue::Text(value.to_string()) + } +} + +impl From for DocValue { + fn from(value: String) -> Self { + DocValue::Text(value) + } +} + +impl From for DocValue { + fn from(value: f64) -> Self { + DocValue::Number(value) + } +} + +impl From for DocValue { + fn from(value: bool) -> Self { + DocValue::Flag(value) + } +} + +impl DocValue { + fn view(&self) -> Val<'_> { + match self { + DocValue::Text(text) => Val::Text(text), + DocValue::Number(n) => Val::Number(*n), + DocValue::Flag(b) => Val::Flag(*b), + } + } +} + +/// Many documents with the same fields, as one flat run of cells: for each row, its free text, +/// then a text value (or none) per column. Cells may borrow from the input they were read from. +/// Matching a table is much cheaper than matching its rows one document at a time. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Table<'a> { + /// Lowercase field paths, one per column. + columns: Vec, + cells: Vec>>, +} + +impl<'a> Table<'a> { + /// An empty table whose columns hold the fields at these paths. + pub fn new(columns: impl IntoIterator>) -> Self { + let columns = columns.into_iter().map(|c| c.into().to_lowercase()).collect(); + Self { columns, cells: Vec::new() } + } + + /// A table from its cells, already flat (text first, then a cell per column, row after + /// row). `None` when the cells don't fill whole rows. + pub fn from_cells( + columns: impl IntoIterator>, + cells: Vec>>, + ) -> Option { + let table = Self { cells, ..Self::new(columns) }; + table.cells.len().is_multiple_of(table.stride()).then_some(table) + } + + /// A table from cells packed the way JavaScript can produce them cheaply: every cell joined + /// into `joined`, with each cell's length in UTF-16 code units in `lengths` (or + /// [`Table::NO_VALUE`] for a cell with no value). Cells borrow from `joined`. `None` when + /// the lengths don't add up to `joined` or the cells don't fill whole rows. + pub fn from_packed( + columns: impl IntoIterator>, + joined: &'a str, + lengths: &[u32], + ) -> Option { + let mut cells = Vec::with_capacity(lengths.len()); + let ascii = joined.is_ascii(); + let mut start = 0; + for &len in lengths { + if len == Self::NO_VALUE { + cells.push(None); + continue; + } + let end = if ascii { + start + len as usize + } else { + // Walk `len` UTF-16 units forward + let mut units = 0; + let mut end = start; + for c in joined.get(start..)?.chars() { + if units >= len as usize { + break; + } + units += c.len_utf16(); + end += c.len_utf8(); + } + if units != len as usize { + return None; + } + end + }; + cells.push(Some(Cow::Borrowed(joined.get(start..end)?))); + start = end; + } + if start != joined.len() { + return None; + } + Self::from_cells(columns, cells) + } + + /// The length [`Table::from_packed`] reads as a cell with no value. + pub const NO_VALUE: u32 = u32::MAX; + + /// Adds a row: its free text, and a value or `None` per column, in column order. Missing + /// trailing cells are `None`; extra ones are dropped. + pub fn push( + &mut self, + text: impl Into>, + cells: impl IntoIterator>>, + ) { + self.cells.push(Some(text.into())); + let mut cells = cells.into_iter(); + for _ in 0..self.columns.len() { + self.cells.push(cells.next().flatten()); + } + } + + pub fn len(&self) -> usize { + self.cells.len() / self.stride() + } + + pub fn is_empty(&self) -> bool { + self.cells.is_empty() + } + + fn stride(&self) -> usize { + self.columns.len() + 1 + } +} + +/// A field value as matching sees it, borrowed from a document or a table. +#[derive(Clone, Copy, Debug)] +enum Val<'a> { + Text(&'a str), + Number(f64), + Flag(bool), +} + +impl Val<'_> { + /// The value as lowercase text. + fn lower(&self) -> Cow<'_, str> { + match self { + Val::Text(text) => Cow::Owned(text.to_lowercase()), + Val::Number(n) => Cow::Owned(format_number(*n)), + Val::Flag(b) => Cow::Borrowed(if *b { "true" } else { "false" }), + } + } + + /// Whether the value, lowercased, contains `needle` (lowercase). + fn contains(&self, needle: &str) -> bool { + match self { + Val::Text(text) => contains_ignore_case(text, needle), + _ => self.lower().contains(needle), + } + } + + fn as_number(&self) -> Option { + match self { + Val::Number(n) => Some(*n), + Val::Text(text) => text.trim().parse().ok(), + Val::Flag(_) => None, + } + } + + fn as_flag(&self) -> Option { + match self { + Val::Flag(b) => Some(*b), + Val::Text(text) => parse_flag(text), + Val::Number(n) => Some(*n != 0.0), + } + } + + fn is_present(&self) -> bool { + match self { + Val::Text(text) => !text.is_empty(), + Val::Number(_) => true, + Val::Flag(b) => *b, + } + } +} + +/// Whether `hay`, lowercased, contains `needle` (lowercase), without copying ASCII text. +fn contains_ignore_case(hay: &str, needle: &str) -> bool { + if !(hay.is_ascii() && needle.is_ascii()) { + return hay.to_lowercase().contains(needle); + } + let (hay, needle) = (hay.as_bytes(), needle.as_bytes()); + needle.is_empty() || hay.windows(needle.len()).any(|w| w.eq_ignore_ascii_case(needle)) +} + +/// `text` lowercased into `out`, without a fresh allocation per call. +fn lowercase_into(text: &str, out: &mut String) { + out.clear(); + if text.is_ascii() { + out.push_str(text); + out.make_ascii_lowercase(); + } else { + out.push_str(&text.to_lowercase()); + } +} + +/// `200`, not `200.0`, so a number field's text matches what the user sees. +fn format_number(n: f64) -> String { + if n.fract() == 0.0 && n.abs() < 1e15 { format!("{}", n as i64) } else { n.to_string() } +} + +/// Where field values come from: a [`Document`], or a row of a [`Table`]. +trait FieldSource { + /// Whether any value under `path` (lowercase) passes `test`. With `any_key`, `path` is a + /// map field and the values of every key under it count. + fn any_value(&self, path: &str, any_key: bool, test: &mut dyn FnMut(Val) -> bool) -> bool; +} + +impl FieldSource for Document { + fn any_value(&self, path: &str, any_key: bool, test: &mut dyn FnMut(Val) -> bool) -> bool { + self.fields.iter().filter(|(key, _)| path_matches(key, path, any_key)).any(|(_, values)| { + match values { + DocValues::One(v) => test(v.view()), + DocValues::Many(vs) => vs.iter().any(|v| test(v.view())), + } + }) + } +} + +struct RowSource<'t, 'a> { + columns: &'t [String], + /// The row's cells, after its text. + cells: &'t [Option>], +} + +impl FieldSource for RowSource<'_, '_> { + fn any_value(&self, path: &str, any_key: bool, test: &mut dyn FnMut(Val) -> bool) -> bool { + self.columns.iter().zip(self.cells).any(|(column, cell)| { + path_matches(column, path, any_key) + && cell.as_deref().is_some_and(|v| test(Val::Text(v))) + }) + } +} + +/// Whether a document key is `path`, or with `any_key`, any key under it (`path.`). Paths +/// are ASCII identifiers, so comparing ASCII case-insensitively is exact. +fn path_matches(key: &str, path: &str, any_key: bool) -> bool { + let (key, path) = (key.as_bytes(), path.as_bytes()); + if key.eq_ignore_ascii_case(path) { + return true; + } + any_key + && key.len() > path.len() + 1 + && key[..path.len()].eq_ignore_ascii_case(path) + && key[path.len()] == b'.' +} + +/// Whether `doc` matches the query. `None` (an empty query) matches everything. To match many +/// documents, build a [`Matcher`] once instead. +pub fn evaluate(root: Option<&Node>, schema: &Schema, doc: &Document) -> bool { + Matcher::new(root, schema).matches(doc) +} + +/// A query resolved against a schema once (fields looked up, text lowercased), to match many +/// documents. +#[derive(Clone, Debug, PartialEq)] +pub struct Matcher { + plan: Option, + fuzzy: bool, + /// Whether any term reads the free text, which then has to be lowercased per document. + uses_text: bool, +} + +#[derive(Clone, Debug, PartialEq)] +enum Plan { + And(Vec), + Or(Vec), + Not(Box), + /// A bare word, lowercased and trimmed. + Term(String), + /// A phrase, lowercased. + Phrase(String), + Field { + /// The lowercase path the document stores the values under. + path: String, + /// A map field with no key: every key's values. + any_key: bool, + kind: FieldKind, + /// With its text and wildcard pattern lowercased. + value: FieldValue, + }, + /// A term that can match nothing: an unknown field, an empty word or phrase. + Never, +} + +/// Buffers reused from one document to the next. +#[derive(Default)] +struct Scratch { + text: String, + fuzzy: FuzzyScratch, +} + +impl Matcher { + pub fn new(root: Option<&Node>, schema: &Schema) -> Self { + let mut uses_text = false; + let plan = root.map(|root| plan_node(root, schema, &mut uses_text)); + Self { plan, fuzzy: schema.free_text == FreeText::Fuzzy, uses_text } + } + + /// Whether the query has no terms, so it matches everything. + pub fn is_empty(&self) -> bool { + self.plan.is_none() + } + + pub fn matches(&self, doc: &Document) -> bool { + self.matches_source(&doc.text, doc, &mut Scratch::default()) + } + + /// The indexes of the rows that match, in order. + pub fn matching_rows(&self, table: &Table) -> Vec { + let mut scratch = Scratch::default(); + let rows = table.cells.chunks_exact(table.stride()); + rows.enumerate() + .filter(|(_, row)| { + let text = row[0].as_deref().unwrap_or_default(); + let source = RowSource { columns: &table.columns, cells: &row[1..] }; + self.matches_source(text, &source, &mut scratch) + }) + .map(|(i, _)| i as u32) + .collect() + } + + fn matches_source(&self, text: &str, source: &impl FieldSource, scratch: &mut Scratch) -> bool { + let Some(plan) = &self.plan else { return true }; + if self.uses_text { + lowercase_into(text, &mut scratch.text); + } + let Scratch { text, fuzzy } = scratch; + self.eval(plan, text, source, fuzzy) + } + + fn eval( + &self, + plan: &Plan, + text: &str, + source: &impl FieldSource, + fuzzy: &mut FuzzyScratch, + ) -> bool { + match plan { + Plan::And(children) => children.iter().all(|c| self.eval(c, text, source, fuzzy)), + Plan::Or(children) => children.iter().any(|c| self.eval(c, text, source, fuzzy)), + Plan::Not(node) => !self.eval(node, text, source, fuzzy), + Plan::Term(term) if self.fuzzy => fuzzy_match_in(text, term, fuzzy), + Plan::Term(term) | Plan::Phrase(term) => text.contains(term.as_str()), + Plan::Field { path, any_key, kind, value } => { + let mut any = + |test: &mut dyn FnMut(Val) -> bool| source.any_value(path, *any_key, test); + match kind { + FieldKind::Text => match_text(&mut any, value), + FieldKind::Number { units } => match_number(&mut any, value, units), + FieldKind::Flag => match_flag(&mut any, value), + } + } + Plan::Never => false, + } + } +} + +fn plan_node(node: &Node, schema: &Schema, uses_text: &mut bool) -> Plan { + let mut children = + |nodes: &[Node]| nodes.iter().map(|n| plan_node(n, schema, uses_text)).collect(); + match node { + Node::And { children: nodes, .. } => Plan::And(children(nodes)), + Node::Or { children: nodes, .. } => Plan::Or(children(nodes)), + Node::Not { node, .. } => Plan::Not(Box::new(plan_node(node, schema, uses_text))), + Node::Term { text, .. } => { + let term = text.trim().to_lowercase(); + if term.is_empty() { + return Plan::Never; + } + *uses_text = true; + Plan::Term(term) + } + Node::Phrase { text, .. } => { + if text.is_empty() { + return Plan::Never; + } + *uses_text = true; + Plan::Phrase(text.to_lowercase()) + } + Node::Field { path, value, .. } => { + let Some(field) = schema.resolve(path) else { + return Plan::Never; + }; + let value = match value { + FieldValue::Text { text, quoted } => { + FieldValue::Text { text: text.to_lowercase(), quoted: *quoted } + } + FieldValue::Wildcard { pattern } => { + FieldValue::Wildcard { pattern: pattern.to_lowercase() } + } + other => other.clone(), + }; + Plan::Field { + path: field.document_path(), + any_key: field.def.map && field.key.is_none(), + kind: field.def.kind.clone(), + value, + } + } + } +} + +/// Asks whether any of a field's values passes a test. +type AnyValue<'s> = dyn FnMut(&mut dyn FnMut(Val) -> bool) -> bool + 's; + +/// `value`'s text and wildcard pattern are already lowercase (see [`plan_node`]). +fn match_text(any: &mut AnyValue, value: &FieldValue) -> bool { + match value { + FieldValue::Text { text: needle, .. } => { + !needle.is_empty() && any(&mut |v| v.contains(needle)) + } + FieldValue::Wildcard { pattern } => any(&mut |v| wildcard_match(pattern, &v.lower())), + FieldValue::Exists => any(&mut |v| v.is_present()), + // A text field holding numbers, such as a status shown as text + FieldValue::Compare { op, number } => { + number.unit.is_none() + && any(&mut |v| v.as_number().is_some_and(|n| op.test(n, number.value))) + } + FieldValue::Range { low, high } => { + low.unit.is_none() + && high.unit.is_none() + && any(&mut |v| v.as_number().is_some_and(|n| n >= low.value && n <= high.value)) + } + } +} + +fn match_number(any: &mut AnyValue, value: &FieldValue, units: &[crate::Unit]) -> bool { + match value { + FieldValue::Text { text, quoted } => { + if text.is_empty() { + return false; + } + // `@status:404` is equality; anything else matches the number as text + match (!quoted).then(|| crate::lexer::parse_number(text)).flatten() { + Some(number) => to_base(&number, units) + .is_some_and(|target| any(&mut |v| v.as_number() == Some(target))), + None => match_text(any, value), + } + } + FieldValue::Wildcard { .. } => match_text(any, value), + FieldValue::Exists => any(&mut |v| v.as_number().is_some()), + FieldValue::Compare { op, number } => to_base(number, units) + .is_some_and(|target| any(&mut |v| v.as_number().is_some_and(|n| op.test(n, target)))), + FieldValue::Range { low, high } => match (to_base(low, units), to_base(high, units)) { + (Some(low), Some(high)) => { + any(&mut |v| v.as_number().is_some_and(|n| n >= low && n <= high)) + } + _ => false, + }, + } +} + +fn match_flag(any: &mut AnyValue, value: &FieldValue) -> bool { + let mut is_set = || any(&mut |v| v.as_flag() == Some(true)); + match value { + FieldValue::Exists => is_set(), + FieldValue::Text { text, .. } => match parse_flag(text) { + Some(true) => is_set(), + Some(false) => !is_set(), + None => false, + }, + _ => false, + } +} + +/// Whether `pattern`, where `*` matches any run of characters, matches the whole of `text`. +pub(crate) fn wildcard_match(pattern: &str, text: &str) -> bool { + let mut parts = pattern.split('*'); + let first = parts.next().unwrap_or_default(); + let Some(mut rest) = text.strip_prefix(first) else { + return false; + }; + let parts: Vec<&str> = parts.collect(); + let Some((last, middle)) = parts.split_last() else { + // No `*` at all: the whole text must equal the pattern + return rest.is_empty(); + }; + for part in middle { + match rest.find(part) { + Some(i) => rest = &rest[i + part.len()..], + None => return false, + } + } + rest.len() >= last.len() && rest.ends_with(last) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::schema::{FieldDef, Unit}; + use crate::{format_field, parse}; + + fn text_field(name: &str) -> FieldDef { + FieldDef { name: name.into(), kind: FieldKind::Text, map: false, info: None } + } + + /// The fields and free-text mode of Yaak's sidebar. + fn sidebar() -> Schema { + Schema::new( + [ + "method", + "url", + "name", + "type", + "folder", + "grpc_service", + "grpc_method", + ] + .into_iter() + .map(text_field) + .collect(), + FreeText::Fuzzy, + ) + } + + fn proxy_like() -> Schema { + Schema::new( + vec![ + text_field("url"), + FieldDef { name: "header".into(), kind: FieldKind::Text, map: true, info: None }, + FieldDef { + name: "status".into(), + kind: FieldKind::Number { units: vec![] }, + map: false, + info: None, + }, + FieldDef { + name: "duration".into(), + kind: FieldKind::Number { + units: vec![ + Unit { name: "ms".into(), factor: 1.0 }, + Unit { name: "s".into(), factor: 1000.0 }, + ], + }, + map: false, + info: None, + }, + FieldDef { name: "error".into(), kind: FieldKind::Flag, map: false, info: None }, + ], + FreeText::Substring, + ) + } + + /// An HTTP request in Yaak's sidebar, built the way `getItemFields` and `getItemText` do. + fn request(method: &str, name: &str, url: &str, folder: Option<&str>) -> Document { + let mut doc = Document::new(format!("{method} {name}")) + .with("method", method.to_uppercase()) + .with("url", url) + .with("name", name) + .with("type", "http"); + if let Some(folder) = folder { + doc = doc.with("folder", folder); + } + doc + } + + fn grpc(name: &str, service: &str, method: &str) -> Document { + Document::new(name) + .with("grpc_method", method) + .with("grpc_service", service) + .with("url", "localhost:50051") + .with("name", name) + .with("type", "grpc") + } + + fn matches(query: &str, schema: &Schema, doc: &Document) -> bool { + evaluate(parse(query).as_ref(), schema, doc) + } + + fn sidebar_matches(query: &str, doc: &Document) -> bool { + matches(query, &sidebar(), doc) + } + + #[test] + fn empty_query_matches_everything() { + assert!(sidebar_matches("", &Document::default())); + assert!(sidebar_matches(" ", &Document::default())); + } + + #[test] + fn bare_words_are_fuzzy_in_the_sidebar() { + let doc = request("GET", "My Request", "https://yaak.app", None); + assert!(sidebar_matches("my", &doc)); + assert!(sidebar_matches("REQ", &doc)); + assert!(sidebar_matches("mr", &doc)); + assert!(sidebar_matches("get", &doc)); + assert!(!sidebar_matches("rq", &doc)); + // Bare words match the free text only, not the URL + assert!(!sidebar_matches("yaak", &doc)); + } + + #[test] + fn bare_words_are_substrings_in_substring_mode() { + let doc = Document::new("GET /v1/charges 200"); + let schema = proxy_like(); + assert!(matches("rges", &schema, &doc)); + assert!(matches("v1 charges", &schema, &doc)); + assert!(!matches("gv", &schema, &doc)); + } + + #[test] + fn phrases_are_case_insensitive_substrings() { + let doc = request("GET", "My Request", "", None); + assert!(sidebar_matches("\"my req\"", &doc)); + assert!(sidebar_matches("\"MY REQ\"", &doc)); + assert!(!sidebar_matches("\"mr\"", &doc)); + assert!(!sidebar_matches("\"\"", &doc)); + } + + #[test] + fn fields() { + let post = request("post", "Create user", "https://api.yaak.app/users", Some("Users")); + let get = request("get", "List users", "https://api.yaak.app/users", Some("Admin")); + let rpc = grpc("Say hello", "helloworld.Greeter", "SayHello"); + + assert!(sidebar_matches("@method:post", &post)); + assert!(sidebar_matches("@METHOD:Po", &post)); + assert!(!sidebar_matches("@method:post", &get)); + assert!(sidebar_matches("@url:yaak.app/users", &post)); + assert!(sidebar_matches("@folder:users", &post)); + assert!(!sidebar_matches("@folder:users", &get)); + assert!(sidebar_matches("@folder:users OR @folder:admin", &get)); + assert!(sidebar_matches("@type:grpc", &rpc)); + assert!(!sidebar_matches("-@type:grpc", &rpc)); + assert!(sidebar_matches("-@type:grpc", &post)); + assert!(sidebar_matches("@grpc_service:greeter @grpc_method:hello", &rpc)); + // A field the document lacks matches nothing, and its negation everything + assert!(!sidebar_matches("@grpc_method:hello", &post)); + assert!(sidebar_matches("-@grpc_method:hello", &post)); + // `@` is required: without it, the term is free text + assert!(!sidebar_matches("method:post", &post)); + assert!(sidebar_matches("localhost:50051", &Document::new("localhost:50051"))); + // An unknown field matches nothing + assert!(!sidebar_matches("@nope:post", &post)); + // A trailing field with no value yet matches nothing, as before + assert!(!sidebar_matches("@method:", &post)); + } + + #[test] + fn groups_and_negation() { + let a = Document::new("alpha"); + let b = Document::new("beta"); + let ac = Document::new("alpha charlie"); + assert!(sidebar_matches("(alpha OR beta) -charlie", &a)); + assert!(sidebar_matches("(alpha OR beta) -charlie", &b)); + assert!(!sidebar_matches("(alpha OR beta) -charlie", &ac)); + assert!(sidebar_matches("-(alpha OR beta)", &Document::new("delta"))); + assert!(sidebar_matches("alpha charlie", &ac)); + assert!(!sidebar_matches("alpha charlie", &a)); + } + + #[test] + fn wildcards() { + assert!(wildcard_match("*.github.com", "api.github.com")); + assert!(!wildcard_match("*.github.com", "github.com")); + assert!(wildcard_match("a*", "a")); + assert!(wildcard_match("*", "")); + assert!(wildcard_match("a*b*c", "aXXbYYc")); + assert!(!wildcard_match("a*b*c", "aXXbYYcd")); + assert!(wildcard_match("a**c", "ac")); + assert!(!wildcard_match("ab*ba", "aba")); + + let doc = Document::default().with("url", "https://API.github.com/repos"); + let schema = proxy_like(); + assert!(matches("@url:https://api.github.com/*", &schema, &doc)); + assert!(matches("@url:*GITHUB*", &schema, &doc)); + assert!(!matches("@url:github*", &schema, &doc)); + // Quoted, `*` is literal + assert!(!matches("@url:\"*github*\"", &schema, &doc)); + assert!(matches("@url:\"*\"", &schema, &Document::default().with("url", "a*b"))); + } + + #[test] + fn presence() { + let schema = proxy_like(); + let with = Document::default().with("header.x-cache", "HIT").with("status", 200.0); + let without = Document::default().with("url", "x").with("status", 200.0); + assert!(matches("@header.x-cache:*", &schema, &with)); + assert!(!matches("@header.x-cache:*", &schema, &without)); + assert!(matches("-@header.x-cache:*", &schema, &without)); + assert!(matches("@status:*", &schema, &with)); + assert!(!matches("@url:*", &schema, &Document::default().with("url", ""))); + } + + #[test] + fn map_fields() { + let schema = proxy_like(); + let doc = Document::default() + .with("header.content-type", "application/json") + .with("header.x-cache", "HIT"); + assert!(matches("@header.content-type:json", &schema, &doc)); + assert!(matches("@header.Content-Type:JSON", &schema, &doc)); + assert!(!matches("@header.content-type:xml", &schema, &doc)); + assert!(!matches("@header.accept:json", &schema, &doc)); + // The map with no key looks at every key + assert!(matches("@header:hit", &schema, &doc)); + assert!(matches("@header:*", &schema, &doc)); + } + + #[test] + fn numbers() { + let schema = proxy_like(); + let doc = Document::default().with("status", 404.0).with("duration", 1500.0); + assert!(matches("@status:404", &schema, &doc)); + assert!(!matches("@status:40", &schema, &doc)); + assert!(matches("@status:40*", &schema, &doc)); + assert!(matches("@status:>=400", &schema, &doc)); + assert!(!matches("@status:<400", &schema, &doc)); + assert!(matches("@status:[400 TO 499]", &schema, &doc)); + assert!(matches("@status:[404 TO 404]", &schema, &doc)); + assert!(!matches("@status:[200 TO 299]", &schema, &doc)); + assert!(matches("@duration:>1s", &schema, &doc)); + assert!(matches("@duration:>1000ms", &schema, &doc)); + assert!(matches("@duration:1.5s", &schema, &doc)); + assert!(matches("@duration:[1s TO 2s]", &schema, &doc)); + assert!(!matches("@duration:>2s", &schema, &doc)); + // A unit the field does not take matches nothing + assert!(!matches("@duration:>1m", &schema, &doc)); + assert!(!matches("@status:>1ms", &schema, &doc)); + } + + #[test] + fn comparisons_on_text_fields() { + let doc = Document::default().with("url", "404"); + assert!(matches("@url:>400", &proxy_like(), &doc)); + assert!(!matches("@url:>400", &proxy_like(), &Document::default().with("url", "x"))); + } + + #[test] + fn flags() { + let schema = proxy_like(); + let failed = Document::default().with("error", true); + let ok = Document::default().with("error", false); + assert!(matches("@error:*", &schema, &failed)); + assert!(!matches("@error:*", &schema, &ok)); + assert!(matches("@error:true", &schema, &failed)); + assert!(matches("@error:false", &schema, &ok)); + assert!(matches("@error:false", &schema, &Document::default())); + assert!(!matches("@error:maybe", &schema, &failed)); + } + + #[test] + fn multiple_values() { + let doc = Document::default().with("url", "a").with("url", "b"); + assert!(matches("@url:b", &proxy_like(), &doc)); + } + + /// A table gives the same answers as matching its rows one document at a time. + #[test] + fn tables_match_like_documents() { + let columns = [ + "method", + "url", + "name", + "type", + "folder", + "grpc_service", + "grpc_method", + ]; + let docs = [ + request("post", "Create user", "https://api.yaak.app/users", Some("Users")), + request("get", "List users", "https://api.yaak.app/users", Some("Admin")), + request("GET", "My Request", "https://yaak.app", None), + grpc("Say hello", "helloworld.Greeter", "SayHello"), + Document::new("Folder"), + ]; + let mut table = Table::new(columns); + for doc in &docs { + let cells = columns.iter().map(|c| match doc.fields.get(*c) { + Some(DocValues::One(DocValue::Text(v))) => Some(Cow::Borrowed(v.as_str())), + _ => None, + }); + table.push(doc.text.as_str(), cells); + } + assert_eq!(table.len(), docs.len()); + + for query in [ + "", + "user", + "mr", + "\"my req\"", + "@method:post", + "-@type:grpc", + "@folder:users OR @folder:admin", + "(users OR hello) -create", + "@grpc_method:*", + "-@folder:*", + "@url:*yaak.app*", + "@nope:x", + "-@nope:x", + "method:post", + ] { + let matcher = Matcher::new(parse(query).as_ref(), &sidebar()); + let expected: Vec = (0..docs.len() as u32) + .filter(|&i| sidebar_matches(query, &docs[i as usize])) + .collect(); + assert_eq!(matcher.matching_rows(&table), expected, "{query:?}"); + } + assert!(Matcher::new(None, &sidebar()).is_empty()); + } + + #[test] + fn table_cells_line_up_with_columns() { + let mut table = Table::new(["Method", "url"]); + table.push("a", [Some("GET".into())]); + table.push("b", [None, Some("x".into()), Some("ignored".into())]); + assert_eq!(table.len(), 2); + let rows = |q: &str| Matcher::new(parse(q).as_ref(), &sidebar()).matching_rows(&table); + assert_eq!(rows("@method:get"), vec![0]); + assert_eq!(rows("@url:x"), vec![1]); + assert_eq!(rows("-@url:*"), vec![0]); + assert_eq!(rows("b"), vec![1]); + } + + #[test] + fn tables_from_flat_cells() { + let cells = vec![ + Some("GET List".into()), + Some("GET".into()), + None, + Some("POST".into()), + ]; + let table = Table::from_cells(["method"], cells.clone()).unwrap(); + assert_eq!(table.len(), 2); + let rows = |q: &str| Matcher::new(parse(q).as_ref(), &sidebar()).matching_rows(&table); + assert_eq!(rows("@method:post"), vec![1]); + assert_eq!(rows("list"), vec![0]); + // Cells that don't fill whole rows + assert!(Table::from_cells(["method", "url"], cells).is_none()); + assert!(Table::from_cells(["method"], Vec::new()).unwrap().is_empty()); + } + + /// Packs cells the way the JavaScript side does: joined, with UTF-16 lengths. + fn pack(cells: &[Option<&str>]) -> (String, Vec) { + let joined = cells.iter().flatten().copied().collect(); + let lengths = cells + .iter() + .map(|c| c.map_or(Table::NO_VALUE, |c| c.encode_utf16().count() as u32)) + .collect(); + (joined, lengths) + } + + #[test] + fn tables_from_packed_cells() { + let rows = |table: &Table, q: &str| { + Matcher::new(parse(q).as_ref(), &sidebar()).matching_rows(table) + }; + + let (joined, lengths) = pack(&[ + Some("GET List"), + Some("GET"), + Some(""), + None, + Some("POST"), + Some("x"), + ]); + let table = Table::from_packed(["method", "url"], &joined, &lengths).unwrap(); + assert_eq!(table.len(), 2); + assert_eq!(rows(&table, "@method:post"), vec![1]); + assert_eq!(rows(&table, "@url:x"), vec![1]); + assert_eq!(rows(&table, "list"), vec![0]); + assert_eq!(rows(&table, "-@method:*"), Vec::::new()); + + // Non-ASCII text, including a character outside the BMP (two UTF-16 units) + let (joined, lengths) = pack(&[ + Some("Über 🦀 crab"), + Some("GET"), + Some("Café"), + Some("x"), + None, + Some("y"), + ]); + let table = Table::from_packed(["method", "name"], &joined, &lengths).unwrap(); + assert_eq!(rows(&table, "🦀"), vec![0]); + assert_eq!(rows(&table, "@name:café"), vec![0]); + assert_eq!(rows(&table, "@name:y"), vec![1]); + + // Lengths that don't add up, or don't fill whole rows + assert!(Table::from_packed(["method"], "abc", &[1, 1]).is_none()); + assert!(Table::from_packed(["method"], "abc", &[2, 2]).is_none()); + assert!(Table::from_packed(["method"], "abc", &[3]).is_none()); + assert!(Table::from_packed(["method"], "é", &[2, 0]).is_none()); + assert!(Table::from_packed(["method"], "", &[]).unwrap().is_empty()); + } + + #[test] + fn non_ascii_text_matches_case_insensitively() { + let doc = Document::new("ÜBER Café").with("name", "ÜBER Café"); + assert!(sidebar_matches("über", &doc)); + assert!(sidebar_matches("@name:café", &doc)); + assert!(sidebar_matches("\"über c\"", &doc)); + assert!(!sidebar_matches("@name:cafe", &doc)); + } + + #[test] + fn document_from_json() { + let doc: Document = serde_json::from_str( + r#"{ "text": "x", "fields": { "url": "a", "tags": ["b", "c"], "status": 200, "error": true } }"#, + ) + .unwrap(); + assert_eq!(doc.fields["url"], DocValues::One(DocValue::Text("a".into()))); + assert_eq!(doc.fields["status"], DocValues::One(DocValue::Number(200.0))); + assert_eq!(doc.fields["error"], DocValues::One(DocValue::Flag(true))); + assert!(matches("@status:200 @error:true", &proxy_like(), &doc)); + } + + // The cases from Yaak's former `format.test.ts`: a formatted field filter matches the value + // it was made from. Values that contain query syntax are now quoted so they stay literal. + + fn matches_formatted_url(value: &str) -> bool { + matches( + &format_field("url", value, false), + &sidebar(), + &Document::default().with("url", value), + ) + } + + #[test] + fn format_keeps_url_like_values_bare() { + assert_eq!(format_field("url", "yaak.app/foo-bar", false), "@url:yaak.app/foo-bar"); + assert!(matches_formatted_url("yaak.app/foo-bar")); + } + + #[test] + fn format_quotes_values_with_query_syntax() { + // `*` is a wildcard and `)` closes a group, so this one is quoted now + assert_eq!(format_field("url", "yaa$&#*@tsrna(*)", false), "@url:\"yaa$&#*@tsrna(*)\""); + assert!(matches_formatted_url("yaa$&#*@tsrna(*)")); + assert_eq!(format_field("url", "yaa$&#@tsrna(", false), "@url:yaa$&#@tsrna("); + assert!(matches_formatted_url("yaa$&#@tsrna(")); + } + + #[test] + fn format_keeps_values_that_start_with_an_operator_token_bare() { + assert_eq!(format_field("url", "-foo", false), "@url:-foo"); + assert!(matches_formatted_url("-foo")); + } + + #[test] + fn format_keeps_boolean_operator_words_bare() { + assert_eq!(format_field("url", "AND", false), "@url:AND"); + assert_eq!(format_field("url", "or", false), "@url:or"); + assert_eq!(format_field("url", "Not", false), "@url:Not"); + assert!(matches_formatted_url("AND")); + } + + #[test] + fn format_escapes_quoted_values() { + assert_eq!(format_field("url", "say \"hi\"", false), "@url:\"say \\\"hi\\\"\""); + assert!(matches_formatted_url("say \"hi\"")); + } + + #[test] + fn format_quotes_values_that_start_with_a_quote() { + assert_eq!(format_field("url", "\"hi\"", false), "@url:\"\\\"hi\\\"\""); + assert!(matches_formatted_url("\"hi\"")); + } + + #[test] + fn formatted_values_round_trip() { + for value in [ + "a b", + "*", + ">5", + "[1 TO 2]", + "x)", + "back\\slash", + "\\\"", + "tab\there", + "é ü", + "a*b", + "<=3ms", + "\\", + ] { + assert!(matches_formatted_url(value), "{value:?} did not round-trip"); + } + } +} diff --git a/crates/yaak-query/src/format.rs b/crates/yaak-query/src/format.rs new file mode 100644 index 000000000..a6cc2d8ba --- /dev/null +++ b/crates/yaak-query/src/format.rs @@ -0,0 +1,89 @@ +//! Writes field terms back out as query text. + +use crate::ast::FieldValue; +use crate::lexer::{classify_bare, is_ident_char}; + +/// A term matching `value` in `field`: `@field:value`, or `-@field:value` when `negated`. The +/// value stays bare when it would read back as the same literal text, and is quoted (with `"` +/// and `\` escaped) when it holds whitespace, a quote, a `)`, a `*`, or the shape of a +/// comparison or range. +pub fn format_field(field: &str, value: &str, negated: bool) -> String { + let sign = if negated { "-" } else { "" }; + format!("{sign}@{field}:{}", format_value(value)) +} + +/// A field value as it should be written after the colon. +pub fn format_value(value: &str) -> String { + if is_bare_value(value) { value.to_string() } else { quote(value) } +} + +/// `value` in double quotes, with `"` and `\` escaped. +pub fn quote(value: &str) -> String { + let mut out = String::with_capacity(value.len() + 2); + out.push('"'); + for c in value.chars() { + if matches!(c, '"' | '\\') { + out.push('\\'); + } + out.push(c); + } + out.push('"'); + out +} + +fn is_bare_value(value: &str) -> bool { + let Some(first) = value.chars().next() else { + return false; + }; + if first == '"' || first == '[' { + return false; + } + if value.chars().any(|c| c.is_whitespace() || c == ')') { + return false; + } + matches!(classify_bare(value), FieldValue::Text { .. }) +} + +/// Whether `name` can be written as a field path: identifiers joined by dots. +pub fn is_valid_path(name: &str) -> bool { + !name.is_empty() + && name.split('.').all(|part| !part.is_empty() && part.chars().all(is_ident_char)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bare_or_quoted() { + assert_eq!(format_value("post"), "post"); + assert_eq!(format_value("a b"), "\"a b\""); + assert_eq!(format_value(""), "\"\""); + assert_eq!(format_value("*"), "\"*\""); + assert_eq!(format_value("a*"), "\"a*\""); + assert_eq!(format_value(">5"), "\">5\""); + assert_eq!(format_value(">x"), ">x"); + assert_eq!(format_value("[1 TO 2]"), "\"[1 TO 2]\""); + assert_eq!(format_value("[x"), "\"[x\""); + assert_eq!(format_value("a)"), "\"a)\""); + assert_eq!(format_value("a(b"), "a(b"); + assert_eq!(format_value("a\"b"), "a\"b"); + assert_eq!(format_value("a\\b"), "a\\b"); + assert_eq!(format_value("x\\ y"), "\"x\\\\ y\""); + } + + #[test] + fn negated() { + assert_eq!(format_field("type", "grpc", true), "-@type:grpc"); + assert_eq!(format_field("name", "My Request", true), "-@name:\"My Request\""); + } + + #[test] + fn paths() { + assert!(is_valid_path("method")); + assert!(is_valid_path("req.header.x-cache")); + assert!(!is_valid_path("")); + assert!(!is_valid_path("a..b")); + assert!(!is_valid_path("a b")); + } +} diff --git a/crates/yaak-query/src/fuzzy.rs b/crates/yaak-query/src/fuzzy.rs new file mode 100644 index 000000000..153606cc8 --- /dev/null +++ b/crates/yaak-query/src/fuzzy.rs @@ -0,0 +1,206 @@ +//! Word-prefix matching, ported from fuzzbunny's `fuzzyMatchSanitized` so Yaak's sidebar keeps +//! the behaviour it had when it called fuzzbunny directly. +//! +//! A search matches when it is a substring of the target. Failing that, it matches when it can +//! be split into runs that each start a word or sit at a punctuation mark, in order: `mr` finds +//! "My Request", `usam` finds "United States of America", but `rq` does not find "request". +//! Spaces in either string never break a run. +//! +//! Only whether it matches is kept; fuzzbunny's scores and highlight ranges are not used. + +/// Whether `search` matches `target`, both already lowercased and trimmed. +pub fn fuzzy_match(target: &str, search: &str) -> bool { + fuzzy_match_in(target, search, &mut FuzzyScratch::default()) +} + +/// Buffers [`fuzzy_match_in`] reuses from one call to the next. +#[derive(Default)] +pub(crate) struct FuzzyScratch { + target: Vec, + search: Vec, + skips: Vec, +} + +/// [`fuzzy_match`], reusing `scratch` instead of allocating per call. ASCII text, the usual case, +/// is matched byte by byte without copying. +pub(crate) fn fuzzy_match_in(target: &str, search: &str, scratch: &mut FuzzyScratch) -> bool { + if target.is_empty() { + return false; + } + if search.is_empty() { + return true; + } + + // A search starting with a quote wants a substring only; the closing quote is optional + let quoted = search.starts_with('"'); + let search = if quoted { + let inner = &search[1..]; + inner.strip_suffix('"').unwrap_or(inner) + } else { + search + }; + + if target.contains(search) { + return true; + } + + // One character that is not a substring can't match fuzzily either + let mut chars = search.chars(); + if quoted || (chars.next().is_some() && chars.next().is_none()) { + return false; + } + + let skips = &mut scratch.skips; + if target.is_ascii() && search.is_ascii() { + return word_prefix_match(target.as_bytes(), search.as_bytes(), skips); + } + scratch.target.clear(); + scratch.target.extend(target.chars()); + scratch.search.clear(); + scratch.search.extend(search.chars()); + word_prefix_match(&scratch.target, &scratch.search, skips) +} + +/// A unit of text the matcher walks: a byte of ASCII text, or a char. +trait Unit: Copy + PartialEq { + /// fuzzbunny counts lowercase letters, digits, and every char code from 127 up. + fn is_alpha_num(self) -> bool; + /// ASCII punctuation: `!` to `/`, `:` to `@`, `[` to `` ` ``, `{` to `~`. + fn is_punctuation(self) -> bool; + fn is_space(self) -> bool; +} + +impl Unit for char { + fn is_alpha_num(self) -> bool { + self.is_ascii_lowercase() || self.is_ascii_digit() || self as u32 >= 127 + } + + fn is_punctuation(self) -> bool { + matches!(self, '!'..='/' | ':'..='@' | '['..='`' | '{'..='~') + } + + fn is_space(self) -> bool { + self == ' ' + } +} + +impl Unit for u8 { + fn is_alpha_num(self) -> bool { + (self as char).is_alpha_num() + } + + fn is_punctuation(self) -> bool { + (self as char).is_punctuation() + } + + fn is_space(self) -> bool { + self == b' ' + } +} + +/// The word-prefix fallback, once `search` is known not to be a plain substring. +fn word_prefix_match(target: &[T], search: &[T], skips: &mut Vec) -> bool { + target_skips(target, skips); + (0..skips.len() - 1) + .any(|skip| target[skips[skip]] == search[0] && prefix_match(skip, search, target, skips)) +} + +/// The indices where a word starts or a punctuation mark sits, then the target's length, so +/// every run lies between `skips[i]` and `skips[i + 1]`. +fn target_skips(target: &[T], skips: &mut Vec) { + skips.clear(); + let mut was_alpha_num = false; + for (i, &c) in target.iter().enumerate() { + let alpha_num = c.is_alpha_num(); + if (alpha_num && !was_alpha_num) || c.is_punctuation() { + skips.push(i); + } + was_alpha_num = alpha_num; + } + skips.push(target.len()); +} + +fn prefix_match(mut skip: usize, search: &[T], target: &[T], skips: &[usize]) -> bool { + let mut search_idx = 0; + while skip < skips.len() - 1 { + let mut target_idx = skips[skip]; + let end = skips[skip + 1]; + while target_idx < end && search_idx < search.len() { + let t = target[target_idx]; + let s = search[search_idx]; + if t == s { + target_idx += 1; + search_idx += 1; + } else if t.is_space() { + target_idx += 1; + } else if s.is_space() { + search_idx += 1; + } else { + break; + } + } + if search_idx == search.len() { + return true; + } + skip += 1; + } + false +} + +#[cfg(test)] +mod tests { + use super::fuzzy_match; + + fn m(target: &str, search: &str) -> bool { + fuzzy_match(&target.to_lowercase(), &search.trim().to_lowercase()) + } + + #[test] + fn substring_first() { + assert!(m("My Request", "req")); + assert!(m("My Request", "quest")); + assert!(m("/v1/charges", "rges")); + assert!(m("abc", "abc")); + } + + #[test] + fn word_prefixes() { + assert!(m("My Request", "mr")); + assert!(m("My Request", "myreq")); + assert!(m("United States of America", "usam")); + assert!(m("api.github.com", "apigi")); + // Each run must continue from the start of a word: "gh" is not a prefix of "github" + assert!(!m("api.github.com", "apigh")); + assert!(m("POST Create User", "postcu")); + assert!(m("get_user_by_id", "gubi")); + } + + #[test] + fn not_any_subsequence() { + assert!(!m("My Request", "rq")); + assert!(!m("request", "rqst")); + assert!(!m("My Request", "mx")); + assert!(!m("abc", "x")); + } + + #[test] + fn spaces_do_not_break_a_run() { + assert!(m("My Request", "my r")); + assert!(m("foo bar", "f b")); + } + + #[test] + fn empty_and_quoted() { + assert!(!m("", "a")); + assert!(m("abc", "")); + assert!(m("las vegas", "\"la")); + assert!(!m("los angeles", "\"la\"")); + assert!(m("los angeles", "la")); + } + + #[test] + fn unicode_is_alphanumeric() { + assert!(m("Über Café", "üc")); + assert!(m("日本 語", "日語")); + } +} diff --git a/crates/yaak-query/src/lexer.rs b/crates/yaak-query/src/lexer.rs new file mode 100644 index 000000000..b28ae24ee --- /dev/null +++ b/crates/yaak-query/src/lexer.rs @@ -0,0 +1,320 @@ +//! Splits a query into tokens. Never fails: anything it does not recognise is text. + +use crate::ast::{CompareOp, FieldValue, NumberLiteral, Span}; + +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum TokKind { + LParen, + RParen, + /// `-` at the start of a term. It negates only what follows it directly. + Minus, + Or, + And, + Word(String), + Phrase { + text: String, + }, + Field { + path: Vec, + path_span: Span, + value: FieldValue, + value_span: Span, + }, +} + +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct Tok { + pub kind: TokKind, + pub span: Span, +} + +pub(crate) fn lex(input: &str) -> Vec { + let mut toks = Vec::new(); + let mut i = 0; + while let Some(c) = input[i..].chars().next() { + if c.is_whitespace() { + i += c.len_utf8(); + continue; + } + let (kind, end) = match c { + '(' => (TokKind::LParen, i + 1), + ')' => (TokKind::RParen, i + 1), + '-' => (TokKind::Minus, i + 1), + '"' => { + let (text, end) = lex_quoted(input, i); + (TokKind::Phrase { text }, end) + } + '@' => match lex_field(input, i) { + Some(field) => field, + None => lex_word(input, i), + }, + _ => lex_word(input, i), + }; + toks.push(Tok { kind, span: Span::new(i, end) }); + i = end; + } + toks +} + +/// Whether `c` ends a bare free-text word. +fn ends_word(c: char) -> bool { + c.is_whitespace() || matches!(c, '(' | ')' | '"') +} + +/// Whether `c` ends a bare field value. Parentheses inside a value are kept, except a closing +/// one, which closes the group the field sits in. +fn ends_value(c: char) -> bool { + c.is_whitespace() || c == ')' +} + +fn run_until(input: &str, start: usize, stop: impl Fn(char) -> bool) -> usize { + input[start..].find(stop).map_or(input.len(), |n| start + n) +} + +fn lex_word(input: &str, start: usize) -> (TokKind, usize) { + // The first character is never a word end (the caller checked), so a word is never empty + let first = input[start..].chars().next().map_or(0, char::len_utf8); + let end = run_until(input, start + first, ends_word); + let text = &input[start..end]; + let kind = match text { + "OR" => TokKind::Or, + "AND" => TokKind::And, + _ => TokKind::Word(text.to_string()), + }; + (kind, end) +} + +/// A double-quoted string starting at `start`. `\"` and `\\` are escapes; any other backslash +/// is kept. Without a closing quote it runs to the end of the input. +pub(crate) fn lex_quoted(input: &str, start: usize) -> (String, usize) { + let mut text = String::new(); + let mut chars = input[start + 1..].char_indices(); + while let Some((n, c)) = chars.next() { + match c { + '"' => return (text, start + 1 + n + 1), + '\\' => match input[start + 1 + n + 1..].chars().next() { + Some(next @ ('"' | '\\')) => { + text.push(next); + chars.next(); + } + _ => text.push('\\'), + }, + c => text.push(c), + } + } + (text, input.len()) +} + +pub(crate) fn is_ident_char(c: char) -> bool { + c.is_ascii_alphanumeric() || c == '_' || c == '-' +} + +/// `@path:value`, or `None` when the text at `start` is not one (no colon after the path). +fn lex_field(input: &str, start: usize) -> Option<(TokKind, usize)> { + let mut path = Vec::new(); + let mut i = start + 1; + loop { + let end = run_until(input, i, |c| !is_ident_char(c)); + if end == i { + return None; + } + path.push(input[i..end].to_string()); + i = end; + match input[i..].chars().next() { + Some('.') => i += 1, + Some(':') => break, + _ => return None, + } + } + let path_span = Span::new(start, i + 1); + let (value, value_end) = lex_value(input, i + 1); + let kind = TokKind::Field { path, path_span, value, value_span: Span::new(i + 1, value_end) }; + Some((kind, value_end)) +} + +fn lex_value(input: &str, start: usize) -> (FieldValue, usize) { + match input[start..].chars().next() { + None => return (FieldValue::Text { text: String::new(), quoted: false }, start), + Some(c) if ends_value(c) => { + return (FieldValue::Text { text: String::new(), quoted: false }, start); + } + Some('"') => { + let (text, end) = lex_quoted(input, start); + return (FieldValue::Text { text, quoted: true }, end); + } + Some('[') => { + if let Some((low, high, end)) = lex_range(input, start) { + return (FieldValue::Range { low, high }, end); + } + } + Some(_) => {} + } + let end = run_until(input, start, ends_value); + (classify_bare(&input[start..end]), end) +} + +/// What a bare (unquoted) field value means: presence, a comparison, a wildcard pattern, or +/// plain text. +pub(crate) fn classify_bare(raw: &str) -> FieldValue { + if raw == "*" { + return FieldValue::Exists; + } + if let Some((op, number)) = parse_compare(raw) { + return FieldValue::Compare { op, number }; + } + if raw.contains('*') { + return FieldValue::Wildcard { pattern: raw.to_string() }; + } + FieldValue::Text { text: raw.to_string(), quoted: false } +} + +fn parse_compare(raw: &str) -> Option<(CompareOp, NumberLiteral)> { + let (op, rest) = if let Some(rest) = raw.strip_prefix(">=") { + (CompareOp::Gte, rest) + } else if let Some(rest) = raw.strip_prefix("<=") { + (CompareOp::Lte, rest) + } else if let Some(rest) = raw.strip_prefix('>') { + (CompareOp::Gt, rest) + } else if let Some(rest) = raw.strip_prefix('<') { + (CompareOp::Lt, rest) + } else { + return None; + }; + Some((op, parse_number(rest)?)) +} + +/// A number with an optional unit: `-?digits(.digits)?letters?`, the whole of `raw`. +pub(crate) fn parse_number(raw: &str) -> Option { + let body = raw.strip_prefix('-').unwrap_or(raw); + let digits_end = body.find(|c: char| !c.is_ascii_digit()).unwrap_or(body.len()); + if digits_end == 0 { + return None; + } + let mut end = digits_end; + if let Some(fraction) = body[end..].strip_prefix('.') { + let fraction_len = fraction.find(|c: char| !c.is_ascii_digit()).unwrap_or(fraction.len()); + if fraction_len == 0 { + return None; + } + end += 1 + fraction_len; + } + let unit = &body[end..]; + if !unit.chars().all(|c| c.is_ascii_alphabetic()) { + return None; + } + let number_len = raw.len() - body.len() + end; + let value: f64 = raw[..number_len].parse().ok()?; + Some(NumberLiteral { + value, + unit: (!unit.is_empty()).then(|| unit.to_string()), + raw: raw.to_string(), + }) +} + +/// `[low TO high]` starting at `start`, or `None` when the text does not have exactly that +/// shape (it is then a literal value). +fn lex_range(input: &str, start: usize) -> Option<(NumberLiteral, NumberLiteral, usize)> { + let skip_ws = |i: usize| run_until(input, i, |c| !c.is_whitespace()); + let mut i = skip_ws(start + 1); + let low_end = run_until(input, i, |c| c.is_whitespace() || c == ']'); + let low = parse_number(&input[i..low_end])?; + i = skip_ws(low_end); + if i == low_end { + return None; + } + input[i..].strip_prefix("TO")?; + i += 2; + let after_to = skip_ws(i); + if after_to == i { + return None; + } + i = after_to; + let high_end = run_until(input, i, |c| c.is_whitespace() || c == ']'); + let high = parse_number(&input[i..high_end])?; + i = skip_ws(high_end); + input[i..].strip_prefix(']')?; + let end = i + 1; + match input[end..].chars().next() { + None => Some((low, high, end)), + Some(c) if ends_value(c) => Some((low, high, end)), + Some(_) => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn kinds(input: &str) -> Vec { + lex(input).into_iter().map(|t| t.kind).collect() + } + + #[test] + fn numbers() { + let n = parse_number("500ms").unwrap(); + assert_eq!(n.value, 500.0); + assert_eq!(n.unit.as_deref(), Some("ms")); + assert_eq!(parse_number("-2.5").unwrap().value, -2.5); + assert_eq!(parse_number("400").unwrap().unit, None); + assert!(parse_number("").is_none()); + assert!(parse_number("ms").is_none()); + assert!(parse_number("1.").is_none()); + assert!(parse_number("1.5.2").is_none()); + assert!(parse_number("12a3").is_none()); + assert!(parse_number("-").is_none()); + } + + #[test] + fn words_and_operators() { + assert_eq!( + kinds("a OR b AND or"), + vec![ + TokKind::Word("a".into()), + TokKind::Or, + TokKind::Word("b".into()), + TokKind::And, + TokKind::Word("or".into()), + ] + ); + } + + #[test] + fn words_stop_at_parens_and_quotes() { + assert_eq!( + kinds("(a)b\"c d\""), + vec![ + TokKind::LParen, + TokKind::Word("a".into()), + TokKind::RParen, + TokKind::Word("b".into()), + TokKind::Phrase { text: "c d".into() }, + ] + ); + } + + #[test] + fn quoted_escapes() { + assert_eq!(lex_quoted(r#""a \"b\" \\ \n""#, 0), (r#"a "b" \ \n"#.to_string(), 15)); + assert_eq!(lex_quoted(r#""open"#, 0), ("open".to_string(), 5)); + assert_eq!(lex_quoted(r#""trailing\"#, 0), ("trailing\\".to_string(), 10)); + } + + #[test] + fn spans_are_bytes() { + let toks = lex("é @url:ü"); + assert_eq!(toks[0].span, Span::new(0, 2)); + assert_eq!(toks[1].span, Span::new(3, 10)); + } + + #[test] + fn range_needs_its_whole_shape() { + assert!(lex_range("[1 TO 2]", 0).is_some()); + assert!(lex_range("[ 1 TO 2 ]", 0).is_some()); + assert!(lex_range("[1 to 2]", 0).is_none()); + assert!(lex_range("[1TO 2]", 0).is_none()); + assert!(lex_range("[1 TO 2", 0).is_none()); + assert!(lex_range("[1 TO 2]x", 0).is_none()); + assert!(lex_range("[a TO b]", 0).is_none()); + assert!(lex_range("[ base_url ]", 0).is_none()); + } +} diff --git a/crates/yaak-query/src/lib.rs b/crates/yaak-query/src/lib.rs new file mode 100644 index 000000000..14cc87e66 --- /dev/null +++ b/crates/yaak-query/src/lib.rs @@ -0,0 +1,40 @@ +//! The search query language of Yaak and Yaak Proxy. +//! +//! The syntax follows Datadog's log search. Terms side by side must all match; `OR` (upper +//! case) between terms or groups means either; parentheses group; `-` directly before a term or +//! group excludes it. A bare word matches the free text, a double-quoted phrase matches it with +//! its spaces kept, and `@path:value` matches a field. A value may be `*` (the field is present), +//! a `*` wildcard pattern, a comparison like `>=400` or `>500ms`, or a range like +//! `[400 TO 499]`. Everything matches case-insensitively, and nothing in the input is an error. +//! +//! [`parse`] gives the tree, which a consumer may walk to compile it elsewhere (to SQL, say). +//! [`evaluate`] runs it in memory against a [`Document`], with a [`Schema`] saying what the +//! fields are. [`tokens`], [`completions`] and [`format_field`] serve the search box. + +// The wasm-bindgen exports expand to unsafe glue; nothing else in the crate may use unsafe +#![cfg_attr( + not(all(feature = "wasm", target_arch = "wasm32")), + forbid(unsafe_code) +)] +#![cfg_attr(all(feature = "wasm", target_arch = "wasm32"), deny(unsafe_code))] + +pub mod ast; +mod complete; +mod eval; +mod format; +pub mod fuzzy; +mod lexer; +mod parser; +mod schema; +mod tokens; +#[cfg(all(feature = "wasm", target_arch = "wasm32"))] +#[allow(unsafe_code)] +mod wasm; + +pub use ast::{CompareOp, FieldValue, Node, NumberLiteral, Span}; +pub use complete::{Completion, CompletionKind, Completions, completions}; +pub use eval::{DocValue, DocValues, Document, Matcher, Table, evaluate}; +pub use format::{format_field, format_value, is_valid_path, quote}; +pub use parser::parse; +pub use schema::{Diagnostic, FieldDef, FieldKind, FreeText, ResolvedField, Schema, Unit}; +pub use tokens::{Token, TokenKind, tokens}; diff --git a/crates/yaak-query/src/parser.rs b/crates/yaak-query/src/parser.rs new file mode 100644 index 000000000..3ef505af7 --- /dev/null +++ b/crates/yaak-query/src/parser.rs @@ -0,0 +1,393 @@ +//! Builds the [`Node`] tree from tokens. +//! +//! Precedence, tightest first: `-`, then AND (implicit or written), then `OR`. The parser is +//! forgiving and never fails: a stray `)` or a dangling `OR`, `AND` or `-` is ignored, an +//! unclosed `(` closes at the end, and an empty group is dropped. Groups and negations nest +//! at most [`MAX_DEPTH`] deep; past that, `(` and `-` are ignored, so pathological input can't +//! exhaust the stack (the WebAssembly build has a small one). + +use crate::ast::{Node, Span}; +use crate::lexer::{Tok, TokKind, lex}; + +/// How deep groups and negations may nest, together. +pub const MAX_DEPTH: usize = 32; + +/// Parses a query. `None` when it has no terms (empty, only whitespace, or only operators). +pub fn parse(input: &str) -> Option { + let toks = lex(input); + let mut parser = Parser { toks: &toks, i: 0 }; + let mut parts = Vec::new(); + // A `)` with no group to close ends parse_or early; skip it and carry on + loop { + if let Some(node) = parser.parse_or(0) { + parts.push(node); + } + if parser.peek().is_none() { + break; + } + parser.i += 1; + } + join(parts, |children, span| Node::And { children, span }) +} + +struct Parser<'a> { + toks: &'a [Tok], + i: usize, +} + +impl<'a> Parser<'a> { + fn peek(&self) -> Option<&'a Tok> { + self.toks.get(self.i) + } + + fn parse_or(&mut self, depth: usize) -> Option { + let mut parts = Vec::new(); + loop { + if let Some(node) = self.parse_and(depth) { + parts.push(node); + } + match self.peek() { + Some(Tok { kind: TokKind::Or, .. }) => self.i += 1, + _ => break, + } + } + join(parts, |children, span| Node::Or { children, span }) + } + + fn parse_and(&mut self, depth: usize) -> Option { + let mut parts = Vec::new(); + while let Some(tok) = self.peek() { + match tok.kind { + TokKind::Or | TokKind::RParen => break, + TokKind::And => self.i += 1, + _ => { + if let Some(node) = self.parse_unary(depth) { + parts.push(node); + } + } + } + } + join(parts, |children, span| Node::And { children, span }) + } + + /// Always consumes at least one token. + fn parse_unary(&mut self, depth: usize) -> Option { + let tok = self.peek()?; + if tok.kind != TokKind::Minus { + return self.parse_primary(depth); + } + self.i += 1; + // `-` negates only a term, group or another `-` written directly after it + let next = self.peek().filter(|next| { + next.span.start == tok.span.end + && !matches!(next.kind, TokKind::RParen | TokKind::Or | TokKind::And) + }); + if next.is_none() || depth >= MAX_DEPTH { + return None; + } + let node = self.parse_unary(depth + 1)?; + let span = tok.span.to(node.span()); + Some(Node::Not { node: Box::new(node), span }) + } + + /// Always consumes at least one token. + fn parse_primary(&mut self, depth: usize) -> Option { + let tok = self.peek()?; + self.i += 1; + match &tok.kind { + TokKind::Word(text) => Some(Node::Term { text: text.clone(), span: tok.span }), + TokKind::Phrase { text } => Some(Node::Phrase { text: text.clone(), span: tok.span }), + TokKind::Field { path, path_span, value, value_span } => Some(Node::Field { + path: path.clone(), + value: value.clone(), + span: tok.span, + path_span: *path_span, + value_span: *value_span, + }), + // Too deep: the group's contents carry on at this level instead + TokKind::LParen if depth >= MAX_DEPTH => None, + TokKind::LParen => { + let inner = self.parse_or(depth + 1); + if let Some(Tok { kind: TokKind::RParen, .. }) = self.peek() { + self.i += 1; + } + inner + } + // Operators are handled by the callers; reaching one here means it is out of place + TokKind::Minus | TokKind::Or | TokKind::And | TokKind::RParen => None, + } + } +} + +/// One node stays itself; several are joined by `make` with a span covering them all. +fn join(mut parts: Vec, make: impl Fn(Vec, Span) -> Node) -> Option { + match parts.len() { + 0 => None, + 1 => parts.pop(), + _ => { + let span = parts[0].span().to(parts[parts.len() - 1].span()); + Some(make(parts, span)) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ast::{CompareOp, FieldValue}; + + /// A compact rendering of the tree, to keep the tests readable. + fn show(input: &str) -> String { + fn go(node: &Node) -> String { + match node { + Node::And { children, .. } => { + format!("(and {})", children.iter().map(go).collect::>().join(" ")) + } + Node::Or { children, .. } => { + format!("(or {})", children.iter().map(go).collect::>().join(" ")) + } + Node::Not { node, .. } => format!("(not {})", go(node)), + Node::Term { text, .. } => text.clone(), + Node::Phrase { text, .. } => format!("{text:?}"), + Node::Field { path, value, .. } => { + let value = match value { + FieldValue::Text { text, quoted: false } => text.clone(), + FieldValue::Text { text, quoted: true } => format!("{text:?}"), + FieldValue::Wildcard { pattern } => format!("wild({pattern})"), + FieldValue::Exists => "exists".to_string(), + FieldValue::Compare { op, number } => { + format!("{}{}", op.as_str(), number.raw) + } + FieldValue::Range { low, high } => format!("[{} TO {}]", low.raw, high.raw), + }; + format!("@{}={}", path.join("."), value) + } + } + } + parse(input).map_or("none".to_string(), |n| go(&n)) + } + + #[test] + fn empty() { + assert_eq!(show(""), "none"); + assert_eq!(show(" "), "none"); + assert_eq!(show("OR AND"), "none"); + assert_eq!(show("()"), "none"); + assert_eq!(show("-"), "none"); + assert_eq!(show(")"), "none"); + } + + #[test] + fn terms_and_phrases() { + assert_eq!(show("foo"), "foo"); + assert_eq!(show("foo bar"), "(and foo bar)"); + assert_eq!(show("\"hi there\""), "\"hi there\""); + assert_eq!(show("\"hi there"), "\"hi there\""); + assert_eq!(show("a\"b c\"d"), "(and a \"b c\" d)"); + assert_eq!(show(r#""say \"hi\"""#), r#""say \"hi\"""#); + } + + #[test] + fn boolean_operators() { + assert_eq!(show("a OR b"), "(or a b)"); + assert_eq!(show("a AND b"), "(and a b)"); + assert_eq!(show("a b OR c"), "(or (and a b) c)"); + assert_eq!(show("a OR b c"), "(or a (and b c))"); + assert_eq!(show("a OR b OR c"), "(or a b c)"); + // Case-sensitive, as Datadog: lowercase is a word + assert_eq!(show("a or b"), "(and a or b)"); + assert_eq!(show("a and b"), "(and a and b)"); + // NOT is not a keyword + assert_eq!(show("NOT a"), "(and NOT a)"); + } + + #[test] + fn dangling_operators_are_ignored() { + assert_eq!(show("a OR"), "a"); + assert_eq!(show("OR a"), "a"); + assert_eq!(show("a OR OR b"), "(or a b)"); + assert_eq!(show("a AND"), "a"); + assert_eq!(show("a -"), "a"); + assert_eq!(show("- a"), "a"); + assert_eq!(show("(a -)"), "a"); + } + + #[test] + fn groups() { + assert_eq!(show("(a OR b) -c"), "(and (or a b) (not c))"); + assert_eq!(show("-(a OR b)"), "(not (or a b))"); + assert_eq!(show("(a OR b"), "(or a b)"); + assert_eq!(show("a) b"), "(and a b)"); + assert_eq!(show("((a))"), "a"); + assert_eq!(show("x (a (b OR c))"), "(and x (and a (or b c)))"); + } + + #[test] + fn negation() { + assert_eq!(show("-foo"), "(not foo)"); + assert_eq!(show("--foo"), "(not (not foo))"); + assert_eq!(show("-\"a b\""), "(not \"a b\")"); + assert_eq!(show("-@type:grpc"), "(not @type=grpc)"); + assert_eq!(show("a-b"), "a-b"); + assert_eq!(show("-OR a"), "a"); + } + + #[test] + fn fields() { + assert_eq!(show("@method:post"), "@method=post"); + assert_eq!(show("@url:\"a b\""), "@url=\"a b\""); + assert_eq!(show("@header.x-cache:HIT"), "@header.x-cache=HIT"); + assert_eq!(show("@req.header.content-type:json"), "@req.header.content-type=json"); + assert_eq!(show("@folder:x OR @folder:y"), "(or @folder=x @folder=y)"); + assert_eq!(show("@url:https://a.b/c?d=e"), "@url=https://a.b/c?d=e"); + } + + #[test] + fn at_is_required() { + assert_eq!(show("method:post"), "method:post"); + assert_eq!(show("localhost:3000"), "localhost:3000"); + assert_eq!(show("@method"), "@method"); + assert_eq!(show("@"), "@"); + assert_eq!(show("@.a:b"), "@.a:b"); + assert_eq!(show("@a.:b"), "@a.:b"); + assert_eq!(show("@a b:c"), "(and @a b:c)"); + } + + #[test] + fn trailing_field_has_empty_value() { + assert_eq!(show("@method:"), "@method="); + assert_eq!(show("@method: post"), "(and @method= post)"); + assert_eq!(show("(@method:)"), "@method="); + } + + #[test] + fn field_values() { + assert_eq!(show("@url:-foo"), "@url=-foo"); + assert_eq!(show("@url:AND"), "@url=AND"); + assert_eq!(show("@url:a(b"), "@url=a(b"); + assert_eq!(show("(@url:a OR @url:b)"), "(or @url=a @url=b)"); + assert_eq!(show("@url:a\"b"), "@url=a\"b"); + assert_eq!(show("@url:\"a"), "@url=\"a\""); + } + + #[test] + fn special_values() { + assert_eq!(show("@status:*"), "@status=exists"); + assert_eq!(show("-@status:*"), "(not @status=exists)"); + assert_eq!(show("@url:*.github.com"), "@url=wild(*.github.com)"); + assert_eq!(show("@url:\"*.github.com\""), "@url=\"*.github.com\""); + assert_eq!(show("@status:>=400"), "@status=>=400"); + assert_eq!(show("@duration:>500ms"), "@duration=>500ms"); + assert_eq!(show("@status:<1"), "@status=<1"); + assert_eq!(show("@status:[400 TO 499]"), "@status=[400 TO 499]"); + assert_eq!(show("(@status:[400 TO 499])"), "@status=[400 TO 499]"); + // Not quite the shape: literal text or a wildcard + assert_eq!(show("@status:>="), "@status=>="); + assert_eq!(show("@status:>abc"), "@status=>abc"); + assert_eq!(show("@status:**"), "@status=wild(**)"); + assert_eq!(show("@status:[400"), "@status=[400"); + } + + #[test] + fn templates_parse_sanely() { + assert_eq!(show("@url:${[ base_url ]}"), "(and @url=${[ base_url ]})"); + assert_eq!(show("@url:[ base_url ]"), "(and @url=[ base_url ])"); + assert_eq!(show("@url:\"${[ base_url ]}/x\""), "@url=\"${[ base_url ]}/x\""); + } + + #[test] + fn compare_value_details() { + let Some(Node::Field { value: FieldValue::Compare { op, number }, .. }) = + parse("@duration:<=1.5s") + else { + panic!("not a comparison"); + }; + assert_eq!(op, CompareOp::Lte); + assert_eq!(number.value, 1.5); + assert_eq!(number.unit.as_deref(), Some("s")); + } + + #[test] + fn spans() { + let input = "-@url:\"a b\" OR (c"; + let node = parse(input).unwrap(); + let Node::Or { children, span } = &node else { + panic!() + }; + assert_eq!(span.slice(input), input); + let Node::Not { node: field, span } = &children[0] else { + panic!() + }; + assert_eq!(span.slice(input), "-@url:\"a b\""); + let Node::Field { span, path_span, value_span, .. } = field.as_ref() else { + panic!() + }; + assert_eq!(span.slice(input), "@url:\"a b\""); + assert_eq!(path_span.slice(input), "@url:"); + assert_eq!(value_span.slice(input), "\"a b\""); + assert_eq!(children[1].span().slice(input), "c"); + + let input = "@method:"; + let Some(Node::Field { value_span, .. }) = parse(input) else { + panic!() + }; + assert_eq!(value_span, Span::new(8, 8)); + } + + #[test] + fn nesting_is_bounded() { + fn depth(node: &Node) -> usize { + match node { + Node::And { children, .. } | Node::Or { children, .. } => { + children.iter().map(depth).max().unwrap_or(0) + 1 + } + Node::Not { node, .. } => depth(node) + 1, + _ => 1, + } + } + let parens = format!("{}a OR b{}", "(".repeat(5000), ")".repeat(5000)); + let negations = format!("{}a", "-".repeat(5000)); + let mixed = format!("{}a{}", "-(".repeat(5000), ")".repeat(5000)); + for input in [parens, negations, mixed] { + let node = parse(&input).unwrap(); + assert!(depth(&node) <= MAX_DEPTH + 2, "too deep for {:?}", &input[..10]); + } + // Within the limit, nesting is kept exactly + let input = format!("{}a{}", "(".repeat(MAX_DEPTH), ")".repeat(MAX_DEPTH)); + assert_eq!(show(&input), "a"); + assert_eq!(show(&format!("{}a", "-".repeat(2))), "(not (not a))"); + } + + #[test] + fn never_panics_on_odd_input() { + for input in [ + "\"", + "\\", + "@", + "@:", + "@a:", + "-", + "--", + "(", + ")", + "((", + "))", + "[", + "@a:[", + "@a:[1 TO", + "@a:\"", + "@a:\"\\", + "é@ü:ö", + "@a.b.c:", + "@a:>", + "@a:<=", + "-(", + "OR", + "AND OR", + "@a:)", + "🦀 @x:🦀 \"🦀", + ] { + let _ = parse(input); + } + } +} diff --git a/crates/yaak-query/src/schema.rs b/crates/yaak-query/src/schema.rs new file mode 100644 index 000000000..17a1c02fb --- /dev/null +++ b/crates/yaak-query/src/schema.rs @@ -0,0 +1,307 @@ +//! What a consumer's fields are, and how its bare words match. + +use crate::ast::{FieldValue, Node, NumberLiteral, Span}; +use crate::lexer::parse_number; +use serde::{Deserialize, Serialize}; + +/// A consumer's searchable fields. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub struct Schema { + pub fields: Vec, + /// Substring when left out. + #[serde(default)] + #[cfg_attr(test, ts(optional, as = "Option"))] + pub free_text: FreeText, +} + +/// How a bare word matches the document's free text. A quoted phrase is always a substring. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub enum FreeText { + /// A substring, falling back to word-prefix matching (`mr` finds "My Request"), as + /// fuzzbunny does. + Fuzzy, + /// A case-insensitive substring. + #[default] + Substring, +} + +/// One field: `@name:value`. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub struct FieldDef { + /// The field's path, dotted when nested: `method`, `req.header`. + pub name: String, + pub kind: FieldKind, + /// A map of named values, such as headers: `@header.:value` is valid for any key, and + /// the key is matched against the document's keys (case-insensitively). + #[serde(default)] + #[cfg_attr(test, ts(optional, as = "Option"))] + pub map: bool, + /// A short note shown with the field in completions. + #[serde(default)] + #[cfg_attr(test, ts(optional))] + pub info: Option, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub enum FieldKind { + Text, + /// A number, with the units a comparison or range may be written in. A number written + /// without a unit is in the field's base unit. + Number { + #[serde(default)] + units: Vec, + }, + /// True or false: `@name:true`, `@name:false`, and `@name:*` for true. + Flag, +} + +/// A unit a number may be written in, and how many of the field's base unit it is worth: +/// `{ name: "s", factor: 1000 }` for a field counted in milliseconds. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub struct Unit { + pub name: String, + pub factor: f64, +} + +/// A field path resolved against the schema. +#[derive(Clone, Debug, PartialEq)] +pub struct ResolvedField<'a> { + pub def: &'a FieldDef, + /// The key under a map field: `x-cache` for `@header.x-cache`. `None` for the map field + /// itself (`@header:...`, which matches any key) and for plain fields. + pub key: Option, +} + +impl ResolvedField<'_> { + /// The lowercase dotted path a document stores this field's values under. + pub fn document_path(&self) -> String { + match &self.key { + Some(key) => format!("{}.{}", self.def.name.to_lowercase(), key.to_lowercase()), + None => self.def.name.to_lowercase(), + } + } +} + +/// Something in a query the schema does not accept. The query still parses and runs; the +/// offending term matches nothing. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub enum Diagnostic { + /// No field has this path. `span` covers `@path:`. + UnknownField { path: String, span: Span }, + /// The value cannot apply to this field: a unit the field does not take, a number for a + /// text-only flag, and so on. `span` covers the value. + InvalidValue { path: String, span: Span }, +} + +impl Schema { + pub fn new(fields: Vec, free_text: FreeText) -> Self { + Self { fields, free_text } + } + + /// Finds the field a path names, case-insensitively. A path under a map field resolves to + /// that field with the rest of the path as the key; the longest matching map wins. + pub fn resolve(&self, path: &[String]) -> Option> { + let joined = path.join(".").to_lowercase(); + if let Some(def) = self.fields.iter().find(|f| f.name.to_lowercase() == joined) { + return Some(ResolvedField { def, key: None }); + } + self.fields + .iter() + .filter(|f| f.map) + .filter_map(|f| { + let name = f.name.to_lowercase(); + let key = joined.strip_prefix(&name)?.strip_prefix('.')?; + (!key.is_empty()).then_some((name.len(), f, key)) + }) + .max_by_key(|(len, ..)| *len) + .map(|(_, def, key)| ResolvedField { def, key: Some(key.to_string()) }) + } + + /// The terms in a query this schema cannot run, for the UI to mark. + pub fn diagnostics(&self, root: &Node) -> Vec { + let mut out = Vec::new(); + root.walk(&mut |node| { + let Node::Field { path, value, path_span, value_span, .. } = node else { + return; + }; + let joined = path.join("."); + match self.resolve(path) { + None => out.push(Diagnostic::UnknownField { path: joined, span: *path_span }), + Some(field) => { + if !value_fits(&field.def.kind, value) { + out.push(Diagnostic::InvalidValue { path: joined, span: *value_span }); + } + } + } + }); + out + } +} + +/// Whether a value can mean anything for a field of this kind. +fn value_fits(kind: &FieldKind, value: &FieldValue) -> bool { + match (kind, value) { + (FieldKind::Number { units }, FieldValue::Compare { number, .. }) => { + to_base(number, units).is_some() + } + (FieldKind::Number { units }, FieldValue::Range { low, high }) => { + to_base(low, units).is_some() && to_base(high, units).is_some() + } + // A bare number is equality, so its unit has to be one the field takes + (FieldKind::Number { units }, FieldValue::Text { text, quoted: false }) => { + parse_number(text).is_none_or(|number| to_base(&number, units).is_some()) + } + // Text holding numbers compares them as plain numbers, without units + (FieldKind::Text, FieldValue::Compare { number, .. }) => number.unit.is_none(), + (FieldKind::Text, FieldValue::Range { low, high }) => { + low.unit.is_none() && high.unit.is_none() + } + (FieldKind::Flag, FieldValue::Text { text, .. }) => { + text.is_empty() || parse_flag(text).is_some() + } + (FieldKind::Flag, FieldValue::Exists) => true, + (FieldKind::Flag, _) => false, + _ => true, + } +} + +/// A number in the field's base unit, or `None` when its unit is not one the field takes. +pub(crate) fn to_base(number: &NumberLiteral, units: &[Unit]) -> Option { + match &number.unit { + None => Some(number.value), + Some(unit) => units + .iter() + .find(|u| u.name.eq_ignore_ascii_case(unit)) + .map(|u| number.value * u.factor), + } +} + +pub(crate) fn parse_flag(text: &str) -> Option { + match text.to_lowercase().as_str() { + "true" | "yes" | "1" => Some(true), + "false" | "no" | "0" => Some(false), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parse; + + fn field(name: &str, kind: FieldKind, map: bool) -> FieldDef { + FieldDef { name: name.to_string(), kind, map, info: None } + } + + fn schema() -> Schema { + Schema::new( + vec![ + field("url", FieldKind::Text, false), + field("header", FieldKind::Text, true), + field("req.header", FieldKind::Text, true), + field( + "duration", + FieldKind::Number { units: vec![Unit { name: "s".into(), factor: 1000.0 }] }, + false, + ), + field("error", FieldKind::Flag, false), + ], + FreeText::Substring, + ) + } + + fn path(p: &str) -> Vec { + p.split('.').map(String::from).collect() + } + + #[test] + fn resolves_plain_and_map_fields() { + let s = schema(); + assert_eq!(s.resolve(&path("URL")).unwrap().def.name, "url"); + assert_eq!(s.resolve(&path("url")).unwrap().key, None); + + let h = s.resolve(&path("header.X-Cache")).unwrap(); + assert_eq!(h.def.name, "header"); + assert_eq!(h.key.as_deref(), Some("x-cache")); + assert_eq!(h.document_path(), "header.x-cache"); + + let r = s.resolve(&path("req.header.accept")).unwrap(); + assert_eq!(r.def.name, "req.header"); + assert_eq!(r.key.as_deref(), Some("accept")); + + // The map itself, with no key + assert_eq!(s.resolve(&path("header")).unwrap().key, None); + + assert!(s.resolve(&path("nope")).is_none()); + assert!(s.resolve(&path("url.x")).is_none()); + assert!(s.resolve(&path("req")).is_none()); + } + + #[test] + fn reports_unknown_fields_and_bad_values() { + let s = schema(); + let input = "@nope:x @url:y @duration:>5m @duration:>5s @error:maybe @error:true"; + let diagnostics = s.diagnostics(&parse(input).unwrap()); + assert_eq!( + diagnostics, + vec![ + Diagnostic::UnknownField { path: "nope".into(), span: Span::new(0, 6) }, + Diagnostic::InvalidValue { path: "duration".into(), span: Span::new(25, 28) }, + Diagnostic::InvalidValue { path: "error".into(), span: Span::new(50, 55) }, + ] + ); + } + + /// Diagnostics flag exactly the unit mistakes the evaluator rejects. + #[test] + fn reports_units_the_evaluator_rejects() { + let s = schema(); + let invalid = |input: &str| !s.diagnostics(&parse(input).unwrap()).is_empty(); + assert!(invalid("@duration:1m")); + assert!(invalid("@duration:[1m TO 2s]")); + assert!(invalid("@url:>1s")); + assert!(invalid("@url:[1 TO 2s]")); + assert!(!invalid("@duration:1s")); + assert!(!invalid("@duration:1500")); + assert!(!invalid("@duration:abc")); + assert!(!invalid("@duration:\"1m\"")); + assert!(!invalid("@url:>1")); + assert!(!invalid("@url:1m")); + } + + #[test] + fn deserializes_from_json() { + let s: Schema = serde_json::from_str( + r#"{ + "fields": [ + { "name": "method", "kind": { "type": "text" } }, + { "name": "header", "kind": { "type": "text" }, "map": true }, + { "name": "size", "kind": { "type": "number", "units": [{ "name": "kb", "factor": 1024 }] } }, + { "name": "error", "kind": { "type": "flag" }, "info": "Failed" } + ], + "freeText": "fuzzy" + }"#, + ) + .unwrap(); + assert_eq!(s.free_text, FreeText::Fuzzy); + assert!(s.fields[1].map); + assert_eq!(s.fields[3].info.as_deref(), Some("Failed")); + } +} diff --git a/crates/yaak-query/src/tokens.rs b/crates/yaak-query/src/tokens.rs new file mode 100644 index 000000000..11ff84243 --- /dev/null +++ b/crates/yaak-query/src/tokens.rs @@ -0,0 +1,100 @@ +//! Spans for syntax highlighting. + +use crate::ast::Span; +use crate::lexer::{TokKind, lex}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub enum TokenKind { + /// `@path:`, through the colon. + Field, + /// A field's value, quoted or not. + Value, + /// `OR` and `AND`. + Operator, + /// A quoted phrase in free text. + Quote, + Paren, + /// A `-` that negates what follows it. + Negation, + /// A bare free-text word. + Text, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(test, derive(ts_rs::TS))] +#[cfg_attr(test, ts(export, export_to = "query.ts"))] +pub struct Token { + pub kind: TokenKind, + pub span: Span, +} + +/// The highlighted pieces of `input`, in order. Whitespace is not covered. +pub fn tokens(input: &str) -> Vec { + let mut out = Vec::new(); + let toks = lex(input); + for (i, tok) in toks.iter().enumerate() { + let kind = match &tok.kind { + TokKind::LParen | TokKind::RParen => TokenKind::Paren, + TokKind::Minus => { + // The parser ignores a `-` with nothing to negate directly after it + let negates = toks.get(i + 1).is_some_and(|next| { + next.span.start == tok.span.end + && !matches!(next.kind, TokKind::RParen | TokKind::Or | TokKind::And) + }); + if negates { TokenKind::Negation } else { TokenKind::Text } + } + TokKind::Or | TokKind::And => TokenKind::Operator, + TokKind::Word(_) => TokenKind::Text, + TokKind::Phrase { .. } => TokenKind::Quote, + TokKind::Field { path_span, value_span, .. } => { + out.push(Token { kind: TokenKind::Field, span: *path_span }); + if value_span.start != value_span.end { + out.push(Token { kind: TokenKind::Value, span: *value_span }); + } + continue; + } + }; + out.push(Token { kind, span: tok.span }); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn show(input: &str) -> Vec<(TokenKind, &str)> { + tokens(input).into_iter().map(|t| (t.kind, t.span.slice(input))).collect() + } + + #[test] + fn kinds_and_spans() { + use TokenKind::*; + assert_eq!( + show("-@method:post OR (\"a b\" c) @url:"), + vec![ + (Negation, "-"), + (Field, "@method:"), + (Value, "post"), + (Operator, "OR"), + (Paren, "("), + (Quote, "\"a b\""), + (Text, "c"), + (Paren, ")"), + (Field, "@url:"), + ] + ); + assert_eq!( + show("@status:[400 TO 499]"), + vec![(Field, "@status:"), (Value, "[400 TO 499]")] + ); + assert_eq!(show("localhost:3000"), vec![(Text, "localhost:3000")]); + assert_eq!(show(""), vec![]); + // A `-` with nothing directly after it negates nothing + assert_eq!(show("- a -"), vec![(Text, "-"), (Text, "a"), (Text, "-")]); + } +} diff --git a/crates/yaak-query/src/wasm.rs b/crates/yaak-query/src/wasm.rs new file mode 100644 index 000000000..fbf0089b3 --- /dev/null +++ b/crates/yaak-query/src/wasm.rs @@ -0,0 +1,161 @@ +//! The `@yaakapp-internal/query` exports. +//! +//! JavaScript counts string positions in UTF-16 code units, so every span going out is +//! converted from bytes, and the cursor coming in is converted to bytes. + +use crate::{Document, Matcher, Schema, Span, Table}; +use serde::Serialize; +use serde::de::DeserializeOwned; +use std::collections::HashMap; +use wasm_bindgen::JsValue; +use wasm_bindgen::prelude::wasm_bindgen; + +/// Byte offset to UTF-16 offset, and back, for one input. +struct Offsets { + /// `utf16[i]` is the UTF-16 offset of byte `i`, for every char boundary and the end. + utf16: Vec, +} + +impl Offsets { + fn new(input: &str) -> Self { + let mut utf16 = vec![0; input.len() + 1]; + let mut units = 0u32; + for (i, c) in input.char_indices() { + for b in 0..c.len_utf8() { + utf16[i + b] = units; + } + units += c.len_utf16() as u32; + } + utf16[input.len()] = units; + Self { utf16 } + } + + fn span(&self, span: Span) -> Span { + Span { start: self.utf16[span.start()], end: self.utf16[span.end()] } + } + + /// The byte offset of UTF-16 offset `pos`, or of the char it falls inside. + fn byte(&self, pos: u32) -> usize { + self.utf16.partition_point(|&u| u < pos).min(self.utf16.len() - 1) + } +} + +fn to_js(value: &T) -> Result { + value + .serialize(&serde_wasm_bindgen::Serializer::json_compatible()) + .map_err(|e| JsValue::from_str(&e.to_string())) +} + +fn from_js(value: JsValue) -> Result { + serde_wasm_bindgen::from_value(value).map_err(|e| JsValue::from_str(&e.to_string())) +} + +/// The parsed tree (`Node | null`), with UTF-16 spans. +#[wasm_bindgen] +pub fn parse_query(input: &str) -> Result { + let offsets = Offsets::new(input); + let mut root = crate::parse(input); + if let Some(root) = &mut root { + root.map_spans(&|s| offsets.span(s)); + } + to_js(&root) +} + +/// Highlighting spans (`Token[]`), in UTF-16 offsets. +#[wasm_bindgen] +pub fn query_tokens(input: &str) -> Result { + let offsets = Offsets::new(input); + let mut tokens = crate::tokens(input); + for token in &mut tokens { + token.span = offsets.span(token.span); + } + to_js(&tokens) +} + +/// `Completions | null` at UTF-16 offset `cursor`. `values` maps field paths to the values seen +/// for them (`Record`). +#[wasm_bindgen] +pub fn query_completions( + input: &str, + cursor: u32, + schema: JsValue, + values: JsValue, +) -> Result { + let schema: Schema = from_js(schema)?; + let values: HashMap> = + if values.is_undefined() || values.is_null() { HashMap::new() } else { from_js(values)? }; + let offsets = Offsets::new(input); + let mut result = crate::completions(input, offsets.byte(cursor), &schema, &values); + if let Some(result) = &mut result { + result.span = offsets.span(result.span); + } + to_js(&result) +} + +/// The terms the schema cannot run (`Diagnostic[]`), with UTF-16 spans. +#[wasm_bindgen] +pub fn query_diagnostics(input: &str, schema: JsValue) -> Result { + let schema: Schema = from_js(schema)?; + let offsets = Offsets::new(input); + let mut diagnostics = + crate::parse(input).map(|root| schema.diagnostics(&root)).unwrap_or_default(); + for diagnostic in &mut diagnostics { + match diagnostic { + crate::Diagnostic::UnknownField { span, .. } + | crate::Diagnostic::InvalidValue { span, .. } => *span = offsets.span(*span), + } + } + to_js(&diagnostics) +} + +#[wasm_bindgen] +pub fn query_format_field(field: &str, value: &str, negated: bool) -> String { + crate::format_field(field, value, negated) +} + +/// A query parsed and resolved once, then matched against many documents. +#[wasm_bindgen] +pub struct QueryMatcher { + matcher: Matcher, +} + +#[wasm_bindgen] +impl QueryMatcher { + #[wasm_bindgen(constructor)] + pub fn new(input: &str, schema: JsValue) -> Result { + let schema: Schema = from_js(schema)?; + Ok(QueryMatcher { matcher: Matcher::new(crate::parse(input).as_ref(), &schema) }) + } + + /// Whether the query has no terms, so it matches everything. + #[wasm_bindgen(getter, js_name = isEmpty)] + pub fn is_empty(&self) -> bool { + self.matcher.is_empty() + } + + /// Whether a `Document` (`{ text, fields }`), as JSON, matches. JSON crosses into wasm far + /// faster than walking a JS object property by property. + pub fn matches(&self, doc_json: &str) -> Result { + let doc: Document = serde_json::from_str(doc_json).map_err(js_error)?; + Ok(self.matcher.matches(&doc)) + } + + /// The indexes of the matching rows, for many documents in one call. The cells (per row, + /// its free text, then one per column) are packed as in [`Table::from_packed`]: joined into + /// one string, with each one's UTF-16 length, or `Table::NO_VALUE` for no value. + #[wasm_bindgen(js_name = matchingRows)] + pub fn matching_rows( + &self, + columns: Vec, + joined: &str, + lengths: &[u32], + ) -> Result, JsValue> { + let table = Table::from_packed(columns, joined, lengths) + .ok_or_else(|| JsValue::from_str("cells do not line up with the columns"))?; + Ok(self.matcher.matching_rows(&table)) + } +} + +fn js_error(e: impl std::fmt::Display) -> JsValue { + JsValue::from_str(&e.to_string()) +} diff --git a/package-lock.json b/package-lock.json index a4539a7a3..9a34d7914 100644 --- a/package-lock.json +++ b/package-lock.json @@ -64,6 +64,7 @@ "crates/yaak-git", "crates/yaak-models", "crates/yaak-plugins", + "crates/yaak-query", "crates/yaak-sse", "crates/yaak-sync", "crates/yaak-templates", @@ -322,6 +323,13 @@ "name": "@yaakapp-internal/plugins", "version": "1.0.0" }, + "crates/yaak-query": { + "name": "@yaakapp-internal/query", + "version": "1.0.0", + "devDependencies": { + "rimraf": "^6.1.2" + } + }, "crates/yaak-sse": { "name": "@yaakapp-internal/sse", "version": "1.0.0", @@ -5241,6 +5249,10 @@ "resolved": "crates-proxy/yaak-proxy-lib", "link": true }, + "node_modules/@yaakapp-internal/query": { + "resolved": "crates/yaak-query", + "link": true + }, "node_modules/@yaakapp-internal/rpc-schema": { "resolved": "crates/common/yaak-rpc-schema", "link": true diff --git a/package.json b/package.json index 9a83a9673..9361af88a 100644 --- a/package.json +++ b/package.json @@ -63,6 +63,7 @@ "crates/yaak-git", "crates/yaak-models", "crates/yaak-plugins", + "crates/yaak-query", "crates/yaak-sse", "crates/yaak-sync", "crates/yaak-templates", diff --git a/vite.config.ts b/vite.config.ts index d009150e4..88ca8c9af 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -10,6 +10,7 @@ export default defineConfig({ ignorePatterns: [ "**/bindings/**", "**/routeTree.gen.ts", + "crates/yaak-query/pkg/**", "crates/yaak-templates/pkg/**", "crates/yaak-wasm/pkg/**", "plugins/importer-bruno/tests/fixtures/upstream/**", @@ -18,6 +19,7 @@ export default defineConfig({ lint: { ignorePatterns: [ "npm/**", + "crates/yaak-query/pkg/**", "crates/yaak-templates/pkg/**", "crates/yaak-wasm/pkg/**", "**/bindings/gen_*.ts", From 77226dbedb0df37d341d465af0e64161abe730a1 Mon Sep 17 00:00:00 2001 From: Gregory Schier Date: Sun, 8 Nov 2026 09:07:04 -0700 Subject: [PATCH 11/45] Put Yaak Proxy's search on the shared query engine (#826) Co-authored-by: Claude Fable 5.1 --- Cargo.lock | 1 + apps/yaak-client/components/Sidebar.tsx | 36 +- .../core/Editor/filter/extension.ts | 88 -- apps/yaak-proxy/components/FlowListEmpty.tsx | 4 +- apps/yaak-proxy/components/ProxyLayout.tsx | 2 + .../components/SavedSearchDialog.tsx | 82 ++ apps/yaak-proxy/components/SearchBar.tsx | 128 +- apps/yaak-proxy/components/Sidebar.tsx | 38 +- apps/yaak-proxy/harness/fixtures.ts | 1 + apps/yaak-proxy/harness/streamFixtures.ts | 1 + apps/yaak-proxy/index.html | 3 + apps/yaak-proxy/lib/flowViews.test.ts | 1 + apps/yaak-proxy/lib/query.test.ts | 107 ++ apps/yaak-proxy/lib/query.ts | 147 ++ apps/yaak-proxy/lib/search.test.ts | 287 +--- apps/yaak-proxy/lib/search.ts | 217 +-- apps/yaak-proxy/lib/searchBox.ts | 79 ++ apps/yaak-proxy/lib/searchChanges.test.ts | 96 ++ apps/yaak-proxy/lib/searchChanges.ts | 105 ++ apps/yaak-proxy/lib/sidebarTree.test.ts | 51 +- apps/yaak-proxy/lib/sidebarTree.ts | 87 +- apps/yaak-proxy/lib/store.ts | 159 ++- apps/yaak-proxy/package.json | 1 + apps/yaak-proxy/vite.config.ts | 6 +- crates-proxy/yaak-proxy-lib/Cargo.toml | 2 + .../yaak-proxy-lib/bindings/gen_models.ts | 33 +- .../yaak-proxy-lib/bindings/gen_rpc.ts | 115 +- .../migrations/00000012_search.sql | 13 + crates-proxy/yaak-proxy-lib/src/control.rs | 7 +- crates-proxy/yaak-proxy-lib/src/db.rs | 34 +- crates-proxy/yaak-proxy-lib/src/flows.rs | 598 +++----- crates-proxy/yaak-proxy-lib/src/lib.rs | 209 ++- crates-proxy/yaak-proxy-lib/src/models.rs | 19 + crates-proxy/yaak-proxy-lib/src/query.rs | 1242 +++++++++++++++++ .../yaak-proxy-lib/src/query/tests.rs | 1001 +++++++++++++ .../yaak-proxy-lib/src/saved_queries.rs | 181 +++ crates-proxy/yaak-proxy-lib/tests/at_rest.rs | 20 +- crates-proxy/yaak-proxy-lib/tests/control.rs | 18 +- crates-proxy/yaak-proxy-lib/tests/flow.rs | 4 +- crates-proxy/yaak-proxy-lib/tests/streams.rs | 2 +- crates/yaak-query/bindings/query.ts | 18 +- crates/yaak-query/pkg/yaak_query_bg.wasm | Bin 340382 -> 339850 bytes crates/yaak-query/src/complete.rs | 77 +- crates/yaak-query/src/condition.rs | 241 ++++ crates/yaak-query/src/eval.rs | 201 +-- crates/yaak-query/src/lib.rs | 8 +- crates/yaak-query/src/offsets.rs | 52 + crates/yaak-query/src/schema.rs | 8 +- crates/yaak-query/src/wasm.rs | 32 +- package-lock.json | 1 + .../src/components/editor/filter/extension.ts | 31 +- .../components/editor/filter/filter.grammar | 78 -- .../components/editor/filter/filter.test.ts | 42 - .../ui/src/components/editor/filter/filter.ts | 22 - .../src/components/editor/filter/highlight.ts | 22 - .../components/editor/query/extension.test.ts | 23 + .../src/components/editor/query/extension.ts | 264 ++++ packages/ui/src/editor.ts | 3 +- 58 files changed, 4754 insertions(+), 1594 deletions(-) delete mode 100644 apps/yaak-client/components/core/Editor/filter/extension.ts create mode 100644 apps/yaak-proxy/components/SavedSearchDialog.tsx create mode 100644 apps/yaak-proxy/lib/query.test.ts create mode 100644 apps/yaak-proxy/lib/query.ts create mode 100644 apps/yaak-proxy/lib/searchBox.ts create mode 100644 apps/yaak-proxy/lib/searchChanges.test.ts create mode 100644 apps/yaak-proxy/lib/searchChanges.ts create mode 100644 crates-proxy/yaak-proxy-lib/migrations/00000012_search.sql create mode 100644 crates-proxy/yaak-proxy-lib/src/query.rs create mode 100644 crates-proxy/yaak-proxy-lib/src/query/tests.rs create mode 100644 crates-proxy/yaak-proxy-lib/src/saved_queries.rs create mode 100644 crates/yaak-query/src/condition.rs create mode 100644 crates/yaak-query/src/offsets.rs delete mode 100644 packages/ui/src/components/editor/filter/filter.grammar delete mode 100644 packages/ui/src/components/editor/filter/filter.test.ts delete mode 100644 packages/ui/src/components/editor/filter/filter.ts delete mode 100644 packages/ui/src/components/editor/filter/highlight.ts create mode 100644 packages/ui/src/components/editor/query/extension.test.ts create mode 100644 packages/ui/src/components/editor/query/extension.ts diff --git a/Cargo.lock b/Cargo.lock index 90c0664ee..73bdd8026 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11965,6 +11965,7 @@ dependencies = [ "yaak-models", "yaak-proxy", "yaak-proxy-helper", + "yaak-query", "yaak-rpc", "yaak-templates", ] diff --git a/apps/yaak-client/components/Sidebar.tsx b/apps/yaak-client/components/Sidebar.tsx index cb3044736..0a94f5426 100644 --- a/apps/yaak-client/components/Sidebar.tsx +++ b/apps/yaak-client/components/Sidebar.tsx @@ -65,7 +65,7 @@ import type { ContextMenuProps, DropdownItem } from "./core/Dropdown"; import { ContextMenu, Dropdown } from "./core/Dropdown"; import type { Node as QueryNode, Schema } from "@yaakapp-internal/query"; import { formatField, parseQuery, QueryMatcher } from "@yaakapp-internal/query"; -import { queryFilter } from "./core/Editor/filter/extension"; +import { queryFilter } from "@yaakapp-internal/ui/editor"; import { HttpMethodTag } from "./core/HttpMethodTag"; import { Icon, @@ -850,18 +850,27 @@ function setSidebarFilterText(text: string) { jotaiStore.set(sidebarFilterAtom, { text, key: `${Math.random()}` }); } -/** The fields `getItemFields` fills in. Bare words match `getItemText` by word prefix */ -const sidebarFilterColumns = [ - "method", - "url", - "name", - "type", - "folder", - "grpc_service", - "grpc_method", +/** + * The fields `getItemFields` fills in, each with what it is and a term using it, as completion + * shows them. Bare words match `getItemText` by word prefix + */ +const sidebarFilterFields: [name: string, info: string, example: string][] = [ + ["method", "GET, POST, …", "@method:post"], + ["url", "Request URL", "@url:*/users*"], + ["name", "Request name", '@name:"Get user"'], + ["type", "http, grpc or ws", "@type:grpc"], + ["folder", "Parent folder", "@folder:auth"], + ["grpc_service", "gRPC service", "@grpc_service:*Greeter"], + ["grpc_method", "gRPC method", "@grpc_method:SayHello"], ]; +const sidebarFilterColumns = sidebarFilterFields.map(([name]) => name); const sidebarFilterSchema: Schema = { - fields: sidebarFilterColumns.map((name) => ({ name, kind: { type: "text" } })), + fields: sidebarFilterFields.map(([name, info, example]) => ({ + name, + kind: { type: "text" }, + info, + example, + })), freeText: "fuzzy", }; @@ -1005,9 +1014,12 @@ const sidebarTreeAtom = atom< matcher.free(); markHidden(root); + // Values offered after `@field:`. URLs and names are left out: nearly every one is unique, + // so a list of them is noise rather than a shortcut. const fieldValues: Record = {}; for (const [name, values] of Object.entries(allFields)) { - fieldValues[name] = Array.from(values).filter((v) => v.length < 20); + if (name === "url" || name === "name") continue; + fieldValues[name] = Array.from(values); } const suggestions = Array.from(suggestionFields) .sort((a, b) => { diff --git a/apps/yaak-client/components/core/Editor/filter/extension.ts b/apps/yaak-client/components/core/Editor/filter/extension.ts deleted file mode 100644 index 003729fe5..000000000 --- a/apps/yaak-client/components/core/Editor/filter/extension.ts +++ /dev/null @@ -1,88 +0,0 @@ -import type { Completion, CompletionContext, CompletionResult } from "@codemirror/autocomplete"; -import { autocompletion, startCompletion } from "@codemirror/autocomplete"; -import { highlightingFor } from "@codemirror/language"; -import type { EditorState, Extension } from "@codemirror/state"; -import { RangeSetBuilder } from "@codemirror/state"; -import type { DecorationSet, ViewUpdate } from "@codemirror/view"; -import { Decoration, EditorView, ViewPlugin } from "@codemirror/view"; -import type { Tag } from "@lezer/highlight"; -import { tags as t } from "@lezer/highlight"; -import type { Schema, TokenKind } from "@yaakapp-internal/query"; -import { queryCompletions, queryTokens } from "@yaakapp-internal/query"; - -export interface QueryFilterOptions { - schema: Schema; - /** The values seen for each field path, offered after `@field:` */ - values?: Record; -} - -const tokenTags: Record = { - field: t.attributeName, - value: t.attributeValue, - operator: t.operatorKeyword, - quote: t.string, - paren: t.paren, - negation: t.operator, - text: null, -}; - -function buildDecorations(state: EditorState): DecorationSet { - const builder = new RangeSetBuilder(); - for (const token of queryTokens(state.doc.toString())) { - const tag = tokenTags[token.kind]; - const className = tag == null ? null : highlightingFor(state, [tag]); - if (className == null || token.span.start === token.span.end) continue; - builder.add(token.span.start, token.span.end, Decoration.mark({ class: className })); - } - return builder.finish(); -} - -const highlight = ViewPlugin.fromClass( - class { - decorations: DecorationSet; - - constructor(view: EditorView) { - this.decorations = buildDecorations(view.state); - } - - update(update: ViewUpdate) { - if (update.docChanged || update.viewportChanged) { - this.decorations = buildDecorations(update.state); - } - } - }, - { decorations: (v) => v.decorations }, -); - -function completionSource({ schema, values }: QueryFilterOptions) { - return (ctx: CompletionContext): CompletionResult | null => { - const result = queryCompletions(ctx.state.doc.toString(), ctx.pos, schema, values); - if (result == null) return null; - - const { start, end } = result.span; - // Field names pop up for something typed; on an empty spot, only when asked for - const onlyFields = result.items.every((item) => item.kind === "field"); - if (onlyFields && start === end && !ctx.explicit) return null; - - const options: Completion[] = result.items.map((item) => ({ - label: item.label, - type: item.kind === "field" ? "property" : "constant", - info: item.info ?? undefined, - apply: (view, _completion, from, to) => { - view.dispatch({ - changes: { from, to, insert: item.insert }, - selection: { anchor: from + item.insert.length }, - }); - if (item.reopen) startCompletion(view); - }, - })); - - // The engine has already filtered and ranked the options - return { from: start, to: end, options, filter: false }; - }; -} - -/** Highlighting and completion for a search box written in Yaak's query language */ -export function queryFilter(options: QueryFilterOptions): Extension { - return [highlight, autocompletion({ override: [completionSource(options)] })]; -} diff --git a/apps/yaak-proxy/components/FlowListEmpty.tsx b/apps/yaak-proxy/components/FlowListEmpty.tsx index 5c3b28a8e..6c51e596a 100644 --- a/apps/yaak-proxy/components/FlowListEmpty.tsx +++ b/apps/yaak-proxy/components/FlowListEmpty.tsx @@ -3,7 +3,7 @@ import classNames from "classnames"; import { useAtomValue, useSetAtom } from "jotai"; import { useEffect, useState } from "react"; import { rpc } from "../lib/rpc"; -import { parseSearch, widerQueries } from "../lib/search"; +import { activeQuery, widerQueries } from "../lib/query"; import { hideSystemAtom, pausedAtAtom, @@ -77,7 +77,7 @@ function NoResults({ text }: { text: string }) { const candidates = widerQueries(text); void Promise.all( candidates.map(async (candidate) => { - const query = parseSearch(candidate, hideSystem); + const query = activeQuery(candidate, hideSystem); const res = await rpc("list_flows", { ...query, limit: 0 }); return { text: candidate, count: query == null ? res.total : (res.matching ?? res.total) }; }), diff --git a/apps/yaak-proxy/components/ProxyLayout.tsx b/apps/yaak-proxy/components/ProxyLayout.tsx index 18601c197..95a4b181b 100644 --- a/apps/yaak-proxy/components/ProxyLayout.tsx +++ b/apps/yaak-proxy/components/ProxyLayout.tsx @@ -22,6 +22,7 @@ import { LockedState } from "./LockedState"; import { ProxyStatus, useProxySetup } from "./ProxyStatus"; import { HelperDialog } from "./HelperDialog"; import { SaveDialog } from "./SaveDialog"; +import { SavedSearchDialog } from "./SavedSearchDialog"; import { SearchBar } from "./SearchBar"; import { SettingsWindow, useOpenSettings } from "./Settings"; import { Sidebar } from "./Sidebar"; @@ -158,6 +159,7 @@ export function ProxyLayout() {

+ diff --git a/apps/yaak-proxy/components/SavedSearchDialog.tsx b/apps/yaak-proxy/components/SavedSearchDialog.tsx new file mode 100644 index 000000000..6fda7826f --- /dev/null +++ b/apps/yaak-proxy/components/SavedSearchDialog.tsx @@ -0,0 +1,82 @@ +import { Dialog, DialogFooter, PlainInput } from "@yaakapp-internal/ui"; +import { atom, useAtom } from "jotai"; +import { useState } from "react"; +import { useRpcMutation } from "../hooks/useRpcMutation"; +import { useShowToast } from "../lib/toast"; + +/** Saving the search box's text under a name, or renaming a saved search */ +export type SavedSearchDialog = + | { mode: "save"; query: string } + | { mode: "rename"; id: string; name: string }; + +export const savedSearchDialogAtom = atom(null); + +const FORM_ID = "saved-search"; + +/** Names a saved search, as `savedSearchDialogAtom` asks */ +export function SavedSearchDialog() { + const [dialog, setDialog] = useAtom(savedSearchDialogAtom); + if (dialog == null) return null; + // Keyed, so each opening starts from its own name. + return ( + setDialog(null)} + /> + ); +} + +function NameDialog({ dialog, onClose }: { dialog: SavedSearchDialog; onClose: () => void }) { + const [name, setName] = useState(dialog.mode === "rename" ? dialog.name : ""); + const showToast = useShowToast(); + const onError = (err: Error) => + showToast({ id: "saved-search", message: err.message, color: "danger" }); + const create = useRpcMutation("create_saved_query", { onSuccess: onClose, onError }); + const rename = useRpcMutation("rename_saved_query", { onSuccess: onClose, onError }); + const busy = create.isPending || rename.isPending; + + const submit = () => { + if (name.trim() === "" || busy) return; + if (dialog.mode === "save") create.mutate({ name, query: dialog.query }); + else rename.mutate({ id: dialog.id, name }); + }; + + return ( + +
{ + e.preventDefault(); + submit(); + }} + > + + + +
+ ); +} diff --git a/apps/yaak-proxy/components/SearchBar.tsx b/apps/yaak-proxy/components/SearchBar.tsx index 44d1cc045..cdcbf7271 100644 --- a/apps/yaak-proxy/components/SearchBar.tsx +++ b/apps/yaak-proxy/components/SearchBar.tsx @@ -1,34 +1,53 @@ -import { Icon, IconButton, PlainInput } from "@yaakapp-internal/ui"; +import { startCompletion } from "@codemirror/autocomplete"; +import type { Extension } from "@codemirror/state"; +import { Compartment } from "@codemirror/state"; +import type { EditorView } from "@codemirror/view"; +import { + Dropdown, + Icon, + IconButton, + Input, + type InputHandle, + LazyCodeEditor, +} from "@yaakapp-internal/ui"; +import { queryFilter } from "@yaakapp-internal/ui/editor"; import { useAtom, useAtomValue, useSetAtom, useStore } from "jotai"; -import { memo, type RefObject, useEffect, useRef } from "react"; +import { memo, useEffect, useMemo, useRef } from "react"; +import { useRpcQuery } from "../hooks/useRpcQuery"; +import { loadSeenValues, savedCompletions, searchProblems } from "../lib/searchBox"; import { allTrafficTotalAtom, lockedAtom, + savedQueriesAtom, searchTermsAtom, searchTextAtom, searchTextVersionAtom, setSearchTextAtom, } from "../lib/store"; +import { savedSearchDialogAtom } from "./SavedSearchDialog"; /** How long typing settles before the search runs */ const SEARCH_DEBOUNCE_MS = 200; /** - * The header's search over every flow in the database, by URL, method, status and content type, - * and the `@app`, `@host` and `@path` terms the sidebar writes. Its placeholder counts the flows. - * ⌘F focuses it and Escape clears it. + * The header's search over every flow in the database, in the query language: bare words match + * the URL, method, status and content type, and `@field:value` terms the rest, highlighted and + * completed as Yaak's sidebar filter is, with the values the proxy has seen. The backend runs + * it and marks what it cannot. Its placeholder counts the flows. ⌘F focuses it and Escape + * clears it. */ export const SearchBar = memo(function SearchBar() { const [text, setText] = useAtom(searchTextAtom); const setTerms = useSetAtom(searchTermsAtom); const setSearchText = useSetAtom(setSearchTextAtom); const version = useAtomValue(searchTextVersionAtom); + const total = useAtomValue(allTrafficTotalAtom); + const store = useStore(); + const setSavedDialog = useSetAtom(savedSearchDialogAtom); // Nothing to search while captured traffic is locked. const locked = useAtomValue(lockedAtom) != null; - // Read, not subscribed: `PlaceholderCount` keeps it current without re-rendering the field. - const total = useStore().get(allTrafficTotalAtom); - // PlainInput hands back its as the ref - const inputRef = useRef(null); + const schema = useRpcQuery("query_schema", {}, { staleTime: Infinity }).data?.schema; + const handleRef = useRef(null); useEffect(() => { const timer = setTimeout(() => setTerms(text), SEARCH_DEBOUNCE_MS); @@ -39,43 +58,69 @@ export const SearchBar = memo(function SearchBar() { const onKeyDown = (e: KeyboardEvent) => { if ((e.metaKey || e.ctrlKey) && !e.shiftKey && !e.altKey && e.key.toLowerCase() === "f") { e.preventDefault(); - inputRef.current?.focus(); - inputRef.current?.select(); + handleRef.current?.selectAll(); } }; window.addEventListener("keydown", onKeyDown); return () => window.removeEventListener("keydown", onKeyDown); }, []); + // The language goes in once the schema is read: into the editor there is, through a + // compartment, and into each one made after, as taking text from the sidebar makes one. + const compartment = useRef(new Compartment()); + const filter = useMemo( + () => + schema == null + ? [] + : queryFilter({ + schema, + loadValues: loadSeenValues, + // Read as completion opens, so saved searches are current without remaking it. + moreCompletions: (typed) => savedCompletions(store.get(savedQueriesAtom), typed), + problems: searchProblems, + }), + [schema, store], + ); + const extensions = useMemo(() => compartment.current.of(filter), [filter]); + useEffect(() => { + handleRef.current?.dispatch({ effects: compartment.current.reconfigure(filter) }); + }, [filter]); + // The field is replaced to take a value from outside, so focus moves to the new one. const clear = () => { setSearchText(""); - requestAnimationFrame(() => inputRef.current?.focus()); + requestAnimationFrame(() => handleRef.current?.focus()); }; return ( - ( + + )} + setRef={(h) => { + handleRef.current = h; + }} label="Search flows" hideLabel size="sm" className="font-mono text-xs" + stateKey={null} + language={null} + wrapLines={false} placeholder={locked ? "Traffic is locked" : placeholderFor(total)} disabled={locked} defaultValue={text} forceUpdateKey={String(version)} onChange={setText} - onKeyDownCapture={(e) => { - if (e.key === "Escape" && text !== "") { + extensions={extensions} + onKeyDown={(e) => { + if (e.key === "Escape" && handleRef.current?.value() !== "") { e.stopPropagation(); clear(); } }} leftSlot={ - <> - - - + } rightSlot={ text === "" ? ( @@ -83,14 +128,26 @@ export const SearchBar = memo(function SearchBar() { ⌘F ) : ( - +
+ , + onSelect: () => setSavedDialog({ mode: "save", query: text.trim() }), + }, + ]} + > + + + +
) } /> @@ -98,15 +155,14 @@ export const SearchBar = memo(function SearchBar() { }); /** - * Keeps the placeholder's count current as flows arrive, alone, so new traffic re-renders this - * and not the whole field + * Ctrl+Space lists what can be typed, fields and saved searches on an empty spot, as it does in + * Yaak's editors, whose shared editor leaves the key to each app */ -function PlaceholderCount({ inputRef }: { inputRef: RefObject }) { - const total = useAtomValue(allTrafficTotalAtom); - useEffect(() => { - if (inputRef.current != null) inputRef.current.placeholder = placeholderFor(total); - }, [inputRef, total]); - return null; +function startCompletionOnCtrlSpace(e: KeyboardEvent, view: EditorView): boolean { + if (!(e.ctrlKey && e.key === " " && !e.metaKey && !e.altKey && !e.shiftKey)) return false; + e.preventDefault(); + startCompletion(view); + return true; } /** "Search 4,102 flows", the title and count that sat in the header's middle */ diff --git a/apps/yaak-proxy/components/Sidebar.tsx b/apps/yaak-proxy/components/Sidebar.tsx index 5bea89e8a..23f3be64e 100644 --- a/apps/yaak-proxy/components/Sidebar.tsx +++ b/apps/yaak-proxy/components/Sidebar.tsx @@ -7,10 +7,11 @@ import { type TreeProps, } from "@yaakapp-internal/ui"; import classNames from "classnames"; -import { atom, useAtom, useAtomValue } from "jotai"; +import { atom, useAtom, useAtomValue, useSetAtom } from "jotai"; import { atomWithStorage } from "jotai/utils"; import { useCallback, useEffect } from "react"; import { useDecryptList, useEditDecryptList } from "../hooks/useDecryptList"; +import { useRpcMutation } from "../hooks/useRpcMutation"; import { SIDEBAR_TREE_ID } from "../lib/atoms"; import { blockerOf, decryptMenuAction, isDecrypted, listEdit, setDecrypted } from "../lib/decrypt"; import { applyCollapsed, type CollapsedMap, collapsedMap, folderIds } from "../lib/sidebarCollapse"; @@ -18,6 +19,7 @@ import { type SidebarItem, SYSTEM_ID } from "../lib/sidebarTree"; import { hideSystemAtom, noteSidebarInput, sidebarTreeAtom } from "../lib/store"; import { useShowToast } from "../lib/toast"; import { AppIcon } from "./AppIcon"; +import { savedSearchDialogAtom } from "./SavedSearchDialog"; /** The apps, sites, hosts and paths the user expanded, by node id, kept across launches */ const expandedAtom = atomWithStorage>("sidebar_expanded", {}, undefined, { @@ -63,6 +65,7 @@ function ItemInner({ item }: { item: SidebarItem }) {
{item.kind === "all" && } {item.kind === "tunnelled" && } + {item.kind === "saved" && } {item.kind === "folder" && } {item.appPath != null && } {item.label} @@ -119,17 +122,46 @@ function HideSystemToggle() { export function Sidebar() { const tree = useAtomValue(sidebarTreeAtom); const getItemKey = useCallback( - (item: SidebarItem) => `${item.id}:${item.count}:${item.countLabel ?? ""}`, + // The label too, as a saved search's changes when it is renamed + (item: SidebarItem) => `${item.id}:${item.count}:${item.countLabel ?? ""}:${item.label}`, [], ); const decryptList = useDecryptList(); const setDecryptList = useEditDecryptList(); const { mutate: saveDecryptList } = setDecryptList; const showToast = useShowToast(); + const setSavedDialog = useSetAtom(savedSearchDialogAtom); + const { mutate: deleteSaved } = useRpcMutation("delete_saved_query", { + onError: (err) => showToast({ id: "saved-search", message: err.message, color: "danger" }), + }); // Decrypt or Stop Decrypting the selected hosts, sites and apps, from new connections on. + // A saved search is renamed or deleted. const getContextMenu = useCallback( (items: SidebarItem[]): DropdownItem[] => { + const saved = items.flatMap((i) => (i.savedId == null ? [] : [{ ...i, id: i.savedId }])); + if (saved.length > 0) { + const [first] = saved; + return [ + { + label: "Rename", + leftSlot: , + hidden: saved.length > 1 || first == null, + onSelect: () => { + if (first != null) + setSavedDialog({ mode: "rename", id: first.id, name: first.label }); + }, + }, + { + label: "Delete", + color: "danger", + leftSlot: , + onSelect: () => { + for (const s of saved) deleteSaved({ id: s.id }); + }, + }, + ]; + } const subjects = items.flatMap((i) => (i.decrypt == null ? [] : [i.decrypt])); const action = decryptMenuAction(decryptList, subjects); if (action == null) return []; @@ -155,7 +187,7 @@ export function Sidebar() { }, ]; }, - [decryptList, saveDecryptList, showToast], + [decryptList, saveDecryptList, showToast, setSavedDialog, deleteSaved], ); const renderContextMenu = useCallback["renderContextMenu"]>>( ({ items, position, onClose }) => diff --git a/apps/yaak-proxy/harness/fixtures.ts b/apps/yaak-proxy/harness/fixtures.ts index 2f052dd74..cc90cf443 100644 --- a/apps/yaak-proxy/harness/fixtures.ts +++ b/apps/yaak-proxy/harness/fixtures.ts @@ -51,6 +51,7 @@ const base: Flow = { grpcStatus: null, grpcMessage: null, tunnelled: false, + agentRead: false, clientAddr: "127.0.0.1:54321", elapsedConnectMs: 18, elapsedTlsMs: 41, diff --git a/apps/yaak-proxy/harness/streamFixtures.ts b/apps/yaak-proxy/harness/streamFixtures.ts index 9fcbe3c9a..47fa878b7 100644 --- a/apps/yaak-proxy/harness/streamFixtures.ts +++ b/apps/yaak-proxy/harness/streamFixtures.ts @@ -50,6 +50,7 @@ const flowBase: Flow = { grpcStatus: null, grpcMessage: null, tunnelled: false, + agentRead: false, clientAddr: null, elapsedConnectMs: null, elapsedTlsMs: null, diff --git a/apps/yaak-proxy/index.html b/apps/yaak-proxy/index.html index 60d859012..cc49b8e4a 100644 --- a/apps/yaak-proxy/index.html +++ b/apps/yaak-proxy/index.html @@ -24,5 +24,8 @@ +
+
+
diff --git a/apps/yaak-proxy/lib/flowViews.test.ts b/apps/yaak-proxy/lib/flowViews.test.ts index 5c7daff3c..6ef8d323c 100644 --- a/apps/yaak-proxy/lib/flowViews.test.ts +++ b/apps/yaak-proxy/lib/flowViews.test.ts @@ -59,6 +59,7 @@ const flow: Flow = { grpcStatus: null, grpcMessage: null, tunnelled: false, + agentRead: false, clientAddr: null, elapsedConnectMs: null, elapsedTlsMs: null, diff --git a/apps/yaak-proxy/lib/query.test.ts b/apps/yaak-proxy/lib/query.test.ts new file mode 100644 index 000000000..dea324fbf --- /dev/null +++ b/apps/yaak-proxy/lib/query.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, test } from "vite-plus/test"; +import { + activeQuery, + anyCountsBackFromNow, + asOneTerm, + countsBackFromNow, + fieldTerm, + pathTerm, + termKeys, + widerQueries, +} from "./query"; +import { savedCompletions } from "./searchBox"; + +describe("terms", () => { + test("quote values that need it", () => { + expect(fieldTerm("app", "Google Chrome")).toBe('@app:"Google Chrome"'); + expect(fieldTerm("host", "api.x.com")).toBe("@host:api.x.com"); + expect(fieldTerm("is", "system", true)).toBe("-@is:system"); + }); + + test("select a path and what is under it", () => { + expect(pathTerm("/v1/charges")).toBe("@path:/v1/charges/*"); + expect(pathTerm("/a b")).toBe('@path:"/a b"'); + }); + + test("are read back as written, any case, with their exclusion", () => { + expect(termKeys('@app:"Google Chrome" -@is:System (a OR -b)')).toEqual([ + '@app:"google chrome"', + "-@is:system", + "a", + "-b", + ]); + expect(termKeys("")).toEqual([]); + }); +}); + +describe("activeQuery", () => { + test("is null for all traffic, and carries hiding System", () => { + expect(activeQuery(" ", false)).toBeNull(); + expect(activeQuery("", true)).toEqual({ query: "", hideSystem: true }); + expect(activeQuery(" stripe ", false)).toEqual({ query: "stripe", hideSystem: false }); + }); +}); + +describe("widerQueries", () => { + test("leaves out each term in turn", () => { + expect(widerQueries('@app:Slack @status:5xx "card declined"')).toEqual([ + '@status:5xx "card declined"', + '@app:Slack "card declined"', + "@app:Slack @status:5xx", + ]); + expect(widerQueries("@app:Slack")).toEqual([]); + }); + + test("widens one OR group at a time, keeping the groups", () => { + const text = "@app:Slack @host:absent.com OR @app:Chrome @host:absent.com"; + expect(widerQueries(text)).toEqual([ + "@host:absent.com OR @app:Chrome @host:absent.com", + "@app:Slack OR @app:Chrome @host:absent.com", + "@app:Slack @host:absent.com OR @host:absent.com", + "@app:Slack @host:absent.com OR @app:Chrome", + ]); + // A group of one term is not emptied, which would take in every flow. + expect(widerQueries("@app:Slack OR @app:Chrome @host:b.com")).toEqual([ + "@app:Slack OR @host:b.com", + "@app:Slack OR @app:Chrome", + ]); + }); + + test("keeps groups and exclusions whole", () => { + expect(widerQueries("(@status:4xx OR @status:5xx) -(a OR b) c")).toEqual([ + "-(a OR b) c", + "(@status:4xx OR @status:5xx) c", + "(@status:4xx OR @status:5xx) -(a OR b)", + ]); + }); +}); + +describe("countsBackFromNow", () => { + test("is a @time comparison or range, which ages", () => { + expect(countsBackFromNow("@time:<5m")).toBe(true); + expect(countsBackFromNow("x -(@time:[1h TO 2h] OR y)")).toBe(true); + expect(countsBackFromNow("@time:2026-11-07")).toBe(false); + expect(countsBackFromNow("@status:<500 time")).toBe(false); + }); + + test("of saved searches, tells when their counts change with time alone", () => { + expect(anyCountsBackFromNow(["@status:4xx", "-@time:>1h stripe"])).toBe(true); + expect(anyCountsBackFromNow(["@status:4xx", "@time:2026-11-07"])).toBe(false); + expect(anyCountsBackFromNow([])).toBe(false); + }); +}); + +describe("saved searches as completions", () => { + test("keep an OR at their top in one group, however it is spaced", () => { + expect(asOneTerm("@status:4xx OR @status:5xx")).toBe("(@status:4xx OR @status:5xx)"); + expect(asOneTerm("(@status:4xx)OR(@status:5xx)")).toBe("((@status:4xx)OR(@status:5xx))"); + expect(asOneTerm("(a OR b) c")).toBe("(a OR b) c"); + const saved = [ + { id: "1", name: "Errors", query: "(@status:4xx)OR(@status:5xx)", sortOrder: 0 }, + { id: "2", name: "Stripe", query: "@host:*.stripe.com", sortOrder: 1 }, + ]; + expect(savedCompletions(saved, "err").map((c) => c.apply)).toEqual([ + "((@status:4xx)OR(@status:5xx))", + ]); + }); +}); diff --git a/apps/yaak-proxy/lib/query.ts b/apps/yaak-proxy/lib/query.ts new file mode 100644 index 000000000..c0eb4021b --- /dev/null +++ b/apps/yaak-proxy/lib/query.ts @@ -0,0 +1,147 @@ +import type { Node } from "@yaakapp-internal/query"; +import { formatField, parseQuery } from "@yaakapp-internal/query"; + +/** + * A search over every flow in the database, as `list_flows` takes it: the search box's text, + * which the backend parses and runs, and whether System's flows are hidden. + */ +export interface FlowQuery { + query: string; + hideSystem: boolean; +} + +/** The search the list shows, or null for all traffic */ +export function activeQuery(text: string, hideSystem: boolean): FlowQuery | null { + if (text.trim() === "" && !hideSystem) return null; + return { query: text.trim(), hideSystem }; +} + +/** `@field:value`, or `-@field:value`, the value quoted when it needs to be */ +export function fieldTerm(field: string, value: string, negated = false): string { + return formatField(field, value, negated); +} + +/** A site, its domain and every host below it: `@host:*.github.com` */ +export function siteTerm(site: string): string { + const term = formatField("host", site); + return term.endsWith('"') ? term : `@host:*.${site}`; +} + +/** + * A path and everything under it, `@path:/v1/charges/*`, as a sidebar row selects it. A path + * that has to be quoted cannot carry the wildcard, so it matches as written. + */ +export function pathTerm(path: string): string { + const term = formatField("path", path); + if (term.endsWith('"')) return term; + return `${term.replace(/\/+$/, "")}/*`; +} + +/** `node` written back as query text, from its source, groups in parentheses where they need */ +export function nodeText(node: Node, source: string, nested = false): string { + switch (node.type) { + case "And": { + const text = node.children.map((c) => nodeText(c, source, true)).join(" "); + return nested ? `(${text})` : text; + } + case "Or": { + const text = node.children.map((c) => nodeText(c, source, true)).join(" OR "); + return nested ? `(${text})` : text; + } + case "Not": + return `-${nodeText(node.node, source, true)}`; + default: + return source.slice(node.span.start, node.span.end); + } +} + +/** The terms of a query, each as written and with a `-` when it is excluded, lowercased */ +export function termKeys(text: string): string[] { + const keys: string[] = []; + const walk = (node: Node, negated: boolean) => { + switch (node.type) { + case "And": + case "Or": + for (const c of node.children) walk(c, negated); + break; + case "Not": + walk(node.node, !negated); + break; + default: + keys.push( + `${negated ? "-" : ""}${text.slice(node.span.start, node.span.end).toLowerCase()}`, + ); + } + }; + const root = parseQuery(text); + if (root != null) walk(root, false); + return keys; +} + +/** + * Wider searches for one that matched nothing: the text with each of its terms left out in + * turn, without repeats. With groups `OR` separates, a term is left out of one group at a time + * and the groups kept, since each group alone is narrower still; a group left with nothing would + * take in every flow, so it is not offered. + */ +export function widerQueries(text: string): string[] { + const root = parseQuery(text); + if (root == null) return []; + const clauses = root.type === "Or" ? root.children : [root]; + const termsOf = (clause: Node) => (clause.type === "And" ? clause.children : [clause]); + const write = (terms: Node[]) => terms.map((t) => nodeText(t, text, true)).join(" "); + const out = new Set(); + clauses.forEach((clause, i) => { + const terms = termsOf(clause); + if (terms.length <= 1) return; + terms.forEach((_, j) => { + const wider = clauses.map((c, k) => + k === i ? write(terms.filter((_, l) => l !== j)) : write(termsOf(c)), + ); + out.add(wider.join(" OR ")); + }); + }); + return [...out]; +} + +/** + * The search has a `@time` term that counts back from now, `@time:<5m`, so which flows it covers + * changes as time passes, not only as flows are written + */ +export function countsBackFromNow(text: string): boolean { + let found = false; + const walk = (node: Node) => { + switch (node.type) { + case "And": + case "Or": + for (const c of node.children) walk(c); + break; + case "Not": + walk(node.node); + break; + case "Field": + if ( + node.path.length === 1 && + node.path[0]?.toLowerCase() === "time" && + (node.value.type === "compare" || node.value.type === "range") + ) { + found = true; + } + break; + default: + } + }; + const root = parseQuery(text); + if (root != null) walk(root); + return found; +} + +/** Any of the searches counts back from now, so its count changes as time passes */ +export function anyCountsBackFromNow(queries: string[]): boolean { + return queries.some(countsBackFromNow); +} + +/** `text` to sit beside other terms as one: in parentheses when it is an `OR` at its top */ +export function asOneTerm(text: string): string { + return parseQuery(text)?.type === "Or" ? `(${text})` : text; +} diff --git a/apps/yaak-proxy/lib/search.test.ts b/apps/yaak-proxy/lib/search.test.ts index 55c838f53..66f930494 100644 --- a/apps/yaak-proxy/lib/search.test.ts +++ b/apps/yaak-proxy/lib/search.test.ts @@ -1,144 +1,6 @@ import type { Flow } from "@yaakapp-internal/proxy-lib"; import { describe, expect, test } from "vite-plus/test"; -import { - appKeyOf, - formatFieldFilter, - isSystemFlow, - matchesQuery, - parseSearch, - sidebarPath, - siteOf, - tokenize, - widerQueries, -} from "./search"; - -function flow(url: string, extra: Partial = {}): Flow { - return { - url, - method: "GET", - status: 200, - contentType: null, - clientBundleId: null, - clientPath: null, - ...extra, - } as Flow; -} - -// The same flows and terms as `search_matches_url_method_status_and_content_type` in -// `flows.rs`, so the live matcher and the database agree. -const stripe = flow("https://api.stripe.com/v1/charges?customer=cus_9", { - method: "POST", - contentType: "application/json; charset=utf-8", -}); -const github = flow("https://api.github.com/repos/yaak", { - status: 404, - contentType: "application/json; charset=utf-8", -}); -const logo = flow("https://images.ctfassets.net/logo.png", { contentType: "image/png" }); -const health = flow("http://localhost:3000/health", { method: "HEAD", status: 503 }); -const all = [stripe, github, logo, health]; - -const search = (q: string) => all.filter((f) => matchesQuery(f, { q })); - -describe("matchesQuery", () => { - test("matches the host, the path and the query, any case", () => { - expect(search("STRIPE")).toEqual([stripe]); - expect(search("/repos/")).toEqual([github]); - expect(search("cus_9")).toEqual([stripe]); - }); - - test("matches the whole method", () => { - expect(search("head")).toEqual([health]); - expect(search("ead")).toEqual([]); - }); - - test("matches a status, its start, or its class", () => { - expect(search("404")).toEqual([github]); - expect(search("50")).toEqual([health]); - expect(search("2xx")).toEqual([stripe, logo]); - }); - - test("matches the content type", () => { - expect(search("png")).toEqual([logo]); - expect(search("json")).toEqual([stripe, github]); - }); - - test("needs every term to match", () => { - expect(search("json get")).toEqual([github]); - expect(search(" json 2xx ")).toEqual([stripe]); - expect(search("json 5xx")).toEqual([]); - expect(search("")).toEqual(all); - }); - - test("keeps flows under a host or path, as the sidebar places them", () => { - const under = (node: string, url: string) => matchesQuery(flow(url), { under: [node] }); - expect(under("api.stripe.com", "https://api.stripe.com/v1/charges")).toBe(true); - expect(under("api.stripe.com", "http://api.stripe.com")).toBe(true); - // Another port is a host node of its own, as in the sidebar. - expect(under("api.stripe.com", "https://api.stripe.com:8443/v1")).toBe(false); - // Empty segments and default ports are left out, as the sidebar leaves them. - expect(under("example.com/v1/users", "https://example.com/v1//users")).toBe(true); - expect(under("x.com/a", "http://x.com:80/a")).toBe(true); - expect(under("api.stripe.com", "https://api.stripe.community/")).toBe(false); - expect(under("api.stripe.com/v1/charges", "https://api.stripe.com/v1/charges/ch_1?x")).toBe( - true, - ); - expect(under("api.stripe.com/v1/charges", "https://api.stripe.com/v1/chargesx")).toBe(false); - expect(under("api.stripe.com:8443", "https://api.stripe.com/v1")).toBe(false); - // The sidebar lowercases hosts; paths keep their case. - expect(under("example.com/A", "http://EXAMPLE.com/A/b")).toBe(true); - expect(under("example.com/a", "http://example.com/A")).toBe(false); - }); - - test("keeps flows from an app, by bundle id or path", () => { - const chrome = flow("https://example.com/", { - clientBundleId: "com.google.Chrome", - clientPath: "/Applications/Google Chrome.app", - }); - const curl = flow("https://example.com/", { clientPath: "/usr/bin/curl" }); - const unknown = flow("https://example.com/"); - const from = (app: string[]) => [chrome, curl, unknown].filter((f) => matchesQuery(f, { app })); - expect(from(["com.google.Chrome"])).toEqual([chrome]); - expect(from(["/usr/bin/curl"])).toEqual([curl]); - expect(from(["/Applications/Google Chrome.app"])).toEqual([]); - }); - - test("keeps flows from an app by name, or a program by the folder it ran in", () => { - // The flows `filters_by_app_name_folder_site_and_system` in `flows.rs` checks. - const slack = flow("https://edgeapi.slack.com/cache", { - clientName: "Slack", - clientBundleId: "com.tinyspeck.slackmacgap", - clientPath: "/Applications/Slack.app", - }); - const node = flow("https://api.github.com:443/repos", { - clientName: "node", - clientPath: "/opt/homebrew/bin/node", - clientCwd: "/Users/me/src/fieldwork", - }); - const daemon = flow("https://github.com/login", { - clientName: "nsurlsessiond", - clientPath: "/usr/libexec/nsurlsessiond", - clientCwd: "/", - }); - const none = flow("http://notgithub.com/"); - const flows = [slack, node, daemon, none]; - const where = (query: Parameters[1]) => - flows.filter((f) => matchesQuery(f, query)); - expect(where({ app: ["slack"] })).toEqual([slack]); - expect(where({ app: ["Fieldwork"] })).toEqual([node]); - expect(where({ app: ["/opt/homebrew/bin/node"] })).toEqual([node]); - expect(where({ under: ["*.github.com"] })).toEqual([node, daemon]); - expect(where({ under: ["api.github.com"] })).toEqual([node]); - expect(where({ system: true })).toEqual([daemon]); - expect(where({ system: false })).toEqual([slack, node, none]); - expect(flows.map(appKeyOf)).toEqual(["Slack", "fieldwork", "nsurlsessiond", null]); - }); - - test("combines terms with the sidebar's selection", () => { - expect(matchesQuery(stripe, { q: "post", under: ["api.stripe.com"] })).toBe(true); - expect(matchesQuery(stripe, { q: "post", under: ["api.github.com"] })).toBe(false); - }); -}); +import { appKeyOf, isSystemFlow, sidebarPath, siteOf } from "./search"; describe("sidebarPath", () => { // The cases `sidebar_path_rules` in `flows.rs` checks, so the two agree. @@ -159,125 +21,6 @@ describe("sidebarPath", () => { }); }); -describe("parseSearch", () => { - test("is null for text that narrows nothing", () => { - expect(parseSearch("")).toBeNull(); - expect(parseSearch(" ")).toBeNull(); - expect(parseSearch("OR")).toBeNull(); - }); - - test("keeps free words as terms, trimmed and collapsed", () => { - expect(parseSearch(" stripe post ")).toEqual({ q: "stripe post" }); - }); - - test("reads apps, hosts and the paths below them", () => { - expect(parseSearch('@app:"Google Chrome" @app:Slack card')).toEqual({ - q: "card", - app: ["Google Chrome", "Slack"], - }); - expect(parseSearch("@app:fieldwork @host:api.stripe.com @path:/v1/payment_intents")).toEqual({ - app: ["fieldwork"], - under: ["api.stripe.com/v1/payment_intents"], - }); - expect(parseSearch("@host:*.github.com")).toEqual({ under: ["*.github.com"] }); - expect(parseSearch("@host:github.com @path:/mountain-loop/yaak/*")).toEqual({ - under: ["github.com/mountain-loop/yaak"], - }); - // A path with no host matches anywhere in the URL. - expect(parseSearch("@path:/v1")).toEqual({ q: "/v1" }); - // Each path stays with the host before it. - expect(parseSearch("@host:a.com @path:/x @host:b.com @path:/y")).toEqual({ - under: ["a.com/x", "b.com/y"], - }); - }); - - test("keeps each clause OR separates as a group of its own", () => { - // Two selected sidebar rows under different apps: Slack on a.com, or Chrome on b.com. - const text = "@app:Slack @host:a.com OR @app:Chrome @host:b.com"; - const query = parseSearch(text); - expect(query).toEqual({ - any: [ - { app: ["Slack"], under: ["a.com"] }, - { app: ["Chrome"], under: ["b.com"] }, - ], - }); - const on = (clientName: string, host: string) => - flow(`https://${host}/x`, { clientName, clientBundleId: `app.${clientName}` }); - const match = (f: Flow) => query != null && matchesQuery(f, query); - expect(match(on("Slack", "a.com"))).toBe(true); - expect(match(on("Chrome", "b.com"))).toBe(true); - // Not every pairing of the two rows' apps and hosts. - expect(match(on("Slack", "b.com"))).toBe(false); - expect(match(on("Chrome", "a.com"))).toBe(false); - - // Words bind to their clause, and hiding System applies to each group that does not ask. - expect(parseSearch("@host:a.com OR @is:system @app:trustd", true)).toEqual({ - any: [ - { under: ["a.com"], system: false }, - { app: ["trustd"], system: true }, - ], - }); - // A clause that narrows nothing takes in every flow. - expect(parseSearch("@app:Slack OR")).toEqual({ app: ["Slack"] }); - expect(parseSearch("@app:Slack OR -@foo:bar")).toBeNull(); - }); - - test("keeps or drops System, which hiding it drops unless the text asks", () => { - expect(parseSearch("@is:system")).toEqual({ system: true }); - expect(parseSearch("-@is:system stripe")).toEqual({ q: "stripe", system: false }); - expect(parseSearch("", true)).toEqual({ system: false }); - expect(parseSearch("@is:system", true)).toEqual({ system: true }); - }); - - test("keeps or drops HTTPS passed through undecrypted", () => { - expect(parseSearch("@is:tunnelled")).toEqual({ tunnelled: true }); - expect(parseSearch("-@is:tunnelled stripe")).toEqual({ q: "stripe", tunnelled: false }); - const tunnel = { url: "https://a.com", method: "CONNECT", tunnelled: true } as Flow; - const decrypted = { url: "https://a.com/x", method: "GET", tunnelled: false } as Flow; - expect(matchesQuery(tunnel, { tunnelled: true })).toBe(true); - expect(matchesQuery(decrypted, { tunnelled: true })).toBe(false); - expect(matchesQuery(decrypted, { tunnelled: false })).toBe(true); - }); - - test("round-trips what the sidebar writes", () => { - const text = [ - formatFieldFilter("app", "Google Chrome"), - formatFieldFilter("host", "api.x.com"), - ].join(" "); - expect(text).toBe('@app:"Google Chrome" @host:api.x.com'); - expect(tokenize(text)).toEqual([ - { field: "app", value: "Google Chrome", negated: false }, - { field: "host", value: "api.x.com", negated: false }, - ]); - }); -}); - -describe("widerQueries", () => { - test("leaves out each term in turn", () => { - expect(widerQueries('@app:Slack @status:5xx "card declined"')).toEqual([ - '@status:5xx "card declined"', - '@app:Slack "card declined"', - "@app:Slack @status:5xx", - ]); - expect(widerQueries("@app:Slack")).toEqual([]); - }); - - test("widens one OR group at a time, keeping the groups", () => { - const text = "@app:Slack @host:absent.com OR @app:Chrome @host:absent.com"; - expect(widerQueries(text)).toEqual([ - "@host:absent.com OR @app:Chrome @host:absent.com", - "@app:Slack OR @app:Chrome @host:absent.com", - "@app:Slack @host:absent.com OR @host:absent.com", - "@app:Slack @host:absent.com OR @app:Chrome", - ]); - // A group of one term is not emptied, which would take in every flow. - expect(widerQueries("@app:Slack OR @app:Chrome @host:b.com")).toEqual([ - "@app:Slack OR @host:b.com", - "@app:Slack OR @app:Chrome", - ]); - }); -}); - describe("isSystemFlow", () => { // The cases `tells_system_daemons_apart` in `flows.rs` checks, so the two agree. test("is a daemon outside an app, under the system's folders or by name", () => { @@ -307,3 +50,31 @@ describe("siteOf", () => { expect(siteOf("149.154.175.50")).toBe("149.154.175.50"); }); }); + +describe("appKeyOf", () => { + test("names an app, a program, or a runtime by the folder it ran in", () => { + const flow = (extra: Partial) => + ({ + clientName: null, + clientBundleId: null, + clientPath: null, + clientCwd: null, + ...extra, + }) as Flow; + const flows = [ + flow({ clientName: "Slack", clientBundleId: "com.tinyspeck.slackmacgap" }), + flow({ + clientName: "node", + clientPath: "/opt/homebrew/bin/node", + clientCwd: "/src/fieldwork", + }), + flow({ + clientName: "nsurlsessiond", + clientPath: "/usr/libexec/nsurlsessiond", + clientCwd: "/", + }), + flow({}), + ]; + expect(flows.map(appKeyOf)).toEqual(["Slack", "fieldwork", "nsurlsessiond", null]); + }); +}); diff --git a/apps/yaak-proxy/lib/search.ts b/apps/yaak-proxy/lib/search.ts index 8cd2f4046..7183554c0 100644 --- a/apps/yaak-proxy/lib/search.ts +++ b/apps/yaak-proxy/lib/search.ts @@ -1,147 +1,6 @@ -import type { Flow, FlowFilter } from "@yaakapp-internal/proxy-lib"; +import type { Flow } from "@yaakapp-internal/proxy-lib"; import { getDomain } from "tldts"; -/** - * A search over every flow in the database: the search box's terms, its field terms, and - * whether System's flows are hidden, as `list_flows` takes them. - */ -export type FlowQuery = Pick< - FlowFilter, - "q" | "under" | "app" | "cwd" | "system" | "tunnelled" | "any" ->; - -/** A word of the search box: a free term, or `@field:value`, maybe negated with `-` */ -export interface SearchToken { - field: string | null; - value: string; - negated: boolean; -} - -/** - * The search box's words. Values may be quoted, `@app:"Google Chrome"`; `OR` and `AND` between - * words are dropped here, and [`clausesOf`] splits on `OR`. - */ -export function tokenize(text: string): SearchToken[] { - return clausesOf(text).flat(); -} - -/** - * The search box's words in the clauses `OR` separates, each a group of terms that must all - * match: `@app:Slack @host:a.com OR @app:Chrome @host:b.com` is two. `AND` is dropped, since - * terms side by side must all match anyway. - */ -export function clausesOf(text: string): SearchToken[][] { - const clauses: SearchToken[][] = [[]]; - const re = /(-)?(?:@([a-z]+):)?(?:"([^"]*)"?|(\S+))/gi; - for (const m of text.matchAll(re)) { - const [, minus, field, quoted, bare] = m; - const value = quoted ?? bare ?? ""; - if (field == null && minus == null && quoted == null && value === "OR") { - clauses.push([]); - continue; - } - if (field == null && minus == null && quoted == null && value === "AND") continue; - if (field == null && value === "" && quoted == null) continue; - clauses.at(-1)?.push({ field: field?.toLowerCase() ?? null, value, negated: minus != null }); - } - return clauses.filter((c) => c.length > 0); -} - -/** `@field:value`, quoting a value with spaces or quotes in it */ -export function formatFieldFilter(field: string, value: string, negated = false): string { - const v = /[\s"]/.test(value) || value === "" ? `"${value.replaceAll('"', "")}"` : value; - return `${negated ? "-" : ""}@${field}:${v}`; -} - -/** - * The search box's text as a query, or null when it narrows nothing. Clauses `OR` separates are - * groups, each judged whole, so two sidebar rows under different apps match what each does and - * not every pairing of their apps and hosts. Within a clause, `@app` terms match any of their - * apps; `@host` terms any of their hosts, each with the `@path` terms after it; `@is:system` - * keeps only macOS's daemons and `-@is:system` drops them, as hiding System does when the - * clause says neither. `@is:tunnelled` keeps only HTTPS passed through undecrypted, and - * `-@is:tunnelled` drops it. Other words match the URL, the method, the status or the type. - */ -export function parseSearch(text: string, hideSystem = false): FlowQuery | null { - const clauses = clausesOf(text); - if (clauses.length <= 1) return parseClause(clauses[0] ?? [], hideSystem); - const any: FlowQuery[] = []; - for (const clause of clauses) { - const group = parseClause(clause, hideSystem); - // A clause that narrows nothing takes in every flow, so the whole does too. - if (group == null) return hideSystem ? { system: false } : null; - any.push(group); - } - return { any }; -} - -function parseClause(tokens: SearchToken[], hideSystem: boolean): FlowQuery | null { - const words: string[] = []; - const apps: string[] = []; - const cwds: string[] = []; - // Each host with the paths written after it. - const hosts: { host: string; paths: string[] }[] = []; - let system: boolean | undefined = hideSystem ? false : undefined; - let tunnelled: boolean | undefined; - for (const t of tokens) { - if (t.field === "app" && !t.negated && t.value !== "") apps.push(t.value); - else if (t.field === "cwd" && !t.negated && t.value !== "") cwds.push(t.value); - else if (t.field === "host" && !t.negated && t.value !== "") { - hosts.push({ host: t.value.replace(/\/+$/, ""), paths: [] }); - } else if (t.field === "path" && !t.negated && t.value !== "") { - const path = t.value.replace(/\/?\*$/, "").replace(/^\/+/, ""); - const host = hosts.at(-1); - // A path with no host to sit under matches anywhere in the URL. - if (host == null) { - if (path !== "") words.push(`/${path}`); - } else { - host.paths.push(path); - } - } else if (t.field === "is" && t.value.toLowerCase() === "system") system = !t.negated; - else if (t.field === "is" && t.value.toLowerCase() === "tunnelled") tunnelled = !t.negated; - else if (t.field == null && !t.negated) words.push(t.value); - // Fields and exclusions the query language will add are ignored until it lands. - } - const under: string[] = []; - for (const { host, paths } of hosts) { - if (paths.length === 0 || host.startsWith("*.")) under.push(host); - else for (const p of paths) under.push(p === "" ? host : `${host}/${p}`); - } - const q = words.join(" ").trim(); - const query: FlowQuery = { - ...(q === "" ? {} : { q }), - ...(under.length > 0 ? { under } : {}), - ...(apps.length > 0 ? { app: apps } : {}), - ...(cwds.length > 0 ? { cwd: cwds } : {}), - ...(system == null ? {} : { system }), - ...(tunnelled == null ? {} : { tunnelled }), - }; - return Object.keys(query).length === 0 ? null : query; -} - -/** - * Wider searches for one that matched nothing: the text with each of its terms left out in turn, - * without repeats. With groups `OR` separates, a term is left out of one group at a time and - * the groups kept, since each group alone is narrower still; a group left with nothing would - * take in every flow, so it is not offered. - */ -export function widerQueries(text: string): string[] { - const format = (t: SearchToken) => - t.field == null - ? `${t.negated ? "-" : ""}${/\s/.test(t.value) ? `"${t.value}"` : t.value}` - : formatFieldFilter(t.field, t.value, t.negated); - const clauses = clausesOf(text).map((c) => c.map(format)); - const out = new Set(); - clauses.forEach((clause, i) => { - if (clause.length <= 1) return; - clause.forEach((_, j) => { - const wider = clauses.map((c, k) => (k === i ? c.filter((_, l) => l !== j) : c)); - out.add(wider.map((c) => c.join(" ")).join(" OR ")); - }); - }); - return [...out]; -} - /** The folder a program outside an app ran in */ export function folderOf(flow: Pick): string | null { if (flow.clientBundleId != null) return null; @@ -202,8 +61,8 @@ const SYSTEM_PROGRAMS = new Set([ /** * A flow from one of macOS's own daemons: a process outside an app bundle whose executable is * under `/System`, `/usr/libexec` and the like, or one of a few daemons by name, but not WebKit's - * helpers, which send apps' traffic. `is_system` in `crates-proxy/yaak-proxy-lib`, which the - * database filters by, follows the same rules. + * helpers, which send apps' traffic. `is_system` in `crates-proxy/yaak-proxy-lib`, which + * `@is:system` searches by, follows the same rules. */ export function isSystemFlow(flow: Pick): boolean { const path = flow.clientPath; @@ -213,66 +72,6 @@ export function isSystemFlow(flow: Pick): return SYSTEM_DIRS.some((dir) => path.startsWith(dir)) || SYSTEM_PROGRAMS.has(name); } -/** - * Whether `flow` is one `query` covers, as `list_flows` decides it in SQL, so new traffic joins a - * search as it arrives. Every term must match the URL, the method, the status or the content type. - */ -export function matchesQuery(flow: Flow, query: FlowQuery): boolean { - for (const term of (query.q ?? "").toLowerCase().split(/\s+/)) { - if (term === "") continue; - const matched = - flow.url.toLowerCase().includes(term) || - flow.method.toLowerCase() === term || - (flow.contentType?.toLowerCase().includes(term) ?? false) || - matchesStatus(flow.status, term); - if (!matched) return false; - } - if (query.under != null && !query.under.some((node) => isUnder(flow.url, node))) return false; - if (query.app != null && !query.app.some((app) => matchesApp(flow, app))) return false; - if (query.cwd != null && !query.cwd.some((cwd) => matchesCwd(flow, cwd))) return false; - if (query.system != null && isSystemFlow(flow) !== query.system) return false; - if (query.tunnelled != null && flow.tunnelled !== query.tunnelled) return false; - // One of the groups, each judged whole, as the database judges them. - if (query.any != null && query.any.length > 0) { - return query.any.some((group) => matchesQuery(flow, group)); - } - return true; -} - -/** By bundle id or path exactly, or by name or folder in any case */ -function matchesApp(flow: Flow, app: string): boolean { - if ((flow.clientBundleId ?? flow.clientPath) === app) return true; - const lower = app.toLowerCase(); - return flow.clientName?.toLowerCase() === lower || folderOf(flow)?.toLowerCase() === lower; -} - -/** By the whole working directory exactly, or its folder's name in any case */ -function matchesCwd(flow: Flow, cwd: string): boolean { - if (flow.clientCwd == null) return false; - return flow.clientCwd === cwd || folderName(flow.clientCwd)?.toLowerCase() === cwd.toLowerCase(); -} - -/** `404` exactly, `40` as the start of one, or a class such as `4xx` */ -function matchesStatus(status: number | null, term: string): boolean { - if (status == null) return false; - const klass = /^([1-9])xx$/.exec(term); - if (klass != null) return Math.trunc(status / 100) === Number(klass[1]); - if (!/^\d{1,3}$/.test(term)) return false; - return term.length === 3 ? status === Number(term) : String(status).startsWith(term); -} - -/** The URL is the host or path `node` names, or something below it, by the sidebar's rules */ -function isUnder(url: string, node: string): boolean { - const path = sidebarPath(url); - if (node.startsWith("*.") && !node.includes("/")) { - const site = node.slice(2).toLowerCase(); - const host = hostnameOf(path.split("/")[0] ?? ""); - return host === site || host.endsWith(`.${site}`); - } - const want = normalizeNode(node); - return path === want || path.startsWith(`${want}/`); -} - /** A host without its port, keeping an IPv6 address's brackets */ export function hostnameOf(host: string): string { if (host.startsWith("[")) { @@ -304,7 +103,7 @@ export function siteOf(host: string): string { * Where the sidebar places a URL: its host, lowercased and without its scheme's default port, * then its path's non-empty segments, joined by `/`, as in `api.stripe.com/v1/charges`. A URL * without a scheme is a host of its own. `sidebar_path` in `crates-proxy/yaak-proxy-lib`, which - * the database filters by, follows the same rules. + * `@host` searches by, follows the same rules. */ export function sidebarPath(url: string): string { const schemeEnd = url.indexOf("://"); @@ -326,11 +125,3 @@ export function sidebarPath(url: string): string { const segments = path.split("/").filter((s) => s !== ""); return [host, ...segments].join("/"); } - -/** A sidebar node as `sidebarPath` writes it, its host lowercased */ -function normalizeNode(node: string): string { - const slash = node.indexOf("/"); - return slash === -1 - ? node.toLowerCase() - : `${node.slice(0, slash).toLowerCase()}${node.slice(slash)}`; -} diff --git a/apps/yaak-proxy/lib/searchBox.ts b/apps/yaak-proxy/lib/searchBox.ts new file mode 100644 index 000000000..ffac3bd24 --- /dev/null +++ b/apps/yaak-proxy/lib/searchBox.ts @@ -0,0 +1,79 @@ +import type { Completion } from "@codemirror/autocomplete"; +import type { QueryDiagnostic, SavedQuery } from "@yaakapp-internal/proxy-lib"; +import type { QueryProblem } from "@yaakapp-internal/ui/editor"; +import { asOneTerm } from "./query"; +import { rpc } from "./rpc"; + +/** How long values read for completion are offered before they are read again */ +const VALUES_MS = 10_000; + +const seenValues = new Map(); +const reading = new Map>(); + +/** + * The values the proxy has seen for field paths, for completing them: read from the database + * the first time each is asked for, and again once they are a few seconds old, so the list + * keeps up with the traffic without a read per keystroke. + */ +export async function loadSeenValues(paths: string[]): Promise> { + const now = Date.now(); + const entries = await Promise.all( + paths.map(async (path): Promise<[string, string[]]> => { + const cached = seenValues.get(path); + if (cached != null && now - cached.at < VALUES_MS) return [path, cached.values]; + let read = reading.get(path); + if (read == null) { + read = rpc("query_values", { path }) + .then((values) => { + seenValues.set(path, { values, at: Date.now() }); + return values; + }) + .catch(() => cached?.values ?? []) + .finally(() => reading.delete(path)); + reading.set(path, read); + } + return [path, await read]; + }), + ); + return Object.fromEntries(entries); +} + +const PROBLEM_LABELS: Record = { + unknownField: "Unknown field", + invalidValue: "Invalid value", + notYet: "Not yet", +}; + +/** The terms of a search the backend cannot run, as the box underlines them */ +export function problemsOf(diagnostics: QueryDiagnostic[]): QueryProblem[] { + return diagnostics.map((d) => ({ + from: d.start, + to: d.end, + label: PROBLEM_LABELS[d.kind], + quiet: d.kind === "notYet", + })); +} + +export async function searchProblems(text: string): Promise { + if (text.trim() === "") return []; + return problemsOf(await rpc("validate_query", { query: text })); +} + +/** + * Saved searches whose names hold the word being typed, each replacing it with its search. One + * with `OR` at its top goes in parentheses, so it stays one group beside the box's other terms. + */ +export function savedCompletions(saved: SavedQuery[], typed: string): Completion[] { + const needle = typed.toLowerCase(); + return saved + .filter((s) => s.name.toLowerCase().includes(needle)) + .map((s) => { + const insert = asOneTerm(s.query); + return { + label: s.name, + detail: s.query, + type: "text", + apply: insert, + }; + }); +} diff --git a/apps/yaak-proxy/lib/searchChanges.test.ts b/apps/yaak-proxy/lib/searchChanges.test.ts new file mode 100644 index 000000000..6801cff8a --- /dev/null +++ b/apps/yaak-proxy/lib/searchChanges.test.ts @@ -0,0 +1,96 @@ +import type { Flow, ModelPayload } from "@yaakapp-internal/proxy-lib"; +import { describe, expect, test } from "vite-plus/test"; +import { emptyWindow, type FlowWindow } from "./flowWindow"; +import { changesFor, FlowEvents } from "./searchChanges"; + +const none = () => {}; + +function write(id: string, matches: boolean, key = "k"): ModelPayload { + return { + model: { id, startedAt: "2026-11-07 12:00:00", updatedAt: "2026-11-07 12:00:00" } as Flow, + change: { type: "upsert", created: false }, + search: { key, matches }, + }; +} + +describe("changesFor", () => { + test("takes matching flows and drops those that stop matching", () => { + const win: FlowWindow = { ...emptyWindow, flows: [write("a", true).model] }; + const kept = changesFor( + "k", + win, + [write("a", false), write("b", true), write("c", false)], + none, + ); + expect(kept.map((c) => [c.model.id, c.change.type])).toEqual([ + ["a", "delete"], + ["b", "upsert"], + ]); + }); + + test("a flow that joins and stops matching in one batch leaves again", () => { + const kept = changesFor("k", emptyWindow, [write("a", true), write("a", false)], none); + expect(kept.map((c) => [c.model.id, c.change.type])).toEqual([ + ["a", "upsert"], + ["a", "delete"], + ]); + }); + + test("leaves flows matched against another search to be matched again", () => { + const later: string[] = []; + const kept = changesFor("k", emptyWindow, [write("a", true, "old")], (unknown) => + later.push(...unknown.map((c) => c.model.id)), + ); + expect([kept, later]).toEqual([[], ["a"]]); + }); +}); + +describe("FlowEvents", () => { + test("drops a late answer for a flow a newer event removed, though the window no longer holds it", () => { + const events = new FlowEvents(); + const old = write("a", true, "old"); + events.arrived([old]); + // Under the new search the flow stops matching and leaves the window... + events.arrived([write("a", false)]); + // ...and the re-check of the old event answers after + expect(events.current([old])).toEqual([]); + }); + + test("drops a late answer for a flow the sweep removed", () => { + const events = new FlowEvents(); + const old = write("a", true, "old"); + events.arrived([old]); + events.removed(["a"]); + expect(events.current([old])).toEqual([]); + }); + + test("keeps an answer no newer event has passed, and one for other flows", () => { + const events = new FlowEvents(); + const a = write("a", true, "old"); + const b = write("b", true, "old"); + events.arrived([a, b]); + events.arrived([write("c", true)]); + expect(events.current([a, b])).toEqual([a, b]); + // Not noted: nothing to judge it by + const unseen = write("d", true, "old"); + expect(events.current([unseen])).toEqual([unseen]); + }); + + test("past what it remembers, drops what it can no longer judge", () => { + const events = new FlowEvents(2); + const a = write("a", true, "old"); + events.arrived([a, write("b", true), write("c", true)]); + expect(events.current([a])).toEqual([]); + const c = write("c", true, "old"); + events.arrived([c]); + expect(events.current([c])).toEqual([c]); + }); + + test("drops answers from before a clear", () => { + const events = new FlowEvents(); + const a = write("a", true, "old"); + events.arrived([a]); + events.reset(); + expect(events.current([a])).toEqual([]); + }); +}); diff --git a/apps/yaak-proxy/lib/searchChanges.ts b/apps/yaak-proxy/lib/searchChanges.ts new file mode 100644 index 000000000..314590306 --- /dev/null +++ b/apps/yaak-proxy/lib/searchChanges.ts @@ -0,0 +1,105 @@ +import type { ModelPayload } from "@yaakapp-internal/proxy-lib"; +import type { FlowWindow } from "./flowWindow"; + +/** + * The changes a search's window `win` takes: flows the backend matched against the search, by + * its `key`, as it sent their events, and every deletion. A flow the window holds that no longer + * matches, such as a pending flow that completed under `@is:pending`, leaves it, even when it + * joined earlier in the same batch. Flows matched against another search, sent as the search + * changed or read from the database while another process owns the engine, go to `unknown` to + * be matched again. + */ +export function changesFor( + key: string, + win: FlowWindow, + changes: ModelPayload[], + unknown: (changes: ModelPayload[]) => void, +): ModelPayload[] { + const others: ModelPayload[] = []; + // What the window holds as each change applies + const held = new Set(win.flows.map((f) => f.id)); + const kept: ModelPayload[] = []; + for (const c of changes) { + if (c.change.type === "delete") { + held.delete(c.model.id); + kept.push(c); + } else if (c.search?.key !== key) { + others.push(c); + } else if (c.search.matches) { + held.add(c.model.id); + kept.push(c); + } else if (held.delete(c.model.id)) { + kept.push({ ...c, change: { type: "delete" } }); + } + } + if (others.length > 0) unknown(others); + return kept; +} + +/** Flows whose newest event `FlowEvents` remembers, at most */ +const REMEMBERED = 20_000; + +/** + * The order flow events arrived in, so an answer that comes later than newer events for the + * same flow can be told apart: a re-check started before the search changed, say, whose flow a + * newer event has since updated or removed, even from the window. + */ +export class FlowEvents { + private count = 0; + /** Each event's place, by the payload as it arrived */ + private seqs = new WeakMap(); + /** The newest event's place, by flow, removals included, the least recent first */ + private latest = new Map(); + /** Places at or below this may have newer events no longer remembered */ + private floor = 0; + + constructor(private readonly remembered = REMEMBERED) {} + + /** Notes events as they arrive */ + arrived(changes: ModelPayload[]) { + for (const c of changes) { + this.count++; + this.seqs.set(c, this.count); + this.note(c.model.id); + } + } + + /** Notes flows removed with no event of their own, as the keep-traffic sweep removes them */ + removed(ids: string[]) { + for (const id of ids) { + this.count++; + this.note(id); + } + } + + private note(id: string) { + // Moved to the end, so the least recent go first + this.latest.delete(id); + this.latest.set(id, this.count); + for (const [oldest, seq] of this.latest) { + if (this.latest.size <= this.remembered) break; + this.latest.delete(oldest); + this.floor = Math.max(this.floor, seq); + } + } + + /** + * Of the changes a late answer is for, those no newer event for their flow has passed. One + * from before the oldest event remembered is dropped, as it can't be judged; the next count + * and reload correct the list. One whose arrival was not noted is kept. + */ + current(changes: T[]): T[] { + return changes.filter((c) => { + const seq = this.seqs.get(c); + if (seq == null) return true; + const newest = this.latest.get(c.model.id); + return newest == null ? seq > this.floor : seq >= newest; + }); + } + + /** Forgets everything, as a clear does */ + reset() { + this.latest.clear(); + this.floor = this.count; + } +} diff --git a/apps/yaak-proxy/lib/sidebarTree.test.ts b/apps/yaak-proxy/lib/sidebarTree.test.ts index 002a7bb15..f1f44dac3 100644 --- a/apps/yaak-proxy/lib/sidebarTree.test.ts +++ b/apps/yaak-proxy/lib/sidebarTree.test.ts @@ -1,7 +1,7 @@ import type { Flow } from "@yaakapp-internal/proxy-lib"; import type { TreeNode } from "@yaakapp-internal/ui"; import { describe, expect, test } from "vite-plus/test"; -import { parseSearch } from "./search"; +import { parseQuery } from "@yaakapp-internal/query"; import { buildSidebarTree, itemsById, @@ -83,8 +83,9 @@ describe("buildSidebarTree", () => { const query = (id: string) => items.get(id)?.query; expect(query("all")).toBe(""); expect(query("app:fieldwork")).toBe("@app:fieldwork"); + // A path row takes in what is under the path. expect(query("app:fieldwork|path:api.github.com/repos")).toBe( - "@app:fieldwork @host:api.github.com @path:/repos", + "@app:fieldwork @host:api.github.com @path:/repos/*", ); expect(query("site:github.com")).toBe("@host:*.github.com"); expect(query("system")).toBe("@is:system"); @@ -93,12 +94,12 @@ describe("buildSidebarTree", () => { expect(query("app:trustd|host:ocsp.apple.com")).toBe( "@is:system @app:trustd @host:ocsp.apple.com", ); - expect(parseSearch(query("app:trustd") ?? "", true)?.system).toBe(true); - // What each writes, read back, covers the flows the row counts. + // What each writes reads back as the terms it wrote. for (const item of items.values()) { - const parsed = parseSearch(item.query); - if (parsed == null || item.kind === "group") continue; + if (item.kind === "group" || item.kind === "all") continue; + const parsed = parseQuery(item.query); expect(parsed, item.id).toBeTruthy(); + expect(parsed?.type, item.id).not.toBe("Or"); } }); }); @@ -164,7 +165,7 @@ describe("programs outside an app", () => { expect(items.get("app:claude|cwd:yaak|host:statsig.anthropic.com")?.query).toBe( "@app:claude @cwd:yaak @host:statsig.anthropic.com", ); - expect(parseSearch(query)).toEqual({ app: ["claude"], cwd: ["yaak"] }); + expect(parseQuery(query)?.type).toBe("And"); }); }); @@ -215,12 +216,10 @@ describe("selection and the search text", () => { pick("app:Slack|site:slack.com"), ]); expect(text).toBe("@app:fieldwork @host:api.github.com OR @app:Slack @host:*.slack.com"); - expect(parseSearch(text)).toEqual({ - any: [ - { app: ["fieldwork"], under: ["api.github.com"] }, - { app: ["Slack"], under: ["*.slack.com"] }, - ], - }); + // Two groups, each judged whole + const parsed = parseQuery(text); + expect(parsed?.type).toBe("Or"); + expect(parsed?.type === "Or" ? parsed.children.map((c) => c.type) : []).toEqual(["And", "And"]); }); test("writes the selected rows' queries, OR'd, or nothing for All traffic", () => { @@ -240,3 +239,29 @@ describe("selection and the search text", () => { expect(queryInText("", "card")).toBe(false); }); }); + +describe("saved searches", () => { + test("follow All traffic, each a row with its count that writes its query", () => { + const saved = [ + { id: "sq_1", name: "Errors", query: "@status:>=400", count: 12 }, + { id: "sq_2", name: "Stripe", query: "@host:*.stripe.com", count: 0 }, + ]; + const root = buildSidebarTree(flows, 4, saved); + expect((root.children ?? []).slice(0, 2).flatMap((c) => shape(c))).toEqual([ + "All traffic 4", + "Saved 0", + " Errors 12", + " Stripe 0", + ]); + const items = itemsById(root); + expect(items.get("saved:sq_1")).toMatchObject({ + kind: "saved", + query: "@status:>=400", + savedId: "sq_1", + }); + // Rows, not folders + const group = (root.children ?? []).find((c) => c.item.id === "saved"); + expect(group?.children?.every((c) => c.children == null)).toBe(true); + expect(itemsById(buildSidebarTree(flows, 4)).has("saved")).toBe(false); + }); +}); diff --git a/apps/yaak-proxy/lib/sidebarTree.ts b/apps/yaak-proxy/lib/sidebarTree.ts index 3e1066689..76eaa535c 100644 --- a/apps/yaak-proxy/lib/sidebarTree.ts +++ b/apps/yaak-proxy/lib/sidebarTree.ts @@ -1,20 +1,13 @@ import type { Flow } from "@yaakapp-internal/proxy-lib"; import type { TreeNode } from "@yaakapp-internal/ui"; import type { DecryptSubject } from "./decrypt"; -import { - appKeyOf, - folderOf, - formatFieldFilter, - isSystemFlow, - runtimeFolderOf, - sidebarPath, - siteOf, - tokenize, -} from "./search"; +import { fieldTerm, pathTerm, siteTerm, termKeys } from "./query"; +import { appKeyOf, folderOf, isSystemFlow, runtimeFolderOf, sidebarPath, siteOf } from "./search"; export type SidebarKind = | "all" | "tunnelled" + | "saved" | "group" | "app" | "folder" @@ -23,8 +16,8 @@ export type SidebarKind = | "path"; /** - * A row of the sidebar: All traffic, a group, an app, a program's working directory, or a site, - * host or path under one + * A row of the sidebar: All traffic, a saved search, a group, an app, a program's working + * directory, or a site, host or path under one */ export interface SidebarItem { id: string; @@ -42,10 +35,21 @@ export interface SidebarItem { query: string; /** What Decrypt in the row's menu decrypts: its host, site or app */ decrypt?: DecryptSubject; + /** The saved search a Saved row is, for renaming and deleting it */ + savedId?: string; +} + +/** A saved search as the sidebar lists it, with how many flows it covers */ +export interface SavedRow { + id: string; + name: string; + query: string; + count: number; } /** The ids of All traffic and the groups */ export const ALL_ID = "all"; +export const SAVED_ID = "saved"; export const APPS_ID = "apps"; export const DOMAINS_ID = "domains"; export const SYSTEM_ID = "system"; @@ -83,13 +87,13 @@ function placeOf(flow: Flow): Place { const [host = "", ...segments] = schemeless ? [path] : path.split("/"); const site = siteOf(host); const app = appKeyOf(flow); - const hostQuery = formatFieldFilter("host", host); + const hostQuery = fieldTerm("host", host); const rows: PlaceRow[] = [ { id: `site:${site}`, kind: "site", label: site, - query: formatFieldFilter("host", `*.${site}`), + query: siteTerm(site), decrypt: { kind: "site", site }, }, { @@ -107,7 +111,7 @@ function placeOf(flow: Flow): Place { id: `path:${host}${p}`, kind: "path", label: `/${segment}`, - query: `${hostQuery} ${formatFieldFilter("path", p)}`, + query: `${hostQuery} ${pathTerm(p)}`, }); } const system = isSystemFlow(flow); @@ -123,7 +127,7 @@ function placeOf(flow: Flow): Place { })); } else { const folderId = `app:${app}|cwd:${folder}`; - const folderQuery = `${appQuery} ${formatFieldFilter("cwd", folder)}`; + const folderQuery = `${appQuery} ${fieldTerm("cwd", folder)}`; appRows = [ { id: folderId, kind: "folder", label: folder, query: folderQuery }, ...rows.map((r) => ({ @@ -144,8 +148,8 @@ function placeOf(flow: Flow): Place { * while System is hidden from All traffic. */ function appQueryFor(app: string, system: boolean): string { - const query = formatFieldFilter("app", app); - return system ? `${formatFieldFilter("is", "system")} ${query}` : query; + const query = fieldTerm("app", app); + return system ? `${fieldTerm("is", "system")} ${query}` : query; } /** A node being built, its children by id */ @@ -209,10 +213,14 @@ function group(id: string, label: string): Trie { * The sidebar: All traffic, then Apps, each app opening to its own sites, hosts and paths, then * Domains by site, host and path, then System, macOS's daemons, apart from the apps. A group with * nothing in it is left out. Not decrypted, after All traffic, gathers the HTTPS passed through - * undecrypted, when there is any. `total` is All traffic's count: every flow it covers in the - * database. + * undecrypted, when there is any, and Saved lists the saved searches, each with its count from + * the database. `total` is All traffic's count: every flow it covers in the database. */ -export function buildSidebarTree(flows: Flow[], total: number): TreeNode { +export function buildSidebarTree( + flows: Flow[], + total: number, + saved: SavedRow[] = [], +): TreeNode { const root: TreeNode = { item: { id: "root", kind: "group", label: "", count: 0, query: "" }, parent: null, @@ -223,7 +231,7 @@ export function buildSidebarTree(flows: Flow[], total: number): TreeNode 0) { + const node: TreeNode = { + item: { id: SAVED_ID, kind: "group", label: "Saved", count: 0, query: "" }, + parent: root, + depth: 1, + children: [], + draggable: false, + }; + node.children = saved.map((s) => ({ + item: { + id: `saved:${s.id}`, + kind: "saved", + label: s.name, + count: s.count, + query: s.query, + savedId: s.id, + }, + parent: node, + depth: 2, + draggable: false, + })); + root.children?.push(node); + } for (const g of [apps, domains, system]) { if (g.children.size === 0) continue; const node: TreeNode = { @@ -318,12 +349,8 @@ export function selectionText(items: SidebarItem[]): string { * stays selected while it is, so typing after selecting keeps it. */ export function queryInText(query: string, text: string): boolean { - const have = new Set(tokenize(text).map(tokenKey)); - const want = tokenize(query); + const have = new Set(termKeys(text)); + const want = termKeys(query); if (want.length === 0) return have.size === 0; - return want.every((t) => have.has(tokenKey(t))); -} - -function tokenKey(t: { field: string | null; value: string; negated: boolean }): string { - return `${t.negated ? "-" : ""}${t.field ?? ""}:${t.value.toLowerCase()}`; + return want.every((t) => have.has(t)); } diff --git a/apps/yaak-proxy/lib/store.ts b/apps/yaak-proxy/lib/store.ts index 59f86eb9b..fc9655c4d 100644 --- a/apps/yaak-proxy/lib/store.ts +++ b/apps/yaak-proxy/lib/store.ts @@ -1,4 +1,9 @@ -import type { Flow, GetProxyStateResponse, ModelPayload } from "@yaakapp-internal/proxy-lib"; +import type { + Flow, + GetProxyStateResponse, + ModelPayload, + SavedQuery, +} from "@yaakapp-internal/proxy-lib"; import { selectedIdsFamily } from "@yaakapp-internal/ui"; import { atom, type createStore } from "jotai"; import { atomWithStorage } from "jotai/utils"; @@ -20,8 +25,9 @@ import { WINDOW_SIZE, windowFromPage, } from "./flowWindow"; +import { activeQuery, anyCountsBackFromNow, countsBackFromNow, type FlowQuery } from "./query"; import { listen, rpc } from "./rpc"; -import { type FlowQuery, matchesQuery, parseSearch } from "./search"; +import { changesFor, FlowEvents } from "./searchChanges"; import { ALL_ID, APPS_ID, @@ -29,6 +35,8 @@ import { DOMAINS_ID, itemsById, queryInText, + SAVED_ID, + type SavedRow, selectionText, } from "./sidebarTree"; @@ -40,6 +48,8 @@ const FLUSH_MS = 100; const RECOUNT_MS = 1000; /** How often the database is read for new flows while another process owns the engine */ const POLL_MS = 1000; +/** How often a search counting back from now is read again, as flows age in and out of it */ +const AGE_REFRESH_MS = 15_000; /** How often the engine's owner is checked while it is another process */ const OWNER_POLL_MS = 2000; @@ -90,9 +100,26 @@ export const hideSystemAtom = atomWithStorage("sidebar_hide_system", false, unde /** The search the list shows, from the search box's settled text, or null for all traffic */ export const activeQueryAtom = atom((get) => - parseSearch(get(searchTermsAtom), get(hideSystemAtom)), + activeQuery(get(searchTermsAtom), get(hideSystemAtom)), ); +/** The saved searches, in their order */ +export const savedQueriesAtom = atom([]); + +/** How many flows each saved search covers, by id, counted in the database */ +export const savedCountsAtom = atom>({}); + +/** The saved searches as the sidebar lists them */ +const savedRowsAtom = atom((get) => { + const counts = get(savedCountsAtom); + return get(savedQueriesAtom).map((s) => ({ + id: s.id, + name: s.name, + query: s.query, + count: counts[s.id] ?? 0, + })); +}); + /** Writes `text` into the search box at once, as the sidebar does, without waiting for typing */ export const setSearchTextAtom = atom(null, (_get, set, text: string) => { set(searchTextAtom, text); @@ -119,7 +146,7 @@ const SIDEBAR_MS = 1000; /** The sidebar's rows, from the flows loaded */ export const sidebarTreeAtom = atom((get) => - buildSidebarTree(get(sidebarFlowsAtom), get(allTrafficTotalAtom)), + buildSidebarTree(get(sidebarFlowsAtom), get(allTrafficTotalAtom), get(savedRowsAtom)), ); const storedColumnPrefsAtom = atomWithStorage( @@ -244,7 +271,7 @@ export const loadOlderFlowsAtom = atom(null, async (get, set) => { const merge = (win: FlowWindow) => { if (win.flows[0]?.id !== first.id) return win; const merged = prependOlder(win, withoutRemoved(res.flows), PAGE_SIZE); - const kept = search == null ? changes : changesFor(search.query, changes); + const kept = search == null ? changes : changesOf(search, merged, changes); // Counted when they first applied. return { ...applyChanges(merged, kept), total: merged.total }; }; @@ -261,9 +288,47 @@ export const loadOlderFlowsAtom = atom(null, async (get, set) => { } }); -/** The changes a search's window takes: flows it matches, and every deletion */ -function changesFor(query: FlowQuery, changes: ModelPayload[]): ModelPayload[] { - return changes.filter((c) => c.change.type === "delete" || matchesQuery(c.model, query)); +/** `changesFor`, with flows matched against another search sent to be matched again */ +function changesOf( + search: Pick, + win: FlowWindow, + changes: ModelPayload[], +): ModelPayload[] { + return changesFor(search.key, win, changes, (unknown) => matchLater(search, unknown)); +} + +/** The store `syncFlows` keeps, for answers that come after the changes they are for */ +let syncedStore: Store | null = null; + +/** The order flow events arrived in, for judging answers that come late */ +const flowEvents = new FlowEvents(); + +/** + * Matches flows against the search in the database, then has its window take them as it takes + * events matched as they were sent. A flow a newer event updated or removed meanwhile, in the + * window or not, keeps what that event left. + */ +function matchLater(search: Pick, changes: ModelPayload[]) { + const store = syncedStore; + if (store == null) return; + const ids = [...new Set(changes.map((c) => c.model.id))]; + rpc("match_flows", { filter: search.query, ids }) + .then((matched) => { + const covered = new Set(matched); + const marked = flowEvents.current(changes).map((c) => ({ + ...c, + search: { key: search.key, matches: covered.has(c.model.id) }, + })); + if (marked.length === 0) return; + const following = store.get(followingAtom); + store.set(searchWindowAtom, (s) => { + if (s?.key !== search.key) return s; + if (s.loading) return { ...s, buffered: [...s.buffered, ...marked] }; + const next = applyChanges(s.win, changesOf(s, s.win, marked)); + return { ...s, win: following ? trimWindow(next) : next }; + }); + }) + .catch((err) => console.warn("Failed to match flows", err)); } /** @@ -274,6 +339,7 @@ function changesFor(query: FlowQuery, changes: ModelPayload[]): ModelPayload[] { * way. */ export function syncFlows(store: Store) { + syncedStore = store; let pending: ModelPayload[] = []; let timer: ReturnType | null = null; let loaded = false; @@ -297,7 +363,7 @@ export function syncFlows(store: Store) { if (s == null) return s; // A matching flow written after the page's snapshot would otherwise be missed. if (s.loading) return { ...s, buffered: [...s.buffered, ...changes] }; - const next = applyChanges(s.win, changesFor(s.query, changes)); + const next = applyChanges(s.win, changesOf(s, s.win, changes)); return { ...s, win: following ? trimWindow(next) : next }; }); const selected = store.get(selectedFlowAtom); @@ -309,6 +375,7 @@ export function syncFlows(store: Store) { } function receive(changes: ModelPayload[]) { + flowEvents.arrived(changes); pending.push(...changes); if (timer == null) timer = setTimeout(flush, FLUSH_MS); } @@ -326,10 +393,11 @@ export function syncFlows(store: Store) { ); }); if (store.get(hideSystemAtom)) { - void rpc("list_flows", { system: false, limit: 0 }).then((res) => { + void rpc("list_flows", { hideSystem: true, limit: 0 }).then((res) => { if (thisLoad === loads) store.set(nonSystemTotalAtom, res.matching ?? res.total); }); } + countSaved(); const search = store.get(searchWindowAtom); if (search == null || search.loading) return; void rpc("list_flows", { ...search.query, limit: 0 }).then((res) => { @@ -340,21 +408,60 @@ export function syncFlows(store: Store) { }); } + // Saved searches, and their counts, which change with the traffic. + function loadSaved() { + rpc("list_saved_queries", {}) + .then((saved) => { + store.set(savedQueriesAtom, saved); + countSaved(); + }) + .catch((err) => console.warn("Failed to list saved searches", err)); + } + let counting = false; + let countAgain = false; + function countSaved() { + if (store.get(savedQueriesAtom).length === 0) return; + // One count at a time; one asked for meanwhile runs after it, with the latest traffic. + if (counting) { + countAgain = true; + return; + } + counting = true; + rpc("count_saved_queries", { hideSystem: store.get(hideSystemAtom) }) + .then((counts) => + store.set(savedCountsAtom, Object.fromEntries(counts.map((c) => [c.id, c.count]))), + ) + .catch((err) => console.warn("Failed to count saved searches", err)) + .finally(() => { + counting = false; + if (countAgain) { + countAgain = false; + countSaved(); + } + }); + } + listen("saved_queries_changed", loadSaved); + loadSaved(); + listen("model_write", (payload) => { seenId = payload.model.id; receive([payload]); }); // Loads again rather than only emptying, for a flow written between the clear and the event. + // Saved searches too, which could not be read while the database was locked. listen("flows_cleared", () => { + flowEvents.reset(); store.set(emptyFlowsAtom); pending = []; load(); + loadSaved(); }); // The oldest flows, swept to keep the capture under its size limit: dropped wherever they // are held, and the totals counted again from the database. listen("flows_removed", ({ ids }) => { + flowEvents.removed(ids); const removed = new Set(ids); const drop = (win: FlowWindow) => win.flows.some((f) => removed.has(f.id)) @@ -399,21 +506,31 @@ export function syncFlows(store: Store) { } reloadFlows = load; - /** Loads the first page of `query`'s flows into a window of its own, or drops it for null */ - function loadSearch(query: FlowQuery | null) { + /** + * Loads the first page of `query`'s flows into a window of its own, or drops it for null. The + * backend matches the flows written from now on against it, and their events say so. + */ + function loadSearch(query: FlowQuery | null, keepRows = false) { if (query == null) { store.set(searchWindowAtom, null); + void rpc("watch_query", { key: "" }).catch(() => {}); return; } // Unique to this load, so a slower answer to an earlier one with the same query, such as // one read before a clear, is dropped. const key = `${++searchLoads}:${JSON.stringify(query)}`; - store.set(searchWindowAtom, { key, query, win: emptyWindow, loading: true, buffered: [] }); + // Read again, it keeps showing its rows until the new ones are in. + const win = (keepRows ? store.get(searchWindowAtom)?.win : null) ?? emptyWindow; + store.set(searchWindowAtom, { key, query, win, loading: true, buffered: [] }); + void rpc("watch_query", { key, query: query.query, hideSystem: query.hideSystem }).catch( + (err) => console.warn("Failed to watch the search", err), + ); const settle = (page: FlowWindow | null) => store.set(searchWindowAtom, (s) => { if (s?.key !== key) return s; // Applied again over the page: a flow already in it is replaced, not added. - const win = applyChanges(page ?? s.win, changesFor(query, s.buffered)); + const base = page ?? s.win; + const win = applyChanges(base, changesOf(s, base, s.buffered)); return { ...s, win: page == null ? win : { ...win, total: page.total }, @@ -451,11 +568,21 @@ export function syncFlows(store: Store) { } }); - // All traffic's count leaves System out while it is hidden. + // All traffic's count, and the saved searches', leave System out while it is hidden. store.sub(hideSystemAtom, () => { if (loaded) recount(); }); + // A search counting back from now, `@time:<5m`, is read again while it is shown and the list + // follows new traffic, so flows leave it and join it as they age. Saved ones are counted + // again, as their counts change with no traffic at all. + setInterval(() => { + if (anyCountsBackFromNow(store.get(savedQueriesAtom).map((q) => q.query))) countSaved(); + const s = store.get(searchWindowAtom); + if (s == null || s.loading || !store.get(followingAtom)) return; + if (countsBackFromNow(s.query.query)) loadSearch(s.query, true); + }, AGE_REFRESH_MS); + let activeKey = JSON.stringify(store.get(activeQueryAtom)); store.sub(activeQueryAtom, () => { const query = store.get(activeQueryAtom); @@ -576,7 +703,7 @@ export function syncSidebarSelection(store: Store) { lastRows = key; if (syncing || Date.now() - lastSidebarInput > SIDEBAR_INPUT_MS) return; // Group headings are not filters. - const rows = ids.filter((id) => id !== APPS_ID && id !== DOMAINS_ID); + const rows = ids.filter((id) => id !== APPS_ID && id !== DOMAINS_ID && id !== SAVED_ID); if (rows.length === 0) return; const items = itemsById(store.get(sidebarTreeAtom)); const selected = rows.flatMap((id) => { diff --git a/apps/yaak-proxy/package.json b/apps/yaak-proxy/package.json index 0da1b2bc0..7e160c0b9 100644 --- a/apps/yaak-proxy/package.json +++ b/apps/yaak-proxy/package.json @@ -35,6 +35,7 @@ "babel-plugin-react-compiler": "^1.0.0", "typescript": "^5.8.3", "vite": "npm:@voidzero-dev/vite-plus-core@^1.4.0", + "vite-plugin-wasm": "^3.5.0", "vite-plus": "^1.4.0" } } diff --git a/apps/yaak-proxy/vite.config.ts b/apps/yaak-proxy/vite.config.ts index 3d56934f6..73206609c 100644 --- a/apps/yaak-proxy/vite.config.ts +++ b/apps/yaak-proxy/vite.config.ts @@ -1,9 +1,13 @@ import react from "@vitejs/plugin-react"; import { defineConfig } from "vite"; +import wasm from "vite-plugin-wasm"; export default defineConfig({ - plugins: [react()], + // The search box's query language is a WebAssembly package + plugins: [wasm(), react()], build: { + // Top-level await, which the WebAssembly package loads with + target: "esnext", outDir: "../../dist/apps/yaak-proxy", emptyOutDir: true, }, diff --git a/crates-proxy/yaak-proxy-lib/Cargo.toml b/crates-proxy/yaak-proxy-lib/Cargo.toml index bf52d7d8c..b0112ecb5 100644 --- a/crates-proxy/yaak-proxy-lib/Cargo.toml +++ b/crates-proxy/yaak-proxy-lib/Cargo.toml @@ -36,6 +36,8 @@ yaak-lifecycle = { workspace = true } yaak-models = { workspace = true, features = ["sqlcipher"] } yaak-proxy = { workspace = true } yaak-proxy-helper = { workspace = true, features = ["ts"] } +# The search query language: parsing, the evaluator for live flows, completion. +yaak-query = { workspace = true } yaak-rpc = { workspace = true } yaak-templates = { workspace = true } diff --git a/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts b/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts index 7e5b70058..98ca43b83 100644 --- a/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts +++ b/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts @@ -131,7 +131,12 @@ grpcMessage: string | null, * client rejected the proxy's certificate: method `CONNECT`, its URL the host, its sizes * the bytes each way, and no headers or bodies. Open until `elapsedMs` or `error` is set. */ -tunnelled: boolean, }; +tunnelled: boolean, +/** + * An agent read the flow through the control API: its detail, a body or its messages. + * Never written by an upsert, which keeps what is stored. + */ +agentRead: boolean, }; export type FlowBody = "request" | "response"; @@ -239,10 +244,34 @@ grpcStatus?: number, */ grpcMessage?: string, }; -export type ModelPayload = { model: Flow, change: ModelChangeEvent, }; +export type ModelPayload = { model: Flow, change: ModelChangeEvent, +/** + * Whether the flow belongs in the list of the search the app shows, decided as the event + * is sent. None while no search is shown. + */ +search?: SearchMatch, }; export type ProxyHeader = { name: string, value: string, }; +export type SavedQuery = { id: string, name: string, +/** + * The search, as the search box takes it. + */ +query: string, +/** + * Where it sits in the list, lowest first. + */ +sortOrder: number, }; + +/** + * A written flow, matched against the search the app shows. + */ +export type SearchMatch = { +/** + * The search's key, as the app gave it with the search. + */ +key: string, matches: boolean, }; + /** * The streaming protocol a flow carries. */ diff --git a/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts b/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts index 2e70e0589..d94a860b5 100644 --- a/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts +++ b/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts @@ -1,5 +1,5 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { Flow, FlowBody, FlowMessage, GrpcSchema, ModelPayload } from "./gen_models"; +import type { Flow, FlowBody, FlowMessage, GrpcSchema, ModelPayload, SavedQuery } from "./gen_models"; export type ActionInvocation = { "scope": "global", action: GlobalAction, }; @@ -58,6 +58,10 @@ path: string, */ cwd: string | null, }; +export type CountSavedQueriesRequest = { hideSystem?: boolean, }; + +export type CreateSavedQueryRequest = { name: string, query: string, }; + export type DecodeGrpcMessageRequest = { flowId: string, seq: number, /** * Decode without a schema even when one is found. @@ -111,6 +115,8 @@ entry: string, */ builtIn: boolean, }; +export type DeleteSavedQueryRequest = { id: string, }; + export type DropReason = "hop_by_hop" | "set_by_sending" | "body_decoded"; export type DroppedHeader = { name: string, value: string, reason: DropReason, }; @@ -146,60 +152,19 @@ export type FilterBodyResponse = { content: string, error: string | null, }; export type FilterLanguage = "jsonpath" | "xpath"; /** - * Which flows a listing or a HAR export covers. Every field narrows it; none is all of them. + * Which flows a listing or a HAR export covers: a search, and a page of it. */ export type FlowFilter = { /** - * Search terms, any case, each of which must match the URL (host, path or query), the - * method, the status (`404`, `40`, `4xx`) or the content type: `stripe post 4xx`. + * A search in the query language (`crate::query`): `stripe @status:4xx`, + * `@host:*.github.com -@is:system`. None or blank covers every flow. */ -q?: string, +query?: string, /** - * Part of the host, any case: `stripe` matches `api.stripe.com`. + * Leaves out macOS's own daemons, as hiding System does, unless the query says + * `@is:system` itself. */ -host?: string, -/** - * Only flows under one of these hosts or paths, as the app's sidebar names them: - * `api.stripe.com` or `api.stripe.com/v1/charges`, with any port after the host. A host - * written `*.github.com` is a site: `github.com` and every host below it. - */ -under?: Array, -/** - * Only flows from one of these apps: by bundle id, by path for a process outside an app - * bundle, by name in any case (`slack`), or, for a process outside an app, by the folder - * it ran in (`fieldwork`). - */ -app?: Array, -/** - * Only flows from a process that ran in one of these working directories: the whole path, - * or its folder's name in any case (`fieldwork`). With `app`, a program in a folder. - */ -cwd?: Array, -/** - * Only flows from macOS's own daemons (true), or none of them (false), as [`is_system`] - * tells them by their executable. - */ -system?: boolean, -/** - * Only HTTPS connections tunnelled undecrypted (true), or none of them (false). - */ -tunnelled?: boolean, -/** - * Only flows one of these groups covers, each judged on its own: several sidebar rows - * selected, `@app:Slack @host:a.com OR @app:Chrome @host:b.com`, are two groups and not every - * pairing of their apps and hosts. A group's `after`, `before` and `limit` do not apply. Not - * carried by a query string. - */ -any?: Array, -/** - * The method, any case. - */ -method?: string, -/** - * A status (`404`), a class of them (`4xx`), or `error` for flows that failed, gRPC calls - * that ended with a status other than OK among them. - */ -status?: string, +hideSystem?: boolean, /** * At most this many flows, the newest. */ @@ -382,6 +347,8 @@ messages: Array, */ total: number, }; +export type ListSavedQueriesRequest = Record; + export type ListSeenRequest = Record; export type ListWorkspacesRequest = Record; @@ -392,6 +359,11 @@ export type ListWorkspacesResponse = { */ yaakFound: boolean, workspaces: Array, }; +/** + * Which of some flows a search covers, for flows whose events were matched against another. + */ +export type MatchFlowsRequest = { filter: FlowFilter, ids: Array, }; + /** * A message of a streaming flow was recorded. */ @@ -436,6 +408,30 @@ export type ProxyState = "running" | "stopped"; export type ProxyStatePayload = { state: ProxyState, }; +/** + * A term the query cannot run, which then matches nothing. `start` and `end` count UTF-16 + * code units, as JavaScript does. + */ +export type QueryDiagnostic = { kind: QueryDiagnosticKind, path: string, start: number, end: number, }; + +/** + * What is wrong with a term, for the search box to mark. + */ +export type QueryDiagnosticKind = "unknownField" | "invalidValue" | "notYet"; + +export type QuerySchemaRequest = Record; + +/** + * The search's fields, for the search box's highlighting and completion. + */ +export type QuerySchemaResponse = { schema: import("@yaakapp-internal/query").Schema, }; + +export type QueryValuesRequest = { +/** + * A field's path: `host`, `header` for header names, `header.accept` for a header's values. + */ +path: string, }; + export type ReflectGrpcHostRequest = { /** * `host` or `host:port`, over TLS on 443 unless a port is given, or either after @@ -445,11 +441,13 @@ host: string, }; export type RemoveGrpcSchemaRequest = { id: string, }; +export type RenameSavedQueryRequest = { id: string, name: string, }; + export type ResetCapturedDataRequest = Record; -export type RpcEventSchema = { decrypt_list_changed: DecryptListChangedPayload, flows_cleared: FlowsClearedPayload, flows_removed: FlowsRemovedPayload, message_write: MessagePayload, model_write: ModelPayload, proxy_state_changed: ProxyStatePayload, }; +export type RpcEventSchema = { decrypt_list_changed: DecryptListChangedPayload, flows_cleared: FlowsClearedPayload, flows_removed: FlowsRemovedPayload, message_write: MessagePayload, model_write: ModelPayload, proxy_state_changed: ProxyStatePayload, saved_queries_changed: SavedQueriesChangedPayload, }; -export type RpcSchema = { execute_action: [ActionInvocation, boolean], format_body: [FormatBodyRequest, string], get_decrypt_list: [GetDecryptListRequest, DecryptList], get_message_payload: [GetMessagePayloadRequest, GetMessagePayloadResponse], list_actions: [ListActionsRequest, ListActionsResponse], list_flows: [FlowFilter, ListFlowsResponse], list_grpc_schemas: [ListGrpcSchemasRequest, Array], list_messages: [ListMessagesRequest, ListMessagesResponse], remove_grpc_schema: [RemoveGrpcSchemaRequest, boolean], set_recording: [SetRecordingRequest, boolean], set_system_proxy_at_launch: [SetSystemProxyAtLaunchRequest, boolean], set_http_version: [SetHttpVersionRequest, HttpVersion], set_capture_body_limit: [SetCaptureBodyLimitRequest, number], set_keep_traffic: [SetKeepTrafficRequest, number | null], add_grpc_schema: [AddGrpcSchemaRequest, GrpcSchema], app_icon: [AppIconRequest, string | null], ca_status: [CaStatusRequest, CaStatus], ca_trust: [CaTrustRequest, CaStatus], clear_flows: [ClearFlowsRequest, number], decode_grpc_message: [DecodeGrpcMessageRequest, DecodedGrpcMessage], export_har: [ExportHarRequest, number], filter_body: [FilterBodyRequest, FilterBodyResponse], get_flow_body: [GetFlowBodyRequest, GetFlowBodyResponse], get_proxy_state: [GetProxyStateRequest, GetProxyStateResponse], helper_register: [HelperRegisterRequest, SystemProxyStatus], helper_unregister: [HelperUnregisterRequest, SystemProxyStatus], edit_decrypt_list: [EditDecryptListRequest, DecryptList], list_seen: [ListSeenRequest, SeenTargets], set_decrypt_list: [SetDecryptListRequest, DecryptList], list_workspaces: [ListWorkspacesRequest, ListWorkspacesResponse], preview_save: [PreviewSaveRequest, SavePreview], reflect_grpc_host: [ReflectGrpcHostRequest, GrpcSchema], save_flow: [SaveFlowToYaakRequest, SavedFlow], save_flow_body: [SaveFlowBodyRequest, SaveFlowBodyResponse], system_proxy_set: [SystemProxySetRequest, SystemProxyStatus], system_proxy_status: [SystemProxyStatusRequest, SystemProxyStatus], take_over: [TakeOverRequest, boolean], set_port: [SetPortRequest, number], get_storage_size: [GetStorageSizeRequest, GetStorageSizeResponse], reset_captured_data: [ResetCapturedDataRequest, boolean], open_terminal: [OpenTerminalRequest, boolean], }; +export type RpcSchema = { execute_action: [ActionInvocation, boolean], format_body: [FormatBodyRequest, string], get_decrypt_list: [GetDecryptListRequest, DecryptList], get_message_payload: [GetMessagePayloadRequest, GetMessagePayloadResponse], list_actions: [ListActionsRequest, ListActionsResponse], list_flows: [FlowFilter, ListFlowsResponse], query_schema: [QuerySchemaRequest, QuerySchemaResponse], validate_query: [ValidateQueryRequest, Array], watch_query: [WatchQueryRequest, boolean], list_saved_queries: [ListSavedQueriesRequest, Array], create_saved_query: [CreateSavedQueryRequest, SavedQuery], rename_saved_query: [RenameSavedQueryRequest, SavedQuery], delete_saved_query: [DeleteSavedQueryRequest, boolean], list_grpc_schemas: [ListGrpcSchemasRequest, Array], list_messages: [ListMessagesRequest, ListMessagesResponse], remove_grpc_schema: [RemoveGrpcSchemaRequest, boolean], set_recording: [SetRecordingRequest, boolean], set_system_proxy_at_launch: [SetSystemProxyAtLaunchRequest, boolean], set_http_version: [SetHttpVersionRequest, HttpVersion], set_capture_body_limit: [SetCaptureBodyLimitRequest, number], set_keep_traffic: [SetKeepTrafficRequest, number | null], add_grpc_schema: [AddGrpcSchemaRequest, GrpcSchema], app_icon: [AppIconRequest, string | null], ca_status: [CaStatusRequest, CaStatus], ca_trust: [CaTrustRequest, CaStatus], clear_flows: [ClearFlowsRequest, number], decode_grpc_message: [DecodeGrpcMessageRequest, DecodedGrpcMessage], export_har: [ExportHarRequest, number], filter_body: [FilterBodyRequest, FilterBodyResponse], get_flow_body: [GetFlowBodyRequest, GetFlowBodyResponse], get_proxy_state: [GetProxyStateRequest, GetProxyStateResponse], helper_register: [HelperRegisterRequest, SystemProxyStatus], helper_unregister: [HelperUnregisterRequest, SystemProxyStatus], edit_decrypt_list: [EditDecryptListRequest, DecryptList], list_seen: [ListSeenRequest, SeenTargets], set_decrypt_list: [SetDecryptListRequest, DecryptList], list_workspaces: [ListWorkspacesRequest, ListWorkspacesResponse], preview_save: [PreviewSaveRequest, SavePreview], reflect_grpc_host: [ReflectGrpcHostRequest, GrpcSchema], save_flow: [SaveFlowToYaakRequest, SavedFlow], save_flow_body: [SaveFlowBodyRequest, SaveFlowBodyResponse], system_proxy_set: [SystemProxySetRequest, SystemProxyStatus], system_proxy_status: [SystemProxyStatusRequest, SystemProxyStatus], take_over: [TakeOverRequest, boolean], set_port: [SetPortRequest, number], get_storage_size: [GetStorageSizeRequest, GetStorageSizeResponse], reset_captured_data: [ResetCapturedDataRequest, boolean], open_terminal: [OpenTerminalRequest, boolean], query_values: [QueryValuesRequest, Array], match_flows: [MatchFlowsRequest, Array], count_saved_queries: [CountSavedQueriesRequest, Array], }; export type SaveFlowBodyRequest = { flowId: string, which: FlowBody, /** @@ -513,6 +511,13 @@ authorization: boolean, }; export type SavedFlow = { requestId: string, name: string, }; +export type SavedQueriesChangedPayload = Record; + +/** + * How many flows a saved query covers. + */ +export type SavedQueryCount = { id: string, count: number, }; + export type SeenApp = { name: string, bundleId: string | null, }; /** @@ -591,4 +596,12 @@ export type SystemProxyStatusRequest = Record; export type TakeOverRequest = Record; +export type ValidateQueryRequest = { query: string, }; + +/** + * The search the app shows, which flows written from now on are matched against, their events + * naming `key`. No query stops it. + */ +export type WatchQueryRequest = { key: string, query?: string, hideSystem?: boolean, }; + export type WorkspaceSummary = { id: string, name: string, folders: Array, }; diff --git a/crates-proxy/yaak-proxy-lib/migrations/00000012_search.sql b/crates-proxy/yaak-proxy-lib/migrations/00000012_search.sql new file mode 100644 index 000000000..939b111ae --- /dev/null +++ b/crates-proxy/yaak-proxy-lib/migrations/00000012_search.sql @@ -0,0 +1,13 @@ +-- An agent read the flow through the control API: its detail, a body or its messages. +ALTER TABLE flows ADD COLUMN agent_read BOOLEAN NOT NULL DEFAULT FALSE; + +-- Searches saved by name, listed in the sidebar by `sort_order`. +CREATE TABLE saved_queries +( + id TEXT NOT NULL PRIMARY KEY, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL, + name TEXT NOT NULL, + query TEXT NOT NULL, + sort_order INTEGER NOT NULL DEFAULT 0 +); diff --git a/crates-proxy/yaak-proxy-lib/src/control.rs b/crates-proxy/yaak-proxy-lib/src/control.rs index b6bdb2e23..61b262eb2 100644 --- a/crates-proxy/yaak-proxy-lib/src/control.rs +++ b/crates-proxy/yaak-proxy-lib/src/control.rs @@ -554,7 +554,9 @@ async fn route(state: &ServerState, req: Request) -> Result { found(ctx, id)?; let limit = body_limit(&query)?; - json(&flows::detail(&ctx.db()?, &ctx.blobs()?, id, limit).await?) + let detail = flows::detail(&ctx.db()?, &ctx.blobs()?, id, limit).await?; + ctx.note_agent_read(id); + json(&detail) } (&Method::GET, ["flows", id, "request-body"]) => raw_body(ctx, id, FlowBody::Request).await, (&Method::GET, ["flows", id, "response-body"]) => { @@ -565,6 +567,7 @@ async fn route(state: &ServerState, req: Request) -> Result { @@ -584,6 +587,7 @@ async fn route(state: &ServerState, req: Request) -> Result json(&grpc::list(&ctx.db()?)?), @@ -687,6 +691,7 @@ async fn raw_body(ctx: &ProxyCtx, id: &str, which: FlowBody) -> Result bool { reason.starts_with(KEY_UNAVAILABLE) } -/// The functions filters use: `sidebar_path(url)`, the app sidebar's place for a URL, and -/// `under_site(url, site)`, for filtering by its nodes; `folder_name(dir)`, for naming a program -/// by its folder; and `is_system(bundle_id, path)`, for macOS's own daemons. +/// The functions searches use: `folder_name(dir)`, for naming a program by its folder; +/// `is_system(bundle_id, path)`, for macOS's own daemons; and those the query compiler reads +/// flows' values and tests them with ([`crate::query::add_functions`]). fn add_functions(conn: &mut rusqlite::Connection) -> rusqlite::Result<()> { let flags = FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC; - conn.create_scalar_function("sidebar_path", 1, flags, |ctx| { - Ok(crate::flows::sidebar_path(&ctx.get::(0)?)) - })?; - conn.create_scalar_function("under_site", 2, flags, |ctx| { - Ok(crate::flows::under_site(&ctx.get::(0)?, &ctx.get::(1)?)) - })?; conn.create_scalar_function("folder_name", 1, flags, |ctx| { let dir = ctx.get::>(0)?; Ok(crate::flows::folder_name(dir.as_deref()).map(str::to_string)) @@ -339,7 +333,8 @@ fn add_functions(conn: &mut rusqlite::Connection) -> rusqlite::Result<()> { let bundle_id = ctx.get::>(0)?; let path = ctx.get::>(1)?; Ok(crate::flows::is_system(bundle_id.as_deref(), path.as_deref())) - }) + })?; + crate::query::add_functions(conn) } fn read_only_pool(path: &Path, key: Option<&DatabaseKey>) -> Result { @@ -447,8 +442,7 @@ mod tests { assert!(!flows[0].cancelled); assert_eq!((flows[0].stream_kind, flows[0].message_count), (None, 0)); // Failed flows are found without the gRPC status column too. - let failed = - crate::flows::FlowFilter { status: Some("error".into()), ..Default::default() }; + let failed = crate::flows::FlowFilter::search("@is:error"); assert_eq!(crate::flows::list(&db, &failed).unwrap().len(), 0); // Filters by app work without the folder column, and count without the clear mark. @@ -465,8 +459,7 @@ mod tests { .unwrap(); drop(conn); let db = ProxyQueryManager::open_read_only(&path, Some(&test_key())).unwrap(); - let filter = - crate::flows::FlowFilter { app: Some(vec!["curl".into()]), ..Default::default() }; + let filter = crate::flows::FlowFilter::search("@app:curl"); let (page, total, matching) = crate::flows::page(&db, &filter).unwrap(); assert_eq!((page.len(), total, matching), (1, 1, Some(1))); assert_eq!(page[0].number, 1); @@ -513,7 +506,8 @@ mod tests { "flow_messages", "flow_sweep_pending", "flows", - "grpc_schemas" + "grpc_schemas", + "saved_queries" ] ); @@ -532,6 +526,7 @@ mod tests { "grpc_method", "grpc_status", "grpc_message", + "agent_read", ] { assert!(flow_columns.contains(&column.to_string()), "{column} in {flow_columns:?}"); } @@ -599,7 +594,12 @@ mod tests { .unwrap() .map(|v| v.unwrap()) .collect(); - assert_eq!(&applied[6..], ["00000007", "00000008", "00000009", "00000010", "00000011"]); + assert_eq!( + &applied[6..], + [ + "00000007", "00000008", "00000009", "00000010", "00000011", "00000012" + ] + ); let mark_columns = columns(&conn, "flow_clear_mark"); assert_eq!(mark_columns, ["id", "max_rowid", "cleaner", "cleaner_at"]); drop(conn); @@ -718,7 +718,7 @@ mod tests { }; db.with_tx(|c| c.upsert(&tunnel, &yaak_database::UpdateSource::Background).map(|_| ())) .unwrap(); - let filter = crate::flows::FlowFilter { tunnelled: Some(true), ..Default::default() }; + let filter = crate::flows::FlowFilter::search("@is:tunnelled"); let listed = crate::flows::list(&db, &filter).unwrap(); assert_eq!(listed.iter().map(|f| f.id.as_str()).collect::>(), ["fl_tunnel"]); let old = crate::flows::find(&db, "fl_old").unwrap(); diff --git a/crates-proxy/yaak-proxy-lib/src/flows.rs b/crates-proxy/yaak-proxy-lib/src/flows.rs index 619fddb66..27876ee6c 100644 --- a/crates-proxy/yaak-proxy-lib/src/flows.rs +++ b/crates-proxy/yaak-proxy-lib/src/flows.rs @@ -1,6 +1,7 @@ use crate::bodies; use crate::db::{ProxyQueryManager, Stores}; use crate::models::{Flow, FlowBody}; +use crate::query::{FlowColumns, FlowQuery}; use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64; use rusqlite::types::Value; @@ -9,52 +10,19 @@ use ts_rs::TS; use yaak_database::UpsertModelInfo; use yaak_models::blob_manager::BlobManager; -/// Which flows a listing or a HAR export covers. Every field narrows it; none is all of them. +/// Which flows a listing or a HAR export covers: a search, and a page of it. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, TS)] #[serde(default, rename_all = "camelCase")] #[ts(export, export_to = "gen_rpc.ts")] pub struct FlowFilter { - /// Search terms, any case, each of which must match the URL (host, path or query), the - /// method, the status (`404`, `40`, `4xx`) or the content type: `stripe post 4xx`. + /// A search in the query language (`crate::query`): `stripe @status:4xx`, + /// `@host:*.github.com -@is:system`. None or blank covers every flow. #[ts(optional)] - pub q: Option, - /// Part of the host, any case: `stripe` matches `api.stripe.com`. - #[ts(optional)] - pub host: Option, - /// Only flows under one of these hosts or paths, as the app's sidebar names them: - /// `api.stripe.com` or `api.stripe.com/v1/charges`, with any port after the host. A host - /// written `*.github.com` is a site: `github.com` and every host below it. - #[ts(optional)] - pub under: Option>, - /// Only flows from one of these apps: by bundle id, by path for a process outside an app - /// bundle, by name in any case (`slack`), or, for a process outside an app, by the folder - /// it ran in (`fieldwork`). - #[ts(optional)] - pub app: Option>, - /// Only flows from a process that ran in one of these working directories: the whole path, - /// or its folder's name in any case (`fieldwork`). With `app`, a program in a folder. - #[ts(optional)] - pub cwd: Option>, - /// Only flows from macOS's own daemons (true), or none of them (false), as [`is_system`] - /// tells them by their executable. - #[ts(optional)] - pub system: Option, - /// Only HTTPS connections tunnelled undecrypted (true), or none of them (false). - #[ts(optional)] - pub tunnelled: Option, - /// Only flows one of these groups covers, each judged on its own: several sidebar rows - /// selected, `@app:Slack @host:a.com OR @app:Chrome @host:b.com`, are two groups and not every - /// pairing of their apps and hosts. A group's `after`, `before` and `limit` do not apply. Not - /// carried by a query string. - #[ts(optional)] - pub any: Option>, - /// The method, any case. - #[ts(optional)] - pub method: Option, - /// A status (`404`), a class of them (`4xx`), or `error` for flows that failed, gRPC calls - /// that ended with a status other than OK among them. - #[ts(optional)] - pub status: Option, + pub query: Option, + /// Leaves out macOS's own daemons, as hiding System does, unless the query says + /// `@is:system` itself. + #[ts(optional, as = "Option")] + pub hide_system: bool, /// At most this many flows, the newest. #[ts(optional, type = "number")] pub limit: Option, @@ -69,30 +37,24 @@ pub struct FlowFilter { } impl FlowFilter { - /// The filter from a query string's `q`, `host`, `under`, `app`, `cwd`, `system`, `method`, - /// `status`, `limit`, `after` and `before`. `under`, `app` and `cwd` may repeat. + /// A filter for a search alone. + pub fn search(query: impl Into) -> Self { + Self { query: Some(query.into()), ..Self::default() } + } + + /// The filter from a query string's `query`, `hide_system`, `limit`, `after` and `before`. pub fn from_query(query: &str) -> Result { let mut filter = Self::default(); for (key, value) in url::form_urlencoded::parse(query.as_bytes()) { let value = Some(value.into_owned()).filter(|v| !v.is_empty()); match key.as_ref() { - "q" => filter.q = value, - "host" => filter.host = value, - "under" => filter.under.get_or_insert_default().extend(value), - "app" => filter.app.get_or_insert_default().extend(value), - "cwd" => filter.cwd.get_or_insert_default().extend(value), - "system" => { - filter.system = value - .map(|v| v.parse().map_err(|_| format!("Invalid system {v:?}"))) + "query" => filter.query = value, + "hide_system" => { + filter.hide_system = value + .map(|v| v.parse().map_err(|_| format!("Invalid hide_system {v:?}"))) .transpose()? + .unwrap_or_default() } - "tunnelled" => { - filter.tunnelled = value - .map(|v| v.parse().map_err(|_| format!("Invalid tunnelled {v:?}"))) - .transpose()? - } - "method" => filter.method = value, - "status" => filter.status = value, "after" => filter.after = value, "before" => filter.before = value, "limit" => { @@ -109,30 +71,10 @@ impl FlowFilter { pub fn to_query(&self) -> String { let mut query = url::form_urlencoded::Serializer::new(String::new()); let limit = self.limit.map(|l| l.to_string()); - let system = self.system.map(|s| s.to_string()); - let tunnelled = self.tunnelled.map(|t| t.to_string()); + let hide_system = self.hide_system.then(|| "true".to_string()); for (key, value) in [ - ("q", &self.q), - ("host", &self.host), - ("system", &system), - ("tunnelled", &tunnelled), - ] { - if let Some(value) = value { - query.append_pair(key, value); - } - } - for (key, values) in [ - ("under", &self.under), - ("app", &self.app), - ("cwd", &self.cwd), - ] { - for value in values.iter().flatten() { - query.append_pair(key, value); - } - } - for (key, value) in [ - ("method", &self.method), - ("status", &self.status), + ("query", &self.query), + ("hide_system", &hide_system), ("limit", &limit), ("after", &self.after), ("before", &self.before), @@ -144,166 +86,21 @@ impl FlowFilter { query.finish() } + /// The search, prepared to run now. + pub fn flow_query(&self) -> FlowQuery { + FlowQuery::new(self.query.as_deref().unwrap_or_default(), self.hide_system) + } + /// Narrows more than `after`, `before` and `limit` do, so it covers fewer than all flows. pub fn narrows(&self) -> bool { - let given = |v: &Option| v.as_deref().is_some_and(|v| !v.trim().is_empty()); - given(&self.q) - || given(&self.host) - || given(&self.method) - || given(&self.status) - || self.under.is_some() - || self.app.is_some() - || self.cwd.is_some() - || self.system.is_some() - || self.tunnelled.is_some() - || self.any.as_ref().is_some_and(|groups| groups.iter().any(FlowFilter::narrows)) + !self.flow_query().is_empty() } - /// The SQL condition for every field but `after`, `before` and `limit`, and its parameters, - /// for the database `conn` reads, which may be from before the folder or gRPC columns, when - /// read without migrating. + /// The SQL condition for the search, and its parameters, for the database `conn` reads, + /// which may be from before later columns, when read without migrating. fn condition_in(&self, conn: &rusqlite::Connection) -> (String, Vec) { - let columns = Columns { - cwd: has_column(conn, "client_cwd"), - grpc: has_column(conn, "grpc_status"), - tunnelled: has_column(conn, "tunnelled"), - }; - self.condition(columns) + self.flow_query().sql(&FlowColumns::of(conn)) } - - /// [`Self::condition_in`], told which of the later columns the database has. - fn condition(&self, columns: Columns) -> (String, Vec) { - let Columns { cwd: has_cwd, grpc, tunnelled } = columns; - let mut sql = String::from("1"); - let mut params: Vec = Vec::new(); - for term in self.q.iter().flat_map(|q| q.split_whitespace()) { - let term = term.to_lowercase(); - sql.push_str( - " AND (instr(lower(url), ?) > 0 OR lower(method) = ? \ - OR instr(lower(content_type), ?) > 0", - ); - params.extend([ - term.clone().into(), - term.clone().into(), - term.clone().into(), - ]); - if let Some(status) = status_condition(&term, &mut params) { - sql.push_str(" OR "); - sql.push_str(&status); - } - sql.push(')'); - } - if let Some(host) = self.host.as_deref().filter(|h| !h.is_empty()) { - sql.push_str(&format!(" AND instr(lower({HOST_SQL}), ?) > 0")); - params.push(host.to_lowercase().into()); - } - if let Some(under) = &self.under { - // The node itself, or anything below it, by the sidebar's rules. - let mut any = Vec::new(); - for node in under { - if let Some(site) = site_of_node(node) { - any.push("under_site(url, ?)"); - params.push(site.into()); - continue; - } - let node = normalize_node(node); - any.push("(sidebar_path(url) = ? OR substr(sidebar_path(url), 1, ?) = ?)"); - let below = format!("{node}/"); - params.extend([ - node.into(), - (below.chars().count() as i64).into(), - below.into(), - ]); - } - let any = if any.is_empty() { "0".to_string() } else { any.join(" OR ") }; - sql.push_str(&format!(" AND ({any})")); - } - if let Some(apps) = &self.app { - let marks = vec!["?"; apps.len()].join(", "); - sql.push_str(&format!( - " AND (coalesce(client_bundle_id, client_path) IN ({marks}) \ - OR lower(client_name) IN ({marks})" - )); - params.extend(apps.iter().map(|a| a.clone().into())); - params.extend(apps.iter().map(|a| a.to_lowercase().into())); - if has_cwd { - sql.push_str(&format!( - " OR (client_bundle_id IS NULL AND lower(folder_name(client_cwd)) IN ({marks}))" - )); - params.extend(apps.iter().map(|a| a.to_lowercase().into())); - } - sql.push(')'); - } - if let Some(cwds) = &self.cwd { - if has_cwd && !cwds.is_empty() { - let marks = vec!["?"; cwds.len()].join(", "); - sql.push_str(&format!( - " AND (client_cwd IN ({marks}) OR lower(folder_name(client_cwd)) IN ({marks}))" - )); - params.extend(cwds.iter().map(|c| c.clone().into())); - params.extend(cwds.iter().map(|c| c.to_lowercase().into())); - } else { - // None, or a database from before the column, which has no folders. - sql.push_str(" AND 0"); - } - } - match self.system { - Some(true) => sql.push_str(" AND is_system(client_bundle_id, client_path)"), - Some(false) => sql.push_str(" AND NOT is_system(client_bundle_id, client_path)"), - None => {} - } - match (self.tunnelled, tunnelled) { - (Some(true), true) => sql.push_str(" AND tunnelled"), - (Some(false), true) => sql.push_str(" AND NOT tunnelled"), - // A database from before tunnelled flows has none. - (Some(true), false) => sql.push_str(" AND 0"), - _ => {} - } - // Each group whole, then any of them. - if let Some(groups) = self.any.as_ref().filter(|g| !g.is_empty()) { - let mut any = Vec::new(); - for group in groups { - let (condition, group_params) = group.condition(columns); - any.push(format!("({condition})")); - params.extend(group_params); - } - sql.push_str(&format!(" AND ({})", any.join(" OR "))); - } - if let Some(method) = self.method.as_deref().filter(|m| !m.is_empty()) { - sql.push_str(" AND lower(method) = ?"); - params.push(method.to_lowercase().into()); - } - match self.status.as_deref().map(str::trim) { - None | Some("") => {} - // A gRPC call that ended with a status other than OK failed too. - Some(s) if s.eq_ignore_ascii_case("error") && grpc => { - sql.push_str(" AND (error IS NOT NULL OR coalesce(grpc_status, 0) != 0)") - } - Some(s) if s.eq_ignore_ascii_case("error") => sql.push_str(" AND error IS NOT NULL"), - // A whole status or a class, not the start of one as a search term may be. - Some(s) if s.len() == 3 => match status_condition(&s.to_ascii_lowercase(), &mut params) - { - Some(status) => { - sql.push_str(" AND "); - sql.push_str(&status); - } - None => sql.push_str(" AND 0"), - }, - Some(_) => sql.push_str(" AND 0"), - } - (sql, params) - } -} - -/// Which of the columns later migrations added a database has, as one opened read-only before -/// they ran may not. -#[derive(Clone, Copy)] -struct Columns { - /// The folder a program ran in. - cwd: bool, - /// gRPC statuses. - grpc: bool, - tunnelled: bool, } /// The host and port of `url`, in SQL: what follows `://` up to the first `/`, `?` or `#`. @@ -341,38 +138,6 @@ pub fn sidebar_path(url: &str) -> String { key } -/// A sidebar node as `sidebar_path` writes it, its host lowercased. -fn normalize_node(node: &str) -> String { - match node.split_once('/') { - Some((host, path)) => format!("{}/{path}", host.to_ascii_lowercase()), - None => node.to_ascii_lowercase(), - } -} - -/// The site a sidebar node names, `github.com` for `*.github.com`, lowercased. None for a host or -/// a path. -fn site_of_node(node: &str) -> Option { - let site = node.strip_prefix("*.")?; - (!site.is_empty() && !site.contains('/')).then(|| site.to_ascii_lowercase()) -} - -/// `url`'s host, without its port, is `site` or a host below it: `api.github.com` is under -/// `github.com`. -pub fn under_site(url: &str, site: &str) -> bool { - let path = sidebar_path(url); - let host = path.split('/').next().unwrap_or_default(); - let host = strip_port(host); - host == site || host.strip_suffix(site).is_some_and(|rest| rest.ends_with('.')) -} - -/// `host` without a `:port` after it, keeping an IPv6 address's brackets. -fn strip_port(host: &str) -> &str { - if host.starts_with('[') { - return host.find(']').map_or(host, |end| &host[..=end]); - } - host.rsplit_once(':').map_or(host, |(h, _)| h) -} - /// The last part of a directory's path, `fieldwork` for `/Users/me/src/fieldwork`. None for the /// root, where daemons run, or no path. pub fn folder_name(path: Option<&str>) -> Option<&str> { @@ -418,26 +183,6 @@ pub fn is_system(bundle_id: Option<&str>, path: Option<&str>) -> bool { SYSTEM_DIRS.iter().any(|dir| path.starts_with(dir)) || SYSTEM_PROGRAMS.contains(&name) } -/// The SQL condition for a status term: `404` exactly, `40` as the start of one, or a class -/// such as `4xx`. None when the term is not one. -fn status_condition(term: &str, params: &mut Vec) -> Option { - if let Some(class) = term.strip_suffix("xx") - && let Ok(class @ 1..=9) = class.parse::() - { - params.push(class.into()); - return Some("status / 100 = ?".into()); - } - if term.is_empty() || term.len() > 3 || !term.chars().all(|c| c.is_ascii_digit()) { - return None; - } - if term.len() == 3 { - params.push(term.parse::().ok()?.into()); - return Some("status = ?".into()); - } - params.push(format!("{term}%").into()); - Some("CAST(status AS TEXT) LIKE ?".into()) -} - /// The flows `filter` covers, newest first: by start, then by id. Reads the table in that order /// through its index and stops at the limit, so a page of a common match costs the same however /// many flows there are. @@ -462,13 +207,6 @@ pub fn page( }) } -/// The database has the gRPC status column, added by a later migration. -/// The flows table has `column`: a database opened read-only before its migrations have run may -/// not. -fn has_column(conn: &rusqlite::Connection, column: &str) -> bool { - conn.prepare(&format!("SELECT {column} FROM flows LIMIT 0")).is_ok() -} - fn list_in(conn: &rusqlite::Connection, filter: &FlowFilter) -> Result, String> { let (condition, mut params) = filter.condition_in(conn); // The rowid, for the flow's number. @@ -511,6 +249,40 @@ fn count_matching(conn: &rusqlite::Connection, filter: &FlowFilter) -> Result Result, String> { + db.with_conn(|c| { + let tx = c.conn().resolve().unchecked_transaction().map_err(|e| e.to_string())?; + let counts = filters.iter().map(|f| count_matching(&tx, f)).collect::>()?; + tx.commit().map_err(|e| e.to_string())?; + Ok(counts) + }) +} + +/// The flows of `ids` that `filter`'s search covers, in any order. Cleared ones are left out. +pub fn matching_ids( + db: &ProxyQueryManager, + filter: &FlowFilter, + ids: &[String], +) -> Result, String> { + if ids.is_empty() { + return Ok(Vec::new()); + } + db.with_conn(|c| { + let conn = c.conn().resolve(); + let (condition, mut params) = filter.condition_in(conn); + let marks = vec!["?"; ids.len()].join(", "); + let mut sql = format!("SELECT id FROM flows WHERE ({condition}) AND id IN ({marks})"); + params.extend(ids.iter().map(|id| Value::from(id.clone()))); + push_clear_mark(conn, &mut sql, &mut params); + let mut stmt = conn.prepare(&sql).map_err(|e| e.to_string())?; + let rows = stmt + .query_map(rusqlite::params_from_iter(params), |r| r.get(0)) + .map_err(|e| e.to_string())?; + rows.collect::, _>>().map_err(|e| e.to_string()) + }) +} + /// Leaves out cleared flows, when a clear is pending; otherwise adds nothing, so SQLite keeps its /// cheapest plan. fn push_clear_mark(conn: &rusqlite::Connection, sql: &mut String, params: &mut Vec) { @@ -1031,6 +803,13 @@ mod tests { list(db, &filter).unwrap().into_iter().map(|f| f.url).collect() } + /// The urls a search covers, sorted. + fn found(db: &ProxyQueryManager, query: &str) -> Vec { + let mut found = urls(db, FlowFilter::search(query)); + found.sort(); + found + } + #[test] fn filters() { let (_dir, db) = db_of(vec![ @@ -1038,18 +817,20 @@ mod tests { flow("http://example.com/missing", "GET", Some(404), None), flow("https://down.example.com/", "GET", None, Some("refused")), ]); - let by = |filter: FlowFilter| urls(&db, filter).len(); - assert_eq!(by(FlowFilter::default()), 3); - assert_eq!(by(FlowFilter { host: Some("STRIPE".into()), ..Default::default() }), 1); - assert_eq!(by(FlowFilter { host: Some("example.com".into()), ..Default::default() }), 2); - assert_eq!(by(FlowFilter { host: Some("charges".into()), ..Default::default() }), 0); - assert_eq!(by(FlowFilter { host: Some("8443".into()), ..Default::default() }), 1); - assert_eq!(by(FlowFilter { method: Some("get".into()), ..Default::default() }), 2); - assert_eq!(by(FlowFilter { status: Some("404".into()), ..Default::default() }), 1); - assert_eq!(by(FlowFilter { status: Some("2xx".into()), ..Default::default() }), 1); - assert_eq!(by(FlowFilter { status: Some("5XX".into()), ..Default::default() }), 0); - assert_eq!(by(FlowFilter { status: Some("40".into()), ..Default::default() }), 0); - assert_eq!(by(FlowFilter { status: Some("error".into()), ..Default::default() }), 1); + let by = |query: &str| urls(&db, FlowFilter::search(query)).len(); + assert_eq!(urls(&db, FlowFilter::default()).len(), 3); + // A host whole, with or without its port; a part of one with wildcards. + assert_eq!(by("@host:API.STRIPE.COM"), 1); + assert_eq!(by("@host:api.stripe.com:8443"), 1); + assert_eq!(by("@host:stripe"), 0); + assert_eq!(by("@host:*stripe*"), 1); + assert_eq!(by("@host:*.example.com"), 2); + assert_eq!(by("@method:get"), 2); + assert_eq!(by("@status:404"), 1); + assert_eq!(by("@status:2xx"), 1); + assert_eq!(by("@status:5XX"), 0); + assert_eq!(by("@status:40"), 0); + assert_eq!(by("@is:error"), 1); } #[test] @@ -1067,12 +848,7 @@ mod tests { }, flow("http://localhost:3000/health", "HEAD", Some(503), None), ]); - let search = |q: &str| { - let filter = FlowFilter { q: Some(q.into()), ..Default::default() }; - let mut found = urls(&db, filter); - found.sort(); - found - }; + let search = |q: &str| found(&db, q); let stripe = "https://api.stripe.com/v1/charges?customer=cus_9"; let github = "https://api.github.com/repos/yaak"; let logo = "https://images.ctfassets.net/logo.png"; @@ -1081,28 +857,28 @@ mod tests { assert_eq!(search("STRIPE"), vec![stripe]); assert_eq!(search("/repos/"), vec![github]); assert_eq!(search("cus_9"), vec![stripe]); - // The method, whole. + // The method and the status, any part of them. assert_eq!(search("head"), vec![health]); - assert_eq!(search("ead"), Vec::<&str>::new()); - // A status, its start, or its class. + assert_eq!(search("ead"), vec![health]); assert_eq!(search("404"), vec![github]); assert_eq!(search("50"), vec![health]); - assert_eq!(search("2xx"), vec![stripe, logo]); // The content type. assert_eq!(search("png"), vec![logo]); assert_eq!(search("json"), vec![github, stripe]); // Every term must match, each anywhere. assert_eq!(search("json get"), vec![github]); - assert_eq!(search(" json 2xx "), vec![stripe]); - assert_eq!(search("json 5xx"), Vec::<&str>::new()); + assert_eq!(search(" json @status:2xx "), vec![stripe]); + assert_eq!(search("json @status:5xx"), Vec::<&str>::new()); // `%` and `_` are plain characters, not patterns. assert_eq!(search("%"), Vec::<&str>::new()); assert_eq!(search("cus_"), vec![stripe]); assert_eq!(search(""), vec![health, github, stripe, logo]); } + /// The terms a sidebar row writes cover what is under it: a host, with any port, and a + /// path with what is below it. #[test] - fn under_matches_a_host_or_a_path_as_the_sidebar_does() { + fn hosts_and_paths_as_the_sidebar_writes_them() { let (_dir, db) = db_of(vec![ flow("https://api.stripe.com/v1/charges", "GET", Some(200), None), flow("https://api.stripe.com/v1/charges/ch_1?expand=x", "GET", Some(200), None), @@ -1110,29 +886,38 @@ mod tests { flow("https://api.stripe.com:8443/v1/charges", "GET", Some(200), None), flow("http://api.stripe.com", "GET", Some(200), None), flow("https://api.stripe.community/", "GET", Some(200), None), - flow("https://example.com/v1//users", "GET", Some(200), None), flow("http://x.com:80/a", "GET", Some(200), None), flow("https://x.com:443/a", "GET", Some(200), None), + flow("https://example.com/v1//users", "GET", Some(200), None), ]); - let under = |nodes: &[&str]| { - let nodes = nodes.iter().map(|n| n.to_string()).collect(); - urls(&db, FlowFilter { under: Some(nodes), ..Default::default() }).len() - }; - // A host on another port is a node of its own, as in the sidebar. - assert_eq!(under(&["api.stripe.com"]), 4); - assert_eq!(under(&["api.stripe.com:8443"]), 1); - assert_eq!(under(&["api.stripe.com/v1/charges"]), 2); - // Empty segments and default ports are left out, as the sidebar leaves them. - assert_eq!(under(&["example.com/v1/users"]), 1); - assert_eq!(under(&["x.com/a"]), 2); - assert_eq!(under(&["api.stripe.com/v1/charges", "example.com"]), 3); - assert_eq!(under(&[]), 0); - // The sidebar lowercases hosts; paths keep their case. - assert_eq!(under(&["API.stripe.com/v1/charges"]), 2); - assert_eq!(under(&["api.stripe.com/V1/charges"]), 0); + let count = |query: &str| found(&db, query).len(); + assert_eq!(count("@host:api.stripe.com"), 5); + assert_eq!(count("@host:api.stripe.com:8443"), 1); + assert_eq!(count("@host:api.stripe.com @path:/v1/charges/*"), 3); + assert_eq!(count("@host:x.com @path:/a"), 2); + assert_eq!(count("@host:api.stripe.com @path:/v1/charges/* OR @host:x.com"), 5); + assert_eq!(count("@host:API.stripe.com @path:/V1/charges/*"), 3); + // A path with empty segments is found by the sidebar's row for it. + assert_eq!(count("@host:example.com @path:/v1/users/*"), 1); } /// The cases `lib/search.test.ts` checks `sidebarPath` against, so the two agree. + /// A search with `OR` at its top still pages and leaves out cleared flows. + #[test] + fn a_top_level_or_keeps_to_the_page_and_the_clear() { + let (_dir, db) = db_of(vec![ + flow("https://a.com/1", "GET", Some(200), None), + flow("https://a.com/2", "GET", Some(404), None), + flow("https://a.com/3", "GET", Some(200), None), + ]); + let query = "@status:200 OR @status:404"; + let before = FlowFilter { before: Some("fl_001".into()), ..FlowFilter::search(query) }; + assert_eq!(urls(&db, before), vec!["https://a.com/1"]); + clear(&db).unwrap(); + assert!(urls(&db, FlowFilter::search(query)).is_empty()); + assert_eq!(page(&db, &FlowFilter::search(query)).unwrap().2, Some(0)); + } + #[test] fn sidebar_path_rules() { for (url, path) in [ @@ -1163,18 +948,15 @@ mod tests { from(None, "/usr/bin/curl"), flow("https://example.com/", "GET", Some(200), None), ]); - let app = |apps: &[&str]| { - let apps = apps.iter().map(|a| a.to_string()).collect(); - urls(&db, FlowFilter { app: Some(apps), ..Default::default() }).len() - }; - assert_eq!(app(&["com.google.Chrome"]), 2); - assert_eq!(app(&["/usr/bin/curl"]), 1); - assert_eq!(app(&["/Applications/Google Chrome.app"]), 0); - assert_eq!(app(&["com.google.Chrome", "/usr/bin/curl"]), 3); + let app = |query: &str| found(&db, query).len(); + assert_eq!(app("@app:com.google.Chrome"), 2); + assert_eq!(app("@app:/usr/bin/curl"), 1); + assert_eq!(app("@app:\"/Applications/Google Chrome.app\""), 0); + assert_eq!(app("@app:com.google.Chrome OR @app:/usr/bin/curl"), 3); } - /// A program in a folder: its working directory by the whole path, or its folder's name in - /// any case, along with whatever else the filter asks. + /// A program in a folder: its working directory by the whole path, or its folder's name, + /// any case, along with whatever else the search asks. #[test] fn cwd_matches_a_path_or_a_folder_name() { let node_in = |cwd: Option<&str>| Flow { @@ -1191,22 +973,14 @@ mod tests { Flow { client_cwd: Some("/tmp/fieldwork".into()), ..node_in(None) }, Flow { client_path: Some("/usr/bin/curl".into()), ..node_in(Some("/tmp/ledger")) }, ]); - let count = |app: Option<&str>, cwds: &[&str]| { - let cwd = Some(cwds.iter().map(|c| c.to_string()).collect()); - let app = app.map(|a| vec![a.to_string()]); - urls(&db, FlowFilter { app, cwd, ..Default::default() }).len() - }; - assert_eq!(count(None, &["/Users/ada/Workspace/fieldwork"]), 2, "the whole path"); - assert_eq!(count(None, &["/users/ada/workspace/fieldwork"]), 0, "a path exactly"); - assert_eq!(count(None, &["FieldWork"]), 3, "a folder name, any case"); - assert_eq!(count(None, &["fieldwork", "ledger"]), 5, "any of them"); - assert_eq!(count(Some("/usr/local/bin/node"), &["ledger"]), 1, "and the program"); - assert_eq!(count(Some("/usr/bin/curl"), &["ledger"]), 1); - assert_eq!(count(None, &[]), 0); - let query = FlowFilter { cwd: Some(vec!["a".into(), "/b".into()]), ..Default::default() }; - assert_eq!(query.to_query(), "cwd=a&cwd=%2Fb"); - assert_eq!(FlowFilter::from_query(&query.to_query()).unwrap(), query); - assert!(query.narrows()); + let count = |query: &str| found(&db, query).len(); + assert_eq!(count("@cwd:/Users/ada/Workspace/fieldwork"), 2, "the whole path"); + assert_eq!(count("@cwd:/users/ada/workspace/fieldwork"), 2, "any case"); + assert_eq!(count("@cwd:FieldWork"), 3, "a folder name"); + assert_eq!(count("@cwd:fieldwork OR @cwd:ledger"), 5, "any of them"); + assert_eq!(count("@app:/usr/local/bin/node @cwd:ledger"), 1, "and the program"); + assert_eq!(count("@app:/usr/bin/curl @cwd:ledger"), 1); + assert_eq!(count("@cwd:work"), 0, "whole"); } #[test] @@ -1221,7 +995,7 @@ mod tests { .collect(), ); let search = |before: Option| FlowFilter { - q: Some("post".into()), + query: Some("post".into()), before, limit: Some(4), ..Default::default() @@ -1245,8 +1019,7 @@ mod tests { let (_dir, db) = db_of(vec![]); let plan: Vec = db.with_conn(|c| { let conn = c.conn().resolve(); - let (condition, params) = - FlowFilter { q: Some("stripe".into()), ..Default::default() }.condition_in(conn); + let (condition, params) = FlowFilter::search("stripe @status:>=400").condition_in(conn); let sql = format!( "EXPLAIN QUERY PLAN SELECT * FROM flows WHERE {condition} \ ORDER BY started_at DESC, id DESC LIMIT 500" @@ -1322,7 +1095,7 @@ mod tests { // The limit counts flows the filter keeps. let ok = FlowFilter { before: Some("fl_e".into()), - status: Some("2xx".into()), + query: Some("@status:2xx".into()), limit: Some(2), ..Default::default() }; @@ -1380,7 +1153,7 @@ mod tests { write_flow("fl_d"); assert_eq!(ids(&FlowFilter::default()), vec!["fl_d"]); assert_eq!(find(&db, "fl_d").map(|f| f.number), Some(4)); - let narrowed = FlowFilter { q: Some("example".into()), ..Default::default() }; + let narrowed = FlowFilter::search("example"); assert_eq!(page(&db, &narrowed).unwrap().1, 1); // Bodies that fail to delete keep their flows, to delete again. @@ -1540,49 +1313,39 @@ mod tests { ids.sort(); ids }; - let app = |a: &str| FlowFilter { app: Some(vec![a.into()]), ..Default::default() }; - assert_eq!(ids(app("slack")), vec!["fl_slack"]); - assert_eq!(ids(app("com.tinyspeck.slackmacgap")), vec!["fl_slack"]); - assert_eq!(ids(app("Fieldwork")), vec!["fl_node"]); - assert_eq!(ids(app("/opt/homebrew/bin/node")), vec!["fl_node"]); - let under = |n: &str| FlowFilter { under: Some(vec![n.into()]), ..Default::default() }; - assert_eq!(ids(under("*.github.com")), vec!["fl_daemon", "fl_node"]); - assert_eq!(ids(under("api.github.com")), vec!["fl_node"]); - let system = |s| FlowFilter { system: Some(s), ..Default::default() }; - assert_eq!(ids(system(true)), vec!["fl_daemon"]); - assert_eq!(ids(system(false)), vec!["fl_node", "fl_none", "fl_slack"]); + let ids = |query: &str| ids(FlowFilter::search(query)); + assert_eq!(ids("@app:slack"), vec!["fl_slack"]); + assert_eq!(ids("@app:com.tinyspeck.slackmacgap"), vec!["fl_slack"]); + assert_eq!(ids("@app:Fieldwork"), vec!["fl_node"]); + assert_eq!(ids("@app:/opt/homebrew/bin/node"), vec!["fl_node"]); + assert_eq!(ids("@host:*.github.com"), vec!["fl_daemon", "fl_node"]); + assert_eq!(ids("@host:api.github.com"), vec!["fl_node"]); + assert_eq!(ids("@is:system"), vec!["fl_daemon"]); + assert_eq!(ids("-@is:system"), vec!["fl_node", "fl_none", "fl_slack"]); // Groups, as two sidebar rows under different apps write them, each judged whole: // flattened, the apps and hosts would pair across rows and match both flows. - let group = |a: &str, n: &str| FlowFilter { - app: Some(vec![a.into()]), - under: Some(vec![n.into()]), - ..Default::default() - }; - let crossed = FlowFilter { - any: Some(vec![ - group("fieldwork", "github.com"), - group("nsurlsessiond", "api.github.com"), - ]), - ..Default::default() - }; - assert_eq!(ids(crossed.clone()), Vec::::new()); - assert_eq!(page(&db, &crossed).unwrap().2, Some(0)); - let flattened = FlowFilter { - app: Some(vec!["fieldwork".into(), "nsurlsessiond".into()]), - under: Some(vec!["github.com".into(), "api.github.com".into()]), - ..Default::default() - }; + let crossed = "@app:fieldwork @host:github.com OR @app:nsurlsessiond @host:api.github.com"; + assert_eq!(ids(crossed), Vec::::new()); + assert_eq!(page(&db, &FlowFilter::search(crossed)).unwrap().2, Some(0)); + let flattened = "(@app:fieldwork OR @app:nsurlsessiond) \ + (@host:github.com OR @host:api.github.com)"; assert_eq!(ids(flattened), vec!["fl_daemon", "fl_node"]); - let either = FlowFilter { - any: Some(vec![ - group("fieldwork", "api.github.com"), - FlowFilter { app: Some(vec!["slack".into()]), ..Default::default() }, - ]), - q: Some("https".into()), - ..Default::default() - }; + let either = "(@app:fieldwork @host:api.github.com OR @app:slack) https"; assert_eq!(ids(either), vec!["fl_node", "fl_slack"]); + + // Hiding System leaves out its daemons, unless the search names them. + let hidden = |query: &str| { + let filter = FlowFilter { hide_system: true, ..FlowFilter::search(query) }; + let mut ids: Vec = + list(&db, &filter).unwrap().into_iter().map(|f| f.id).collect(); + ids.sort(); + ids + }; + assert_eq!(hidden(""), vec!["fl_node", "fl_none", "fl_slack"]); + assert_eq!(hidden("@host:*.github.com"), vec!["fl_node"]); + assert_eq!(hidden("@is:system"), vec!["fl_daemon"]); + assert_eq!(hidden("@host:*.github.com OR @is:SYSTEM"), vec!["fl_daemon", "fl_node"]); } #[test] @@ -1603,35 +1366,24 @@ mod tests { assert!(!is_system(None, None)); assert_eq!(folder_name(Some("/Users/me/src/fieldwork/")), Some("fieldwork")); assert_eq!(folder_name(Some("/")), None); - assert!(under_site("https://api.github.com:8443/x", "github.com")); - assert!(!under_site("https://notgithub.com/", "github.com")); } #[test] fn query_round_trip() { let filter = FlowFilter { - q: Some("post json".into()), - host: Some("api.stripe.com".into()), - under: Some(vec!["api.stripe.com/v1".into(), "example.com".into()]), - app: Some(vec!["com.google.Chrome".into()]), - status: Some("4xx".into()), + query: Some("post @status:4xx".into()), + hide_system: true, limit: Some(20), ..Default::default() }; - assert_eq!( - filter.to_query(), - "q=post+json&host=api.stripe.com&under=api.stripe.com%2Fv1&under=example.com\ - &app=com.google.Chrome&status=4xx&limit=20" - ); + assert_eq!(filter.to_query(), "query=post+%40status%3A4xx&hide_system=true&limit=20"); assert_eq!(FlowFilter::from_query(&filter.to_query()).unwrap(), filter); assert_eq!(FlowFilter::from_query("").unwrap(), FlowFilter::default()); assert!(FlowFilter::from_query("limit=lots").is_err()); - - let tunnelled = FlowFilter { tunnelled: Some(true), ..Default::default() }; - assert_eq!(tunnelled.to_query(), "tunnelled=true"); - assert_eq!(FlowFilter::from_query("tunnelled=true").unwrap(), tunnelled); - assert!(tunnelled.narrows()); - assert!(FlowFilter::from_query("tunnelled=maybe").is_err()); + assert!(FlowFilter::from_query("hide_system=maybe").is_err()); + assert!(filter.narrows()); + assert!(FlowFilter { hide_system: true, ..Default::default() }.narrows()); + assert!(!FlowFilter::search(" ").narrows()); } #[test] diff --git a/crates-proxy/yaak-proxy-lib/src/lib.rs b/crates-proxy/yaak-proxy-lib/src/lib.rs index 80e16fb68..fc935c80d 100644 --- a/crates-proxy/yaak-proxy-lib/src/lib.rs +++ b/crates-proxy/yaak-proxy-lib/src/lib.rs @@ -14,7 +14,9 @@ pub mod helper; pub mod macos; pub mod messages; pub mod models; +pub mod query; pub mod save; +pub mod saved_queries; pub mod settings; pub mod terminal; @@ -47,7 +49,7 @@ use std::time::{Duration, SystemTime}; use ts_rs::TS; pub use yaak_crypto::MasterKeySource; use yaak_crypto::data_key::DataKey; -use yaak_database::{ModelChangeEvent, UpdateSource, UpsertModelInfo}; +use yaak_database::{UpdateSource, UpsertModelInfo}; pub use yaak_grpc::GrpcConfig; use yaak_models::blob_manager::{BlobManager, BodyChunkWriter}; use yaak_proxy::{CaStore, KeyCipher, ProxyConfig, ProxyEvent, ProxyHandle}; @@ -107,6 +109,8 @@ pub struct ProxyCtx { pub events: RpcEventEmitter, /// gRPC schemas as decoded, and the protoc that compiles attached protos. pub grpc: GrpcState, + /// The search the app shows, which written flows' events are matched against. + live_search: Arc, } impl ProxyCtx { @@ -148,6 +152,7 @@ impl ProxyCtx { sweeper: Default::default(), events, grpc: GrpcState::default(), + live_search: Default::default(), }; if db::needs_encrypting(data_dir) { let unlocking = ctx.clone(); @@ -418,6 +423,7 @@ impl ProxyCtx { db: self.db()?, blobs: self.blobs()?, events: self.events.clone(), + live_search: self.live_search.clone(), attribution, recording: self.recording.clone(), finished: Default::default(), @@ -1118,16 +1124,38 @@ impl ProxyCtx { } let (flow, created) = tx.upsert(&flow, &UpdateSource::Background)?; // Sent before the commit, so it cannot reach the app after a clear that follows. - self.events.emit( - "model_write", - &ModelPayload { model: flow, change: ModelChangeEvent::Upsert { created } }, - ); + self.events.emit("model_write", &self.live_search.payload(flow, created)); Ok::<_, yaak_database::Error>(()) }) .map_err(|e| format!("Failed to record the saved request on the flow: {e}"))?; Ok(saved) } + /// Records that an agent read the flow through the control API, for `@is:read`, and tells + /// the app. Only the first read writes anything. + pub fn note_agent_read(&self, flow_id: &str) { + let noted = self.db().map_err(String::from).and_then(|db| { + db.with_tx(|tx| { + let conn = tx.conn().resolve(); + let changed = conn.execute( + "UPDATE flows SET agent_read = TRUE WHERE id = ? AND NOT agent_read", + [flow_id], + )?; + // Sent before the commit, as every flow's event is. + if changed > 0 + && let Some(flow) = flows::find_in(conn, flow_id) + { + self.events.emit("model_write", &self.live_search.payload(flow, false)); + } + Ok::<_, yaak_database::Error>(()) + }) + .map_err(|e| e.to_string()) + }); + if let Err(e) = noted { + warn!("Failed to note that an agent read flow {flow_id}: {e}"); + } + } + /// Clears every flow at once and tells the app so its list empties, then deletes them and /// their bodies in the background. Returns how many there were. pub fn clear_flows(&self) -> Result { @@ -1385,6 +1413,91 @@ pub struct ListFlowsResponse { pub matching: Option, } +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct QuerySchemaRequest {} + +/// The search's fields, for the search box's highlighting and completion. +#[derive(Serialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct QuerySchemaResponse { + #[ts(type = "import(\"@yaakapp-internal/query\").Schema")] + pub schema: serde_json::Value, +} + +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct ValidateQueryRequest { + pub query: String, +} + +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct QueryValuesRequest { + /// A field's path: `host`, `header` for header names, `header.accept` for a header's values. + pub path: String, +} + +/// The search the app shows, which flows written from now on are matched against, their events +/// naming `key`. No query stops it. +#[derive(Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct WatchQueryRequest { + pub key: String, + #[ts(optional)] + pub query: Option, + #[serde(default)] + #[ts(optional, as = "Option")] + pub hide_system: bool, +} + +/// Which of some flows a search covers, for flows whose events were matched against another. +#[derive(Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct MatchFlowsRequest { + pub filter: flows::FlowFilter, + pub ids: Vec, +} + +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct ListSavedQueriesRequest {} + +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct CreateSavedQueryRequest { + pub name: String, + pub query: String, +} + +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct RenameSavedQueryRequest { + pub id: String, + pub name: String, +} + +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct DeleteSavedQueryRequest { + pub id: String, +} + +#[derive(Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct CountSavedQueriesRequest { + #[serde(default)] + #[ts(optional, as = "Option")] + pub hide_system: bool, +} + +#[derive(Serialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct SavedQueriesChangedPayload {} + #[derive(Deserialize, TS)] #[ts(export, export_to = "gen_rpc.ts")] pub struct ClearFlowsRequest {} @@ -2326,6 +2439,73 @@ async fn clear_flows(ctx: ProxyCtx, _req: ClearFlowsRequest) -> Result Result { + let schema = serde_json::to_value(query::schema()).map_err(rpc_error)?; + Ok(QuerySchemaResponse { schema }) +} + +fn validate_query( + _ctx: &ProxyCtx, + req: ValidateQueryRequest, +) -> Result, RpcError> { + Ok(query::diagnostics(&req.query)) +} + +async fn query_values(ctx: ProxyCtx, req: QueryValuesRequest) -> Result, RpcError> { + blocking(move || query::seen_values(&ctx.db()?, &req.path)).await +} + +fn watch_query(ctx: &ProxyCtx, req: WatchQueryRequest) -> Result { + let watched = req.query.filter(|q| !q.trim().is_empty() || req.hide_system); + ctx.live_search.watch(watched.map(|q| (req.key, q, req.hide_system))); + Ok(true) +} + +async fn match_flows(ctx: ProxyCtx, req: MatchFlowsRequest) -> Result, RpcError> { + blocking(move || flows::matching_ids(&ctx.db()?, &req.filter, &req.ids)).await +} + +fn list_saved_queries( + ctx: &ProxyCtx, + _req: ListSavedQueriesRequest, +) -> Result, RpcError> { + saved_queries::list(&ctx.db()?).map_err(rpc_error) +} + +fn create_saved_query( + ctx: &ProxyCtx, + req: CreateSavedQueryRequest, +) -> Result { + let saved = saved_queries::create(&ctx.db()?, &req.name, &req.query).map_err(rpc_error)?; + ctx.events.emit("saved_queries_changed", &SavedQueriesChangedPayload {}); + Ok(saved) +} + +fn rename_saved_query( + ctx: &ProxyCtx, + req: RenameSavedQueryRequest, +) -> Result { + let saved = saved_queries::rename(&ctx.db()?, &req.id, &req.name).map_err(rpc_error)?; + ctx.events.emit("saved_queries_changed", &SavedQueriesChangedPayload {}); + Ok(saved) +} + +fn delete_saved_query(ctx: &ProxyCtx, req: DeleteSavedQueryRequest) -> Result { + let deleted = saved_queries::delete(&ctx.db()?, &req.id).map_err(rpc_error)?; + ctx.events.emit("saved_queries_changed", &SavedQueriesChangedPayload {}); + Ok(deleted) +} + +async fn count_saved_queries( + ctx: ProxyCtx, + req: CountSavedQueriesRequest, +) -> Result, RpcError> { + blocking(move || saved_queries::counts(&ctx.db()?, req.hide_system)).await +} + // -- Event loop -- /// Why the engine cannot be taken over from another app. @@ -2429,6 +2609,8 @@ struct FlowSink { db: ProxyQueryManager, blobs: BlobManager, events: RpcEventEmitter, + /// The search the app shows, which each flow written is matched against. + live_search: Arc, attribution: Attribution, recording: Arc>, /// Flows that have ended, waiting to be written together. @@ -3076,10 +3258,7 @@ impl FlowSink { let (saved, created) = ctx.upsert(&flow, &UpdateSource::Background)?; // Sent while this holds the writer, before the commit, so a clear's event, sent after // its own commit, always comes after it. - self.events.emit( - "model_write", - &ModelPayload { model: saved, change: ModelChangeEvent::Upsert { created } }, - ); + self.events.emit("model_write", &self.live_search.payload(saved, created)); Ok(true) } @@ -3142,6 +3321,13 @@ define_rpc! { get_message_payload(GetMessagePayloadRequest) -> GetMessagePayloadResponse, list_actions(ListActionsRequest) -> ListActionsResponse, list_flows(flows::FlowFilter) -> ListFlowsResponse, + query_schema(QuerySchemaRequest) -> QuerySchemaResponse, + validate_query(ValidateQueryRequest) -> Vec, + watch_query(WatchQueryRequest) -> bool, + list_saved_queries(ListSavedQueriesRequest) -> Vec, + create_saved_query(CreateSavedQueryRequest) -> saved_queries::SavedQuery, + rename_saved_query(RenameSavedQueryRequest) -> saved_queries::SavedQuery, + delete_saved_query(DeleteSavedQueryRequest) -> bool, list_grpc_schemas(ListGrpcSchemasRequest) -> Vec, list_messages(ListMessagesRequest) -> ListMessagesResponse, remove_grpc_schema(RemoveGrpcSchemaRequest) -> bool, @@ -3179,6 +3365,9 @@ define_rpc! { get_storage_size(GetStorageSizeRequest) -> GetStorageSizeResponse, reset_captured_data(ResetCapturedDataRequest) -> bool, open_terminal(OpenTerminalRequest) -> bool, + query_values(QueryValuesRequest) -> Vec, + match_flows(MatchFlowsRequest) -> Vec, + count_saved_queries(CountSavedQueriesRequest) -> Vec, } events { decrypt_list_changed(DecryptListChangedPayload), @@ -3187,6 +3376,7 @@ define_rpc! { message_write(MessagePayload), model_write(ModelPayload), proxy_state_changed(ProxyStatePayload), + saved_queries_changed(SavedQueriesChangedPayload), } } @@ -3256,6 +3446,7 @@ mod tests { db: ctx.db().unwrap(), blobs: ctx.blobs().unwrap(), events: ctx.events.clone(), + live_search: ctx.live_search.clone(), attribution: Attribution::new(0), recording: ctx.recording.clone(), finished: Default::default(), diff --git a/crates-proxy/yaak-proxy-lib/src/models.rs b/crates-proxy/yaak-proxy-lib/src/models.rs index a11671ef3..2d922fd07 100644 --- a/crates-proxy/yaak-proxy-lib/src/models.rs +++ b/crates-proxy/yaak-proxy-lib/src/models.rs @@ -107,6 +107,9 @@ pub struct Flow { /// client rejected the proxy's certificate: method `CONNECT`, its URL the host, its sizes /// the bytes each way, and no headers or bodies. Open until `elapsedMs` or `error` is set. pub tunnelled: bool, + /// An agent read the flow through the control API: its detail, a body or its messages. + /// Never written by an upsert, which keeps what is stored. + pub agent_read: bool, } /// The streaming protocol a flow carries. @@ -194,6 +197,21 @@ pub fn find_header<'a>(headers: &'a [ProxyHeader], name: &str) -> Option<&'a str pub struct ModelPayload { pub model: Flow, pub change: ModelChangeEvent, + /// Whether the flow belongs in the list of the search the app shows, decided as the event + /// is sent. None while no search is shown. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub search: Option, +} + +/// A written flow, matched against the search the app shows. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_models.ts")] +pub struct SearchMatch { + /// The search's key, as the app gave it with the search. + pub key: String, + pub matches: bool, } impl UpsertModelInfo for Flow { @@ -357,6 +375,7 @@ impl UpsertModelInfo for Flow { grpc_status: added_column(r, "grpc_status")?, grpc_message: added_column(r, "grpc_message")?, tunnelled: added_column(r, "tunnelled")?.unwrap_or_default(), + agent_read: added_column(r, "agent_read")?.unwrap_or_default(), }) } } diff --git a/crates-proxy/yaak-proxy-lib/src/query.rs b/crates-proxy/yaak-proxy-lib/src/query.rs new file mode 100644 index 000000000..a91185092 --- /dev/null +++ b/crates-proxy/yaak-proxy-lib/src/query.rs @@ -0,0 +1,1242 @@ +//! Yaak Proxy's search: the query language of `yaak-query` over flows. +//! +//! A query is parsed by the crate, then prepared here: the shorthands the proxy keeps +//! (`@status:4xx`, `@host:*.github.com` for a site, `@path:/v1/*` for a folder and what is under +//! it, `@time:<5m`) become plain terms, and the hidden System flows are left out. The prepared +//! tree then runs two ways that must agree: compiled to a SQLite condition for listing and +//! counting ([`FlowQuery::sql`]), and in memory over one flow ([`FlowQuery::matches`]), for a +//! flow written while a search is shown. Both resolve each field term to the crate's +//! [`Condition`], and the values they test are made by the same functions here, which the +//! database calls as SQL functions ([`add_functions`]). + +use crate::flows::{folder_name, is_system, sidebar_path}; +use crate::models::{Flow, ModelPayload, ProxyHeader, SearchMatch, find_header}; +use chrono::{DateTime, Local, NaiveDate, NaiveDateTime, NaiveTime, TimeZone, Utc}; +use rusqlite::functions::FunctionFlags; +use rusqlite::types::{Value, ValueRef}; +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; +use std::sync::{OnceLock, PoisonError, RwLock}; +use ts_rs::TS; +use yaak_database::ModelChangeEvent; +use yaak_query::{ + CompareOp, Condition, DocValue, DocValues, Document, FieldDef, FieldKind, FieldValue, FreeText, + Matcher, Node, NumberLiteral, Schema, Span, Unit, Utf16Offsets, contains_ignore_case, + format_number, read_number, wildcard_match, +}; + +/// The flags `@is:` takes, in the order completion offers them. +pub const FLAGS: &[&str] = &[ + "error", + "pending", + "system", + "tunnelled", + "saved", + "read", + "loop", + "ws", + "sse", + "grpc", +]; + +/// Flags `@is:` will take once flows have what they need. Valid, matching nothing. +const FLAGS_NOT_YET: &[&str] = &["marked"]; + +/// Fields declared so completion shows them, matching nothing until they are built. +const FIELDS_NOT_YET: &[&str] = &["body", "req.body", "res.body"]; + +/// The values `@proto:` takes. +pub const PROTOS: &[&str] = &["h1", "h2", "ws", "sse", "grpc"]; + +/// The proxy's fields. +pub fn schema() -> &'static Schema { + static SCHEMA: OnceLock = OnceLock::new(); + SCHEMA.get_or_init(|| { + let units = |units: &[(&str, f64)]| FieldKind::Number { + units: units + .iter() + .map(|(name, factor)| Unit { name: (*name).into(), factor: *factor }) + .collect(), + }; + // As the app shows sizes: a kilobyte is 1000 bytes. + let bytes = || units(&[("b", 1.0), ("kb", 1e3), ("mb", 1e6), ("gb", 1e9)]); + // Each with what it is and a term using it, as completion shows them + let field = |name: &str, kind: FieldKind, info: &str, example: &str| FieldDef { + name: name.into(), + kind, + map: false, + info: Some(info.into()), + example: (!example.is_empty()).then(|| example.into()), + }; + let map = |name: &str, info: &str, example: &str| FieldDef { + map: true, + ..field(name, FieldKind::Text, info, example) + }; + let text = || FieldKind::Text; + let number = || FieldKind::Number { units: vec![] }; + let ms = || units(&[("ms", 1.0), ("s", 1e3), ("m", 6e4)]); + Schema::new( + vec![ + field("host", text(), "Host, wildcards allowed", "@host:*.github.com"), + field("path", text(), "URL path, /* for below", "@path:/v1/*"), + field("method", text(), "GET, POST, …", "@method:post"), + field("status", number(), "Response status", "@status:4xx @status:>=400"), + field("type", text(), "Content type", "@type:json"), + field("app", text(), "App or program", "@app:Slack"), + field("is", text(), "error, pending, system, …", "@is:error -@is:system"), + map("header", "A header on either side", "@header.content-type:json"), + map("req.header", "A request header", "@req.header.accept:*"), + map("res.header", "A response header", "@res.header.x-cache:hit"), + field("size", bytes(), "Response body size", "@size:>100kb"), + field("req.size", bytes(), "Request body size", "@req.size:>1kb"), + field("res.size", bytes(), "Response body size", "@res.size:<1mb"), + field("duration", ms(), "Total time", "@duration:>500ms"), + field("time", units(TIME_UNITS), "When the flow started", "@time:<5m"), + field("url", text(), "The whole URL", "@url:*token=*"), + map("query", "A query parameter", "@query.page:2"), + field("scheme", text(), "http or https", "@scheme:https"), + field("port", number(), "Port", "@port:8080"), + field("proto", text(), "h1, h2, ws, sse, grpc", "@proto:h2"), + field("id", number(), "Flow number", "@id:>1200"), + field("cwd", text(), "Working folder", "@cwd:myapp"), + field("pid", number(), "Process id", "@pid:4121"), + field("body", text(), "Not yet", ""), + field("req.body", text(), "Not yet", ""), + field("res.body", text(), "Not yet", ""), + ], + FreeText::Substring, + ) + }) +} + +/// How long ago `@time` counts in, in milliseconds. +const TIME_UNITS: &[(&str, f64)] = &[ + ("ms", 1.0), + ("s", 1e3), + ("m", 6e4), + ("h", 3.6e6), + ("d", 8.64e7), + ("w", 6.048e8), +]; + +// -- Preparing -- + +/// A query prepared for running: its shorthands written out, System left out when hidden. +#[derive(Debug, Clone, PartialEq)] +pub struct FlowQuery { + root: Option, +} + +impl FlowQuery { + /// `text` as of now. With `hide_system`, macOS's own daemons are left out, unless the text + /// says `@is:system` itself. + pub fn new(text: &str, hide_system: bool) -> Self { + Self::at(text, hide_system, Utc::now()) + } + + /// [`Self::new`], with `now` for the times `@time` counts back from. + pub fn at(text: &str, hide_system: bool, now: DateTime) -> Self { + let parsed = yaak_query::parse(text); + let mentions_system = parsed.as_ref().is_some_and(mentions_system); + let mut root = parsed.map(|root| prepare(&root, now, &mut Vec::new())); + if hide_system && !mentions_system { + let span = Span::default(); + let not_system = Node::Not { node: Box::new(is_flag("system", span)), span }; + root = Some(match root { + None => not_system, + Some(root) => Node::And { children: vec![root, not_system], span }, + }); + } + Self { root } + } + + /// Covers every flow. + pub fn is_empty(&self) -> bool { + self.root.is_none() + } + + /// Whether `flow` is one the query covers, as [`Self::sql`] decides it in the database. + pub fn matches(&self, flow: &Flow) -> bool { + self.matcher().matches(&document(flow)) + } + + /// The query resolved for matching many flows in memory. + pub fn matcher(&self) -> Matcher { + Matcher::new(self.root.as_ref(), schema()) + } + + /// The SQL condition on `flows` and its parameters, for a database with `columns`. In + /// parentheses, so a caller may add to it with `AND`. + pub fn sql(&self, columns: &FlowColumns) -> (String, Vec) { + let mut sql = Sql { columns, params: Vec::new() }; + let condition = match &self.root { + None => "1".to_string(), + Some(root) => sql.node(root), + }; + (format!("({condition})"), sql.params) + } +} + +/// `@is:`, as a prepared term. +fn is_flag(flag: &str, span: Span) -> Node { + Node::Field { + path: vec!["is".into()], + value: FieldValue::Wildcard { pattern: flag.into() }, + span, + path_span: span, + value_span: span, + } +} + +/// The query names `@is:system`, either way, so hiding System leaves it be. +fn mentions_system(root: &Node) -> bool { + let mut found = false; + root.walk(&mut |node| { + if let Node::Field { path, value, .. } = node + && path.len() == 1 + && path[0].eq_ignore_ascii_case("is") + && let FieldValue::Text { text, .. } | FieldValue::Wildcard { pattern: text } = value + && text.eq_ignore_ascii_case("system") + { + found = true; + } + }); + found +} + +/// What the schema alone does not catch, found while preparing. +#[derive(Debug, Clone, PartialEq)] +struct Issue { + kind: QueryDiagnosticKind, + path: String, + span: Span, +} + +/// A term that matches nothing, in place of one that cannot run. +fn never(span: Span) -> Node { + Node::Phrase { text: String::new(), span } +} + +fn number(value: f64) -> NumberLiteral { + NumberLiteral { value, unit: None, raw: format_number(value) } +} + +/// `node` with the proxy's shorthands written out as plain terms, keeping their spans. +fn prepare(node: &Node, now: DateTime, issues: &mut Vec) -> Node { + match node { + Node::And { children, span } => Node::And { + children: children.iter().map(|c| prepare(c, now, issues)).collect(), + span: *span, + }, + Node::Or { children, span } => Node::Or { + children: children.iter().map(|c| prepare(c, now, issues)).collect(), + span: *span, + }, + Node::Not { node: inner, span } => { + Node::Not { node: Box::new(prepare(inner, now, issues)), span: *span } + } + Node::Term { .. } | Node::Phrase { .. } => node.clone(), + Node::Field { path, value, span, path_span, value_span } => { + let Some(field) = schema().resolve(path) else { + return node.clone(); + }; + let with = |value: FieldValue| Node::Field { + path: path.clone(), + value, + span: *span, + path_span: *path_span, + value_span: *value_span, + }; + let mut issue = |kind| { + let marked = if kind == QueryDiagnosticKind::NotYet { *span } else { *value_span }; + issues.push(Issue { kind, path: path.join("."), span: marked }); + never(*span) + }; + match (field.def.name.as_str(), value) { + // A class of statuses: `4xx` + ("status", FieldValue::Text { text, quoted: false }) => match status_class(text) { + Some(class) => with(FieldValue::Range { + low: number(class * 100.0), + high: number(class * 100.0 + 99.0), + }), + None => node.clone(), + }, + // A site: `*.github.com` is `github.com` and every host below it + ("host", FieldValue::Wildcard { pattern }) if pattern.starts_with("*.") => { + Node::Or { + children: vec![ + with(FieldValue::Wildcard { pattern: pattern[2..].to_string() }), + node.clone(), + ], + span: *span, + } + } + // A folder: `/v1/*` is `/v1` and everything under it + ("path", FieldValue::Wildcard { pattern }) if pattern.ends_with("/*") => Node::Or { + children: vec![ + with(FieldValue::Wildcard { pattern: pattern[..pattern.len() - 2].into() }), + node.clone(), + ], + span: *span, + }, + // Named whole, as the sidebar names them: a part needs wildcards + ("host" | "scheme" | "app" | "cwd", FieldValue::Text { text, .. }) + if !text.is_empty() => + { + with(FieldValue::Wildcard { pattern: text.clone() }) + } + ("is", FieldValue::Text { text, .. }) if !text.is_empty() => { + let flag = text.to_lowercase(); + if FLAGS.contains(&flag.as_str()) { + with(FieldValue::Wildcard { pattern: flag }) + } else if FLAGS_NOT_YET.contains(&flag.as_str()) { + issue(QueryDiagnosticKind::NotYet) + } else { + issue(QueryDiagnosticKind::InvalidValue) + } + } + // Not written yet + ("time", FieldValue::Text { text, .. }) if text.is_empty() => node.clone(), + ("time", value) => match time_node(value, now, &with) { + Some(node) => node, + None => issue(QueryDiagnosticKind::InvalidValue), + }, + (name, _) if FIELDS_NOT_YET.contains(&name) => issue(QueryDiagnosticKind::NotYet), + _ => node.clone(), + } + } + } +} + +/// The class `4xx` names, 4, or None when `text` is not one. +fn status_class(text: &str) -> Option { + let class = text.strip_suffix("xx").or_else(|| text.strip_suffix("XX"))?; + match class.as_bytes() { + [d @ b'1'..=b'9'] => Some(f64::from(d - b'0')), + _ => None, + } +} + +/// `@time`'s value as plain comparisons of when flows started, in Unix milliseconds. A +/// comparison or range of durations is how long ago (`<5m`: in the last five minutes); a date, +/// `2026-11-07`, a time today, `14:30`, or both, `2026-11-07T14:30`, all local, or an ISO time +/// with its offset, is the span it names, compared whole. None when the value is none of these. +fn time_node( + value: &FieldValue, + now: DateTime, + with: &impl Fn(FieldValue) -> Node, +) -> Option { + let now_ms = now.timestamp_millis() as f64; + let compare = |op, ms: f64| with(FieldValue::Compare { op, number: number(ms) }); + match value { + FieldValue::Exists => Some(with(FieldValue::Exists)), + FieldValue::Compare { op, number: age } => { + // Less long ago is later + let op = match op { + CompareOp::Gt => CompareOp::Lt, + CompareOp::Gte => CompareOp::Lte, + CompareOp::Lt => CompareOp::Gt, + CompareOp::Lte => CompareOp::Gte, + }; + Some(compare(op, now_ms - age_ms(age)?)) + } + FieldValue::Range { low, high } => { + let (a, b) = (age_ms(low)?, age_ms(high)?); + Some(with(FieldValue::Range { + low: number(now_ms - a.max(b)), + high: number(now_ms - a.min(b)), + })) + } + FieldValue::Text { text, .. } => { + let (op, point) = split_op(text); + let (start, end) = time_span(point, now)?; + Some(match op { + None => { + let span = Span::default(); + Node::And { + children: vec![compare(CompareOp::Gte, start), compare(CompareOp::Lt, end)], + span, + } + } + Some(CompareOp::Gt) => compare(CompareOp::Gte, end), + Some(CompareOp::Gte) => compare(CompareOp::Gte, start), + Some(CompareOp::Lt) => compare(CompareOp::Lt, start), + Some(CompareOp::Lte) => compare(CompareOp::Lt, end), + }) + } + FieldValue::Wildcard { .. } => None, + } +} + +/// A duration in milliseconds, which must have a unit. +fn age_ms(number: &NumberLiteral) -> Option { + let unit = number.unit.as_deref()?; + let (_, factor) = TIME_UNITS.iter().find(|(name, _)| name.eq_ignore_ascii_case(unit))?; + Some(number.value * factor) +} + +fn split_op(text: &str) -> (Option, &str) { + for (prefix, op) in [ + (">=", CompareOp::Gte), + ("<=", CompareOp::Lte), + (">", CompareOp::Gt), + ("<", CompareOp::Lt), + ] { + if let Some(rest) = text.strip_prefix(prefix) { + return (Some(op), rest); + } + } + (None, text) +} + +/// The span of time `text` names, as Unix milliseconds from its start to just past its end. +fn time_span(text: &str, now: DateTime) -> Option<(f64, f64)> { + let local = |dt: NaiveDateTime| Local.from_local_datetime(&dt).earliest(); + let ms = |dt: DateTime| dt.timestamp_millis() as f64; + if let Ok(dt) = DateTime::parse_from_rfc3339(text) { + let start = dt.timestamp_millis() as f64; + return Some((start, start + 1e3)); + } + if let Ok(date) = NaiveDate::parse_from_str(text, "%Y-%m-%d") { + let start = local(date.and_time(NaiveTime::MIN))?; + let end = local(date.succ_opt()?.and_time(NaiveTime::MIN))?; + return Some((ms(start), ms(end))); + } + let today = now.with_timezone(&Local).date_naive(); + for (format, length) in [ + ("%Y-%m-%dT%H:%M:%S", 1e3), + ("%Y-%m-%dT%H:%M", 6e4), + ("%H:%M:%S", 1e3), + ("%H:%M", 6e4), + ] { + let parsed = NaiveDateTime::parse_from_str(text, format) + .ok() + .or_else(|| NaiveTime::parse_from_str(text, format).ok().map(|t| today.and_time(t))); + if let Some(dt) = parsed { + let start = ms(local(dt)?); + return Some((start, start + length)); + } + } + None +} + +// -- Live flows -- + +/// The search the app shows, while it shows one. Each flow written meanwhile is matched +/// against it as its event is sent, so the event says whether it belongs in the search's list +/// and the app needs no matching of its own. +#[derive(Debug, Default)] +pub struct LiveSearch(RwLock>); + +#[derive(Debug)] +struct Watched { + key: String, + text: String, + hide_system: bool, +} + +impl LiveSearch { + /// Matches flows against `text` from now on, their events naming `key`; None stops. + pub fn watch(&self, watched: Option<(String, String, bool)>) { + let watched = watched.map(|(key, text, hide_system)| Watched { key, text, hide_system }); + *self.0.write().unwrap_or_else(PoisonError::into_inner) = watched; + } + + /// `flow` matched against the search shown, if any. + pub fn check(&self, flow: &Flow) -> Option { + let watched = self.0.read().unwrap_or_else(PoisonError::into_inner); + let Watched { key, text, hide_system } = watched.as_ref()?; + // Prepared again for each flow, so `@time:<5m` counts back from now. + let matches = FlowQuery::new(text, *hide_system).matches(flow); + Some(SearchMatch { key: key.clone(), matches }) + } + + /// The event for `flow`, written: created, or updated. + pub fn payload(&self, flow: Flow, created: bool) -> ModelPayload { + let search = self.check(&flow); + ModelPayload { model: flow, change: ModelChangeEvent::Upsert { created }, search } + } +} + +// -- Seen values, for completion -- + +/// The newest flows completion looks through for values. +const SEEN_FLOWS: i64 = 2000; + +/// Values offered per field at most. +const SEEN_VALUES: i64 = 200; + +/// A header's values are offered only when it has had at most this many: more, and they are +/// near-unique (dates, ids), which is noise. +const HEADER_VALUES: i64 = 20; + +/// The values seen for a field path, for completing it, most recent first: hosts, methods, +/// statuses, content types, protocols, schemes, apps, working folders, `@is` flags, and header +/// names, or a header's values when it has had few. Nothing for fields whose values are +/// near-unique (URLs, paths, ids, sizes, times, query parameters). +pub fn seen_values(db: &crate::db::ProxyQueryManager, path: &str) -> Result, String> { + let path = path.to_lowercase(); + let column = match path.as_str() { + "proto" => return Ok(PROTOS.iter().map(|p| p.to_string()).collect()), + "is" => return Ok(FLAGS.iter().map(|f| f.to_string()).collect()), + "host" => "url_host(url)", + "method" => "method", + "status" => "CAST(status AS TEXT)", + "type" => "coalesce(content_type, header_value(request_headers, 'content-type'))", + "scheme" => "url_scheme(url)", + "app" => "client_name", + "cwd" => "folder_name(client_cwd)", + "header" | "req.header" | "res.header" => { + return seen_headers(db, &path, None); + } + _ => { + for map in ["req.header.", "res.header.", "header."] { + if let Some(name) = path.strip_prefix(map) { + return seen_headers(db, &map[..map.len() - 1], Some(name)); + } + } + return Ok(Vec::new()); + } + }; + let sql = format!( + "SELECT v FROM (SELECT {column} AS v, rid FROM ({})) WHERE v IS NOT NULL AND v != '' \ + GROUP BY v ORDER BY max(rid) DESC LIMIT {SEEN_VALUES}", + newest_flows() + ); + let mut values = select_strings(db, &sql, None)?; + if path == "type" { + // The media type alone, as a search names it + let mut seen = HashSet::new(); + values = values + .into_iter() + .filter_map(|t| t.split(';').next().map(|m| m.trim().to_lowercase())) + .filter(|m| !m.is_empty() && seen.insert(m.clone())) + .collect(); + } + Ok(values) +} + +/// The newest flows not cleared, with their rowid as `rid`. +fn newest_flows() -> String { + format!( + "SELECT rowid AS rid, * FROM flows WHERE rowid > (SELECT coalesce(max(max_rowid), 0) \ + FROM flow_clear_mark) ORDER BY rowid DESC LIMIT {SEEN_FLOWS}" + ) +} + +/// Header names under a map, or one header's values when it has had few. +fn seen_headers( + db: &crate::db::ProxyQueryManager, + map: &str, + name: Option<&str>, +) -> Result, String> { + let sides = match map { + "req.header" => vec!["request_headers"], + "res.header" => vec!["response_headers"], + _ => vec!["request_headers", "response_headers"], + }; + let pairs = sides + .iter() + .map(|side| { + format!( + "SELECT q_lower(json_extract(kv.value, '$.name')) AS name, \ + json_extract(kv.value, '$.value') AS value, f.rid AS rid \ + FROM ({}) AS f, json_each(f.{side}) AS kv", + newest_flows() + ) + }) + .collect::>() + .join(" UNION ALL "); + match name { + None => { + let sql = format!( + "SELECT name FROM ({pairs}) WHERE name IS NOT NULL AND name != '' \ + GROUP BY name ORDER BY max(rid) DESC LIMIT {SEEN_VALUES}" + ); + select_strings(db, &sql, None) + } + Some(name) => { + let sql = format!( + "SELECT value FROM ({pairs}) WHERE name = ? AND value IS NOT NULL AND value != '' \ + GROUP BY value ORDER BY max(rid) DESC LIMIT {}", + HEADER_VALUES + 1 + ); + let values = select_strings(db, &sql, Some(name.to_lowercase()))?; + Ok(if values.len() as i64 > HEADER_VALUES { Vec::new() } else { values }) + } + } +} + +fn select_strings( + db: &crate::db::ProxyQueryManager, + sql: &str, + param: Option, +) -> Result, String> { + db.with_conn(|c| { + let conn = c.conn().resolve(); + let mut stmt = conn.prepare(sql).map_err(|e| e.to_string())?; + let params: Vec = param.into_iter().map(Value::from).collect(); + let rows = stmt + .query_map(rusqlite::params_from_iter(params), |r| r.get::<_, String>(0)) + .map_err(|e| e.to_string())?; + rows.collect::, _>>().map_err(|e| e.to_string()) + }) +} + +// -- Diagnostics -- + +/// What is wrong with a term, for the search box to mark. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_rpc.ts")] +pub enum QueryDiagnosticKind { + /// No field has this path. + UnknownField, + /// The value cannot apply to the field: a unit it does not take, a flag it does not have. + InvalidValue, + /// Searchable later; matches nothing for now. + NotYet, +} + +/// A term the query cannot run, which then matches nothing. `start` and `end` count UTF-16 +/// code units, as JavaScript does. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct QueryDiagnostic { + pub kind: QueryDiagnosticKind, + pub path: String, + pub start: u32, + pub end: u32, +} + +/// The terms of `text` that cannot run, in order. +pub fn diagnostics(text: &str) -> Vec { + let Some(root) = yaak_query::parse(text) else { + return Vec::new(); + }; + let mut issues = Vec::new(); + let prepared = prepare(&root, Utc::now(), &mut issues); + for diagnostic in schema().diagnostics(&prepared) { + issues.push(match diagnostic { + yaak_query::Diagnostic::UnknownField { path, span } => { + Issue { kind: QueryDiagnosticKind::UnknownField, path, span } + } + yaak_query::Diagnostic::InvalidValue { path, span } => { + Issue { kind: QueryDiagnosticKind::InvalidValue, path, span } + } + }); + } + issues.sort_by_key(|i| (i.span.start, i.span.end)); + let offsets = Utf16Offsets::new(text); + issues + .into_iter() + .map(|Issue { kind, path, span }| { + let span = offsets.span(span); + QueryDiagnostic { kind, path, start: span.start, end: span.end } + }) + .collect() +} + +// -- The values a flow is matched by -- + +/// The parts of a URL the fields read. A URL without a scheme, as an undecrypted connection's +/// may be, is all host. +struct UrlParts<'a> { + scheme: Option<&'a str>, + authority: &'a str, + path: Option<&'a str>, + query: Option<&'a str>, +} + +fn url_parts(url: &str) -> UrlParts<'_> { + let Some((scheme, rest)) = url.split_once("://") else { + return UrlParts { scheme: None, authority: url, path: None, query: None }; + }; + let end = rest.find(['/', '?', '#']).unwrap_or(rest.len()); + let (authority, rest) = rest.split_at(end); + let rest = rest.split('#').next().unwrap_or_default(); + let (path, query) = match rest.split_once('?') { + Some((path, query)) => (path, Some(query)), + None => (rest, None), + }; + UrlParts { scheme: Some(scheme), authority, path: Some(path), query } +} + +/// The authority less any user info. +fn host_port(authority: &str) -> &str { + authority.rsplit_once('@').map_or(authority, |(_, host)| host) +} + +/// The host, lowercased and without its port. +pub fn url_hostname(url: &str) -> String { + let host = host_port(url_parts(url).authority); + let host = if host.starts_with('[') { + host.find(']').map_or(host, |end| &host[..=end]) + } else { + host.rsplit_once(':').map_or(host, |(h, _)| h) + }; + host.to_lowercase() +} + +/// The host as the sidebar names it: with its port when that is not the scheme's. +pub fn url_host(url: &str) -> String { + sidebar_path(url).split('/').next().unwrap_or_default().to_string() +} + +/// The port the URL names, or its scheme's. +pub fn url_port(url: &str) -> Option { + let parts = url_parts(url); + let host = host_port(parts.authority); + let after_host = if host.starts_with('[') { + host.find(']').map(|end| &host[end + 1..]).unwrap_or_default() + } else { + host + }; + if let Some((_, port)) = after_host.rsplit_once(':') + && let Ok(port) = port.parse() + { + return Some(port); + } + match parts.scheme?.to_ascii_lowercase().as_str() { + "http" | "ws" => Some(80), + "https" | "wss" => Some(443), + _ => None, + } +} + +pub fn url_scheme(url: &str) -> Option { + url_parts(url).scheme.map(str::to_lowercase) +} + +/// The path, without the query or fragment. +pub fn url_path(url: &str) -> Option<&str> { + url_parts(url).path +} + +/// The path as the sidebar names it, its empty segments left out (`/v1//users` is +/// `/v1/users`), when that differs from the path, so a sidebar row's term finds it. +pub fn url_sidebar_path(url: &str) -> Option { + let path = url_path(url)?; + let segments: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect(); + let normalized = format!("/{}", segments.join("/")); + (normalized != path && !path.is_empty()).then_some(normalized) +} + +/// The query's parameters, decoded, their names lowercased. +pub fn url_query(url: &str) -> Vec<(String, String)> { + let Some(query) = url_parts(url).query else { + return Vec::new(); + }; + url::form_urlencoded::parse(query.as_bytes()) + .filter(|(name, _)| !name.is_empty()) + .map(|(name, value)| (name.to_lowercase(), value.into_owned())) + .collect() +} + +/// What bare words search: the URL, the method, the status and the content type. +pub fn flow_text( + url: &str, + method: &str, + status: Option, + content_type: Option<&str>, +) -> String { + let status = status.map(|s| s.to_string()).unwrap_or_default(); + format!("{url}\n{method}\n{status}\n{}", content_type.unwrap_or_default()) +} + +/// `h1` or `h2`, from the version the client spoke. +pub fn http_proto(version: &str) -> Option<&'static str> { + match version { + "HTTP/2" | "HTTP/2.0" => Some("h2"), + "HTTP/3" | "HTTP/3.0" => Some("h3"), + v if v.starts_with("HTTP/1") || v.starts_with("HTTP/0") => Some("h1"), + _ => None, + } +} + +/// `ws`, `sse` or `grpc`, from a stream's kind as stored. +pub fn stream_proto(kind: &str) -> Option<&'static str> { + match kind { + "websocket" => Some("ws"), + "sse" => Some("sse"), + "grpc" | "grpc_web" => Some("grpc"), + _ => None, + } +} + +/// The flags `@is:` finds on `flow`. +fn flags(flow: &Flow) -> Vec<&'static str> { + let stream = flow.stream_kind.map(|k| k.as_str()); + let set = |flag: &str| match flag { + "error" => flow.error.is_some() || flow.grpc_status.unwrap_or(0) != 0, + "pending" => { + flow.status.is_none() + && flow.error.is_none() + && !flow.cancelled + && flow.elapsed_ms.is_none() + } + "system" => is_system(flow.client_bundle_id.as_deref(), flow.client_path.as_deref()), + "tunnelled" => flow.tunnelled, + "saved" => flow.saved_request_id.is_some(), + "read" => flow.agent_read, + "loop" => flow.status == Some(508), + "ws" => stream == Some("websocket"), + "sse" => stream == Some("sse"), + "grpc" => matches!(stream, Some("grpc" | "grpc_web")), + _ => false, + }; + FLAGS.iter().copied().filter(|f| set(f)).collect() +} + +/// The response's content type, or the request's when it has none. +fn content_type(flow: &Flow) -> Option<&str> { + flow.content_type.as_deref().or_else(|| find_header(&flow.request_headers, "content-type")) +} + +/// `flow` as the evaluator sees it, with the values [`FlowQuery::sql`] reads in the database. +pub fn document(flow: &Flow) -> Document { + let mut fields: HashMap> = HashMap::new(); + let mut add = |path: &str, value: DocValue| { + fields.entry(path.to_lowercase()).or_default().push(value); + }; + let text = |s: &str| DocValue::Text(s.to_string()); + let num = |n: f64| DocValue::Number(n); + + add("host", DocValue::Text(url_hostname(&flow.url))); + add("host", DocValue::Text(url_host(&flow.url))); + if let Some(port) = url_port(&flow.url) { + add("port", num(port as f64)); + } + if let Some(scheme) = url_scheme(&flow.url) { + add("scheme", DocValue::Text(scheme)); + } + if let Some(path) = url_path(&flow.url) { + add("path", text(path)); + } + if let Some(path) = url_sidebar_path(&flow.url) { + add("path", DocValue::Text(path)); + } + add("url", text(&flow.url)); + for (name, value) in url_query(&flow.url) { + add(&format!("query.{name}"), DocValue::Text(value)); + } + add("method", text(&flow.method)); + if let Some(t) = content_type(flow) { + add("type", text(t)); + } + if let Some(p) = http_proto(&flow.client_http_version) { + add("proto", text(p)); + } + if let Some(p) = flow.stream_kind.and_then(|k| stream_proto(k.as_str())) { + add("proto", text(p)); + } + add("id", num(flow.number as f64)); + if let Some(status) = flow.status { + add("status", num(f64::from(status))); + } + add("size", num(flow.response_size as f64)); + add("res.size", num(flow.response_size as f64)); + add("req.size", num(flow.request_size as f64)); + if let Some(ms) = flow.elapsed_ms { + add("duration", num(f64::from(ms))); + } + add("time", num(started_ms(&flow.started_at))); + if let Some(app) = flow.client_bundle_id.as_deref().or(flow.client_path.as_deref()) { + add("app", text(app)); + } + if let Some(name) = &flow.client_name { + add("app", text(name)); + } + if flow.client_bundle_id.is_none() + && let Some(folder) = folder_name(flow.client_cwd.as_deref()) + { + add("app", text(folder)); + } + if let Some(cwd) = &flow.client_cwd { + add("cwd", text(cwd)); + if let Some(folder) = folder_name(Some(cwd)) { + add("cwd", text(folder)); + } + } + if let Some(pid) = flow.client_pid { + add("pid", num(f64::from(pid))); + } + for (side, headers) in [ + ("req", &flow.request_headers), + ("res", &flow.response_headers), + ] { + for ProxyHeader { name, value } in headers.iter().filter(|h| !h.name.is_empty()) { + add(&format!("header.{name}"), text(value)); + add(&format!("{side}.header.{name}"), text(value)); + } + } + for flag in flags(flow) { + add("is", text(flag)); + } + + Document { + text: flow_text( + &flow.url, + &flow.method, + flow.status.map(i64::from), + flow.content_type.as_deref(), + ), + fields: fields.into_iter().map(|(path, values)| (path, DocValues::Many(values))).collect(), + } +} + +/// When a flow started, in Unix milliseconds, fractions kept, as `@time` compares it. +fn started_ms(started_at: &NaiveDateTime) -> f64 { + let utc = started_at.and_utc(); + utc.timestamp() as f64 * 1e3 + f64::from(utc.timestamp_subsec_nanos()) / 1e6 +} + +/// Unix milliseconds as `started_at` is stored, so the database compares them as text in its +/// index's order: equal times are equal text, and later ones sort after. +fn stored_time(ms: f64) -> String { + let nanos = (ms * 1e6).round() as i64; + let dt = DateTime::::from_timestamp_nanos(nanos).naive_utc(); + dt.format("%Y-%m-%d %H:%M:%S%.f").to_string() +} + +// -- Compiling to SQL -- + +/// The columns of a database's flows table: one opened read-only before its migrations ran may +/// lack later ones, which then read as the model reads them, empty. +#[derive(Debug, Clone)] +pub struct FlowColumns(HashSet); + +impl FlowColumns { + pub fn of(conn: &rusqlite::Connection) -> Self { + let names = conn + .prepare("SELECT name FROM pragma_table_info('flows')") + .and_then(|mut stmt| { + stmt.query_map([], |r| r.get::<_, String>(0))?.collect::>() + }) + .unwrap_or_default(); + Self(names) + } + + /// `name`, or what the model reads in its place when the table lacks it. + fn col(&self, name: &str) -> String { + if self.0.contains(name) { + return name.to_string(); + } + match name { + "cancelled" | "tunnelled" | "agent_read" => "0".into(), + _ => "NULL".into(), + } + } +} + +struct Sql<'a> { + columns: &'a FlowColumns, + params: Vec, +} + +impl Sql<'_> { + fn param(&mut self, value: impl Into) -> &'static str { + self.params.push(value.into()); + "?" + } + + fn node(&mut self, node: &Node) -> String { + match node { + Node::And { children, .. } => self.join(children, " AND ", "1"), + Node::Or { children, .. } => self.join(children, " OR ", "0"), + Node::Not { node, .. } => format!("NOT ({})", self.node(node)), + Node::Term { text, .. } => self.free_text(text.trim().to_lowercase()), + Node::Phrase { text, .. } => self.free_text(text.to_lowercase()), + Node::Field { path, value, .. } => { + let Some(field) = schema().resolve(path) else { + return "0".into(); + }; + let condition = Condition::new(&field.def.kind, value); + if condition == Condition::Never { + return "0".into(); + } + self.field(&field.def.name, field.key.as_deref(), &condition) + } + } + } + + fn join(&mut self, children: &[Node], op: &str, empty: &str) -> String { + if children.is_empty() { + return empty.into(); + } + let parts: Vec = children.iter().map(|c| format!("({})", self.node(c))).collect(); + parts.join(op) + } + + fn free_text(&mut self, needle: String) -> String { + if needle.is_empty() { + return "0".into(); + } + let p = self.param(needle); + format!("coalesce(q_contains(flow_text(url, method, status, content_type), {p}), 0)") + } + + fn field(&mut self, name: &str, key: Option<&str>, condition: &Condition) -> String { + let c = |n: &str| self.columns.col(n); + let (values, numeric): (Vec, bool) = match name { + "host" => (vec!["url_hostname(url)".into(), "url_host(url)".into()], false), + "port" => (vec!["url_port(url)".into()], true), + "scheme" => (vec!["url_scheme(url)".into()], false), + "path" => (vec!["url_path(url)".into(), "url_sidebar_path(url)".into()], false), + "url" => (vec!["url".into()], false), + "method" => (vec!["method".into()], false), + "type" => ( + vec![ + "coalesce(content_type, header_value(request_headers, 'content-type'))".into(), + ], + false, + ), + "proto" => ( + vec![ + "http_proto(client_http_version)".into(), + format!("stream_proto({})", c("stream_kind")), + ], + false, + ), + "id" => (vec!["flows.rowid".into()], true), + "status" => (vec!["status".into()], true), + "size" | "res.size" => (vec!["response_size".into()], true), + "req.size" => (vec!["request_size".into()], true), + "duration" => (vec!["elapsed_ms".into()], true), + "pid" => (vec![c("client_pid")], true), + "app" => { + let (bundle, path, name) = + (c("client_bundle_id"), c("client_path"), c("client_name")); + let folder = + format!("CASE WHEN {bundle} IS NULL THEN folder_name({}) END", c("client_cwd")); + (vec![format!("coalesce({bundle}, {path})"), name, folder], false) + } + "cwd" => { + let cwd = c("client_cwd"); + (vec![cwd.clone(), format!("folder_name({cwd})")], false) + } + "time" => return self.time(condition), + "is" => return self.flags(condition), + "header" => { + let both = [ + self.pairs("request_headers", key, condition), + self.pairs("response_headers", key, condition), + ]; + return format!("({} OR {})", both[0], both[1]); + } + "req.header" => return self.pairs("request_headers", key, condition), + "res.header" => return self.pairs("response_headers", key, condition), + "query" => return self.pairs("url_query_json(url)", key, condition), + _ => return "0".into(), + }; + self.any_value(&values, numeric, condition) + } + + /// Any of the values passes. + fn any_value(&mut self, values: &[String], numeric: bool, condition: &Condition) -> String { + let parts: Vec = values.iter().map(|v| self.test(v, numeric, condition)).collect(); + if parts.len() == 1 { + parts.into_iter().next().unwrap_or_default() + } else { + format!("({})", parts.join(" OR ")) + } + } + + /// `condition` on one value, never NULL. `numeric` for a value stored as a number, compared + /// without reading it as one first. + fn test(&mut self, value: &str, numeric: bool, condition: &Condition) -> String { + let as_number = if numeric { value.to_string() } else { format!("q_number({value})") }; + let test = match condition { + Condition::Contains(needle) => { + format!("q_contains({value}, {})", self.param(needle.clone())) + } + Condition::Wildcard(pattern) => { + format!("q_wildcard({value}, {})", self.param(pattern.clone())) + } + Condition::Present => format!("q_present({value})"), + Condition::IsNumber => format!("{as_number} IS NOT NULL"), + Condition::Equals(n) => format!("{as_number} = {}", self.param(*n)), + Condition::Compare(op, n) => format!("{as_number} {} {}", op.as_str(), self.param(*n)), + Condition::Range(low, high) => { + format!("{as_number} BETWEEN {} AND {}", self.param(*low), self.param(*high)) + } + // The proxy has no flag fields; `@is` is text. + Condition::Flag(_) | Condition::Never => return "0".into(), + }; + format!("coalesce({test}, 0)") + } + + /// `@time`, prepared into comparisons of Unix milliseconds, compared with `started_at` as + /// stored, through its index. + fn time(&mut self, condition: &Condition) -> String { + match condition { + Condition::Present | Condition::IsNumber => "1".into(), + Condition::Equals(ms) => format!("started_at = {}", self.param(stored_time(*ms))), + Condition::Compare(op, ms) => { + format!("started_at {} {}", op.as_str(), self.param(stored_time(*ms))) + } + Condition::Range(low, high) => format!( + "started_at BETWEEN {} AND {}", + self.param(stored_time(*low)), + self.param(stored_time(*high)) + ), + // Preparing leaves `@time` nothing else. + _ => "0".into(), + } + } + + /// A flag's condition on the flow's columns. + fn flag(&self, flag: &str) -> String { + let c = |n: &str| self.columns.col(n); + match flag { + "error" => format!("(error IS NOT NULL OR coalesce({}, 0) != 0)", c("grpc_status")), + "pending" => format!( + "(status IS NULL AND error IS NULL AND NOT {} AND elapsed_ms IS NULL)", + c("cancelled") + ), + "system" => format!("is_system({}, {})", c("client_bundle_id"), c("client_path")), + "tunnelled" => c("tunnelled"), + "saved" => format!("{} IS NOT NULL", c("saved_request_id")), + "read" => c("agent_read"), + "loop" => "status = 508".into(), + "ws" => format!("{} = 'websocket'", c("stream_kind")), + "sse" => format!("{} = 'sse'", c("stream_kind")), + "grpc" => format!("{} IN ('grpc', 'grpc_web')", c("stream_kind")), + _ => "0".into(), + } + } + + /// `@is`: each flag set is a value, its name. A flag named whole is its column condition. + fn flags(&mut self, condition: &Condition) -> String { + if let Condition::Wildcard(pattern) = condition + && !pattern.contains('*') + { + return match FLAGS.contains(&pattern.as_str()) { + true => format!("coalesce({}, 0)", self.flag(pattern)), + false => "0".into(), + }; + } + let values: Vec = FLAGS + .iter() + .map(|f| format!("CASE WHEN coalesce({}, 0) THEN '{f}' END", self.flag(f))) + .collect(); + self.any_value(&values, false, condition) + } + + /// A map's values in a JSON array of `{ name, value }`: under `key` when given, any + /// otherwise. + fn pairs(&mut self, json: &str, key: Option<&str>, condition: &Condition) -> String { + let name = match key { + Some(key) => { + format!( + "q_lower(json_extract(kv.value, '$.name')) = {} AND ", + self.param(key.to_lowercase()) + ) + } + None => String::new(), + }; + let test = self.test("json_extract(kv.value, '$.value')", false, condition); + format!("EXISTS (SELECT 1 FROM json_each({json}) AS kv WHERE {name}{test})") + } +} + +// -- SQL functions -- + +/// A value as the evaluator reads text from it: text as it is, a number as `200`. +fn value_text(value: ValueRef<'_>) -> Option> { + match value { + ValueRef::Text(t) => Some(String::from_utf8_lossy(t)), + ValueRef::Integer(i) => Some(format_number(i as f64).into()), + ValueRef::Real(f) => Some(format_number(f).into()), + ValueRef::Null | ValueRef::Blob(_) => None, + } +} + +/// Adds the functions [`FlowQuery::sql`] compiles to: the value tests of the crate's +/// conditions, and the values fields read from a flow's columns. +pub(crate) fn add_functions(conn: &rusqlite::Connection) -> rusqlite::Result<()> { + let flags = FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC; + let text_arg = |ctx: &rusqlite::functions::Context, i| ctx.get::>(i); + + conn.create_scalar_function("q_contains", 2, flags, |ctx| { + let needle = ctx.get::(1)?; + Ok(match ctx.get_raw(0) { + ValueRef::Text(t) => contains_ignore_case(&String::from_utf8_lossy(t), &needle), + v => value_text(v).is_some_and(|t| t.contains(needle.as_str())), + }) + })?; + conn.create_scalar_function("q_wildcard", 2, flags, |ctx| { + let pattern = ctx.get::(1)?; + Ok(match ctx.get_raw(0) { + ValueRef::Text(t) => { + wildcard_match(&pattern, &String::from_utf8_lossy(t).to_lowercase()) + } + v => value_text(v).is_some_and(|t| wildcard_match(&pattern, &t)), + }) + })?; + conn.create_scalar_function("q_present", 1, flags, |ctx| { + Ok(match ctx.get_raw(0) { + ValueRef::Text(t) => !t.is_empty(), + ValueRef::Integer(_) | ValueRef::Real(_) => true, + ValueRef::Null | ValueRef::Blob(_) => false, + }) + })?; + conn.create_scalar_function("q_number", 1, flags, |ctx| { + Ok(match ctx.get_raw(0) { + ValueRef::Integer(i) => Some(i as f64), + ValueRef::Real(f) => Some(f), + ValueRef::Text(t) => read_number(&String::from_utf8_lossy(t)), + ValueRef::Null | ValueRef::Blob(_) => None, + }) + })?; + conn.create_scalar_function("q_lower", 1, flags, move |ctx| { + Ok(text_arg(ctx, 0)?.map(|t| t.to_lowercase())) + })?; + conn.create_scalar_function("flow_text", 4, flags, move |ctx| { + let url = text_arg(ctx, 0)?.unwrap_or_default(); + let method = text_arg(ctx, 1)?.unwrap_or_default(); + let content_type = text_arg(ctx, 3)?; + Ok(flow_text(&url, &method, ctx.get::>(2)?, content_type.as_deref())) + })?; + conn.create_scalar_function("url_hostname", 1, flags, move |ctx| { + Ok(url_hostname(&text_arg(ctx, 0)?.unwrap_or_default())) + })?; + conn.create_scalar_function("url_host", 1, flags, move |ctx| { + Ok(url_host(&text_arg(ctx, 0)?.unwrap_or_default())) + })?; + conn.create_scalar_function("url_port", 1, flags, move |ctx| { + Ok(url_port(&text_arg(ctx, 0)?.unwrap_or_default())) + })?; + conn.create_scalar_function("url_scheme", 1, flags, move |ctx| { + Ok(url_scheme(&text_arg(ctx, 0)?.unwrap_or_default())) + })?; + conn.create_scalar_function("url_path", 1, flags, move |ctx| { + Ok(url_path(&text_arg(ctx, 0)?.unwrap_or_default()).map(str::to_string)) + })?; + conn.create_scalar_function("url_sidebar_path", 1, flags, move |ctx| { + Ok(url_sidebar_path(&text_arg(ctx, 0)?.unwrap_or_default())) + })?; + conn.create_scalar_function("url_query_json", 1, flags, move |ctx| { + let pairs: Vec = url_query(&text_arg(ctx, 0)?.unwrap_or_default()) + .into_iter() + .map(|(name, value)| ProxyHeader { name, value }) + .collect(); + serde_json::to_string(&pairs).map_err(|e| rusqlite::Error::UserFunctionError(e.into())) + })?; + conn.create_scalar_function("header_value", 2, flags, move |ctx| { + let headers: Vec = + serde_json::from_str(&text_arg(ctx, 0)?.unwrap_or_default()).unwrap_or_default(); + let name = ctx.get::(1)?; + Ok(find_header(&headers, &name).map(str::to_string)) + })?; + conn.create_scalar_function("http_proto", 1, flags, move |ctx| { + Ok(text_arg(ctx, 0)?.and_then(|v| http_proto(&v))) + })?; + conn.create_scalar_function("stream_proto", 1, flags, move |ctx| { + Ok(text_arg(ctx, 0)?.and_then(|v| stream_proto(&v))) + }) +} + +#[cfg(test)] +mod tests; diff --git a/crates-proxy/yaak-proxy-lib/src/query/tests.rs b/crates-proxy/yaak-proxy-lib/src/query/tests.rs new file mode 100644 index 000000000..499ecf52a --- /dev/null +++ b/crates-proxy/yaak-proxy-lib/src/query/tests.rs @@ -0,0 +1,1001 @@ +//! The compiler's tests. Each term kind against flows made for it, then the drift guard: for a +//! sample of varied flows and a battery of queries, the database's answer and the evaluator's +//! must be the same flows. + +use super::*; +use crate::db::ProxyQueryManager; +use crate::models::StreamKind; +use chrono::Duration; +use yaak_database::UpdateSource; + +/// When the tests run: every flow's start and every `@time` counts from here. +fn now() -> DateTime { + DateTime::parse_from_rfc3339("2026-11-07T12:00:00Z").unwrap().with_timezone(&Utc) +} + +fn header(name: &str, value: &str) -> ProxyHeader { + ProxyHeader { name: name.into(), value: value.into() } +} + +/// A database holding `flows`, numbered in order, with what they read back as. +fn db_of(flows: Vec) -> (tempfile::TempDir, ProxyQueryManager, Vec) { + let dir = tempfile::tempdir().unwrap(); + let db = ProxyQueryManager::new(&dir.path().join("proxy.db"), &crate::db::test_key()).unwrap(); + let mut stored = Vec::new(); + for flow in flows { + let read = flow.agent_read; + let (mut saved, _) = db.with_tx(|tx| tx.upsert(&flow, &UpdateSource::Background)).unwrap(); + if read { + db.with_tx(|tx| { + tx.conn() + .resolve() + .execute("UPDATE flows SET agent_read = TRUE WHERE id = ?", [&saved.id]) + .map_err(yaak_database::Error::from) + }) + .unwrap(); + saved.agent_read = true; + } + stored.push(saved); + } + (dir, db, stored) +} + +/// The ids the database finds for `query`, sorted. +fn sql_ids(db: &ProxyQueryManager, query: &FlowQuery) -> Vec { + db.with_conn(|c| { + let conn = c.conn().resolve(); + let (condition, params) = query.sql(&FlowColumns::of(conn)); + let sql = format!("SELECT id FROM flows WHERE {condition} ORDER BY id"); + let mut stmt = conn.prepare(&sql).unwrap_or_else(|e| panic!("{e}: {sql}")); + stmt.query_map(rusqlite::params_from_iter(params), |r| r.get(0)) + .unwrap() + .collect::>>() + .unwrap() + }) +} + +/// The ids the evaluator finds for `query`, sorted. +fn memory_ids(flows: &[Flow], query: &FlowQuery) -> Vec { + let matcher = query.matcher(); + let mut ids: Vec = + flows.iter().filter(|f| matcher.matches(&document(f))).map(|f| f.id.clone()).collect(); + ids.sort(); + ids +} + +/// Flows made for the tests below, each named by what sets it apart. +fn fixture() -> Vec { + let at = |minutes_ago: i64| (now() - Duration::minutes(minutes_ago)).naive_utc(); + let base = |id: &str, url: &str| Flow { + id: id.into(), + url: url.into(), + method: "GET".into(), + client_http_version: "HTTP/1.1".into(), + started_at: at(1), + ..Default::default() + }; + vec![ + Flow { + status: Some(200), + content_type: Some("application/json; charset=utf-8".into()), + request_headers: vec![header("Accept", "application/json")], + response_headers: vec![ + header("X-Cache", "HIT"), + header("Content-Type", "application/json"), + ], + response_size: 250_000, + request_size: 120, + elapsed_ms: Some(820), + client_http_version: "HTTP/2".into(), + client_name: Some("Google Chrome".into()), + client_bundle_id: Some("com.google.Chrome".into()), + client_path: Some("/Applications/Google Chrome.app".into()), + client_pid: Some(501), + saved_request_id: Some("rq_1".into()), + started_at: at(2), + ..base("fl_github", "https://api.github.com/repos/yaak?page=2&sort=Stars") + }, + Flow { + method: "POST".into(), + status: Some(404), + content_type: Some("text/html".into()), + request_headers: vec![header("Content-Type", "application/x-www-form-urlencoded")], + response_headers: vec![header("x-cache", "MISS")], + response_size: 900, + elapsed_ms: Some(40), + client_name: Some("curl".into()), + client_path: Some("/usr/bin/curl".into()), + client_cwd: Some("/Users/ada/src/fieldwork".into()), + started_at: at(30), + ..base("fl_stripe", "http://api.stripe.com:8080/v1/charges/ch_1") + }, + Flow { + status: Some(503), + error: None, + request_headers: vec![header("content-type", "application/json")], + response_size: 2_000_000, + elapsed_ms: Some(3_000), + client_name: Some("nsurlsessiond".into()), + client_path: Some("/usr/libexec/nsurlsessiond".into()), + client_cwd: Some("/".into()), + started_at: at(90), + ..base("fl_daemon", "https://github.com/login") + }, + Flow { + method: "CONNECT".into(), + tunnelled: true, + response_size: 5_000, + elapsed_ms: Some(10_000), + started_at: at(60 * 25), + ..base("fl_tunnel", "https://bank.example.com") + }, + Flow { + error: Some("connection refused".into()), + started_at: at(3), + ..base("fl_error", "http://localhost:3000/health") + }, + Flow { + started_at: at(0), + agent_read: true, + ..base("fl_pending", "https://example.com/slow") + }, + Flow { + status: Some(101), + stream_kind: Some(StreamKind::Websocket), + elapsed_ms: Some(50), + ..base("fl_ws", "wss://ws.example.com/socket") + }, + Flow { + method: "POST".into(), + status: Some(200), + stream_kind: Some(StreamKind::Grpc), + grpc_status: Some(5), + client_http_version: "HTTP/2".into(), + elapsed_ms: Some(12), + ..base("fl_grpc", "https://grpc.example.com/pkg.Service/Method") + }, + Flow { status: Some(508), elapsed_ms: Some(1), ..base("fl_loop", "http://neverssl.com/") }, + ] +} + +/// The fixture's ids `query` finds, after checking the database and the evaluator agree. +fn finds(query: &str) -> Vec { + finds_with(query, false) +} + +fn finds_with(query: &str, hide_system: bool) -> Vec { + let (_dir, db, flows) = db_of(fixture()); + let prepared = FlowQuery::at(query, hide_system, now()); + let sql = sql_ids(&db, &prepared); + assert_eq!(sql, memory_ids(&flows, &prepared), "the two disagree on {query:?}"); + sql +} + +fn ids(ids: &[&str]) -> Vec { + let mut ids: Vec = ids.iter().map(|id| id.to_string()).collect(); + ids.sort(); + ids +} + +#[test] +fn bare_words_search_url_method_status_and_type() { + assert_eq!(finds("GITHUB"), ids(&["fl_daemon", "fl_github"])); + assert_eq!(finds("post"), ids(&["fl_grpc", "fl_stripe"])); + assert_eq!(finds("404"), ids(&["fl_stripe"])); + assert_eq!(finds("html"), ids(&["fl_stripe"])); + assert_eq!(finds("\"api.github.com/repos\""), ids(&["fl_github"])); + assert_eq!(finds("github repos"), ids(&["fl_github"])); + // Headers are not free text + assert_eq!(finds("HIT"), ids(&[])); +} + +#[test] +fn url_fields() { + assert_eq!(finds("@host:api.github.com"), ids(&["fl_github"])); + assert_eq!(finds("@host:github"), ids(&[])); + assert_eq!(finds("@host:*github*"), ids(&["fl_daemon", "fl_github"])); + // A site is its domain and every host below it + assert_eq!(finds("@host:*.github.com"), ids(&["fl_daemon", "fl_github"])); + assert_eq!(finds("@host:*.example.com"), ids(&["fl_grpc", "fl_pending", "fl_tunnel", "fl_ws"])); + // A host on another port, with or without it + assert_eq!(finds("@host:api.stripe.com"), ids(&["fl_stripe"])); + assert_eq!(finds("@host:api.stripe.com:8080"), ids(&["fl_stripe"])); + assert_eq!(finds("@port:8080"), ids(&["fl_stripe"])); + assert_eq!(finds("@port:443 @scheme:wss"), ids(&["fl_ws"])); + assert_eq!(finds("@port:<1000 @scheme:http"), ids(&["fl_loop"])); + assert_eq!(finds("@scheme:http"), ids(&["fl_error", "fl_loop", "fl_stripe"])); + assert_eq!(finds("@path:/v1"), ids(&["fl_stripe"])); + assert_eq!(finds("@path:/v1/*"), ids(&["fl_stripe"])); + assert_eq!(finds("@path:/v1/charges/*"), ids(&["fl_stripe"])); + assert_eq!(finds("@path:/v1/charges/ch_1/*"), ids(&["fl_stripe"])); + assert_eq!(finds("@path:/v1/charge/*"), ids(&[])); + assert_eq!( + finds("@path:/"), + ids(&[ + "fl_daemon", + "fl_error", + "fl_github", + "fl_grpc", + "fl_loop", + "fl_pending", + "fl_stripe", + "fl_ws" + ]) + ); + assert_eq!(finds("@url:*page=2*"), ids(&["fl_github"])); + assert_eq!(finds("@query.page:2"), ids(&["fl_github"])); + assert_eq!(finds("@query.SORT:stars"), ids(&["fl_github"])); + assert_eq!(finds("@query.page:*"), ids(&["fl_github"])); + assert_eq!(finds("@query.page:>1"), ids(&["fl_github"])); + assert_eq!(finds("@query:stars"), ids(&["fl_github"])); + assert_eq!( + finds("-@query.page:*"), + ids(&[ + "fl_daemon", + "fl_error", + "fl_grpc", + "fl_loop", + "fl_pending", + "fl_stripe", + "fl_tunnel", + "fl_ws" + ]) + ); +} + +#[test] +fn request_and_response_fields() { + assert_eq!(finds("@method:post"), ids(&["fl_grpc", "fl_stripe"])); + assert_eq!(finds("@method:CONN*"), ids(&["fl_tunnel"])); + assert_eq!(finds("@type:json"), ids(&["fl_daemon", "fl_github"])); + // The request's only when the response has none + assert_eq!(finds("@type:form"), ids(&[])); + assert_eq!(finds("@proto:h2"), ids(&["fl_github", "fl_grpc"])); + assert_eq!(finds("@proto:ws"), ids(&["fl_ws"])); + assert_eq!(finds("@proto:grpc"), ids(&["fl_grpc"])); + assert_eq!(finds("@proto:h1 @method:post"), ids(&["fl_stripe"])); + assert_eq!(finds("@id:2"), ids(&["fl_stripe"])); + assert_eq!(finds("@id:>=8"), ids(&["fl_grpc", "fl_loop"])); +} + +#[test] +fn statuses() { + assert_eq!(finds("@status:404"), ids(&["fl_stripe"])); + assert_eq!(finds("@status:4xx"), ids(&["fl_stripe"])); + assert_eq!(finds("@status:5XX"), ids(&["fl_daemon", "fl_loop"])); + assert_eq!(finds("@status:>=400"), ids(&["fl_daemon", "fl_loop", "fl_stripe"])); + assert_eq!(finds("@status:[200 TO 299]"), ids(&["fl_github", "fl_grpc"])); + assert_eq!(finds("@status:50*"), ids(&["fl_daemon", "fl_loop"])); + assert_eq!( + finds("@status:*"), + ids(&[ + "fl_daemon", + "fl_github", + "fl_grpc", + "fl_loop", + "fl_stripe", + "fl_ws" + ]) + ); + assert_eq!(finds("-@status:*"), ids(&["fl_error", "fl_pending", "fl_tunnel"])); + assert_eq!(finds("@status:40"), ids(&[])); +} + +#[test] +fn sizes_and_durations_with_units() { + assert_eq!(finds("@size:>100kb"), ids(&["fl_daemon", "fl_github"])); + assert_eq!(finds("@size:>1mb"), ids(&["fl_daemon"])); + assert_eq!(finds("@res.size:[1kb TO 10kb]"), ids(&["fl_tunnel"])); + assert_eq!(finds("@req.size:120b"), ids(&["fl_github"])); + assert_eq!(finds("@size:>1.5gb"), ids(&[])); + assert_eq!(finds("@duration:>500ms"), ids(&["fl_daemon", "fl_github", "fl_tunnel"])); + assert_eq!(finds("@duration:>=3s"), ids(&["fl_daemon", "fl_tunnel"])); + assert_eq!(finds("@duration:<1m @duration:>1s"), ids(&["fl_daemon", "fl_tunnel"])); + // A unit the field does not take matches nothing + assert_eq!(finds("@duration:>1h"), ids(&[])); + assert_eq!(finds("-@duration:*"), ids(&["fl_error", "fl_pending"])); +} + +#[test] +fn times_ago_and_absolute() { + assert_eq!( + finds("@time:<5m"), + ids(&[ + "fl_error", + "fl_github", + "fl_grpc", + "fl_loop", + "fl_pending", + "fl_ws" + ]) + ); + assert_eq!(finds("@time:<=30m @time:>5m"), ids(&["fl_stripe"])); + assert_eq!(finds("@time:>1h"), ids(&["fl_daemon", "fl_tunnel"])); + assert_eq!(finds("@time:>1d"), ids(&["fl_tunnel"])); + assert_eq!(finds("@time:[1h TO 2h]"), ids(&["fl_daemon"])); + assert_eq!(finds("@time:<1w"), finds("@time:*")); + // An instant with its offset + assert_eq!(finds("@time:<2026-11-07T11:00:00Z"), ids(&["fl_daemon", "fl_tunnel"])); + assert_eq!( + finds("@time:>=2026-11-07T11:30:00Z @time:<2026-11-07T11:58:00Z"), + ids(&["fl_error", "fl_stripe"]) + ); + // Local: the day the fixture's now falls on holds all but the tunnel + let today = now().with_timezone(&Local).format("%Y-%m-%d").to_string(); + let yesterday_ish = finds(&format!("@time:<{today}")); + let on_day = finds(&format!("@time:{today}")); + assert_eq!(yesterday_ish.len() + on_day.len(), fixture().len(), "{yesterday_ish:?} {on_day:?}"); + assert!(yesterday_ish.contains(&"fl_tunnel".to_string())); + // A time today, local: the minute two minutes before now holds the GitHub flow + let minute = (now() - Duration::minutes(2)).with_timezone(&Local).format("%H:%M").to_string(); + let by_minute = finds(&format!("@time:{minute}")); + if (now() - Duration::minutes(2)).with_timezone(&Local).date_naive() + == now().with_timezone(&Local).date_naive() + { + assert_eq!(by_minute, ids(&["fl_github"])); + } + // A time that is none of these matches nothing + assert_eq!(finds("@time:soon"), ids(&[])); + assert_eq!(finds("@time:5m"), ids(&[])); +} + +#[test] +fn sources() { + assert_eq!(finds("@app:\"google chrome\""), ids(&["fl_github"])); + assert_eq!(finds("@app:com.google.Chrome"), ids(&["fl_github"])); + assert_eq!(finds("@app:chrome"), ids(&[])); + assert_eq!(finds("@app:*chrome*"), ids(&["fl_github"])); + // A program outside an app, by its name, its path or its folder + assert_eq!(finds("@app:curl"), ids(&["fl_stripe"])); + assert_eq!(finds("@app:/usr/bin/curl"), ids(&["fl_stripe"])); + assert_eq!(finds("@app:FIELDWORK"), ids(&["fl_stripe"])); + assert_eq!(finds("@cwd:fieldwork"), ids(&["fl_stripe"])); + assert_eq!(finds("@cwd:/Users/ada/src/fieldwork"), ids(&["fl_stripe"])); + assert_eq!(finds("@cwd:*"), ids(&["fl_daemon", "fl_stripe"])); + assert_eq!(finds("@pid:501"), ids(&["fl_github"])); +} + +#[test] +fn headers_on_either_side_or_one() { + assert_eq!(finds("@header.x-cache:hit"), ids(&["fl_github"])); + assert_eq!(finds("@header.X-CACHE:*"), ids(&["fl_github", "fl_stripe"])); + assert_eq!(finds("@res.header.x-cache:*"), ids(&["fl_github", "fl_stripe"])); + assert_eq!(finds("@req.header.x-cache:*"), ids(&[])); + assert_eq!(finds("@header.content-type:json"), ids(&["fl_daemon", "fl_github"])); + assert_eq!(finds("@req.header.content-type:json"), ids(&["fl_daemon"])); + assert_eq!(finds("@res.header.content-type:json"), ids(&["fl_github"])); + assert_eq!(finds("@header.accept:application/*"), ids(&["fl_github"])); + assert_eq!(finds("@header:miss"), ids(&["fl_stripe"])); + assert_eq!( + finds("-@header.x-cache:hit"), + ids(&[ + "fl_daemon", + "fl_error", + "fl_grpc", + "fl_loop", + "fl_pending", + "fl_stripe", + "fl_tunnel", + "fl_ws" + ]) + ); +} + +#[test] +fn flags() { + assert_eq!(finds("@is:system"), ids(&["fl_daemon"])); + assert_eq!(finds("@is:tunnelled"), ids(&["fl_tunnel"])); + assert_eq!(finds("@is:error"), ids(&["fl_error", "fl_grpc"])); + assert_eq!(finds("@is:pending"), ids(&["fl_pending"])); + assert_eq!(finds("@is:saved"), ids(&["fl_github"])); + assert_eq!(finds("@is:read"), ids(&["fl_pending"])); + assert_eq!(finds("@is:loop"), ids(&["fl_loop"])); + assert_eq!(finds("@is:ws"), ids(&["fl_ws"])); + assert_eq!(finds("@is:sse"), ids(&[])); + assert_eq!(finds("@is:GRPC"), ids(&["fl_grpc"])); + assert_eq!( + finds("@is:*"), + ids(&[ + "fl_daemon", + "fl_error", + "fl_github", + "fl_grpc", + "fl_loop", + "fl_pending", + "fl_tunnel", + "fl_ws" + ]) + ); + assert_eq!(finds("@is:s*"), ids(&["fl_daemon", "fl_github"])); + // Not yet, and not a flag + assert_eq!(finds("@is:marked"), ids(&[])); + assert_eq!(finds("@is:nope"), ids(&[])); + assert_eq!(finds("-@is:nope").len(), fixture().len()); +} + +#[test] +fn groups_and_negation() { + assert_eq!(finds("@status:4xx OR @is:tunnelled"), ids(&["fl_stripe", "fl_tunnel"])); + assert_eq!(finds("(@status:4xx OR @is:tunnelled) -@method:connect"), ids(&["fl_stripe"])); + assert_eq!(finds("-(@status:* OR @is:pending)"), ids(&["fl_error", "fl_tunnel"])); + assert_eq!(finds("github -@host:github.com"), ids(&["fl_github"])); + assert_eq!( + finds("@host:*.example.com AND -@is:ws"), + ids(&["fl_grpc", "fl_pending", "fl_tunnel"]) + ); +} + +#[test] +fn bodies_and_unknown_fields_match_nothing() { + assert_eq!(finds("@body:x"), ids(&[])); + assert_eq!(finds("@res.body:*"), ids(&[])); + assert_eq!(finds("@nope:x"), ids(&[])); + assert_eq!(finds("-@body:x").len(), fixture().len()); +} + +#[test] +fn hiding_system() { + let all = fixture().len(); + assert_eq!(finds_with("", true).len(), all - 1); + assert!(!finds_with("", true).contains(&"fl_daemon".to_string())); + assert_eq!(finds_with("github", true), ids(&["fl_github"])); + // The search names System itself, either way + assert_eq!(finds_with("@is:system", true), ids(&["fl_daemon"])); + assert_eq!(finds_with("github OR @is:System", true), ids(&["fl_daemon", "fl_github"])); + assert_eq!(finds_with("-@is:system", true).len(), all - 1); + assert!(FlowQuery::new("", true).matcher().matches(&document(&fixture()[0]))); + assert!(!FlowQuery::new("", true).is_empty()); + assert!(FlowQuery::new(" ", false).is_empty()); +} + +#[test] +fn diagnostics_mark_what_cannot_run() { + let kinds = + |q: &str| diagnostics(q).into_iter().map(|d| (d.kind, d.start, d.end)).collect::>(); + use QueryDiagnosticKind::*; + assert_eq!(kinds("@status:4xx @time:<5m @host:*.a.com"), vec![]); + assert_eq!(kinds("@nope:x"), vec![(UnknownField, 0, 6)]); + assert_eq!(kinds("@size:>5parsecs"), vec![(InvalidValue, 6, 15)]); + assert_eq!(kinds("@time:soon"), vec![(InvalidValue, 6, 10)]); + assert_eq!(kinds("@is:nope"), vec![(InvalidValue, 4, 8)]); + assert_eq!(kinds("@is:marked"), vec![(NotYet, 0, 10)]); + assert_eq!(kinds("x @body:token"), vec![(NotYet, 2, 13)]); + // Spans count UTF-16 units + assert_eq!(kinds("é😀 @nope:x"), vec![(UnknownField, 4, 10)]); + // Typing a field is not yet a mistake + assert_eq!(kinds("@is:"), vec![]); + assert_eq!(kinds("@time:"), vec![]); +} + +#[test] +fn time_is_compared_as_stored() { + // The database compares `started_at` as text: what this writes must be how it is stored. + let (_dir, db, flows) = db_of(fixture()); + let stored: Vec = db.with_conn(|c| { + let conn = c.conn().resolve(); + let mut stmt = conn.prepare("SELECT started_at FROM flows ORDER BY rowid").unwrap(); + stmt.query_map([], |r| r.get(0)).unwrap().map(|r| r.unwrap()).collect() + }); + for (flow, stored) in flows.iter().zip(stored) { + assert_eq!(stored_time(started_ms(&flow.started_at)), stored); + } + let precise = + NaiveDateTime::parse_from_str("2026-11-07 10:00:00.123456", "%Y-%m-%d %H:%M:%S%.f") + .unwrap(); + assert_eq!(stored_time(started_ms(&precise)), "2026-11-07 10:00:00.123456"); +} + +#[test] +fn reads_url_parts() { + assert_eq!(url_hostname("https://user:pw@API.Example.com:8443/x"), "api.example.com"); + assert_eq!(url_host("https://API.Example.com:8443/x"), "api.example.com:8443"); + assert_eq!(url_hostname("http://[::1]:8080/"), "[::1]"); + assert_eq!(url_port("http://[::1]:8080/"), Some(8080)); + assert_eq!(url_port("https://a.com/"), Some(443)); + assert_eq!(url_port("bank.example.com:443"), Some(443)); + assert_eq!(url_scheme("bank.example.com:443"), None); + assert_eq!(url_path("https://a.com/x/y?q=1#f"), Some("/x/y")); + assert_eq!(url_path("https://a.com?q=1"), Some("")); + assert_eq!(url_sidebar_path("https://a.com/v1//users/"), Some("/v1/users".into())); + assert_eq!(url_sidebar_path("https://a.com/v1/users"), None); + assert_eq!(url_sidebar_path("https://a.com"), None); + assert_eq!( + url_query("https://a.com/?A=1&b=%20two&a=3#f"), + vec![ + ("a".to_string(), "1".to_string()), + ("b".to_string(), " two".to_string()), + ("a".to_string(), "3".to_string()), + ] + ); +} + +// -- The drift guard -- + +/// A small deterministic generator, so a failure names a flow that comes back every run. +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + self.0 >> 33 + } + + fn pick<'a, T>(&mut self, items: &'a [T]) -> &'a T { + &items[self.next() as usize % items.len()] + } + + fn chance(&mut self, percent: u64) -> bool { + self.next() % 100 < percent + } +} + +/// Flows as varied as the fields can tell apart: hosts with and without ports and schemes, +/// queries and their encodings, any case, text outside ASCII, every flag, and the empty cases. +fn sample(count: usize) -> Vec { + let mut rng = Rng(7); + let hosts = [ + "https://api.github.com", + "https://github.com", + "http://API.Stripe.com:8080", + "https://api.stripe.community", + "http://localhost:3000", + "https://[::1]:8443", + "wss://ws.example.com", + "https://café.example.org", + "http://neverssl.com:80", + "https://user:pw@intranet.local", + "bank.example.com:443", + ]; + let paths = [ + "", + "/", + "/v1/charges", + "/v1/charges/ch_1", + "/v1//users", + "/repos/yaak", + "/ÜBER/x", + "/a%20b", + ]; + let queries = [ + "", + "?page=2", + "?page=10&sort=Stars", + "?q=%C3%A9t%C3%A9", + "?a=1&a=2", + "?flag", + "?page=two", + ]; + let methods = ["GET", "POST", "put", "DELETE", "CONNECT", "HEAD"]; + let statuses = [ + None, + Some(101), + Some(200), + Some(204), + Some(301), + Some(404), + Some(500), + Some(503), + Some(508), + ]; + let types = [ + None, + Some("application/json"), + Some("Application/JSON; charset=UTF-8"), + Some("text/html"), + Some("image/png"), + Some(""), + ]; + let header_names = [ + "Content-Type", + "accept", + "X-Cache", + "x-trace", + "Set-Cookie", + "ÉTAG", + ]; + let header_values = [ + "application/json", + "HIT", + "miss", + "", + "a=1; Path=/", + "200", + "Wörld", + "text/*", + ]; + // Name, bundle id, path and working directory + type App<'a> = (Option<&'a str>, Option<&'a str>, Option<&'a str>, Option<&'a str>); + let apps: [App; 7] = [ + (None, None, None, None), + ( + Some("Google Chrome"), + Some("com.google.Chrome"), + Some("/Applications/Google Chrome.app"), + None, + ), + (Some("curl"), None, Some("/usr/bin/curl"), Some("/Users/ada/src/fieldwork")), + (Some("node"), None, Some("/opt/homebrew/bin/node"), Some("/Users/ada/src/Ledger/")), + (Some("nsurlsessiond"), None, Some("/usr/libexec/nsurlsessiond"), Some("/")), + (Some("trustd"), None, Some("/somewhere/trustd"), None), + ( + Some("Slack"), + Some("com.tinyspeck.slackmacgap"), + Some("/Applications/Slack.app"), + Some("/tmp"), + ), + ]; + let versions = ["HTTP/1.1", "HTTP/2", "HTTP/1.0", ""]; + let streams = [ + None, + None, + None, + Some(StreamKind::Websocket), + Some(StreamKind::Sse), + Some(StreamKind::Grpc), + Some(StreamKind::GrpcWeb), + ]; + let sizes = [0, 1, 999, 1000, 1001, 250_000, 1_000_000, 2_500_000_000]; + let elapsed = [ + None, + Some(0), + Some(12), + Some(500), + Some(501), + Some(1500), + Some(60_000), + Some(61_000), + ]; + let ages_s = [ + 0, 30, 59, 60, 61, 299, 300, 301, 3599, 3600, 7200, 86_399, 86_400, 90_000, 700_000, + ]; + + (0..count) + .map(|i| { + let tunnelled = rng.chance(8); + let url = if tunnelled { + rng.pick(&["bank.example.com:443", "https://bank.example.com"]).to_string() + } else { + format!("{}{}{}", rng.pick(&hosts), rng.pick(&paths), rng.pick(&queries)) + }; + let headers = |rng: &mut Rng| { + (0..rng.next() % 4) + .map(|_| { + header(rng.pick::<&str>(&header_names), rng.pick::<&str>(&header_values)) + }) + .collect::>() + }; + let (name, bundle, path, cwd) = *rng.pick(&apps); + let age = *rng.pick(&ages_s) as i64; + let started_at = (now() + - Duration::seconds(age) + - Duration::microseconds((rng.next() % 3) as i64 * 250)) + .naive_utc(); + Flow { + id: format!("fl_{i:03}"), + url, + method: rng.pick(&methods).to_string(), + status: *rng.pick(&statuses), + content_type: rng.pick(&types).map(String::from), + request_headers: headers(&mut rng), + response_headers: headers(&mut rng), + request_size: *rng.pick(&sizes), + response_size: *rng.pick(&sizes), + elapsed_ms: *rng.pick(&elapsed), + started_at, + client_http_version: rng.pick(&versions).to_string(), + client_name: name.map(String::from), + client_bundle_id: bundle.map(String::from), + client_path: path.map(String::from), + client_cwd: cwd.map(String::from), + client_pid: rng.chance(50).then(|| (rng.next() % 1000) as i32), + stream_kind: *rng.pick(&streams), + grpc_status: *rng.pick(&[None, Some(0), Some(5)]), + error: rng.chance(15).then(|| "connection reset".to_string()), + cancelled: rng.chance(10), + saved_request_id: rng.chance(15).then(|| "rq_x".to_string()), + tunnelled, + agent_read: rng.chance(20), + ..Default::default() + } + }) + .collect() +} + +/// Queries over every field kind and value shape, and the ways they combine. +const BATTERY: &[&str] = &[ + "", + "github", + "GITHUB stripe", + "json", + "post", + "50", + "é", + "café", + "\"api.github.com/repos\"", + "\"GET\"", + "%", + "_", + "@host:api.github.com", + "@host:API.STRIPE.COM", + "@host:api.stripe.com:8080", + "@host:*.github.com", + "@host:*.example.com", + "@host:*stripe*", + "@host:café.example.org", + "@host:[::1]", + "@host:*", + "-@host:*.github.com", + "@port:443", + "@port:>=8000", + "@port:[80 TO 3000]", + "@port:*", + "-@port:*", + "@scheme:https", + "@scheme:ws*", + "-@scheme:*", + "@path:/v1", + "@path:/v1/*", + "@path:/v1/charges/*", + "@path:\"/v1/charges\"", + "@path:*users*", + "@path:/über/*", + "@path:/", + "@path:*", + "@url:*page=*", + "@url:\"%20\"", + "@query.page:2", + "@query.page:>3", + "@query.page:[1 TO 5]", + "@query.page:*", + "@query.sort:STARS", + "@query.q:été", + "@query.a:2", + "@query.flag:*", + "@query:2", + "-@query:*", + "@method:get", + "@method:P*", + "@method:*", + "@type:json", + "@type:application/json", + "@type:*json*", + "@type:*", + "-@type:*", + "@proto:h1", + "@proto:h2", + "@proto:ws", + "@proto:grpc", + "@proto:sse", + "-@proto:*", + "@id:7", + "@id:>20", + "@id:[5 TO 9]", + "@status:200", + "@status:2xx", + "@status:5XX", + "@status:1xx", + "@status:>=400", + "@status:<300", + "@status:[300 TO 499]", + "@status:50*", + "@status:*", + "-@status:*", + "@status:\"50\"", + "@status:abc", + "@status:4xy", + "@size:>100kb", + "@size:1000", + "@size:1kb", + "@size:>=1mb", + "@size:[1b TO 1kb]", + "@size:>2gb", + "@req.size:<1kb", + "@res.size:*", + "@size:>5parsecs", + "@duration:>500ms", + "@duration:500", + "@duration:>=1.5s", + "@duration:[1s TO 1m]", + "@duration:*", + "-@duration:*", + "@duration:>1h", + "@time:<5m", + "@time:<=1m", + "@time:>1h", + "@time:>=1d", + "@time:[1m TO 1h]", + "@time:[1h TO 1m]", + "@time:<500ms", + "@time:*", + "@time:<2026-11-07T11:00:00Z", + "@time:>2026-11-07T11:59:00Z", + "@time:2026-11-07T11:59:59Z", + "@time:2026-11-07", + "@time:<2026-11-07", + "@time:>=2026-11-06", + "@time:11:59", + "@time:>11:00", + "@time:<=11:59:30", + "@time:soon", + "@app:\"google chrome\"", + "@app:com.google.Chrome", + "@app:curl", + "@app:fieldwork", + "@app:ledger", + "@app:/usr/bin/curl", + "@app:*", + "@app:*o*", + "-@app:*", + "@cwd:fieldwork", + "@cwd:/tmp", + "@cwd:*", + "@cwd:/", + "@pid:>500", + "@pid:*", + "@header.content-type:json", + "@header.Content-Type:*", + "@header.x-cache:hit", + "@header.x-cache:*", + "-@header.x-cache:*", + "@header.accept:text/*", + "@header.étag:wörld", + "@header.set-cookie:\"Path=/\"", + "@header:miss", + "@header:*", + "@header:200", + "@header.x-trace:>100", + "@req.header.content-type:json", + "@res.header.x-cache:*", + "@res.header:hit", + "-@req.header:*", + "@is:system", + "-@is:system", + "@is:tunnelled", + "@is:error", + "@is:pending", + "@is:saved", + "@is:read", + "@is:loop", + "@is:ws", + "@is:sse", + "@is:grpc", + "@is:*", + "-@is:*", + "@is:s*", + "@is:marked", + "@is:nope", + "@body:x", + "-@res.body:*", + "@nope:x", + "-@nope:x", + "@status:4xx OR @is:tunnelled", + "(@status:4xx OR @is:error) -@method:post", + "-(@status:* OR @is:pending)", + "github -@host:github.com", + "@host:*.example.com AND -@is:ws", + "(@app:curl OR @app:node) (@host:*.github.com OR @path:/v1/*)", + "@app:Slack @host:a.com OR @app:Chrome @host:api.github.com", + "-(-@is:system)", + "json @size:>1kb OR @duration:<100ms @proto:h2", +]; + +#[test] +fn the_database_and_the_evaluator_find_the_same_flows() { + let (_dir, db, flows) = db_of(sample(400)); + let mut checked = 0; + for query in BATTERY { + for hide_system in [false, true] { + let prepared = FlowQuery::at(query, hide_system, now()); + let sql = sql_ids(&db, &prepared); + let memory = memory_ids(&flows, &prepared); + if sql != memory { + let only_sql: Vec<_> = sql.iter().filter(|id| !memory.contains(id)).collect(); + let only_memory: Vec<_> = memory.iter().filter(|id| !sql.contains(id)).collect(); + let example = only_sql + .first() + .or(only_memory.first()) + .map(|id| flows.iter().find(|f| &&f.id == id).cloned()); + panic!( + "{query:?} (hide_system {hide_system}): only in SQL {only_sql:?}, \ + only in memory {only_memory:?}; e.g. {example:#?}" + ); + } + checked += 1; + } + } + assert_eq!(checked, BATTERY.len() * 2); + // The battery means something only if the sample spreads its answers. + let spread = BATTERY + .iter() + .map(|q| sql_ids(&db, &FlowQuery::at(q, false, now())).len()) + .filter(|n| *n > 0 && *n < flows.len()) + .count(); + assert!(spread > BATTERY.len() * 2 / 3, "only {spread} queries split the sample"); +} + +/// Live flows are matched as the list is: a flow's event and a listing agree. +#[test] +fn live_search_matches_as_the_list_does() { + let live = LiveSearch::default(); + let flow = fixture().remove(0); + assert_eq!(live.check(&flow), None); + live.watch(Some(("k1".into(), "@status:2xx @host:*.github.com".into(), false))); + assert_eq!(live.check(&flow), Some(SearchMatch { key: "k1".into(), matches: true })); + live.watch(Some(("k2".into(), "@status:4xx".into(), false))); + let payload = live.payload(flow, true); + assert_eq!(payload.search, Some(SearchMatch { key: "k2".into(), matches: false })); + live.watch(None); + assert_eq!(live.check(&payload.model), None); +} + +#[test] +fn seen_values_for_completion() { + let (_dir, db, _) = db_of(fixture()); + let values = |path: &str| seen_values(&db, path).unwrap(); + let mut hosts = values("host"); + hosts.sort(); + assert!(hosts.contains(&"api.stripe.com:8080".to_string()), "{hosts:?}"); + assert_eq!(values("method").len(), 3); + assert!(values("status").contains(&"404".to_string())); + let types = values("type"); + assert!(types.contains(&"application/json".to_string()), "{types:?}"); + assert!(types.contains(&"text/html".to_string()), "{types:?}"); + assert_eq!(values("cwd"), vec!["fieldwork".to_string()]); + assert!(values("app").contains(&"Google Chrome".to_string())); + assert_eq!(values("is"), FLAGS.iter().map(|f| f.to_string()).collect::>()); + let mut names = values("header"); + names.sort(); + assert_eq!(names, vec!["accept", "content-type", "x-cache"]); + let mut response_names = values("res.header"); + response_names.sort(); + assert_eq!(response_names, vec!["content-type", "x-cache"]); + let mut cache = values("header.X-Cache"); + cache.sort(); + assert_eq!(cache, vec!["HIT", "MISS"]); + // Near-unique fields offer nothing + for path in [ + "url", + "path", + "id", + "pid", + "size", + "duration", + "time", + "query", + "query.page", + ] { + assert!(values(path).is_empty(), "{path}"); + } +} + +#[test] +fn a_header_with_many_values_offers_none() { + let flows = (0..25) + .map(|i| Flow { + id: format!("fl_{i}"), + url: "https://a.com/".into(), + response_headers: vec![ + header("Date", &format!("day {i}")), + header("Server", "nginx"), + ], + ..Default::default() + }) + .collect(); + let (_dir, db, _) = db_of(flows); + assert!(seen_values(&db, "header.date").unwrap().is_empty()); + assert_eq!(seen_values(&db, "res.header.server").unwrap(), vec!["nginx"]); +} + +/// Completion shows each field with what it is and a term using it, and every such term runs. +#[test] +fn fields_have_descriptions_and_examples_that_run() { + for field in &schema().fields { + assert!(field.info.as_deref().is_some_and(|i| !i.is_empty()), "{}", field.name); + if FIELDS_NOT_YET.contains(&field.name.as_str()) { + continue; + } + let example = field.example.as_deref().unwrap_or_else(|| panic!("{}", field.name)); + assert!(example.contains(&format!("@{}", field.name)), "{example}"); + assert_eq!(diagnostics(example), vec![], "{example}"); + } +} diff --git a/crates-proxy/yaak-proxy-lib/src/saved_queries.rs b/crates-proxy/yaak-proxy-lib/src/saved_queries.rs new file mode 100644 index 000000000..7524e86be --- /dev/null +++ b/crates-proxy/yaak-proxy-lib/src/saved_queries.rs @@ -0,0 +1,181 @@ +//! Searches saved by name, which the sidebar lists with a count each. + +use crate::db::ProxyQueryManager; +use crate::flows::{self, FlowFilter}; +use serde::{Deserialize, Serialize}; +use ts_rs::TS; +use yaak_database::generate_prefixed_id; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_models.ts")] +pub struct SavedQuery { + pub id: String, + pub name: String, + /// The search, as the search box takes it. + pub query: String, + /// Where it sits in the list, lowest first. + #[ts(type = "number")] + pub sort_order: i64, +} + +/// How many flows a saved query covers. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct SavedQueryCount { + pub id: String, + #[ts(type = "number")] + pub count: usize, +} + +fn failed(what: &str, e: impl std::fmt::Display) -> String { + format!("Failed to {what} the saved search: {e}") +} + +fn read(row: &rusqlite::Row) -> rusqlite::Result { + Ok(SavedQuery { + id: row.get("id")?, + name: row.get("name")?, + query: row.get("query")?, + sort_order: row.get("sort_order")?, + }) +} + +/// Every saved query, in order. None for a database from before them, read without migrating. +pub fn list(db: &ProxyQueryManager) -> Result, String> { + db.with_conn(|c| { + let conn = c.conn().resolve(); + let Ok(mut stmt) = + conn.prepare("SELECT * FROM saved_queries ORDER BY sort_order, created_at, id") + else { + return Ok(Vec::new()); + }; + stmt.query_map([], read) + .and_then(|rows| rows.collect::>>()) + .map_err(|e| failed("list", e)) + }) +} + +fn name_of(name: &str) -> Result { + let name = name.trim(); + if name.is_empty() { + return Err("A saved search needs a name".into()); + } + Ok(name.to_string()) +} + +/// Saves `query` as `name`, after the others. +pub fn create(db: &ProxyQueryManager, name: &str, query: &str) -> Result { + let name = name_of(name)?; + let query = query.trim(); + if query.is_empty() { + return Err("Nothing to save".into()); + } + let id = generate_prefixed_id("sq"); + db.with_tx(|c| { + let conn = c.conn().resolve(); + conn.query_row( + "INSERT INTO saved_queries (id, name, query, sort_order) \ + VALUES (?1, ?2, ?3, (SELECT coalesce(max(sort_order), -1) + 1 FROM saved_queries)) \ + RETURNING *", + rusqlite::params![id, name, query], + read, + ) + .map_err(yaak_database::Error::from) + }) + .map_err(|e| failed("save", e)) +} + +pub fn rename(db: &ProxyQueryManager, id: &str, name: &str) -> Result { + let name = name_of(name)?; + db.with_tx(|c| { + c.conn() + .resolve() + .query_row( + "UPDATE saved_queries SET name = ?2, updated_at = CURRENT_TIMESTAMP WHERE id = ?1 \ + RETURNING *", + rusqlite::params![id, name], + read, + ) + .map_err(yaak_database::Error::from) + }) + .map_err(|e| failed("rename", e)) +} + +/// Deletes one. False when there was none. +pub fn delete(db: &ProxyQueryManager, id: &str) -> Result { + db.with_tx(|c| { + c.conn() + .resolve() + .execute("DELETE FROM saved_queries WHERE id = ?", [id]) + .map(|n| n > 0) + .map_err(yaak_database::Error::from) + }) + .map_err(|e| failed("delete", e)) +} + +/// Each saved query's count of flows, with System's left out when `hide_system`, as the list +/// would show it, read from one snapshot. +pub fn counts(db: &ProxyQueryManager, hide_system: bool) -> Result, String> { + let saved = list(db)?; + let filters: Vec = saved + .iter() + .map(|s| FlowFilter { hide_system, ..FlowFilter::search(s.query.clone()) }) + .collect(); + let counts = flows::count_each(db, &filters)?; + Ok(saved + .into_iter() + .zip(counts) + .map(|(s, count)| SavedQueryCount { id: s.id, count }) + .collect()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn saves_renames_counts_and_deletes() { + use crate::models::Flow; + use yaak_database::UpdateSource; + let dir = tempfile::tempdir().unwrap(); + let db = + ProxyQueryManager::new(&dir.path().join("proxy.db"), &crate::db::test_key()).unwrap(); + for (i, status) in [200, 404, 500].into_iter().enumerate() { + let flow = Flow { + id: format!("fl_{i}"), + url: format!("https://example.com/{i}"), + status: Some(status), + ..Default::default() + }; + db.with_tx(|tx| tx.upsert(&flow, &UpdateSource::Background)).unwrap(); + } + + let errors = create(&db, " Errors ", "@status:>=400").unwrap(); + let ok = create(&db, "OK", "@status:2xx").unwrap(); + assert_eq!((errors.name.as_str(), errors.sort_order, ok.sort_order), ("Errors", 0, 1)); + assert!(create(&db, " ", "x").is_err()); + assert!(create(&db, "x", " ").is_err()); + + let renamed = rename(&db, &ok.id, "Fine").unwrap(); + assert_eq!(renamed.name, "Fine"); + assert_eq!( + list(&db).unwrap().iter().map(|s| s.name.as_str()).collect::>(), + ["Errors", "Fine"] + ); + + let counts = counts(&db, false).unwrap(); + assert_eq!( + counts, + vec![ + SavedQueryCount { id: errors.id.clone(), count: 2 }, + SavedQueryCount { id: ok.id.clone(), count: 1 }, + ] + ); + + assert!(delete(&db, &errors.id).unwrap()); + assert!(!delete(&db, &errors.id).unwrap()); + assert_eq!(list(&db).unwrap(), vec![renamed]); + } +} diff --git a/crates-proxy/yaak-proxy-lib/tests/at_rest.rs b/crates-proxy/yaak-proxy-lib/tests/at_rest.rs index 3c5dd487c..65cb8f72a 100644 --- a/crates-proxy/yaak-proxy-lib/tests/at_rest.rs +++ b/crates-proxy/yaak-proxy-lib/tests/at_rest.rs @@ -267,17 +267,13 @@ fn search_reaches_inside_the_encrypted_database() { let db = ctx.db().unwrap(); // Query strings are searchable, being inside the encrypted file. - let q = |q: &str| FlowFilter { q: Some(q.into()), limit: Some(50), ..Default::default() }; + let q = |q: &str| FlowFilter { limit: Some(50), ..FlowFilter::search(q) }; let (page, total, matching) = flows::page(&db, &q(QUERY_SECRET)).unwrap(); assert_eq!((page.len(), total, matching), (50, FLOWS, Some(FLOWS))); let (page, _, matching) = flows::page(&db, &q("items/2999")).unwrap(); assert_eq!((page[0].id.as_str(), matching), ("fl_02999", Some(1))); - let posts_not_found = FlowFilter { - method: Some("post".into()), - status: Some("4xx".into()), - limit: Some(1), - ..Default::default() - }; + let posts_not_found = + FlowFilter { limit: Some(1), ..FlowFilter::search("@method:post @status:4xx") }; let (_, _, matching) = flows::page(&db, &posts_not_found).unwrap(); assert_eq!(matching, Some((0..FLOWS).filter(|i| i % 3 == 0 && i % 10 == 0).count())); @@ -295,11 +291,11 @@ fn search_reaches_inside_the_encrypted_database() { } assert_eq!(seen, FLOWS); - // The sidebar's site and path filters, which run the database's own functions. - let under = |u: &str| FlowFilter { under: Some(vec![u.into()]), ..Default::default() }; - assert_eq!(flows::list(&db, &under("api.example.com")).unwrap().len(), FLOWS); - assert_eq!(flows::list(&db, &under("*.example.com")).unwrap().len(), FLOWS); - assert_eq!(flows::list(&db, &under("api.example.com/v1/items/42")).unwrap().len(), 1); + // The sidebar's host, site and path terms, which run the database's own functions. + let found = |q: &str| flows::list(&db, &FlowFilter::search(q)).unwrap().len(); + assert_eq!(found("@host:api.example.com"), FLOWS); + assert_eq!(found("@host:*.example.com"), FLOWS); + assert_eq!(found("@host:api.example.com @path:/v1/items/42/*"), 1); // A full-text table, as body search will want, made and searched in the encrypted file. let hits: i64 = db diff --git a/crates-proxy/yaak-proxy-lib/tests/control.rs b/crates-proxy/yaak-proxy-lib/tests/control.rs index 65185eaff..7290e1ef1 100644 --- a/crates-proxy/yaak-proxy-lib/tests/control.rs +++ b/crates-proxy/yaak-proxy-lib/tests/control.rs @@ -144,14 +144,18 @@ async fn round_trip() { assert_eq!(flows.iter().map(|f| &f.id).collect::>(), vec![&flow.id]); let following = FlowFilter { after: Some(flow.id.clone()), ..Default::default() }; assert!(client.flows(&following).await.unwrap().is_empty()); - let filter = FlowFilter { status: Some("4xx".into()), ..Default::default() }; + let filter = FlowFilter::search("@status:4xx"); assert!(client.flows(&filter).await.unwrap().is_empty()); - let search = FlowFilter { q: Some("CHARGES post 2xx".into()), ..Default::default() }; + let search = FlowFilter::search("CHARGES post @status:2xx"); assert_eq!(client.flows(&search).await.unwrap().len(), 1); - let search = FlowFilter { q: Some("charges get".into()), ..Default::default() }; + let search = FlowFilter::search("charges @method:get"); assert!(client.flows(&search).await.unwrap().is_empty()); + // Listing is not reading; an agent reading the flow marks it. + let read = FlowFilter::search("@is:read"); + assert!(client.flows(&read).await.unwrap().is_empty()); let detail = client.flow(&flow.id, 8).await.unwrap(); + assert_eq!(client.flows(&read).await.unwrap().len(), 1); assert_eq!(detail.flow.method, "POST"); let request_body = detail.request_body.unwrap(); assert_eq!((request_body.data.as_str(), request_body.truncated), (r#"{"amount"#, true)); @@ -283,7 +287,7 @@ async fn offline_reads_proxy_db() { assert_eq!(detail.response_body.unwrap().data, r#"{"id":"ch_1","amount":42}"#); let body = connection.body(&flow.id, FlowBody::Request).await.unwrap(); assert_eq!(body.data, br#"{"amount":42}"#); - let posts = FlowFilter { method: Some("post".into()), ..Default::default() }; + let posts = FlowFilter::search("@method:post"); assert_eq!(connection.har(&posts).await.unwrap().log.entries.len(), 1); let empty = tempfile::tempdir().unwrap(); @@ -602,7 +606,7 @@ async fn export_har_writes_the_flows_asked_for() { assert_eq!(read(&selected)["log"]["entries"].as_array().unwrap().len(), 1); let searched = dir.path().join("searched.har"); - let filter = json!({ "q": "charges", "limit": 1 }); + let filter = json!({ "query": "charges", "limit": 1 }); let written = rpc(&ctx, "export_har", json!({ "path": searched, "filter": filter })).await.unwrap(); // The whole search, not the page the list had loaded. @@ -613,7 +617,7 @@ async fn export_har_writes_the_flows_asked_for() { let none = dir.path().join("none.har"); let written = - rpc(&ctx, "export_har", json!({ "path": none, "filter": { "q": "nothing-matches" } })) + rpc(&ctx, "export_har", json!({ "path": none, "filter": { "query": "nothing-matches" } })) .await .unwrap(); assert_eq!(written, 0); @@ -657,7 +661,7 @@ async fn flows_name_the_app_that_sent_them() { assert_eq!(flow.client_path.as_deref(), Some("/usr/bin/curl")); // The search and the sidebar's Apps group find it by its path. - let filter = FlowFilter { app: Some(vec!["/usr/bin/curl".into()]), ..Default::default() }; + let filter = FlowFilter::search("@app:/usr/bin/curl"); assert_eq!(yaak_proxy_lib::flows::list(&ctx.db().unwrap(), &filter).unwrap().len(), 1); } diff --git a/crates-proxy/yaak-proxy-lib/tests/flow.rs b/crates-proxy/yaak-proxy-lib/tests/flow.rs index 612183589..b08109fc1 100644 --- a/crates-proxy/yaak-proxy-lib/tests/flow.rs +++ b/crates-proxy/yaak-proxy-lib/tests/flow.rs @@ -502,10 +502,10 @@ async fn a_tunnelled_connect_is_recorded_with_its_sizes() { assert_eq!(done.error, None); assert!(done.request_headers.is_empty() && done.response_headers.is_empty()); - let tunnelled = rpc(&ctx, "list_flows", json!({"tunnelled": true})).await.unwrap(); + let tunnelled = rpc(&ctx, "list_flows", json!({"query": "@is:tunnelled"})).await.unwrap(); assert_eq!(tunnelled["flows"].as_array().unwrap().len(), 1); assert_eq!(tunnelled["flows"][0]["tunnelled"], true); - let decrypted = rpc(&ctx, "list_flows", json!({"tunnelled": false})).await.unwrap(); + let decrypted = rpc(&ctx, "list_flows", json!({"query": "-@is:tunnelled"})).await.unwrap(); assert_eq!(decrypted["flows"].as_array().unwrap().len(), 0); } diff --git a/crates-proxy/yaak-proxy-lib/tests/streams.rs b/crates-proxy/yaak-proxy-lib/tests/streams.rs index b34184fa0..d2a25040c 100644 --- a/crates-proxy/yaak-proxy-lib/tests/streams.rs +++ b/crates-proxy/yaak-proxy-lib/tests/streams.rs @@ -370,7 +370,7 @@ async fn grpc_web_calls_are_stored_and_decoded() { (failed.grpc_status, failed.grpc_message.as_deref()), (Some(12), Some("not here")) ); - let errors = rpc(&ctx, "list_flows", json!({ "status": "error" })).await.unwrap(); + let errors = rpc(&ctx, "list_flows", json!({ "query": "@is:error" })).await.unwrap(); let ids: Vec<_> = errors["flows"].as_array().unwrap().iter().map(|f| f["id"].clone()).collect(); assert_eq!(ids, vec![json!(failed.id)], "a non-zero status is a failed call"); diff --git a/crates/yaak-query/bindings/query.ts b/crates/yaak-query/bindings/query.ts index a9efad919..b64cf5d0d 100644 --- a/crates/yaak-query/bindings/query.ts +++ b/crates/yaak-query/bindings/query.ts @@ -10,7 +10,15 @@ label: string, /** * What replaces the span: `@method:`, `@header.`, or a value quoted when it needs to be. */ -insert: string, kind: CompletionKind, info: string | null, +insert: string, kind: CompletionKind, +/** + * What a field is, from its definition. + */ +info: string | null, +/** + * A term using the field, from its definition. + */ +example: string | null, /** * The term is not finished after inserting (a field still needs its value, a map field its * key), so completion should open again. @@ -57,9 +65,13 @@ name: string, kind: FieldKind, */ map?: boolean, /** - * A short note shown with the field in completions. + * What the field is, in a few words, shown with it in completions: `GET, POST, …`. */ -info?: string, }; +info?: string, +/** + * A term using it, shown with it in completions: `@method:post`. + */ +example?: string, }; export type FieldKind = { "type": "text" } | { "type": "number", units: Array, } | { "type": "flag" }; diff --git a/crates/yaak-query/pkg/yaak_query_bg.wasm b/crates/yaak-query/pkg/yaak_query_bg.wasm index c71e7d627d96d264cd5b1696e568d644dd54f9ad..2ad25df49c147b4d9400f72174bd26f14ef57ea8 100644 GIT binary patch delta 73910 zcmd3P31C!3w)Q=BZ!bxACpTmvk^r~c011l(jR6rQmmLAYZNzOr0mXpqE;EC6KvY!J z=tT<@6%-W_BpNh`TTn-Nii+!?gNQpSjyU5eDkJ}Qs%|IUh%@uv```P2naRDk)>Eg> zId#sdQ`L8TRB+Av1ySSd*%y#u81$6c|Dos@$`;AF#(4apq<01_r@r1ME8nXTA+Jaj z(NX?ad+MX!W{M7e+TEgypSDAE^xy6XM!Zrp&u`;FGvbZJ*U_li&b!1+&@rlw2hCDC zR=w>oJ5r^e7V?U%3;nith#YU7m8V+0%x`;IR>(U&P>!!}0`0up1BdwOF9tfRv?Khw zeS#Qp{9fHqXX>Y*n;$ycZ}0n1ca@g!^$w5o)4mKBdQXRo zTfTbuZ~KF7ykoOU{nn;s_4L!HWwnuOF0>-vD_L!j^g-4@Kci3f;eO+nWgm%ZA7pis z$&0P_!q8}lL{9bQ2MWBJh+`gZ9IjA%y~kt`R7Ii%RwQIb%-l(#EGrU>WCe4I3X6(*6h$M! zU@0nAMDRBnEJp1}u8AIlkwb#WD-2ozJd3z)u4P4oZ6cu{p{t;UZi)b%X_`hTH&_}e z2;slr5g}CeK8O~aWLlA2D=!$#1@I9J5C3&XvynV2SWr-sS5S~=<^>Bvd7+S*m&XuW zxxo;I9?A`cx|(JL;Fp>~GY=i`f6)Td_Qu3Y4#^AUT4oT#F{9CF2&K$W6l04-ysKgz z+J@Q`6qrFf6c3uwF0n58R?rNGqj}y_v7^i*d-d{uj9qKar}^H(oTJA-N0Aw4&ze1H z+{_Cn&HA13j+v)k#!Z-h!HlVs=A1u$+HB+HRN}<*&pvP3^x1RHpD^2a!)&8Vo;!Wk z1!vD0ckcOvWny4ZNl%o{iE;&aZQHu1bk)5cwJ_SC7vK}*^GvBt}w6*)}O8StQ*Wd z<`Q$hc+7gsy2pCJ{Ki~rUM}vU&*@%kr?t}ht9h%rKzv}`Z~a*$f3`j~e-JN-KZ@(k z56zwC)#m%=73NjuHRi|WmF7b81M>^>BeUN8oA|`sBi=LX%-!ba=4Ybo16KELJ^o5- zt!3gtYn^qI^^onrOs z^N;4e;tg?CS>PY$&*olpg}G3?WPNQlSzF9E#b2%4t;@lJFI(A9JQv&3I4N%@bv?eF zW4SA-)*)NHZWj7AN)cr2Hfx=ryMI5;w2$|u=TEbjEO^GS|4v54$###O8PD;acIGyo zUvO()Q0FvebskE|pDusKaND^mX})DdT*E1K%k?)`b`Me@@~;Z~%N8irLQ?f%I3=kB z^~Bcu?LNVvsDzI>q1xTjlCb9+B$T?l zTdFj3LVI_6OG4cqRbopj;fF4wG-+OTsNpsTrz8SphN$6xjUy}iQ^E1=n1%vw1Bfd{ zLydQKJ(|S!?;5vvJC0H$)4j)3O8QN&`8EZf-}w_Q2|jpRS`z#LJduLN^@q5oCE-9E z?PR(v$)HPr_TmSfpY-~TeR@8Z6h>}?dI>ldkgxVT9iK0R%-XY z=~o@xr%E;UIOchxRo=J99*)0V`d>q*dk^>TLnk$U**{yfUm9>Jp6kS1vOlyEMKN+> z@uG;6>lE4hO|Q*>!)U$d4j3J*Ti$Hg%NlPP@Dw+8%Alcil6U){6Y1H;J%dJ3;i|7) zmu3!*1)^5O$#Ef+dKjy`_?ULy)FIcK*AXSbt@?Guh%$Q0 zn=xWEt@D~k6c@fGFfh|?5ViJGEKTb9rSYo~eMRw3RRe-hftBM3w|)f=)$4zJ=Mf6z za+u&AHP&YOlf93%LO2gBwUF%e@DL7&o zZ2({}YJrpE#HQFQyidmbNMCw8PZ-m0Pe#WI_>7b`8O>E|nZy^oS z-}vx}(I9>7Jw0{??P;t$tp}-(GrY%6&!;`!%cmbhZ+ek4&ZV~-XPwcN@VNaLK*I#2a+(VMv{QZm)J|Bq0XJ3~pl8?tyWWWg~wq z@t!)jUHOvT15x3UnXc|!#KSBvIVFL)Wq{&gZt)Cug9ud%v>6RzJ@vtIFLQ$B zqV(dK)nb`9_PpL_a-ZSBF-rAmwDgILESH?HYhE-b5tzwfIay9%R<&S2!mWUW<5>&{ zrzFBYAX)fa;(dAEN!c0_0Z6=IlaB%x=1;zumU}-=eiH8+&R>uVWC5yB3sfU&Yfjuv zQ4vXpBnU`^f+VaV2|8gP65;NpR-6PiPLR}slprI;d#`rV;XGK(+2|NX$pc=gzemJa zZdn_&NwvTS zgRGlg!#O;cb>8b!Dw9uePxy3#jxjn$tl$Kz1yXrx_IIon&8pErwOI58x1Lok7N{4L zs^b@hS4^6EUxLZ57Aw=zm#FvUDt%2_ndRzzRZ?ZFPb;)ay*H@zCv+k8-k{z$sQ2g6 z%4|^Y%_@CMTA60`zFDQenO0`Adf%$jw||l35$Hl&(=xWH_id`s$7%07)%#ABzB}!G zmwMl&(!Wi6-=p65sPuhl?|aqz-Y=f50=gD_$w1%(?{(_E?n{1Ol=i+*y)RVhi@p6* z+a{N(lqD)lN#)P>ahX7#>Ry>Cw|v(0rnEK7uNhC~QKA{a^{1hqs6IYD0{V3u1Xf)Nj~L{Lc>ycUxod@u1<&72+s8ivG7 z2nZIrUfC>%-tq>|8o=EJGrGfK1j)dn(C<#PbZ4e@XT}A0r;-v_Awg}mae?n8-aE6Z ziXd=+b@avJ0is4>d5yPaPJ6FrcBm5zeZdk0ST?65LT)ZZ6IZC_^+3NY_omM7k_$-& z#G&EZ1SG^N@2=TLAEDaxrx~k)Zd;W@$tk|P!;~e2nqSmg>DhBSIvcP=m@JXdjV22% z8o6JK@hN?` zVPrG}y>12#yN3NI(T-)QQDJTdM28HZxxM*~6Se}a?c9ZEJsxovuk@NPK8Ch=KVLk% zARU-6!x9B3S6m_^tYaxK?U%eRzZ*?Adl&!ia=P97>32Pj*axM^4cfPxtd|asneM*3 zRl{oHk3(=+N6))oM3Tsa>dI3ag#ZM7(H+St-N>C*1B$*a3`T*m0eaBC-{ zwZC23vwwPPV>#nY-ByeT4Q||u9^73d);0swOE_b$-O#w`vN|eH zN9r)n8L3;(jkpbrsd9H49`<~%s;+a)PuZnY&Z|3tr1x0eb;qVR(uW)AryFU*AcAxi zh?NXsddW&ISp^6e%1P2(s%?8L2O=`s;HSg{Do$bd&A_65)lY7)Y!HZu0+fI8K1(vOdMT;0F z5H1PHsu&|`gXS-<0nKf_ymJOjH4M`rz?8k9lt7pZ$_toK$)O%bu}igmjbd=mipB{G zh%!ic`V|!!4cXjK0UBC!#ZdI~?G;ICXq7-C$Jcn$PYw26dau=`Lw=r)5WEeDKlpX9xHb>EZfZUZ*S9%C;JD9y{# zCS#OUt*HC$$8S9ltTqhfx-co(m^8Ufli73LZ8gY3n^v`$gCyvDMi1kP+q|={=}U{f zW!H2}zQw{E(l`gSC8v}rvDwmreP7qKtxi~w+ z6e4En$B-1PNT0;iHA?-jkn!+8BuSqo%chF zH^RbV3q8yCMwBn^jr{S83)}4=jh<4CICiP*GN!s=VD^T^>w=5dDVa&P6nUrLd}#XT z(#+3o>7PfX7yQjF8TD??{CqR>Ghb#DK2esXH+55H?rU;)Yi^;pZAla;`D96_*2E&O z^U^X*+lZwZiHnvNrk8ngY4or*VKD+fs^5oN1ROH=oTMhuZxQsBLITYll4sKwqB!6Zy>u}1U{ zw;e{FSH3Jk8yiO~n@GQT7~&;>r9K{XvLQIYIYZ;YXf_yla4fr(XOWQNUndyL;sUvD zU}#K$skf%JA>6G-tzGX0?l=u_jk{xDO}$`AnBBv$(1o837IXh*42>}lIHJNF8)uh^ zswtc{V~`FCN;UQ29bJRFmSJ#Py*77F12fFK^U&}@j23X;@2$CW0x)7P|0>I4;bYg@ zo;T{QGIZW;MM;Nxjg+_&a@p?9DGA5O29y(53`4=SD-Olq*H-kv-~HSn%>NN3%slE{vB;^G*@<|(4O1T^4{CMs(Z?d8o2qbrt(ZcdZe;u-P0MPSb9%K z%<^ORBv{M2@y!uQ`>E){r4R7YrS(wrXSD!%JAAaC0ie;eE`({mnciEa)P!HX^aZORU zUJRgkan$l&Ta*8b(uKcF32)&}g6Je?Z8dryymm?|X_dEOZ5&DOt{u#z#P%Gf|LX@! zvU8w)2IeR^HR{05J3S)qun9IxYhxA?)sH8SJ?X$c#ALh_HbYcAOO}yk3$BFroth zB@Qn(;6+xEn(Z+2jT69x;(v*3`#!Jx=B{+Ick1SNa*Pw1f=axkuzG@qYIH+*8>CtT zXX!l4v6%>RK!wNii>75OeT6zfN0F+_fDE> zq-}beYDxj8+svAXp4S(6(^NL#bWC19)=0kl`5Nd5Tg*V z#9%*bShBtPmx_|+3`UzCZW$v8R+;AI(pvj-9$>j}Fi}-gc^u&i6z29KZk;%qDxHW^ z)LWF}fn}%+kHX%f6c12!G0KE%omhow=nA&aX%KCGTG+`+!mQ$&tync8Jj5)fLmPUW zR+F*P0ghtk92i#dEPOI+%)tY$+!=TfRz-#p=CxuVFt}nMc78EW_zaKP!KAeCPjiw# zdanSrgSnhU4UT9zVv!b2a+Lrg2ZUCTgCx6!8itFK&T&3mpqH zkO;6i70DXZ4fHT7G2F0|BH^kQQo^i(l-TE$k$bHsG@Xk)#o7T-fKONCKXbTsxLppU zs{pu3(MytP%O{u|Mm$Ov|FSgJj1(b)9eK=v2+LH3sLG%&TZ5t*j0s*a4TK?|Tay&W zrKm26#>$wm4kSAZY^JB%VfoH8+e&=_pZWrjt)RXD&n{?)X|WWK9Hu_Vqgu!IDJ=k^ z^JgNB`oPjjMSa=u?Rtuu|7=jp0XMlfIN0A`)iZuzZ4!- zCIh84sbPfK`>0*7*rP-@o6S8PDR9AbNDo z=wqN0`2&)2VIE=iu`m!$mGknbT#n15aDJA1S#2WAp9&>?v%Oc}>X(Elt97y~45PPU zD0bjk%4S3$XJ58qumXsl00Z~MI!PTXB`#=iN<7MmZgMs>soHpq6&R=qY=`389bap& zB`h--ngwX`?@L5xCbHsT)l^n3`W#T_{l}*TB{>dKQ5ABF_X1hLiRW3EbI&;WD?cC}&a@_3XR1*!4{)kc9Xk4L$|LRtY-Tn)QY zwU0-+QL4%(QKGckDAg58)o?*)1ig)?O(P0U2Zuhnk$_?gcZ2M#MkJQ6I`fB@?l$_7!CsW6_yxMmiO+bRLd{&=Ws-ne(# z2ck|+JlmW1PGOLFFp(|S6;Kr>i`DPE6dQo$!x+jGu-AQw7pV2cP*lQN~A<#f}5Jrdn8dK*0beV0L88sS{#chO|}I zrPdvfo$y*0+dfPsh}IBO7-y_1MtcB%Y8gsVKvd68m`MzBiMxR$;BEr0(>P{b9en7c}+~ zTUUC^8!NHB^IGEo9erW~eef519oJt_xQl?~G!Hv_KDK!)*7u>Cym!}+Dtz)F`76AB zkCY+*>_^;@KQ5sRiDwH}k!Bc2z=fVHstvg_J+x!%;~nCHH}K6)!7x z&Ke#IppehR0WdMXd=`dL%rwd@oc&7{hD`ajl^L!p2tv)>2r$vyjg>Is_4%l{9QK4r z_|QikO>#1xir{*K@~ou}E9HAu znE!uE>FElk@xQ0u#Qrnt4IqP&AM@3lJgwg3DLsev8l{!sq1Bsgn9PRKYw}oIvGA$Y z8&yH6H>v_yORG1kf>Li(g(%l&^+uIf>WwN7ylM4Dl~?MGDqo$`G3dlRJ zm8N-3#dix$1CSJT|8wmmi3x|D|8J@%5q4l?>L@}F9`*v!}$6R{*+=KZ0mLwI*n zv*BjhgS{0^*luq8ThqOSUFTaLTNhlo0dC`EjiWa%px`R~{?6lKb#e3-0vr2vzym5+UwOa80TV<{NH1r)_X0P!ow)W_@MDV7Js{gFY zFzsal1)h?k3Ob#=)SI`pbIH}1MA)6G!7&qXty@n&NRS-^4%S@Ph>YC*(}uN3vmR#f#W zyxFW@BfOAzg!>fjUc|e&6s{BT;{Ee>kjd#);15&`<_C^PR zP1VQWI60M4_+}aX!|VHIyPogm(jiGW+bgFef@N^ZGhPk7X4}s){)6z$_pEid-pAu4 z+0dB^WOf<-rp7ldB8;D)TTmO1rc(T8mV3E2K71EY4Er8%11a%u$8+VfGAd3&o@pH0pr5YUgJM8C`YRh?urC5p&?%P2b4!!{779I`+7NSMo z$L3lOBM#3PcBadq-{Z@PBY42ccP|ekipj~hKNU&VYdCaVIEDrj!*>j)4cdZwiqf{b zK$*!lO7nNV|IM{g8Hk5F2Hs7z3OB&&kUPkT9(PN?=!!Vbo2+rEP!Oj&}vo zYOKZ6H!8B8FzrdP!+>-60P<5yF6Z?M;*|UQwk-M zC$T|KV*H>3OKTz!F9cW&QcoCP=9p_=nu~Ua zb@*2Uo5YOPfk(zvfzctdNV7ql4=$v!9a|;ro&%rpRtLPL8WlWsEo4g=uuqZfZM$C2 z*D7YCLb<=3vXU_*r$mwwSEfiI>ho?&DxH0I`HCW(B5Yynhv0437d9HHK3N zsPVwo8IJ)+GG=l#h)+ch3?iR!FmJ^z$){$u5ERm*sMua@FLglZ3cSUJ(@uSs@+bTp z?CVgHichs*#2Q72D97|OSEdAo9^p}p6Yf|=FR`ajxhy*%B^8mA)taX^HA(>v^5pv! zbV#s>Ew{Gv=L&K-wg(u;zL@0{u{WYfc_WnPrY%scLnzb+TRv;Z>=I(zCB+>kdcl!2fvH274rtOvD#ZSEFw!e@%>DBPfGyXv| z|D@N>mIVK#*P_3sO86(emhoXDb#~1^>GdX#u;~QFdT$U;LF2hETk_*hE^d zuRDutV+Zk`_e%c1be&%@ASAENqYnRzhGdKP96Qt*-gA6o=fqwGCScp>|9kH_!{6#X zhf|up=gLd2yyw1oshpn&wepgj1y|31(aK{txa#P?>SIp+drszm;bX?WR!Dh14scRY ztCPz2G`F~^d{1-5ulT93AROR~@_K#LvGc$7C(lj!ld(nqOCJ>Q%s!}ee{vMNa#KKX z|J0xS&mB;`4{$*J!k^3zsC0ias0O~NsC>GR5-GRRuR4W++(@Qd=>VrNr1HP-6K2!; zqnHTmL%qvsinr*ywlFhRem69E&{$RBzKNIvjx}fvQ_XIPQ4xrw79-h|y}ub{;M;5o zR2k3y%iiW#Q3JV5IPw8odjU3)_*}!jT?69*e%Xr`sF>W!-VVD4vIc?0g}0*28m?$B z3@AOtvaj$Ke(%sL-u>SvU~q2v{>Y%#g1yk*KJ<`Rv$qR^ohI)+kuLQf-#dsl-t@zv zc!#&8$sscn+rsq#UMF z_{-E^UgYPVsCe|x7vV(4!#^L5lRkg>c>?~PxPL6tAK2eXH5mFugGfnh5MQYVf8LJ< zZ$6AppqFm0qwDA;nL~6T9tdH-kRF#m5_Q8@XF;QBiM&WqM>X zd8NQxzV|!c>mOAH#G!T`vo==tR!TBE$vC>$2zwJWvavl1-mv zQ6IRBe&hvS9vY#gsrP3hG?i{|>KUcW%rSd8UqPA;*<`u9TE3&Kc&OT}EO6k}wqN;5 zFDbt$rQ@?VuYo#m!9ZFq2ehXi;Sc!Rw6B*Jx2F!tEh-lVHHha4b~iOczrm7$cI+YS zl;8+3JZhN1*l7OvzzWz);ggamzXmQV?4sxjeNq+voKYc~!!8#Dr-Nz%SttcsNe6(o zOBcOfj_E)h62AbAx0tZW#hXm{Q-C4Vcc8-G?9S)_t@Bn%21qAskj5}b0gT}QkYX7i zr6Br=A=<-j2hmey$AdwH@S6i5!XdYVK*R@UG9a3&A)3q(vESqXh@uCD=+7Ogdw45D z#M77SP=_w9I1+mLl!Fdo6z&^b4lzXR+`+BtJwLpmhiO7Q;mjCQOKaEMgm#Iw z%ufm#O%A$#pPLlu`pbiMUCti8Uv^y?r;5(H>oVQ7!(D?J59~Ti+SkGYYlu@&-WsQY zDbfvsbU%wz&+vB)KmY)ANKm5u7h_)zs?rer#05brpiBE6IXOYyYfj@qFCz`X5*)hV z^#a?=x=GN#k3jaA>P9qT&QsuWH zy@T%8jAM|9Oy}@Y_RXt3f6Km2b~u7cX`ej$2ksUmBk1U!&%j;DZZnRA*CG03e%s5IKZnvacO-L<>?4!OUIUe2!f6YP5V_K|enfsyQTN;P#r*s;8}n!3zBGa!X*$0z;Xar}d-pR4-rcM{V)=tRL;cV;iEG=_wgImT*giJp5SnI6_`_ zES-?Mt9}4nb3lrN_*VNWx#L(mU+k)v!~4^3>8hqx{pk+Mx{8fT(BwN!qg|>C1ozeR z^3wFy5c*667ha)0o*7OHg}p?5V2QjdA|gpKS1vtKl*o>+fe$~gq)3A8Ph(g~j%kp9 zsVmqyn*Prbl8zFE$$&vY!NbM=b3uaOYljHJHfa>{4EzWjwgd?DLIOv_FnJ=iO#3Q0 zK*+C>jr~ywNp4B!@zDuywBJ3K3D)501_LpU|>l;1c3Iv?XgOiHiqCiIwER(H5GHEF9Wmr(F-~(fI z4FO}&g4q>AaK)e;+|{+H5_KIoC!#pH#{oz)-FCxEX7LOBJ)9Ad>o$p^By^x+91nJ~ z&>1j*N$Cc>hJe+(5LClHPW>-!Lg0JlF|b~x$B@mLrFtmEa0Z7s@(iOuv>?J>cocZ! zQDgzSm>vb=0E4Lczch+4j{=*g=!_NfFgxg&G<0APzd2z&CU%tZ@o|RPrzw9x&wH84@R5-C?C|WA-90|kVL%De*{l==Z>{n!B z6!ib|y=uX6HeEChGTy3sHXzLH;! zrWe^1d29?-;8vp>Un#__x&^iNCb@qM6~oRbJ^>xq%X3el{#iHh#xHc18{~s0P%~?&ipF~H};p$2zHaFaI{7b)}DVX{`Ah*dng=#6e;CuK*g*~Mm zvk$qBa_1WEB2G42lA8@beF>)rTGH!y6AcvPuHp3T)^y>gzsc$OoQ~-r`#Ngp7&s`- z%76khPa8>TP&fVj8(J@dYt(%~O&d<55~AVq&C{X$!onlaG~b2vp^TqFHK`P#QZ6`y zPCQHj$x*PDYpOjahNgZ^< zyhqPnHJ*vpdQ;Q>GwBF3skZt*eE>D`-932NPhgRB|JM?Z`ET_r|1zEqrI+QRbLrB2 z6@Lar9}H&jAIol#rRUL*zG)dP0c~+!|JkpDdBj-@KKSJ@c5%ALUA_`%P!~JMW#`f1 z>Bs5iJLgepe?Cv&;s|k+7pObxE<`=vXTcn)qxCoi1I-LxAVlzS$UFY>j{MECYBCh7 z+vQD@>BvJA5rp*2ScudH9QeX?%bh5dhokmEP^7MJ+E)sB{P|G1Hpsl(J6*4%5##Jr_76lh+ zCTW^YTWOPAIERMd@#-Af65e+QrX2TaHN}2Um&19}bmLqo&uVEvd@9G`V&e+CO!r+1 z9IS7;=`#9`0Ls;MG%SqW5UB9z`<*&knFmlXLWq284#M~ur0eHF44xwIn@7Lbxj2Hv z5y?9D@cGmw`xLBY7R$v`+0?#02RR3@ zd5OEHpekjlz$SzWsCSZ+&ZbVip0|+4Artl{{l+ekfIWb>Q_oaU(s&umXAdBx5Lh!( zCMQx`v7u2enMm=-7L-;O2iSMrat^i2zs8DZp;NU#r(!^X?H=b)ap`iAqDbZv|5QhW zC1=!<6VIW6NgP=!he8zvlt_+>ntyWjKqb7n6=p+Y=FI~+EiPTSV=K&6oQF1z$WHJP;qenI5k*+&Hjw+IAzHT zG4(zPbUW-Wlf8E3IJl`1wSa&Dd@n>4l%@t1D8t?u=DlwwQXv8YQDAI5#9}IxGR&9` zF?c=+q@u}D@CxR{kJjidYwL; zqpqk4Xs2g(Ut@;fBnD*fgsBUO-|I>U+?(ZS{kY&t>VU6B`fK%-a43AO%YAqyC9s|a zuA;suP;(WPaTIwAgxGX~_KskA)##;HQB_Y}(TuCRE^!+PA38K403oi@;k3CXE`E_a zucE>U={XMPAY`ALz%td+1j6hxo2cZ12;t{~Sa|R;OuHx4L2V}xc{fY}jvZ?@rX(ym zekAT;7+(>TPJ0SL$=5MYijLW@%iA%K??G+~-*!C2;pYLjd};!l9K7iPs}oLBUIEnE zCXPVBf)10#9?SYDz(We2eB2BcAMSGSlW@2umBe`9Qb~QdP89>YsWb$!`o;M4SFvuR zycQjX_zooOAmLyW>`Wyn{8$bZzrkm<;ez0c{MfIAF9QzMe>yV6-v3Vx?cW+%66pE$ zq0M7!?pKEfiKuXoS&C}(^O{<&CkeU`oT6z6JovmM6~+%TG5-vDLkFm6z&H+(jmY$^ zf!}Mu_|&R{KNcK7c))ZGf{fKEETmV6;@%<&?NrC~=m?xv8c<@{U<+GOrP5JaDl=ii zA;^BE1wM}0_*Uppo>u5{%#detzs0HOF!m^hYgw%Obe0I=G^-*~K4=Jd;K5RDpjJV` zbjx_(0P=Gr`azftLCjiwu~V3>cOVx{*&87sTF?y60WbLEs}5qSo?4KJl59Aq#Cn2~ zjtL(aqLRuS0(LM;Xc9c1N*Po4t&n@A9-?2_KdCzednuS?xKOz`t7kCl*+z#xsVtt< z3t;yfC}ug5_5}_SE%iP;Tnn_)kyjjekZ zQh9hcn`9Pt2i{&t9b#Mbm)_WtbT#Flb-nU!umcJbO6`B}?Q4EDT@W+`^n%fY?nZ7t zh@CNaei+e74wYbg1lj~?k2-JZ2-k-}V25Jn)qtUyD}@2WimRzZJOc*aTw^(;WSSx= z6x(?<6?Wgul|rdKIMoHbp9CoQzMr&i2;Ic5p?LbxM_xk};g7im-Wa*`nuG1>sSPp6 z?C;Pblq8J+wHt<|-u;{_o2l;6=VtV&#i(Cnq^>cHl?<-2n`>C98VX?TN>ibDvfZ_G zbof;+!#ihZT}$250rsw?icxQIZphX%GDvUA)Nu3cciM??04Qme+loJba|U~jI(s|U z2cL0N_iCB7h{_MugR9bQRqD1ZcZUW8vx_F=$&08mjWUqbbmJnLFKCr4xPfr-v>bW^ zy;iWh#q_t_eRv0UZkOHbaTgTC8{CFU9r;k6UQfe1DKi1BLkAIQ*C}%fA5q(`nreEX zo+=2Qp>J-4$@F1U&P{Odz%J_S(SCj`rgPY!x@Ix84?TqbaDAD4a53R#8u`{@>O@uY zr^Ph9pFUOqb}NQL2`c6rN5U);SXd~n9>*_zrh$D4`;Kso9dsnzk+SY)7@?I?-b_QG zN4$SC4LhE#*f8Hm2v;}XN5Y4XVZIt#T(MZyQOcO|kst|{IXKtOw^yRV6r{OT)8b^- z00iMy$_sD7j{NQNkGH^SwNV}}!M{~|XFuze_ZNE*PSO)Q~qAqK&6ou(8lk0HMnq|`sjN%tV?Q;gH}>s@uMN@R#LouLd8J_u+;OV zUghqZx@N4p5SSqLC*)HrX)YJ-cMruWA%A-hmB#8aN?dght*2LIzk6{(1PWewFU80F zWMnkj2kq@kC6%gvDv^}xx!CXdt@L)`-FZ2LFj8ndJ4s%;LKMr(mQWCiHP!^))iBvD zf*a-rK(5x+q4TMNkqA)=W6%U6#Dkom5>bW6w9A;vh641C9Lpr^#9(rf%vVvUOP_I2 zgxMBkej+pZC2kPYg*NeyyPEkiHOVe`+{z9u{!5-%w8^p-Su^|&kc(E3u8$1hNQ&xU zj#>iYZe{opF^3brV0tWfTou|7H_n9E`j3<5B(iegjWt+vWR?ymnS0SYmMFcpf`0HQLg9z_2h8%Hv}uu}%U!pawUw@JwKL8WX_@xRXH-AD%{5 zAw&8=FR%hG#1|i*>8%#!f|LkMA+ZGzh5?P@R^1dJ>;xfo=_`&$WZs^YF@&KYT z5iaeM6W>Mx7=J;aOq{p`0OA4QMqwN(uz8JQtfXxSI-wi^ou0)A)L@V>nQI)*06757s;y(bS}O?|`{^X1glaK+s$m#(IxXrX*#HN@#FvgQGr znXB$5CXjN7L)%u#<_D+?y(M=&K*MN*?6rm}X}LUS4K4KVSjPk`m*1|T0ofb)_F}HI zQr4`+nVh;WHW}_=ZUY`V`nmaYm9Pg7o$${~$hX&0k8E|RFPk}QWb8qjLifqvJxIf1 zH?n35A44fGrKRcvUWuHBTc0&7`0 z9LIQcPhdy0TvW27B8D-m3pDxCQFc_s9-Lkn?%u)&g*O<&%1fB?2< z$#EO0s7!%`ir`;Vw6OTb!K4tvFu5QCz%N)@*EG_t)tP-O_YTj8>YGUiumpLvhyMxt z?dXc(sFHKn(^)A1)CJig%pW)o0Q_Y=6}1Nder=!t*Ff{tXbdQe_bM5Ggc8tXdOkr% z@Y?>nN9ZAh?#u3vQm3|?KN(QVS6BC+xi(=hybt;l^`sX+k&_>#E@Inv^2SGL&9QZR zzt>J_+cWX<61ZTN%cV{XAQsOnk)v@*R4^6bbtd7I~U*Rnqc=K6?@#$M9 zXFztH6I|=qSK@A}*W^hX=nwX5#5~OYlX|YRSnk+BaZY8~h}7jeHTN;PrClaI)8|q? z{TN-F!r8wwX^0&@=%h`Q|JX=J9?(-Gs=4 zwuvKU_ye2FzEw_p8j(ZGpM?CmLw^1w#Nh3+4R&Y3z?Os(0eZ3+te2R|g zx>sm>87cborNyT&AxiqN7J>56^%W3H<@TrOkn$G^TgJ#>k2zmPqa1{=H2qVVYz?ym z=tbH7Y3ihpRDyG%O7T+5#xp@}Ofl_9$rfsZ*3-cKE~nQ!35}3Y$Q$ygn)mSt$(&Tv697 za@h{@ooC7Eyyw1V!|i2%3@&if-tP-)58O`nz?B*D23wSpkWIbmNZHhfY`L#DzVu)PQhdRpLy#dY(eE^Qh;9xe?)MkK#+nyL4o`rO9(vs7`K{6{nEW|-U zutU_<^boUV#H?{(Zm~;GN6kQ_@@;)ex+U4M62N+oe(*qiHJ6070l^aNVHk;EdQc56 z|5x!foE!eNTQ?j1Xr2LHkg zFoK(V3@6FfhO$-=E)*vx95;ZP02U!!I}Ip@VA~WZVx%x2x~zYrCA=LBkNv&sA7zHH zy#P?c`bSTY=dXqq8wX{;Xyn068Wqxcjt+xB8T}lT)4_7abBJ+hY`X zX$}PIqc5iR_&L}FhHTT$7ZEauoUdP^tMIsZ3;jtk{mTG%Q`5|sDNMObcMb%%usO$` zr6qFVD^yeb@lO5?2z5msOlfzSa94ivETS6Z&R6Js`YX*MBC~knN-1Xe}t8(mCI-2U_VlFuEqg25l27`j^ zxK9;)dn@fP#O4A=%U6Mpc>e=`c>m*z*DxcO$U|PIb23W+V-QU!QU5ypD3{1*U#G_a z)unGxS>dDUP1#(Kw;LXPgL-9VL79fGWj8eKdjlqMf&Q&7=ooNN1emq(Q(bcC+W=sN zyy$H@9#{49f}oa3wn+djYliQMEvQ%4*@#}$&5ibHXF zvGoXgKX9hKRQ|^{>e~MGbn*Zf-#Syp#Sw zXEpu)16oY#wad++p?$b8Y%3G&IdjR0Xt|_~Iut zgiW_LpVEmP-cvG<50-$QS=54nmAP1s#|?7+r_=-1V#BAYMFr zY65E7*w3hIT&Zb-MY~`X4Yq{CDjK5Y%6U<;o8{doR)MQFI7?r(f$R#iJQ3ilHlXS_ zSY_d#;6|XWefJri2}SLsUDPk8)H0n3oopGBPSJ{(U_}g7^=%4*6*qw^6x3qFx)=0g z4pZP|YLU=NndvKKoB;(QO)0Y%$sV6mSvPgZMG8t@rhF*1LY4{T+|TJht&g#H-Pgzd z_BkDz-{4<0&fvRsieqKP7g{SD{sm2dt9R`e*tK6SpZ|i`hW^VJfB#TD;y!Fl{UJOQ_2cucr;_|>5}$qrqphe`6J+v+#3qXLkvLJ$O{=?!w z{0+_T@|0i+h+pE+_vylamBoA0slvDsRWCaaeM7HdsxJSlUQHm({pO&PsU<@`@mK1? zt4V8?miXvt?_cRth?V#MM(3KC_95S6i+xqoBxol1iAZ_nUP|C`|6ZsLAIXz{p!jh= z&mYj&$Yu4+wJvgVvF@mpe3cSTOKGE0qG>64D#cEv$ftjxk>cn1GWH{_PhP;R;}+Fo zy;Z^JOlL<^k*ZQ~k>3k`>-B(s+pCoHeoIwKdcQc&fLbm60{(WYDdv!8CqLGnj-h);g9&-ZhnouUC+MDglju&5Th5pSVT*o4yb{G}lKEWGn>t+o;gHD(gF1IT) zDeLH`FcR0M*ghBsoGa$Vn z2`4*IfofwV+9WwLcCj#TXvMyT9X_WgU!L800{X!08BL>bI?0oJRQg|a3KJ?O{ld^ zd&X69*4vcxhjn8VRrZUmy?H!7jKoYuIl(i`er5;1Ag@Nm*9?W1;K(v-EpmKNjE36L z5ELaAmi{f-;`(6y^UPgM<0Asm%HPYmQ86;ufIG5b2D}m#J&?37Dy{}uUl|imLP8D7 z5nb_^o+C~Z&)3Tr@#$=t}V(o z9bxaQv5kjBvgzx2oab0#o?+jW^t|$CGR=OYXDjW%(9tl*N$_#9^Q;6vvikg}a%H zhW%z!M=?>9O(hrJ`j(%=4RT&4pLImPuHWt& zsBR#)f8uRI{>RGO&RqkQ+6P4nubi@Lm#m13xcGRN92pmD2kqvF*jKSU>k7NM0=O3O zMIY~a7K@Slt$Nrs><&jrjylT54Sl=|Kk)0fcuYb((g9TgEX~9{(0J8~>>h>+o6CE4 zN&8SyRj|Q|W8w25Q#m_#HH|$KGO6|G2#7o7aAB<>l>X(e0>k5%;FSwzK3j4iI=`xX zwoJT~{b!3}yjt%uiG-`NN=9hkntXq1P^ZhQr(2tw5HT`CcW z)7VXcXt%UPn}6Yg7uJ?U#3TZE{IUfLi&QFH*j&lhkVdh6!+=4>(17yqULb+j9scD3 zgevkFV8t>-#O9XW8@cRac5*mAjFWlCIQ456#Nr~VayoGqBtoR699TX(H9GXbgb|e| zj1z@zRs4%!>jWWyDxRSkZCi5YAkl$Oc5w)8nOxLKoJ-#~{jHP87Kf=I8sCYgf;stk zp?2sZJkJTJU388ltW=ZRtz10Jezo_@MY~i04lhd|%0LCwzfJPk3UQtZIAqrqqCfmg zm}li}u-r;^<$PZuM(9%%P)pF_M@`3f6<1JM9hf_%-s%W`K+8HC4!)W09_G-+P2YDD zd}(IW_lJoI1QX;U>;{uuxAN-W7-j7b!L{$S#0$5s1jW= zvrr$ih^+Iggp--&a2CRE^))hz;ggDP1n^8LI*)m``;{bSqe?%dzFpRS-i< z<>^O?ICd-M9w~0jTEnb>lTPa-t}r_XTJ!I!1%6Cb-cv1H`iuOmT3pI_ZHo01Yw4JU zxaL&j3r3u~moF++#z8i|wzYh*+ZQ*Bv-y-e0%nQ3OJ&LLj}b{)DK9=&+>-yeiItKA zjhHa)f%wRlC-fJExU!?Vzo^9Hy#AsW9!vGti~8}Cesr%ubajvdus-1V=?q z8z`>AKFbFKMR(p#VQ(Ey!4(Y>-AyBYC=?+#hRn~A{ zPcK`h%Q~zQBTSUn;-GsQmuLwJ)LlataXG?C{Ya)#bJP&jZzwoE0{oMw2=iv`bza!5 zgVEK)FiPTqBv#e*F77;N7cFPUd4x&<0D8+A$KCxdQX!vOLC)v`J*Z#~e-L+b4=tgX zJa;_h?rKivu#{f4_Mep^sped*Dr)>_M(rJAE)WdE$_2>>+L)cf9Gr|cn&mlz#f3+& zm(PJge}yKyJVXoqVkZI1`tanc%j1&P;>Or zUGj_}qIcWYY8;^6(vG};h&UBqa~di-bh!)hbZl7ixpb@<9L(5SjlEs^OAZ+-x^!&K z0jyX9?!%=&J&e25Fi=-MFA#5J+r%Z`JiTp>VK4zrQ;|%t#^4YC)QQOM}`v5NQa4#OejC zTrsj0X>1~Cq#i!2sL?3*j1qm0dQKnhV$lt+mi;zjJeY6zm@L#DIKXgk@-^6T;tLR1q+z-Z zdmbMvKKpo4L3hb(_2ZG_MRPU^e#^K1?pr03qlFu6z7I%#R6ajiJcoOct{sClV5R(Q zjJO70vrZ7zh{(DB1ktf8U;4!Re*jEh`ov!%;|r9?dGfC(h%Tts?nKeQJ0!WmRsj=Y z-3zJt1nb#q>j~-`bC7@K(iJC)$7!^5PZD>*y8ZYhQK3&}vZ{sU9Jge{uK^*)2XSTR zlf?j?4ZTZe4DJ#tAt(0V)@{ zFK|LwCqRCZ`zj}xIssNBxvz7=8J13g9!u`qoZ`0%iAe5;oRF(aV1~*4h7)2s0W3`J z&z#`*i5>$z6PKAZAQ9g3281C=asZc5ussdr~W?cSQ@f*5Z){cdW zy3xv&BgcyN`n!EUC~PQ|isoa58x;|aW#7%7RCOE`RZ^+E4TgM<(c*?uERtJ zTo6BenCO|z1}0$VK|7AK+=mv7t1pxtj!S5DK!uJeMKWvOTreBmq>AWO%iaZ)qGwcx zFM@wP0s)ndPZLMb7J2cbTN|Ynr98lkJ@&2`HugM^6t}dE6%O* z=rcs3C#SP5XTQV(&Q-_auo2`RM8qvr2#QXNOq7exfS=8kTh4&MRDSJp*>I-VPCI1% z`Qk>|d%S2hf?w=qk)Ukyqj(fZ_ zL3Bv6ZLL(npuUw1@G0v&LwCjRPbt!51qwmk?w#CXS}pK?%9F#tgbeD!>ZFU_ol#W# zkKk$&?$rt^ecRpdC#He5dg4LAV*ePADFduB-Iv3C@qxbs!D1V$6|z!YnwM);IukN> z3vr|&Q;)kqp1+79a@r&jmZwh=7sD8PeUjjdLg2nAmigzx_WD%zKUW;b#*3+qmz&NN z91HgJxq>eO>3$wOidY=4JWpKSd7AdzVX_>>*$CCdq%yN9wDKRG^q!N&5xfJJ%4j+7 zdEsQSWau>Qr-Q4K8{whp!=bv7)VWs^{BH~}c|nDXyXC|I3c-NGp52r3#c3#{3o-88Wr_grz{z+%}vLlhP)QSa=T;@=$H3!R>G zW{K{}FHM56gAElnTT=f_yfJkYV}D2)G59Gy*1d+6AOY9O7`hYu`EjshgkkAI(mrWNxnU<9Qz|q7Q&~pc-cQlBe*zQ zH-okg+DvOJkW@Gk$#K0j8w85hm=3j@k~guidlAJh2p$L$a6<5Rz@MfiH#$au7iu_h zKnZ?_t%q{ZCsdGJ52FmE&VDsC6Ve+ediy4TQCaq9so+FIA0h&pFjfvqXjQw#Pd zT*@Wf&xBPkneB)FSlQWP8r&DHZp?4tfpa-S#ok%7p5M{F4!apQd5zI zp&1Y4MB zj!&psqkx`4hX3C=fF8;cU}I9$JgK40pzLy?=#-X< zMcjZ`Jo7@)_e52eBRSP%;p!nB{iZfg4Es62Um*rDFR@cwxdsjUDU<;VqLDfkU=6Q^ zD$kxK`#HJmLUGIpZkHqJaJj>MP*4=2SRsYaiW#5x)9{8ZF`l!^wF8_h9fObDfnYe} z{wtb+3U1YYLC&~H^h%@}WUA5aacj!O<`|L7EegxZLR=Y$qar3p7=e`Rm88c7 zX}H1R=~HfSxZ80rf(s?`o+8mHiNh_*aa0LV0DTo;#G?-%Zn4muI#!Skcp3GWUx1Ys z=Z>z`FR`FTsehUTXhG?y{sCUDfINS18gINeE&RP{b zap+@TYKCA@0i|j8X8@mR=W2YlRWG$^YO9b>UIe{$mHfv=Sf>$^dNCIIw`Bi|MQJLs zO1R)#Ke7tM%8^wjBC9UB`2W=Q9`I2W>;LecnK>a4AR8bgfsjTEeOor!NLx`5LDk%%~2c?wvjm(lzI{GpFGbD_q*kH+s zGFS>N#OffN88HLKPrPXrD^-;JfM|VZ1=}kKR45R(Qs@UPE3oYqX1P^OZRy2>8f|%{ zEw~Vd`ucT3L}pmIETeS~h(R5mS+kju4K8TpDGhxQ7}>k^j0GyOqef%HI#Kvp|A|*B zGwU{R0rs2e?gvF9;{=U;Q1k}#bMQgY1iskOq9|F58cek{PGiJmVI-O`8Vd6_>Gjd# z-P)6&{beN0bA$iz^1{@I#5f!x8vn4U7abq5pMr%|AuWAaR5LEpiw}z@!%GpU6iSIK zB6NR5%&i!OF|bNF_lRgwF)Fg`;eAvzsu&esz@Q-@!Z^5T-J@c3%zQpC_bYOgh!1M7 z-3~L2EHQJJ(QI=U#fHwf0v$jw2!Xr~C-mA+*jsP#ORZ z^-Shrm*_8yyfw2KD?$U66!t$AhnbRP7%piuK9Y|OVIV!^?|8|-X&@d_-YE**$&!6-jGU^X1F_Bw$pf<>I#19zX6-{kwq~`Y4Fn3bj==}ETsV-6ymr~B zWHW0NHTpqv)Pc7ohpGlYbBNJ?2$HyR5R}=fJW>D=#zdz;!Qo>{FgKgYj(qe}LSQ(o^8ook(9A?K?grk*)ZjMSS2G)D|A99@t%iAozk}Wg zg}5`k3f*S97-oDx%cl!>Vh~L$^~NG#Y$lcT`*c`ht)`|kL_r8Bsvri^ z2ENFOv9F}XGlaL~2L-{;{x2>S8hgF?yi^*7Wz3*W4?#cCU=7jskGNzlY%zd^=)LHO zK*(7c{)tQ*fJGoLfO0UGtCfg~EgD*)o2~BWu0+FiXbJG64z_?G91N~Xhgwh&4h4eH zp%xhEP(JTNhgopIVLX~p?seol@oCgeKEvT@Ksy3S~lWecH$bn6gg*-Ypl57!JqfO!}{=%BV7gi{1C{u)O8 zu|5!YMuq?dkVCE>4r>XKany|6+*1q!)*&*Od^klWZjq!-T}9o}zh;OSBWzi~%<3Vs z&l2g*Pfl(@P9QzAfP@YI1$h0RF@=fm??1twPfl$Kok<6Fi5D&U%mO`HK<~~9t99C9 z!%ef^Y!E*a|d9c%bV@oGW^ONEOc& zc_9GQd@dyW>I1W?0svp0D;jDe4F5Bw7^!S)=O0wYT%Qw@^lo(P~?!{>>cjn8P? zJRrKs^zV7%x$u6)*(82MDUZ(=?G)oaY{?Z4`gJ}`6QQ7ApwU`~1t5%NrS~lm9Ya@s z9|qij9zs`Y|?OV&x`@I@y=0b!f_tNl^NJNT*blWdv{JhUsZ+-I4l;ss?Y&H>W~ znVh?hEysFRI3}YN9zlItE6bA z3&_d3&ERI9g?>P>iIT=y5JJMHNvvevV^I1+4C@fHCs!EQa|m99Pu;+b z)mIMbFCv&kUxTKDBACQvTBL&_I6~h)9uq+rp`!TI7LX~OG=gDZV}ym`5Gq0U^`{}P z!Mdc3mc1tGc{f`a0~ErtD0)v?_T=jTy;7Tl0Gl#`&{8ZiXZI%Bun1;JkJ2xTM6S`9 zS}hjsc`0GgVxf;?gut6EX5f}Se$O$ov^Y3lAIBJj!R3PiAbkN;Kz`m|ESh1d^~_>X zzZNDEqNC{sHCQ4VCr6|)K2|qM8lyXxh`uogPQ$X`7uvo=e4c(ryV+XUrs=yDWSG$I zu@b$#RNTb-y)t;DDP@^>7sgfJE)z2uQ4L)#Tvy!stY2B`5hHh6n(h2ng z*#ua~j$GC7)k?ztn98KUR9v8QcDd*X-pI8=v`PEVnNi6KAi(ML>I$s@Ke$5N-}RI8 zS7OU7G?l1xDI_j_%aPe4I{a%aQC%*)b#yQ-e_AA_M1K}nA$m8WZ;6hU(b46V_?tRh z8q8MkK$V}IC*L!8e{r4)pAlVf@Q~O53i11MA{Y-L*8eCN3whuDkj3ZIpy$OkzC}6^ z9i7U7@WqXnmPwLbA2G`lPoT5I|8&QLRZphP&5g$Km6X!7v@VYKWV2Ohp_xZe)1`T8k47SUGOj7W)}hwNAK98luDxZfo|iwXIA4dS0Z7%Pyj|c~R7ljzBM> zOo-TLGh{N2dQtR^j*YCDoq16tMn{Af-Aa>JiQ@)V`P6E0JtwgAt5IZi`f5>^9$GD` zRa^HAdgC^zJLeJtJr>KtigRXJcCQxg^4RWiiZo&wx9tDQXXNJOpkzxiX1C;M zCDmLbZm6<7TK;=rjnGGOvvS#D6}PPsR~rvfjkUsEWx`qv(0DB>Ceqz&MI0ze-?gxF zc^!Eg_8CljL4@=PwfLS{v=`Tkr?3t5>UE->xG;_;tON4G`?+-@*d}!1$s31(!^g37 zu^lddy8lQdVDt9~$nyWD11NvJ7=;qot%uN3M*G)`uDDBwiE(x&TZywA1-2wxv_v;B zdFk{rOfSmlsh1(lmeGcnMK^GXRbCOD!FXQ(3Nl$h55FRALfoEL#Bggk;{xtF{lT36+( z*)fyslj+kpQ1Et=ZweP!SMQtX-H++vH$_31S#@j%qj=#>SQE^lTgyaR{W)KB^2d$L z5TFda%qfOjFA#<~G`&ouARWJa(di(+AzFV3p;~kX+7Vp=8IZeKv=jb0U$D|ptLEm= z`7-DbHqo7LiEG4!&*`=@(I9og=M_t3iJiaA<`gr^aOrdCT`M}Jm4Grgi2Bthd>$!c z0$sB~+#^<=rMuq33eha1~k^z{S}^ zKVxJFy+Q^0#0}_&P*wr(st*c4!!j|OxsgbSb20$X8a-rbzABlQeY7GDYgnu>Gb~Wo zcMk|AUY11;KDBNJVdfq9j|qW5yW;*7LYC0(4dO~QhE41eH;T6bKc8$Aqe?q&fdp4~ z-Ez!vz2^!T)xV5=Fk+;Czbo3ormpc;*oFA$x~(Dye9ri-U;*Evx3(f~I{m&?OMs2u z6NRxe*mkN4?4+i@hf01<>)sP}<1UsPjnQ6&XA`FqFu6kCLTAM)5)d1U;`pOer@SxH z+I(RhQ)tT(HZ0BjO;SG7o!7MdUr_P;qGkHR@-jI}#QD!2K%q7LV^B7ZMq7_WjbYEh zS{Lkyu*}Rq(E$zwK+%J702ggWpM99r; zvcl0y{V~6h3^fNhv%JUviW-7v-w8P%$9Zi0K-^k$%uz-m((0H+l(_?_6kKA4!ZOAp zx_yWE%OYQ2h18eBJ`~L@wjoTup8rtXTrs4|PSL)kRL>I&(irAq?WLM{aYe8^WDzgM zJerEx;*JSeP-uZtFLs9z0GKumf`(bXnN@O={-f-`qlp>qrQT;@B~yG{)U7trx}R0r zAEU>Qi<}IImj?f((-#nF%*r|ED(d6Ee-~N4C#$*ac6{Kl5q@FaK=v2#KI|M zJ4I_gE^DW7hV5K3K7xVvdV1g^5WO;5^^pj`Ub@CE@SyX*w1`RQU2FcAXWVSZa#PTdC{7+cZ!{}{)b2EAt zs7?D0u&;jDcb@TG)(YuLwjs`|S!_SFviKC0k5z&Per`!yTHCk zHWS|pC*`<#6@1UIFH%G)`yy4_%f9yE;-_zbuTI$aH2XZ-cRMouoV_i>k>fYQ*C_0J zo_)HeT>NY7ixjw>eW~G;huO!y7I#EI!A!m017zK}Pt^55Zm$ET4jLNJJzgf@g{o}Z zmq+|w8|D`x8rbwO!*J%?;E}FSmb*H1`fa;NQBxk{bCnD z;VcdOSU6a_VE+`z?J9D9iUpo|^vI{spO(>c)@%2t;tKfUJ`*i!>V+Bqr?}X`-x$2& zRiBAYIN5jNXCklUy|E}Sq)+7gm4NO+iJ+L}T{yh!P9NEe#S%VM&T1=Lg|OnAeRls_ z<1WbnbNZiVU`Gx=N9RB{d@fo$Dg>ccdxfP!viLf%9vZovS&atO&_|h{sx2M#?dPHj z5Wo+gV_CfXhEf<)QmrpUV_Y_z_l0QF@lXD$qQN9Eb#n^Zkr9&oe^C41b<6*2?Qi|N+H*xve=O%}F`0qifP`uFM5GD-|MmVK z|E&Hce^-B9`~TSc{}RG15m>?Je|#YjZ)L^-)xsvQl?$9}z7D}~905DV(kCa&#-3tu zaPYDFFs|C(^tIS`-C^E@JRn0@%Z&IMw#}R$QZ*A3U+b95`<5-=4EBX~D(mgh;EGnh zwi&1Q+WbfG#yr{o4TRYx)a+Z)hRsu^@Or!6<6)8mDiY(X#lzj^$3mw!cCz>~b zI$122`?6llAY+j@-5%NNXuQz^|3GU_W^Mk7@6IF&Fauz2Au|Wt4nSI1?us8LaMxG8xtyk=5*b4q~OaLR5I2y!82l zV3DDmefyBOvDFC}nE7Gyfn_efYl>AiAlgIVY`&TewK#||Tpl-r+I=rFuYQAZrM{SF z8@EbtGSX`ZaAmL#H)MpQi8^wN1XNHzKmEP%@#YmF)h`$`ahUb=)6ZT$IraUW{A z=7iWmz5XS7{jGbam1gx3G|R`vAZfF=&a;za8#WT3JA%Cy8z}LpxC#S$=TR}T`VQeA zw*rm!*}oFx+%A%!fv2C2iU)Y82OZNxJ@XhOuo*P)m>y;LQea>yEJ;tst)QBR#cjg= zSt%-a8+T>QDbY!kS^Ji$-zjk;>!&p=FvF_(pWons{qL2wT=Clx`tp=$#ms3HK!XWr zu%Cv-$S%6}wCIex9A}*tH5LNZD}2a_&KXJqo323?OgV-BN1Oj3*Od>A6#idqSayy0ez5s} z3i&YPm;p!;-9B*st?(W4wJ5(O&g8F5d-0_$)@ zOJ)QM@)->~qgO3wo)Oo@jCme&{B=5fMqJOM^dH0u>T%W@B|xQ~vrnAmQF{KJvjV3b z(Yrs2et5O|Nz9G1xj*{{5JVY`{S{Vhh&(TnS}=~ZXpm$ZA_;RDwuZP|APqT>nog(b z=k*SenLmqyf1bC#*t&Aw<$?cG-ZL*(@AAO^DeqaA%ey@Af69CIOB9^udMo z5C|1uXUnLKA=&QWTI+SMA^!~DQ{P|6LTq^5C8RgIExN|ue?X=sq`-a~EXVe*fM7qw zpA7TLl3TKD_5HiQqLxy6V1^oyvR-Ov?|FH=uJeIWxRrD>daQ~&#;Pwg} ze5F%4kUM;To5w4a@|ep^ZbKWrsAM0kqS(Kby^N9LNFHT{rv|lyj&6^+hP0y&5hc$Cq`Bg69jcwFVb+< z3Sg5@C&3zoP4k1i8o|P9gyCSjZ&d0UZ6eQ071mBnxU%WHx@bY!yd;^F)Cdb2k>w0P z9PUtOgiNiIWWyY5r5v9mBP+cm#g+=pesXcC;o?vi*7$|a0;A7j164K zSeY((3z)$yCM)y!5AO{-RZU)j%NTYh$_(ljC!5vBnG|3_d1oGTqO9nL4In0S40cLQ zLjo{j-RjB)^l6+7Bpx`j1w@sHlK-$vL052Xb=f~r-$}qbBB75k&O*tyzIRoG5J--ZZj6SKRWHY|^YD#aFkg*<#c$tH3=y~ySMGfq>*2r3G zS?N2wbhTa1!ZmDQ{4cTtYj?V5V64wQ!vh6IOk1eL_(Mqz|0+%@>@(=P1X(Yf1K6VI z9O_s(SjSx;2gv`C9Hf6uIESYbWJ`4Ro&@Qtwr(M^#fDzobOr4|GBv6t)2qIO3TxwW z>R4OWY2<&46Ge{SK)V5+tX=SL;s`LB_SH0@mKyQz~XB+0$!T19uU(icU!e3ttker^06^z=foj&@rZJZQJb!}7m4NEq; z@WWnuC%fnHuBY>Y-c6N#VxQqT&UdJ_Xdp9VpJyMi9^KqPUJdO7XQS6QY;h+Hw!skP zYm`>|HO|VC7A_y*2N?1$%$lanF!j3GIij z3MaSg4{T~69TD?^y=k(&r8R)a2HbOqI;P2XtTiY} zll4NWjS2K&n!M`LboJN>bM zNT*@d#|heyF5O*!!!i=v+p#+2!ii4knuHh|Qz1=J!C#)kt_0+e%^ySSqch=veA@|p z8ai{quzx_<5%ru!3M-Kx5Xy$p5$IW_A?v7Vro6LtHUnQ4Yn1DR_$?TN!e;?v20C#w zt1$$aE&DcujD@0Zcz;YR^PA{9U>p!VrU2h&%1Kbt1RBdae3>|FNDfi^hBAlV$ddgo z7qbQ|%*lqbx&0z9Sn!Mk#>Howsc9oQuA*WlqO*Cskz8@P%>KTZ92(R_rljzgVVVO5 zVbY#~b7s(y*l2 zQ)GsnGVWWsM~QJ-x{*y0v5mEjx0w zPvM&-@S@meGTpUDA%WJvTi5|(fbY>EFoWTifxPJzI3)7_#yY=kbVoBeC}bp!jDRLs zqQKFgE9oobSZm_zn{9qq*gje-sN$N-Tm}bLa2=v<&1KW8wrQ}@G8NM+032u??*iqz zjW;uyw}_bAYPJ>i(C7=93vHQtXiEgmb2^M7hlf^j{m*bTOcH2ND{#4OTZ4ft zqX$|`zK!e6){@t44z-r88Cd~xvnjs~@Y^Qp(MBeR0GKAVk(m|aH&=)+a>|_YCNFo& z4rvy#!RDeXGBG}v^!G-GOsZ+^h2>Kl0ViAOkV7`9$9rD^EeOEgR}E)6ULZcpDewQk zmGYNUdU|1Dz6LaQe84%J05KpytkP%6uAo1#K^YLSO~4LRj%2d{z0%Jftn&ld0l=us zW!C&@x=W_Sd}A%}uW`vSiPj|;fElg$3*Um#$t{b(>#TIk1@$k>g&XzIOR?0mx~xM( zJ+eM?-A{Vtb+}>kYme+hALPiEH5d=_46)A86kd4)kIGG6*)WEuS%#!_ZDbZL@ye`p zD}bkh0jP!eXTwXwu;kIe^+EFLmh)h0hDH*5=YAaZeeTQ04#7OqgMr zhD|A{EmI5@%TK-Y+Nv1tH5=N^238z+eBYZ-BM>b_pf3MHh_eC<2ewIXmeMJYezV}b zTPH=IvP^0C$FtB_#gZwl&h!JDF|G&2)}sl|gj%(eZQFco^@>~P5LAX_(H$a5Y2#hM^=s#berr_OvP4zC+fvx5S~(ejPZN=)k@ z9RqkKi}IO_Ppz43@WFg2WdJK8w4IN(+OcLbaNkJH5=CY*>T;Qx+zIP^pj6gWmbAR1 ztQ#A~?zE|+yoI`Sk}g@@{uV8}Qlm!N@0UHLr((bjW~|>{F({p;`60Ph`YyR*BK|Jp z(x6JI(rM;Za!z@@W?mZgx@Jl>hY;N>Np!J?OrRE>kK zuf=q+i|occR$SqKse3&de~lcP_Mh+3q=c@rO>0XdsMQA+)-l71MAPcOXPHdv%`SY8 z@!+n~6V`y%>;}OY=K*zXLcA>{uarT`IgZXv_7ooBIQ{d$hy0CPI?RTO4gn zSvnts6QWJ+Z;&nBU-E~E|NX;LQ6Db8K{}ZG04@jp`t$~wlffkRuKxmLdV)+VZ_wPH zaOU`n#+*BfPejgu@bF}23x>;`IhlWbl8bx6RP z!PTOY9VdVR^I4>f2HqqyJ7_(gt|Rt;GT}vadFv!TGq4SEwYA|*;LX|wj)lQFpK=G& zw;Q9xlXJXa8J4wP`!)R-zsmyS2}_FHksVyCyRH7hF32%pvLyIAbg^ur3yyV zxwuvIil**x2FJZgdyxXPg(r=!;U8YdU9WpK1 z!in0jD%O8=0lHs<%}3Pc4%wn&?5B~~_jPQ^-+Zg_X_x`#F6Lvaz=SfTI%A!X0ZAM# zG*f6%3}g@Stym~$@|a=C=hhcTAi1;n&u_uZfDzZ;fal4DX~m%%>`b?5F4Mz!B$#V0PFIQ(x8>iuul=>x(6~?8TGqIHfDf9sbHl; zla7|$BRj(c{D*sFX8q8Dl4Lp%8$MTuWnE0!%-%A)UPzF~=Mi%#W`!7%3wz_hVc-;o z^g}eGH#pJvXhU!57(-btXiHaFmu~1IQ!W=#^-2u7?f`o2 zW077j)I#oz%8i`T2Ou+>UgFm*`lyfWopbN3Wc?-=Ad50{jvIaWY6GpX!-`Jzgz7e}#L$x81?p9R zrtPPa0$Hy%_7VX4aa=B|_s~ml@O8CysCl7Gg*2j3yhdf{@Q~bVWI#RvGHw+?#(J|S zLO%zb@Zj0UL4t*U27OnEvZj(zghic|-KajaIXkldmRC1vWwfNPY!G_!9{f0*Pg{Z;|Gw`F8as!o^aI@P zqU?T^)3U}iu7NO0j3F7e#=uOC4Lj6b_T#j?A1L*4`l_F7P=ayCuwvXrTmw`Hav>Y# zZ9U|-w$#DBLA#@4KZ%OnX~ovn-KiChfMo7(`%e6+A62{^@*9I$xHuixG*mZxOvT7r zs0(`67>Q$>G=^NpF;`~r`RKG$Drtsgy-~?TCz+e4kO3vj!t$X=T z(Wo@tcijiblq;fIk%YnEs-ltzg)oJ>D0^XC@@2!{H)z=anV5A}#U*kJdFD>G4(#Fy zYm2L+L*KwQ(9Z`*R|*edT?7F!X7L{lLa57G${Hy1F^0Vc$|PJ~80mzhQ1{qdRcuFX zoiDOoI#8zM?*$lkiQ_)zJQ?lU_gxzFu?~voFVQ@n`}W@S<3O34gaQ$UmfGSHb%FNk znFji`;~<&H5EYJ02vr!xfXxPv7Kadg#a{ejBYh5m6Z0}U4LqS>!@m6=q(RP1Cb*tB z@zP!pAIi3JCiaCtt(k&hxdSLz?#hvi@K=!W?l+BaCH^B^EiV;&=%^T5$FR&Cb(1GR5l zbp&u<+5ZsnaXi2gF!)`@)eQSNUg~~j0GSGl$u1G!lNEf^D)^?G{Wr=d-_7B>L-$A6JS!!m$~V#4GubBwfpDRu}#vm6C zzI&Mw7lT7l(VG%W!=}x?RPo6sKNb4jtGTw7(dl3`D=vy{Ng&-q(P6# zHYLBI+i!;VRkVS-{bmeH8s2x|sTbw%4R=PA{~fq9qx>Jhofzfc1$Ro6pNEX|4d>q- z?z%QxNhIKUE5PQz{;EjWg9uBG`sg9JdECO8d;>RU9QGfCyIPe0DBKOB{KqTBpS0a- zvvH?~Gnfc>w0{Bo(Y??ml)-~l?6wAQC!#00qr?6>@W;bX-Nwm0>NrkzDv8A1iMUk6 z@h?0`oR-TQF>?5*!jTzQ4JjNoC^&dz1Q7o@pjwluFEVa~{Q3(5Df`1792% z2n0NQR1j|NaRyFqDFZ7{Kc3>@AD5Rwd&kKuTm*P#BwURAYvTRi`<>d4mvtjZJPdab z>}^{mvo?Bm>%78#1|H0KJt4!@;wm$6q*JeH(qDa0zoLxx83TiZZ4O1%rpeBBhqBu| zNbnRMp4oJGvfSVZXUQ$h#1rlB2!HE9#)#msQNh85!9d2Kz))KKr0mez_5*6x4bcPe zLD(_ku;{8_@qm7Vi%?L% z!No%&uowRJw@@Az@CGThT@3kUvfaj%qIlHk(GQLoRRAQ=Z}7dar$abb$A@=SyhlDb zG#HxDR613D*`RrfRjJl^b7edsza|5;Zjx*oi0+Zg^$@;_z|b;$9{pHERX-i~BBco=^87Y5NW^`dZ$aw0|#N9{|7V@4b`pDy>++6`d5 z6UipvNx-wa3e}u}zFz}aU{duf#*5n&_MeBJ>l^k9Q87La{^n8f@$hrI!s+Y4&;1+* z`YX_agINkG@!gs@$xC6kh{ zzdlN3oD}wF!0(RoH-f)?l)pLr`BDD&ksqU)@b_EbzdFi)zm>nRC6sO<`FT|ux=|)S zKYzLGWYnbvhb2|*jnsYc1WJ8B$PThsG2G#C2N>G=;$cM0zcBvk4>wPrfp`Yt8H^{4 z5r@D%6wfd`!}0Jh{M`uhw^WIA?1)Sg!8odRRJLyrM9=WJl%m&oI!2d2l7dI&s*<~h z*=+;yJc4IBo~3x!;&}(pCwLCxIgLjRx7+M^>f>?Xxf;*Scna{0!m|+1$9PWQkt6K3 zcs$8?vhcX@bj5QUo*oDy{fg9x__dW538%HTHu# zT#w)h_}<_VsN1{Wy1VWI|O-KDpx@`{kNzemk{>vujGFOaon(Qyc_v3N(%d-_R^>$>>n8M(~g#^HJA4X z%1sH!ufn&{<*nfsrP3X(RDD-C&Du(7s-aHNW!8Y7(G~K3u9fP{Wt_shB>eqLq3<8X z;PVuZ3$-ciuU^T|`Ei}X@x$R~Bo_AH4}Y5|KLZof@^E}H{FuHak$~<9$cjo34}Ww4 zH7ogfQbfn|1c~jc!Wg*u2ZX4fv89Rksy0Td?I*iU;8FZW96wd?Kpk|1`t(%|#Koyp z(pRNad3vgZ9Dn=VJ?8DDzxGvMBpCU${6SThUV2D1GV-YABg#pGA69nZ>q=+W0`^Xf zR;`doNBqeVS2(EW$Er59en>Si+CpSk*BI?+??VdrBn_gmkEomJ;}Vr3I$TS}11gP{ zJd6xFat3}}4yN*b(~7Y zwU5Q)l-uY*PmNPfqbGegP7N?_q-)2cz?~!^nO^29LPyYFfTTWzp0LXwB`^;}MmfkXP(=I$e1=MZO%L!{aHw zgVs$@BaJ($@nfouxT}zQyofPr{~5Y`>JzwW@P)@P2zPs}Qg=Uwol3o^#pB9j+(UgH zS1zMB&Bjag$))b&QB>mNDiJH!??0|Oi2@wWm1?G?7Y*z^JQx@_a7bZcsE$Sb0{0Fc zGGb)E!VyK4OH2q97Ucv2!Q6rZUrv#`$Q7LTfJ*0atw~=#p;AS0|Gcb5yU__JPxO;D9`0_6&4oc z`F#B;Z<4xJxq1!A$BEZi8C^L^r5FS0ok;++L3C!4$`^w>az{6%nQQ!)eaFUkYiX6Gc?p&`o&+E$dAC9M^Q}I!obp z2^up?4R1Ho@I?<_NTZjZf6K^Xr}qX+zZafjd4-Og;(}ba$CKlA7Zo^XQPbI~v7BwR z=Web`!PzQX&N13pDCoJ_7?-)UdA1sa({ws4QcuzLIm(XH#ye7AvC5`db5%0VmsvGe zxy1s5_BJrDr5>w+KBmt@X8Y%=KInao&HBEpuAyqnRebFNho>OuLUW39eMLFN?nNm3 zz#&zi-kql!V-D~7Ugc3wQuXDMaLi;<4dqf}fQ8G{e3gU~FH+~L+i@51sQExVD`?ex z)m=Uf6qY$xHKrM-QHLsPkPesB&Qon*ZhLjbn0m%@)DB~gspNA7gP85PfgD#sPT})3 zdVy+(HTYwTRFl$4PpQuZodl}0JA#G2VwXGDlUr1rTacGSOBbsS(%I`3TDDkq?DDG7 zDS8wxkLNVB1V{89JZexua5x4%2jG_D@Dw^d&O)~{;CPKQYcJisUZ;bL)it$Bjp`Q4 z8aZS@aPWvXfUj#jiwSn!5-k3`X|$%9(^L{Yu>>tGqoqsKRj}tgyhM$X8;twtKwYy3 zy|h#{5gRLwO$@bI27K@~wOt0t+(Zv8Q?2D@WNuB-<*J_8VlZkiqSNoAE5BTh)bCgj zFkuDU@6y~AK($*9+VYM{mhTxU^wDXRQs;fcuIpzy-snV zFQ+&V2ox1Mi}QR1!4Hf%206b+Qx-j~9uYf`HbjZ);_l}yFRD(o;~5mYlfHfihr^SX`#AtnlbvzTV|IOkDUXiLqz|7*$9`!rUCg4YD^(-$6`K8^x}Ba_ z32x?V-NChd&R}tIF?uB@uOK%k;Q7X&o}<;h6!U_rE58LxfNIwv_Y3%Zzd?>4l+T{y z@^V`Xiyb+EqMV!@+VYZGNbO!kg%5D|-6RiUPVaeHB~zPKDoGp)bubTZZAX#Qmm6>w zc?yaPJw-0h_eOV``5b8G$Ez@N4ug)vmqra=QmwB3!SMfcC?6SoU?gC?sL0_1Sr6tF zf?9Z-o_|r#m()=CZ^KDF9{~-0Yqd%fN1}UrGz5L)C{0q%JUZI6%@H#1qFe^U#R}e$|uiZ;%WY9?rCMG^)G{G`IUCPtn%b< z;HQ9{+zFn7A|Jy`FxOq+b`?5)r}Vihr%vVIyPUoL&@qXZ#JGB$H|YF6btS#`3c&P& zK^;p$;?%3EzVRnzysC!bHltaus@}Ne23VmXCB6pG9wR94HI<({R@AbPO7W24n5eyr zStT}3bR*{lVC`kEDUTd4t}FM&pe7S42Do}XRx!Zc>v3_vg>jNfRa4^$>QJh>%8Ajb zs@&!2HAze`BK%VVJ@JORQce~H)_9e^seF|7rm}aQBK{9>Ry;6pFB8zhJWo-P$D3E= zal0Kp_f!$VA~7@txymP_3hIh8n~XAbwVW<)hF&PS}T>^39xHbHn~28&q2m(3du-c2!7t+7|W=(Ai8go#HpD1klgS zjcS-&Aj+ZkI$ZdBVWY~G3x$`ANx+MPzg4kx%iAhRE)qQ{VLhh7__sl)7Sooul}{`Y zj3YGsUrA@)S0?q_ga$7a!z`M)XA^kgWkO?%+Ciracq))r?8A7Y*zK; z3Q=f{MwiX1n|xZdw)onW9QTaqWMQ(6PpK+&a0nyFI?*V6iXostCy!`?skEZC?|A$09QJgQTI? zQd!37>vZM#3YmfM_}T9LpkVoxaG$17w#SS;%#v)f8T~C>;M+p#OXR3 zn|1O8bTFicVzz3LU59oQ|Qg}K_&}2+(m^h@4Mnk+Ixag z#0Ox;4}S=A+^wP&^&G^Ac_)zZd$eaKiYwlUs=P05rM;PEV_LEkg4#BY=^(d@JX%kx z&b%Y4Iz93cIEN2J56XW>b)e5bQVDVgaEP9ZC(a_D;k!WVKeQ_L!!9&+r{(tThWjJ> zZnx@4li`-TgrBo*L|b-40^ZFrJz&)6*aPl#FLm9cTyh_hl-GU99vEIJ~ z?u9J!si3j5RKdJ`kP<%Qa&9X9eIH1rd>JBjB?2_4o>`BM)i;|nvs)#3By)1Ef+BAq z=wlwhgHH1v5I57o-71H^`BcT0KKUv7mG*M)Bz=Jrj!@uB)ttWi0xdbJ-3d9Kydr0z zH{dFA`ErW`zGGIB#4nNLxRoT&IO>~DEJ0x zez&q*{SBi3z#%Lw_TqFsRn0h0N54@Hc>!6Lqq-};MJa!Z@|+vWzoLA#>FsaPqZc)T z7>TPd_Wpsc9l0O%9ZUNUq1(&$LxLD5y;h}v-470VJjH(p6h9#x*#A2OKBhmlJ952l zU!Wk*!k+3tS>C|1ZGj95BQ^CS59X|8=f z-FE~%@C+S1qEZur0kE+_m(vFwg##?#vpSgeaFXZP-4X<9_fhn~N(P7&8h#X=@&X67 zptp|#K3{|>fJVa&k3pHUiu>9_qmKcJzGS&SWcO-1atxSdO}Itgb`vfu>D%d^E*89 zb$O#D7+-Z#)dRKYeNqjQZ^!|(zNVQ^7xw@ur=J2fep6DyN6J^HIM-3+&hZvOW-Tbn zEpnF0o^2`q5rD<>r=UK2%j%L}Pa*3K=ywep8K+T`jg)s9lkshs{s0kcNxDhmCYjK? z_x%M}F}*i97%C6eQ*EXXPpcN#cEkN&+np24%Z2LB=W#pSpepZFXk+0SdnNQB3T#@RU*Ov(8&T#vAa$+a1Mz783lvVD z(6C=H&Yz-=ngDJ11$6f_+0F8abE=8_T()Gmu1}qjvGE1neomzsU()^O04-n1+qlq1 z^wv2v_XSoSymvYN-<()B#i};k=qGPe~7*JqP%F^}I@xr=_0`{0=be zRL87CF&9w88L5>vDYWSVZaOuK1@h^3k`~%1szDCsH zB6!U6k{-FJ?vxkg9h5dqbxX(qTL(*>;lbjJ5qeZ|hr}dBeMSP!^ctb^D<_#uH!9qrIYnhq z$6o+4Q)#KfhV3V5i!wbRDt%1zChBUMbwO0%?q|+WL)$hX=X=16kC7I=$YgOW+7&BGQQ8&#Spsu2eF=jmIPJ9)V zJdd1s#gIzB#p|Ko%1ng_q-(Lg3Tm-{zN>=%dy4F_rdKWma@T|W=zYpwt1t8-L5R(H z1)iKB3id9d39+U_E(QbKGsdhurJ1#E~m5ofPwb-U@s zI27{Sr3qZUo~NF1rbn&>BFDVXqVM9&Dj+Z1TNGE_w3{!g{!tz5r6JYL6uGK$Rt?2V zYWBMLM9>-iX?zXSZmhA~SFwAo<$e%uxh|>$8i3HC294{wi?KccMMKb6>?|&Jy{z)v zHW@LJW}ZZ^^hq`oD6J;4c|~Va+voIoyxzhX;dHG9Fbb3$?#mk+rb#7Fe6`EUQJm>}CQ)mmzjD zS8h}Vba9aCNq-+ zK$qI4y~=jgy=~EhgK0}6%+Bre!TQcau>1g2k@0!l@KGa1_8uA-+^=wl{^&GfjSu1E z{iV)20N#(HMx_Q#u7j_4p<6YzeWi}+0`2)1lFL16pf%xJC7QR(z3700Rm=u-au_Q5 zO(LfMJ~bpV3u34S^t$q6^-v_}CfZroj8~ti7obI(3YFU6rK&wSl91V_aO_M1^7)LM zsb&lEC7U&|M-brMEWzvb`GQ#M@C5@o1#Vy77dn_$B?CskL|)Ny&!5TW0P`!gszRXF z#$mxLhPI}dH<@3n)t5#?R=UglMy;t3gWQ_40Oh6r^-M}ysp894V?7l;-eKa;1 zRiB#*I6Gj~aaSt(`XKE{GCP63eU}OZ_q~FqBGqgf8)6*~Q+@*g%@1^M1Jeznwy**4 zz`tov1GBF@0_~`Ok?KIB)68D-sOm+|Ip7C=Of#>R$HKlF)8RX=dRuX$w}U%ecM?qN z(R7UC2|yH=X73$jtC8M)1A~hO28W+iv<0#yoj!@7pO9gu7^i4ShM8lW=DPa9ET5?W z6Zx6Q3rq|J9*0z1!;Gh)2!R+jHq-15F>F_+ITaGv-YoDB_hp%Pg3Q4^N}W?<4Qu$p zXV#{R_o>dbp&=mUS9QM?-sgUZU*1OOvfsE5o6}Hs|88~P3yo07A0a5nDRu-3ymi-7GG~_davZ;BGoMg_UnS-GeYQ_Q=# zQr&4rGjzmMTGq^5A)hpt^Y^pqi{=QNMn{^PcggAIy|g^R%%po;n4RSea}ZZpqnj(K zW=j+?(;TLQG`e|fgz_g;WC~`5;%Qw=^Ay7^h72?1u delta 74912 zcmeFa33wGn)<4{*y6={IlXM^nfduG#4Uj+(5Dbf`X?6q^92eYJ6chu>s^ScKgP;N; zqKy=;ETSTUqQMEyFd7vVl~J5QWfT<@6%Us7zWjvCq5n{kL5d@^%JN`E!GgUq|2I z8Gfh)&HZk&1@Lt9e+#zs0m%v->$ipVl|)IKR<(S;zZe9*Q*a_GGp4-wLDWP&Wnc7g2|f^C(*2A>_Rs?UC$k z9<;u*a`U25D?dM&AGD4yist3!<>dxr`FVK-(fopFlPLa$qIvnjXl^JJj74*UxzSK8 zucMtC%tO$QMXlm!e$X!H7&JpAoEte}!C(+YgCPt9N+B0wOtUan5Iim#jpbX>keM4w z1|wE97>xvT3JVL7p#+7>qftC!MTpNegP~~iU^Hk3gH|9IEHn|!&9y8fH8G>1AfXit z4Fn5wqq$K7jg>_6(OxLnB^V0kBGuIY&;e8xDnj;Xu9X*L0MY5wqIr3!8_ng0qR~7n zn4e#qm!F?!<^}UZd0dg1mlr}+7P?_aqb6d2g!ZNxLVrq3bT}{MHxbJ>z4Nk*OG1ds zwag$o7&Bur&WPM0ldCdAH=8FnXnkP98IL+PG0ut{gY@8e^B4rvjrco^<)-iQ}eUHtC9K#a=MCZI@($vexOdoa0W#cA}HQqI2I(GV`3FA=o1DX4wRVa+bUR7>0 z^KWKz8UMT0ypOTjv{gQoymWePBV(g{kt{8Rsn28f7U2HsW zHpz&bI!?#FX3A%?0tLc&!rPkLftq)smkBalirz}Mn={@pKM?1h?Xe}^r zF=qwdv)-{ju>NIzV$}q8TOV5=T6?TD*7xQ;=Fha#T4&u)kC}Vupn0$PPkO}q)Ji^P zy({Jf_M1P7=S01@&D?8#V$L@|Hg7cNnm3!DnKzm9%ume&=04LizZIXGUyF~-S?1T~ zm*!s4{&B0KL&yD8YuzC>SWj9@tvc%~bG`Y3SR{ULJ!!5G-sxFs;#nq&RTTxDJ_-mv~*{mFXWd_x?x?ziS)HvZMh z`gn8pceTUvc2WCbEgZ{TMb!@3D!4`HV2q+j*l$)lLHE!hnqm+0&TD!_aL1hI4EsRs zYfYUdsqnXL22%2mh<97KDtWGDL|wxvam)1k0mA!HGDLqZ!>`srsTz{14Z|r;MX2|( z)SEOFMCt6D>X?q{Zv773QK^md{<}BBXZE5 zqNRIJI->r7%CSI2)V3@?m-2_TVA=?{>p?Ur+Nho1{sa>Dd|&%ghf^p;7|l9Pq@-WL zel)G2@|*ZM9pMA9;2)_RKA1nJBm9o+Pe&Y%iRDav#f_-XAEWqDMp z{YTHSqVOKoGd%&qNL#DYyh**!Da`C?15JB7dshZG=^Bboe359c_rXcW;kU`j^XUTb z-jjRK*|l$-oF!T=3%C@|b+TQu_gaao?61P8SX9#r*vT?bX7%IZNQ4y)2mRr&VdKcbv> z%K0ewZ+(rdBMi*c}Q+=tu`?L!V>t+V#T6k@T)wJKysGjUb^#DRl-+c5kdxE{v+cxY+ z`pkQE_^{qv8WmzB>{Zy+SQ1W?M%6ILF~qFWBXG_cy<&fkHqPN<&pyU3JE%6cf=*NP=mcgf%2VC!EH~ zL25usP$LCVX{-blR<=}+JC4zgkvu&cz2pw>(eFbrG0C;6-8o-8&jbBf4Z<}wGpE@9 z@J3B=PC5Y0Q<(@SA|QlHfmh&GVf0%bN5^o)km5>#7Y12Zy`NKfgzLROP3WBbnOkB8 z5Oj>w!8K4puu>qFhvrc0N>TF#R~o1kTR-DKq*82F0pzOZfWj*VO@()WCM(6BjQAZY zyi3LJ&&ab&h4&^^!nYZj_Nwp!75}r&q{0VO_@D~U`J5Zpc@C=hn$Nl61u9pXK+c#(>~=W}n_<%P+mDr%{UT9pxAuENVz{MwB0Y876s;x}Z3*Q@Y)6~8GX zyitWWs`$+r;d&LWuT%+J^@|E`QQ<8rygehlO@+6q_?_NWSFBCCi}rKJ@8OYm=c@1` z6<(^st5jI$S*pUzRs7nFJj+#hwTj=6k!Q6EuUGM#GV-if;f*SO)aHy#8&$YoC2Y;e zRIkEYRQ&dgJX<&n;VZYAC#d1hs7_!ufH@&K=;7f%v040gun!*nqTa5OK>bpUadTzyurrC^9O0IkmjQ+5seZP9{xrBQCq{i1K8c^{hDokJY4=!?Gb z7VfqO>y3ES-T9#R$kiv(8{WRFr{!k?6lQp00B7fJD|_Dr<*{r8a>>g#H#EEB1NIAy4=Bsc0Fgh1$)xyyktd#=~Irfs$#vtGxX zQ8S$G^;Xr~a#Cg`J-CwIx{@Xg2}oK&(wPCt%-NZ9c0*?-&1%!A`wl0RB4PEc4t+CA zlyO2us+Vp!9zp{w7^w!JCp2!Lh_jcVp2F+L4LBMJ0=kWucbH|=B{ z*Mr+=sw@Ikde2_p*4`izQAmk;cEs#6z3;Cd*`}W9%F=YDnGw?x=VX4(^C)Ixdd#i@ zdH!*B+b)^?s$xL<>3)Gp;bl9;L7X61epNaJNPA9MK9f2*)X^w%sk)a@g!y-8?P+s} z8cl-pZYXbrP@AjF2O{JR18J%E!3{}zpmx-alL+?R*KaBg-m{F=A@9IVC(uZ*V(zry zRvq!c+^dnJ<-FqHCLM9oycWT=IyiD(WrwGQ0z9s0yb8bqHncPdN+2|rpj!{(vmDHH z`)u!-dA)wK6GfpNoMA5uDACH&`eTe`shIoio;M!}RvHFUUFesrS(;p@$@BBQWmQN* zomio!BML&D(b3qw%o{qt7kS=I^IISHSLX1d%E>7cMeg-AH8nH9aU27}AA%TSG{Ftl zTPV);*IQoR!lq4tQWPSFK|pduy!X%f#m%9aF?G2Ml(FMm^hAX*hFxwRb>?iW2n@h4 zQ5Er0X>M6w)!ob>ExUoVIJusa*Yhk@6z1)@c?pg8ez)N47EeQmU`(!}M3ezDfkbE| z7GV2)K~=LIq?4BtV+b8Xj~MPgw+wIbrW%tfjXG4(4{2o+Rnc{~3@FjPMSP847Qs1; zV=kY&u%OS8=m5!OX(Ue7j{(^MsQOt7g|R@?g}ALo9DxAq|CcPhiN5y!zVLYbmfpIM z7J9X}p3~x1?kJm%EbiG}N@VeKKiMO_(nZ}uOxZ!VnKx=tv)JD2>v^b=*?!Kueo$vrHqPjxo~6(x67WYwkfduQA=pPu&KyX!WL?B|>_txIqk}WA&Y=P2b z=>)SQ>fH?t%tkBjHb1W!XAN?jQEkune!BNuVB@U&`sU9UpkpViqhVzvuf4BDllicF z49iw@kt((0zV^ZT6&Uolz3{RtXufy#vSY%#Rw_`|dMlS*jGhLUe-T;8+{CW77kWQ0 zFGV}?6~$ujO7El<#VB*$ib06Ff5kEQeQ8BU{Qg6Qo3AVlazS0Hi17p*6uEb3cE;9~ zA_hIVvar+A`_X2FyBQDrS(y1)@~jj+6c=+KVN|${a(J6pc54{Y0aVH|K7=Wo5bRl;ZIk{}S3HsxG(SvO;o^v03io%;QmsnWy z2Cps__3OP$R<~`Lio-OAsmwK?iKJw<`t`LBukK5ox2ay3@hmouh&adq3oehaU=dnH zfQw=!%>^4J5K_sS3or)PtT`sUoV#n;YrR!#h64rPt+|l08)I$s+IY(pBC?oD(9!9B zM-M16x^HcZBg*9G9ESYSYxA?+Roq(8UJFp8-s9_a z!c3`!i`!l-^fuPt!2)YK(cwfE<;Fa_qESL;PGI4$@VSjAW|pYp1SJcZjBUY^>N&0E=_bA zQinHm^W_B}QDZTSr{o*nYnyw}&0gq*QwyFtO8TANWiON>ec=m3nr9*ihCHRKfTuxO zVl0JUJdxWU_+mmEG%=}lCqcF7WoF@FT$ojX+g=>f2HGO?JEv+5*`h$cVu{RzI?m`M|^3T38hk`Xvsqm1$&ZgiZ7527o zZ<~DpLASHJoMYyCp*Op?22o-D2y6^ORidaNJ4M90 zT@8yr7{^$C+Fg(*%+iw+VCfRU8HpU0G7y@HJq)lHg(`kcM|3r zR4HcQ(XARk(D&>MMM^eJ9-(z3%NB5qqX?%>(!wbk@P$+KRm-M*yPG4>V3Y3VFz#s& z1_yH~ZXq(eQcT7pRw*VjOc+lwy*dFoeE}soLIdp}0>JVsfJMFg4Xja%xf@{@;&r81 zq{AwHPDZ@yYS>*y$TJ<8!SV3R9Uad=jE>v<7@NqATN=ybnL0rtVExm9)?5IKG&Q_o z7F3)7MiT!evg`+Mf1*A0^Ex~cPh#vRpbU#QtPr5eZn_?XdT?b*{#I$0lgq{oO+5-A zP*791xuPZ>2)s4{jB}_O#&fi9qCVAXrRatb(5O0<*WKiSW7%nLa&|q)Np6lM29?GuR*)b!)^4 zRLW0YHk4e4x}BVI)6nU*KWx$1oprnfx;Fr`CwY`Z1HCaziPtUl|gvj(sY+(j90QQ#01IlYKVNg z!>B3B(F1e|!_ZIDT~$0+V`@;E>ZZpbvNA2io1z$_I946U4j?A%BoYt#5DdKG3U)pd`FL=PDGNL5CMCksSzK~;!3gN=vT$1G}QwPj-=P>PWls^l+BUNt$TyWDh6g4Pw)dEDEIdFd29sJb8nE^7skK6}=CHe* zs!I*;o1HzgcbdG*gtZG8{L$y!=*XM(D#$g{8c$+?m@e1ynr4%SHXn!fQ>vh-n_1M@PzyEntB5- zKW7;%q9A0?3xL)*#A~!S@}u_X(!7uQG(gr$z{v5{M`t#~6lH^&ARU4D;4t-_H+fGB zF+0cms;-3>+2{r$NQ-!Ax(LQgygl}FT~X*m&WPb6X;4a_zr8?kpl zVEP2OfH0<#)9_Lqw^@X8LKv7_dmTNLh)zjF;$eVod9VLeGAPf%8!~{ERbxWraZ`ER zv=hQsb9K34^r@M3{hYarmeKs<(yvl6~|ll2Rj*>;)VwU z9jF5R1#qY?0#?-gszkJxVK~`fFEhAIwWHpVF9m7@%^w0T!IaC4d_aST4N<{%BISnN z-LM@S@PK`*2%|e{i(m=G1My)UpG2s;fz^i-_z|t-a%D!5t_Xoj*hjE=UTNg(RJ)&Xf%_0Rmzw=~@0piT0$0UViAodf z1`?Q#Aq3IIct9fVLC|Li+hkA>m>hZ9MZugE&j*b;5&Q3o>hpDdSQ*)bpw@>}Q!$+s zMiNWN7~^;Zw4`jsY@%~wKSvF()1R8P&U3JzZzvHv$V&&)Ki1|PCgLn7$IE-UIFRe) z#4WGO%LU$Fzb*7Gc)64Z1eBVUA}-K(m`PeuZuXQ!%=_S9tqNmK6IL3UaE(n+1VhA0 zU7n5?N1S|3mNAeQB9#}BF^JpuPCrzT#4b#M({wgy1;wi2Dh0C4=JB&Jk{H@3$Wpx{ z*(b|Hj0Z9&O{Rb-J*F|eT&Bk~29S_3J*F|Ny?V5B-DGtlcMQ)S3?~*m3lE4$(BhrE z#)IpG7+)diy@Ba&yJ~kd{4A)&!X8R`h%^>1rB!gu}KnRX8JUH=xR zBYZE%o#`U`*=4b#dk>r)b)|rfl^dtM<@SJz10`?QL8QH^UEp-uVM$)UFUy1L8BSR{ z?aTHQT+6j>uf6{O{2e!OnEe5PIe3h!H_!h13+wr8k>k-fXL;v+b9%Dqx zSGKfn4R!a(34$5nj!#XHmgZs8;~20lo*uxW&wY6Tm;g&`wVEM$QxZ`%L!vxG@B@(4 z48cU@q((C&3Q9%fVKd|+&}VvvAWI(4kUX9tY^K8u31fz+cV_BXW6*XY%F`f6&lrB^ zY2nj-l(E9oBK7XXy!o#bQ^afgU2E@!S8|!E;qKQ!)!?c}uC)EUCtoc+auM}%zi;KW z`>6{rn&XeMXksN}bRMy2Dtm&zXabo+p%C23UpLVQ)*rAe^15j&+{PNMo8C2B3y=Gc zu9Dfzkr>%AMs{AqDyiQ6RgwXNh0$LnbAeNEvRqysGgm{r@@fbj*Ix{C^zzbq73#_0Buzhf?~7H}i+K-ao(oxJAPv6+3K^@_zoN1u{1Kb_D&S zcJjAh!#q`c@pnb0E}ZjVju}ph>%>f0owm`EYpA7MUQK~yPS7A>S^=e8VD9B70`8k@ z-po?|7jYbP(R3GpgoizBUfhw%jx&cqL=Y2vmoX9CUoD)VT3BT{R*6rR3#&625;P0D z3Ucf&-xE#6;|+pb5*<~BVH6^FuThN+T8KX2N@`Jn5T_ONp^j34!gyd0J*5yS`I7-U z9O}q_73w6f_W(Qc-vj&CpjHw|rT7J?!7_c+lqI3+DfBQtcLB!?5B8ISX>>tPWHf4Y zp}L z$xM&BJ!Qq=>cOcMhM|n`+Txd1jS19ixO4o{EEnm1!351M<0od1WWJ^U7>3DOT4UY7 z!aQh|*qt0a8t(n zFUSh95Gr8@vqwCKJrPE*cNBv)R3~aE6p^}Hp|--}fTpKo@-w7y)t?^KNl+$Rp*Yov zbvFm$f!(Wqc;xgnJL4gG8N<-HkH#`a1VI^&V;7YLL=zxf86XTRho(i;3!ogMMVQ=~Q*@OgwUX zg4d(R$?j$kJRwtc!^0{!VOuFnIL8zq1Vyh- z0y`CKQ5Ve4+7Dqx8pU9%2HT4Ie5|B_k7fCyKTi?%p9MfX>mVuwk^l;k@et6F<(RxV z7yvf0b*Kjf^1K599UgH^%&_(=Dh{GUekIz5s3;o*14k@I;$$XuJ~6-)$`6773&s~x zj87>0Y!sHaSS(3VP1kXB4S z3`7G&f7F2p2GHRsA7(?tjHoooFdZlTY!L8u*9;WWC7p(-GC!)UAxe=MD(l=3)x*!# zBNau!H{?H1CAc{Hqc$Zle}YbycFi%|;ps`=hy^I}W?l)Dz_SfV)cX_`doYz6@CswY zpzmwpgndUJ6}Y1#wWZJRIFkk@kLnd87+;6^T4km897G=&iHCx)wmni0J2ZB;(i;z> z!OXKTiGY1xg>LT6ut|9o5SsL1)l)a)O|Kcv~;%m@@1Q zsH%Sjf_?HQK@;gcIo_mWXpvlihgkZBeB7jFosa4RnZ|_Q?Ks#h{Y?m@~yctDVDLxvqeT0pDixDb^cyZ33d)X{+bYkDWz`&`x3XWj60juD7CR>km} zso&%uL(~aHKM&DW^o+bHOefGo^1d)#jE9KO2&A1Kp*E`8&=J)lCZk%srK){2qN;sA zLc?i`?3_in;PG@8U5Q7xC?3zqHBst-x3{8nI^8MT#;7$S2gFq5)EJG&+q*Hkg1(VB z+|#ab?@KT(suO@A=DDtZmp5h8*}Ma6fjx$JH?wYEHmwo#xm;{hEgpmN=v0{eq|5^- z^>Rv6%9sDlqvi-VYeL=d7}|u&@VK%Goklmw%}qczNNY;)Y>+25rLK6O6r4Tk?q~`M z3m*KDHB0%|e7Z9go>~BcSzh-+0bOVQX5&vtSCD3b3tH~#^gC8jcP_svy_7fGmj1$9 zqnsOKM;IeJ_B><`fhQNR|NjAlJ>C2PVNMchW6f7{& zfIR>hhoMa=ne4QX&1Oeki#)JSG)uxV`D?`>Zp7FbAO1Jfux_Kc5vLdom;g7zuT@Zn zm6*>`DO?hiRBBo*y0JtM;Kh;~<(1JTIILkC3-5qmBzgJAz~h`4OG4TU4N@e-kGkxB z5EHgRkYmx(zFGM%4?_iE-g7kvW;!TO0j4Y%H zWPwPVh-BlaAY3qb^OyXL{j)=(=mwE^||CJ8;lw z{^2!pg*XbdCtaULc0LqrRj4tf@GHm!b#b~B6n3;*nrb(o%G#F>65B~a5* z$sk5B@5q}>Fe$8Rnn;2u{sV{w&JIIt3a<-hrI4@u#hI2s9*(f@h9|abL(peDreiRC zywwX-sSRRAS+IjSq9;5-RH&$@SpzCW(Hi!Qn5cr->#r2!j|Kr0PIp5Z0*%xB1ZFG5 zMMgl!q)`OQSBeu*1Xme*INq?IjNO_bwu}=v(&g(`n8et_4?+P=Wcm6G;3yX*?DYe1 zi&B^zsE#F|UyX>UtvqTOMh;1@^T% zZ5qFrtQwtqa2b;Ete_}#O>T^?8AsqKi5h@D_h5=H(>^%hO}W zo<*8uk3z8~1q?PxF_|}>{yep7QhoF7O1{MkM^_F}lF#wN`Ifo$zbov7!CR+cu$bSl z7E6lneXTemGfpId-zv5V&dPPM?lA<>P^B^LxEQ?iN_j&mg1V+e-^nH`DQlb4psWRt z*2B-%HSP5>(1JRKAK-V>UMzdHpjOF6D%FP9PQZSST?FfS62Z^~b!{N~3iN{u7amp2 z?15@`_2C7e0`s9?Z)dAKZKl>*@F4=<+JMJD9k&eTP~C3KO) zK!-6n1N0Q+PZtC74P1H9)-BJ})y&{(g8vrs14o5?x-LIimp=;RJUJWTyxT-wWPGZ~ zVUR;UItt|Xl~Bd0iy3l|gTgtxLICweTn(RW@o{ct;9O`j;hCtCC6|ZD!u~m{WwVx4 zP7g@8B^8u5z`$j+`$~G?Q`q}$OR5Op&vgipYVU-PnZwYYuR~ig6fgO#-3~`o^I=@PR~=N?rh<`>OLc&svGxN_hXG z9z+1Dt?)hq6gZ0m*T-defe|a{AP=9)qa-#n8d}80;3yb8-2gvS7LK-*pLIFP_{A0h=J!TVArZ{@5=L@ZdJ6<6YC6L)qP`#U&-_{l`% z4dhtZ01EQO2iS|6qa0Oo?N8Ku6aO7qIVZf0;5o;_ys5YQlY%kY z=7kO)4aSTjfp~s{ax-#|xP}@g*g9#eS(w2I(;l9-X8V;EQpj`W7FZpSwZcQ)% zA)OD7P25q2rT{1?1b$Ho?>A0yv054KJ6wtB_iygLSW;7f>Y@Fk$r%&bt{AASdVaS- zl5><1jN;9lqImYvI;LEMv*SfPppClH3@F!7MAF+S?t6Y$%(ypuZ+<6i+xTOnsI7<* z%rl4m12q9Q8Uk<2-j)^3xtFDJb$61Y*q{qIQ^;@82u3e&U$coV0Ej{q#hXA6(ddRA z)QF7q41%Trq6NX?TuWSG7hwdUNTDox*f5L@L&UV}578rX)*FQL+@FO zV%?nX5JdR`AySiuoqzmZ#!_^xAS>#L6T}g1r&pdeJnLH^M#*RmJz~O!s{*nhu7EQY zdVl-eNwSB%#oGXuQ^4B*1!@~0OJzU-eXLATZxeJCK1+w-Q)UcQm;0nT&*MY>ge5g$ zLl}wK;UUNc9H}`CBIwNO_QF_7-CoY>_Wy{!J^24l-)>Y42spaF&D@wZJ{Dmi&}b%1 zyp0D59i#tWP{x!0RvTxQbht7O2BD-%Um4#8xk54do0aG}8Up5jM;V77^}o@^|AWf7 z11R$~s1O?-6@ZgDT2e>iv;w%7iv6#s+o%j+J3w=3Mzxl%;3)caX5#^!%h#uqdL|8I zaT8UtLX9dBVwEJbSWp-Hbq)F`>c|n6iH3@k)hTSC-#&&I8KHDvlxu>Mp(BF;4pWgK zZ~H2;qpb?it;Y8>=PJdxgIU(o#9<=_k_>9@%`pO3$zXvMx#%q=%4 z#gJ%~Qrr!vSJFx`q~nBv>U%?8$yACfx|^fb`gNF63}1HEiW?A84FONlf!2Z@Xu)u( z^|j!D!v|5AQz|VO7aXt_jKZu1V@I+AEu$y+1*nH>!Eim`a==Xur7@CJC}B)#LIhf{ zX~Y09C&Nl2ZI$p>zacY&<5Q|le%FVD`6sMM{g1h5XnhHk~RF(?&%m`na z5xyfq<*7WI5#K6ZJ=S`5DT0s>tz8+yENB#2$6y=qpR(OC)bX&an3w8OJ!EBd;$Lzq z^0aMS9cUHR^)jmPs0aNCSEtJtj-eCsUWDP3Wf~?jbs${kl~Pd(x=y9Y@{-C@k&)$h zrP$Bo(QJsW+f#3Rmx%kVdRtO~UD#9Q$O;-w+vGD9)C`ZeE9d|oTRLLz@DK8z z9jO$L#Ib1bG0}YsV0eh}|?O3`@tiDb5>_j7JZr##ObRR`9#W{oh zm%5X?P)#7X={DXotb3u7J{7?&H>j5cG0Dv}M z{y2b6j?Ck8o={Qe$vy+A0$Zn(22u^K7_IwiAf3c_jn;J;Ovlr4`bJtZqHY=fGOx4@ zrrvoAH6yid=_yoA$<4S16^qnFC3|D#QSzSQZslk*9bIqu@jEy^kdCj>mp>ZrevZ#- zh!=kT#c(=yn{vD&!CR<>!;TY-1O;ZE-rCKC>-M3vK?LWlVj`^@dO8&o4VB}EVJX4B zG(mjjgQa`rlf$Sg6(dv(Hl)uu_69y(3tuAbjX}U38v+)q+BgYUO4QYsJA0YBH|2tONL0y*lc;5sPlC+}<9@%G zkPl9xHeI$V%LfzkOF9Gz8Pn6{MfC!8_7s5;d~z3cLyR6O|1_1Fi5+X@TT^kJ4}@s2 zOI#;o!!$LSS~Oi`abu2BNBD{rC^hnm$yC((VYpT44Np|a25A348*6rvSU|3vOnsAC z@bglN8V>tYJSM9C&2==L;dfJRu3uXPzomwYj&jn@H_&Oz&DER?0ffW2gp1FM2Av$Z zsj-oxW*r8bwgEZj;9_Kg@56)y^57r|mN(9zjuxv7C4;fm z%T4njplOgd6g)12|K*`6RCKQII0tnX*=Ai|14rW-?yPVenxb}A1AJp~yft(>%s>c_ z2Pg=0J6S#;(6xei<;!=phZkG-v*hHd)Vg$)BA6Iwfhio#7F~prdVa0EV;ZBbeMSz#E!SX?px1m9t(B9$jt!jU@=!Pkxc&lWGzUW<$iZ zsjP1Yd>fi_C&akx?m5_3(yL7fyf1=V4zc_rgq;Y4dHj_$my2lzLa71$YX2GDa$b~r zJCexVjgb_Qza}3%o33rT1Tr`P=erl+g2ng$ZjhDd(16T@)O`=~)^n(3*4_uvn7SuV z{{9>~?vh{djl-ee7-lJeZJ3~GpD;(TUt0y5=6mtj6I5i(Qso_oyDeA);w*4iT`W%< z0m0)wxpo8{-(0OBA`KEfgg2}f@<-&#!zE=w%n@DQv=5OSe=gz18hOjP^e|3A z%kJk9?pKyqoJY7#S3YqbmFMvYHg0hLc~o5VBF}%APl-3DVZcvaQ6^iTPowBXnLHm$ z+fsS=`Sckrls8_0d<*467f@gAf5qbo|10_31rY5P%2pTBTu#3b&@7Z&FNBxZLRm5r za_vIdb0nRK$HI|t!&)d;kEDv$a2z$uMGTI{kS)S-5DUELz#T$-D9BzYzZprhXq3cd zg?G?dGCG>hjQ}%yVg5#wmyf2~S%{%AsQ%l!R%2)-Vf}mdB8pIOk6t*pT}<0(lUy;D z2H>%GENxA0S1DK?_l7BygrCI$19vN8bz!wGq2K^FV^0K2ROvrPjRAZ^kAdkJ69j=U zc9bkWIu~H$G#`VUF(n%YTo7Q{I@_P53EBr>M_SfCH2qH$jIXwlam9-ZW?eKBZ7;KBCa zF??9B;z91GgDzrSGO>8*S59*h#BLZ!U^ff_lLXi)pV>Cxh9UU;GRLtO1~QkTurdk* zRcsIjVR2)ua2m=T19d0y4s6B^Mk9M@)Yu zU}i*T4kn_(T)7ASsQHKw>JuxBcqTN)8oSj3!w?hT9Hg9UfJZhdGayRyLMFA@%$`Ef z&ESOXIxc1b@SiYo(K;Vno#M6_3|(!i3UX@rmr^nGyt3kQ9v)SPAp$f$(Ir8{aFPGt zCc8A%;nI-*1ld7v&m=pw+ny#nZRF8}#@&WN3j7716pB!^?%8p4ouJ`$RpX(js&y6T zZg9s9m>!FhXI})3S5gPr?JByeQRyC>2X~Unr>~-xSbuiv$B+6^bTzd?WcRC8+F4ga z$^Qy0!d@zGxtbDKMAzdMOl0`{YAWT6ozhLikj;m2Hkc}N*$nEI)CGBG5K{ryO!Cz@ zCbJcb@2*Ohw7DcM(;$KjNK83itXok7mKuIik$_7>jNb0^_C-ldmXh6&v^{R!jM;1Uq`Jj zYS;kx{Xnp;90?Nc0O8Im{{RMQ0oM^qd5@?zvmn<3EK_+9rQ0PSqAW@AOvL;qN#)7a zTm!b@mHR=a%LXD`He4Rv<%5L_rr5905>y=xmh#)cy5p|nO3f5ls0dOBW}X^iU1F#% zF{pr%%SG065z8+k?HU>y_BaYY9dc|9Rb&EwR}Gbi@8r}FQeUk(n$H5fVl@2AIUn2m zu)5wa?O9ZIj0T~bb|2`>5b*6@D|K_)dqAE)i#lgF3Xy!;>jkVw#d8Q>29daW z_Kp0_tLn9J4Haxh7`lPftqV)zN5rHP4GPmpstd=Yy zK+2IbZ-ykhTyDJ?=CgX)6LSU-|84<|)cE3~OU$1?UO;`go8`Aq2X%>}1>z|+qIZNT z#-e7un2<1)ORpv0mDC)IA&ZhY>t(Vcu^m*|heehe>Mpht+J?mg;uj_eSUDfO!Y>?6 zS!qQW(}HCfWJ4ubjWBg4Dv=~vonu*s1Fq`Mkdu(?Zi!bsQQEeQdyk@CocyCyaS}1@}1a_;&?V+_Qv;s zgkikkbKo&07CK7b4uc?oP_Q~CnyhSfw;`Gh!%TvQh0|^fL}r^|Senx9;ff|4k{k_G zZd0fL@m=lvge>8CrBu9L6N>2B6z{J2teo07jkq4HF{ z7+l2_`C2Yif&BtKvg+<_HsLdwV0z&aIPAPeoE%oYBYN{6R8fxlyTi*Tg3hQO0)415 ziptcjC91(xTS}?`CBSwDUoqG=u;Lv?jPS@NpFsGoeC}16)K5s$I#aqYpU}=$$ z-a#`tYsC_ZQ>8p@36*4TX_VvYCA5Lwlob*qg^c4Q#rw^pMwRmYck@$GC90jyh)T6w z9xBLxcs8Q`gdZo`$M{VmRO~)HTbA8R zgIcdaByI)7IaPNNj)?|YjDqr1DVB27>+;rnah&Zh^4WXo_=-389F~`&s0THZt~Oi@ zKN!C$nYfQmq)X-K`)CpR`qh2ZL#&xCJ1oNn!<%x%GU`e5_5s|?xz}hQs%6}LAVd)v8(7V`a!#R{Q$M^081p4Qg*(AR9+@3 z67aMw6P?)%nTQ9U zmRPpPfomz_puFZmD(=SYkhkAp0%QoP@qBw|(rWlSg zdBVeV;UFIXb^8OP3jkr+IQ)|CiFQd>q5uH$YXeGP_u;EY8E|IrWpdZUlz>q5_x043 zrG*}A=yBZ2BR^k5ZJN#fq)#<}vgG6`>k{_dhgc!+MDst9?H-|aV(IsC*dw&|r1}Gx zGbQd$JoG&}`#GZ2UBuVosOx*)#!PeL?($RwpnAh_^|d*&*IL?8TF*(_VFz;zmgHPR z8RB#LWYsvqb&h?blV`soo2;YX*{|?5&b)%U+qjIB_pYNj$1<}=>@FSq+B#a&qA`9I z{rF=muX~hk{_hMTPKF(I5XZ_*kJ0gmwUjxMGP<5-RQz{FOFp(9`;?7FK5V$*Tpk{K^6W~uP{d%B zw*Nt-Gw56WEPEvmOXDhWUX+j@YJ@7jOx|5f&C6aOI65JLx5wDrrA+d$ncWK4N(`;R z%m~8v3vy>Iwb8fsahLS1y6UcjL5Z-9TlH{+nD5>x2GaKL4XZm;bF}dlx z`sAt&VAAi(tsCHG@hvF>PlY(=UHN^phhtVcYwOx(7FrNA!g3^)%rBT3+-tJrUf% z@UN_E@eHk@{PgBFfJ<$4*X^i>!J=D3QY4*}y70=+s&>H*jA80smOn?+L813Om)iDa zJ2P~Sx-HMaTLLL>KTkK|F={itt(eLSkh(V2o%;fXDfe%?`X&Ncn?V9l|98mAFH%*} zs@?dq9<+V1Q)oTzJ;L4d@$=Z|mRnw=%k0PB$t<$|9qGPACl@Vvw^5nxlq$3IB^rn2 z^9&J;|WQBK~2E(qCStW9b@6uh2z}a{yzI?T}-}E6{bXk&nDWPtid+?o}!+ zfH9|`D)_D|YmU6{RqE0>NnN-B6H(osS0P>H>knRnc|&Oou?t|zCgR{vwsU8q;j___h@+RcUZEGZf_08dbsI!R`Z;FSksR6y&q|0;Jf&r8p?uL6YL#1r{7O=luj(-bNq@SGk7H0J0b#K4LGx~8kVF!(Bh0O#I zh_dQ3@j5J_FxRSC!Bbp5`7ZsAM%Q)ONsEbImM^}CO9Wn)1@GfRhL`JBz7J8S#fd4w zHp239Bf+-qE*h92P^%1Dpq{mh2C$I(@-AFQ@TM%;4VLq|?75pdLV~_@Hza{KrQD6n z3FgUHcT=w{e{~Y}kN9#owU4W%NkFZDY>8zF@)i93;F|;gOV}6$)}-^0tsM8)auN>> zcT1F;Wr+aaUkmo`=mFtMutKf-kS@UDl=Tty&Q?p28ZCi!=r@QuPA@)!7atTYQA8kVWCYyks@WZxj4{fL^y8rB`e?D>dFJE)stQ$Vpm;sd4EoFjm0CQsQz z?f$##4%-I(b!W{UI;QC(>QYzsT`H+|H|x7NcI{EC&ytVn;uiQgo0_noRo9&2vL*{L z3EIvsdF{vG`>)FfJ|^cBrFbY23TlV`-W|}&=k104I!Fw?VUJY(d}0kde#nmy_IfN)#nD^j#h*~8)`(MXN^sa}^u;-D4quCt zkAFh>19dOBIsm@EuMQv*RLxc4vvmpURD5O)VN}AO$z-uh@H>BmF5s3A#>ZCcW$aVx zmXC58F#cYhZf2J}^HXyAzxOOjNIFNyx z%J4}FzN%{|9yqu=NJsMiZF`nURa$R6aHSla5;$i!Jd!Z#;F}+urvl?(Oh8YB%e)20 zjqO);Pk^2>?9Uo}<$w|;&_3@tA|A(eG2hJT36P0}s{!>_b}`CfSX(g+f=zX-U22Pw z+5*Q2>WyVwJe1Tq{JrHfm0Za~jIP1I1{i~~2k0>ZL%4m%fccD%`-cEQ1%{+*P&TA! zegiy6X88HKDh7*(hA~J;u&}Ai_8%Y+DO|tKj$`0@*P8pqdDX76quN#dLs!g(FAM;v zd=N8Rv>JAtp-m!})@%i6miGY(yJo5dkiqhMcs;@L^j-kYZt7OT&Jz*=n20GZPu@#+ zH-*>|;B9tzvfKQFEhzcLUb?>BQ-Wy?-(BO=0_y6~2ya-WGUM)cEiA3OPFyU1*oQlH z4$ASLX+emMO}b7T2v!?na>ZxVjs>BHB!sju_1VgsSWm)LyUTX)Kr z4;h;IZ%kq~4vf=T{Uz<>}dddKtM`GBKuCxw#OTR7_J96V8ZfqGDnhF?lM+ zPQ}PoU(rzU^lbU{SFm_pAs_mhTI=eYA5neFBdRYsqIzs#v$Bw`-u^~aUpMj_$}{OC zIr%#@vs^CynVy$>zo&O_{m=6U(R)hDH9ykD$t{mzy!?B7U|LVV@yhbQkQV>d2Iab+ z;2GSTJ2F>Z@iUdRt{A zxUzmNKMsna!3E0@cR+RziH?XG7ZUS;oG(M-PZ;oZVbLCs9bs{Ji9;07AND;I!FAFa_nxEE7FeLXXlEt;yn+gMkWh8L}|zFLl#A4r>jK) z4q0G~Aevw~w(nIFy%cOI=AVFe8f?wJSDXWd3A~n<4z+8H4_G)s?E}f&0+Q_M5UsL! zqmj2D64(`BuSCB2R=Xa?d82_ZWL2+8wdBJ;vJwm?3(`HiF)B6$u(YdV8A6caO<0Uv z$e;EPFu6br*SjVh?47{pB@M8m82BFKD}=I`2_UH-fCO}+3XIhRlY@kuZ9fQJj@b>s z&9Xr(%}xP_1K_yK`!2q*gxKGMVSz_vz{M9}0(!;bO3^~pw-!Xz8M+AkB{6%VGB!gL zv01Y14ACYDv!sHuvqn-k>?@$j9&DBbFxr5K{T0mu53dvO^ST)0#!#w+FTr4r8TLQ^ zr!*@h*g63rz8Wv}jR_PINiKj9^;xVs!U$H+#~a~Cqmd1>75iU+>!Rx!o`ifkPJ=zv zcb8=W@o+pC!U*ta3e@o1}CWUwW^K37~7T&)SZ!WJ8X8+G^-TfF8IxBNq% zIEikr`yo%fPldM=79DJ*bQeliU_+*`Z?F5JsrZS48(&iM?e7KR-kdEejJfnj3lfja zyM~&|)0(5F>+2RZ7fXrO%kpB;D){q#D6>+YS}c0E->|PQJ8I#!KkdUMs#uC!zOOIS z3X4z(a?I*|vbI>n#j1Vstzxlm@Rlc0$9C8tbb+;807!*Av})h;Sc}{Z>S3P@p^c^Z z081)d(W-rY6WEaC8vDX#YAk6f3gqq*ad#`!0YaoHxS1Y7Eib`=)Yjrvd1XS3 zrLXJWNr)_=w`zT}i9Ap$TCld+S=ol0v=L9J4|}=!5ES9;%M%GtSdW!g7JJJ;y2IMR z+r0bRh#p{8ZQ6=US$SXBR`ic59TQiIvXAEbZN(7n9trUk_3x`YzMZ&%O1G>{@hHSA z&l(^($&zCdkZ)9}b?=u6{zO*Y`yIr^1d~dij^aBo@t((ukf0^f?JSyNJL9a*Vh!6q zc$i=vf$(AT4@(F6T|~RaNlId3v&gV6u$3*5Q@W@+ZopebqdFkqDQN%_pxXTsE=~tFHvI8~8Y(~ACAx=}5HP&UkdtijYva&}`{ z9A-3d{o&LRziGTxsqT`pOp~|pb$UKHjuI`|Q$cOGpimL0Mm5RcFegM@EY%s8cZv{^ zad;4IMW zP<EJfK7Vg9BQ3S~szdPTciWid8}DaJQx3SWsA>e#6#|gKtl;sGmCovb9^Ul4K9K z7T_w;jg?|a?lUH64s*b82rFG5d0}_a9M^kZ-CcCXV|91Y1&??1TeOFY?5!V{>&Jci z@p=!@Ri1N#c!_q`weAf`REnNr0Np4D_Y^l_gW!#xqJsAV*l3D_Xu)2h!o)snW|YWE==1X#mIyDY@9 za@|mj4M_S;e&vdG@rD$D2I=l0+~nHP_)tNOve8H7JHthXmS6-*kWx$u-de^)*h3Ig zge!L~+3F0@u44e@)nv{ITb5y(Bm!VZK{ha`vMj`M3C@-!FFixFqAYpi8KM(Bx7DqI z-~;%K5y1@x-)UuY0k{{w*b7-7>B<2H(YzzW8JT^hvVd_xq_98trBW#8{9)f1JEQRD zo!J*f$!!p!Sx9CSz>I<@8eBqP*wB%4eQE{SUsFnu8Ep@V;7=$C7|XY%S61gt@xnv-*k2DMGf=gseju zq{a4*Gt251QoCy;lGZV?jP7{T#56c883z<5Y;hw6deY&PM?`qT> zZf?^e7|%4}ex@@WY@7*qq;Ye?>aH?i zb%H~RA-I3CmEC#pJGdA-7iAOql$M$r>gpfCdm#CsG?=!bFR^nLmJ!X?aMMpYNbz8)`{0Y%5X3hg`PY}INSnXX%EvV_Be5Ry?lQhopl|;~FI0%H zv_doaSmL-fUKBAL=j!jiA_f<9qE%k`)F)%G0P;kHFjj6B!3qJGz)EA~_@)6zv#J+O zzrfm`<-ah`W#v6z#uK8T87p&WN8?jDE!U%hm~7ciBm%qG;37{sPdq@^$^++#Lb|pt zbiVkI_|PRZ4OroTZT`VVyZreAv4-x!p+V95gp71xODVJ_PAT#1apAC9ssR*i=}y&y zI|;OId!TcpgEJQseHoqz@Csuv5w+ z4O`2wQ{*n=8Z%0P)B^sMm2gm1w2{hsf1|7nw8`Wn+<6KreVL{+Dg&%xqeLs`aIk>% zMqtT>qeNk^Ms=fNj8g+7IKsz}r~?-K0gXnG$oEHyJE5=77%lh~i4~(oUwU7DJX)N> z@-oX3ke7Rm5$roNZj9gyBK|T)j9}A7#YJLv+e!Ll4AAdrq*?sEa}$D&LKOe-Fur+_ z=*rs_sf5&t8CiU>Sc*!(64hZCu7rWt$7^&Y&}8)({_SE|C^pI;FNSA%Ch3%t%(?_- zv}JPeC8`%wFA+VP-iV3IQ&=zwgXnRGo_zWe(KYpPi0?0f^i?mo+@EX zcT5c4pu=LhQ7*q!^y1#`yi{y0x)$K^o$ROM2?EguL9(x{s~<0xi{N%Pw@FD;L_z)@ z6=o5NZywmJQ*NCgD(aGx#7qkATHcWNfC{JbZqq3lc^9+ukYDdA6;9=yqf~y6l zKVb#rBy_|`-KmEo_R1>2LNtrbRxbnmpJ%vzgNapAab+OLu%D3Lm7+~XEM}qsoNc*M z^lGj0vf-Ei5dFONX>scf&h(6oUnL4!ZAK(lZrC>}hqK;55NBY^HS(N!l{j$-SHva{ zY$!eihQ-RO9(-JC0Je%_9q?O$l*Kgbo}|2%4fcbs(g|PPSdU(Ct@d*`_a(ZNq$=d~ z$M@Al24&TC>vG>U z!pT;9<1n}I{n@)@;cvxp!#~CT-2Z9rO~9inmj2;6-F?C`Kqd(Zfj}}rz_8CuCXY_5nfNhX5qo45yLk$q95KoL+xTu?#Lpdtobt|$U3S3pt06=Zk0prHPL)o0FR z0_eT>`R@BXU-U3t-FglPyv-N#xBV1l4hH_^&Faf=idn*%ll*LC#4ouY|xjDEaR48*eNbC+la z==QsyBdwq>?gE*vpkMC7hWsVE_-^rL(o8;O2wOVrfKg-M?{|xPVT+k~k7y8^9F>{i*Bkshb6%N+{dUTT5nlybSho$RtX6f7rsBZ7RSKMiClfku1>GyjfJx!-Z zlSPqniY87LecIq$y#P(nxCMo0E>opm*3CG)=>XJ^m1Guj&RjIb(l{{n0R_#Oq9wUO zQ$$Zt&}XNJSH;?mG-Rp>7(2^XOce!&v5P*NCc5Jjto#KWmZ~()XYulZrE05|AcsF31s}axM5rZtVWgRbAVME$C_wd=3-JC>kicI z&NoDGvG!M_+(y-t6%h3Jh@-9mGC5GdDwfwUkd%c7uqRzB7Ke3UM^rg?0e%bz5@yQ~ z9q~RNzhE4)j$g#_ck4j~*aZwEz`+M^AgRu0J6Sngl#R4vc?9RK>+>O&reS(uiZR5|($_gohdv zB7$GR_6bSC}+Cr z+iAw7f=j?;g1Ck#xP}9!Ymx3qt;`;lfTO)y&|+|gXkJHfb!=W3L%7BG3K^TubgQcc zPjS_@Ktx}56v;!ZE;r;ZyCY{ZZP~~&0#AbB4W1U*KBs9%9K9ikly@e!JPccFM^j); zwcww27h}$+((!%5Z+K|+bTKcM{6xu9_ZcEP;k2RmLO>m$JgyslndZz8?Ei3fhA4=A zElRbz&%~`657K)xvHh7#f6UY?x%n({6QJp{Kr}1p*;yjB+UdtYQvM|tl|u~PKTE{5 zdJOZ=YYQSiYQw6+%7|Q2s@VY$30KF?XR5TD{(KqMv5RMm?hs=}&lYSvd0@8a40-31 z*?KeBevUSp44wmP_z{{lM+}2lQvT~4(Z@jEp7RiWkVeiE{%QwTp~S}&^;;lD8*kA& z3q*F);|36n&ra~BDAr7>FVMvjWL8Y+_rsXKj(XlNit2(fgJ)Gk3dS98G{$v*wB>#g za2-}ohtqWgLOznqh3HFz8m$)5)m>Uro4xsmiwIMJOmCgpPqe4JYhShU;nW95w-MODmvh2 z=2G}kU^l!}Pn-D&NPR_l`$t5fQ5O<29I7E10V$%KYW_0OvIXyk1$*vdH;f5GdgOjJ zMkE#oG;tC*M^DREptkk2;8AgF?e&-`%L0wpY28ZEfR7Ssg2TiA8tqvLt0WA?kBeNR z8+CtNbmT4kjK_t(#z?c0`M7787xY4-vG!?LtgkU*x}?#DKc>Yf^3~(eqkToSR*AGk zw3ZvmScPhIa{emOG&O23e?D@(ROH|g-MUH)j$3g8YV#BH%_^}wZmkvG{Deq~n{S0b zeM0nQaoq7`Lk|T8GQ=)D9-%vKIKLfX=3Yxq|gySLC2k$`-RL11rYYCim@zDzEdrXe2JE$mqCEq{Sv+$zV7j*ABz^uGoaA%Hq#; zA~`m})_Ig){DL@a;3A8p_2Np-pnX~~xlvNGY(16l3u7 zpQM@ZE6wO&GJoL-!!iBMl+XRwXLG3m%tD`ZS#QeBD z{58W9RQ(O{R^q!l3=`EHiK~WKrlyhWjK($I*18068>&%(8F-U&D})dH_Ld4D-ZolW zA&P7ci{oFZ8)$8KrnW|7`-*Pbpx=Qy_jRc7Uq7-Ri_DDT=s32(v~E%KWbn-kKE z&d_R?Xi#g{9`-QTN}pZSb_>M1KWXk30b9^+nzdOpN_%T})jIgdCaOXaZ;|Uw7_6KytO-8!&`LMo8l&M@FdOKA{x`JZ;Ni$t*o2h7MqOOH1QoVzU-8+E+4DF zcsPi92~t0uKxRaWX%5!DbII@X`CsOI9{Xg?EdkTBP*g69A9eiG=QkNr;) z9L4-T`WYyERYN);NJO@Pmfj(XdXhD0`eHx4Q!8rV#2(yMTVu}!?8UYZx9Q_X1T7mN zrmp4or*+S|kfXdy<}C~#8DdsuXe`7A%tAG4^0H{16xs1vw@E;=jL)n+P}s?xV^_g` z?qKfx$qgB-V4bq_tPsk&6qN7MP}r`qnj8mAmXll#bw*EWv0St%dkqQ@-nR%WL^cRO zGgjuV-q}4J*JtPhN_sN}vmh^7e0doLP+bn_3R@%eXWF;v4K_}->qKP1otf6f4fvj? zj@uyY&-daz+}0^_PlG`FCCcFXw#v3_zHC>K8>yAe0Bk^ZC~byw<<*W$#+B> zqv3q^HP_C$z_Q>(hS(PQt>8~S{nl4YvTTiVY68Iuqt^wwdc!UhF+Q=CQeoKFB6)Dc`m3B_b zcu%;qc5^NPvmMm^FmmKdZJ4llcQ7{h&F_iyw%<7{g=LO#KxO$CS>ZsaZN9J)_il8WD zaeLvRE}%9D%tt`51_}yJ;GK?}eu%!?CR)WWMw)g*<|8AE4^id^qMkrv!3QFv+iW<1 zV$?xN;TNb}<(6#7LOZA%&og)q@*s?tr*lA8#`-nv+F^&|h}nl|#RsB6Z?uB#rvi8T z1|3#H7qmr0?s!FOt!T7g8>#pgucLwNu#0YkP;g_QM}2$)5Pugrw~K4)RKTrOpG)D) z`)K5L(WV9jKevKLb{{>lUHoCmOb1vTN}-A!qIp%p>6aa%U)7kacZ!Z}UeLN=3xzB_ z&d~toi3YYI z8GGm-AB%LcXRpNrBfp7z_EvH-&7t;C%UvR^)}Fml2DOL!>;en?gVya5zE-Ohd;uY+ zIvQgQyCy+n$AfXfuXG=x_{wZ3e@)07B(aJ{e;KSG-z*udQP}YG?yPX*TpRLk(=`C`0vaXxeLl*g&t} zyPu&X8(LKb+Q1Oo<>R$a(>qa+tKY>?w3z)>(vDSu&N39OV%|nT(WX7cP)ED^?-{CV zLuVNBYp6FWeExNSt?b0l7;0iee=?+7%1xWI2~f1c6$~}Bb8ch^qejPeTc+{4EZHkQ zt;DtkRxf8cmK+KO99VoTQaYMD790xZY7pTWRv6-|HmLClMeH>gVy)>3i|PqB(=Z)5 z8G$iz9HqA{5sG>AVO%whEMQCvj7-ctb93wsv4`R9i>2SNJYpQ|TLY~V@NNF#pstV! z!-gXWbFDxAbuHRR6s<(7rnw%x&9sHwdxZ(5ezU#8Q=cu>CxkG{jJqI^vcQFNgDe@* zvx8(t3opM6+5j(>#xh@~a*BY|w4|%U;cxcK&{|!J8G#`W(#Z^&T$be)S}b$r3OCxg z>k+1{px2#OK#q#b)hFl#L^A8}97wk&+%NId2hw*I zEQm|CF7M&IJjBcgFl#3hr&VA~nHOw9b_APC3a&HiUCnPhDz_2Xu>>I^VuTwT)-6ZD zwPla)*++Dae*^rjDBnSK4bn0Z z>n&rXSQ%U!TWk}nSU0k=-NmwyM6#;jOR-sDDB)7GkX6g=o|t}_!B=$)$|AN1`z}M| z1@=aqB_hSf1-Ecs_!{WOwzrDSV}Tk7%==X!Khp_&Yw(@Gr@X{Hyn00;U#-ulVQ>|m za0ouX8N>#QV?ZLFYzo1JrF+hoh2toYbpUbraTa$l7vPEFfrPC|H1;d;(iMnx%b~nH zST~EZ;2)sN0u>eeUnCAaCqytoVd<5+k3N!EYHF_D7)^Vm07v5P7G zTl_pjo^K&6Z==EAiU)zwe|!s5?=qYn7nfR6i)By8XDYrEBUnw;^m{l~pQh)(hY@%t z9r#|K6Zz$P;ZL4vV7$+@2A6j{2&ogg+p7yXHFPGs7^7cRNBEc!vzy9lEz3a#T3c$d{V=c}I!eh>k^A_s`TTVVq` z#OK&GlIzy4kF$Odx1vqPkK#Src32Gf%kWk!PuJtrnlfXqW6JR5Q(yU%!C8EEF}`v{zx>8W2KMtG5X-JxK_A!m1EehW4`vZI8{xh)r{@IhSt7ln=d0=dzi<6~UrqiVH<1nflj(=*ve;Nj zSJse$tb9$UhGybz&@1hf8eH)iSl&wf<2E~&UESD`7F_)qt*IgX#@BS9hHTI<;tLN3 z$D1MMS=eFnRt>J@RIjGIy!PpXU6@~L!_E;JQByV&vkjV4Q>K^Av92Ecjqd zx1(G-${l=R8Y-{OWWSpJ#uLJ`bsnf>t7BHIy#t@Ess5K`oS9m+j0c5Q-ct*wyd1|P z#yjN=mgmE%O3w#a$xk|EYLdl>!Fk!wVgI8THLWa3CT}g-98=i8mh3j*#llOm{ZUyC zma!b`U&P0l4Y!uaplra6ju2H<@VOT;EPQDd?1(JmvBSa!j+3sBSkMa^$O-9_{XFqc z#x23etuq~e){<kxD<44B0jMM7iBn7v5PW(v_}pU>OlQC z*F#h{2?M@`Iwr|`l2;ts3;}|bI{XiZlxqQfm?UpW);D6Zw?32ue3=dIKydv`z3Kw* zCLX4db!E%=eJseu1yAsgaE+#?>dGAUCD>nAHsy;*_0o4|V$%7R7wqF8?(>O%QxsYU zNVQxBN!ODsd)1403W#8*DhKO{lI8e>RV>8l6Aw(z*nHP0MY4ZNj}#fG7V$!%=_xV? zCt^0G$j4G~@IzBRtSy0&hP?n*l54x({8%rz9k|hkFIgHz=?g?kV|?_$ zwT@42!E**aGao#Qq3fhQOzgRcGGF}d3{;E;pd!YZ>b}N=)JRYJxE3anOyrT&SCG9*|RI(ebj)@0z~Gj z8~5pYe`#0asljG2EYqJ1YO_x?(ABk@V{H~I-RO`7wwQ>geymeJ+gKLF=GIlyjSYZz zdYQQ|$r=XpErz(EJ_DlduyODc+K4sGtFLC2)rSvj@pT%YPi&jx%?y@jHC$eMyn#%y z7GxueYY5`hAuU=3%}SH04Z8hhdGvakY#IBq+Hb@d#A|e9O_8>(Y16VsN`#jRXedv`7<@At$rcH3MudI6?|}CAFx}8dUK&?<=UOUlY9wo7 zr@FO~Y=6$%l$S1D*f!vmOu8jqHm&vzyTrd}(DL;nmELG1YZ$vP^kTy53soB_W1*(I zF=bGBx@=Ik7sJdL=+deqQHU;MGXUi{%MPI6tYrlCb;sd=5nngVF>vSQOsa7;4uc*e z3vR=akL@@{Ns~>wp$a%79>w=*R4_frgiXwUgr|q#gWW zQge%hYCJz+iQ3S_cfB%E;2%CIin)Y3K;LkMDV7jd*ih>HHcBjQ)2Uu0z$Djxro-^^ zf?=3>c|>B;-A6b21*b!~c^k|?s1fImxSKq*!v0Qx(w{u$!$~#)WiJU$XPM2gXQ1{7 zW+DvE+j$a-b7@Ov@Q6k<{sc2d<>xLS!AGJb8-u6SJz)PBBCtltUaYzV=IM4KjDA`4 z6?&HHs1e_(p{Lz)kasz)6;0BBlP zTqL`m|1PUy+BT8(jOVCZ6WPMO{wQu&36}N5__Ml%Wo|_A?C^ek!%onVuG=AuJU#GKK^4>~y z)&tGuV;3&uuZq7?+Ykg62-`C;$@-fd@YlyQ4fe?_TG|r3k(a438|DMVRrq9v_XEZq zT=Rk6+y7}g0QDk}f8uR?(T9k==z|Ie%1pi+gezjof{EgzO?oW`p-<%qzx_IHSjBA~ zd@>G~dFZ4jl}V6Z^+(WjDf}TaESX3_JTMSkn55a2MAMuv8T3LcS=5kaL~aF>HxP$| z5V=@r^GMACb@^`D%R$kn7i@`{zx3U;$kWWCkMH2K9ZJS|g^;*=U zLQg`Zfqa8skYx)@^@O(amZ-ZiEZtU1*p2#^wlbrvYD?fpjZVD;rN4F6AEX?ZqIg=_ z2Fr7TR|p zDvzvLIhY}jYynASwnsL+7{jKx$&7dw62#C`7-sEZ+u@OcvLg{!3f`B-I#NVzdyu&8 zJ+ojMg$c3{NPt6FP8j-4)F*=5zS`^v#=gjy$)mw{3gDJ%PS!piST7FU!5}^>rMnlQ z^)DJi#7+ycNk(q|FFFT2Y?X6>iKzaUlYmVUPlDD{>p5j%3p2+PqL-JZ-+I<#Cj~QG zf@O_KL8ufVObSB9dZ}|LtXH-*DUud=C104g-75?66Axm9pFE#z!*mF=sRO=K9z&K2 z?Pz+nrL1q$D!M&e=2TATDZlL0g{_PDwC63ZJW!ViL#)JmhjXQ2ZM+~SMa-$-9M+LR zeht7Il;@Yt8t{ohEMr9A#Gtm@=yZYf+x&7u)oRoKzFO*+BmHGl;I9G&I^So8AZ8cC z%C0E*dM0oVhMK773hpxDM0Q1C3ZXe8d_^ENFg3wfODwAmzUzwsb5;DZNQ|?IOnLCd zPLM#SGahFlg6p;42oQj(1?1hy*1h0RRq#}4!GbRaUlNcd@J;$7Aea2b9(+wM_R`pB zf1N9@z;|Q3d2&$M?8sPx9w;m|&;TZvAd6_IMnB1fI9N|3y<((y1%H}W!EsQWZx%Lg zz>x;_RJMx2Y8uS7vV(r|8@N2ZY5~7bu?widB>>BUZkm-ReYKhAGCkEt=jpvXc@-l{ zTE4u9e+%>FW&Arc9}0{41|7dvdVzCUt-t~+s9P)9oOu|A1p6RfdXPoI9-L0+45oY9 z2LFFd_uP>7KayE%`5 zG%yVmuub1mATux8A4_mp+(FNzhk-ZvqZZB71mJYU;JyUp=Aa-M;Z_Ss!w?Way_Skq@S@cAnpT&6}lggNx4E`UG zN?<=cEpWGpHHi#r+F6zS9quUGw>@Buii}`7pThgfjOcr_IEIb3n6ntdr1QZjvrp6nTJv5yYm2?idEjnlL{ZuG;D7G?)f^VF5i?iq) z*xWgE4%J_f&Q+yq6RJ)rhPw>n2oRIS5m4Jq$2qL9syL$ly*L7Ng8+#M4aF=)hjUbv zkC^P1qc7OjVv~dfM`mYvb=mDz+xCBjZW`M{vFz13g7<&)b643~9dzIv;k?e4guk<^ zyvk(FX_ctn(p5Er^JJYLm0C5bw{%yH=!?BXOidXy;1ap8x-Cmm#g(!JU394&$z&KU z$!Iv_uf~vv@cQ1^q1@0-uF3anj$p47zCFYnV~?U8dy{vparPP@C@ZayMv>v-t0ty|}xanLhDvKfFbSm&>+o zEOQ5TzBZ#XZ7Ydsb?WG}3RZAI_rw>|JaW18XH0yI+a+y68QBX(H$v2VkW8Vt9@6`t z$V=UiLr`K_JA>1D`D%Ar>Cr6Ig7n}!o7PrebiItv^alcb1DT~ zS1>}ncvi@Jg>3on@<-@aW&RH5P5fV!{}z3Ah0LsMBv9>|{}@Uk|5dWEO7;}G|NN*E zSINt>XRrgb3*L)SPFvaI*v-JShcI~&4m-S;Y?Zx)Urhe@FTNi0;^E{7b+iZS<*v{6C$h;fCTh2hS?2X1>W{7^nC~l{PI?x zQ!9ACjQtnwyjnJe&&j%eFtYVwkusBR?5B;gxAl{J*Dg|eM>5dlt7W3`4K2D_UR?di zy^J1^zTUoCdhiVvE`gd{Ba3k`c=|PRG$fhduE7#~ml|Iy`{GEE&hYrPGPT*S){Y;F zYt}tAMk8IP;QcHLE!8(RRQS-f-$aMiVfwU(AQo#^U_{1sU)b9lpG z6)6UDG%dYeW~5r11nsC>J@~jK6hb`RO!KdoEvu$(I48BN%Da`R?+MJb;CaFrIs5RL zNJl1iT`YX8g=P|SiLDbt=z`Enp&%N?9(YhIvXJApG>q08PeS*|`S}+GqwUU)79vx@ z4M4!<)c*zG&a~`0L*cErFk#<<}bDrOJZo&GXtdC>Ol2a zV{LVGVE+JF_r?=2@m6ZpoL4yh8jUn%iNzxp>i!m$F270g`Myav$)=3InEuLjPcPpj zFU18?zuzP?(<1%|k~wX4O98|3g{B($17#L}{EAl;Ubj3i5VG~WcMgxa%&VjlX?x3p7W#X(f9D?#-^@X2uQSmvO;oT%8Kva zWPLf5hdKns9<<<#=#h|Yma;aQrbBkx-VjX9(4-VeR;ctUlFcrLccIpeOO}}10at!r+Yp1}5m}hULqr z7I-!D+W?*ZXhr|kO9H;d&`jUQR&r1U4oE>Yie;O!S(fjJrp1`9h`!8lW7`Y0M$0HS zv6d4iN^e}WyM^JpR%=8ey0U+h_qN!)t?hCfL=HMSTszr>1)O3D3^R>6y{#BT^L}~= z&BP^J^l&i@r3>hEvGj@G@FCn1*%rPRH`;HE>2S8P z>(Y8v6s?m#FTu*&O87~o#$lP59Ib!9Kue`X*=S5yUX(lwr8IN!FayT>Xb_Mqzh0vED zJc}Q&Iv6uL zF{!h4*qNe7j@3|%i#)WhkMK`gO!Y2ovl*l8>U0|A-6CtH#l$A+5^%vRt_#!Db>J;Tlm zz;gRhFJ8kY9aRfgZAzGnxBZqMq$_bfOrc@2e!*^_ zTQ?^UDWf0OsKd4A{HVP;DxTj&*LgX+ifH{Xnag8?IE;eBnXD^x!Oq8Hb53k>)!2kc z`%!CTL=b~Q7{ot(=*)S+JV=Zk$1*EZq%=LW=frGRP7CGqaF@hKijBU3bbiD2XpCX* z0#(-qqOa zmm&o>t2!L#zs9p2JVA3u$fn&ja$ucnROdznXG1B0iulSRXDvkN@EH;B;E^)3j7OKzhTsw3O^-E0i>p9Osz47{ftG4WH-}AEc5}E!Oc^k; z7CbzhP6iFvGP42b2Esw@*N3KG&285)G50S4p@VN72cf(uxCcL)t4F%jTyi-R1-clV z^4*(`V0MJ-13rj58-gtXPp5jLWc#wW`1}{&Ys~#OfFQ1cokW<~9 zxu!LQXCS;qf-5V7)4}b*=kMqHpwhIxxQ=5`@L2|TI3h5HVB!G`YK>^s&oRiCMOzK~ z4)Yh2dYnPNIND16i9x<zO#=ZgC0=EMU8&0UvQrt;{i_^ zIH=C|5&bq=b}M^LI%`C*A{siX?6_mm(8*}%cr&MAiwWBnQUsfnjX1XlRv$BD;0C$1J?5sBj;ez!KBGP|_u0QluPc;fKz zpB)#EFsJdKU5N`}PPhNKzyv(iA`$kxe^*|eqHy^mW95^gWy1IoLr099Fv5LnXv~n% zh;i;g6{&+-xyKC-j|sa&ct(u0lPKX%nOffeR(yVeym!hb7;8cO6+get3AqI7cGlxNIJtBs_KTB;&FFcz9B1!W`Mo zSBmD~?69L0^TeaGGS1~_hu?27XZ%_$oT*{jKV7C=Z^sWt9o$SC{s3ijdB37u&VL1- zj2QR^gqy~|oe^#v1Fxl|NzjUSodmxEME@E2%X9G*SN_@6{dF^~ZtOuftY%xV-CVmQ zwARtU2Jp*pS_B@#zZ}6sZ2vLL(`Wx(2tK#K^hjpz0#6B#2mjfyn+D!1TPC+ia7F93 z5jK;S-z)27@(A%Czs=7@3`G981x;x8y|QiDKWexf{Q$1Vn&C>aFqvF=e>~hM8y<+T zI|kl@aApj=6=B8~JO6fs>&L*{94^;R?}=~*PWn+twl$=7hUv(_Lu|uK5RM&%Zd?F$n<`r-HmvD# z^al7C2AA8=uqI8JDx1k(NiO=ZxshDPgUo1a|1oUi3CA(q3=bMS5oJuo{7qaTtB2<_so0DVV!kFyt6z)~!pPi3M0Fz4_=GB1};PBwF zDX$k{AD(k5Z?=POn6u-gZw&Pq3>tO z!;PM+o<_L;I%`YX!ixEx7C@zlmMd`OAAqkC9*1j5Rs*)wo6 zn=)MvKj4S()TB-`WDs-lY%Q1LIz;5);c2mFdrt#v{hB;Xov71X*(LGZN(R=Vg>&Uc zWy^pPHz3;-JS;od@cj|k9UA8zHe^KD4ftL(f(h*@l*{O2!)E~V4A`&`Rnwh-c{$kW z@qih-0e8{`)U^M#+5I2|_a;%>`7o9ZJ1Em}nIg?TC_6Ut15bGT%YplhV6pYzOWz-q&y+14 z?Q*Qa^E#d#c)rAQ9FG{|a@56>iKi`|u6TOk>5pd=o~d{i;&}|uOL#uRlQh=lXoaT> zo~!U&ho=b77(6rZ5S~?dUcmDXp6z2@%^ZgiIEAO$I0%Y(e0VzG>5FF&o}qYdrQ9E6 zMu!n#TihKcpjmh>88c?&82nDgV-G8%^He<3@YsKtGRJ-Nzz?#2<8&;!%K*;AwDHQx z2WB(89CrUUPN?@yi`l(Nd*m6^$?L?->9(QgpHBVuzi&0cDzaZ1@!`H_dIT zYCDIAZnKEaY-f4TW-8rm{04aWO~`&6!_*AFHvDxIrr9l33-N}e4K0kKK z_=|I3Mquuqot};InE2Q*l(w24*zm9@Og&qvHeBCbtyGIdJN+5FYt=^^TB(jcJO24- zyw#7IXh&>4wE;8TL3vrN)umkSDEyY$@2`ox-?o~|F%V&Aq#rh1>l~QN<968TV*oda zfhPcN8v{=P%)1sle<@%r(6VSmPee41$q)}Xwt_n6z)VSE)0wKo!c0|SVWupx@D{)| zn9(|Du8F55p3ya^SE&jl%te_0K=Sl6 zubfI%Tf<%WlgrT=Pggv^D*aUBjKL{~-gG!q=~zJ27Z1)SW3Z~PUYRFXmZuL^*@<*w zmMWkccPf|2^N@FkN}^GBszydW&An4yW)x8GJ5)QI9vOX~y3tkY$;mCs%PYqJ#s1Pj zdkWsIytL^KyxgG&&3!_(qA%}K8TewmPUwMBjZRkBb2q|;v}m%*NGQ(r2D0(B4W~a|P(A42J#cWiY&bo-28CTYQ&mq0`E#=a#h%>!Jio6rT%6n8DxlOK%J-J! zgu-5LZb_~`|MJ^u?F!YDPCTkQ(#SHjq6bYYQ#m-$zOhX8H?E-6i5P?{soGNtH-}DC zO^mC^*raY)^r(uXoJr_@ug={4bP7*GVsDx`NoB)hV$&q$t=l&Ng5H?h2NsVUK5AHa z+>ntY#`dEVlhjb-YP#`WlzR;=zE^n*uWf&OC^U4S&RkFszBM$gpkR1tT=C$6AV|kP=r-Ui0xp5=4o1)r^0mZa`gX&Bxrl|yaV+v;ICfYd#AA1{^ zO9SpjU*DOkl8r%BeX8=~SbvA9%4ZbODE=v?eN$BzqlDbkRP*F;q>m*-LN|{XId{P-6=UMuaB-qe{GaMZVJf;!vKi z*yk(FTR=}epqj$`yW;^>h;PQ>S}m_!h)G!dlDdKxEL7<@SiE+jdK#aue(pt;lT=dV z%g^(( zTd%7Yi!eA#X)R${<&tVbla{D9a+%SWLWflzEhLo&uM{K~!g=u|so}6i4_vISGajQ2 zi?MiD(y_&=r+geF1+8dGyO*d&aI-F;NiP7(cu1Lvz9N6Nx1=cFQ<7Wi4f%?n&|}^z z!IJ};5eVh^i?Y4moc!F?+|P#+ioM>F{6J1|I494OgJF7-ep{-Vl`nZn?G`kAnd%_D z174tE%T!0xH{eC|e$o!rjQ+Vyx#fBzNZVHdWwVxJNxnqgmn)Ba8BMd1#*Om{Z!A~i zMfa_MRr?isbfxNp@kqgmk$Rv{#%pxVqZo&cK&2ILsT8{HEp-|Fx=6(* z<@>Yqva5{JA4b^wR;n6`Nm{o9Gb)I{&GCskAVAy^jro=JzFR1J)eDDElMO@0ho zadMLCN86uPHR#!=luPcy!ql{X{?qDOT$6tAX;mOTF{t8Ol}`hnK{Iz-BUOD3a_@=l zMM8GKlN0uabBjXZlAIiW?p_{^E_5Su8lTeeHOebKi;ULJcfcsla1r~!RM5Rf)bd%? zR(=jn11OX3c~-T6(0%k-HOlx>*O-)-Ula~xdrH0j{QSJ!9RGee0Rna1^{nL@&IL~c ze*+21&d>9C4$w`{f$4ljYnP~QxaWiimwtE-ZTW`IJcr(Y3sQ9QA=R9E)i;|HJr6eU z9c_MI)tBF6ko7?R@jQm^pgkl#)?!EwahnF=s(cjm1Kqt&b;XhIE$fi)2zuh#0Zom3 z0reiG+!s{0>|;jTSVCiQv1^!>4I4QjJO*S2)SQMBhiQQK>ZVMU|fXlW|*YB^FVn=NGAf&k7e5 z^kYKV1L}^B1qEZn(EAS?az_}o=LdYAaDHCc=kpZ>yu~Fy8@O!?9B}J;^zRq?biL{> z{$()V8AMZGLfk1@^pYAVPg_ehlj^?=&z)Z(9__3Ex9$D1YMT0+k!Vrp(vf4pum_g1 zw&e_N{oVOBh}x)MRW;hW0aJe#f<%?51k*d<_wyrt1O7NaGJC+E#srHnq-{_ai;03h zTd!Kv=nX1OP7{>LhVmcFL4RcPrseu9W1f4-So?1gDun z$6rT#7A)GNM$7wS`%dp|!azSDa;!HSJfvzSWqWdqAnTQ6dvc2M!CDuJ0DX82?f1W- z>d6O1Z^q)fRP=`G08X~z4V5oRM5qFmKxbkyMG4a6jbyC63^jlZGg`Pi7gWggT;m3a9Ej2Rv2{G<3 zmP6|*%i)}*5GX3~m6U`_!`XSo9&nD;x`Kq_?0~Ph2rQ&1FOV1V1fI0&ZS@Z1rKf}@ zj0xfFVxU)XUP+F>*q`I|KTRFpQSHSu5d;FYOA%`-fw;utqHJGI$X^;N2^Zyfiv7>V zBqn+CJsy8{ekkPe__D*HlIKJxp1w?fRf#v?&&~1ra!NwpoZQmfaMAO)1BV4`EhC4= z3G!n>HCbs|KYp90ZX*2rv0&%dkE~8$t;unRF zGSkfzx_ql@DA$WiEgm{+D+K$OXf5td#xamy?<)^Adk<4lZiR0~SZ=U0FMSW0U!l$K zK}vho%FyY3WPXjqg>aZ&@V@FRUl*5Il1Z&8Dlu^|B#84WnE#t@sW?=M_7^M_PY?Y1$IWG@OAq1uZzD z@{_zpD7G{oy$zLQhqFuG5_y#HD$2#nwQ!xcN|kSmUS#~LI?-<*sD#e%i2qI?_U8I? zJVoB(a2{$3`2Qg$aPR6<#ToFv*4rVNz03XXLR+_ED7ISRMmrFGkJ{`|y^Qy%YzMX! z+ZZZrQR)u|LiwK3u+QVk^+GNXDFX*$BNQGpVobPnV9A(~qXv#0Hl#Sbom%gNoV|kv z?NnXFPC*&zW+tzvY`Utp8ApHaL=_*B>qF(2ABlY0{3&FaPxiul^kZBTR4FL0{t)?g zSWJAC=spI9U6@_V!Meb~m9LVPRLwgk5oNT_+O1$ zxTq+4)YIONYR4h8m$e&qS5>QJ8YPqaYKTw;D{o+qz36w3EQtP2zc4o0T8-)`l) z>X7IWyZB?bjU&bnFJhKkWf@fg2ImHH^ZZ_4aVZuS7GU@X>b*y`NXW?!=VGVk@qtC= zm*i(3*3mQ%6;iJ~swJ6wFjhyc%z34r(ok;L6UYX|$jYmtkhmpSJND`%+> zR3O(E@@D(7^Uo_eNn)?+B!0B|laLe61!E2r6&LxkOLKiiKUpQL*^80*Inw8>Pf^@2 zl>aGqvYS3t9jM|{NHouJ>}i^`4|}R+pJ8SEikXF2mrmDyruvBAA}{Ynk~kxH%i5S~ z?^D&_ke<8`g81)P3z{X|ybtp4A6!s-e2YYv_rpGvH&KqYR@#Z*zzeS0qPo+d&r#09h?u;`L->mh*;6_QY2Z*$HEE(^0I(F9G3l+&{m9 zHe?C4`x*l5L$v*C)dSbrz40ydQ?GxG&bIzmU4b3eBh=(*j0~4N zZk6dDuuArdlGJv_6ISgh-=jaPY0~$qVL~`m4CM8B^S#)&czoU`bu^9QEKhN`mGQJ5 zw4}T|Pf00wXuGhu2R# z4yuO6S}W{6gz!4*a0pcI1@2{@d{K5FkISr2S7n+>wEK`skn1Hj;9si-Wc~o0eu*-E z!05e<5!Z-5_Xl8MIqf*6e6;5WEQ<}29^I{)CzgO0<^^)RAPj;0l5pNDRy-`_`(Ghr z>9Ekve@mx8978#QY+rT`^p2%Po=uYA%+#6EkD^s?SX0o0!xePnQEWvw)9j;I*IQ%{ zi^G0%R5eR}Qzi@?IH3ref}6u51`ZDm8#c1|Ey_5iS{iR#6Fv19*7`d!V^obcAqpE@ z3{+?Nt}KcQYf!E^&18D=INsO_9G^5FLp^7fs!3lS2Qt4$e;&uK=zadVh^{$-T-)TW zRT*M)T6aRVF+QOECsf0f?J_Z@bz{Q1c{?car0R(aj0c`nzQPaXf5lt;dD%rJVX)4S zC!449qX=Gxz4`g2IYr*ECod;IJNIMYB?N&~T5}Q&-$mO_LS6MwYvC6K0)D^Wo0C^s zR2s_3&i#a=+ahyd{)v7+t!5fO)AV0e0KD#v zUm+Czi~cwRGH};#$Z(40|AxxE15R_GoB6A}xqUNBmIHh0Eq}v=YGN?p3`E{DvRNg) zlV{~MQ6%hO{LZCz1;f*YPdE!K{Zrmy6}9ZFDwPwl4c0;;ASX5Z9hVADQZ(duXmsvX z*VBqoV8gCIRJ@q1DDw~H6I1kWW18>>Xv$PYGycFboThH3%)#oqM0ZqvbB`T7WcYom zFUKcSyFXP&IUSP^Zh*6?f2t&KzbF2LzH}xX{S%eVQa90OO+nw<3{imxmzeQ#Hgr%d z7C-m~zBoCDa6w^zd=z$oVcsd{sUoZYUycFmXd}#cd@Z!2Fq`A-%3xtOmiL225BL$t z{s0A?W|IWJ7v!tN6AF8BJ;fex=|T#bW?#yYW(riCI=mcVbCDVqGtlj*hB7mfNX5sj z+tM-N@M2vm4TNt4xY;OW`oYasDf4Reu*$fU+@_fhy=Eg5cNslGo%m-NS29E{hhUG1 zYe3(dW?i`g^Uc$^=vK(jjpEE?`6#**iE+_QaVGP$QE{dRd&5WLOrLxlV#&n$7|+9T zsB;y$tC@Y|6RJ0L8wOfgUJb*xTJ@*Htx?7Lk)Zs^)e-fix`aiBrsQ^-F8ZQ6TKyEA zNQRDR5r?0qUbW22fNjrM0pEZ%R{s{&KSvnicI zvbk2>VvnA?rr8f1Oc$IDm1#vyjP#3EQD-QFgOT8cH_Sf?l2Oz?Sii?ZF zd4Yg8f4l10zU1~1w0S=^WW@<)Te$-h{^(D*7;{u;#E{~hR<}P(z({<^VGnesjS|fO zZi?5#erqBIWEV8eT3%a`XnMgSK1wt_@)I?T7vg|(7x!-TT_f|{BwR4CM~$>mH;#Ty zGJDCr>P|bRFKwucMfxedTi0A8KZ6l&Mgy~>v5zX!%^)xjchECBewmEceWAk%{?fc~ zI6Dk!0~?7@vF}SAot*Ct7w16O^7wMHi?fS8`_)rM`>574i(W}FuQd)(LOt_2u#Jj( z=2~VOspdNK8}&j=&gozrTxWtO-xnwim-ry_<&@+G^1id0(!K$j@;&WuY_^~Y4X}s~ zs*5%5)8PhY4S5I)nOA~_C_T-r24>PM4fB4O`lgxLU?KOVnSOCp(eo={xr7K+o01!% zoMY-?>b(?O@Y06n0C`*uurwH7G&C=fCv2!=BS0sC-L`h4oBR=|SXpCyx>*N&Bt6|6 z3O+J39Vqq-4<_>wJ)O-mOjos2D!VjVZuv$%uAFYD)tQ+k;6NzRyyGPPF^!==wWayVzEFCO=R}(Dg<4lvkbEPM>e`D19 zrxkt>VKLF9lRVoy8=LWBl1T>|n|b119hsF_l$}@VhYGg{yHjsYD1WjQPj_F0H>X5^ zat5ZFwDltMN;wTP#C(F555vway9rk1eTcepBo@LThNhcdt5tKFn5klhNo|<|gLOD* z8`8mnzGz~0#jXoKe!8|Pnmd~YH^ow%1NifuDChO2W*V5!-lpbFa-O+}yjf;GEgzz; zI(JRXH?OA?L%=iBFUHs{F#B6g`+gV}^<5lwzq#Cc`-3`QcKa_juLO^3n`J&O7vj}H z@rXR0Wfp=fd76V^EHa04+Z&R545-s@%~1lGqphf{7UtF1Q{B_T94nXDksv2m%ZJSI zR*xYc)TX&D&8x7@`l6*dK`u4*3~A!hlPX$aQ9eRDTbb9$Wv1RWrBk=osARd>(ZaR~ mt#Nz93R>LSycp`riq>WpSkj@^W^1|9v=&5pZX2_&@&5n{I$~-7 diff --git a/crates/yaak-query/src/complete.rs b/crates/yaak-query/src/complete.rs index 9849c50c2..4c68b45e5 100644 --- a/crates/yaak-query/src/complete.rs +++ b/crates/yaak-query/src/complete.rs @@ -25,7 +25,10 @@ pub struct Completion { /// What replaces the span: `@method:`, `@header.`, or a value quoted when it needs to be. pub insert: String, pub kind: CompletionKind, + /// What a field is, from its definition. pub info: Option, + /// A term using the field, from its definition. + pub example: Option, /// The term is not finished after inserting (a field still needs its value, a map field its /// key), so completion should open again. pub reopen: bool, @@ -115,6 +118,7 @@ fn missing_at(word: &str, typed: &str, schema: &Schema) -> Option { insert: term, kind: CompletionKind::Field, info: None, + example: None, // With no value yet, go on to the field's values reopen: value.is_empty(), }) @@ -143,7 +147,8 @@ fn field_completions( insert: format!("@{path}:"), label: path, kind: CompletionKind::Field, - info: None, + info: map.info.clone(), + example: map.example.clone(), reopen: true, }); } @@ -157,6 +162,7 @@ fn field_completions( insert, kind: CompletionKind::Field, info: field.info.clone(), + example: field.example.clone(), reopen: true, }); } @@ -191,6 +197,7 @@ fn value_completions( insert: format_value(value), kind: CompletionKind::Value, info: None, + example: None, reopen: false, }) .collect(); @@ -236,14 +243,27 @@ mod tests { kind: FieldKind::Text, map: false, info: None, + example: None, }; Schema::new( vec![ text("method"), text("url"), text("name"), - FieldDef { name: "header".into(), kind: FieldKind::Text, map: true, info: None }, - FieldDef { name: "error".into(), kind: FieldKind::Flag, map: false, info: None }, + FieldDef { + name: "header".into(), + kind: FieldKind::Text, + map: true, + info: None, + example: None, + }, + FieldDef { + name: "error".into(), + kind: FieldKind::Flag, + map: false, + info: None, + example: None, + }, ], FreeText::Fuzzy, ) @@ -369,6 +389,57 @@ mod tests { assert_eq!(at("a|b"), Some(("a".into(), inserts(&["@name:", "@header."])))); } + /// Field names that start with what is typed come first, then those holding it, in the + /// schema's order; nothing matches loosely. + #[test] + fn prefix_matches_first() { + let field = |name: &str| FieldDef { + name: name.into(), + kind: FieldKind::Text, + map: false, + info: None, + example: None, + }; + let s = Schema::new( + vec![ + field("time"), + field("scheme"), + field("method"), + field("host"), + ], + FreeText::Substring, + ); + let labels = |input: &str| { + let c = completions(input, input.len(), &s, &HashMap::new()).unwrap(); + c.items.into_iter().map(|i| i.label).collect::>() + }; + assert_eq!(labels("m"), ["method", "time", "scheme"]); + assert_eq!(labels("@h"), ["host", "scheme", "method"]); + assert!(completions("mt", 2, &s, &HashMap::new()).is_none(), "no fuzzy match"); + } + + /// A field's description and example go with it, and with the keys under a map. + #[test] + fn descriptions_and_examples() { + let s = Schema::new( + vec![FieldDef { + name: "header".into(), + kind: FieldKind::Text, + map: true, + info: Some("A header on either side".into()), + example: Some("@header.content-type:json".into()), + }], + FreeText::Substring, + ); + let values = HashMap::from([("header".to_string(), vec!["accept".to_string()])]); + let c = completions("@he", 3, &s, &values).unwrap(); + assert_eq!(c.items[0].info.as_deref(), Some("A header on either side")); + assert_eq!(c.items[0].example.as_deref(), Some("@header.content-type:json")); + let c = completions("@header.", 8, &s, &values).unwrap(); + assert_eq!(c.items[0].label, "header.accept"); + assert_eq!(c.items[0].example.as_deref(), Some("@header.content-type:json")); + } + #[test] fn cursor_is_clamped() { let c = completions("é", 1, &schema(), &values()); diff --git a/crates/yaak-query/src/condition.rs b/crates/yaak-query/src/condition.rs new file mode 100644 index 000000000..657931217 --- /dev/null +++ b/crates/yaak-query/src/condition.rs @@ -0,0 +1,241 @@ +//! What a field term asks of a field's values, resolved once from the term's value and the +//! field's kind. +//! +//! The evaluator matches with these, and a consumer that compiles queries elsewhere (to SQL, +//! say) translates them rather than deriving the rules again, so the two agree. The helpers +//! below are the value tests the conditions are made of, for a compiler that has to run them +//! where the values are. + +use crate::ast::{CompareOp, FieldValue}; +use crate::lexer::parse_number; +use crate::schema::{FieldKind, Unit, parse_flag, to_base}; + +/// A field term's test, applied to each of the field's values: the term matches when any value +/// passes, except [`Condition::Flag`] `false`, which matches when none is set. +#[derive(Clone, Debug, PartialEq)] +pub enum Condition { + /// The value, as lowercase text, contains this lowercase text, which is never empty. + Contains(String), + /// The whole value, as lowercase text, matches this lowercase pattern, where `*` matches + /// any run of characters. A pattern without `*` must equal the value. + Wildcard(String), + /// The value is there: text that is not empty, any number, or a flag that is set. + Present, + /// The value reads as a number. + IsNumber, + /// The value, read as a number, equals this one, in the field's base unit. + Equals(f64), + /// The value, read as a number, compares so with this one, in the field's base unit. + Compare(CompareOp, f64), + /// The value, read as a number, is in this inclusive range, in the field's base unit. + Range(f64, f64), + /// With `true`, some value is a set flag; with `false`, none is. + Flag(bool), + /// Nothing matches: a value the field cannot take, such as a unit it does not have. + Never, +} + +impl Condition { + /// The condition `value` sets for a field of `kind`. + pub fn new(kind: &FieldKind, value: &FieldValue) -> Self { + match kind { + FieldKind::Text => text_condition(value), + FieldKind::Number { units } => number_condition(value, units), + FieldKind::Flag => flag_condition(value), + } + } + + /// Whether a text value passes. + pub fn matches_text(&self, text: &str) -> bool { + match self { + Condition::Contains(needle) => contains_ignore_case(text, needle), + Condition::Wildcard(pattern) => wildcard_match(pattern, &text.to_lowercase()), + Condition::Present => !text.is_empty(), + Condition::Flag(_) => parse_flag(text) == Some(true), + _ => self.matches_parsed_number(read_number(text)), + } + } + + /// Whether a number value passes. + pub fn matches_number(&self, n: f64) -> bool { + match self { + Condition::Contains(needle) => format_number(n).contains(needle.as_str()), + Condition::Wildcard(pattern) => wildcard_match(pattern, &format_number(n)), + Condition::Present => true, + Condition::Flag(_) => n != 0.0, + _ => self.matches_parsed_number(Some(n)), + } + } + + /// Whether a flag value passes. + pub fn matches_flag(&self, b: bool) -> bool { + let text = if b { "true" } else { "false" }; + match self { + Condition::Contains(needle) => text.contains(needle.as_str()), + Condition::Wildcard(pattern) => wildcard_match(pattern, text), + Condition::Present | Condition::Flag(_) => b, + _ => false, + } + } + + fn matches_parsed_number(&self, n: Option) -> bool { + match self { + Condition::IsNumber => n.is_some(), + Condition::Equals(target) => n == Some(*target), + Condition::Compare(op, target) => n.is_some_and(|n| op.test(n, *target)), + Condition::Range(low, high) => n.is_some_and(|n| n >= *low && n <= *high), + _ => false, + } + } +} + +fn text_condition(value: &FieldValue) -> Condition { + match value { + FieldValue::Text { text, .. } if text.is_empty() => Condition::Never, + FieldValue::Text { text, .. } => Condition::Contains(text.to_lowercase()), + FieldValue::Wildcard { pattern } => Condition::Wildcard(pattern.to_lowercase()), + FieldValue::Exists => Condition::Present, + // A text field holding numbers, such as a status shown as text + FieldValue::Compare { op, number } if number.unit.is_none() => { + Condition::Compare(*op, number.value) + } + FieldValue::Range { low, high } if low.unit.is_none() && high.unit.is_none() => { + Condition::Range(low.value, high.value) + } + FieldValue::Compare { .. } | FieldValue::Range { .. } => Condition::Never, + } +} + +fn number_condition(value: &FieldValue, units: &[Unit]) -> Condition { + match value { + FieldValue::Text { text, .. } if text.is_empty() => Condition::Never, + // `@status:404` is equality; anything else matches the number as text + FieldValue::Text { text, quoted: false } => match parse_number(text) { + Some(number) => to_base(&number, units).map_or(Condition::Never, Condition::Equals), + None => text_condition(value), + }, + FieldValue::Text { quoted: true, .. } | FieldValue::Wildcard { .. } => { + text_condition(value) + } + FieldValue::Exists => Condition::IsNumber, + FieldValue::Compare { op, number } => to_base(number, units) + .map_or(Condition::Never, |target| Condition::Compare(*op, target)), + FieldValue::Range { low, high } => match (to_base(low, units), to_base(high, units)) { + (Some(low), Some(high)) => Condition::Range(low, high), + _ => Condition::Never, + }, + } +} + +fn flag_condition(value: &FieldValue) -> Condition { + match value { + FieldValue::Exists => Condition::Flag(true), + FieldValue::Text { text, .. } => parse_flag(text).map_or(Condition::Never, Condition::Flag), + _ => Condition::Never, + } +} + +/// Whether `hay`, lowercased, contains `needle`, which is lowercase already. Copies nothing +/// when both are ASCII. +pub fn contains_ignore_case(hay: &str, needle: &str) -> bool { + if !(hay.is_ascii() && needle.is_ascii()) { + return hay.to_lowercase().contains(needle); + } + let (hay, needle) = (hay.as_bytes(), needle.as_bytes()); + needle.is_empty() || hay.windows(needle.len()).any(|w| w.eq_ignore_ascii_case(needle)) +} + +/// Whether `pattern`, where `*` matches any run of characters, matches the whole of `text`. +/// Case-sensitive: both are lowercase already when matching a query. +pub fn wildcard_match(pattern: &str, text: &str) -> bool { + let mut parts = pattern.split('*'); + let first = parts.next().unwrap_or_default(); + let Some(mut rest) = text.strip_prefix(first) else { + return false; + }; + let parts: Vec<&str> = parts.collect(); + let Some((last, middle)) = parts.split_last() else { + // No `*` at all: the whole text must equal the pattern + return rest.is_empty(); + }; + for part in middle { + match rest.find(part) { + Some(i) => rest = &rest[i + part.len()..], + None => return false, + } + } + rest.len() >= last.len() && rest.ends_with(last) +} + +/// A text value read as a number, as comparisons read it: the whole text, spaces around it +/// aside. +pub fn read_number(text: &str) -> Option { + text.trim().parse().ok() +} + +/// A number as text, as a field's text matching sees it: `200`, not `200.0`. +pub fn format_number(n: f64) -> String { + if n.fract() == 0.0 && n.abs() < 1e15 { format!("{}", n as i64) } else { n.to_string() } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ast::Node; + use crate::parse; + + fn condition(kind: FieldKind, input: &str) -> Condition { + let Some(Node::Field { value, .. }) = parse(input) else { + panic!("not a field") + }; + Condition::new(&kind, &value) + } + + fn ms() -> FieldKind { + FieldKind::Number { + units: vec![ + Unit { name: "ms".into(), factor: 1.0 }, + Unit { name: "s".into(), factor: 1000.0 }, + ], + } + } + + #[test] + fn resolves_text_values() { + assert_eq!(condition(FieldKind::Text, "@a:Foo"), Condition::Contains("foo".into())); + assert_eq!(condition(FieldKind::Text, "@a:"), Condition::Never); + assert_eq!(condition(FieldKind::Text, "@a:F*o"), Condition::Wildcard("f*o".into())); + assert_eq!(condition(FieldKind::Text, "@a:*"), Condition::Present); + assert_eq!(condition(FieldKind::Text, "@a:>5"), Condition::Compare(CompareOp::Gt, 5.0)); + assert_eq!(condition(FieldKind::Text, "@a:>5s"), Condition::Never); + } + + #[test] + fn resolves_number_values_in_base_units() { + assert_eq!(condition(ms(), "@d:1.5s"), Condition::Equals(1500.0)); + assert_eq!(condition(ms(), "@d:1m"), Condition::Never); + assert_eq!(condition(ms(), "@d:<2s"), Condition::Compare(CompareOp::Lt, 2000.0)); + assert_eq!(condition(ms(), "@d:[1s TO 2s]"), Condition::Range(1000.0, 2000.0)); + assert_eq!(condition(ms(), "@d:*"), Condition::IsNumber); + assert_eq!(condition(ms(), "@d:\"15\""), Condition::Contains("15".into())); + assert_eq!(condition(ms(), "@d:abc"), Condition::Contains("abc".into())); + } + + #[test] + fn resolves_flags() { + assert_eq!(condition(FieldKind::Flag, "@f:*"), Condition::Flag(true)); + assert_eq!(condition(FieldKind::Flag, "@f:no"), Condition::Flag(false)); + assert_eq!(condition(FieldKind::Flag, "@f:maybe"), Condition::Never); + } + + #[test] + fn tests_values() { + let c = Condition::Wildcard("40*".into()); + assert!(c.matches_number(404.0)); + assert!(!c.matches_text("504")); + assert!(Condition::Equals(200.0).matches_text(" 200 ")); + assert!(Condition::Contains("é".into()).matches_text("CAFÉ")); + assert!(!Condition::Present.matches_text("")); + assert!(Condition::Flag(true).matches_flag(true)); + } +} diff --git a/crates/yaak-query/src/eval.rs b/crates/yaak-query/src/eval.rs index 85dd998e8..f494678c6 100644 --- a/crates/yaak-query/src/eval.rs +++ b/crates/yaak-query/src/eval.rs @@ -1,8 +1,11 @@ //! Runs a parsed query in memory, against one document or a whole table of them. -use crate::ast::{FieldValue, Node}; +use crate::ast::Node; +use crate::condition::Condition; +#[cfg(test)] +use crate::condition::wildcard_match; use crate::fuzzy::{FuzzyScratch, fuzzy_match_in}; -use crate::schema::{FieldKind, FreeText, Schema, parse_flag, to_base}; +use crate::schema::{FreeText, Schema}; use serde::{Deserialize, Serialize}; use std::borrow::Cow; use std::collections::HashMap; @@ -208,55 +211,13 @@ enum Val<'a> { } impl Val<'_> { - /// The value as lowercase text. - fn lower(&self) -> Cow<'_, str> { + fn passes(&self, condition: &Condition) -> bool { match self { - Val::Text(text) => Cow::Owned(text.to_lowercase()), - Val::Number(n) => Cow::Owned(format_number(*n)), - Val::Flag(b) => Cow::Borrowed(if *b { "true" } else { "false" }), + Val::Text(text) => condition.matches_text(text), + Val::Number(n) => condition.matches_number(*n), + Val::Flag(b) => condition.matches_flag(*b), } } - - /// Whether the value, lowercased, contains `needle` (lowercase). - fn contains(&self, needle: &str) -> bool { - match self { - Val::Text(text) => contains_ignore_case(text, needle), - _ => self.lower().contains(needle), - } - } - - fn as_number(&self) -> Option { - match self { - Val::Number(n) => Some(*n), - Val::Text(text) => text.trim().parse().ok(), - Val::Flag(_) => None, - } - } - - fn as_flag(&self) -> Option { - match self { - Val::Flag(b) => Some(*b), - Val::Text(text) => parse_flag(text), - Val::Number(n) => Some(*n != 0.0), - } - } - - fn is_present(&self) -> bool { - match self { - Val::Text(text) => !text.is_empty(), - Val::Number(_) => true, - Val::Flag(b) => *b, - } - } -} - -/// Whether `hay`, lowercased, contains `needle` (lowercase), without copying ASCII text. -fn contains_ignore_case(hay: &str, needle: &str) -> bool { - if !(hay.is_ascii() && needle.is_ascii()) { - return hay.to_lowercase().contains(needle); - } - let (hay, needle) = (hay.as_bytes(), needle.as_bytes()); - needle.is_empty() || hay.windows(needle.len()).any(|w| w.eq_ignore_ascii_case(needle)) } /// `text` lowercased into `out`, without a fresh allocation per call. @@ -270,11 +231,6 @@ fn lowercase_into(text: &str, out: &mut String) { } } -/// `200`, not `200.0`, so a number field's text matches what the user sees. -fn format_number(n: f64) -> String { - if n.fract() == 0.0 && n.abs() < 1e15 { format!("{}", n as i64) } else { n.to_string() } -} - /// Where field values come from: a [`Document`], or a row of a [`Table`]. trait FieldSource { /// Whether any value under `path` (lowercase) passes `test`. With `any_key`, `path` is a @@ -351,9 +307,8 @@ enum Plan { path: String, /// A map field with no key: every key's values. any_key: bool, - kind: FieldKind, - /// With its text and wildcard pattern lowercased. - value: FieldValue, + /// Never [`Condition::Never`], which plans as [`Plan::Never`]. + condition: Condition, }, /// A term that can match nothing: an unknown field, an empty word or phrase. Never, @@ -418,14 +373,10 @@ impl Matcher { Plan::Not(node) => !self.eval(node, text, source, fuzzy), Plan::Term(term) if self.fuzzy => fuzzy_match_in(text, term, fuzzy), Plan::Term(term) | Plan::Phrase(term) => text.contains(term.as_str()), - Plan::Field { path, any_key, kind, value } => { - let mut any = - |test: &mut dyn FnMut(Val) -> bool| source.any_value(path, *any_key, test); - match kind { - FieldKind::Text => match_text(&mut any, value), - FieldKind::Number { units } => match_number(&mut any, value, units), - FieldKind::Flag => match_flag(&mut any, value), - } + Plan::Field { path, any_key, condition } => { + let any = source.any_value(path, *any_key, &mut |v| v.passes(condition)); + // `@flag:false`: no value is set + if *condition == Condition::Flag(false) { !any } else { any } } Plan::Never => false, } @@ -458,117 +409,27 @@ fn plan_node(node: &Node, schema: &Schema, uses_text: &mut bool) -> Plan { let Some(field) = schema.resolve(path) else { return Plan::Never; }; - let value = match value { - FieldValue::Text { text, quoted } => { - FieldValue::Text { text: text.to_lowercase(), quoted: *quoted } - } - FieldValue::Wildcard { pattern } => { - FieldValue::Wildcard { pattern: pattern.to_lowercase() } - } - other => other.clone(), - }; + let condition = Condition::new(&field.def.kind, value); + if condition == Condition::Never { + return Plan::Never; + } Plan::Field { path: field.document_path(), any_key: field.def.map && field.key.is_none(), - kind: field.def.kind.clone(), - value, + condition, } } } } -/// Asks whether any of a field's values passes a test. -type AnyValue<'s> = dyn FnMut(&mut dyn FnMut(Val) -> bool) -> bool + 's; - -/// `value`'s text and wildcard pattern are already lowercase (see [`plan_node`]). -fn match_text(any: &mut AnyValue, value: &FieldValue) -> bool { - match value { - FieldValue::Text { text: needle, .. } => { - !needle.is_empty() && any(&mut |v| v.contains(needle)) - } - FieldValue::Wildcard { pattern } => any(&mut |v| wildcard_match(pattern, &v.lower())), - FieldValue::Exists => any(&mut |v| v.is_present()), - // A text field holding numbers, such as a status shown as text - FieldValue::Compare { op, number } => { - number.unit.is_none() - && any(&mut |v| v.as_number().is_some_and(|n| op.test(n, number.value))) - } - FieldValue::Range { low, high } => { - low.unit.is_none() - && high.unit.is_none() - && any(&mut |v| v.as_number().is_some_and(|n| n >= low.value && n <= high.value)) - } - } -} - -fn match_number(any: &mut AnyValue, value: &FieldValue, units: &[crate::Unit]) -> bool { - match value { - FieldValue::Text { text, quoted } => { - if text.is_empty() { - return false; - } - // `@status:404` is equality; anything else matches the number as text - match (!quoted).then(|| crate::lexer::parse_number(text)).flatten() { - Some(number) => to_base(&number, units) - .is_some_and(|target| any(&mut |v| v.as_number() == Some(target))), - None => match_text(any, value), - } - } - FieldValue::Wildcard { .. } => match_text(any, value), - FieldValue::Exists => any(&mut |v| v.as_number().is_some()), - FieldValue::Compare { op, number } => to_base(number, units) - .is_some_and(|target| any(&mut |v| v.as_number().is_some_and(|n| op.test(n, target)))), - FieldValue::Range { low, high } => match (to_base(low, units), to_base(high, units)) { - (Some(low), Some(high)) => { - any(&mut |v| v.as_number().is_some_and(|n| n >= low && n <= high)) - } - _ => false, - }, - } -} - -fn match_flag(any: &mut AnyValue, value: &FieldValue) -> bool { - let mut is_set = || any(&mut |v| v.as_flag() == Some(true)); - match value { - FieldValue::Exists => is_set(), - FieldValue::Text { text, .. } => match parse_flag(text) { - Some(true) => is_set(), - Some(false) => !is_set(), - None => false, - }, - _ => false, - } -} - -/// Whether `pattern`, where `*` matches any run of characters, matches the whole of `text`. -pub(crate) fn wildcard_match(pattern: &str, text: &str) -> bool { - let mut parts = pattern.split('*'); - let first = parts.next().unwrap_or_default(); - let Some(mut rest) = text.strip_prefix(first) else { - return false; - }; - let parts: Vec<&str> = parts.collect(); - let Some((last, middle)) = parts.split_last() else { - // No `*` at all: the whole text must equal the pattern - return rest.is_empty(); - }; - for part in middle { - match rest.find(part) { - Some(i) => rest = &rest[i + part.len()..], - None => return false, - } - } - rest.len() >= last.len() && rest.ends_with(last) -} - #[cfg(test)] mod tests { use super::*; - use crate::schema::{FieldDef, Unit}; + use crate::schema::{FieldDef, FieldKind, Unit}; use crate::{format_field, parse}; fn text_field(name: &str) -> FieldDef { - FieldDef { name: name.into(), kind: FieldKind::Text, map: false, info: None } + FieldDef { name: name.into(), kind: FieldKind::Text, map: false, info: None, example: None } } /// The fields and free-text mode of Yaak's sidebar. @@ -594,12 +455,19 @@ mod tests { Schema::new( vec![ text_field("url"), - FieldDef { name: "header".into(), kind: FieldKind::Text, map: true, info: None }, + FieldDef { + name: "header".into(), + kind: FieldKind::Text, + map: true, + info: None, + example: None, + }, FieldDef { name: "status".into(), kind: FieldKind::Number { units: vec![] }, map: false, info: None, + example: None, }, FieldDef { name: "duration".into(), @@ -611,8 +479,15 @@ mod tests { }, map: false, info: None, + example: None, + }, + FieldDef { + name: "error".into(), + kind: FieldKind::Flag, + map: false, + info: None, + example: None, }, - FieldDef { name: "error".into(), kind: FieldKind::Flag, map: false, info: None }, ], FreeText::Substring, ) diff --git a/crates/yaak-query/src/lib.rs b/crates/yaak-query/src/lib.rs index 14cc87e66..e0fc05bd3 100644 --- a/crates/yaak-query/src/lib.rs +++ b/crates/yaak-query/src/lib.rs @@ -9,7 +9,9 @@ //! //! [`parse`] gives the tree, which a consumer may walk to compile it elsewhere (to SQL, say). //! [`evaluate`] runs it in memory against a [`Document`], with a [`Schema`] saying what the -//! fields are. [`tokens`], [`completions`] and [`format_field`] serve the search box. +//! fields are. [`tokens`], [`completions`] and [`format_field`] serve the search box. A compiler +//! resolves each field term to a [`Condition`] and translates that, so it matches as +//! [`evaluate`] does. // The wasm-bindgen exports expand to unsafe glue; nothing else in the crate may use unsafe #![cfg_attr( @@ -20,10 +22,12 @@ pub mod ast; mod complete; +mod condition; mod eval; mod format; pub mod fuzzy; mod lexer; +mod offsets; mod parser; mod schema; mod tokens; @@ -33,8 +37,10 @@ mod wasm; pub use ast::{CompareOp, FieldValue, Node, NumberLiteral, Span}; pub use complete::{Completion, CompletionKind, Completions, completions}; +pub use condition::{Condition, contains_ignore_case, format_number, read_number, wildcard_match}; pub use eval::{DocValue, DocValues, Document, Matcher, Table, evaluate}; pub use format::{format_field, format_value, is_valid_path, quote}; +pub use offsets::Utf16Offsets; pub use parser::parse; pub use schema::{Diagnostic, FieldDef, FieldKind, FreeText, ResolvedField, Schema, Unit}; pub use tokens::{Token, TokenKind, tokens}; diff --git a/crates/yaak-query/src/offsets.rs b/crates/yaak-query/src/offsets.rs new file mode 100644 index 000000000..999c339ac --- /dev/null +++ b/crates/yaak-query/src/offsets.rs @@ -0,0 +1,52 @@ +//! Byte offsets to UTF-16 offsets and back, for spans handed to JavaScript, which counts string +//! positions in UTF-16 code units. + +use crate::ast::Span; + +/// Byte offset to UTF-16 offset, and back, for one input. +pub struct Utf16Offsets { + /// `utf16[i]` is the UTF-16 offset of byte `i`, for every char boundary and the end. + utf16: Vec, +} + +impl Utf16Offsets { + pub fn new(input: &str) -> Self { + let mut utf16 = vec![0; input.len() + 1]; + let mut units = 0u32; + for (i, c) in input.char_indices() { + for b in 0..c.len_utf8() { + utf16[i + b] = units; + } + units += c.len_utf16() as u32; + } + utf16[input.len()] = units; + Self { utf16 } + } + + /// A byte span as a UTF-16 one. + pub fn span(&self, span: Span) -> Span { + Span { start: self.utf16[span.start()], end: self.utf16[span.end()] } + } + + /// The byte offset of UTF-16 offset `pos`, or of the char it falls inside. + pub fn byte(&self, pos: u32) -> usize { + self.utf16.partition_point(|&u| u < pos).min(self.utf16.len() - 1) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn converts_both_ways() { + let input = "é@a:😀x"; + let offsets = Utf16Offsets::new(input); + // `é` is two bytes and one unit; `😀` is four bytes and two units + assert_eq!(offsets.span(Span::new(2, 5)), Span { start: 1, end: 4 }); + assert_eq!(offsets.span(Span::new(5, 10)), Span { start: 4, end: 7 }); + assert_eq!(offsets.byte(4), 5); + assert_eq!(offsets.byte(6), 9); + assert_eq!(offsets.byte(99), input.len()); + } +} diff --git a/crates/yaak-query/src/schema.rs b/crates/yaak-query/src/schema.rs index 17a1c02fb..b37abc7a5 100644 --- a/crates/yaak-query/src/schema.rs +++ b/crates/yaak-query/src/schema.rs @@ -45,10 +45,14 @@ pub struct FieldDef { #[serde(default)] #[cfg_attr(test, ts(optional, as = "Option"))] pub map: bool, - /// A short note shown with the field in completions. + /// What the field is, in a few words, shown with it in completions: `GET, POST, …`. #[serde(default)] #[cfg_attr(test, ts(optional))] pub info: Option, + /// A term using it, shown with it in completions: `@method:post`. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[cfg_attr(test, ts(optional))] + pub example: Option, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -207,7 +211,7 @@ mod tests { use crate::parse; fn field(name: &str, kind: FieldKind, map: bool) -> FieldDef { - FieldDef { name: name.to_string(), kind, map, info: None } + FieldDef { name: name.to_string(), kind, map, info: None, example: None } } fn schema() -> Schema { diff --git a/crates/yaak-query/src/wasm.rs b/crates/yaak-query/src/wasm.rs index fbf0089b3..85019be6a 100644 --- a/crates/yaak-query/src/wasm.rs +++ b/crates/yaak-query/src/wasm.rs @@ -3,43 +3,13 @@ //! JavaScript counts string positions in UTF-16 code units, so every span going out is //! converted from bytes, and the cursor coming in is converted to bytes. -use crate::{Document, Matcher, Schema, Span, Table}; +use crate::{Document, Matcher, Schema, Table, Utf16Offsets as Offsets}; use serde::Serialize; use serde::de::DeserializeOwned; use std::collections::HashMap; use wasm_bindgen::JsValue; use wasm_bindgen::prelude::wasm_bindgen; -/// Byte offset to UTF-16 offset, and back, for one input. -struct Offsets { - /// `utf16[i]` is the UTF-16 offset of byte `i`, for every char boundary and the end. - utf16: Vec, -} - -impl Offsets { - fn new(input: &str) -> Self { - let mut utf16 = vec![0; input.len() + 1]; - let mut units = 0u32; - for (i, c) in input.char_indices() { - for b in 0..c.len_utf8() { - utf16[i + b] = units; - } - units += c.len_utf16() as u32; - } - utf16[input.len()] = units; - Self { utf16 } - } - - fn span(&self, span: Span) -> Span { - Span { start: self.utf16[span.start()], end: self.utf16[span.end()] } - } - - /// The byte offset of UTF-16 offset `pos`, or of the char it falls inside. - fn byte(&self, pos: u32) -> usize { - self.utf16.partition_point(|&u| u < pos).min(self.utf16.len() - 1) - } -} - fn to_js(value: &T) -> Result { value .serialize(&serde_wasm_bindgen::Serializer::json_compatible()) diff --git a/package-lock.json b/package-lock.json index 9a34d7914..1238d6bc1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -272,6 +272,7 @@ "babel-plugin-react-compiler": "^1.0.0", "typescript": "^5.8.3", "vite": "npm:@voidzero-dev/vite-plus-core@^1.4.0", + "vite-plugin-wasm": "^3.5.0", "vite-plus": "^1.4.0" } }, diff --git a/packages/ui/src/components/editor/filter/extension.ts b/packages/ui/src/components/editor/filter/extension.ts index 869a50853..a522e3fe6 100644 --- a/packages/ui/src/components/editor/filter/extension.ts +++ b/packages/ui/src/components/editor/filter/extension.ts @@ -1,8 +1,5 @@ import type { Completion, CompletionContext, CompletionResult } from "@codemirror/autocomplete"; import { autocompletion, startCompletion } from "@codemirror/autocomplete"; -import { LanguageSupport, LRLanguage, syntaxTree } from "@codemirror/language"; -import type { SyntaxNode } from "@lezer/common"; -import { parser } from "./filter"; export interface FieldDef { name: string; @@ -87,17 +84,7 @@ function fieldCompletionFrom( return null; } -function inPhrase(ctx: CompletionContext): boolean { - // Lezer node names from your grammar: Phrase is the quoted token - let n: SyntaxNode | null = syntaxTree(ctx.state).resolveInner(ctx.pos, -1); - while (n) { - if (n.name === "Phrase") return true; - n = n.parent; - } - return false; -} - -// While typing an incomplete quote, there's no Phrase token yet. +/** Inside a quote, opened and not yet closed */ function inUnclosedQuote(doc: string, pos: number): boolean { let quotes = 0; for (let i = 0; i < pos; i++) { @@ -200,7 +187,7 @@ export function makeCompletionSource(opts: FilterOptions) { const { state, pos } = ctx; const doc = state.doc.toString(); - if (inPhrase(ctx) || inUnclosedQuote(doc, pos)) { + if (inUnclosedQuote(doc, pos)) { return null; } @@ -250,14 +237,6 @@ export function makeCompletionSource(opts: FilterOptions) { }; } -const language = LRLanguage.define({ - name: "filter", - parser, - languageData: { - autocompletion: {}, - }, -}); - /** * Field and value suggestions, as the sidebar's filter offers them, for a field of another * language: a plain input of `host:` and `app:` entries, say @@ -265,9 +244,3 @@ const language = LRLanguage.define({ export function filterAutocomplete(options: FilterOptions) { return autocompletion({ override: [makeCompletionSource(options)] }); } - -/** Public extension */ -export function filter(options: FilterOptions) { - const source = makeCompletionSource(options); - return new LanguageSupport(language, [autocompletion({ override: [source] })]); -} diff --git a/packages/ui/src/components/editor/filter/filter.grammar b/packages/ui/src/components/editor/filter/filter.grammar deleted file mode 100644 index dcf8b81ec..000000000 --- a/packages/ui/src/components/editor/filter/filter.grammar +++ /dev/null @@ -1,78 +0,0 @@ -@top Query { Expr } - -@skip { space+ } -@tokens { - space { $[ \t\r\n]+ } - - LParen { "(" } - RParen { ")" } - At { "@" } - Colon { ":" } - Not { "-" | "NOT" } - - // Keywords (case-insensitive) - And { "AND" } - Or { "OR" } - - // "quoted phrase" with simple escapes: \" and \\ - Phrase { '"' (!["\\] | "\\" _)* '"' } - - // Bare words run until filter syntax or whitespace. Leading '-' remains unary - // negation, but '-' may appear after the first character. - Word { ![ \t\r\n():"@-] ![ \t\r\n():"@]* } - FieldValueWord { ![ \t\r\n"] ![ \t\r\n]* } - - @precedence { Not, And, Or, Word } -} - -@detectDelim - -// Precedence: NOT (highest) > AND > OR (lowest) -// We also allow implicit AND in your parser/evaluator, but for highlighting, -// this grammar parses explicit AND/OR/NOT + adjacency as a sequence (Seq). -Expr { - OrExpr -} - -OrExpr { - AndExpr (Or AndExpr)* -} - -AndExpr { - Unary (And Unary | Unary)* // allow implicit AND by adjacency: Unary Unary -} - -Unary { - Not Unary -| Primary -} - -Primary { - Group -| Field -| Phrase -| Term -} - -Group { - LParen Expr RParen -} - -Field { - FieldName Colon FieldValue -} - -FieldName { - At? Word -} - -FieldValue { - Phrase -| FieldValueWord -} - -Term { - Word -} - -@external propSource highlight from "./highlight" diff --git a/packages/ui/src/components/editor/filter/filter.test.ts b/packages/ui/src/components/editor/filter/filter.test.ts deleted file mode 100644 index 6002175f6..000000000 --- a/packages/ui/src/components/editor/filter/filter.test.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { describe, expect, test } from "vite-plus/test"; -import { parser } from "./filter"; - -function getNodeNames(input: string): string[] { - const tree = parser.parse(input); - const nodes: string[] = []; - const cursor = tree.cursor(); - do { - if (cursor.name !== "Query") { - nodes.push(cursor.name); - } - } while (cursor.next()); - return nodes; -} - -describe("filter grammar", () => { - test("parses URL-like field values as one value", () => { - const nodes = getNodeNames("@url:yaak.app/foo-bar"); - - expect(nodes).not.toContain("⚠"); - expect(nodes).toContain("FieldValue"); - expect(nodes).toContain("FieldValueWord"); - }); - - test("parses punctuation-heavy field values as one value", () => { - const nodes = getNodeNames("@url:yaa$&#*@tsrna(*)"); - - expect(nodes).not.toContain("⚠"); - expect(nodes).toContain("FieldValue"); - expect(nodes).toContain("FieldValueWord"); - }); - - test("parses operator-looking field values as one value", () => { - const negativeValueNodes = getNodeNames("@url:-foo"); - const operatorWordNodes = getNodeNames("@url:AND"); - - expect(negativeValueNodes).not.toContain("⚠"); - expect(negativeValueNodes).toContain("FieldValueWord"); - expect(operatorWordNodes).not.toContain("⚠"); - expect(operatorWordNodes).toContain("FieldValueWord"); - }); -}); diff --git a/packages/ui/src/components/editor/filter/filter.ts b/packages/ui/src/components/editor/filter/filter.ts deleted file mode 100644 index 717a6daa4..000000000 --- a/packages/ui/src/components/editor/filter/filter.ts +++ /dev/null @@ -1,22 +0,0 @@ -// This file was generated by lezer-generator. You probably shouldn't edit it. -import {LRParser} from "@lezer/lr" -import {highlight} from "./highlight" -export const parser = LRParser.deserialize({ - version: 14, - states: "%WOVQPOOPhOPOOOVQPO'#CfOmQPO'#ChO!_QPO'#ChO!dQPO'#CgOOQO'#Cc'#CcOVQPO'#CaOOQO'#Ca'#CaO!iQPO'#C`O!yQPO'#C_OOQO'#C^'#C^QOQPOOPOOO'#Cr'#CrP#UOPO)C>lO#]QPO,59QOOQO,59S,59SO#bQQO,59ROOQO,58{,58{OVQPO'#CsOOQO'#Cs'#CsO#jQPO,58zOVQPO'#CtO#zQPO,58yPOOO-E6p-E6pOOQO1G.l1G.lOOQO'#Cl'#ClOOQO1G.m1G.mOOQO,59_,59_OOQO-E6q-E6qOOQO,59`,59`OOQO-E6r-E6r", - stateData: "$]~OkPQ~OUVOXQO]SO^ROaUO~Ok]O~OUcXXcX]cX^cX_[XacXdcXecXicXWcX~O^`O~O_aO~OdcOeSXiSXWSX~PVOefOiRXWRX~Ok]O~Qj]WiO~OajObjO~OdcOeSaiSaWSa~PVOefOiRaWRa~OUde^e~", - goto: "#^iPPjpt{P![PP!e!e!nPPP!wPP!ePP!z#Q#WQ[OR_QTZOQSYOQRnfUXOQfQbVSdXeRlc_WOQVXcef_UOQVXcef_TOQVXcefRkaQ^PRh^QeXRmeQgYRog", - nodeNames: "⚠ Query Expr OrExpr AndExpr Unary Not Primary RParen LParen Group Field FieldName At Word Colon FieldValue Phrase FieldValueWord Term And Or", - maxTerm: 27, - nodeProps: [ - ["openedBy", 8,"LParen"], - ["closedBy", 9,"RParen"] - ], - propSources: [highlight], - skippedNodes: [0,22], - repeatNodeCount: 3, - tokenData: "2h~RiOX!pXY$hYZ$hZ]!p]^$h^p!ppq$hqr!prs$ysx!pxy&gyz'Qz}!p}!O'k!O![!p![!](U!]!b!p!b!c(o!c!d)Y!d!p!p!p!q,q!q!r0Y!r;'S!p;'S;=`$b<%lO!pR!w^bQ^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c;'S!p;'S;=`$b<%lO!pQ#xUbQOX#sZ]#s^p#sq;'S#s;'S;=`$[<%lO#sQ$_P;=`<%l#sR$eP;=`<%l!p~$mSk~XY$hYZ$h]^$hpq$h~$|VOr$yrs%cs#O$y#O#P%h#P;'S$y;'S;=`&a<%lO$y~%hOa~~%kRO;'S$y;'S;=`%t;=`O$y~%wWOr$yrs%cs#O$y#O#P%h#P;'S$y;'S;=`&a;=`<%l$y<%lO$y~&dP;=`<%l$yR&nUbQXPOX#sZ]#s^p#sq;'S#s;'S;=`$[<%lO#sR'XUbQWPOX#sZ]#s^p#sq;'S#s;'S;=`$[<%lO#sR'rUbQUPOX#sZ]#s^p#sq;'S#s;'S;=`$[<%lO#sR(]U_PbQOX#sZ]#s^p#sq;'S#s;'S;=`$[<%lO#sR(vU]PbQOX#sZ]#s^p#sq;'S#s;'S;=`$[<%lO#sR)a`bQ^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c!p!p!p!q*c!q;'S!p;'S;=`$b<%lO!pR*j`bQ^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c!f!p!f!g+l!g;'S!p;'S;=`$b<%lO!pR+u^bQdP^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c;'S!p;'S;=`$b<%lO!pR,x`bQ^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c!q!p!q!r-z!r;'S!p;'S;=`$b<%lO!pR.R`bQ^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c!v!p!v!w/T!w;'S!p;'S;=`$b<%lO!pR/^^bQUP^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c;'S!p;'S;=`$b<%lO!pR0a`bQ^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c!t!p!t!u1c!u;'S!p;'S;=`$b<%lO!pR1l^bQeP^POX!pZ]!p^p!pqr!prs#ssx!pxz#sz![!p![!]#s!]!b!p!b!c#s!c;'S!p;'S;=`$b<%lO!p", - tokenizers: [0, 1], - topRules: {"Query":[0,1]}, - tokenPrec: 145 -}) diff --git a/packages/ui/src/components/editor/filter/highlight.ts b/packages/ui/src/components/editor/filter/highlight.ts deleted file mode 100644 index 9c05d7267..000000000 --- a/packages/ui/src/components/editor/filter/highlight.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { styleTags, tags as t } from "@lezer/highlight"; - -export const highlight = styleTags({ - // Boolean operators - And: t.operatorKeyword, - Or: t.operatorKeyword, - Not: t.operatorKeyword, - - // Structural punctuation - LParen: t.paren, - RParen: t.paren, - Colon: t.punctuation, - Minus: t.operator, - - // Literals - Phrase: t.string, // "quoted string" - - // Fields - "FieldName/At": t.attributeName, - "FieldName/Word": t.attributeName, - "FieldValue/FieldValueWord": t.attributeValue, -}); diff --git a/packages/ui/src/components/editor/query/extension.test.ts b/packages/ui/src/components/editor/query/extension.test.ts new file mode 100644 index 000000000..16a4be1d1 --- /dev/null +++ b/packages/ui/src/components/editor/query/extension.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, test } from "vite-plus/test"; +import { pathsAt } from "./extension"; + +describe("pathsAt", () => { + const at = (text: string) => pathsAt(text, text.length); + + test("is the field whose value is being typed, quoted values with spaces too", () => { + expect(at("@host:api")).toEqual(["host"]); + expect(at('x @app:"Google C')).toEqual(["app"]); + expect(at('@app:"Google Chrome" ')).toEqual([]); + expect(at("@header.content-type:")).toEqual(["header.content-type"]); + }); + + test("is a map's name while its key is typed", () => { + expect(at("@header.x-")).toEqual(["header"]); + expect(at("@res.header.")).toEqual(["res.header"]); + }); + + test("is nothing outside a field", () => { + expect(at("stripe ")).toEqual([]); + expect(at("")).toEqual([]); + }); +}); diff --git a/packages/ui/src/components/editor/query/extension.ts b/packages/ui/src/components/editor/query/extension.ts new file mode 100644 index 000000000..aed6e0f52 --- /dev/null +++ b/packages/ui/src/components/editor/query/extension.ts @@ -0,0 +1,264 @@ +import type { Completion, CompletionContext, CompletionResult } from "@codemirror/autocomplete"; +import { autocompletion, startCompletion } from "@codemirror/autocomplete"; +import { highlightingFor } from "@codemirror/language"; +import type { EditorState, Extension } from "@codemirror/state"; +import { RangeSetBuilder, StateEffect, StateField } from "@codemirror/state"; +import type { DecorationSet, ViewUpdate } from "@codemirror/view"; +import { Decoration, EditorView, ViewPlugin } from "@codemirror/view"; +import type { Tag } from "@lezer/highlight"; +import { tags as t } from "@lezer/highlight"; +import type { Schema, TokenKind } from "@yaakapp-internal/query"; +import { queryCompletions, queryTokens } from "@yaakapp-internal/query"; + +/** A term the search cannot run, to underline: `from` and `to` count UTF-16 units */ +export interface QueryProblem { + from: number; + to: number; + /** Shown on hover, a few words */ + label: string; + /** Searchable later rather than wrong: marked more quietly */ + quiet?: boolean; +} + +export interface QueryFilterOptions { + schema: Schema; + /** The values seen for each field path, offered after `@field:` */ + values?: Record; + /** + * Values for field paths `values` lacks, read as completion opens: `host` for `@host:`, and a + * map's name, `header`, for the keys after `@header.`. Null for a path with none to offer. + */ + loadValues?: (paths: string[]) => Promise>; + /** + * More completions for a bare word or an empty spot, after the engine's, such as saved + * searches. `typed` is the word, and the completions replace it. + */ + moreCompletions?: (typed: string) => Completion[]; + /** The terms of a query the search cannot run, underlined as typing settles */ + problems?: (text: string) => Promise; +} + +const tokenTags: Record = { + field: t.attributeName, + value: t.attributeValue, + operator: t.operatorKeyword, + quote: t.string, + paren: t.paren, + negation: t.operator, + text: null, +}; + +function buildDecorations(state: EditorState): DecorationSet { + const builder = new RangeSetBuilder(); + for (const token of queryTokens(state.doc.toString())) { + const tag = tokenTags[token.kind]; + const className = tag == null ? null : highlightingFor(state, [tag]); + if (className == null || token.span.start === token.span.end) continue; + builder.add(token.span.start, token.span.end, Decoration.mark({ class: className })); + } + return builder.finish(); +} + +const highlight = ViewPlugin.fromClass( + class { + decorations: DecorationSet; + + constructor(view: EditorView) { + this.decorations = buildDecorations(view.state); + } + + update(update: ViewUpdate) { + if (update.docChanged || update.viewportChanged) { + this.decorations = buildDecorations(update.state); + } + } + }, + { decorations: (v) => v.decorations }, +); + +/** + * The field paths whose values the term at `pos` would offer: its field's, a quoted value with + * spaces included, or its map's keys + */ +export function pathsAt(text: string, pos: number): string[] { + const before = text.slice(0, pos); + const value = /@([A-Za-z0-9_.-]+):(?:"(?:[^"\\]|\\.)*|[^\s()"]*)$/.exec(before); + if (value?.[1] != null) return [value[1].toLowerCase()]; + const key = /@([A-Za-z0-9_.-]+)\.[A-Za-z0-9_-]*$/.exec(before); + if (key?.[1] != null) return [key[1].toLowerCase()]; + return []; +} + +/** The bare word being typed at `pos`, or empty at the start of a term */ +function wordAt(text: string, pos: number): { word: string; from: number } | null { + const before = text.slice(0, pos); + const m = /(^|[\s(])([^\s()"@:-][^\s()":]*)?$/.exec(before); + if (m == null) return null; + const word = m[2] ?? ""; + return { word, from: pos - word.length }; +} + +/** The panel beside a completion: what the field is, and a term using it */ +function infoOf(info: string | null, example: string | null): Completion["info"] { + if (example == null) return undefined; + return () => { + const el = document.createElement("div"); + if (info != null) { + const what = document.createElement("div"); + what.textContent = info; + el.append(what); + } + const term = document.createElement("code"); + term.className = "cm-query-example"; + term.textContent = example; + el.append(term); + return el; + }; +} + +function completionSource({ schema, values, loadValues, moreCompletions }: QueryFilterOptions) { + return async (ctx: CompletionContext): Promise => { + const text = ctx.state.doc.toString(); + let known = values ?? {}; + const wanted = pathsAt(text, ctx.pos).filter((p) => known[p] == null); + if (loadValues != null && wanted.length > 0) { + known = { ...known, ...(await loadValues(wanted)) }; + if (ctx.aborted) return null; + } + const result = queryCompletions(text, ctx.pos, schema, known); + + const options: Completion[] = (result?.items ?? []).map((item) => ({ + label: item.label, + // Not a CodeMirror type: it names the kind for `onlyFields` below, and draws no icon + type: item.kind === "field" ? "field" : "value", + detail: item.info ?? undefined, + info: infoOf(item.info, item.example), + apply: (view, _completion, from, to) => { + view.dispatch({ + changes: { from, to, insert: item.insert }, + selection: { anchor: from + item.insert.length }, + }); + if (item.reopen) startCompletion(view); + }, + })); + + let from = result?.span.start ?? ctx.pos; + let to = result?.span.end ?? ctx.pos; + const word = moreCompletions == null ? null : wordAt(text, ctx.pos); + // On an empty spot, only when asked for + if (moreCompletions != null && word != null && (word.word !== "" || ctx.explicit)) { + const more = moreCompletions(word.word); + if (more.length > 0) { + // Both replace the word being typed + if (options.length === 0) [from, to] = [word.from, ctx.pos]; + if (from === word.from) options.push(...more); + } + } + if (options.length === 0) return null; + + // Field names pop up for something typed; on an empty spot, only when asked for + const onlyFields = options.every((o) => o.type === "field"); + if (onlyFields && from === to && !ctx.explicit) return null; + + // The engine has already filtered and ranked the options + return { from, to, options, filter: false }; + }; +} + +const setProblems = StateEffect.define(); + +/** Underlines what the search cannot run, as `problems` last answered for the text */ +const problemField = StateField.define({ + create: () => Decoration.none, + update(decorations, tr) { + let next = decorations.map(tr.changes); + for (const effect of tr.effects) { + if (!effect.is(setProblems)) continue; + const builder = new RangeSetBuilder(); + const length = tr.state.doc.length; + for (const p of [...effect.value].sort((a, b) => a.from - b.from)) { + const from = Math.min(p.from, length); + // An empty value, `@size:`, marks the colon before it + const to = Math.min(Math.max(p.to, p.from + 1), length); + const at = p.to === p.from ? Math.max(0, from - 1) : from; + if (at >= to) continue; + const className = p.quiet ? "cm-query-quiet" : "cm-query-problem"; + builder.add(at, to, Decoration.mark({ class: className, attributes: { title: p.label } })); + } + next = builder.finish(); + } + return next; + }, + provide: (f) => EditorView.decorations.from(f), +}); + +/** How long typing settles before the search's problems are asked for */ +const PROBLEMS_MS = 150; + +function problemChecker(problems: (text: string) => Promise) { + return ViewPlugin.fromClass( + class { + timer: ReturnType | null = null; + + constructor(readonly view: EditorView) { + this.check(); + } + + update(update: ViewUpdate) { + if (!update.docChanged) return; + if (this.timer != null) clearTimeout(this.timer); + this.timer = setTimeout(() => this.check(), PROBLEMS_MS); + } + + check() { + this.timer = null; + const text = this.view.state.doc.toString(); + problems(text) + .then((found) => { + // Answered for text since changed: the next check covers it + if (this.view.state.doc.toString() !== text) return; + this.view.dispatch({ effects: setProblems.of(found) }); + }) + .catch(() => {}); + } + + destroy() { + if (this.timer != null) clearTimeout(this.timer); + } + }, + ); +} + +const problemTheme = EditorView.baseTheme({ + ".cm-query-problem": { + textDecoration: "underline wavy var(--color-danger)", + textUnderlineOffset: "3px", + }, + ".cm-query-quiet": { + textDecoration: "underline dotted var(--color-text-subtlest, currentColor)", + textUnderlineOffset: "3px", + }, +}); + +const completionTheme = EditorView.baseTheme({ + ".cm-query-example": { + display: "block", + marginTop: "0.25rem", + color: "var(--color-text)", + whiteSpace: "pre", + }, +}); + +/** Highlighting and completion for a search box written in Yaak's query language */ +export function queryFilter(options: QueryFilterOptions): Extension { + const extensions: Extension[] = [ + highlight, + // No type icons: the fields and values of a search have no kind the editor's icons name + autocompletion({ override: [completionSource(options)], icons: false }), + completionTheme, + ]; + if (options.problems != null) { + extensions.push(problemField, problemChecker(options.problems), problemTheme); + } + return extensions; +} diff --git a/packages/ui/src/editor.ts b/packages/ui/src/editor.ts index 347d2139b..df2ae1b56 100644 --- a/packages/ui/src/editor.ts +++ b/packages/ui/src/editor.ts @@ -3,10 +3,11 @@ export { CodeEditor } from "./components/editor/CodeEditor"; export type { LanguageSupports } from "./components/editor/extensions"; export type { FieldDef, FilterOptions } from "./components/editor/filter/extension"; export { - filter, filterAutocomplete, makeCompletionSource as filterCompletionSource, } from "./components/editor/filter/extension"; +export type { QueryFilterOptions, QueryProblem } from "./components/editor/query/extension"; +export { queryFilter } from "./components/editor/query/extension"; export { closeBracketsExtensions, getLanguageExtension, From c78732ecafc44f5f789af09476322b6292ece756 Mon Sep 17 00:00:00 2001 From: Gregory Schier Date: Mon, 9 Nov 2026 10:45:46 -0700 Subject: [PATCH 12/45] Polish Yaak Proxy's flow list, sidebar and header Columns keep fixed widths, Path included, and the list scrolls sideways with the header in step instead of dropping columns or capping Host. Resize handles sit on every column boundary, a double-click resets one, and a third click on a sorted header goes back to start time, with empty values sorting lowest. The search box takes up to 44rem and wraps over the list while focused, its completions as wide as it. The list/detail handle fills the gutter. Native title tooltips become Yaak's tooltip, shared buttons gain an opt-in for it, and folder and program rows in the sidebar get Reveal in Finder and Copy path. Co-Authored-By: Claude Opus 5.5 --- .../components/ClearFlowsButton.tsx | 2 + .../components/DecryptListEditor.tsx | 55 ++-- apps/yaak-proxy/components/FirstRun.tsx | 21 +- apps/yaak-proxy/components/FlowCard.tsx | 30 +- apps/yaak-proxy/components/FlowDetail.tsx | 6 +- apps/yaak-proxy/components/FlowList.tsx | 233 +++++++++++----- apps/yaak-proxy/components/FlowListEmpty.tsx | 2 +- apps/yaak-proxy/components/FlowListHeader.tsx | 114 ++++---- .../components/FlowRequestBody.test.tsx | 2 + .../yaak-proxy/components/FlowRequestBody.tsx | 16 +- apps/yaak-proxy/components/FlowTimeline.tsx | 23 +- apps/yaak-proxy/components/NotDecrypted.tsx | 26 +- apps/yaak-proxy/components/ProxyLayout.tsx | 9 +- apps/yaak-proxy/components/SaveDialog.tsx | 15 +- apps/yaak-proxy/components/SearchBar.tsx | 264 +++++++++++++----- apps/yaak-proxy/components/Sidebar.tsx | 53 +++- apps/yaak-proxy/harness/backend.ts | 73 ++++- apps/yaak-proxy/lib/columns.test.ts | 87 ++++-- apps/yaak-proxy/lib/columns.ts | 100 +++---- apps/yaak-proxy/lib/opener.test.ts | 8 +- apps/yaak-proxy/lib/opener.ts | 7 + apps/yaak-proxy/lib/sidebarTree.test.ts | 13 + apps/yaak-proxy/lib/sidebarTree.ts | 19 +- packages/ui/src/components/Button.tsx | 21 +- packages/ui/src/components/ResizeHandle.tsx | 20 +- packages/ui/src/components/SplitLayout.tsx | 37 ++- packages/ui/src/components/Tooltip.tsx | 170 +++++++---- packages/ui/src/components/resizeDrag.test.ts | 18 ++ packages/ui/src/components/resizeDrag.ts | 7 +- packages/ui/src/index.ts | 2 +- 30 files changed, 1032 insertions(+), 421 deletions(-) diff --git a/apps/yaak-proxy/components/ClearFlowsButton.tsx b/apps/yaak-proxy/components/ClearFlowsButton.tsx index 08cc5cd2a..3e0058f89 100644 --- a/apps/yaak-proxy/components/ClearFlowsButton.tsx +++ b/apps/yaak-proxy/components/ClearFlowsButton.tsx @@ -54,6 +54,7 @@ export function ClearFlowsButton() { className="pointer-events-auto" icon="trash" title={locked != null ? "Clear flows (traffic is locked)" : "Clear flows (none yet)"} + tooltip size="sm" iconColor="secondary" disabled @@ -67,6 +68,7 @@ export function ClearFlowsButton() { className="pointer-events-auto" icon="trash" title="Clear flows" + tooltip size="sm" iconColor="secondary" /> diff --git a/apps/yaak-proxy/components/DecryptListEditor.tsx b/apps/yaak-proxy/components/DecryptListEditor.tsx index fa788ee95..779b4d323 100644 --- a/apps/yaak-proxy/components/DecryptListEditor.tsx +++ b/apps/yaak-proxy/components/DecryptListEditor.tsx @@ -1,7 +1,7 @@ import { Prec } from "@codemirror/state"; import { type EditorView, keymap } from "@codemirror/view"; import type { DecryptListEntry, SeenTargets } from "@yaakapp-internal/proxy-lib"; -import { Icon, IconButton, Input } from "@yaakapp-internal/ui"; +import { CONTROL_TOOLTIP_DELAY, Icon, IconButton, Input, Tooltip } from "@yaakapp-internal/ui"; import { filterAutocomplete } from "@yaakapp-internal/ui/editor"; import classNames from "classnames"; import { useMemo, useRef, useState } from "react"; @@ -71,28 +71,29 @@ function EntryRow({ entry.builtIn && "text-text-subtlest", )} > - + + + {entryLabel(entry.entry)} {entry.builtIn && ( - - built-in - + + + built-in + + )} - setError(null)} - /> - {error != null &&
{error}
} -
+ +
+ setError(null)} + /> + {error != null &&
{error}
} +
+
); } diff --git a/apps/yaak-proxy/components/FirstRun.tsx b/apps/yaak-proxy/components/FirstRun.tsx index a4b8faec4..f6efbc927 100644 --- a/apps/yaak-proxy/components/FirstRun.tsx +++ b/apps/yaak-proxy/components/FirstRun.tsx @@ -1,4 +1,11 @@ -import { Button, Icon, InlineCode, LoadingIcon } from "@yaakapp-internal/ui"; +import { + Button, + CONTROL_TOOLTIP_DELAY, + Icon, + InlineCode, + LoadingIcon, + Tooltip, +} from "@yaakapp-internal/ui"; import classNames from "classnames"; import { useAtomValue } from "jotai"; import type { ReactNode } from "react"; @@ -127,9 +134,17 @@ export function FirstRun() { title="What to decrypt" description={ everything ? ( - Everything + + Everything + ) : ( - Pick as they appear + + Pick as they appear + ) } action={ diff --git a/apps/yaak-proxy/components/FlowCard.tsx b/apps/yaak-proxy/components/FlowCard.tsx index 977b04326..6fd906c9c 100644 --- a/apps/yaak-proxy/components/FlowCard.tsx +++ b/apps/yaak-proxy/components/FlowCard.tsx @@ -1,19 +1,21 @@ import { formatSize } from "@yaakapp-internal/lib/formatSize"; import type { Flow, FlowBody } from "@yaakapp-internal/proxy-lib"; import { + CONTROL_TOOLTIP_DELAY, CountBadge, Dropdown, EmptyStateText, type ExtraPaneTab, + type HttpBodyResult, HttpRequestPaneView, HttpResponsePaneView, - type HttpBodyResult, Icon, IconButton, parseCookieHeaders, ResponseCookies, type ResponseViewMode, SplitLayout, + Tooltip, } from "@yaakapp-internal/ui"; import classNames from "classnames"; import { useSetAtom } from "jotai"; @@ -181,17 +183,18 @@ function FlowRequestHalf({ flow }: { flow: Flow }) { /** The method and URL, at the status line's height so the two header lines align */ function UrlLine({ flow }: { flow: Flow }) { return ( -
- {flow.method} - - {flow.url} - -
+ // The whole URL on hover, when it runs past the card + +
+ {flow.method} + {flow.url} +
+
); } @@ -295,10 +298,11 @@ function FlowResponseHalf({ flow, className }: { flow: Flow; className: string } icon="bookmark" iconColor={saved ? "primary" : "secondary"} title={saved ? "Saved to Yaak" : "Save to Yaak"} + tooltip onClick={() => openSave(flow)} /> - +
} diff --git a/apps/yaak-proxy/components/FlowDetail.tsx b/apps/yaak-proxy/components/FlowDetail.tsx index 5c31ac522..60fe3199e 100644 --- a/apps/yaak-proxy/components/FlowDetail.tsx +++ b/apps/yaak-proxy/components/FlowDetail.tsx @@ -7,8 +7,8 @@ import { FlowCard } from "./FlowCard"; import { NotDecrypted } from "./NotDecrypted"; /** - * The detail pane: the selected flow's card, or what the pane holds before one is selected. Side - * by side, a wider gutter separates the card from the list and it sits a little under the header. + * The detail pane: the selected flow's card, or what the pane holds before one is selected. The + * split's gutter separates it from the list; side by side it sits a little under the header. */ export function FlowDetail({ style, sideBySide }: { style?: CSSProperties; sideBySide: boolean }) { const flow = useAtomValue(selectedFlowAtom); @@ -16,7 +16,7 @@ export function FlowDetail({ style, sideBySide }: { style?: CSSProperties; sideB return (
(null); - const { width } = useContainerSize(containerRef); const scrollRef = useRef(null); + const headerScrollRef = useRef(null); + const lastScrollTop = useRef(0); + // The list's vertical scrollbar, which the header leaves room for so the two scroll as far + const [scrollbarWidth, setScrollbarWidth] = useState(0); const ignoreScroll = useRef(false); const [following, setFollowing] = useAtom(followAtom); const [heldAt, setHeldAt] = useState(null); @@ -101,10 +106,12 @@ export function FlowList({ style, className }: Props) { // Where the platform cannot tell which app sent a flow, there is no App column. const hasApps = useAtomValue(hasAppsAtom); const columns = useMemo( - () => shownColumns(prefs, width).filter((c) => hasApps || c.id !== "app"), - [prefs, width, hasApps], + () => shownColumns(prefs).filter((c) => hasApps || c.id !== "app"), + [prefs, hasApps], ); const template = useMemo(() => gridTemplate(columns), [columns]); + // A row's grid, its padding and the list's margins: `px-1.5` and `left-1.5 right-1.5` + const contentWidth = useMemo(() => `calc(${gridWidth(columns)} + 1.5rem)`, [columns]); const arrivals = useArrivals(flows, following && sort == null); const virtualizer = useVirtualizer({ @@ -173,13 +180,29 @@ export function FlowList({ style, className }: Props) { if (latest.current.sort == null) void loadOlder(); }, [hold, loadOlder]); + useEffect(() => { + const el = scrollRef.current; + if (el == null) return; + const measure = () => setScrollbarWidth(el.offsetWidth - el.clientWidth); + measure(); + const observer = new ResizeObserver(measure); + observer.observe(el); + return () => observer.disconnect(); + }, []); + const handleScroll = useCallback(() => { + const el = scrollRef.current; + if (el == null) return; + if (headerScrollRef.current != null) headerScrollRef.current.scrollLeft = el.scrollLeft; + // Sideways only: following and holding go by the vertical position. + const top = el.scrollTop; + const moved = top !== lastScrollTop.current; + lastScrollTop.current = top; if (ignoreScroll.current) { ignoreScroll.current = false; return; } - const el = scrollRef.current; - if (el == null) return; + if (!moved) return; const atEnd = el.scrollHeight - el.scrollTop - el.clientHeight < 2; if (atEnd && latest.current.sort == null) { setFollowing(true); @@ -192,6 +215,13 @@ export function FlowList({ style, className }: Props) { } }, [hold, loadOlder, setFollowing, ROW_HEIGHT]); + // The header scrolls the list sideways, as it has no scrollbar of its own. + const handleHeaderWheel = useCallback((e: WheelEvent) => { + const el = scrollRef.current; + if (el == null || e.deltaX === 0) return; + el.scrollLeft += e.deltaX; + }, []); + // A list too short to scroll still loads older flows on scrolling up. const handleWheel = useCallback( (e: WheelEvent) => { @@ -247,12 +277,18 @@ export function FlowList({ style, className }: Props) { [select], ); + // A third click on a column goes back to start time, following as Sort by Start Time does. const handleSort = useCallback( (column: ColumnId) => { + const next = nextSort(latest.current.sort, column); + if (next == null) { + follow(); + return; + } hold(); - setSort((prev) => nextSort(prev, column)); + setSort(next); }, - [hold, setSort], + [follow, hold, setSort], ); const newCount = following || sort != null ? 0 : countNewFlows(flows, heldAt); @@ -261,18 +297,29 @@ export function FlowList({ style, className }: Props) { return (
- + {/* oxlint-disable-next-line jsx-a11y/no-static-element-interactions -- scrolls the list sideways */} +
+ +
{flows.length === 0 ? ( - +
+ +
) : ( -
+
{virtualizer.getVirtualItems().map((item) => { const flow = flows[item.index]; if (flow == null) return null; @@ -311,7 +363,7 @@ export function FlowList({ style, className }: Props) {
{divider.count === 1 ? "1 new" : `${divider.count} new`} @@ -436,8 +488,8 @@ const FlowRow = memo(function FlowRow({ : {}), }} className={classNames( - "absolute top-0 inset-x-0 grid items-center gap-x-3 px-1.5 rounded-md whitespace-nowrap", - "[&>*]:overflow-hidden [&>*]:text-ellipsis [&>*]:min-w-0 tabular-nums", + "absolute top-0 left-1.5 right-1.5 grid items-center px-1.5 rounded-md whitespace-nowrap", + "tabular-nums", flow.cancelled && "text-text-subtlest", selected ? "text-text bg-surface-highlight group-focus-within/flows:bg-surface-active" @@ -469,9 +521,8 @@ function FlowMark({ flow }: { flow: Flow }) { if (open) states.push("Streaming"); if (flow.responseTruncated) states.push("Body cut at the capture limit"); if (flow.savedRequestId != null) states.push("Saved to Yaak"); - const title = states.length === 0 ? undefined : states.join("\n"); - return ( - + const mark = ( + {flow.error != null ? ( ) : flow.cancelled ? ( @@ -487,10 +538,28 @@ function FlowMark({ flow }: { flow: Flow }) { ) : null} ); + if (states.length === 0) return mark; + return ( + + {mark} + + ); } const mono = "font-mono text-xs"; +/** Every cell: its padding matching the header's, and its text cut short at its width */ +const cell = "min-w-0 px-1.5 overflow-hidden text-ellipsis"; + +/** A cell with its full text in a tooltip, shown when the cell cuts it short */ +function Truncated({ text, children }: { text: string; children: ReactElement }) { + return ( + + {children} + + ); +} + /** An undecrypted tunnel still open: written as it opens, done once it closes */ function isOpenTunnel(flow: Flow): boolean { return flow.tunnelled && flow.elapsedMs == null && flow.error == null; @@ -507,18 +576,25 @@ function FlowCell({ column, flow, span }: { column: ColumnId; flow: Flow; span: switch (column) { case "number": return ( - + {flow.number || ""} ); case "app": { - if (flow.clientPath == null) return ; - const folder = folderOf(flow); + if (flow.clientPath == null) return ; const name = flow.clientName ?? flow.clientPath; + // A program outside an app says which folder it ran in, in full. + const folder = folderOf(flow) == null ? null : flow.clientCwd; return ( - - - + + + + + ); } case "method": @@ -526,63 +602,74 @@ function FlowCell({ column, flow, span }: { column: ColumnId; flow: Flow; span: ); - case "host": + case "host": { + const host = hostOf(flow.url); // Cut from the front, so the part that names the service stays. return ( - - {hostOf(flow.url)} - + + + {host} + + ); + } case "path": { // A tunnel's only path is its host; what it carried is unread. - if (flow.tunnelled) return Not decrypted; + if (flow.tunnelled) { + return Not decrypted; + } const { path, query } = splitPath(flow.url); if (flow.streamKind == null) { return ( - - {path} - {query} - + + + {path} + {query} + + ); } // A stream's message count rides the end of the path, so it survives the narrow width, // ticking while the stream is open. return ( - - - {path} - {query} + + + + {path} + {query} + + - - + ); } case "status": return (flow.cancelled || flow.tunnelled) && flow.status == null ? ( - -- + -- ) : ( - + ); case "type": return failed && noResponse ? ( - - {flow.error} - + + + {flow.error} + + ) : flow.cancelled && noResponse ? ( - Cancelled + Cancelled ) : ( - + {shortContentType(flow.contentType) ?? "--"} ); @@ -590,6 +677,7 @@ function FlowCell({ column, flow, span }: { column: ColumnId; flow: Flow; span: return ( + {Number.isFinite(open) ? formatMillis(Math.max(0, open)) : "--"} ); @@ -611,6 +699,7 @@ function FlowCell({ column, flow, span }: { column: ColumnId; flow: Flow; span: return ( {flow.remoteAddr ?? "--"} + + {flow.remoteAddr ?? "--"} + ); case "version": return ( - + {flow.upstreamHttpVersion ?? flow.clientHttpVersion} ); diff --git a/apps/yaak-proxy/components/FlowListEmpty.tsx b/apps/yaak-proxy/components/FlowListEmpty.tsx index 6c51e596a..8003f045c 100644 --- a/apps/yaak-proxy/components/FlowListEmpty.tsx +++ b/apps/yaak-proxy/components/FlowListEmpty.tsx @@ -49,7 +49,7 @@ export function PausedRow({ style }: { style?: React.CSSProperties }) {
diff --git a/apps/yaak-proxy/components/FlowListHeader.tsx b/apps/yaak-proxy/components/FlowListHeader.tsx index 45971fe0c..61bcb8847 100644 --- a/apps/yaak-proxy/components/FlowListHeader.tsx +++ b/apps/yaak-proxy/components/FlowListHeader.tsx @@ -15,6 +15,7 @@ import { type ColumnId, type ColumnPrefs, type FlowSort, + resetWidth, resetWidths, resizeColumn, toggleColumn, @@ -25,6 +26,8 @@ import { columnPrefsAtom } from "../lib/store"; interface Props { columns: Column[]; template: string; + /** The rows' width, as CSS, so the header scrolls sideways with them */ + width: string; prefs: ColumnPrefs; sort: FlowSort | null; onSort: (column: ColumnId) => void; @@ -32,13 +35,15 @@ interface Props { } /** - * The list's only control surface: a click on a label sorts by it and a second reverses it, - * dragging the edge after a label resizes its column, and the picker sits in the empty corner on - * hover, or anywhere on a right-click. + * The list's only control surface: a click on a label sorts by it, a second reverses it and a + * third goes back to start time. Dragging the edge after a label resizes its column and a + * double-click on it resets it. The picker sits in the empty corner on hover, or anywhere on a + * right-click. */ export const FlowListHeader = memo(function FlowListHeader({ columns, template, + width, prefs, sort, onSort, @@ -82,67 +87,68 @@ export const FlowListHeader = memo(function FlowListHeader({ }, []); return ( - // oxlint-disable-next-line jsx-a11y/no-static-element-interactions -- a right-click opens the same menu as the picker -
- - - - - - {columns.map((c) => ( - - - {c.id !== "path" && ( + // The same margins and padding as the rows, so each label sits over its cells. +
+ {/* oxlint-disable-next-line jsx-a11y/no-static-element-interactions -- a right-click opens the same menu as the picker */} +
+ + + + + + {columns.map((c) => ( + + + {/* On the column's right edge, the boundary with the next, resizing this one */} { startWidth.current = c.current; }} onResizeMove={(e) => setPrefs((p) => resizeColumn(p, c.id, startWidth.current + e.x - e.xStart)) } - onReset={() => - setPrefs((p) => ({ ...p, widths: { ...p.widths, [c.id]: undefined } })) - } + onReset={() => setPrefs((p) => resetWidth(p, c.id))} /> - )} - - ))} - setMenu(null)} /> + + ))} + setMenu(null)} /> +
); }); diff --git a/apps/yaak-proxy/components/FlowRequestBody.test.tsx b/apps/yaak-proxy/components/FlowRequestBody.test.tsx index a4656d337..b6d4f3a5d 100644 --- a/apps/yaak-proxy/components/FlowRequestBody.test.tsx +++ b/apps/yaak-proxy/components/FlowRequestBody.test.tsx @@ -11,6 +11,8 @@ vi.mock("@yaakapp-internal/ui", () => ({ ), EmptyStateText: ({ children }: { children: ReactNode }) =>
{children}
, LoadingIcon: () => LOADING, + CONTROL_TOOLTIP_DELAY: 0, + Tooltip: ({ children }: { children: ReactNode }) => <>{children}, })); const data = (truncated = false): FlowBodyData => ({ diff --git a/apps/yaak-proxy/components/FlowRequestBody.tsx b/apps/yaak-proxy/components/FlowRequestBody.tsx index af41d3bc8..162a0adcd 100644 --- a/apps/yaak-proxy/components/FlowRequestBody.tsx +++ b/apps/yaak-proxy/components/FlowRequestBody.tsx @@ -1,5 +1,11 @@ import { formatSize } from "@yaakapp-internal/lib/formatSize"; -import { Banner, EmptyStateText, LoadingIcon } from "@yaakapp-internal/ui"; +import { + Banner, + CONTROL_TOOLTIP_DELAY, + EmptyStateText, + LoadingIcon, + Tooltip, +} from "@yaakapp-internal/ui"; import type { HttpBodyResult } from "@yaakapp-internal/ui/httpTypes"; import type { ReactNode } from "react"; import type { FlowBodyData } from "../lib/flowViews"; @@ -39,9 +45,11 @@ export function FlowRequestBody({ body, editor, binaryViewer, requestSize }: Pro return (
- - First {formatSize(body.data.storedSize)} of {formatSize(requestSize)} kept - + + + First {formatSize(body.data.storedSize)} of {formatSize(requestSize)} kept + +
{editor}
diff --git a/apps/yaak-proxy/components/FlowTimeline.tsx b/apps/yaak-proxy/components/FlowTimeline.tsx index a0b0927b4..064baef73 100644 --- a/apps/yaak-proxy/components/FlowTimeline.tsx +++ b/apps/yaak-proxy/components/FlowTimeline.tsx @@ -1,10 +1,12 @@ import type { Flow } from "@yaakapp-internal/proxy-lib"; import { + CONTROL_TOOLTIP_DELAY, EventViewer, EventViewerRow, formatMillis, Icon, type IconProps, + Tooltip, } from "@yaakapp-internal/ui"; import { useMemo } from "react"; import { type FlowPhase, flowPhases, type PhaseKind } from "../lib/phases"; @@ -35,12 +37,21 @@ export function FlowTimeline({ flow }: { flow: Flow }) { onClick={onClick} icon={} content={ - - {event.label} - {event.detail != null && ( - {` ${event.detail}`} - )} - + event.detail == null ? ( + {event.label} + ) : ( + + + {event.label} + {` ${event.detail}`} + + + ) } columns="auto minmax(0,1fr) 6rem 6rem" cells={{ diff --git a/apps/yaak-proxy/components/NotDecrypted.tsx b/apps/yaak-proxy/components/NotDecrypted.tsx index 13b28ab3f..b45e4bd8f 100644 --- a/apps/yaak-proxy/components/NotDecrypted.tsx +++ b/apps/yaak-proxy/components/NotDecrypted.tsx @@ -1,6 +1,6 @@ import { formatSize } from "@yaakapp-internal/lib/formatSize"; import type { Flow } from "@yaakapp-internal/proxy-lib"; -import { Button, formatMillis, Icon } from "@yaakapp-internal/ui"; +import { Button, CONTROL_TOOLTIP_DELAY, formatMillis, Icon, Tooltip } from "@yaakapp-internal/ui"; import { useDecryptList, useEditDecryptList } from "../hooks/useDecryptList"; import { type DecryptSubject, detailActions, listEdit, setDecrypted } from "../lib/decrypt"; import { sidebarPath } from "../lib/search"; @@ -46,13 +46,23 @@ export function NotDecrypted({ flow }: { flow: Flow }) {
{facts.join(" · ")}
{flow.error != null &&
{flow.error}
} {actions.listed ? ( -
- Decrypting new connections -
+ +
Decrypting new connections
+
) : actions.blocked != null ? ( -
- Excluded by {actions.blocked} -
+ +
+ Excluded by {actions.blocked} +
+
) : (
{actions.host && ( @@ -60,6 +70,7 @@ export function NotDecrypted({ flow }: { flow: Flow }) { size="xs" variant="border" title={`Decrypt new connections to ${host}`} + tooltip disabled={save.isPending} onClick={() => decrypt(hostSubject)} > @@ -71,6 +82,7 @@ export function NotDecrypted({ flow }: { flow: Flow }) { size="xs" variant="border" title={`Decrypt new connections from ${app}`} + tooltip disabled={save.isPending} onClick={() => decrypt(appSubject)} > diff --git a/apps/yaak-proxy/components/ProxyLayout.tsx b/apps/yaak-proxy/components/ProxyLayout.tsx index 95a4b181b..4f4ffd8be 100644 --- a/apps/yaak-proxy/components/ProxyLayout.tsx +++ b/apps/yaak-proxy/components/ProxyLayout.tsx @@ -63,13 +63,14 @@ export function ProxyLayout() { included, falls through to the header, which drags the window. */}
{ @@ -93,6 +94,7 @@ export function ProxyLayout() { className="pointer-events-auto" icon={flowLayout === "horizontal" ? "columns_2" : "rows_2"} title={`Change to ${flowLayout === "horizontal" ? "vertical" : "horizontal"} layout`} + tooltip size="sm" iconColor="secondary" onClick={() => @@ -103,6 +105,7 @@ export function ProxyLayout() { className="pointer-events-auto" icon="settings" title="Settings" + tooltip size="sm" iconColor="secondary" onClick={() => openSettings()} @@ -139,6 +142,7 @@ export function ProxyLayout() { setFloatingSidebarHidden(true)} @@ -188,7 +192,8 @@ function FlowsOrFirstRun({ flowLayout }: { flowLayout: SplitLayoutLayout }) { return ( } diff --git a/apps/yaak-proxy/components/SaveDialog.tsx b/apps/yaak-proxy/components/SaveDialog.tsx index 653632474..229a6f5a6 100644 --- a/apps/yaak-proxy/components/SaveDialog.tsx +++ b/apps/yaak-proxy/components/SaveDialog.tsx @@ -6,11 +6,13 @@ import type { } from "@yaakapp-internal/proxy-lib"; import { Banner, + CONTROL_TOOLTIP_DELAY, Dialog, DialogFooter, HttpMethodTagRaw, LoadingIcon, PlainInput, + Tooltip, } from "@yaakapp-internal/ui"; import { unnamedRequestName } from "@yaakapp-internal/ui/resolvedModelName"; import { openUrl } from "@tauri-apps/plugin-opener"; @@ -181,13 +183,16 @@ function SaveForm({
{p.droppedHeaders.map((h) => ( - - {h.name} - + + {h.name} + + ))}
diff --git a/apps/yaak-proxy/components/SearchBar.tsx b/apps/yaak-proxy/components/SearchBar.tsx index cdcbf7271..77c45fe94 100644 --- a/apps/yaak-proxy/components/SearchBar.tsx +++ b/apps/yaak-proxy/components/SearchBar.tsx @@ -1,7 +1,7 @@ -import { startCompletion } from "@codemirror/autocomplete"; +import { autocompletion, startCompletion } from "@codemirror/autocomplete"; import type { Extension } from "@codemirror/state"; import { Compartment } from "@codemirror/state"; -import type { EditorView } from "@codemirror/view"; +import { EditorView, tooltips, ViewPlugin, type ViewUpdate } from "@codemirror/view"; import { Dropdown, Icon, @@ -11,8 +11,17 @@ import { LazyCodeEditor, } from "@yaakapp-internal/ui"; import { queryFilter } from "@yaakapp-internal/ui/editor"; +import classNames from "classnames"; import { useAtom, useAtomValue, useSetAtom, useStore } from "jotai"; -import { memo, useEffect, useMemo, useRef } from "react"; +import { + type CSSProperties, + memo, + useEffect, + useLayoutEffect, + useMemo, + useRef, + useState, +} from "react"; import { useRpcQuery } from "../hooks/useRpcQuery"; import { loadSeenValues, savedCompletions, searchProblems } from "../lib/searchBox"; import { @@ -34,7 +43,8 @@ const SEARCH_DEBOUNCE_MS = 200; * the URL, method, status and content type, and `@field:value` terms the rest, highlighted and * completed as Yaak's sidebar filter is, with the values the proxy has seen. The backend runs * it and marks what it cannot. Its placeholder counts the flows. ⌘F focuses it and Escape - * clears it. + * clears it. Focused, it wraps a long query over as many lines as it needs, as Yaak's URL bar + * does, over what is below; blurred, it is one line again. */ export const SearchBar = memo(function SearchBar() { const [text, setText] = useAtom(searchTextAtom); @@ -48,6 +58,24 @@ export const SearchBar = memo(function SearchBar() { const locked = useAtomValue(lockedAtom) != null; const schema = useRpcQuery("query_schema", {}, { staleTime: Infinity }).data?.schema; const handleRef = useRef(null); + const boxRef = useRef(null); + const [focused, setFocused] = useState(false); + // Where the box is while focused, so it grows over the list from the header's place for it + const [place, setPlace] = useState(null); + + useLayoutEffect(() => { + if (!focused) { + setPlace(null); + return; + } + const measure = () => { + const rect = boxRef.current?.getBoundingClientRect(); + if (rect != null) setPlace({ top: rect.top, left: rect.left, width: rect.width }); + }; + measure(); + window.addEventListener("resize", measure); + return () => window.removeEventListener("resize", measure); + }, [focused]); useEffect(() => { const timer = setTimeout(() => setTerms(text), SEARCH_DEBOUNCE_MS); @@ -81,7 +109,7 @@ export const SearchBar = memo(function SearchBar() { }), [schema, store], ); - const extensions = useMemo(() => compartment.current.of(filter), [filter]); + const extensions = useMemo(() => [compartment.current.of(filter), completionAcrossBox], [filter]); useEffect(() => { handleRef.current?.dispatch({ effects: compartment.current.reconfigure(filter) }); }, [filter]); @@ -93,67 +121,177 @@ export const SearchBar = memo(function SearchBar() { }; return ( - ( - - )} - setRef={(h) => { - handleRef.current = h; - }} - label="Search flows" - hideLabel - size="sm" - className="font-mono text-xs" - stateKey={null} - language={null} - wrapLines={false} - placeholder={locked ? "Traffic is locked" : placeholderFor(total)} - disabled={locked} - defaultValue={text} - forceUpdateKey={String(version)} - onChange={setText} - extensions={extensions} - onKeyDown={(e) => { - if (e.key === "Escape" && handleRef.current?.value() !== "") { - e.stopPropagation(); - clear(); - } - }} - leftSlot={ - - } - rightSlot={ - text === "" ? ( - - ⌘F - - ) : ( -
- , - onSelect: () => setSavedDialog({ mode: "save", query: text.trim() }), - }, - ]} - > - - - -
- ) - } - /> + // Holds the header's place for the box, which floats over the list while focused +
+
+ ( + + )} + setRef={(h) => { + handleRef.current = h; + }} + label="Search flows" + hideLabel + size="sm" + className="font-mono text-xs" + containerClassName={classNames(focused && "bg-surface shadow-lg")} + stateKey={null} + language={null} + wrapLines={focused} + onFocus={() => setFocused(true)} + onBlur={() => setFocused(false)} + placeholder={locked ? "Traffic is locked" : placeholderFor(total)} + disabled={locked} + defaultValue={text} + forceUpdateKey={String(version)} + onChange={setText} + extensions={extensions} + onKeyDown={(e) => { + if (e.key === "Escape" && handleRef.current?.value() !== "") { + e.stopPropagation(); + clear(); + } + }} + leftSlot={ + + } + rightSlot={ + text === "" ? ( + + ⌘F + + ) : ( +
+ , + onSelect: () => setSavedDialog({ mode: "save", query: text.trim() }), + }, + ]} + > + + + +
+ ) + } + /> +
+
); }); +/** The search box's width, which its completions take */ +const BOX_WIDTH_VAR = "--proxy-search-width"; + +/** The search box an editor is in */ +function boxOf(view: EditorView): HTMLElement | null { + return view.dom.closest(".x-theme-input"); +} + +/** Pixels between a completion's info and the window's edge */ +const INFO_MARGIN = 8; + +/** + * Where a completion's info goes: beside the list, in the window's room either side of the box, + * or, with none, under the option as CodeMirror puts it in a narrow space + */ +function positionInfo( + view: EditorView, + list: { left: number; right: number; top: number; bottom: number }, + option: { top: number; bottom: number }, + info: { left: number; right: number; top: number; bottom: number }, +): { style: string; class: string } { + const win = view.dom.ownerDocument.defaultView ?? window; + const width = info.right - info.left; + const height = info.bottom - info.top; + const right = win.innerWidth - list.right - INFO_MARGIN; + const left = list.left - INFO_MARGIN; + if (Math.max(left, right) >= width) { + const top = Math.max(0, Math.min(option.top, win.innerHeight - height)) - list.top; + const onRight = right >= width; + return { + style: `top: ${top}px; max-width: ${onRight ? right : left}px`, + class: onRight ? "cm-completionInfo-right" : "cm-completionInfo-left", + }; + } + return { + style: `top: ${option.bottom - list.top}px; max-width: 400px`, + class: "cm-completionInfo-right-narrow", + }; +} + +/** + * The completions under the search box, as wide as it and level with its left edge, rather + * than a narrow list wherever the cursor is + */ +const completionAcrossBox: Extension = [ + autocompletion({ positionInfo }), + tooltips({ + tooltipSpace: (view) => { + const win = view.dom.ownerDocument.defaultView ?? window; + const rect = boxOf(view)?.getBoundingClientRect(); + return { + top: 0, + bottom: win.innerHeight, + left: rect?.left ?? 0, + right: rect?.right ?? win.innerWidth, + }; + }, + }), + ViewPlugin.fromClass( + class { + observer: ResizeObserver | null = null; + + constructor(view: EditorView) { + this.watch(view); + } + + // The editor is put in the page after it is made, so the box may only be found later. + update(update: ViewUpdate) { + if (this.observer == null) this.watch(update.view); + } + + watch(view: EditorView) { + const box = boxOf(view); + if (box == null) return; + const root = view.dom.ownerDocument.documentElement; + this.observer = new ResizeObserver(() => { + root.style.setProperty(BOX_WIDTH_VAR, `${box.getBoundingClientRect().width}px`); + }); + this.observer.observe(box); + } + + destroy() { + this.observer?.disconnect(); + } + }, + ), + EditorView.theme({ + ".cm-tooltip.cm-tooltip-autocomplete": { width: `var(${BOX_WIDTH_VAR})` }, + ".cm-tooltip.cm-tooltip-autocomplete > ul": { minWidth: "0", maxWidth: "none" }, + }), +]; + /** * Ctrl+Space lists what can be typed, fields and saved searches on an empty spot, as it does in * Yaak's editors, whose shared editor leaves the key to each app diff --git a/apps/yaak-proxy/components/Sidebar.tsx b/apps/yaak-proxy/components/Sidebar.tsx index 23f3be64e..6b7eb8043 100644 --- a/apps/yaak-proxy/components/Sidebar.tsx +++ b/apps/yaak-proxy/components/Sidebar.tsx @@ -1,8 +1,10 @@ import { ContextMenu, + CONTROL_TOOLTIP_DELAY, type DropdownItem, Icon, IconButton, + Tooltip, Tree, type TreeProps, } from "@yaakapp-internal/ui"; @@ -14,9 +16,11 @@ import { useDecryptList, useEditDecryptList } from "../hooks/useDecryptList"; import { useRpcMutation } from "../hooks/useRpcMutation"; import { SIDEBAR_TREE_ID } from "../lib/atoms"; import { blockerOf, decryptMenuAction, isDecrypted, listEdit, setDecrypted } from "../lib/decrypt"; +import { revealLabel, revealPath } from "../lib/opener"; import { applyCollapsed, type CollapsedMap, collapsedMap, folderIds } from "../lib/sidebarCollapse"; import { type SidebarItem, SYSTEM_ID } from "../lib/sidebarTree"; import { hideSystemAtom, noteSidebarInput, sidebarTreeAtom } from "../lib/store"; +import { getOsType } from "../lib/tauri"; import { useShowToast } from "../lib/toast"; import { AppIcon } from "./AppIcon"; import { savedSearchDialogAtom } from "./SavedSearchDialog"; @@ -68,7 +72,14 @@ function ItemInner({ item }: { item: SidebarItem }) { {item.kind === "saved" && } {item.kind === "folder" && } {item.appPath != null && } - {item.label} + {item.kind === "folder" && item.path != null ? ( + // Named by its last part; the whole path on hover + + {item.label} + + ) : ( + {item.label} + )} {item.sublabel != null && ( {item.sublabel} )} @@ -89,7 +100,9 @@ function DecryptedMark({ item }: { item: SidebarItem }) { const list = useDecryptList(); if (item.decrypt == null || !isDecrypted(list, item.decrypt)) return null; return ( - + + + ); } @@ -107,6 +120,7 @@ function HideSystemToggle() { icon={hidden ? "eye_closed" : "eye"} iconColor="secondary" title={hidden ? "Show System in All traffic" : "Hide System from All traffic"} + tooltip className={classNames( "!h-4 !w-4 mr-1", !hidden && "opacity-0 [.tree-item:hover_&]:opacity-100 focus-visible:opacity-100", @@ -135,10 +149,39 @@ export function Sidebar() { onError: (err) => showToast({ id: "saved-search", message: err.message, color: "danger" }), }); + // A program's folder, or the program, is shown in Finder, and a folder's path copied. + const fail = useCallback( + (message: string) => showToast({ id: "sidebar-path", message, color: "danger" }), + [showToast], + ); + const pathItems = useCallback( + (item: SidebarItem | undefined): DropdownItem[] => { + const path = item?.path; + if (item == null || path == null) return []; + return [ + { + label: revealLabel(getOsType()), + leftSlot: , + onSelect: () => void revealPath(path, fail), + }, + { + label: "Copy path", + leftSlot: , + hidden: item.kind !== "folder", + onSelect: () => { + navigator.clipboard.writeText(path).catch((err) => fail(String(err))); + }, + }, + ]; + }, + [fail], + ); + // Decrypt or Stop Decrypting the selected hosts, sites and apps, from new connections on. // A saved search is renamed or deleted. const getContextMenu = useCallback( (items: SidebarItem[]): DropdownItem[] => { + const paths = items.length === 1 ? pathItems(items[0]) : []; const saved = items.flatMap((i) => (i.savedId == null ? [] : [{ ...i, id: i.savedId }])); if (saved.length > 0) { const [first] = saved; @@ -164,8 +207,10 @@ export function Sidebar() { } const subjects = items.flatMap((i) => (i.decrypt == null ? [] : [i.decrypt])); const action = decryptMenuAction(decryptList, subjects); - if (action == null) return []; + if (action == null) return paths; return [ + ...paths, + ...(paths.length > 0 ? [{ type: "separator" } as const] : []), { label: action.label, icon: action.on ? "lock_open" : "lock", @@ -187,7 +232,7 @@ export function Sidebar() { }, ]; }, - [decryptList, saveDecryptList, showToast, setSavedDialog, deleteSaved], + [decryptList, saveDecryptList, showToast, setSavedDialog, deleteSaved, pathItems], ); const renderContextMenu = useCallback["renderContextMenu"]>>( ({ items, position, onClose }) => diff --git a/apps/yaak-proxy/harness/backend.ts b/apps/yaak-proxy/harness/backend.ts index 14a7c5f05..701f2091f 100644 --- a/apps/yaak-proxy/harness/backend.ts +++ b/apps/yaak-proxy/harness/backend.ts @@ -1,4 +1,4 @@ -import type { GetProxyStateResponse } from "@yaakapp-internal/proxy-lib"; +import type { Flow, GetProxyStateResponse } from "@yaakapp-internal/proxy-lib"; import { fixtures } from "./fixtures"; /** @@ -42,7 +42,9 @@ function answer(cmd: string, payload: unknown): unknown { const req = (payload ?? {}) as Record; switch (cmd) { case "get_flow_body": { - const fixture = fixtures.find((f) => f.flow.id === req.flowId); + // A copy from `?many` reads its original's body + const id = String(req.flowId).split("~")[0]; + const fixture = fixtures.find((f) => f.flow.id === id); if (fixture == null) throw new Error(`No fixture for ${String(req.flowId)}`); return req.which === "request" ? fixture.requestBody : fixture.responseBody; } @@ -86,12 +88,77 @@ function answer(cmd: string, payload: unknown): unknown { case "list_actions": return { actions: [] }; case "list_flows": { - const flows = fixtures.map((f, i) => ({ ...f.flow, number: 1401 + i })).reverse(); + const many = Number(new URLSearchParams(location.search).get("many") ?? 0); + const flows = (many > 0 ? manyFlows(many) : fixtures.map((f) => f.flow)) + .map((f, i) => ({ ...f, number: 1401 + i })) + .reverse(); return { flows, total: flows.length, matching: null }; } case "list_seen": return { hosts: [], apps: [] }; + // A few of the engine's fields, so the search box highlights and completes + case "query_schema": + return { + schema: { + fields: [ + { + name: "host", + kind: { type: "text" }, + info: "The request's host", + example: "@host:api.example.com", + }, + { + name: "method", + kind: { type: "text" }, + info: "GET, POST, …", + example: "@method:post", + }, + { + name: "status", + kind: { type: "number", units: [] }, + info: "The response's status", + example: "@status:>=400", + }, + { + name: "header", + kind: { type: "text" }, + map: true, + info: "A request or response header", + example: "@header.accept:*json*", + }, + ], + }, + }; + case "query_values": + return []; + case "validate_query": + return []; default: throw new Error(`Not in the harness: ${cmd}`); } } + +/** + * `?many=3000`: the fixtures over and over, a second apart, for checking the list with a lot of + * traffic. Every fourth has no response yet, and a program runs in a few working folders. + */ +function manyFlows(count: number): Flow[] { + const start = Date.parse("2026-11-09T09:00:00Z"); + const folders = ["/Users/you/src/yaak", "/Users/you/.codex/worktrees/1a79/yaak", null]; + return Array.from({ length: count }, (_, i) => { + const base = fixtures[i % fixtures.length]?.flow as Flow; + const pending = i % 4 === 3; + return { + ...base, + id: `${base.id}~${i}`, + startedAt: new Date(start + i * 1000).toISOString().replace("Z", ""), + url: + i % 7 === 0 ? `${base.url}/a/rather/long/path/segment/${i}?page=${i}&q=search` : base.url, + status: pending ? null : base.status, + contentType: pending ? null : base.contentType, + responseSize: pending ? 0 : base.responseSize, + elapsedMs: pending ? null : base.elapsedMs, + clientCwd: base.clientBundleId == null ? (folders[i % folders.length] ?? null) : null, + }; + }); +} diff --git a/apps/yaak-proxy/lib/columns.test.ts b/apps/yaak-proxy/lib/columns.test.ts index f3bd1c272..6703b589d 100644 --- a/apps/yaak-proxy/lib/columns.test.ts +++ b/apps/yaak-proxy/lib/columns.test.ts @@ -4,14 +4,15 @@ import { type ColumnPrefs, defaultColumnPrefs, gridTemplate, + gridWidth, nextSort, parseColumnPrefs, + resetWidth, resetWidths, resizeColumn, shownColumns, sortFlows, toggleColumn, - WIDE_MIN_WIDTH, } from "./columns"; function flow(number: number, extra: Partial = {}): Flow { @@ -33,11 +34,11 @@ function flow(number: number, extra: Partial = {}): Flow { } as Flow; } -const ids = (prefs: ColumnPrefs, width: number) => shownColumns(prefs, width).map((c) => c.id); +const ids = (prefs: ColumnPrefs) => shownColumns(prefs).map((c) => c.id); describe("shownColumns", () => { - test("shows the wide set when there is room, and the narrow set when not", () => { - expect(ids(defaultColumnPrefs, WIDE_MIN_WIDTH)).toEqual([ + test("shows every visible column, however narrow the list", () => { + expect(ids(defaultColumnPrefs)).toEqual([ "number", "app", "method", @@ -48,39 +49,31 @@ describe("shownColumns", () => { "size", "time", ]); - expect(ids(defaultColumnPrefs, WIDE_MIN_WIDTH - 1)).toEqual([ - "app", - "method", - "host", - "path", - "status", - "time", - ]); }); test("follows the picker, and always keeps the path", () => { let prefs = toggleColumn(defaultColumnPrefs, "remote"); prefs = toggleColumn(prefs, "type"); prefs = toggleColumn(prefs, "path"); - expect(ids(prefs, 2000)).toContain("remote"); - expect(ids(prefs, 2000)).not.toContain("type"); - expect(ids(prefs, 2000)).toContain("path"); - // Off by default, and outside the narrow set. - expect(ids(prefs, 400)).not.toContain("remote"); + expect(ids(prefs)).toContain("remote"); + expect(ids(prefs)).not.toContain("type"); + expect(ids(prefs)).toContain("path"); }); - test("lays out fixed widths with the path taking the rest", () => { - const prefs = resizeColumn(defaultColumnPrefs, "host", 240.4); - const template = gridTemplate(shownColumns(prefs, 1200)); - expect(template).toBe("22px 46px 18px 50px 240px minmax(80px,1fr) 46px 50px 72px 64px"); + test("lays out every column at a fixed width, the path included", () => { + let prefs = resizeColumn(defaultColumnPrefs, "host", 240.4); + prefs = resizeColumn(prefs, "path", 500); + const columns = shownColumns(prefs); + const pad = (w: number) => `calc(${w}px + 0.75rem)`; + expect(gridTemplate(columns)).toBe( + ["24px", ...[46, 18, 50, 240, 500, 46, 50, 72, 64].map(pad)].join(" "), + ); + expect(gridWidth(columns)).toBe("calc(1110px + 9 * 0.75rem)"); }); - test("lets the host give way to the path in a narrow list", () => { - const host = (width: number) => - shownColumns(defaultColumnPrefs, width).find((c) => c.id === "host"); - expect(host(2000)?.current).toBe(190); - expect(host(500)?.current).toBe(120); - expect(host(100)?.current).toBe(60); + test("keeps the host's width in a narrow list", () => { + const host = shownColumns(defaultColumnPrefs).find((c) => c.id === "host"); + expect(host?.current).toBe(190); }); }); @@ -96,19 +89,36 @@ describe("column preferences", () => { expect(parseColumnPrefs(null)).toEqual(defaultColumnPrefs); expect(resetWidths(prefs).widths).toEqual({}); }); + + test("a double-click resets one column's width", () => { + let prefs = resizeColumn(defaultColumnPrefs, "host", 300); + prefs = resizeColumn(prefs, "path", 400); + expect(resetWidth(prefs, "path").widths).toEqual({ host: 300 }); + }); }); describe("sorting", () => { - test("sorts numbers largest first and names from A, then reverses", () => { + test("sorts numbers largest first and names from A, reverses, then goes back to start", () => { expect(nextSort(null, "size")).toEqual({ column: "size", descending: true }); expect(nextSort(null, "host")).toEqual({ column: "host", descending: false }); expect(nextSort({ column: "host", descending: false }, "host")).toEqual({ column: "host", descending: true, }); + expect(nextSort({ column: "host", descending: true }, "host")).toBeNull(); + expect(nextSort({ column: "size", descending: true }, "size")).toEqual({ + column: "size", + descending: false, + }); + expect(nextSort({ column: "size", descending: false }, "size")).toBeNull(); + // Another column starts over. + expect(nextSort({ column: "host", descending: true }, "size")).toEqual({ + column: "size", + descending: true, + }); }); - test("sorts the loaded flows, those without a value last, ties in arrival order", () => { + test("sorts the loaded flows, those without a value lowest, ties in arrival order", () => { const a = flow(1, { responseSize: 10, elapsedMs: 5, url: "https://b.com/" }); const b = flow(2, { responseSize: 300, elapsedMs: null, url: "https://a.com/" }); const c = flow(3, { responseSize: 10, elapsedMs: 50, url: "https://c.com/" }); @@ -117,12 +127,29 @@ describe("sorting", () => { const by = (column: Parameters[1], descending: boolean) => sortFlows(flows, { column, descending }).map((f) => f.number); expect(by("size", true)).toEqual([2, 1, 3, 4]); - expect(by("size", false)).toEqual([1, 3, 2, 4]); + expect(by("size", false)).toEqual([4, 1, 3, 2]); expect(by("time", true)).toEqual([3, 1, 2, 4]); + expect(by("time", false)).toEqual([2, 4, 1, 3]); expect(by("host", false)).toEqual([2, 4, 1, 3]); expect(by("number", true)).toEqual([4, 3, 2, 1]); // Failures sort below every status. expect(by("status", true)).toEqual([1, 2, 3, 4]); + expect(by("status", false)).toEqual([4, 1, 2, 3]); expect(sortFlows(flows, null)).toBe(flows); }); + + test("flips flows with no status, type, size or time to the other end", () => { + const done = flow(1, { status: 200, contentType: "text/html", responseSize: 5, elapsedMs: 9 }); + const pending = flow(2, { status: null, contentType: null, responseSize: 0, elapsedMs: null }); + const cancelled = flow(3, { status: null, cancelled: true, contentType: "" }); + const flows = [done, pending, cancelled]; + for (const column of ["status", "type", "size", "time"] as const) { + expect(sortFlows(flows, { column, descending: true }).map((f) => f.number)).toEqual([ + 1, 2, 3, + ]); + expect(sortFlows(flows, { column, descending: false }).map((f) => f.number)).toEqual([ + 2, 3, 1, + ]); + } + }); }); diff --git a/apps/yaak-proxy/lib/columns.ts b/apps/yaak-proxy/lib/columns.ts index 549a84b41..1492b7b07 100644 --- a/apps/yaak-proxy/lib/columns.ts +++ b/apps/yaak-proxy/lib/columns.ts @@ -18,14 +18,12 @@ export interface ColumnDef { id: ColumnId; /** In the header and the picker */ label: string; - /** Pixels, at the default interface size; the path takes what is left */ + /** Pixels of content, at the default interface size; each cell adds `CELL_PADDING` */ width: number; minWidth: number; align: "left" | "right"; /** Shown until the person hides it */ defaultVisible: boolean; - /** Kept when the list is narrow; the rest drop out below `WIDE_MIN_WIDTH` */ - narrow: boolean; } export const COLUMNS: ColumnDef[] = [ @@ -36,7 +34,6 @@ export const COLUMNS: ColumnDef[] = [ minWidth: 30, align: "right", defaultVisible: true, - narrow: false, }, { id: "app", @@ -45,7 +42,6 @@ export const COLUMNS: ColumnDef[] = [ minWidth: 18, align: "left", defaultVisible: true, - narrow: true, }, { id: "method", @@ -54,7 +50,6 @@ export const COLUMNS: ColumnDef[] = [ minWidth: 40, align: "left", defaultVisible: true, - narrow: true, }, { id: "host", @@ -63,16 +58,14 @@ export const COLUMNS: ColumnDef[] = [ minWidth: 60, align: "left", defaultVisible: true, - narrow: true, }, { id: "path", label: "Path", - width: 0, + width: 360, minWidth: 80, align: "left", defaultVisible: true, - narrow: true, }, { id: "status", @@ -81,7 +74,6 @@ export const COLUMNS: ColumnDef[] = [ minWidth: 34, align: "left", defaultVisible: true, - narrow: true, }, { id: "type", @@ -90,7 +82,6 @@ export const COLUMNS: ColumnDef[] = [ minWidth: 34, align: "left", defaultVisible: true, - narrow: false, }, { id: "size", @@ -99,7 +90,6 @@ export const COLUMNS: ColumnDef[] = [ minWidth: 44, align: "right", defaultVisible: true, - narrow: false, }, { id: "time", @@ -108,7 +98,6 @@ export const COLUMNS: ColumnDef[] = [ minWidth: 44, align: "right", defaultVisible: true, - narrow: true, }, { id: "remote", @@ -117,7 +106,6 @@ export const COLUMNS: ColumnDef[] = [ minWidth: 60, align: "left", defaultVisible: false, - narrow: false, }, { id: "version", @@ -126,7 +114,6 @@ export const COLUMNS: ColumnDef[] = [ minWidth: 44, align: "left", defaultVisible: false, - narrow: false, }, ]; @@ -138,11 +125,11 @@ export function columnDef(id: ColumnId): ColumnDef { return def; } -/** Below this width the list keeps only the narrow set: what a glance needs */ -export const WIDE_MIN_WIDTH = 760; - /** The mark column, first and fixed */ -export const MARK_WIDTH = 22; +export const MARK_WIDTH = 24; + +/** Each cell's padding, both sides together, added to its column's width; `px-1.5` per side */ +export const CELL_PADDING = "0.75rem"; /** What the person chose: which columns show, and the widths they dragged, by id */ export interface ColumnPrefs { @@ -153,32 +140,18 @@ export interface ColumnPrefs { export const defaultColumnPrefs: ColumnPrefs = { visible: {}, widths: {} }; export interface Column extends ColumnDef { - /** Its width now, in pixels; zero for the path, which takes the rest */ + /** Its width now, in pixels of content */ current: number; } /** - * The columns shown at `listWidth`, in order: the visible ones, less those outside the narrow set - * when the list is narrower than `WIDE_MIN_WIDTH`. The path always shows. Measured, not by - * layout, so a wide window side by side still gets size and type. + * The columns shown, in order: the visible ones, the path always. Each keeps its width however + * narrow the list is; the list scrolls sideways instead. */ -export function shownColumns(prefs: ColumnPrefs, listWidth: number): Column[] { - const wide = listWidth >= WIDE_MIN_WIDTH; - return COLUMNS.filter((c) => { - if (c.id === "path") return true; - if (!(prefs.visible[c.id] ?? c.defaultVisible)) return false; - return wide || c.narrow; - }).map((c) => ({ ...c, current: currentWidth(prefs, c.id, listWidth) })); -} - -/** The host gives way to the path in a narrow list, at most this share of its width */ -const HOST_SHARE = 0.24; - -function currentWidth(prefs: ColumnPrefs, id: ColumnId, listWidth: number): number { - if (id === "path") return 0; - const width = columnWidth(prefs, id); - if (id !== "host" || listWidth <= 0) return width; - return Math.max(columnDef(id).minWidth, Math.min(width, Math.round(listWidth * HOST_SHARE))); +export function shownColumns(prefs: ColumnPrefs): Column[] { + return COLUMNS.filter((c) => c.id === "path" || (prefs.visible[c.id] ?? c.defaultVisible)).map( + (c) => ({ ...c, current: columnWidth(prefs, c.id) }), + ); } export function columnWidth(prefs: ColumnPrefs, id: ColumnId): number { @@ -186,14 +159,18 @@ export function columnWidth(prefs: ColumnPrefs, id: ColumnId): number { return Math.max(def.minWidth, Math.round(prefs.widths[id] ?? def.width)); } -/** The grid template for the header and every row */ +/** The grid template for the header and every row: the mark, then each column and its padding */ export function gridTemplate(columns: Column[]): string { - const cells = columns.map((c) => - c.id === "path" ? `minmax(${c.minWidth}px,1fr)` : `${c.current}px`, - ); + const cells = columns.map((c) => `calc(${c.current}px + ${CELL_PADDING})`); return [`${MARK_WIDTH}px`, ...cells].join(" "); } +/** The width of a row's grid, as CSS: the list scrolls sideways when it is wider than the list */ +export function gridWidth(columns: Column[]): string { + const px = columns.reduce((sum, c) => sum + c.current, MARK_WIDTH); + return `calc(${px}px + ${columns.length} * ${CELL_PADDING})`; +} + export function toggleColumn(prefs: ColumnPrefs, id: ColumnId): ColumnPrefs { if (id === "path") return prefs; const shown = prefs.visible[id] ?? columnDef(id).defaultVisible; @@ -201,7 +178,6 @@ export function toggleColumn(prefs: ColumnPrefs, id: ColumnId): ColumnPrefs { } export function resizeColumn(prefs: ColumnPrefs, id: ColumnId, width: number): ColumnPrefs { - if (id === "path") return prefs; const w = Math.max(columnDef(id).minWidth, Math.round(width)); return { ...prefs, widths: { ...prefs.widths, [id]: w } }; } @@ -210,6 +186,12 @@ export function resetWidths(prefs: ColumnPrefs): ColumnPrefs { return { ...prefs, widths: {} }; } +/** One column back to its default width */ +export function resetWidth(prefs: ColumnPrefs, id: ColumnId): ColumnPrefs { + const { [id]: _, ...widths } = prefs.widths; + return { ...prefs, widths }; +} + /** * Reads stored preferences, dropping what no longer applies: unknown ids, and values of the * wrong type, so a change to the columns never breaks the list. @@ -238,10 +220,15 @@ export function firstDirection(column: ColumnId): boolean { return column === "number" || column === "size" || column === "time" || column === "status"; } -/** A click on a column's label: sort by it, or reverse the sort it already has */ -export function nextSort(sort: FlowSort | null, column: ColumnId): FlowSort { - if (sort?.column === column) return { column, descending: !sort.descending }; - return { column, descending: firstDirection(column) }; +/** + * A click on a column's label: sort by it, reverse the sort it has, then, from the reversed sort, + * go back to start time + */ +export function nextSort(sort: FlowSort | null, column: ColumnId): FlowSort | null { + const first = firstDirection(column); + if (sort?.column !== column) return { column, descending: first }; + if (sort.descending === first) return { column, descending: !first }; + return null; } const collator = new Intl.Collator(undefined, { numeric: true, sensitivity: "base" }); @@ -276,20 +263,19 @@ function sortKey(flow: Flow, column: ColumnId): Key { } /** - * The flows in `sort`'s order, without a value last either way, then by start so equal values - * keep arrival order. Sorts the flows given, which are the ones loaded, not the whole database. + * The flows in `sort`'s order, those without a value (none, empty or zero) lowest, so first + * ascending and last descending, then by start so equal values keep arrival order. Sorts the + * flows given, which are the ones loaded, not the whole database: the database only ever reads + * by start time, so no `ORDER BY` follows the column sort. */ export function sortFlows(flows: Flow[], sort: FlowSort | null): Flow[] { if (sort == null) return flows; const dir = sort.descending ? -1 : 1; - const keyed = flows.map((flow, i) => ({ flow, i, key: sortKey(flow, sort.column) })); + const keyed = flows.map((flow, i) => ({ flow, i, key: sortKey(flow, sort.column) || null })); keyed.sort((a, b) => { - if (a.key == null || b.key == null) { - if (a.key == null && b.key == null) return a.i - b.i; - return a.key == null ? 1 : -1; - } let c: number; - if (typeof a.key === "number" && typeof b.key === "number") c = a.key - b.key; + if (a.key == null || b.key == null) c = Number(b.key == null) - Number(a.key == null); + else if (typeof a.key === "number" && typeof b.key === "number") c = a.key - b.key; else c = collator.compare(String(a.key), String(b.key)); return c === 0 ? a.i - b.i : c * dir; }); diff --git a/apps/yaak-proxy/lib/opener.test.ts b/apps/yaak-proxy/lib/opener.test.ts index a59946036..e62892a3e 100644 --- a/apps/yaak-proxy/lib/opener.test.ts +++ b/apps/yaak-proxy/lib/opener.test.ts @@ -6,7 +6,7 @@ const opener = vi.hoisted(() => ({ })); vi.mock("@tauri-apps/plugin-opener", () => opener); -const { openLink, revealPath } = await import("./opener"); +const { openLink, revealLabel, revealPath } = await import("./opener"); describe("opener", () => { beforeEach(() => { @@ -39,4 +39,10 @@ describe("opener", () => { await openLink("https://yaak.app", report); expect(report).toHaveBeenCalledWith("Failed to open https://yaak.app: Error: denied"); }); + + test("names the file manager by platform", () => { + expect(revealLabel("macos")).toBe("Reveal in Finder"); + expect(revealLabel("windows")).toBe("Show in Explorer"); + expect(revealLabel("linux")).toBe("Show in Folder"); + }); }); diff --git a/apps/yaak-proxy/lib/opener.ts b/apps/yaak-proxy/lib/opener.ts index 7d653095c..a448b5970 100644 --- a/apps/yaak-proxy/lib/opener.ts +++ b/apps/yaak-proxy/lib/opener.ts @@ -20,3 +20,10 @@ export async function openLink(url: string, report: ReportFailure): Promise { expect(items.get("app:curl|site:example.com")?.kind).toBe("site"); }); + test("folder rows carry their full path, and programs their executable", () => { + expect(items.get("app:claude|cwd:yaak")?.path).toBe("/src/yaak"); + expect(items.get("app:claude|cwd:fieldwork")?.path).toBe("/src/fieldwork"); + expect(items.get("app:claude")?.path).toBe("/usr/local/bin/claude"); + // Hosts and paths under a folder are not on disk. + expect(items.get("app:claude|cwd:yaak|host:statsig.anthropic.com")?.path).toBeUndefined(); + }); + test("a folder row selects that program in that folder", () => { const query = items.get("app:claude|cwd:yaak")?.query ?? ""; expect(query).toBe("@app:claude @cwd:yaak"); @@ -169,6 +177,11 @@ describe("programs outside an app", () => { }); }); +test("an app in a bundle has no path to reveal", () => { + const items = itemsById(buildSidebarTree([flow("https://slack.com/", slack)], 1)); + expect(items.get("app:Slack")?.path).toBeUndefined(); +}); + describe("decrypting from the sidebar", () => { const tunnel = flow("https://bank.example.com", { ...slack, diff --git a/apps/yaak-proxy/lib/sidebarTree.ts b/apps/yaak-proxy/lib/sidebarTree.ts index 76eaa535c..ff16027f2 100644 --- a/apps/yaak-proxy/lib/sidebarTree.ts +++ b/apps/yaak-proxy/lib/sidebarTree.ts @@ -37,6 +37,11 @@ export interface SidebarItem { decrypt?: DecryptSubject; /** The saved search a Saved row is, for renaming and deleting it */ savedId?: string; + /** + * On disk: a folder row's working directory, in full, and a program's executable, for Reveal + * in Finder and, on a folder, Copy path + */ + path?: string; } /** A saved search as the sidebar lists it, with how many flows it covers */ @@ -62,6 +67,8 @@ interface PlaceRow { label: string; query: string; decrypt?: DecryptSubject; + /** A folder row's working directory, in full */ + path?: string; } /** @@ -128,8 +135,15 @@ function placeOf(flow: Flow): Place { } else { const folderId = `app:${app}|cwd:${folder}`; const folderQuery = `${appQuery} ${fieldTerm("cwd", folder)}`; + const folderRow: PlaceRow = { + id: folderId, + kind: "folder", + label: folder, + query: folderQuery, + }; + if (flow.clientCwd != null) folderRow.path = flow.clientCwd; appRows = [ - { id: folderId, kind: "folder", label: folder, query: folderQuery }, + folderRow, ...rows.map((r) => ({ ...r, id: `${folderId}|${r.id}`, @@ -169,6 +183,7 @@ function child(parent: Trie, row: PlaceRow): Trie { children: new Map(), }; if (row.decrypt != null) node.item.decrypt = row.decrypt; + if (row.path != null) node.item.path = row.path; parent.children.set(row.id, node); } node.item.count++; @@ -257,6 +272,8 @@ export function buildSidebarTree( } if (folder != null) item.sublabel = folder; if (flow.clientPath != null) item.appPath = flow.clientPath; + // A program outside an app bundle: its executable + if (flow.clientBundleId == null && flow.clientPath != null) item.path = flow.clientPath; app = { item, children: new Map() }; parent.children.set(id, app); } diff --git a/packages/ui/src/components/Button.tsx b/packages/ui/src/components/Button.tsx index b23a6871c..814e0df46 100644 --- a/packages/ui/src/components/Button.tsx +++ b/packages/ui/src/components/Button.tsx @@ -4,6 +4,10 @@ import type { HTMLAttributes, ReactNode } from "react"; import { forwardRef } from "react"; import { Icon } from "./Icon"; import { LoadingIcon } from "./LoadingIcon"; +import { Tooltip } from "./Tooltip"; + +/** How long the pointer rests on a control before its tooltip shows, in milliseconds */ +export const CONTROL_TOOLTIP_DELAY = 300; type ButtonVariant = "border" | "solid" | "input"; type ButtonSize = "2xs" | "xs" | "sm" | "md" | "auto"; @@ -22,6 +26,8 @@ export type ButtonProps = Omit, "color" | "onC forDropdown?: boolean; disabled?: boolean; title?: string; + /** Shows `title` in Yaak's tooltip, quickly, rather than the browser's */ + tooltip?: boolean; leftSlot?: ReactNode; rightSlot?: ReactNode; }; @@ -43,6 +49,7 @@ export const Button = forwardRef(function Button rightSlot, disabled, title, + tooltip, onClick, ...props }: ButtonProps, @@ -50,8 +57,9 @@ export const Button = forwardRef(function Button ) { const resolvedColor = color ?? tone ?? "default"; const isDisabled = disabled || isLoading; + const withTooltip = tooltip === true && title != null && title !== ""; - return ( + const button = ( ); + + return withTooltip ? ( + + {button} + + ) : ( + button + ); }); diff --git a/packages/ui/src/components/ResizeHandle.tsx b/packages/ui/src/components/ResizeHandle.tsx index 114ea8c60..6e362e527 100644 --- a/packages/ui/src/components/ResizeHandle.tsx +++ b/packages/ui/src/components/ResizeHandle.tsx @@ -19,6 +19,8 @@ interface Props { onReset?: () => void; side: "left" | "right" | "top"; justify: "center" | "end" | "start"; + /** Fills the space it is given, all of it taking the pointer, with its bar centred in it */ + fill?: boolean; } export function ResizeHandle({ @@ -30,6 +32,7 @@ export function ResizeHandle({ onResizeMove, onReset, side, + fill = false, }: Props) { const vertical = side === "top"; const [isResizing, setIsResizing] = useState(false); @@ -89,8 +92,12 @@ export function ResizeHandle({ onPointerDown={handlePointerDown} className={classNames( className, - "group z-10 flex select-none transition-colors hover:bg-surface-active rounded-full", - vertical ? "w-full h-1.5 cursor-row-resize" : "h-full w-1.5 cursor-col-resize", + "group z-10 flex select-none", + vertical ? "cursor-row-resize" : "cursor-col-resize", + fill + ? "w-full h-full items-center" + : "transition-colors hover:bg-surface-active rounded-full", + !fill && (vertical ? "w-full h-1.5" : "h-full w-1.5"), justify === "center" && "justify-center", justify === "end" && "justify-end", justify === "start" && "justify-start", @@ -99,6 +106,15 @@ export function ResizeHandle({ side === "top" && "top-0", )} > + {fill && ( +
+ )} {isResizing && (
( () => ({ ...style, - gridTemplate: splitGridTemplate(vertical, vertical ? renderedHeight : renderedWidth), + gridTemplate: splitGridTemplate(vertical, vertical ? renderedHeight : renderedWidth, gutter), }), - [style, vertical, renderedHeight, renderedWidth], + [style, vertical, renderedHeight, renderedWidth, gutter], ); // While dragging, the split is written straight to the DOM and only saved (and rendered) when // the drag ends, so the panes are not re-rendered on every move. const liveDrag = useRef(null); - const applySplit = useCallback((vertical: boolean, ratio: number) => { - if (containerRef.current == null) return; - containerRef.current.style.gridTemplate = splitGridTemplate(vertical, ratio); - }, []); + const applySplit = useCallback( + (vertical: boolean, ratio: number) => { + if (containerRef.current == null) return; + containerRef.current.style.gridTemplate = splitGridTemplate(vertical, ratio, gutter); + }, + [gutter], + ); // Something else re-rendering mid-drag (a response arriving, say) may write the saved split // back; put the live one back before it is painted. @@ -102,9 +112,16 @@ export function SplitLayout({ const $c = containerRef.current; if ($c == null) return 0; const { paddingLeft, paddingRight, paddingTop, paddingBottom } = getComputedStyle($c); + // The panes share what the gutter leaves return vertical - ? $c.clientHeight - Number.parseFloat(paddingTop) - Number.parseFloat(paddingBottom) - : $c.clientWidth - Number.parseFloat(paddingLeft) - Number.parseFloat(paddingRight); + ? $c.clientHeight - + Number.parseFloat(paddingTop) - + Number.parseFloat(paddingBottom) - + gutter + : $c.clientWidth - + Number.parseFloat(paddingLeft) - + Number.parseFloat(paddingRight) - + gutter; }, apply: (ratio) => applySplit(vertical, ratio), commit: (ratio) => (vertical ? setHeight(ratio) : setWidth(ratio)), @@ -115,6 +132,7 @@ export function SplitLayout({ renderedWidth, minHeightPx, minWidthPx, + gutter, applySplit, setHeight, setWidth, @@ -151,8 +169,9 @@ export function SplitLayout({ style={areaD} className={classNames( resizeHandleClassName, - vertical ? "-translate-y-1" : "-translate-x-1", + gutter === 0 && (vertical ? "-translate-y-1" : "-translate-x-1"), )} + fill={gutter > 0} onResizeStart={handleResizeStart} onResizeMove={handleResizeMove} onResizeEnd={handleResizeEnd} diff --git a/packages/ui/src/components/Tooltip.tsx b/packages/ui/src/components/Tooltip.tsx index d32624cc2..0fc913b46 100644 --- a/packages/ui/src/components/Tooltip.tsx +++ b/packages/ui/src/components/Tooltip.tsx @@ -1,6 +1,6 @@ import classNames from "classnames"; import type { CSSProperties, KeyboardEvent, ReactNode } from "react"; -import { useRef, useState } from "react"; +import { useEffect, useLayoutEffect, useRef, useState } from "react"; import { generateId } from "@yaakapp-internal/lib/generateId"; import { Portal } from "./Portal"; @@ -10,6 +10,15 @@ export interface TooltipProps { tabIndex?: number; size?: "md" | "lg"; className?: string; + /** How long the pointer rests before it shows, in milliseconds */ + delay?: number; + /** + * Wraps a control or a cell without a box of its own: shown on hover alone, hidden on a press, + * and placed against the first element inside, which keeps its own focus and clicks + */ + wrap?: boolean; + /** Shown only while something inside is cut short, as a long host in a narrow column is */ + onlyWhenTruncated?: boolean; } const hiddenStyles: CSSProperties = { @@ -27,27 +36,46 @@ interface TooltipOpenState { position: TooltipPosition; } -export function Tooltip({ children, className, content, tabIndex, size = "md" }: TooltipProps) { +export function Tooltip({ + children, + className, + content, + tabIndex, + size = "md", + delay = 500, + wrap = false, + onlyWhenTruncated = false, +}: TooltipProps) { + // Open, then placed once measured; the tooltip is only in the page while open. + const [open, setOpen] = useState(false); const [openState, setOpenState] = useState(null); - const triggerRef = useRef(null); + const triggerRef = useRef(null); const tooltipRef = useRef(null); const showTimeout = useRef>(undefined); - const handleOpenImmediate = () => { - if (triggerRef.current == null || tooltipRef.current == null) return; - clearTimeout(showTimeout.current); - const triggerRect = triggerRef.current.getBoundingClientRect(); + useEffect(() => () => clearTimeout(showTimeout.current), []); + + useLayoutEffect(() => { + if (!open) { + setOpenState(null); + return; + } + const trigger = anchorOf(triggerRef.current, wrap); + if (trigger == null || tooltipRef.current == null) return; + const triggerRect = trigger.getBoundingClientRect(); const tooltipRect = tooltipRef.current.getBoundingClientRect(); const viewportHeight = document.documentElement.clientHeight; + const viewportWidth = document.documentElement.clientWidth; const margin = 8; const spaceAbove = Math.max(0, triggerRect.top - margin); const spaceBelow = Math.max(0, viewportHeight - triggerRect.bottom - margin); const preferBottom = spaceAbove < tooltipRect.height + margin && spaceBelow > spaceAbove; const position: TooltipPosition = preferBottom ? "bottom" : "top"; + const centred = triggerRect.left + triggerRect.width / 2 - tooltipRect.width / 2; const styles: CSSProperties = { - left: Math.max(0, triggerRect.left + triggerRect.width / 2 - tooltipRect.width / 2), + left: Math.max(0, Math.min(centred, viewportWidth - tooltipRect.width)), maxHeight: position === "top" ? spaceAbove : spaceBelow, ...(position === "top" ? { bottom: viewportHeight - triggerRect.top } @@ -55,25 +83,31 @@ export function Tooltip({ children, className, content, tabIndex, size = "md" }: }; setOpenState({ styles, position }); + }, [open, wrap]); + + const handleOpenImmediate = () => { + clearTimeout(showTimeout.current); + if (onlyWhenTruncated && !isTruncated(anchorOf(triggerRef.current, wrap))) return; + setOpen(true); }; const handleOpen = () => { clearTimeout(showTimeout.current); - showTimeout.current = setTimeout(handleOpenImmediate, 500); + showTimeout.current = setTimeout(handleOpenImmediate, delay); }; const handleClose = () => { clearTimeout(showTimeout.current); - setOpenState(null); + setOpen(false); }; const handleToggleImmediate = () => { - if (openState) handleClose(); + if (open) handleClose(); else handleOpenImmediate(); }; - const handleKeyDown = (e: KeyboardEvent) => { - if (openState && e.key === "Escape") { + const handleKeyDown = (e: KeyboardEvent) => { + if (open && e.key === "Escape") { e.preventDefault(); e.stopPropagation(); handleClose(); @@ -84,52 +118,88 @@ export function Tooltip({ children, className, content, tabIndex, size = "md" }: return ( <> - - - - {/* oxlint-disable-next-line jsx-a11y/prefer-tag-over-role -- Needs to be usable in other buttons */} - - {children} - + + )} + {wrap ? ( + // oxlint-disable-next-line jsx-a11y/no-static-element-interactions -- hover only; what it wraps keeps its own focus and clicks + + {children} + + ) : ( + // oxlint-disable-next-line jsx-a11y/prefer-tag-over-role -- Needs to be usable in other buttons + + {children} + + )} ); } +/** What the tooltip is placed against: the trigger, or a wrapping one's first element */ +function anchorOf(trigger: HTMLElement | null, wrap: boolean): Element | null { + if (trigger == null) return null; + return wrap ? (trigger.firstElementChild ?? trigger) : trigger; +} + +/** Whether `el`, or anything in it, is cut short */ +function isTruncated(el: Element | null): boolean { + if (el == null) return false; + if (el.scrollWidth > el.clientWidth + 1) return true; + for (const child of el.querySelectorAll("*")) { + if (child.scrollWidth > child.clientWidth + 1) return true; + } + return false; +} + function Triangle({ className, position }: { className?: string; position: "top" | "bottom" }) { const isBottom = position === "bottom"; diff --git a/packages/ui/src/components/resizeDrag.test.ts b/packages/ui/src/components/resizeDrag.test.ts index e5f3ca4ea..923a752b8 100644 --- a/packages/ui/src/components/resizeDrag.test.ts +++ b/packages/ui/src/components/resizeDrag.test.ts @@ -6,6 +6,7 @@ import { createHandleDrag, createSidebarDrag, createSplitDrag, + splitGridTemplate, } from "./resizeDrag"; let frames: Map; @@ -225,6 +226,23 @@ describe("clampSplitRatio", () => { }); }); +describe("splitGridTemplate", () => { + const tracks = (t: string) => t.replace(/\s+/g, " ").trim(); + + test("puts the gutter between the panes, the handle's track", () => { + expect(tracks(splitGridTemplate(false, 0.4, 20))).toBe( + "' left drag right' minmax(0,1fr) / 0.6fr 20px 0.4fr", + ); + expect(tracks(splitGridTemplate(true, 0.5, 8))).toBe( + "' left' minmax(0,0.5fr) ' drag' 8px ' right' minmax(0,0.5fr) / 1fr", + ); + }); + + test("lets the panes meet without one", () => { + expect(tracks(splitGridTemplate(false, 0.5))).toContain("0.5fr 0px 0.5fr"); + }); +}); + describe("createSidebarDrag", () => { function sidebar() { const apply = vi.fn(); diff --git a/packages/ui/src/components/resizeDrag.ts b/packages/ui/src/components/resizeDrag.ts index f248b8072..fc0f2b75f 100644 --- a/packages/ui/src/components/resizeDrag.ts +++ b/packages/ui/src/components/resizeDrag.ts @@ -93,17 +93,18 @@ export function createHandleDrag({ }; } -export function splitGridTemplate(vertical: boolean, ratio: number): string { +/** The split's grid: the two panes by `ratio`, and between them `gutter` pixels for the handle */ +export function splitGridTemplate(vertical: boolean, ratio: number, gutter = 0): string { return vertical ? ` ' left' minmax(0,${1 - ratio}fr) - ' drag' 0 + ' drag' ${gutter}px ' right' minmax(0,${ratio}fr) / 1fr ` : ` ' left drag right' minmax(0,1fr) - / ${1 - ratio}fr 0 ${ratio}fr + / ${1 - ratio}fr ${gutter}px ${ratio}fr `; } diff --git a/packages/ui/src/index.ts b/packages/ui/src/index.ts index 3e5c69a72..a004464d3 100644 --- a/packages/ui/src/index.ts +++ b/packages/ui/src/index.ts @@ -1,7 +1,7 @@ export type { BannerProps } from "./components/Banner"; export { Banner } from "./components/Banner"; export type { ButtonProps } from "./components/Button"; -export { Button } from "./components/Button"; +export { Button, CONTROL_TOOLTIP_DELAY } from "./components/Button"; export type { CheckboxProps } from "./components/Checkbox"; export { Checkbox } from "./components/Checkbox"; export type { CopyIconButtonProps } from "./components/CopyIconButton"; From 13ef9c60f18f1bbb7f748126c14f3fb48e596b93 Mon Sep 17 00:00:00 2001 From: Gregory Schier Date: Mon, 9 Nov 2026 10:54:09 -0700 Subject: [PATCH 13/45] Fix review findings in the flow list polish Search actions keep the editor focused so a wrapped box does not collapse under the click. Header scrolls from focus move the rows too. Timeline details always show their tooltip. A sidebar row shared by same-named folders or programs in different places offers no path rather than an arbitrary one. Co-Authored-By: Claude Opus 5.5 --- apps/yaak-proxy/components/FlowList.tsx | 9 +++++++++ apps/yaak-proxy/components/FlowTimeline.tsx | 1 - apps/yaak-proxy/components/SearchBar.tsx | 8 +++++++- apps/yaak-proxy/lib/sidebarTree.test.ts | 18 ++++++++++++++++++ apps/yaak-proxy/lib/sidebarTree.ts | 15 +++++++++++++-- 5 files changed, 47 insertions(+), 4 deletions(-) diff --git a/apps/yaak-proxy/components/FlowList.tsx b/apps/yaak-proxy/components/FlowList.tsx index 383fe64aa..b4f42b7bb 100644 --- a/apps/yaak-proxy/components/FlowList.tsx +++ b/apps/yaak-proxy/components/FlowList.tsx @@ -215,6 +215,14 @@ export function FlowList({ style, className }: Props) { } }, [hold, loadOlder, setFollowing, ROW_HEIGHT]); + // Focus moving to a sort button out of view scrolls the header itself; the rows follow. + const handleHeaderScroll = useCallback(() => { + const header = headerScrollRef.current; + const el = scrollRef.current; + if (header == null || el == null || header.scrollLeft === el.scrollLeft) return; + el.scrollLeft = header.scrollLeft; + }, []); + // The header scrolls the list sideways, as it has no scrollbar of its own. const handleHeaderWheel = useCallback((e: WheelEvent) => { const el = scrollRef.current; @@ -307,6 +315,7 @@ export function FlowList({ style, className }: Props) {
diff --git a/apps/yaak-proxy/components/FlowTimeline.tsx b/apps/yaak-proxy/components/FlowTimeline.tsx index 064baef73..51e93bb56 100644 --- a/apps/yaak-proxy/components/FlowTimeline.tsx +++ b/apps/yaak-proxy/components/FlowTimeline.tsx @@ -42,7 +42,6 @@ export function FlowTimeline({ flow }: { flow: Flow }) { ) : ( diff --git a/apps/yaak-proxy/components/SearchBar.tsx b/apps/yaak-proxy/components/SearchBar.tsx index 77c45fe94..ff7dba372 100644 --- a/apps/yaak-proxy/components/SearchBar.tsx +++ b/apps/yaak-proxy/components/SearchBar.tsx @@ -165,7 +165,13 @@ export const SearchBar = memo(function SearchBar() { ⌘F ) : ( -
+ // oxlint-disable-next-line jsx-a11y/no-static-element-interactions -- keeps focus in the editor +
e.preventDefault()} + className="flex items-center self-center ml-1 mr-0.5" + > { }); }); +test("a row shared by folders or programs in different places names no path", () => { + const tool = { clientName: "claude", clientPath: "/usr/local/bin/claude" }; + const items = itemsById( + buildSidebarTree( + [ + flow("https://a.com/", { ...tool, clientCwd: "/src/yaak" }), + flow("https://b.com/", { ...tool, clientCwd: "/worktrees/123/yaak" }), + flow("https://c.com/", { ...tool, clientPath: "/opt/bin/claude", clientCwd: "/src/ml" }), + ], + 3, + ), + ); + expect(items.get("app:claude|cwd:yaak")?.kind).toBe("folder"); + expect(items.get("app:claude|cwd:yaak")?.path).toBeUndefined(); + expect(items.get("app:claude|cwd:ml")?.path).toBe("/src/ml"); + expect(items.get("app:claude")?.path).toBeUndefined(); +}); + test("an app in a bundle has no path to reveal", () => { const items = itemsById(buildSidebarTree([flow("https://slack.com/", slack)], 1)); expect(items.get("app:Slack")?.path).toBeUndefined(); diff --git a/apps/yaak-proxy/lib/sidebarTree.ts b/apps/yaak-proxy/lib/sidebarTree.ts index ff16027f2..b8728fc0f 100644 --- a/apps/yaak-proxy/lib/sidebarTree.ts +++ b/apps/yaak-proxy/lib/sidebarTree.ts @@ -166,6 +166,11 @@ function appQueryFor(app: string, system: boolean): string { return system ? `${fieldTerm("is", "system")} ${query}` : query; } +/** A program outside an app bundle: its executable, for Reveal in Finder */ +function programPath(flow: Flow): string | undefined { + return flow.clientBundleId == null ? (flow.clientPath ?? undefined) : undefined; +} + /** A node being built, its children by id */ interface Trie { item: SidebarItem; @@ -185,6 +190,9 @@ function child(parent: Trie, row: PlaceRow): Trie { if (row.decrypt != null) node.item.decrypt = row.decrypt; if (row.path != null) node.item.path = row.path; parent.children.set(row.id, node); + } else if (node.item.path !== row.path) { + // Folders of the same name in different places share a row, which then names no one path. + delete node.item.path; } node.item.count++; return node; @@ -272,10 +280,13 @@ export function buildSidebarTree( } if (folder != null) item.sublabel = folder; if (flow.clientPath != null) item.appPath = flow.clientPath; - // A program outside an app bundle: its executable - if (flow.clientBundleId == null && flow.clientPath != null) item.path = flow.clientPath; + const program = programPath(flow); + if (program != null) item.path = program; app = { item, children: new Map() }; parent.children.set(id, app); + } else if (app.item.path !== programPath(flow)) { + // Programs of the same name in different places share a row, which names no one path. + delete app.item.path; } app.item.count++; parent.item.count++; From 4fc95d264caedf9b3aa40a88e590814c14379c71 Mon Sep 17 00:00:00 2001 From: Gregory Schier Date: Mon, 9 Nov 2026 11:00:58 -0700 Subject: [PATCH 14/45] Clear Yaak Proxy traffic at launch and follow the real system proxy Captured traffic is now cleared as the app launches by default, with a "Keep between launches" choice that brings the size limit with it. An install that had a size limit set keeps its traffic. The Proxy settings tab is regrouped into Proxy, Decryption, Traffic and Terminal, and the per-body limit row is gone. The app now watches macOS proxy settings and follows them: when another app or the user points the system proxy elsewhere, the override shows as off and the settings saved for restoring are dropped, so quitting puts back nothing the app didn't set. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 2 + apps/yaak-proxy/components/SettingsProxy.tsx | 91 +++--- apps/yaak-proxy/harness/LockedHarness.tsx | 1 + apps/yaak-proxy/harness/backend.ts | 1 + apps/yaak-proxy/lib/settings.test.ts | 29 +- apps/yaak-proxy/lib/settings.ts | 6 - crates-proxy/yaak-proxy-lib/Cargo.toml | 3 + .../yaak-proxy-lib/bindings/gen_rpc.ts | 16 +- crates-proxy/yaak-proxy-lib/src/db.rs | 15 + crates-proxy/yaak-proxy-lib/src/flows.rs | 21 ++ crates-proxy/yaak-proxy-lib/src/lib.rs | 272 +++++++++++++++++- crates-proxy/yaak-proxy-lib/src/macos.rs | 222 ++++++++++++++ crates-proxy/yaak-proxy-lib/src/settings.rs | 79 ++++- crates-proxy/yaak-proxy-lib/tests/at_rest.rs | 43 ++- crates-proxy/yaak-proxy-lib/tests/flow.rs | 1 + 15 files changed, 726 insertions(+), 76 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 73bdd8026..979e3c573 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11927,6 +11927,7 @@ dependencies = [ "base64 0.23.1", "bytes", "chrono", + "core-foundation 0.9.4", "encoding_rs", "flate2", "futures-util", @@ -11949,6 +11950,7 @@ dependencies = [ "sea-query", "serde", "serde_json", + "system-configuration", "tempfile", "tokio", "tonic", diff --git a/apps/yaak-proxy/components/SettingsProxy.tsx b/apps/yaak-proxy/components/SettingsProxy.tsx index 5ed520827..fdc785a73 100644 --- a/apps/yaak-proxy/components/SettingsProxy.tsx +++ b/apps/yaak-proxy/components/SettingsProxy.tsx @@ -1,6 +1,10 @@ import { useQueryClient } from "@tanstack/react-query"; import { formatSize } from "@yaakapp-internal/lib/formatSize"; -import type { HttpVersion, SystemProxyStatus } from "@yaakapp-internal/proxy-lib"; +import type { + HttpVersion, + SystemProxyStatus, + TrafficRetention, +} from "@yaakapp-internal/proxy-lib"; import { Button, Dialog, @@ -31,7 +35,6 @@ import { type TerminalShell, } from "../lib/env"; import { - CAPTURE_LIMIT_OPTIONS, KEEP_TRAFFIC_OPTIONS, sizeFromOptionValue, sizeOptionsFor, @@ -44,16 +47,15 @@ import { useShowToast } from "../lib/toast"; import { DecryptListEditor } from "./DecryptListEditor"; /** - * The proxy's own settings tab: how it listens, what it decrypts, the system proxy and its - * helper, how much traffic it keeps, and the environment for one terminal. Everything set once - * and left alone, which is what left the state pill's menu. + * The proxy's own settings tab: how it listens and the system proxy, what it decrypts, what + * happens to captured traffic, and the environment for one terminal. Everything set once and + * left alone, which is what left the state pill's menu. */ export function SettingsProxy() { const proxyState = useAtomValue(proxyStateAtom); const store = useStore(); const refresh = { onSettled: () => void refreshProxyState(store) }; const setHttpVersion = useRpcMutation("set_http_version", refresh); - const setCaptureLimit = useRpcMutation("set_capture_body_limit", refresh); const setAtLaunch = useRpcMutation("set_system_proxy_at_launch", refresh); const systemProxy = useRpcQuery("system_proxy_status", {}); // Another process owning the engine wouldn't see changes until it restarts @@ -63,10 +65,10 @@ export function SettingsProxy() {
Proxy -

Listening, decryption, storage and terminals

+

Listening, decryption, traffic and terminals

- + name="httpVersion" @@ -80,40 +82,27 @@ export function SettingsProxy() { { label: "HTTP/1.1 only", value: "http1Only" }, ]} /> - { - const bytes = sizeFromOptionValue(v); - if (bytes != null) setCaptureLimit.mutate({ bytes }); - }} - options={sizeOptionsFor(CAPTURE_LIMIT_OPTIONS, proxyState?.captureBodyLimit ?? null) - .filter((o) => o.bytes != null) - .map(({ label, value }) => ({ label, value }))} - /> + {systemProxy.data?.supported === true && ( + <> + setAtLaunch.mutate({ enabled })} + /> + + + )} - + - {systemProxy.data?.supported === true && ( - - setAtLaunch.mutate({ enabled })} - /> - - - )} - - - + + + {proxyState?.traffic === "keep" && } @@ -225,6 +214,30 @@ function HelperRow({ status }: { status: SystemProxyStatus }) { ); } +/** Whether captured traffic is cleared as the app launches or kept between launches */ +function TrafficRow() { + const proxyState = useAtomValue(proxyStateAtom); + const store = useStore(); + const setTraffic = useRpcMutation("set_traffic", { + onSettled: () => void refreshProxyState(store), + }); + return ( + + name="traffic" + title="Captured traffic" + selectClassName="w-56!" + value={proxyState?.traffic ?? "clearOnLaunch"} + disabled={proxyState?.otherOwner != null} + onChange={(traffic) => setTraffic.mutate({ traffic })} + options={[ + { label: "Clear on launch", value: "clearOnLaunch" }, + { label: "Keep between launches", value: "keep" }, + ]} + /> + ); +} + +/** The most traffic kept between launches, beside what is stored now */ function KeepTrafficRow() { const proxyState = useAtomValue(proxyStateAtom); const store = useStore(); @@ -234,7 +247,7 @@ function KeepTrafficRow() { }); const keep = proxyState?.keepTrafficBytes ?? null; return ( - +
{size.data != null && ( @@ -243,7 +256,7 @@ function KeepTrafficRow() { )} setKeep.mutate({ bytes: sizeFromOptionValue(v) })} diff --git a/apps/yaak-proxy/harness/LockedHarness.tsx b/apps/yaak-proxy/harness/LockedHarness.tsx index f8c77a071..7c0b2e4eb 100644 --- a/apps/yaak-proxy/harness/LockedHarness.tsx +++ b/apps/yaak-proxy/harness/LockedHarness.tsx @@ -23,6 +23,7 @@ const STATE: GetProxyStateResponse = { locked: { message: LOCKED, keychain: true }, otherOwner: null, captureBodyLimit: 10 * 1024 * 1024, + traffic: "clearOnLaunch", keepTrafficBytes: null, }; diff --git a/apps/yaak-proxy/harness/backend.ts b/apps/yaak-proxy/harness/backend.ts index 701f2091f..391268241 100644 --- a/apps/yaak-proxy/harness/backend.ts +++ b/apps/yaak-proxy/harness/backend.ts @@ -31,6 +31,7 @@ export const HARNESS_STATE: GetProxyStateResponse = { systemProxyAtLaunch: true, httpVersion: "auto", captureBodyLimit: 10 * 1024 * 1024, + traffic: "keep", keepTrafficBytes: 2 * 1024 * 1024 * 1024, startError: null, encrypting: null, diff --git a/apps/yaak-proxy/lib/settings.test.ts b/apps/yaak-proxy/lib/settings.test.ts index 158c7dea7..aa892b73b 100644 --- a/apps/yaak-proxy/lib/settings.test.ts +++ b/apps/yaak-proxy/lib/settings.test.ts @@ -1,6 +1,5 @@ import { describe, expect, test } from "vite-plus/test"; import { - CAPTURE_LIMIT_OPTIONS, DEFAULT_SETTINGS_VALUES, KEEP_TRAFFIC_OPTIONS, proxySettingsView, @@ -78,13 +77,6 @@ describe("size options", () => { }); test("labels in the units the limits are set in", () => { - expect(CAPTURE_LIMIT_OPTIONS.map((o) => o.label)).toEqual([ - "1 MB", - "5 MB", - "10 MB", - "50 MB", - "100 MB", - ]); expect(KEEP_TRAFFIC_OPTIONS.map((o) => o.label)).toEqual([ "Unlimited", "500 MB", @@ -97,22 +89,25 @@ describe("size options", () => { }); test("values round trip", () => { - for (const o of [...KEEP_TRAFFIC_OPTIONS, ...CAPTURE_LIMIT_OPTIONS]) { + for (const o of KEEP_TRAFFIC_OPTIONS) { expect(sizeFromOptionValue(sizeOptionValue(o.bytes))).toBe(o.bytes); } expect(sizeFromOptionValue("abc")).toBeNull(); }); test("a size set elsewhere gets its own option, in order", () => { - const options = sizeOptionsFor(CAPTURE_LIMIT_OPTIONS, 3 * 1024 * 1024); + const MB = 1024 * 1024; + const options = sizeOptionsFor(KEEP_TRAFFIC_OPTIONS, 750 * MB); expect(options.map((o) => o.bytes)).toEqual([ - 1024 * 1024, - 3 * 1024 * 1024, - 5 * 1024 * 1024, - 10 * 1024 * 1024, - 50 * 1024 * 1024, - 100 * 1024 * 1024, + 500 * MB, + 750 * MB, + 1024 * MB, + 2048 * MB, + 5120 * MB, + 10240 * MB, + 20480 * MB, + null, ]); - expect(sizeOptionsFor(CAPTURE_LIMIT_OPTIONS, 10 * 1024 * 1024)).toBe(CAPTURE_LIMIT_OPTIONS); + expect(sizeOptionsFor(KEEP_TRAFFIC_OPTIONS, 1024 * MB)).toBe(KEEP_TRAFFIC_OPTIONS); }); }); diff --git a/apps/yaak-proxy/lib/settings.ts b/apps/yaak-proxy/lib/settings.ts index 7c22db897..117d83ad9 100644 --- a/apps/yaak-proxy/lib/settings.ts +++ b/apps/yaak-proxy/lib/settings.ts @@ -141,12 +141,6 @@ export const KEEP_TRAFFIC_OPTIONS = sizeOptions( "Unlimited", ); -/** How much of each body is stored */ -export const CAPTURE_LIMIT_OPTIONS = sizeOptions( - [MB, 5 * MB, 10 * MB, 50 * MB, 100 * MB], - "Unlimited", -); - /** * The option a stored size selects. A size set elsewhere (the CLI or `settings.json`) that no * option has is added as its own, so the select still shows it. diff --git a/crates-proxy/yaak-proxy-lib/Cargo.toml b/crates-proxy/yaak-proxy-lib/Cargo.toml index b0112ecb5..ccac0f160 100644 --- a/crates-proxy/yaak-proxy-lib/Cargo.toml +++ b/crates-proxy/yaak-proxy-lib/Cargo.toml @@ -71,6 +71,9 @@ objc2-service-management = { version = "0.3.2", default-features = false, featur "SMAppService", ] } plist = "1.7" +# Told when the system proxy changes underneath the app, as Proxyman or System Settings change it. +core-foundation = "0.9" +system-configuration = "0.7" [dev-dependencies] base64 = "0.23.1" diff --git a/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts b/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts index d94a860b5..c97e3c210 100644 --- a/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts +++ b/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts @@ -262,7 +262,12 @@ httpVersion: HttpVersion, */ captureBodyLimit: number, /** - * The most captured traffic kept, in bytes, or null to keep everything. + * Captured traffic is cleared at launch or kept between launches. + */ +traffic: TrafficRetention, +/** + * The most captured traffic kept, in bytes, or null to keep everything. Applies only + * while traffic is kept. */ keepTrafficBytes: number | null, /** @@ -447,7 +452,7 @@ export type ResetCapturedDataRequest = Record; export type RpcEventSchema = { decrypt_list_changed: DecryptListChangedPayload, flows_cleared: FlowsClearedPayload, flows_removed: FlowsRemovedPayload, message_write: MessagePayload, model_write: ModelPayload, proxy_state_changed: ProxyStatePayload, saved_queries_changed: SavedQueriesChangedPayload, }; -export type RpcSchema = { execute_action: [ActionInvocation, boolean], format_body: [FormatBodyRequest, string], get_decrypt_list: [GetDecryptListRequest, DecryptList], get_message_payload: [GetMessagePayloadRequest, GetMessagePayloadResponse], list_actions: [ListActionsRequest, ListActionsResponse], list_flows: [FlowFilter, ListFlowsResponse], query_schema: [QuerySchemaRequest, QuerySchemaResponse], validate_query: [ValidateQueryRequest, Array], watch_query: [WatchQueryRequest, boolean], list_saved_queries: [ListSavedQueriesRequest, Array], create_saved_query: [CreateSavedQueryRequest, SavedQuery], rename_saved_query: [RenameSavedQueryRequest, SavedQuery], delete_saved_query: [DeleteSavedQueryRequest, boolean], list_grpc_schemas: [ListGrpcSchemasRequest, Array], list_messages: [ListMessagesRequest, ListMessagesResponse], remove_grpc_schema: [RemoveGrpcSchemaRequest, boolean], set_recording: [SetRecordingRequest, boolean], set_system_proxy_at_launch: [SetSystemProxyAtLaunchRequest, boolean], set_http_version: [SetHttpVersionRequest, HttpVersion], set_capture_body_limit: [SetCaptureBodyLimitRequest, number], set_keep_traffic: [SetKeepTrafficRequest, number | null], add_grpc_schema: [AddGrpcSchemaRequest, GrpcSchema], app_icon: [AppIconRequest, string | null], ca_status: [CaStatusRequest, CaStatus], ca_trust: [CaTrustRequest, CaStatus], clear_flows: [ClearFlowsRequest, number], decode_grpc_message: [DecodeGrpcMessageRequest, DecodedGrpcMessage], export_har: [ExportHarRequest, number], filter_body: [FilterBodyRequest, FilterBodyResponse], get_flow_body: [GetFlowBodyRequest, GetFlowBodyResponse], get_proxy_state: [GetProxyStateRequest, GetProxyStateResponse], helper_register: [HelperRegisterRequest, SystemProxyStatus], helper_unregister: [HelperUnregisterRequest, SystemProxyStatus], edit_decrypt_list: [EditDecryptListRequest, DecryptList], list_seen: [ListSeenRequest, SeenTargets], set_decrypt_list: [SetDecryptListRequest, DecryptList], list_workspaces: [ListWorkspacesRequest, ListWorkspacesResponse], preview_save: [PreviewSaveRequest, SavePreview], reflect_grpc_host: [ReflectGrpcHostRequest, GrpcSchema], save_flow: [SaveFlowToYaakRequest, SavedFlow], save_flow_body: [SaveFlowBodyRequest, SaveFlowBodyResponse], system_proxy_set: [SystemProxySetRequest, SystemProxyStatus], system_proxy_status: [SystemProxyStatusRequest, SystemProxyStatus], take_over: [TakeOverRequest, boolean], set_port: [SetPortRequest, number], get_storage_size: [GetStorageSizeRequest, GetStorageSizeResponse], reset_captured_data: [ResetCapturedDataRequest, boolean], open_terminal: [OpenTerminalRequest, boolean], query_values: [QueryValuesRequest, Array], match_flows: [MatchFlowsRequest, Array], count_saved_queries: [CountSavedQueriesRequest, Array], }; +export type RpcSchema = { execute_action: [ActionInvocation, boolean], format_body: [FormatBodyRequest, string], get_decrypt_list: [GetDecryptListRequest, DecryptList], get_message_payload: [GetMessagePayloadRequest, GetMessagePayloadResponse], list_actions: [ListActionsRequest, ListActionsResponse], list_flows: [FlowFilter, ListFlowsResponse], query_schema: [QuerySchemaRequest, QuerySchemaResponse], validate_query: [ValidateQueryRequest, Array], watch_query: [WatchQueryRequest, boolean], list_saved_queries: [ListSavedQueriesRequest, Array], create_saved_query: [CreateSavedQueryRequest, SavedQuery], rename_saved_query: [RenameSavedQueryRequest, SavedQuery], delete_saved_query: [DeleteSavedQueryRequest, boolean], list_grpc_schemas: [ListGrpcSchemasRequest, Array], list_messages: [ListMessagesRequest, ListMessagesResponse], remove_grpc_schema: [RemoveGrpcSchemaRequest, boolean], set_recording: [SetRecordingRequest, boolean], set_system_proxy_at_launch: [SetSystemProxyAtLaunchRequest, boolean], set_http_version: [SetHttpVersionRequest, HttpVersion], set_capture_body_limit: [SetCaptureBodyLimitRequest, number], set_traffic: [SetTrafficRequest, TrafficRetention], set_keep_traffic: [SetKeepTrafficRequest, number | null], add_grpc_schema: [AddGrpcSchemaRequest, GrpcSchema], app_icon: [AppIconRequest, string | null], ca_status: [CaStatusRequest, CaStatus], ca_trust: [CaTrustRequest, CaStatus], clear_flows: [ClearFlowsRequest, number], decode_grpc_message: [DecodeGrpcMessageRequest, DecodedGrpcMessage], export_har: [ExportHarRequest, number], filter_body: [FilterBodyRequest, FilterBodyResponse], get_flow_body: [GetFlowBodyRequest, GetFlowBodyResponse], get_proxy_state: [GetProxyStateRequest, GetProxyStateResponse], helper_register: [HelperRegisterRequest, SystemProxyStatus], helper_unregister: [HelperUnregisterRequest, SystemProxyStatus], edit_decrypt_list: [EditDecryptListRequest, DecryptList], list_seen: [ListSeenRequest, SeenTargets], set_decrypt_list: [SetDecryptListRequest, DecryptList], list_workspaces: [ListWorkspacesRequest, ListWorkspacesResponse], preview_save: [PreviewSaveRequest, SavePreview], reflect_grpc_host: [ReflectGrpcHostRequest, GrpcSchema], save_flow: [SaveFlowToYaakRequest, SavedFlow], save_flow_body: [SaveFlowBodyRequest, SaveFlowBodyResponse], system_proxy_set: [SystemProxySetRequest, SystemProxyStatus], system_proxy_status: [SystemProxyStatusRequest, SystemProxyStatus], take_over: [TakeOverRequest, boolean], set_port: [SetPortRequest, number], get_storage_size: [GetStorageSizeRequest, GetStorageSizeResponse], reset_captured_data: [ResetCapturedDataRequest, boolean], open_terminal: [OpenTerminalRequest, boolean], query_values: [QueryValuesRequest, Array], match_flows: [MatchFlowsRequest, Array], count_saved_queries: [CountSavedQueriesRequest, Array], }; export type SaveFlowBodyRequest = { flowId: string, which: FlowBody, /** @@ -567,6 +572,8 @@ export type SetRecordingRequest = { enabled: boolean, }; export type SetSystemProxyAtLaunchRequest = { enabled: boolean, }; +export type SetTrafficRequest = { traffic: TrafficRetention, }; + /** * How a change gets administrator rights. */ @@ -596,6 +603,11 @@ export type SystemProxyStatusRequest = Record; export type TakeOverRequest = Record; +/** + * What happens to captured traffic between launches. + */ +export type TrafficRetention = "clearOnLaunch" | "keep"; + export type ValidateQueryRequest = { query: string, }; /** diff --git a/crates-proxy/yaak-proxy-lib/src/db.rs b/crates-proxy/yaak-proxy-lib/src/db.rs index 91b96581e..9c5ab4530 100644 --- a/crates-proxy/yaak-proxy-lib/src/db.rs +++ b/crates-proxy/yaak-proxy-lib/src/db.rs @@ -270,6 +270,21 @@ pub fn remove_captured(data_dir: &Path) -> std::result::Result<(), String> { Ok(()) } +/// Deletes `blobs.sqlite`, every body and message payload, for one made empty as it opens. +/// The data key stays, as `proxy.db` is still encrypted with it. Nothing may have it open. +pub fn remove_blobs(data_dir: &Path) -> std::result::Result<(), String> { + for suffix in ["", "-wal", "-shm"] { + let path = data_dir.join(format!("{BLOBS_DB}{suffix}")); + match std::fs::remove_file(&path) { + Err(e) if e.kind() != std::io::ErrorKind::NotFound => { + return Err(format!("Failed to delete {}: {e}", path.display())); + } + _ => {} + } + } + Ok(()) +} + /// Either database in `data_dir` is plain, left by a build from before encryption. pub fn needs_encrypting(data_dir: &Path) -> bool { [PROXY_DB, BLOBS_DB] diff --git a/crates-proxy/yaak-proxy-lib/src/flows.rs b/crates-proxy/yaak-proxy-lib/src/flows.rs index 27876ee6c..a01c4c97d 100644 --- a/crates-proxy/yaak-proxy-lib/src/flows.rs +++ b/crates-proxy/yaak-proxy-lib/src/flows.rs @@ -324,6 +324,27 @@ pub fn clear(db: &ProxyQueryManager) -> Result { .map_err(|e| format!("Failed to clear flows: {e}")) } +/// Deletes every flow and message from `proxy.db` in one transaction, with what was left of a +/// clear or sweep, for a launch that starts empty. Saved searches and gRPC schemas stay. Only +/// before anything else writes: unlike [`clear`], flows being written are not spared. Their +/// bodies are in `blobs.sqlite`, which goes whole ([`crate::db::remove_blobs`]). +pub fn delete_all(db: &ProxyQueryManager) -> Result<(), String> { + db.with_tx(|c| { + let conn = c.conn().resolve(); + // Unconditional, so SQLite drops each table's pages whole rather than row by row. + for table in [ + "flow_messages", + "flows", + "flow_clear_mark", + "flow_sweep_pending", + ] { + conn.execute(&format!("DELETE FROM {table}"), [])?; + } + Ok::<_, yaak_database::Error>(()) + }) + .map_err(|e| format!("Failed to delete captured traffic: {e}")) +} + /// Flows deleted per transaction while deleting cleared ones, so a flow being written never /// waits long behind it. pub const DELETE_BATCH: usize = 500; diff --git a/crates-proxy/yaak-proxy-lib/src/lib.rs b/crates-proxy/yaak-proxy-lib/src/lib.rs index fc935c80d..f03a6b84c 100644 --- a/crates-proxy/yaak-proxy-lib/src/lib.rs +++ b/crates-proxy/yaak-proxy-lib/src/lib.rs @@ -33,7 +33,7 @@ use crate::macos::{SystemProxyBackend, SystemProxyStatus}; use crate::messages::FlowMessage; use crate::models::{Flow, FlowBody, ModelPayload, ProxyHeader, StreamKind, find_header}; use crate::save::{ClientDb, SaveFlowRequest, SaveFlowResponse, WorkspaceSummary}; -use crate::settings::{DecryptList, HttpVersion, ProxySettings}; +use crate::settings::{DecryptList, HttpVersion, ProxySettings, TrafficRetention}; use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64; use chrono::{DateTime, Utc}; @@ -106,6 +106,15 @@ pub struct ProxyCtx { resets: Arc, /// The thread keeping captured traffic under the keep-traffic cap, while there is one. sweeper: Arc<(Mutex, Condvar)>, + /// Captured traffic is to be emptied as the databases are first unlocked, this launch + /// being set to clear it. Taken once it is. + clear_at_launch: Arc, + /// The system proxy watch is running, so it is started once. + watching_system_proxy: Arc, + /// System proxy changes decided but not yet made, from when `system_proxy_on` is set until + /// the change is done. Until then the system proxy is not yet as meant, and following it + /// would undo the change before it is made. + system_proxy_pending: Arc, pub events: RpcEventEmitter, /// gRPC schemas as decoded, and the protoc that compiles attached protos. pub grpc: GrpcState, @@ -150,10 +159,18 @@ impl ProxyCtx { lock_reason: Arc::new(Mutex::new(None)), resets: Arc::new(AtomicU64::new(0)), sweeper: Default::default(), + clear_at_launch: Arc::new(AtomicBool::new(false)), + watching_system_proxy: Arc::new(AtomicBool::new(false)), + system_proxy_pending: Arc::new(AtomicU64::new(0)), events, grpc: GrpcState::default(), live_search: Default::default(), }; + // Unreadable settings keep the traffic: nothing is deleted on a guess. Nor is traffic + // another process is capturing as this launches, which this one may take over later. + let clear = ctx.settings().is_ok_and(|s| s.traffic() == TrafficRetention::ClearOnLaunch) + && control::live_owner(data_dir).is_none_or(|o| o.pid == std::process::id()); + ctx.clear_at_launch.store(clear, Ordering::SeqCst); if db::needs_encrypting(data_dir) { let unlocking = ctx.clone(); std::thread::spawn(move || { @@ -251,12 +268,24 @@ impl ProxyCtx { } let db_key = db::database_key(&key); + // Before converting, so bodies about to be deleted aren't encrypted first. + let clearing = self.take_clear_at_launch(); + if clearing && let Err(e) = db::remove_blobs(&self.data_dir) { + warn!("{e}"); + } + let converting = converting && db::needs_encrypting(&self.data_dir); if converting { let converted = self.encrypt_existing(&db_key); self.set_encrypting(None); converted?; } let stores = Stores::open(&self.data_dir, &db_key)?; + if clearing { + match flows::delete_all(&stores.db) { + Ok(()) => log::info!("Cleared captured traffic at launch"), + Err(e) => warn!("{e}"), + } + } // SQLCipher's exit handler is registered by now, so the gate's runs before it. exit_gate::install(); *self.stores.write().unwrap_or_else(PoisonError::into_inner) = Some(stores); @@ -273,6 +302,23 @@ impl ProxyCtx { Ok(key) } + /// Whether this unlock empties the captured traffic: the first to get this far since a + /// launch set to clear it. Not while another process owns the engine, as the CLI daemon + /// may: the traffic is its, still being written, and this app takes it over as it is. Asked + /// at launch too, as the first unlock may come later, once the keychain allows it. + fn take_clear_at_launch(&self) -> bool { + if !self.clear_at_launch.swap(false, Ordering::SeqCst) { + return false; + } + match control::live_owner(&self.data_dir) { + Some(owner) if owner.pid != std::process::id() => { + log::info!("Keeping captured traffic: {}", control::already_owned(&owner)); + false + } + _ => true, + } + } + /// Converts the plain databases, telling the app how far it has got a few times a second. fn encrypt_existing(&self, key: &yaak_database::encryption::DatabaseKey) -> Result<(), String> { let started = std::time::Instant::now(); @@ -440,11 +486,13 @@ impl ProxyCtx { *last_port = Some(bound); } let route = system_proxy.unwrap_or(settings.system_proxy_at_launch); + let pending = route.then(|| self.pending_system_proxy_change()); self.system_proxy_on.store(route, Ordering::SeqCst); - if route { + if let Some(pending) = pending { // The password dialog can sit open for a while; the engine is up regardless. let ctx = self.detached(); std::thread::spawn(move || { + let _pending = pending; if let Err(e) = ctx.change_system_proxy(true, bound, true) { warn!("Failed to turn on the system proxy: {e}"); ctx.system_proxy_on.store(false, Ordering::SeqCst); @@ -486,11 +534,13 @@ impl ProxyCtx { /// Stops the engine and turns off a system proxy that points at it. pub fn stop(&self) -> Result<(), String> { + let pending = self.pending_system_proxy_change(); self.system_proxy_on.store(false, Ordering::SeqCst); let port = self.take_handle()?; if let Some(port) = port { let ctx = self.detached(); std::thread::spawn(move || { + let _pending = pending; if let Err(e) = ctx.change_system_proxy(false, port, true) { warn!("Failed to turn off the system proxy: {e}"); } @@ -565,6 +615,8 @@ impl ProxyCtx { warn!("Failed to reset a stale system proxy: {e}"); } }); + // The owner turns the system proxy on and off, so it follows changes made underneath. + self.watch_system_proxy(); Ok(()) } @@ -692,6 +744,7 @@ impl ProxyCtx { /// password dialog. Not saved: the `systemProxyAtLaunch` setting decides whether it turns /// on as the engine starts. Blocks. pub fn set_system_proxy(&self, enabled: bool) -> Result { + let _pending = self.pending_system_proxy_change(); let Some(port) = self.port() else { if enabled { return Err("The proxy is not running, so there is nothing to route to".into()); @@ -799,6 +852,18 @@ impl ProxyCtx { Ok(bytes) } + /// Whether captured traffic is cleared at the next launch or kept, saved in `settings.json`. + /// Kept, the keep-traffic cap applies from now on; cleared, it is remembered but not + /// applied, and nothing already captured goes until the next launch. + pub fn set_traffic(&self, traffic: TrafficRetention) -> Result { + let mut settings = self.settings()?; + settings.traffic = Some(traffic); + settings.save(&self.settings_path())?; + self.sweep_soon(); + self.emit_state(); + Ok(traffic) + } + /// Bytes the captured traffic takes, as the keep-traffic cap counts them: the databases' /// used pages, so it shrinks as flows are deleted though the files do not. Locked, the /// files' sizes instead, as nothing in them can be read. @@ -916,9 +981,9 @@ impl ProxyCtx { } } - /// The keep-traffic cap, as `settings.json` has it. + /// The keep-traffic cap, as `settings.json` has it, while traffic is kept between launches. fn keep_traffic(&self) -> Option { - self.settings().ok().and_then(|s| s.keep_traffic_bytes) + self.settings().ok().and_then(|s| s.keep_traffic_cap()) } /// Sweeps the oldest flows while the captured traffic is over the cap, then waits for @@ -1093,6 +1158,88 @@ impl ProxyCtx { Ok(()) } + /// Marks a system proxy change as decided and not yet made, until the guard is dropped. + fn pending_system_proxy_change(&self) -> PendingChange { + self.system_proxy_pending.fetch_add(1, Ordering::SeqCst); + PendingChange(self.system_proxy_pending.clone()) + } + + /// Follows the system proxy as macOS reports changes to it, for as long as the process + /// runs, so a change made underneath the app, by hand or by another app such as Proxyman, + /// shows at once. Started once; a failure leaves the app reading it when the window is + /// focused, as before. + pub fn watch_system_proxy(&self) { + if self.watching_system_proxy.swap(true, Ordering::SeqCst) { + return; + } + let (tx, rx) = std::sync::mpsc::channel::<()>(); + if let Err(e) = macos::watch(move || { + let _ = tx.send(()); + }) { + warn!("Not following the system proxy: {e}"); + return; + } + let ctx = self.detached(); + let spawned = + std::thread::Builder::new().name("system-proxy-follow".into()).spawn(move || { + while rx.recv().is_ok() { + // One change comes as several notices, a service at a time. + while rx.recv_timeout(SYSTEM_PROXY_SETTLE).is_ok() {} + while !ctx.follow_system_proxy() { + std::thread::sleep(SYSTEM_PROXY_SETTLE); + } + } + }); + if let Err(e) = spawned { + warn!("Not following the system proxy: {e}"); + } + } + + /// Takes the system proxy as it is now: meant to point at the running engine while every + /// service sends its traffic there, and not otherwise. With nothing left pointing at the + /// engine's port, the settings saved for putting back are dropped, so stopping or quitting + /// restores nothing over a change someone else made. Tells the app either way. Returns + /// false, doing nothing, while a change of the app's own is under way, to be tried again. + /// Blocks. + fn follow_system_proxy(&self) -> bool { + if self.system_proxy_pending.load(Ordering::SeqCst) > 0 { + return false; + } + // Held so no change starts meanwhile, nor a change of ours is mistaken for another's. + let _turn = match self.system_proxy_turn.try_lock() { + Ok(turn) => turn, + Err(std::sync::TryLockError::Poisoned(turn)) => turn.into_inner(), + Err(std::sync::TryLockError::WouldBlock) => return false, + }; + if self.system_proxy_pending.load(Ordering::SeqCst) > 0 { + return false; + } + let running = self.port(); + let Some(port) = running.or_else(|| self.settings().ok().map(|s| s.port)) else { + return true; + }; + match macos::routing(port) { + Ok(routing) => { + if running.is_some() { + let was = self.system_proxy_on.swap(routing.every, Ordering::SeqCst); + if was != routing.every { + let now = if routing.every { "on" } else { "off" }; + debug!("The system proxy is {now} at port {port} after a change"); + } + } + if !routing.any + && let Err(e) = + macos::forget(port, self.system_proxy_backend(), &self.snapshot_path()) + { + warn!("Failed to forget the replaced proxy settings: {e}"); + } + } + Err(e) => warn!("Failed to read the system proxy: {e}"), + } + self.emit_state(); + true + } + /// Yaak has run on this machine, so there is a database to save into. pub fn client_exists(&self) -> bool { ClientDb::exists(&self.client_data_dir) @@ -1273,6 +1420,19 @@ fn remove_captured_when_closed(data_dir: &Path) -> Result<(), String> { } } +/// A system proxy change decided and not yet made, counted while it is held. +struct PendingChange(Arc); + +impl Drop for PendingChange { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::SeqCst); + } +} + +/// How long the system proxy is left to settle after a change is reported before it is read, +/// and how long a read waits for a change of the app's own to finish. +const SYSTEM_PROXY_SETTLE: Duration = Duration::from_millis(500); + /// Tries at deleting the captured databases, 100 ms apart, before a reset gives up. const REMOVE_TRIES: u32 = 50; @@ -1535,7 +1695,10 @@ pub struct GetProxyStateResponse { /// Bytes of each body stored per flow. #[ts(type = "number")] pub capture_body_limit: usize, - /// The most captured traffic kept, in bytes, or null to keep everything. + /// Captured traffic is cleared at launch or kept between launches. + pub traffic: TrafficRetention, + /// The most captured traffic kept, in bytes, or null to keep everything. Applies only + /// while traffic is kept. #[ts(type = "number | null")] pub keep_traffic_bytes: Option, /// Why the engine last failed to start, until it starts. @@ -1635,6 +1798,12 @@ pub struct SetCaptureBodyLimitRequest { pub bytes: u64, } +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct SetTrafficRequest { + pub traffic: TrafficRetention, +} + #[derive(Deserialize, TS)] #[ts(export, export_to = "gen_rpc.ts")] pub struct SetKeepTrafficRequest { @@ -2075,6 +2244,7 @@ async fn get_proxy_state( system_proxy_at_launch: settings.system_proxy_at_launch, http_version: settings.http_version, capture_body_limit: settings.capture_body_limit, + traffic: settings.traffic(), keep_traffic_bytes: settings.keep_traffic_bytes, start_error: ctx.start_error(), encrypting: ctx.encrypting(), @@ -2116,6 +2286,10 @@ fn set_capture_body_limit( ctx.set_capture_body_limit(bytes).map_err(rpc_error) } +fn set_traffic(ctx: &ProxyCtx, req: SetTrafficRequest) -> Result { + ctx.set_traffic(req.traffic).map_err(rpc_error) +} + /// The cap as saved, in bytes, or null to keep everything. fn set_keep_traffic(ctx: &ProxyCtx, req: SetKeepTrafficRequest) -> Result, RpcError> { let bytes = ctx.set_keep_traffic(req.bytes).map_err(rpc_error)?; @@ -3335,6 +3509,7 @@ define_rpc! { set_system_proxy_at_launch(SetSystemProxyAtLaunchRequest) -> bool, set_http_version(SetHttpVersionRequest) -> HttpVersion, set_capture_body_limit(SetCaptureBodyLimitRequest) -> usize, + set_traffic(SetTrafficRequest) -> TrafficRetention, set_keep_traffic(SetKeepTrafficRequest) -> Option, } async_commands { @@ -4396,9 +4571,17 @@ mod tests { assert_eq!(removed.len(), 20 - kept, "the app hears of each"); assert_eq!(removed, ids[..removed.len()], "the oldest, in order"); - // Set, the cap starts the sweeper, which finds nothing more to do; taken away, it stops. + // Clearing at launch, a cap is saved but not applied. assert_eq!(ctx.set_keep_traffic(Some(cap)).unwrap(), Some(cap)); assert_eq!(ctx.settings().unwrap().keep_traffic_bytes, Some(cap)); + assert!(!sweeper_running(&ctx), "no sweeping while traffic is cleared at launch"); + // Kept, the cap starts the sweeper, which finds nothing more to do; taken away, it + // stops, as it does when traffic goes back to being cleared at launch. + ctx.set_traffic(TrafficRetention::Keep).unwrap(); + assert!(sweeper_running(&ctx)); + ctx.set_traffic(TrafficRetention::ClearOnLaunch).unwrap(); + wait_until("the sweeper stops", || !sweeper_running(&ctx)); + ctx.set_traffic(TrafficRetention::Keep).unwrap(); assert!(sweeper_running(&ctx)); assert_eq!(ctx.set_keep_traffic(None).unwrap(), None); wait_until("the sweeper stops", || !sweeper_running(&ctx)); @@ -4632,6 +4815,75 @@ mod tests { ctx.stop().unwrap(); } + /// Opens the proxy in `dir` as a launch does, with `keys`, after writing two flows with + /// bodies and a saved search into it from a launch before. + fn relaunch_with_traffic( + dir: &Path, + keys: &MasterKeySource, + settings: Option, + ) -> ProxyCtx { + { + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir, events, keys.clone()).unwrap(); + flow_with_response(&ctx, "fl_a", "text/plain", b"one"); + flow_with_response(&ctx, "fl_b", "text/plain", b"two"); + saved_queries::create(&ctx.db().unwrap(), "Errors", "@status:>=400").unwrap(); + assert_eq!(flows::count(&ctx.db().unwrap()).unwrap(), 2); + } + if let Some(settings) = settings { + settings.save(&dir.join("settings.json")).unwrap(); + } + let (events, _rx) = RpcEventEmitter::new(); + ProxyCtx::new(dir, events, keys.clone()).unwrap() + } + + /// By default a launch starts with no traffic, bodies included, and keeps saved searches. + #[test] + fn a_launch_clears_captured_traffic_by_default() { + let dir = tempfile::tempdir().unwrap(); + let keys = MasterKeySource::memory(); + let ctx = relaunch_with_traffic(dir.path(), &keys, None); + let db = ctx.db().unwrap(); + assert_eq!(flows::count(&db).unwrap(), 0); + assert_eq!(blobs_of(&ctx, "fl_a", 0), 0); + let saved = saved_queries::list(&db).unwrap(); + assert_eq!(saved.iter().map(|s| s.name.as_str()).collect::>(), ["Errors"]); + // New traffic is written as before. + flow_with_response(&ctx, "fl_c", "text/plain", b"three"); + assert_eq!(flows::count(&db).unwrap(), 1); + } + + /// Kept between launches, or with a cap set before the choice existed, traffic stays. + #[test] + fn a_launch_keeps_traffic_when_set_to() { + let keys = MasterKeySource::memory(); + let keep = ProxySettings { traffic: Some(TrafficRetention::Keep), ..Default::default() }; + // From before the choice: a cap and no `traffic` key. + let capped: ProxySettings = + serde_json::from_str(r#"{"keepTrafficBytes": 1073741824}"#).unwrap(); + assert_eq!(capped.traffic, None); + for settings in [keep, capped] { + let dir = tempfile::tempdir().unwrap(); + let ctx = relaunch_with_traffic(dir.path(), &keys, Some(settings)); + assert_eq!(flows::count(&ctx.db().unwrap()).unwrap(), 2); + assert_eq!(blobs_of(&ctx, "fl_a", 0), 1); + } + } + + /// Only a launch clears: unlocking again, after a reset, keeps what was captured since. + #[test] + fn only_the_first_unlock_clears() { + let dir = tempfile::tempdir().unwrap(); + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + assert!(!ctx.take_clear_at_launch(), "taken by the launch"); + flow_with_response(&ctx, "fl_a", "text/plain", b"one"); + ctx.reset_captured_data().unwrap(); + flow_with_response(&ctx, "fl_b", "text/plain", b"two"); + assert!(ctx.unlock().is_ok()); + assert_eq!(flows::count(&ctx.db().unwrap()).unwrap(), 1); + } + /// The capture limit is saved and shows in the proxy's state, as the keep-traffic cap does. #[tokio::test(flavor = "multi_thread")] async fn the_state_has_the_capture_limit_and_the_traffic_cap() { @@ -4644,6 +4896,14 @@ mod tests { assert_eq!(set_keep_traffic(&ctx, req).unwrap(), Some(1 << 30)); let state = get_proxy_state(ctx.clone(), GetProxyStateRequest {}).await.unwrap(); assert_eq!((state.capture_body_limit, state.keep_traffic_bytes), (4096, Some(1 << 30))); + assert_eq!( + state.traffic, + TrafficRetention::ClearOnLaunch, + "a cap set now keeps the choice" + ); + set_traffic(&ctx, SetTrafficRequest { traffic: TrafficRetention::Keep }).unwrap(); + let state = get_proxy_state(ctx.clone(), GetProxyStateRequest {}).await.unwrap(); + assert_eq!(state.traffic, TrafficRetention::Keep); let size = get_storage_size(ctx.clone(), GetStorageSizeRequest {}).await.unwrap(); assert!(size.bytes > 0); ctx.set_keep_traffic(None).unwrap(); diff --git a/crates-proxy/yaak-proxy-lib/src/macos.rs b/crates-proxy/yaak-proxy-lib/src/macos.rs index 7a319306b..6224a5329 100644 --- a/crates-proxy/yaak-proxy-lib/src/macos.rs +++ b/crates-proxy/yaak-proxy-lib/src/macos.rs @@ -152,6 +152,61 @@ fn set_scripted( } } +/// Where the system proxy points, as far as one port is concerned. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Routing { + /// Every enabled service sends HTTP and HTTPS to the port: the override is on. + pub every: bool, + /// Some service, enabled or not, still sends one of them to the port: something the app + /// set may be left to put back. + pub any: bool, +} + +/// [`Routing`] at `port` through `ns`. Disabled services are read only when no enabled one +/// routes everything, which is the common case worth the extra `networksetup` calls. +#[cfg_attr(not(target_os = "macos"), allow(dead_code))] +fn routing_with(ns: &impl Networksetup, port: u16) -> Result { + let enabled = read_services(ns, false)?; + let every = !enabled.is_empty() + && enabled.iter().all(|s| s.http.points_at(port) && s.https.points_at(port)); + if every { + return Ok(Routing { every, any: true }); + } + let any = read_services(ns, true)?.iter().any(|s| s.points_at(port)); + Ok(Routing { every, any }) +} + +/// Drops the settings the password dialog or `sudo` saved for putting back once routing at +/// `port` ends, when nothing points at it any more: someone else changed the system proxy +/// since, and what is there now is theirs. Saved settings for a service no longer present stay +/// parked, as a restore would leave them. Never asks for rights: it only rewrites the file. +/// Returns whether anything was dropped. +#[cfg_attr(not(target_os = "macos"), allow(dead_code))] +fn forget_scripted( + ns: &impl Networksetup, + port: u16, + snapshot_path: &Path, +) -> Result { + let Some(snapshot) = load_snapshot(snapshot_path)? else { + return Ok(false); + }; + // Settings parked at port 0, a restore halfway through, or routing at another port are + // left to what handles them. + if snapshot.port == 0 || snapshot.port != port || !snapshot.restoring.is_empty() { + return Ok(false); + } + let services = read_services(ns, true)?; + let ports = snapshot.ports(); + if services.iter().any(|s| ports.iter().any(|p| s.points_at(*p))) { + return Ok(false); + } + match snapshot.leftover(&services) { + Some(leftover) => save_snapshot(snapshot_path, &leftover)?, + None => remove_snapshot(snapshot_path)?, + } + Ok(true) +} + /// The calls as one `sh` script that stops at the first failure. #[cfg_attr(not(target_os = "macos"), allow(dead_code))] fn script(commands: &[Vec]) -> String { @@ -269,6 +324,104 @@ pub fn set( } } +/// Where the system proxy points now, judged against `port`. Blocks. +#[cfg(target_os = "macos")] +pub fn routing(port: u16) -> Result { + routing_with(&yaak_proxy_helper::networksetup::SystemNetworksetup, port) +} + +/// Drops what was saved for putting back once routing at `port` ends, when nothing points at +/// it any more, so stopping or quitting restores nothing over someone else's change. Never +/// asks for rights: the helper is root already, and the other paths only rewrite their file. +/// Blocks. +#[cfg(target_os = "macos")] +pub fn forget(port: u16, backend: SystemProxyBackend, snapshot_path: &Path) -> Result<(), String> { + match backend { + // With nothing pointing at `port`, turning off there changes nothing and drops what + // the helper saved. + SystemProxyBackend::Helper => crate::helper::set(false, port), + SystemProxyBackend::AdminPrompt | SystemProxyBackend::Sudo => { + let ns = yaak_proxy_helper::networksetup::SystemNetworksetup; + if forget_scripted(&ns, port, snapshot_path)? { + log::info!( + "The system proxy no longer points at port {port}; forgot what it replaced" + ); + } + Ok(()) + } + } +} + +/// Calls `changed`, on a thread of its own, each time macOS reports a change to a proxy +/// setting: by this app, another one such as Proxyman, or the user in System Settings. Several +/// may come for one change. Runs for as long as the process does. +#[cfg(target_os = "macos")] +pub fn watch(changed: impl Fn() + Send + 'static) -> Result<(), String> { + use core_foundation::array::CFArray; + use core_foundation::runloop::{CFRunLoop, kCFRunLoopCommonModes}; + use core_foundation::string::CFString; + use system_configuration::dynamic_store::{ + SCDynamicStore, SCDynamicStoreBuilder, SCDynamicStoreCallBackContext, + }; + + type Changed = Box; + fn callout(_store: SCDynamicStore, _keys: CFArray, changed: &mut Changed) { + changed(); + } + + let changed: Changed = Box::new(changed); + let (ready_tx, ready_rx) = std::sync::mpsc::channel(); + std::thread::Builder::new() + .name("system-proxy-watch".into()) + .spawn(move || { + let context = SCDynamicStoreCallBackContext { callout, info: changed }; + let Some(store) = + SCDynamicStoreBuilder::new("yaak-proxy").callback_context(context).build() + else { + let _ = ready_tx.send(Err("Failed to open the dynamic store".to_string())); + return; + }; + // The proxies in effect, and each service's own, as `networksetup` changes them. + let keys = CFArray::from_CFTypes(&[CFString::from("State:/Network/Global/Proxies")]); + let patterns = + CFArray::from_CFTypes(&[CFString::from("Setup:/Network/Service/[^/]+/Proxies")]); + if !store.set_notification_keys(&keys, &patterns) { + let _ = ready_tx.send(Err("Failed to watch the proxy settings".to_string())); + return; + } + let Some(source) = store.create_run_loop_source() else { + let _ = ready_tx.send(Err("Failed to watch the proxy settings".to_string())); + return; + }; + // SAFETY: a constant CoreFoundation exports. + CFRunLoop::get_current().add_source(&source, unsafe { kCFRunLoopCommonModes }); + let _ = ready_tx.send(Ok(())); + CFRunLoop::run_current(); + }) + .map_err(|e| format!("Failed to start watching the system proxy: {e}"))?; + ready_rx.recv().map_err(|_| "The system proxy watch stopped".to_string())? +} + +#[cfg(not(target_os = "macos"))] +pub fn routing(_port: u16) -> Result { + Ok(Routing { every: false, any: false }) +} + +#[cfg(not(target_os = "macos"))] +pub fn forget( + _port: u16, + _backend: SystemProxyBackend, + _snapshot_path: &Path, +) -> Result<(), String> { + Ok(()) +} + +/// No system proxy to watch here. +#[cfg(not(target_os = "macos"))] +pub fn watch(_changed: impl Fn() + Send + 'static) -> Result<(), String> { + Ok(()) +} + #[cfg(not(target_os = "macos"))] pub fn status( port: u16, @@ -478,6 +631,75 @@ mod tests { assert_eq!(ethernet.http.server, HOST); } + #[test] + fn routing_follows_what_the_services_point_at() { + let ns = fake(); + let prompts = std::cell::Cell::new(0); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(SNAPSHOT_FILE); + assert_eq!(routing_with(&ns, 9090).unwrap(), Routing { every: false, any: false }); + + set_scripted(&ns, runner(&ns, &prompts), true, 9090, &path).unwrap(); + assert_eq!(routing_with(&ns, 9090).unwrap(), Routing { every: true, any: true }); + + // Another app moves one proxy of one service to its own port. + let wifi = ns.proxies()[0].clone(); + ns.set_proxy("Wi-Fi", setting(true, HOST, 9091), wifi.https); + assert_eq!(routing_with(&ns, 9090).unwrap(), Routing { every: false, any: true }); + + // A disabled service still pointing here counts as something to put back. + ns.set_proxy("Wi-Fi", setting(true, HOST, 9091), setting(true, HOST, 9091)); + ns.set_enabled("Ethernet", false); + assert_eq!(routing_with(&ns, 9090).unwrap(), Routing { every: false, any: true }); + } + + #[test] + fn a_proxy_changed_underneath_forgets_what_it_replaced() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(SNAPSHOT_FILE); + let ns = fake(); + let prompts = std::cell::Cell::new(0); + set_scripted(&ns, runner(&ns, &prompts), true, 9090, &path).unwrap(); + + // Still pointing here: kept. + assert!(!forget_scripted(&ns, 9090, &path).unwrap()); + assert!(path.exists()); + + // Another app, Proxyman say, points everything at its own port. + let theirs = (setting(true, HOST, 9091), setting(true, HOST, 9091)); + ns.set_proxy("Wi-Fi", theirs.0.clone(), theirs.1.clone()); + ns.set_proxy("Ethernet", theirs.0.clone(), theirs.1.clone()); + // Checked against another port, nothing changes. + assert!(!forget_scripted(&ns, 8080, &path).unwrap()); + assert!(forget_scripted(&ns, 9090, &path).unwrap()); + assert!(!path.exists()); + + // Quitting now restores nothing over theirs, and asks for nothing. + let before = prompts.get(); + set_scripted(&ns, runner(&ns, &prompts), false, 9090, &path).unwrap(); + assert_eq!(prompts.get(), before); + assert!(ns.proxies().iter().all(|s| s.http == theirs.0 && s.https == theirs.1)); + } + + #[test] + fn forgetting_keeps_settings_parked_for_a_missing_service() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(SNAPSHOT_FILE); + let ns = fake(); + let prompts = std::cell::Cell::new(0); + set_scripted(&ns, runner(&ns, &prompts), true, 9090, &path).unwrap(); + ns.remove("Ethernet"); + ns.set_proxy("Wi-Fi", ProxySetting::default(), ProxySetting::default()); + + assert!(forget_scripted(&ns, 9090, &path).unwrap()); + let kept = load_snapshot(&path).unwrap().unwrap(); + assert_eq!(kept.port, 0, "parked, routing nothing"); + assert_eq!(kept.services.len(), 1); + assert_eq!(kept.services[0].name, "Ethernet"); + // Parked settings aren't forgotten again. + assert!(!forget_scripted(&ns, 9090, &path).unwrap()); + } + #[test] fn quotes_service_names() { let commands = vec![vec![ diff --git a/crates-proxy/yaak-proxy-lib/src/settings.rs b/crates-proxy/yaak-proxy-lib/src/settings.rs index 969d04523..d32f55ac7 100644 --- a/crates-proxy/yaak-proxy-lib/src/settings.rs +++ b/crates-proxy/yaak-proxy-lib/src/settings.rs @@ -15,8 +15,13 @@ pub struct ProxySettings { pub system_proxy_at_launch: bool, /// Bytes of each request and response body stored per flow. pub capture_body_limit: usize, + /// Whether captured traffic is cleared as the app or daemon launches or kept between + /// launches. Missing, as in a file from before the choice, it is read from + /// `keepTrafficBytes`: a cap set there means the traffic was meant to be kept. + #[serde(default)] + pub traffic: Option, /// The most captured traffic kept, in bytes of the databases' used pages; the oldest flows - /// are deleted past it. None keeps everything. + /// are deleted past it. None keeps everything. Only applies while traffic is kept. pub keep_traffic_bytes: Option, /// The HTTP versions offered to clients in intercepted HTTPS. pub http_version: HttpVersion, @@ -34,6 +39,7 @@ impl Default for ProxySettings { port: 9090, system_proxy_at_launch: true, capture_body_limit: DEFAULT_CAPTURE_BODY_LIMIT, + traffic: Some(TrafficRetention::ClearOnLaunch), keep_traffic_bytes: None, http_version: HttpVersion::Auto, decrypt: Vec::new(), @@ -42,6 +48,18 @@ impl Default for ProxySettings { } } +/// What happens to captured traffic between launches. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_rpc.ts")] +pub enum TrafficRetention { + /// Emptied as the app or daemon launches, before anything reads it. + #[default] + ClearOnLaunch, + /// Kept, under the keep-traffic cap when there is one. + Keep, +} + /// One entry of the decrypt list, as the app shows it. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, TS)] #[serde(rename_all = "camelCase")] @@ -76,6 +94,25 @@ pub enum HttpVersion { } impl ProxySettings { + /// Whether traffic is cleared at launch or kept. A file from before the choice keeps it + /// when it has a keep-traffic cap, so an update doesn't delete a capture someone meant to + /// keep. + pub fn traffic(&self) -> TrafficRetention { + self.traffic.unwrap_or(if self.keep_traffic_bytes.is_some() { + TrafficRetention::Keep + } else { + TrafficRetention::ClearOnLaunch + }) + } + + /// The keep-traffic cap, while traffic is kept. Cleared at launch, nothing is swept. + pub fn keep_traffic_cap(&self) -> Option { + match self.traffic() { + TrafficRetention::Keep => self.keep_traffic_bytes, + TrafficRetention::ClearOnLaunch => None, + } + } + /// The user's entries, then the built-in exclusions they kept. pub fn decrypt_list(&self) -> DecryptList { let user = @@ -157,8 +194,12 @@ impl ProxySettings { /// The settings at `path`, or the defaults when there is no file. pub fn load(path: &Path) -> Result { match std::fs::read_to_string(path) { - Ok(contents) => serde_json::from_str(&contents) - .map_err(|e| format!("Invalid settings in {}: {e}", path.display())), + Ok(contents) => { + let mut settings: Self = serde_json::from_str(&contents) + .map_err(|e| format!("Invalid settings in {}: {e}", path.display()))?; + settings.traffic = Some(settings.traffic()); + Ok(settings) + } Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(Self::default()), Err(e) => Err(format!("Failed to read {}: {e}", path.display())), } @@ -182,7 +223,35 @@ mod tests { assert_eq!(settings.port, 9090); assert!(settings.system_proxy_at_launch); assert_eq!(settings.capture_body_limit, 10 * 1024 * 1024); - assert_eq!(settings.keep_traffic_bytes, None, "everything is kept"); + assert_eq!(settings.keep_traffic_bytes, None, "no cap"); + assert_eq!(settings.traffic(), TrafficRetention::ClearOnLaunch); + } + + #[test] + fn a_file_from_before_the_choice_keeps_traffic_when_it_had_a_cap() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("settings.json"); + std::fs::write(&path, r#"{"port": 9090, "keepTrafficBytes": 1073741824}"#).unwrap(); + let settings = ProxySettings::load(&path).unwrap(); + assert_eq!(settings.traffic(), TrafficRetention::Keep); + assert_eq!(settings.keep_traffic_cap(), Some(1 << 30)); + + std::fs::write(&path, r#"{"port": 9090, "keepTrafficBytes": null}"#).unwrap(); + let settings = ProxySettings::load(&path).unwrap(); + assert_eq!(settings.traffic(), TrafficRetention::ClearOnLaunch); + + // Chosen once, the choice holds whatever the cap. + std::fs::write(&path, r#"{"traffic": "clearOnLaunch", "keepTrafficBytes": 1024}"#).unwrap(); + let settings = ProxySettings::load(&path).unwrap(); + assert_eq!(settings.traffic(), TrafficRetention::ClearOnLaunch); + assert_eq!(settings.keep_traffic_cap(), None, "the cap applies only while keeping"); + assert_eq!(settings.keep_traffic_bytes, Some(1024), "but is remembered"); + + // Saved, the choice is written out, so setting a cap later doesn't change it. + settings.save(&path).unwrap(); + let json: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap(); + assert_eq!(json["traffic"], "clearOnLaunch"); } #[test] @@ -202,6 +271,7 @@ mod tests { port: 8080, system_proxy_at_launch: false, capture_body_limit: 1024, + traffic: Some(TrafficRetention::Keep), keep_traffic_bytes: Some(1 << 30), http_version: HttpVersion::Http1Only, decrypt: vec!["host:api.example.com".into()], @@ -218,6 +288,7 @@ mod tests { "port": 8080, "systemProxyAtLaunch": false, "captureBodyLimit": 1024, + "traffic": "keep", "keepTrafficBytes": 1073741824, "httpVersion": "http1Only", "decrypt": ["host:api.example.com"], diff --git a/crates-proxy/yaak-proxy-lib/tests/at_rest.rs b/crates-proxy/yaak-proxy-lib/tests/at_rest.rs index 65cb8f72a..68d18aace 100644 --- a/crates-proxy/yaak-proxy-lib/tests/at_rest.rs +++ b/crates-proxy/yaak-proxy-lib/tests/at_rest.rs @@ -16,7 +16,7 @@ use yaak_proxy_lib::client::ProxyConnection; use yaak_proxy_lib::db::{BLOBS_DB, DATA_KEY, PROXY_DB, database_key}; use yaak_proxy_lib::flows::{self, FlowFilter}; use yaak_proxy_lib::models::{Flow, FlowBody, ProxyHeader}; -use yaak_proxy_lib::settings::ProxySettings; +use yaak_proxy_lib::settings::{ProxySettings, TrafficRetention}; use yaak_proxy_lib::{MasterKeySource, ProxyCtx}; use yaak_rpc::RpcEventEmitter; @@ -25,8 +25,15 @@ const QUERY_SECRET: &str = "QUERYSECRET456"; const BODY_SECRET: &str = "BODYSECRET789"; const FLOWS: usize = 3_000; +/// The proxy in `dir`, launched with traffic kept between launches, so what one launch stored +/// the next reads back. fn open_ctx(dir: &Path, keys: MasterKeySource) -> ProxyCtx { - let settings = ProxySettings { port: 0, system_proxy_at_launch: false, ..Default::default() }; + let settings = ProxySettings { + port: 0, + system_proxy_at_launch: false, + traffic: Some(TrafficRetention::Keep), + ..Default::default() + }; settings.save(&dir.join("settings.json")).unwrap(); let (events, _rx) = RpcEventEmitter::new(); ProxyCtx::new(dir, events, keys).unwrap() @@ -436,6 +443,38 @@ fn waits_for_an_older_owner_before_converting() { } } +/// Launched while the CLI daemon captures, set to clear traffic at launch, the app keeps the +/// daemon's traffic: it is still being written, and is what taking over shows. +#[cfg(unix)] +#[test] +fn a_launch_under_another_owner_keeps_its_traffic() { + use yaak_proxy_lib::control::{ControlInfo, Owner, control_path}; + let dir = plain_capture(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let owner = ControlInfo { + pid: std::os::unix::process::parent_id(), + control_port: listener.local_addr().unwrap().port(), + proxy_port: None, + token: "tok".into(), + owner: Owner::Cli, + started_at: chrono::Utc::now(), + }; + let control = control_path(dir.path()); + std::fs::write(&control, serde_json::to_string(&owner).unwrap()).unwrap(); + let settings = ProxySettings { port: 0, system_proxy_at_launch: false, ..Default::default() }; + assert_eq!(settings.traffic(), TrafficRetention::ClearOnLaunch); + settings.save(&dir.path().join("settings.json")).unwrap(); + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + + // Taken over once the daemon lets go: every flow it captured is there. + drop(listener); + std::fs::remove_file(&control).unwrap(); + ctx.start().unwrap(); + assert_eq!(flows::count(&ctx.db().unwrap()).unwrap(), FLOWS); + ctx.stop().unwrap(); +} + /// The data key's file gone from beside encrypted databases is refused too, rather than a new /// key being made that could open nothing and would replace the trusted CA. #[test] diff --git a/crates-proxy/yaak-proxy-lib/tests/flow.rs b/crates-proxy/yaak-proxy-lib/tests/flow.rs index b08109fc1..f00815e71 100644 --- a/crates-proxy/yaak-proxy-lib/tests/flow.rs +++ b/crates-proxy/yaak-proxy-lib/tests/flow.rs @@ -337,6 +337,7 @@ async fn body_rpcs() { "systemProxyAtLaunch": false, "httpVersion": "auto", "captureBodyLimit": CAPTURE_LIMIT, + "traffic": "clearOnLaunch", "keepTrafficBytes": null, "startError": null, "encrypting": null, From 52d9c9596bd5a8c1034ce618498a33680a7117cc Mon Sep 17 00:00:00 2001 From: Gregory Schier Date: Mon, 9 Nov 2026 11:10:41 -0700 Subject: [PATCH 15/45] Guard the launch clear and system proxy watch against races A launch now clears captured traffic only when it can take traffic.lock alone, and every process with the databases open holds it shared, so an overlapping launch can't delete a database another has open. Changes the app decides to the system proxy are made under a lock the watch also takes to apply what it read, so a reading never overwrites them. Co-Authored-By: Claude Opus 5.5 --- crates-proxy/yaak-proxy-lib/src/lib.rs | 134 +++++++++++++++++++++++-- 1 file changed, 123 insertions(+), 11 deletions(-) diff --git a/crates-proxy/yaak-proxy-lib/src/lib.rs b/crates-proxy/yaak-proxy-lib/src/lib.rs index f03a6b84c..90eceaa73 100644 --- a/crates-proxy/yaak-proxy-lib/src/lib.rs +++ b/crates-proxy/yaak-proxy-lib/src/lib.rs @@ -109,6 +109,12 @@ pub struct ProxyCtx { /// Captured traffic is to be emptied as the databases are first unlocked, this launch /// being set to clear it. Taken once it is. clear_at_launch: Arc, + /// `traffic.lock`, locked shared for as long as this process has the databases open, so + /// another launch can't delete them from under it: clearing them takes it alone. + traffic_lock: Arc>>, + /// Held while `system_proxy_on` is set and changes are counted pending, and while the + /// watch applies what it read, so what it read never overwrites a change decided since. + system_proxy_intent: Arc>, /// The system proxy watch is running, so it is started once. watching_system_proxy: Arc, /// System proxy changes decided but not yet made, from when `system_proxy_on` is set until @@ -160,6 +166,8 @@ impl ProxyCtx { resets: Arc::new(AtomicU64::new(0)), sweeper: Default::default(), clear_at_launch: Arc::new(AtomicBool::new(false)), + traffic_lock: Arc::new(Mutex::new(None)), + system_proxy_intent: Arc::new(Mutex::new(())), watching_system_proxy: Arc::new(AtomicBool::new(false)), system_proxy_pending: Arc::new(AtomicU64::new(0)), events, @@ -269,7 +277,9 @@ impl ProxyCtx { let db_key = db::database_key(&key); // Before converting, so bodies about to be deleted aren't encrypted first. - let clearing = self.take_clear_at_launch(); + let clearing = self.lock_traffic(self.take_clear_at_launch()); + // Shared again once cleared, or on the way out should opening fail. + let _share = clearing.then(|| ShareOnDrop(&self.traffic_lock)); if clearing && let Err(e) = db::remove_blobs(&self.data_dir) { warn!("{e}"); } @@ -319,6 +329,41 @@ impl ProxyCtx { } } + /// Takes `traffic.lock` for this process's life, the first time the databases open: alone + /// when `clear` asks to empty them and no other process has them open, so the clear goes + /// ahead; shared otherwise, waiting out another launch's clear. Returns whether to clear. + /// A launch racing this one, before either has claimed the engine, then never deletes a + /// database the other already has open. + fn lock_traffic(&self, clear: bool) -> bool { + let mut held = self.traffic_lock.lock().unwrap_or_else(PoisonError::into_inner); + if held.is_some() { + return false; + } + let path = self.data_dir.join(TRAFFIC_LOCK); + let file = match std::fs::OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(&path) + { + Ok(file) => file, + Err(e) => { + warn!("Failed to open {}, keeping captured traffic: {e}", path.display()); + return false; + } + }; + let alone = clear && file.try_lock().is_ok(); + if clear && !alone { + log::info!("Keeping captured traffic: another process has it open"); + } + if !alone && let Err(e) = file.lock_shared() { + warn!("Failed to lock {}: {e}", path.display()); + } + *held = Some(file); + alone + } + /// Converts the plain databases, telling the app how far it has got a few times a second. fn encrypt_existing(&self, key: &yaak_database::encryption::DatabaseKey) -> Result<(), String> { let started = std::time::Instant::now(); @@ -486,9 +531,8 @@ impl ProxyCtx { *last_port = Some(bound); } let route = system_proxy.unwrap_or(settings.system_proxy_at_launch); - let pending = route.then(|| self.pending_system_proxy_change()); - self.system_proxy_on.store(route, Ordering::SeqCst); - if let Some(pending) = pending { + let (pending, _) = self.intend_system_proxy(route); + if route { // The password dialog can sit open for a while; the engine is up regardless. let ctx = self.detached(); std::thread::spawn(move || { @@ -534,8 +578,7 @@ impl ProxyCtx { /// Stops the engine and turns off a system proxy that points at it. pub fn stop(&self) -> Result<(), String> { - let pending = self.pending_system_proxy_change(); - self.system_proxy_on.store(false, Ordering::SeqCst); + let (pending, _) = self.intend_system_proxy(false); let port = self.take_handle()?; if let Some(port) = port { let ctx = self.detached(); @@ -744,7 +787,6 @@ impl ProxyCtx { /// password dialog. Not saved: the `systemProxyAtLaunch` setting decides whether it turns /// on as the engine starts. Blocks. pub fn set_system_proxy(&self, enabled: bool) -> Result { - let _pending = self.pending_system_proxy_change(); let Some(port) = self.port() else { if enabled { return Err("The proxy is not running, so there is nothing to route to".into()); @@ -753,7 +795,7 @@ impl ProxyCtx { self.change_system_proxy(false, self.settings()?.port, false)?; return self.system_proxy_status(); }; - let previous = self.system_proxy_on.swap(enabled, Ordering::SeqCst); + let (_pending, previous) = self.intend_system_proxy(enabled); if let Err(e) = self.change_system_proxy(enabled, port, enabled) { self.system_proxy_on.store(previous, Ordering::SeqCst); return Err(e); @@ -1158,10 +1200,14 @@ impl ProxyCtx { Ok(()) } - /// Marks a system proxy change as decided and not yet made, until the guard is dropped. - fn pending_system_proxy_change(&self) -> PendingChange { + /// Decides that this Mac's web traffic is meant to point at the running engine, or not, + /// returning what was meant before. The change counts as pending until the guard is + /// dropped, once it is made: until then the watch leaves `system_proxy_on` alone. + fn intend_system_proxy(&self, on: bool) -> (PendingChange, bool) { + let _intent = self.system_proxy_intent.lock().unwrap_or_else(PoisonError::into_inner); self.system_proxy_pending.fetch_add(1, Ordering::SeqCst); - PendingChange(self.system_proxy_pending.clone()) + let previous = self.system_proxy_on.swap(on, Ordering::SeqCst); + (PendingChange(self.system_proxy_pending.clone()), previous) } /// Follows the system proxy as macOS reports changes to it, for as long as the process @@ -1220,6 +1266,13 @@ impl ProxyCtx { }; match macos::routing(port) { Ok(routing) => { + // Applied only if nothing was decided, and the engine didn't move, while it was + // read: a change asked for meanwhile wins, and is read again once made. + let intent = + self.system_proxy_intent.lock().unwrap_or_else(PoisonError::into_inner); + if self.system_proxy_pending.load(Ordering::SeqCst) > 0 || self.port() != running { + return false; + } if running.is_some() { let was = self.system_proxy_on.swap(routing.every, Ordering::SeqCst); if was != routing.every { @@ -1227,6 +1280,7 @@ impl ProxyCtx { debug!("The system proxy is {now} at port {port} after a change"); } } + drop(intent); if !routing.any && let Err(e) = macos::forget(port, self.system_proxy_backend(), &self.snapshot_path()) @@ -1420,6 +1474,24 @@ fn remove_captured_when_closed(data_dir: &Path) -> Result<(), String> { } } +/// Beside the databases, locked shared by each process that has them open. +const TRAFFIC_LOCK: &str = "traffic.lock"; + +/// Turns the exclusive `traffic.lock` a launch took to clear back into a shared one as it is +/// dropped, so other processes can open the databases again. +struct ShareOnDrop<'a>(&'a Mutex>); + +impl Drop for ShareOnDrop<'_> { + fn drop(&mut self) { + let held = self.0.lock().unwrap_or_else(PoisonError::into_inner); + if let Some(file) = held.as_ref() + && let Err(e) = file.lock_shared() + { + warn!("Failed to share {TRAFFIC_LOCK}: {e}"); + } + } +} + /// A system proxy change decided and not yet made, counted while it is held. struct PendingChange(Arc); @@ -4870,6 +4942,46 @@ mod tests { } } + /// A launch that overlaps another, before either has claimed the engine, leaves the + /// databases the other has open alone, so what it writes is not lost. + #[test] + fn an_overlapping_launch_keeps_the_other_launch_s_traffic() { + let dir = tempfile::tempdir().unwrap(); + let keys = MasterKeySource::memory(); + let (events, _rx) = RpcEventEmitter::new(); + let first = ProxyCtx::new(dir.path(), events, keys.clone()).unwrap(); + flow_with_response(&first, "fl_a", "text/plain", b"one"); + + let (events, _rx) = RpcEventEmitter::new(); + let second = ProxyCtx::new(dir.path(), events, keys.clone()).unwrap(); + assert_eq!(flows::count(&second.db().unwrap()).unwrap(), 1); + // The first still writes bodies into the database both read. + flow_with_response(&first, "fl_b", "text/plain", b"two"); + assert_eq!(blobs_of(&second, "fl_a", 0), 1); + assert_eq!(blobs_of(&second, "fl_b", 0), 1); + + // Once both have let go, the next launch clears. + drop((first, second)); + let (events, _rx) = RpcEventEmitter::new(); + let third = ProxyCtx::new(dir.path(), events, keys).unwrap(); + assert_eq!(flows::count(&third.db().unwrap()).unwrap(), 0); + } + + /// A change the app decides while the watch reads the system proxy wins over what the + /// watch read, which is read again once the change is made. + #[test] + fn the_watch_never_overwrites_a_change_under_way() { + let dir = tempfile::tempdir().unwrap(); + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + let (pending, previous) = ctx.intend_system_proxy(true); + assert!(!previous); + assert!(!ctx.follow_system_proxy(), "left to try again"); + assert!(ctx.system_proxy_on.load(Ordering::SeqCst), "the change stands"); + drop(pending); + assert_eq!(ctx.system_proxy_pending.load(Ordering::SeqCst), 0); + } + /// Only a launch clears: unlocking again, after a reset, keeps what was captured since. #[test] fn only_the_first_unlock_clears() { From 3f3fd15959bb99ad76d7da4bd9856418ff22a0cd Mon Sep 17 00:00:00 2001 From: Gregory Schier Date: Mon, 9 Nov 2026 11:19:16 -0700 Subject: [PATCH 16/45] Release the clear's lock fully and clear conversion leftovers The exclusive traffic.lock a launch takes to clear is let go before it is taken shared, since Windows would otherwise hold both. Clearing the blob store also deletes what an interrupted encryption left beside it, so opening doesn't recover the old bodies. Co-Authored-By: Claude Opus 5.5 --- crates-proxy/yaak-proxy-lib/src/db.rs | 23 +++++++++++++++++++++-- crates-proxy/yaak-proxy-lib/src/lib.rs | 4 +++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/crates-proxy/yaak-proxy-lib/src/db.rs b/crates-proxy/yaak-proxy-lib/src/db.rs index 9c5ab4530..4a7c81ac4 100644 --- a/crates-proxy/yaak-proxy-lib/src/db.rs +++ b/crates-proxy/yaak-proxy-lib/src/db.rs @@ -271,9 +271,11 @@ pub fn remove_captured(data_dir: &Path) -> std::result::Result<(), String> { } /// Deletes `blobs.sqlite`, every body and message payload, for one made empty as it opens. -/// The data key stays, as `proxy.db` is still encrypted with it. Nothing may have it open. +/// What an interrupted conversion to encryption left beside it goes too, or opening would +/// recover it, bodies and all. The data key stays, as `proxy.db` is still encrypted with it. +/// Nothing may have it open. pub fn remove_blobs(data_dir: &Path) -> std::result::Result<(), String> { - for suffix in ["", "-wal", "-shm"] { + for suffix in ["", "-wal", "-shm", "-plain", "-encrypting"] { let path = data_dir.join(format!("{BLOBS_DB}{suffix}")); match std::fs::remove_file(&path) { Err(e) if e.kind() != std::io::ErrorKind::NotFound => { @@ -739,4 +741,21 @@ mod tests { let old = crate::flows::find(&db, "fl_old").unwrap(); assert!(!old.tunnelled); } + + /// Removing the blob store takes what an interrupted conversion left too, which opening + /// would otherwise recover, and leaves `proxy.db` and the key. + #[test] + fn removing_the_blobs_takes_a_conversion_s_leftovers() { + let dir = tempfile::tempdir().unwrap(); + let blobs = ["", "-wal", "-shm", "-plain", "-encrypting"].map(|s| format!("{BLOBS_DB}{s}")); + let kept = [PROXY_DB, DATA_KEY]; + for name in blobs.iter().map(String::as_str).chain(kept) { + std::fs::write(dir.path().join(name), b"x").unwrap(); + } + remove_blobs(dir.path()).unwrap(); + assert!(blobs.iter().all(|name| !dir.path().join(name).exists())); + assert!(kept.iter().all(|name| dir.path().join(name).exists())); + // Nothing there is fine too. + remove_blobs(dir.path()).unwrap(); + } } diff --git a/crates-proxy/yaak-proxy-lib/src/lib.rs b/crates-proxy/yaak-proxy-lib/src/lib.rs index 90eceaa73..e24b6f9a4 100644 --- a/crates-proxy/yaak-proxy-lib/src/lib.rs +++ b/crates-proxy/yaak-proxy-lib/src/lib.rs @@ -1484,8 +1484,10 @@ struct ShareOnDrop<'a>(&'a Mutex>); impl Drop for ShareOnDrop<'_> { fn drop(&mut self) { let held = self.0.lock().unwrap_or_else(PoisonError::into_inner); + // Let go first: Windows would otherwise add the shared lock beside the exclusive one. + // Unix's flock lets go between the two as well, so this is no wider a gap. if let Some(file) = held.as_ref() - && let Err(e) = file.lock_shared() + && let Err(e) = file.unlock().and_then(|()| file.lock_shared()) { warn!("Failed to share {TRAFFIC_LOCK}: {e}"); } From a43c3f7c2c78975a9defbbc57bf7e33e789f3cbb Mon Sep 17 00:00:00 2001 From: Gregory Schier Date: Mon, 9 Nov 2026 12:04:21 -0700 Subject: [PATCH 17/45] Take over a system proxy that comes back to the engine When routing returns to the running engine without the app setting it, as Proxyman leaves it in putting back what it found, the watch now records it as the app's: the helper is asked to point it there, which makes the app its owner, and the password and sudo paths save a snapshot for the port. Stopping or quitting then turns it off as it does when the app set it. Tests cover the exact sequence on both paths, with and without the takeover. Co-Authored-By: Claude Opus 5.5 --- crates-proxy/yaak-proxy-lib/src/helper.rs | 8 +- crates-proxy/yaak-proxy-lib/src/lib.rs | 10 ++ crates-proxy/yaak-proxy-lib/src/macos.rs | 116 +++++++++++++++++++ crates-tauri/yaak-proxy-helper/src/daemon.rs | 33 ++++++ 4 files changed, 166 insertions(+), 1 deletion(-) diff --git a/crates-proxy/yaak-proxy-lib/src/helper.rs b/crates-proxy/yaak-proxy-lib/src/helper.rs index 3ef2988df..7b836509c 100644 --- a/crates-proxy/yaak-proxy-lib/src/helper.rs +++ b/crates-proxy/yaak-proxy-lib/src/helper.rs @@ -191,6 +191,12 @@ mod imp { response.into_result() } + /// What the helper has done to the system proxy: whether it will restore it, and at which + /// port. + pub fn status() -> Result { + request(&Request::Get) + } + /// Points the system proxy at `port`, or puts back what was there, owned by this process. pub fn set(enabled: bool, port: u16) -> Result<(), String> { let owner_pid = std::process::id() as i32; @@ -199,7 +205,7 @@ mod imp { } #[cfg(target_os = "macos")] -pub use imp::{diagnose, register, set, state, unregister}; +pub use imp::{diagnose, register, set, state, status, unregister}; #[cfg(not(target_os = "macos"))] pub fn state() -> HelperState { diff --git a/crates-proxy/yaak-proxy-lib/src/lib.rs b/crates-proxy/yaak-proxy-lib/src/lib.rs index e24b6f9a4..cbec53467 100644 --- a/crates-proxy/yaak-proxy-lib/src/lib.rs +++ b/crates-proxy/yaak-proxy-lib/src/lib.rs @@ -1281,6 +1281,16 @@ impl ProxyCtx { } } drop(intent); + // Back on the running engine without the app setting it, as Proxyman leaves it + // in putting back what it found: recorded as the app's, so it is turned off as + // the engine stops or the app quits. + if running.is_some() + && routing.every + && let Err(e) = + macos::adopt(port, self.system_proxy_backend(), &self.snapshot_path()) + { + warn!("Failed to take over the system proxy pointing here: {e}"); + } if !routing.any && let Err(e) = macos::forget(port, self.system_proxy_backend(), &self.snapshot_path()) diff --git a/crates-proxy/yaak-proxy-lib/src/macos.rs b/crates-proxy/yaak-proxy-lib/src/macos.rs index 6224a5329..21ead6c06 100644 --- a/crates-proxy/yaak-proxy-lib/src/macos.rs +++ b/crates-proxy/yaak-proxy-lib/src/macos.rs @@ -207,6 +207,26 @@ fn forget_scripted( Ok(true) } +/// Records that the system proxy points at `port` for the app, when routing has come back to +/// it without the app setting it, as another app such as Proxyman does in putting back what it +/// found: the settings now there are saved as the ones to put back, with ours turned off, so +/// stopping or quitting turns it off. A snapshot already saved for `port` is kept. Never asks +/// for rights: it only writes the file. Returns whether anything was recorded. +#[cfg_attr(not(target_os = "macos"), allow(dead_code))] +fn adopt_scripted(ns: &impl Networksetup, port: u16, snapshot_path: &Path) -> Result { + let saved = load_snapshot(snapshot_path)?; + if saved.as_ref().is_some_and(|s| s.port == port) { + return Ok(false); + } + let services = read_services(ns, false)?; + // Settings parked for a missing service, or left at an old port, are carried along, as + // turning on does. + let mut snapshot = saved.unwrap_or_default(); + snapshot.extend(&services, port); + save_snapshot(snapshot_path, &snapshot)?; + Ok(true) +} + /// The calls as one `sh` script that stops at the first failure. #[cfg_attr(not(target_os = "macos"), allow(dead_code))] fn script(commands: &[Vec]) -> String { @@ -352,6 +372,30 @@ pub fn forget(port: u16, backend: SystemProxyBackend, snapshot_path: &Path) -> R } } +/// Records that the system proxy points at `port` for the app, when routing came back to it +/// without the app setting it, so stopping or quitting turns it off. The helper is asked to +/// point it there, as it already does, which records the app as its owner and restores it +/// should the app exit; the other paths write their file. Never asks for rights. Blocks. +#[cfg(target_os = "macos")] +pub fn adopt(port: u16, backend: SystemProxyBackend, snapshot_path: &Path) -> Result<(), String> { + match backend { + SystemProxyBackend::Helper => { + let status = crate::helper::status()?; + if status.active && status.port == Some(port) { + return Ok(()); + } + crate::helper::set(true, port) + } + SystemProxyBackend::AdminPrompt | SystemProxyBackend::Sudo => { + let ns = yaak_proxy_helper::networksetup::SystemNetworksetup; + if adopt_scripted(&ns, port, snapshot_path)? { + log::info!("The system proxy points at port {port} again; turning it off on quit"); + } + Ok(()) + } + } +} + /// Calls `changed`, on a thread of its own, each time macOS reports a change to a proxy /// setting: by this app, another one such as Proxyman, or the user in System Settings. Several /// may come for one change. Runs for as long as the process does. @@ -416,6 +460,15 @@ pub fn forget( Ok(()) } +#[cfg(not(target_os = "macos"))] +pub fn adopt( + _port: u16, + _backend: SystemProxyBackend, + _snapshot_path: &Path, +) -> Result<(), String> { + Ok(()) +} + /// No system proxy to watch here. #[cfg(not(target_os = "macos"))] pub fn watch(_changed: impl Fn() + Send + 'static) -> Result<(), String> { @@ -681,6 +734,69 @@ mod tests { assert!(ns.proxies().iter().all(|s| s.http == theirs.0 && s.https == theirs.1)); } + /// Proxyman turns its override on over ours, so ours is forgotten, then off again, putting + /// back what it found: ours. Quitting must still turn it off, or the Mac is left offline. + #[test] + fn quitting_after_another_app_restores_ours_turns_it_off() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(SNAPSHOT_FILE); + let ns = fake(); + let prompts = std::cell::Cell::new(0); + set_scripted(&ns, runner(&ns, &prompts), true, 9090, &path).unwrap(); + + // Proxyman on: it saves what it found, ours, and points everything at its port. + let found = ns.proxies(); + for s in &found { + ns.set_proxy(&s.name, setting(true, HOST, 9091), setting(true, HOST, 9091)); + } + assert!(forget_scripted(&ns, 9090, &path).unwrap()); + assert!(!path.exists()); + + // Proxyman off: it puts back what it found, so everything points at us again. + for s in &found { + ns.set_proxy(&s.name, s.http.clone(), s.https.clone()); + } + assert_eq!(routing_with(&ns, 9090).unwrap(), Routing { every: true, any: true }); + assert!(adopt_scripted(&ns, 9090, &path).unwrap(), "the app owns it again"); + assert_eq!(saved_port(&path), Some(9090)); + + // Quit. + set_scripted(&ns, runner(&ns, &prompts), false, 9090, &path).unwrap(); + assert!(ns.proxies().iter().all(|s| !s.points_at(9090)), "{:?}", ns.proxies()); + assert!(!path.exists()); + } + + /// The same, had nothing been recorded when routing came back: turning off at the port + /// still turns off whatever points there. + #[test] + fn turning_off_without_a_snapshot_still_turns_off_what_points_here() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(SNAPSHOT_FILE); + let ns = fake(); + for name in ["Wi-Fi", "Ethernet"] { + ns.set_proxy(name, setting(true, HOST, 9090), setting(true, HOST, 9090)); + } + let prompts = std::cell::Cell::new(0); + set_scripted(&ns, runner(&ns, &prompts), false, 9090, &path).unwrap(); + assert!(ns.proxies().iter().all(|s| !s.points_at(9090))); + assert_eq!(prompts.get(), 1); + } + + /// Adopting keeps a snapshot already saved for the port, and asks for nothing. + #[test] + fn adopting_keeps_what_was_saved_for_the_port() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(SNAPSHOT_FILE); + let ns = fake(); + let before = ns.proxies(); + let prompts = std::cell::Cell::new(0); + set_scripted(&ns, runner(&ns, &prompts), true, 9090, &path).unwrap(); + assert!(!adopt_scripted(&ns, 9090, &path).unwrap()); + set_scripted(&ns, runner(&ns, &prompts), false, 9090, &path).unwrap(); + assert_eq!(ns.proxies()[0], before[0], "the user's corp proxy came back"); + assert_eq!(prompts.get(), 2); + } + #[test] fn forgetting_keeps_settings_parked_for_a_missing_service() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates-tauri/yaak-proxy-helper/src/daemon.rs b/crates-tauri/yaak-proxy-helper/src/daemon.rs index 4d703c354..3e69d3e33 100644 --- a/crates-tauri/yaak-proxy-helper/src/daemon.rs +++ b/crates-tauri/yaak-proxy-helper/src/daemon.rs @@ -478,6 +478,39 @@ mod tests { daemon.ns.proxies().iter().all(|s| s.http.points_at(port) && s.https.points_at(port)) } + /// Proxyman turns its override on over the app's, so the app has the helper forget its + /// snapshot, then off, putting back what it found: the app's. Quitting must still turn + /// it off, whether or not the app took it over again in between. + #[test] + fn quitting_after_another_app_restores_ours_turns_it_off() { + for adopt in [true, false] { + let dir = tempfile::tempdir().unwrap(); + let daemon = daemon(dir.path()); + daemon.handle(&set(true, 9090)).into_result().unwrap(); + + let found = daemon.ns.proxies(); + for s in &found { + daemon.ns.set_proxy(&s.name, setting(true, HOST, 9091), setting(true, HOST, 9091)); + } + // Nothing points at 9090, so turning off there only forgets. + let status = daemon.handle(&set(false, 9090)).into_result().unwrap(); + assert!(!status.active); + assert!(all_point_at(&daemon, 9091), "Proxyman's settings left alone"); + + for s in &found { + daemon.ns.set_proxy(&s.name, s.http.clone(), s.https.clone()); + } + assert!(all_point_at(&daemon, 9090)); + if adopt { + let status = daemon.handle(&set(true, 9090)).into_result().unwrap(); + assert!(status.active && status.port == Some(9090)); + } + + daemon.handle(&set(false, 9090)).into_result().unwrap(); + assert!(daemon.ns.proxies().iter().all(|s| !s.points_at(9090)), "adopt {adopt}"); + } + } + #[test] fn set_then_off_restores_exactly() { let dir = tempfile::tempdir().unwrap(); From b5951b36aefa584c2eca13266649a0eb37180dc8 Mon Sep 17 00:00:00 2001 From: Gregory Schier Date: Tue, 10 Nov 2026 00:24:03 -0700 Subject: [PATCH 18/45] Flow list columns, multi-column sorting and header and cell menus (#833) Co-authored-by: Claude Opus 5.5 --- .../yaak-proxy/components/AddColumnDialog.tsx | 97 +++ apps/yaak-proxy/components/AppIcon.tsx | 8 +- apps/yaak-proxy/components/FlowInfo.tsx | 2 +- apps/yaak-proxy/components/FlowList.tsx | 373 +++++++++-- apps/yaak-proxy/components/FlowListHeader.tsx | 233 +++++-- apps/yaak-proxy/components/ProxyLayout.tsx | 4 + apps/yaak-proxy/components/SearchBar.tsx | 3 + apps/yaak-proxy/components/Sidebar.tsx | 2 +- apps/yaak-proxy/harness/FlowPanesHarness.tsx | 26 +- apps/yaak-proxy/harness/backend.ts | 31 +- apps/yaak-proxy/harness/fixtures.ts | 44 ++ apps/yaak-proxy/harness/main.tsx | 25 +- apps/yaak-proxy/harness/streamFixtures.ts | 1 + apps/yaak-proxy/lib/columns.test.ts | 271 ++++++-- apps/yaak-proxy/lib/columns.ts | 575 ++++++++++------ apps/yaak-proxy/lib/flowActions.ts | 62 +- apps/yaak-proxy/lib/flowViews.test.ts | 1 + apps/yaak-proxy/lib/follow.test.ts | 27 +- apps/yaak-proxy/lib/follow.ts | 25 + apps/yaak-proxy/lib/sortedWindow.test.ts | 89 +++ apps/yaak-proxy/lib/sortedWindow.ts | 97 +++ apps/yaak-proxy/lib/store.ts | 166 ++++- apps/yaak-proxy/lib/tauri.test.ts | 65 ++ apps/yaak-proxy/lib/tauri.ts | 20 +- .../yaak-proxy-lib/bindings/gen_models.ts | 5 + .../yaak-proxy-lib/bindings/gen_rpc.ts | 36 +- .../00000013_flow_graphql_operation.sql | 3 + crates-proxy/yaak-proxy-lib/src/convert.rs | 43 +- crates-proxy/yaak-proxy-lib/src/db.rs | 3 +- crates-proxy/yaak-proxy-lib/src/flows.rs | 297 ++++++++- crates-proxy/yaak-proxy-lib/src/lib.rs | 153 ++++- crates-proxy/yaak-proxy-lib/src/models.rs | 6 + crates-proxy/yaak-proxy-lib/src/query.rs | 64 +- .../yaak-proxy-lib/src/query/tests.rs | 68 +- crates-tauri/yaak-app-proxy/src/lib.rs | 5 + .../yaak-app-proxy/src/window_menu.rs | 9 + crates/yaak-http/src/graphql.rs | 612 ++++++++++++++++++ crates/yaak-http/src/lib.rs | 1 + 38 files changed, 3073 insertions(+), 479 deletions(-) create mode 100644 apps/yaak-proxy/components/AddColumnDialog.tsx create mode 100644 apps/yaak-proxy/lib/sortedWindow.test.ts create mode 100644 apps/yaak-proxy/lib/sortedWindow.ts create mode 100644 apps/yaak-proxy/lib/tauri.test.ts create mode 100644 crates-proxy/yaak-proxy-lib/migrations/00000013_flow_graphql_operation.sql create mode 100644 crates/yaak-http/src/graphql.rs diff --git a/apps/yaak-proxy/components/AddColumnDialog.tsx b/apps/yaak-proxy/components/AddColumnDialog.tsx new file mode 100644 index 000000000..2cba433ad --- /dev/null +++ b/apps/yaak-proxy/components/AddColumnDialog.tsx @@ -0,0 +1,97 @@ +import { Compartment, type Extension } from "@codemirror/state"; +import type { Schema } from "@yaakapp-internal/query"; +import { + Dialog, + DialogFooter, + Input, + type InputHandle, + LazyCodeEditor, +} from "@yaakapp-internal/ui"; +import { queryFilter } from "@yaakapp-internal/ui/editor"; +import { atom, useAtom, useSetAtom } from "jotai"; +import { useEffect, useMemo, useRef, useState } from "react"; +import { useRpcQuery } from "../hooks/useRpcQuery"; +import { addCustomColumn, customPath } from "../lib/columns"; +import { loadSeenValues } from "../lib/searchBox"; +import { columnPrefsAtom } from "../lib/store"; + +/** The Add Column dialog is open */ +export const addColumnDialogAtom = atom(false); + +const FORM_ID = "add-column"; + +/** + * Adds a column for a header or query parameter, named by its field as the search names it, + * `header.x-request-id` or `query.page`, completed as the search box completes it + */ +export function AddColumnDialog() { + const [open, setOpen] = useAtom(addColumnDialogAtom); + if (!open) return null; + return setOpen(false)} />; +} + +function AddColumnForm({ onClose }: { onClose: () => void }) { + const [text, setText] = useState(""); + const setPrefs = useSetAtom(columnPrefsAtom); + const handleRef = useRef(null); + const schema = useRpcQuery("query_schema", {}, { staleTime: Infinity }).data?.schema; + // Only the fields a column can show: the maps of headers and query parameters. It goes in + // once the schema is read, through a compartment, as the search box's does. + const compartment = useRef(new Compartment()); + const filter = useMemo(() => { + if (schema == null) return []; + const maps: Schema = { ...schema, fields: schema.fields.filter((f) => f.map) }; + return queryFilter({ schema: maps, loadValues: loadSeenValues }); + }, [schema]); + const extensions = useMemo(() => [compartment.current.of(filter)], [filter]); + useEffect(() => { + handleRef.current?.dispatch({ effects: compartment.current.reconfigure(filter) }); + }, [filter]); + const path = customPath(text); + + const submit = () => { + if (path == null) return; + setPrefs((p) => addCustomColumn(p, path)); + onClose(); + }; + + return ( + +
{ + e.preventDefault(); + submit(); + }} + > + } + setRef={(h) => { + handleRef.current = h; + }} + label="Field" + autoFocus + className="font-mono text-xs" + stateKey={null} + language={null} + placeholder="@header.x-request-id" + defaultValue="" + onChange={setText} + extensions={extensions} + onKeyDown={(e) => { + if (e.key === "Enter" && !e.defaultPrevented) { + e.preventDefault(); + submit(); + } + }} + /> + + +
+ ); +} diff --git a/apps/yaak-proxy/components/AppIcon.tsx b/apps/yaak-proxy/components/AppIcon.tsx index e132ee021..c90b0a824 100644 --- a/apps/yaak-proxy/components/AppIcon.tsx +++ b/apps/yaak-proxy/components/AppIcon.tsx @@ -24,8 +24,8 @@ function loadIcon(path: string): Promise { return promise; } -/** An app's icon, read once per path for the whole session */ -export function AppIcon({ path, className }: { path: string; className?: string }) { +/** An app's icon, read once per path for the whole session; `className` sets its size */ +export function AppIcon({ path, className = "size-4" }: { path: string; className?: string }) { const [url, setUrl] = useState(() => icons.get(path)); useEffect(() => { if (icons.has(path)) { @@ -42,8 +42,8 @@ export function AppIcon({ path, className }: { path: string; className?: string }, [path]); return url == null ? ( - + ) : ( - + ); } diff --git a/apps/yaak-proxy/components/FlowInfo.tsx b/apps/yaak-proxy/components/FlowInfo.tsx index 601f997fd..7286729d9 100644 --- a/apps/yaak-proxy/components/FlowInfo.tsx +++ b/apps/yaak-proxy/components/FlowInfo.tsx @@ -35,7 +35,7 @@ export function FlowInfo({ flow }: { flow: Flow }) { "Name", flow.clientName == null ? null : ( - {flow.clientPath != null && } + {flow.clientPath != null && } {flow.clientName} {folder != null && {folder}} diff --git a/apps/yaak-proxy/components/FlowList.tsx b/apps/yaak-proxy/components/FlowList.tsx index b4f42b7bb..949261c9d 100644 --- a/apps/yaak-proxy/components/FlowList.tsx +++ b/apps/yaak-proxy/components/FlowList.tsx @@ -1,10 +1,11 @@ import { useVirtualizer } from "@tanstack/react-virtual"; import { formatSize } from "@yaakapp-internal/lib/formatSize"; -import type { Flow } from "@yaakapp-internal/proxy-lib"; +import type { Flow, FlowSortKey, SortColumn } from "@yaakapp-internal/proxy-lib"; import { ContextMenu, CONTROL_TOOLTIP_DELAY, CountBadge, + type DropdownItem, formatMillis, HttpMethodTagRaw, HttpStatusTagRaw, @@ -12,7 +13,7 @@ import { Tooltip, } from "@yaakapp-internal/ui"; import classNames from "classnames"; -import { useAtom, useAtomValue, useSetAtom } from "jotai"; +import { useAtom, useAtomValue, useSetAtom, useStore } from "jotai"; import type { CSSProperties, KeyboardEvent, MouseEvent, ReactElement, WheelEvent } from "react"; import { memo, useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from "react"; import { selectedFlowAtom, selectedFlowIdAtom } from "../lib/atoms"; @@ -21,33 +22,49 @@ import { utcTimestamp } from "../lib/flowViews"; import { type Column, type ColumnId, + cellFilterTerm, + customValue, gridTemplate, gridWidth, hostOf, + isCustomColumn, nextSort, + operationOf, + showOperationFor, shownColumns, + tlsVersionOf, + toggleSortKey, + unavailableColumns, + urlParts, } from "../lib/columns"; import { useExportShownHar, useFlowActions } from "../lib/flowActions"; import { compareFlows } from "../lib/flowWindow"; -import { countNewFlows } from "../lib/follow"; +import { countNewFlows, isAtEnd, resumesFollowing } from "../lib/follow"; +import { asOneTerm } from "../lib/query"; import { folderOf } from "../lib/search"; import { columnPrefsAtom, hasAppsAtom, flowSortAtom, followAtom, + loadMoreSortedAtom, loadOlderFlowsAtom, newestStartedAtAtom, oldestFlowAtom, proxyStateAtom, + searchLoadingAtom, + searchTextAtom, + setSearchTextAtom, + flowsAtom, shownFlowsAtom, } from "../lib/store"; +import { useShowToast } from "../lib/toast"; import { AppIcon } from "./AppIcon"; import { FlowListEmpty, PausedRow } from "./FlowListEmpty"; import { FlowListHeader } from "./FlowListHeader"; const BOTTOM_PADDING = 6; -/** Rows from the top at which older flows start loading */ +/** Rows from the top at which older flows start loading, or from the bottom, sorted, more */ const LOAD_OLDER_ROWS = 20; /** How long a row that arrives while following stays tinted */ const ARRIVAL_MS = 1400; @@ -68,20 +85,37 @@ interface Props { /** * Follows new traffic until a flow is selected or the list is scrolled, then holds. Scrolling to * the top loads older flows from the database. Catching up leaves a divider where the hold - * began. Sorting by a column sorts the flows loaded and stops following. The columns keep their - * widths, and the list scrolls sideways when they are wider than it, the header with it. + * began. Sorting by columns reads the flows from the database in that order, a page at a time on + * scrolling down, and stops following. The columns keep their widths, and the list scrolls + * sideways when they are wider than it, the header with it. */ export function FlowList({ style, className }: Props) { const flows = useAtomValue(shownFlowsAtom); - const [menu, setMenu] = useState<{ flow: Flow; x: number; y: number } | null>(null); + const [menu, setMenu] = useState<{ + flow: Flow; + column: ColumnId | null; + x: number; + y: number; + } | null>(null); const flowActions = useFlowActions(); + const cellActions = useCellActions(); const exportShown = useExportShownHar(); const selectedId = useAtomValue(selectedFlowIdAtom); const setSelectedFlow = useSetAtom(selectedFlowAtom); const loadOlder = useSetAtom(loadOlderFlowsAtom); + const loadMoreSorted = useSetAtom(loadMoreSortedAtom); const oldestFlow = useAtomValue(oldestFlowAtom); - const prefs = useAtomValue(columnPrefsAtom); + const [prefs, setPrefs] = useAtom(columnPrefsAtom); const [sort, setSort] = useAtom(flowSortAtom); + const sorted = sort.length > 0; + const allFlows = useAtomValue(flowsAtom); + + // Operation shows itself the first time a flow has one, unless the person set it already. + useEffect(() => { + if (prefs.visible.operation !== undefined) return; + const next = showOperationFor(prefs, allFlows); + if (next !== prefs) setPrefs(next); + }, [prefs, allFlows, setPrefs]); const proxyState = useAtomValue(proxyStateAtom); const paused = proxyState?.state === "running" && proxyState.recording === false; const scrollRef = useRef(null); @@ -89,6 +123,8 @@ export function FlowList({ style, className }: Props) { const lastScrollTop = useRef(0); // The list's vertical scrollbar, which the header leaves room for so the two scroll as far const [scrollbarWidth, setScrollbarWidth] = useState(0); + // The list's visible height, so a resize that leaves it at its end follows again + const [viewportHeight, setViewportHeight] = useState(0); const ignoreScroll = useRef(false); const [following, setFollowing] = useAtom(followAtom); const [heldAt, setHeldAt] = useState(null); @@ -105,14 +141,11 @@ export function FlowList({ style, className }: Props) { // Where the platform cannot tell which app sent a flow, there is no App column. const hasApps = useAtomValue(hasAppsAtom); - const columns = useMemo( - () => shownColumns(prefs).filter((c) => hasApps || c.id !== "app"), - [prefs, hasApps], - ); + const columns = useMemo(() => shownColumns(prefs, unavailableColumns(hasApps)), [prefs, hasApps]); const template = useMemo(() => gridTemplate(columns), [columns]); // A row's grid, its padding and the list's margins: `px-1.5` and `left-1.5 right-1.5` const contentWidth = useMemo(() => `calc(${gridWidth(columns)} + 1.5rem)`, [columns]); - const arrivals = useArrivals(flows, following && sort == null); + const arrivals = useArrivals(flows, following && !sorted); const virtualizer = useVirtualizer({ count: flows.length, @@ -142,48 +175,83 @@ export function FlowList({ style, className }: Props) { const follow = useCallback(() => { const { flows, heldAt, following, sort } = latest.current; // Catching up from a hold marks where it began. - if (!following && sort == null) { + if (!following && sort.length === 0) { const count = countNewFlows(flows, heldAt); const first = flows[flows.length - count]; setDivider(count > 0 && first != null ? { id: first.id, count } : null); } - setSort(null); + setSort([]); setFollowing(true); setHeldAt(null); setScrollTop(Number.MAX_SAFE_INTEGER); }, [setFollowing, setScrollTop, setSort]); + // A sorted list never follows, whether the sort was just set, kept from the last launch, or + // kept through a clear. useLayoutEffect(() => { - if (following) setScrollTop(Number.MAX_SAFE_INTEGER); - }, [following, flows, paused, setScrollTop]); + if (sorted && following) setFollowing(false); + }, [sorted, following, setFollowing]); + + // A held list follows again once it is at its end, not only when scrolled there: a sidebar row, + // search or resize can leave it there with no scroll to make, as one that fits its viewport + // always is, and its flows would otherwise count as new while all are on screen. Judged when + // which flows are listed changes, or the list's height, not when one listed is updated or + // `following` itself changes, so selecting a row at the end still holds; and not while a + // search's first page loads, as its empty list is at its end only until the page is in. + const searchLoading = useAtomValue(searchLoadingAtom); + const listed = `${flows.length}:${flows[0]?.id ?? ""}:${flows.at(-1)?.id ?? ""}`; + useLayoutEffect(() => { + const el = scrollRef.current; + if (el == null || searchLoading) return; + if (!resumesFollowing({ following: latest.current.following, sorted, el })) return; + setFollowing(true); + setHeldAt(null); + }, [listed, sorted, searchLoading, viewportHeight, setFollowing]); + + useLayoutEffect(() => { + if (following && !sorted) setScrollTop(Number.MAX_SAFE_INTEGER); + }, [following, sorted, flows, paused, setScrollTop]); + + // Sorted, rows that leave, deleted or moved past what is read, can leave the end of the list + // in view with no scroll to load the next page; it loads as they go. + useEffect(() => { + const el = scrollRef.current; + if (!sorted || el == null) return; + const toEnd = el.scrollHeight - el.scrollTop - el.clientHeight; + if (toEnd < LOAD_OLDER_ROWS * ROW_HEIGHT) void loadMoreSorted(); + }, [sorted, flows, loadMoreSorted, ROW_HEIGHT]); // A new sort starts from its top. + const sortKey = JSON.stringify(sort); useLayoutEffect(() => { - if (sort != null) setScrollTop(0); - }, [sort, setScrollTop]); + if (sortKey !== "[]") setScrollTop(0); + }, [sortKey, setScrollTop]); // Older flows that loaded in above push the rows down; scroll by as much, so they stay put. const anchor = useRef(null); useLayoutEffect(() => { const previous = anchor.current; anchor.current = oldestFlow; - if (following || sort != null || previous == null || oldestFlow == null) return; + if (following || sorted || previous == null || oldestFlow == null) return; if (compareFlows(oldestFlow, previous) >= 0) return; const added = flows.findIndex((f) => compareFlows(f, previous) >= 0); const el = scrollRef.current; if (added > 0 && el != null) setScrollTop(el.scrollTop + added * ROW_HEIGHT); - }, [oldestFlow, flows, following, sort, setScrollTop, ROW_HEIGHT]); + }, [oldestFlow, flows, following, sorted, setScrollTop, ROW_HEIGHT]); // Sorted, the flows loaded are what is sorted; older ones would land anywhere. const loadOlderHeld = useCallback(() => { hold(); - if (latest.current.sort == null) void loadOlder(); + if (latest.current.sort.length === 0) void loadOlder(); }, [hold, loadOlder]); useEffect(() => { const el = scrollRef.current; if (el == null) return; - const measure = () => setScrollbarWidth(el.offsetWidth - el.clientWidth); + const measure = () => { + setScrollbarWidth(el.offsetWidth - el.clientWidth); + setViewportHeight(el.clientHeight); + }; measure(); const observer = new ResizeObserver(measure); observer.observe(el); @@ -203,17 +271,20 @@ export function FlowList({ style, className }: Props) { return; } if (!moved) return; - const atEnd = el.scrollHeight - el.scrollTop - el.clientHeight < 2; - if (atEnd && latest.current.sort == null) { + const sortedNow = latest.current.sort.length > 0; + if (isAtEnd(el) && !sortedNow) { setFollowing(true); setHeldAt(null); } else { hold(); - if (el.scrollTop < LOAD_OLDER_ROWS * ROW_HEIGHT && latest.current.sort == null) { + if (el.scrollTop < LOAD_OLDER_ROWS * ROW_HEIGHT && !sortedNow) { void loadOlder(); } } - }, [hold, loadOlder, setFollowing, ROW_HEIGHT]); + // Sorted, the next page of the sort loads near the bottom. + const toEnd = el.scrollHeight - el.scrollTop - el.clientHeight; + if (sortedNow && toEnd < LOAD_OLDER_ROWS * ROW_HEIGHT) void loadMoreSorted(); + }, [hold, loadOlder, loadMoreSorted, setFollowing, ROW_HEIGHT]); // Focus moving to a sort button out of view scrolls the header itself; the rows follow. const handleHeaderScroll = useCallback(() => { @@ -269,9 +340,12 @@ export function FlowList({ style, className }: Props) { const index = flows.findIndex((f) => f.id === selectedId); if (index === -1) select(e.key === "ArrowDown" ? 0 : flows.length - 1); else if (index === 0 && e.key === "ArrowUp") loadOlderHeld(); - else select(e.key === "ArrowDown" ? index + 1 : index - 1); + else { + if (sorted && index >= flows.length - 2 && e.key === "ArrowDown") void loadMoreSorted(); + select(e.key === "ArrowDown" ? index + 1 : index - 1); + } }, - [flows, follow, loadOlderHeld, select, selectedId], + [flows, follow, loadOlderHeld, loadMoreSorted, select, selectedId, sorted], ); const openMenu = useCallback( @@ -280,16 +354,21 @@ export function FlowList({ style, className }: Props) { if (flow == null) return; e.preventDefault(); select(index); - setMenu({ flow, x: e.clientX, y: e.clientY }); + // The cell under the pointer, for the menu's items on its value + const cellEl = (e.target as HTMLElement).closest("[data-column]"); + const column = (cellEl?.dataset.column as ColumnId | undefined) ?? null; + setMenu({ flow, column, x: e.clientX, y: e.clientY }); }, [select], ); - // A third click on a column goes back to start time, following as Sort by Start Time does. + // A click sorts by the column alone; a third goes back to arrival order, following as Reset + // Sort does. Option-click adds the column to the sort, reverses it, then takes it out. const handleSort = useCallback( - (column: ColumnId) => { - const next = nextSort(latest.current.sort, column); - if (next == null) { + (column: SortColumn, add: boolean) => { + const current = latest.current.sort; + const next = add ? toggleSortKey(current, column) : nextSort(current, column); + if (next.length === 0) { follow(); return; } @@ -299,7 +378,30 @@ export function FlowList({ style, className }: Props) { [follow, hold, setSort], ); - const newCount = following || sort != null ? 0 : countNewFlows(flows, heldAt); + const handleSetSort = useCallback( + (next: FlowSortKey[]) => { + if (next.length === 0) { + follow(); + return; + } + hold(); + setSort(next); + }, + [follow, hold, setSort], + ); + + const menuItems = useMemo((): DropdownItem[] => { + if (menu == null) return []; + // The cell's value first, as it is what was right-clicked + const cell = menu.column == null ? [] : cellActions(menu.flow, menu.column); + return [ + ...(cell.length > 0 ? [...cell, { type: "separator" } as const] : []), + ...flowActions(menu.flow), + exportShown(), + ]; + }, [menu, flowActions, cellActions, exportShown]); + + const newCount = following || sorted ? 0 : countNewFlows(flows, heldAt); const dividerIndex = divider == null ? -1 : flows.findIndex((f) => f.id === divider.id); const totalSize = virtualizer.getTotalSize(); @@ -326,7 +428,8 @@ export function FlowList({ style, className }: Props) { prefs={prefs} sort={sort} onSort={handleSort} - onSortByStart={follow} + onSetSort={handleSetSort} + onResetSort={follow} />
@@ -398,11 +501,7 @@ export function FlowList({ style, className }: Props) { {newCount === 1 ? "1 new flow" : `${newCount} new flows`} )} - setMenu(null)} - /> + setMenu(null)} />
); @@ -475,8 +574,9 @@ const FlowRow = memo(function FlowRow({ // Failed or cancelled before any response; after the headers, the body was cut short mid-way. const noResponse = (failed || flow.cancelled) && flow.status == null; const ids = columns.map((c) => c.id); - // A failure's reason, or Cancelled, takes the type and size cells together. - const spanReason = noResponse && ids.includes("type") && ids.includes("size"); + // A failure's reason, or Cancelled, takes the type and size cells together, side by side. + const typeAt = ids.indexOf("type"); + const spanReason = noResponse && typeAt !== -1 && ids[typeAt + 1] === "size"; const sinceArrival = arrivedAt == null ? null : Date.now() - arrivedAt; return ( @@ -510,7 +610,10 @@ const FlowRow = memo(function FlowRow({ {columns.map((c) => { if (spanReason && c.id === "size") return null; return ( - + // Names the cell's column for its menu, without a box of its own in the row's grid +
+ +
); })}
@@ -585,9 +688,7 @@ function FlowCell({ column, flow, span }: { column: ColumnId; flow: Flow; span: switch (column) { case "number": return ( - - {flow.number || ""} - + {flow.number || ""} ); case "app": { if (flow.clientPath == null) return ; @@ -601,7 +702,7 @@ function FlowCell({ column, flow, span }: { column: ColumnId; flow: Flow; span: delay={CONTROL_TOOLTIP_DELAY} > - +
); @@ -632,26 +733,20 @@ function FlowCell({ column, flow, span }: { column: ColumnId; flow: Flow; span: if (flow.tunnelled) { return Not decrypted; } - const { path, query } = splitPath(flow.url); + const path = pathOf(flow.url); if (flow.streamKind == null) { return ( - - - {path} - {query} - + + {path} ); } // A stream's message count rides the end of the path, so it survives the narrow width, // ticking while the stream is open. return ( - + - - {path} - {query} - + {path} ); + case "query": { + const query = flow.tunnelled ? "" : (urlParts(flow.url).query ?? ""); + return ( + + {query} + + ); + } + case "started": { + const at = new Date(utcTimestamp(flow.startedAt)); + if (Number.isNaN(at.getTime())) return ; + return ( + + {clockTime(at)} + + ); + } + case "requestSize": + return ( + + {flow.requestSize === 0 ? "--" : formatSize(flow.requestSize)} + + ); + case "latency": + return ( + + {flow.elapsedHeadersMs == null ? "--" : formatMillis(flow.elapsedHeadersMs)} + + ); + case "tls": + return ( + {tlsVersionOf(flow)} + ); + case "operation": { + const operation = operationOf(flow) ?? ""; + return ( + + {operation} + + ); + } + default: { + if (!isCustomColumn(column)) return ; + const value = customValue(flow, column.slice("field:".length)) ?? ""; + return ( + + {value} + + ); + } } } -function splitPath(url: string): { path: string; query: string } { +/** The URL's path as the browser reads it, `/` for none, without the query */ +function pathOf(url: string): string { try { - const u = new URL(url); - return { path: u.pathname, query: u.search }; + return new URL(url).pathname; } catch { - return { path: url, query: "" }; + return url; + } +} + +/** `14:03:27.481`, in local time */ +function clockTime(at: Date): string { + const two = (n: number) => String(n).padStart(2, "0"); + const ms = String(at.getMilliseconds()).padStart(3, "0"); + return `${two(at.getHours())}:${two(at.getMinutes())}:${two(at.getSeconds())}.${ms}`; +} + +/** + * The items a cell's menu adds for its value: a search term for it, either way, where its column + * has a field to match it by, and copying it + */ +function useCellActions(): (flow: Flow, column: ColumnId) => DropdownItem[] { + const store = useStore(); + const setSearchText = useSetAtom(setSearchTextAtom); + const showToast = useShowToast(); + return useCallback( + (flow: Flow, column: ColumnId) => { + const add = (term: string) => { + // A search with `OR` at its top is grouped first, so the term narrows all of it. + const text = store.get(searchTextAtom).trim(); + setSearchText(text === "" ? term : `${asOneTerm(text)} ${term}`); + }; + const include = cellFilterTerm(flow, column, false); + const exclude = cellFilterTerm(flow, column, true); + const value = cellText(flow, column); + const items: DropdownItem[] = []; + if (include != null && exclude != null) { + items.push( + { label: "Filter by This Value", icon: "filter", onSelect: () => add(include) }, + { label: "Exclude This Value", icon: "minus_circle", onSelect: () => add(exclude) }, + ); + } + if (value !== "") { + items.push({ + label: "Copy Value", + icon: "copy", + onSelect: async () => { + try { + await navigator.clipboard.writeText(value); + showToast({ message: "Copied value" }); + } catch (err) { + showToast({ message: `Failed to copy value: ${String(err)}`, color: "danger" }); + } + }, + }); + } + return items; + }, + [store, setSearchText, showToast], + ); +} + +/** A cell's value as text, as it shows it, for copying */ +function cellText(flow: Flow, column: ColumnId): string { + if (isCustomColumn(column)) return customValue(flow, column.slice("field:".length)) ?? ""; + switch (column) { + case "number": + return flow.number ? String(flow.number) : ""; + case "app": + return flow.clientName ?? flow.clientPath ?? ""; + case "method": + return flow.method; + case "host": + return hostOf(flow.url); + case "path": + return flow.tunnelled ? "" : pathOf(flow.url); + case "query": + return flow.tunnelled ? "" : (urlParts(flow.url).query ?? ""); + case "status": + return flow.status == null ? "" : String(flow.status); + case "type": + return flow.contentType ?? ""; + case "size": + return String(flow.responseSize); + case "requestSize": + return String(flow.requestSize); + case "time": + return flow.elapsedMs == null ? "" : String(flow.elapsedMs); + case "latency": + return flow.elapsedHeadersMs == null ? "" : String(flow.elapsedHeadersMs); + case "started": { + const at = new Date(utcTimestamp(flow.startedAt)); + return Number.isNaN(at.getTime()) ? "" : at.toISOString(); + } + case "remote": + return flow.remoteAddr ?? ""; + case "version": + return flow.upstreamHttpVersion ?? flow.clientHttpVersion; + case "tls": + return tlsVersionOf(flow); + case "operation": + return operationOf(flow) ?? ""; } } diff --git a/apps/yaak-proxy/components/FlowListHeader.tsx b/apps/yaak-proxy/components/FlowListHeader.tsx index 61bcb8847..65033f37c 100644 --- a/apps/yaak-proxy/components/FlowListHeader.tsx +++ b/apps/yaak-proxy/components/FlowListHeader.tsx @@ -1,3 +1,4 @@ +import type { FlowSortKey, SortColumn } from "@yaakapp-internal/proxy-lib"; import { ContextMenu, Dropdown, @@ -7,21 +8,27 @@ import { ResizeHandle, } from "@yaakapp-internal/ui"; import classNames from "classnames"; -import { useSetAtom } from "jotai"; +import { useAtomValue, useSetAtom } from "jotai"; import { memo, type MouseEvent, useCallback, useRef, useState } from "react"; import { + addSortKey, + allColumns, type Column, - COLUMNS, + type ColumnDef, type ColumnId, type ColumnPrefs, - type FlowSort, + canHide, + hideOtherColumns, + isCustomColumn, + removeCustomColumn, resetWidth, resetWidths, resizeColumn, toggleColumn, + unavailableColumns, } from "../lib/columns"; -import { useExportShownHar } from "../lib/flowActions"; -import { columnPrefsAtom } from "../lib/store"; +import { columnPrefsAtom, hasAppsAtom } from "../lib/store"; +import { addColumnDialogAtom } from "./AddColumnDialog"; interface Props { columns: Column[]; @@ -29,16 +36,19 @@ interface Props { /** The rows' width, as CSS, so the header scrolls sideways with them */ width: string; prefs: ColumnPrefs; - sort: FlowSort | null; - onSort: (column: ColumnId) => void; - onSortByStart: () => void; + sort: FlowSortKey[]; + /** A click on a label, or with `add`, an Option-click */ + onSort: (column: SortColumn, add: boolean) => void; + /** Sets the sort from the menus; empty goes back to arrival order */ + onSetSort: (sort: FlowSortKey[]) => void; + onResetSort: () => void; } /** - * The list's only control surface: a click on a label sorts by it, a second reverses it and a - * third goes back to start time. Dragging the edge after a label resizes its column and a - * double-click on it resets it. The picker sits in the empty corner on hover, or anywhere on a - * right-click. + * The list's only control surface: a click on a label sorts by it alone, a second reverses it + * and a third goes back to arrival order; Option-click adds it to the sort as a further key. + * Dragging the edge after a label resizes its column and a double-click on it resets it. A + * right-click on a label opens the column's menu; the picker sits in the empty corner on hover. */ export const FlowListHeader = memo(function FlowListHeader({ columns, @@ -47,49 +57,113 @@ export const FlowListHeader = memo(function FlowListHeader({ prefs, sort, onSort, - onSortByStart, + onSetSort: setSort, + onResetSort, }: Props) { const setPrefs = useSetAtom(columnPrefsAtom); - const exportShown = useExportShownHar(); - const [menu, setMenu] = useState<{ x: number; y: number } | null>(null); + const unavailable = unavailableColumns(useAtomValue(hasAppsAtom)); + const openAddColumn = useSetAtom(addColumnDialogAtom); + const [menu, setMenu] = useState<{ x: number; y: number; column: ColumnDef | null } | null>(null); // Widths when a drag began, so each move sets the width from where it started. const startWidth = useRef(0); - const items: DropdownItem[] = [ + const pickerItems: DropdownItem[] = [ { type: "content", label:
Columns
, }, - ...COLUMNS.map((c): DropdownItem => ({ - label: c.label, - keepOpenOnSelect: true, - disabled: c.id === "path", - leftSlot: - (prefs.visible[c.id] ?? c.defaultVisible) || c.id === "path" ? ( + ...allColumns(prefs).map((c): DropdownItem => { + const shown = prefs.visible[c.id] ?? c.defaultVisible; + return { + label: c.label, + keepOpenOnSelect: true, + disabled: shown && !canHide(prefs, c.id, unavailable), + leftSlot: shown ? ( ) : ( ), - rightSlot: - c.id === "path" ? always : undefined, - onSelect: () => setPrefs((p) => toggleColumn(p, c.id)), - })), + onSelect: () => setPrefs((p) => toggleColumn(p, c.id, unavailable)), + ...(isCustomColumn(c.id) + ? { + submenuTrigger: "button" as const, + submenu: [ + { + label: "Remove Column", + icon: "trash" as const, + color: "danger" as const, + onSelect: () => setPrefs((p) => removeCustomColumn(p, c.id as `field:${string}`)), + }, + ], + } + : {}), + }; + }), + { label: "Add Column…", icon: "plus", onSelect: () => openAddColumn(true) }, { type: "separator" }, - { label: "Sort by Start Time", disabled: sort == null, onSelect: onSortByStart }, + { label: "Reset Sort", disabled: sort.length === 0, onSelect: onResetSort }, { label: "Reset Column Widths", onSelect: () => setPrefs((p) => resetWidths(p)) }, - { type: "separator" }, - exportShown(), ]; - const openMenu = useCallback((e: MouseEvent) => { - e.preventDefault(); - setMenu({ x: e.clientX, y: e.clientY }); - }, []); + const columnItems = (c: ColumnDef): DropdownItem[] => { + const column = c.sortable ? (c.id as SortColumn) : null; + const sortBy = (descending: boolean) => { + if (column != null) setSort([{ column, descending }]); + }; + const addTo = (descending: boolean) => { + if (column != null) setSort(addSortKey(sort, column, descending)); + }; + return [ + ...(column == null + ? [] + : ([ + { label: "Sort Ascending", onSelect: () => sortBy(false) }, + { label: "Sort Descending", onSelect: () => sortBy(true) }, + { label: "Add to Sort Ascending", onSelect: () => addTo(false) }, + { label: "Add to Sort Descending", onSelect: () => addTo(true) }, + ] satisfies DropdownItem[])), + { label: "Clear Sorting", disabled: sort.length === 0, onSelect: onResetSort }, + { type: "separator" }, + { + label: "Hide Column", + disabled: !canHide(prefs, c.id, unavailable), + onSelect: () => setPrefs((p) => toggleColumn(p, c.id, unavailable)), + }, + { + label: "Hide Other Columns", + disabled: columns.length <= 1, + onSelect: () => setPrefs((p) => hideOtherColumns(p, c.id)), + }, + ...(isCustomColumn(c.id) + ? [ + { + label: "Remove Column", + color: "danger" as const, + onSelect: () => setPrefs((p) => removeCustomColumn(p, c.id as `field:${string}`)), + }, + ] + : []), + { type: "separator" }, + { label: "Reset Column Widths", onSelect: () => setPrefs((p) => resetWidths(p)) }, + ]; + }; + + // A right-click on a label is for its column; anywhere else in the header, the picker. + const openMenu = useCallback( + (e: MouseEvent) => { + e.preventDefault(); + const el = (e.target as HTMLElement).closest("[data-column]"); + const id = el?.dataset.column as ColumnId | undefined; + const column = columns.find((c) => c.id === id) ?? null; + setMenu({ x: e.clientX, y: e.clientY, column }); + }, + [columns], + ); return ( // The same margins and padding as the rows, so each label sits over its cells.
- {/* oxlint-disable-next-line jsx-a11y/no-static-element-interactions -- a right-click opens the same menu as the picker */} + {/* oxlint-disable-next-line jsx-a11y/no-static-element-interactions -- a right-click opens the column's menu */}
- + - {columns.map((c) => ( - - - {/* On the column's right edge, the boundary with the next, resizing this one */} - { - startWidth.current = c.current; - }} - onResizeMove={(e) => - setPrefs((p) => resizeColumn(p, c.id, startWidth.current + e.x - e.xStart)) - } - onReset={() => setPrefs((p) => resetWidth(p, c.id))} - /> - - ))} - setMenu(null)} /> + + ); + const labelClass = classNames( + "flex items-center gap-0.5 min-w-0 w-full h-full whitespace-nowrap", + c.align === "right" && "justify-end", + key != null && "text-text-subtle", + ); + return ( + + {c.sortable ? ( + + ) : ( + {label} + )} + {/* On the column's right edge, the boundary with the next, resizing this one */} + { + startWidth.current = c.current; + }} + onResizeMove={(e) => + setPrefs((p) => resizeColumn(p, c.id, startWidth.current + e.x - e.xStart)) + } + onReset={() => setPrefs((p) => resetWidth(p, c.id))} + /> + + ); + })} + setMenu(null)} + />
); diff --git a/apps/yaak-proxy/components/ProxyLayout.tsx b/apps/yaak-proxy/components/ProxyLayout.tsx index 4f4ffd8be..0cd38b228 100644 --- a/apps/yaak-proxy/components/ProxyLayout.tsx +++ b/apps/yaak-proxy/components/ProxyLayout.tsx @@ -12,8 +12,10 @@ import { useState } from "react"; import { useLocalStorage } from "react-use"; import { flowLayoutAtom } from "../lib/atoms"; import { setupIncomplete } from "../lib/firstRun"; +import { useMenuExportHar } from "../lib/flowActions"; import { lockedAtom, noFlowsAtom, systemProxyEverOnAtom } from "../lib/store"; import { getOsType } from "../lib/tauri"; +import { AddColumnDialog } from "./AddColumnDialog"; import { ClearFlowsButton } from "./ClearFlowsButton"; import { FirstRun } from "./FirstRun"; import { FlowDetail } from "./FlowDetail"; @@ -32,6 +34,7 @@ export function ProxyLayout() { const os = getOsType(); const [flowLayout, setFlowLayout] = useAtom(flowLayoutAtom); const openSettings = useOpenSettings(); + useMenuExportHar(); const [sidebarWidth, setSidebarWidth] = useLocalStorage("sidebar_width", 250); const [sidebarHidden, setSidebarHidden] = useLocalStorage("sidebar_hidden", false); const [floatingSidebarHidden, setFloatingSidebarHidden] = useLocalStorage( @@ -164,6 +167,7 @@ export function ProxyLayout() { + diff --git a/apps/yaak-proxy/components/SearchBar.tsx b/apps/yaak-proxy/components/SearchBar.tsx index ff7dba372..fd0a1c0f6 100644 --- a/apps/yaak-proxy/components/SearchBar.tsx +++ b/apps/yaak-proxy/components/SearchBar.tsx @@ -23,6 +23,7 @@ import { useState, } from "react"; import { useRpcQuery } from "../hooks/useRpcQuery"; +import { useExportShownHar } from "../lib/flowActions"; import { loadSeenValues, savedCompletions, searchProblems } from "../lib/searchBox"; import { allTrafficTotalAtom, @@ -54,6 +55,7 @@ export const SearchBar = memo(function SearchBar() { const total = useAtomValue(allTrafficTotalAtom); const store = useStore(); const setSavedDialog = useSetAtom(savedSearchDialogAtom); + const exportShown = useExportShownHar(); // Nothing to search while captured traffic is locked. const locked = useAtomValue(lockedAtom) != null; const schema = useRpcQuery("query_schema", {}, { staleTime: Infinity }).data?.schema; @@ -179,6 +181,7 @@ export const SearchBar = memo(function SearchBar() { leftSlot: , onSelect: () => setSavedDialog({ mode: "save", query: text.trim() }), }, + exportShown(), ]} > } {item.kind === "saved" && } {item.kind === "folder" && } - {item.appPath != null && } + {item.appPath != null && } {item.kind === "folder" && item.path != null ? ( // Named by its last part; the whole path on hover diff --git a/apps/yaak-proxy/harness/FlowPanesHarness.tsx b/apps/yaak-proxy/harness/FlowPanesHarness.tsx index dc01632b0..e24dcada3 100644 --- a/apps/yaak-proxy/harness/FlowPanesHarness.tsx +++ b/apps/yaak-proxy/harness/FlowPanesHarness.tsx @@ -1,5 +1,6 @@ import { useSetAtom } from "jotai"; -import { useLayoutEffect, useState } from "react"; +import { useEffect, useLayoutEffect, useState } from "react"; +import { AddColumnDialog } from "../components/AddColumnDialog"; import { FlowDetail } from "../components/FlowDetail"; import { FlowList } from "../components/FlowList"; import { selectedFlowAtom } from "../lib/atoms"; @@ -34,6 +35,7 @@ export function FlowPanesHarness() { ))}
+
); } @@ -54,6 +56,28 @@ function FixtureCard({ fixture }: { fixture: FlowFixture }) { setReady(true); // oxlint-disable-next-line react-hooks/exhaustive-deps -- once per fixture }, [fixture]); + + // `&arrive=3`: that many new flows, one a second, as traffic would bring them + useEffect(() => { + const count = Number(params.get("arrive") ?? 0); + let added = 0; + const timer = setInterval(() => { + if (added >= count) return clearInterval(timer); + added++; + const source = fixtures[added % fixtures.length]?.flow; + if (source == null) return; + const flow = { + ...source, + // The fixture's id first, so the backend finds its bodies, as `manyFlows` does + id: `${source.id}~arrived_${added}`, + number: 1500 + added, + startedAt: new Date().toISOString().slice(0, 23), + }; + setWindow((w) => ({ ...w, flows: [...w.flows, flow], total: w.total + 1 })); + }, 1000); + return () => clearInterval(timer); + // oxlint-disable-next-line react-hooks/exhaustive-deps -- once per fixture + }, [fixture]); if (!ready) return null; if (params.has("list")) { diff --git a/apps/yaak-proxy/harness/backend.ts b/apps/yaak-proxy/harness/backend.ts index 391268241..92f001571 100644 --- a/apps/yaak-proxy/harness/backend.ts +++ b/apps/yaak-proxy/harness/backend.ts @@ -1,5 +1,5 @@ import type { Flow, GetProxyStateResponse } from "@yaakapp-internal/proxy-lib"; -import { fixtures } from "./fixtures"; +import { appIconPng, fixtures } from "./fixtures"; /** * Dev only: answers the engine's RPCs the harness pages need, from fixtures, so the card and @@ -13,8 +13,9 @@ export function installFixtureBackend() { if (cmd.startsWith("plugin:event|")) return 0; throw new Error(`Not in the harness: ${cmd}`); }; - (window as unknown as { __TAURI_OS_PLUGIN_INTERNALS__: Record }) - .__TAURI_OS_PLUGIN_INTERNALS__ = { + ( + window as unknown as { __TAURI_OS_PLUGIN_INTERNALS__: Record } + ).__TAURI_OS_PLUGIN_INTERNALS__ = { os_type: "macos", platform: "macos", family: "unix", @@ -58,7 +59,7 @@ function answer(cmd: string, payload: unknown): unknown { } } case "app_icon": - return null; + return appIconPng; case "get_proxy_state": return HARNESS_STATE; case "get_storage_size": @@ -127,11 +128,29 @@ function answer(cmd: string, payload: unknown): unknown { info: "A request or response header", example: "@header.accept:*json*", }, + { + name: "res.header", + kind: { type: "text" }, + map: true, + info: "A response header", + example: "@res.header.x-cache:hit", + }, + { + name: "query", + kind: { type: "text" }, + map: true, + info: "A query parameter", + example: "@query.page:2", + }, ], }, }; - case "query_values": - return []; + // Header and query parameter names, for completing a map's keys + case "query_values": { + const path = (payload as { path?: string } | undefined)?.path ?? ""; + if (path.endsWith("header")) return ["x-cache", "x-request-id", "content-type"]; + return path === "query" ? ["expand", "limit"] : []; + } case "validate_query": return []; default: diff --git a/apps/yaak-proxy/harness/fixtures.ts b/apps/yaak-proxy/harness/fixtures.ts index cc90cf443..7368f528c 100644 --- a/apps/yaak-proxy/harness/fixtures.ts +++ b/apps/yaak-proxy/harness/fixtures.ts @@ -50,6 +50,7 @@ const base: Flow = { grpcMethod: null, grpcStatus: null, grpcMessage: null, + graphqlOperation: null, tunnelled: false, agentRead: false, clientAddr: "127.0.0.1:54321", @@ -69,6 +70,10 @@ const jsonResponse = JSON.stringify({ status: "succeeded", charges: { data: [{ id: "ch_1", paid: true }] }, }); +const graphqlRequest = JSON.stringify({ + query: "query GetViewer { viewer { login } }", + operationName: "GetViewer", +}); const formRequest = "grant_type=client_credentials&client_id=yaak-proxy&scope=read+write"; const htmlResponse = "

Signed in

You can close this window.

"; @@ -176,4 +181,43 @@ export const fixtures: FlowFixture[] = [ requestBody: body(""), responseBody: body(""), }, + { + label: "GraphQL POST", + flow: { + ...base, + id: "fl_graphql", + method: "POST", + url: "https://api.github.com/graphql", + contentType: "application/json", + requestHeaders: [{ name: "Content-Type", value: "application/json" }], + responseHeaders: [{ name: "X-Cache", value: "MISS" }], + requestSize: graphqlRequest.length, + responseSize: 64, + graphqlOperation: "GetViewer", + }, + requestBody: body(graphqlRequest), + responseBody: body('{"data":{"viewer":{"login":"yaak"}}}'), + }, + { + label: "gRPC call", + flow: { + ...base, + id: "fl_grpc", + method: "POST", + url: "https://grpc.example.com/pkg.Greeter/SayHello", + contentType: "application/grpc", + clientHttpVersion: "HTTP/2", + upstreamHttpVersion: "HTTP/2", + requestHeaders: [{ name: "Content-Type", value: "application/grpc" }], + responseHeaders: [{ name: "Content-Type", value: "application/grpc" }], + grpcService: "pkg.Greeter", + grpcMethod: "SayHello", + grpcStatus: 0, + }, + requestBody: body(""), + responseBody: body(""), + }, ]; + +/** An app icon for every flow in the harness, as base64 PNG */ +export const appIconPng = btoa(String.fromCharCode(...png)); diff --git a/apps/yaak-proxy/harness/main.tsx b/apps/yaak-proxy/harness/main.tsx index 8fa7d599a..e22ce2ceb 100644 --- a/apps/yaak-proxy/harness/main.tsx +++ b/apps/yaak-proxy/harness/main.tsx @@ -1,6 +1,7 @@ import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { applyThemeToDocument, defaultDarkTheme, defaultLightTheme } from "@yaakapp-internal/theme"; import { createStore, Provider } from "jotai"; +import { LazyMotion, MotionConfig } from "motion/react"; import { StrictMode } from "react"; import { createRoot } from "react-dom/client"; import { installFixtureBackend } from "./backend"; @@ -11,6 +12,8 @@ import { SettingsHarness } from "./SettingsHarness"; import "../main.css"; const params = new URLSearchParams(location.search); +// As the app loads them, so dialogs fade in +const motionFeatures = () => import("framer-motion").then((mod) => mod.domAnimation); const dark = params.get("theme") !== "light"; applyThemeToDocument(dark ? defaultDarkTheme : defaultLightTheme); if (params.has("locked")) installLockedBackend(); @@ -20,15 +23,19 @@ createRoot(document.getElementById("root") as HTMLElement).render( - {params.has("locked") || params.has("window") ? ( - - ) : params.has("settings") ? ( - - ) : params.has("stream") ? ( - - ) : ( - - )} + + + {params.has("locked") || params.has("window") ? ( + + ) : params.has("settings") ? ( + + ) : params.has("stream") ? ( + + ) : ( + + )} + + , diff --git a/apps/yaak-proxy/harness/streamFixtures.ts b/apps/yaak-proxy/harness/streamFixtures.ts index 47fa878b7..73d9d29be 100644 --- a/apps/yaak-proxy/harness/streamFixtures.ts +++ b/apps/yaak-proxy/harness/streamFixtures.ts @@ -49,6 +49,7 @@ const flowBase: Flow = { grpcMethod: null, grpcStatus: null, grpcMessage: null, + graphqlOperation: null, tunnelled: false, agentRead: false, clientAddr: null, diff --git a/apps/yaak-proxy/lib/columns.test.ts b/apps/yaak-proxy/lib/columns.test.ts index 6703b589d..5f4120a10 100644 --- a/apps/yaak-proxy/lib/columns.test.ts +++ b/apps/yaak-proxy/lib/columns.test.ts @@ -1,35 +1,59 @@ -import type { Flow } from "@yaakapp-internal/proxy-lib"; +import type { Flow, FlowSortKey } from "@yaakapp-internal/proxy-lib"; import { describe, expect, test } from "vite-plus/test"; import { + addCustomColumn, + addSortKey, + cellFilterTerm, type ColumnPrefs, + customPath, + customValue, defaultColumnPrefs, gridTemplate, gridWidth, + hideOtherColumns, nextSort, + operationOf, parseColumnPrefs, + removeCustomColumn, resetWidth, resetWidths, resizeColumn, + showOperationFor, shownColumns, sortFlows, toggleColumn, + toggleSortKey, + urlParts, } from "./columns"; function flow(number: number, extra: Partial = {}): Flow { return { - id: `fl_${number}`, + id: `fl_${String(number).padStart(3, "0")}`, number, + startedAt: `2026-11-09T12:00:${String(number).padStart(2, "0")}`, url: "https://example.com/", method: "GET", status: 200, error: null, + requestSize: 0, responseSize: 0, elapsedMs: null, + elapsedHeadersMs: null, contentType: null, clientName: null, clientBundleId: null, clientPath: null, clientCwd: null, + clientHttpVersion: "HTTP/1.1", + upstreamHttpVersion: null, + upstreamTlsVersion: null, + remoteAddr: null, + requestHeaders: [], + responseHeaders: [], + grpcService: null, + grpcMethod: null, + graphqlOperation: null, + tunnelled: false, ...extra, } as Flow; } @@ -37,13 +61,14 @@ function flow(number: number, extra: Partial = {}): Flow { const ids = (prefs: ColumnPrefs) => shownColumns(prefs).map((c) => c.id); describe("shownColumns", () => { - test("shows every visible column, however narrow the list", () => { + test("shows every visible column, Query after Path", () => { expect(ids(defaultColumnPrefs)).toEqual([ "number", "app", "method", "host", "path", + "query", "status", "type", "size", @@ -51,29 +76,33 @@ describe("shownColumns", () => { ]); }); - test("follows the picker, and always keeps the path", () => { + test("follows the picker, Path included, but keeps the last column", () => { let prefs = toggleColumn(defaultColumnPrefs, "remote"); prefs = toggleColumn(prefs, "type"); prefs = toggleColumn(prefs, "path"); expect(ids(prefs)).toContain("remote"); expect(ids(prefs)).not.toContain("type"); - expect(ids(prefs)).toContain("path"); + expect(ids(prefs)).not.toContain("path"); + const alone = hideOtherColumns(defaultColumnPrefs, "host"); + expect(ids(alone)).toEqual(["host"]); + expect(toggleColumn(alone, "host")).toBe(alone); + // Where App has nothing to show, the last other column is the last one. + const withApp = hideOtherColumns(defaultColumnPrefs, "app"); + const both = toggleColumn(withApp, "host"); + expect(toggleColumn(both, "host", ["app"])).toBe(both); + expect(ids(both)).toEqual(["app", "host"]); + expect(shownColumns(both, ["app"]).map((c) => c.id)).toEqual(["host"]); }); - test("lays out every column at a fixed width, the path included", () => { + test("lays out every column at a fixed width", () => { let prefs = resizeColumn(defaultColumnPrefs, "host", 240.4); prefs = resizeColumn(prefs, "path", 500); const columns = shownColumns(prefs); const pad = (w: number) => `calc(${w}px + 0.75rem)`; expect(gridTemplate(columns)).toBe( - ["24px", ...[46, 18, 50, 240, 500, 46, 50, 72, 64].map(pad)].join(" "), + ["24px", ...[46, 20, 50, 240, 500, 120, 46, 50, 72, 64].map(pad)].join(" "), ); - expect(gridWidth(columns)).toBe("calc(1110px + 9 * 0.75rem)"); - }); - - test("keeps the host's width in a narrow list", () => { - const host = shownColumns(defaultColumnPrefs).find((c) => c.id === "host"); - expect(host?.current).toBe(190); + expect(gridWidth(columns)).toBe("calc(1232px + 10 * 0.75rem)"); }); }); @@ -81,11 +110,32 @@ describe("column preferences", () => { test("persist through storage and drop what no longer applies", () => { let prefs = resizeColumn(defaultColumnPrefs, "host", 10); prefs = toggleColumn(prefs, "number"); + prefs = addCustomColumn(prefs, "header.x-cache"); + prefs = { ...prefs, sort: [{ column: "status", descending: true }] }; const stored = JSON.parse(JSON.stringify(prefs)); - expect(parseColumnPrefs(stored)).toEqual({ visible: { number: false }, widths: { host: 60 } }); + expect(parseColumnPrefs(stored)).toEqual({ + visible: { number: false, "field:header.x-cache": true }, + widths: { host: 60 }, + custom: ["field:header.x-cache"], + sort: [{ column: "status", descending: true }], + }); expect( - parseColumnPrefs({ visible: { gone: true, size: "yes" }, widths: { time: -1, size: 90 } }), - ).toEqual({ visible: {}, widths: { size: 90 } }); + parseColumnPrefs({ + visible: { gone: true, size: "yes" }, + widths: { time: -1, size: 90 }, + custom: ["field:nope.x", 3, "field:query.page"], + sort: [ + { column: "gone", descending: true }, + { column: "host" }, + { column: "time", descending: false }, + ], + }), + ).toEqual({ + visible: {}, + widths: { size: 90 }, + custom: ["field:query.page"], + sort: [{ column: "time", descending: false }], + }); expect(parseColumnPrefs(null)).toEqual(defaultColumnPrefs); expect(resetWidths(prefs).widths).toEqual({}); }); @@ -95,61 +145,162 @@ describe("column preferences", () => { prefs = resizeColumn(prefs, "path", 400); expect(resetWidth(prefs, "path").widths).toEqual({ host: 300 }); }); + + test("custom columns are added at the end and removed with their settings", () => { + let prefs = addCustomColumn(defaultColumnPrefs, "res.header.x-cache"); + prefs = resizeColumn(prefs, "field:res.header.x-cache", 200); + expect(ids(prefs).at(-1)).toBe("field:res.header.x-cache"); + prefs = removeCustomColumn(prefs, "field:res.header.x-cache"); + expect(prefs).toEqual(defaultColumnPrefs); + }); + + test("Operation shows itself once, unless the person set it", () => { + const grpc = flow(1, { grpcService: "pkg.Greeter", grpcMethod: "SayHello" }); + const anonymous = flow(2, { graphqlOperation: "" }); + expect(showOperationFor(defaultColumnPrefs, [flow(3), anonymous])).toBe(defaultColumnPrefs); + expect(showOperationFor(defaultColumnPrefs, [grpc]).visible.operation).toBe(true); + const hidden = { ...defaultColumnPrefs, visible: { operation: false } }; + expect(showOperationFor(hidden, [grpc])).toBe(hidden); + }); }); describe("sorting", () => { - test("sorts numbers largest first and names from A, reverses, then goes back to start", () => { - expect(nextSort(null, "size")).toEqual({ column: "size", descending: true }); - expect(nextSort(null, "host")).toEqual({ column: "host", descending: false }); - expect(nextSort({ column: "host", descending: false }, "host")).toEqual({ - column: "host", - descending: true, - }); - expect(nextSort({ column: "host", descending: true }, "host")).toBeNull(); - expect(nextSort({ column: "size", descending: true }, "size")).toEqual({ - column: "size", - descending: false, - }); - expect(nextSort({ column: "size", descending: false }, "size")).toBeNull(); - // Another column starts over. - expect(nextSort({ column: "host", descending: true }, "size")).toEqual({ - column: "size", - descending: true, - }); + const one = (column: FlowSortKey["column"], descending: boolean) => [{ column, descending }]; + + test("a click sorts by a column alone, reverses it, then goes back to arrival order", () => { + expect(nextSort([], "size")).toEqual(one("size", true)); + expect(nextSort([], "host")).toEqual(one("host", false)); + expect(nextSort(one("host", false), "host")).toEqual(one("host", true)); + expect(nextSort(one("host", true), "host")).toEqual([]); + expect(nextSort(one("size", true), "size")).toEqual(one("size", false)); + expect(nextSort(one("size", false), "size")).toEqual([]); + // Another column, or a sort on several, starts over. + expect(nextSort(one("host", true), "size")).toEqual(one("size", true)); + const two = [...one("host", false), ...one("size", true)]; + expect(nextSort(two, "host")).toEqual(one("host", false)); }); - test("sorts the loaded flows, those without a value lowest, ties in arrival order", () => { - const a = flow(1, { responseSize: 10, elapsedMs: 5, url: "https://b.com/" }); - const b = flow(2, { responseSize: 300, elapsedMs: null, url: "https://a.com/" }); - const c = flow(3, { responseSize: 10, elapsedMs: 50, url: "https://c.com/" }); - const d = flow(4, { error: "DNS lookup failed", status: null, url: "https://a.com/" }); - const flows = [a, b, c, d]; - const by = (column: Parameters[1], descending: boolean) => - sortFlows(flows, { column, descending }).map((f) => f.number); + test("Option-click adds a key ascending, reverses it, then takes it out", () => { + let sort = toggleSortKey(one("status", true), "host"); + expect(sort).toEqual([...one("status", true), ...one("host", false)]); + sort = toggleSortKey(sort, "host"); + expect(sort).toEqual([...one("status", true), ...one("host", true)]); + expect(toggleSortKey(sort, "host")).toEqual(one("status", true)); + expect(addSortKey(sort, "status", false)).toEqual([ + ...one("status", false), + ...one("host", true), + ]); + expect(addSortKey([], "time", true)).toEqual(one("time", true)); + }); + + // The same flows and orders as `a_sort_on_several_columns_pages_in_its_order` in the lib's + // `flows.rs`, so the app places a flow where the database would. + const flows = [ + flow(0, { url: "https://B.com/x", status: 200 }), + flow(1, { url: "https://a.com/y", status: 404 }), + flow(2, { url: "https://a.com/z", status: null }), + flow(3, { url: "https://c.com/", status: 200 }), + flow(4, { url: "https://a.com/", status: null, error: "boom" }), + flow(5, { url: "https://b.com/q", status: 200 }), + flow(6, { url: "https://A.com/", status: 404 }), + flow(7, { url: "https://c.com/", status: 200 }), + ]; + const numbers = (sort: FlowSortKey[]) => sortFlows(flows, sort).map((f) => f.number); + + test("sorts on several columns as the database does", () => { + const status = [...one("status", true), ...one("host", false)]; + expect(numbers(status)).toEqual([1, 6, 0, 5, 3, 7, 4, 2]); + const ascending = [...one("status", false), ...one("host", false)]; + expect(numbers(ascending)).toEqual([2, 4, 0, 5, 3, 7, 1, 6]); + expect(numbers(one("query", true))).toEqual([0, 1, 2, 3, 4, 5, 6, 7]); + expect(sortFlows(flows, [])).toBe(flows); + }); + + test("sizes, times and the rest put flows without a value lowest", () => { + const a = flow(1, { responseSize: 10, elapsedMs: 5, requestSize: 3, elapsedHeadersMs: 2 }); + const b = flow(2, { responseSize: 300, elapsedMs: null, requestSize: 0 }); + const c = flow(3, { responseSize: 10, elapsedMs: 50, elapsedHeadersMs: 40 }); + const d = flow(4, { error: "DNS lookup failed", status: null, responseSize: 99 }); + const four = [a, b, c, d]; + const by = (column: FlowSortKey["column"], descending: boolean) => + sortFlows(four, one(column, descending)).map((f) => f.number); expect(by("size", true)).toEqual([2, 1, 3, 4]); expect(by("size", false)).toEqual([4, 1, 3, 2]); expect(by("time", true)).toEqual([3, 1, 2, 4]); - expect(by("time", false)).toEqual([2, 4, 1, 3]); - expect(by("host", false)).toEqual([2, 4, 1, 3]); + expect(by("latency", true)).toEqual([3, 1, 2, 4]); + expect(by("requestSize", true)).toEqual([1, 2, 3, 4]); expect(by("number", true)).toEqual([4, 3, 2, 1]); // Failures sort below every status. expect(by("status", true)).toEqual([1, 2, 3, 4]); expect(by("status", false)).toEqual([4, 1, 2, 3]); - expect(sortFlows(flows, null)).toBe(flows); - }); - - test("flips flows with no status, type, size or time to the other end", () => { - const done = flow(1, { status: 200, contentType: "text/html", responseSize: 5, elapsedMs: 9 }); - const pending = flow(2, { status: null, contentType: null, responseSize: 0, elapsedMs: null }); - const cancelled = flow(3, { status: null, cancelled: true, contentType: "" }); - const flows = [done, pending, cancelled]; - for (const column of ["status", "type", "size", "time"] as const) { - expect(sortFlows(flows, { column, descending: true }).map((f) => f.number)).toEqual([ - 1, 2, 3, - ]); - expect(sortFlows(flows, { column, descending: false }).map((f) => f.number)).toEqual([ - 2, 3, 1, - ]); - } + }); +}); + +describe("values", () => { + test("splits URLs as the lib does", () => { + expect(urlParts("https://a.com:8080/v1/x?page=2&q=1#top")).toEqual({ + authority: "a.com:8080", + path: "/v1/x", + query: "page=2&q=1", + }); + expect(urlParts("https://a.com?x")).toEqual({ authority: "a.com", path: "", query: "x" }); + expect(urlParts("bank.example.com:443")).toEqual({ + authority: "bank.example.com:443", + path: "", + query: null, + }); + }); + + test("reads a custom column's path from what was typed", () => { + expect(customPath("@header.X-Request-Id:")).toBe("header.x-request-id"); + expect(customPath("query.page")).toBe("query.page"); + expect(customPath(" res.header.x-cache ")).toBe("res.header.x-cache"); + expect(customPath("@host")).toBeNull(); + expect(customPath("header.")).toBeNull(); + expect(customPath("header.a b")).toBeNull(); + expect(customPath("query.filter[status]")).toBeNull(); + }); + + test("a custom column shows the first value, the response's header before the request's", () => { + const f = flow(1, { + url: "https://a.com/?Page=2&page=3", + requestHeaders: [ + { name: "X-Cache", value: "req" }, + { name: "Accept", value: "a" }, + { name: "accept", value: "b" }, + ], + responseHeaders: [{ name: "x-cache", value: "HIT" }], + }); + expect(customValue(f, "header.x-cache")).toBe("HIT"); + expect(customValue(f, "req.header.x-cache")).toBe("req"); + expect(customValue(f, "header.accept")).toBe("a"); + expect(customValue(f, "query.page")).toBe("2"); + expect(customValue(f, "query.nope")).toBeNull(); + }); + + test("a cell's filter terms, where its column has a field to match", () => { + const f = flow(1, { + url: "https://api.stripe.com:443/v1/charges?x=1", + status: 404, + clientName: "Google Chrome", + contentType: "application/json; charset=utf-8", + responseHeaders: [{ name: "X-Cache", value: "HIT" }], + }); + expect(cellFilterTerm(f, "host", false)).toBe("@host:api.stripe.com"); + expect(cellFilterTerm(f, "status", true)).toBe("-@status:404"); + expect(cellFilterTerm(f, "app", false)).toBe('@app:"Google Chrome"'); + expect(cellFilterTerm(f, "path", false)).toBe("@path:/v1/charges"); + expect(cellFilterTerm(f, "type", false)).toBe("@type:application/json"); + expect(cellFilterTerm(f, "field:header.x-cache", false)).toBe("@header.x-cache:HIT"); + for (const column of ["number", "size", "time", "query", "started"] as const) { + expect(cellFilterTerm(f, column, false)).toBeNull(); + } + const grpc = flow(2, { grpcService: "pkg.Greeter", grpcMethod: "SayHello" }); + expect(operationOf(grpc)).toBe("pkg.Greeter/SayHello"); + expect(cellFilterTerm(grpc, "operation", true)).toBe( + "-(@grpc.service:pkg.Greeter @grpc.method:SayHello)", + ); + const graphql = flow(3, { graphqlOperation: "GetUser" }); + expect(cellFilterTerm(graphql, "operation", false)).toBe("@graphql.operation:GetUser"); }); }); diff --git a/apps/yaak-proxy/lib/columns.ts b/apps/yaak-proxy/lib/columns.ts index 1492b7b07..b7f0e4492 100644 --- a/apps/yaak-proxy/lib/columns.ts +++ b/apps/yaak-proxy/lib/columns.ts @@ -1,18 +1,15 @@ -import type { Flow } from "@yaakapp-internal/proxy-lib"; -import { appKeyOf } from "./search"; +import type { Flow, FlowSortKey, SortColumn } from "@yaakapp-internal/proxy-lib"; +import { compareFlows } from "./flowWindow"; +import { fieldTerm } from "./query"; +import { sidebarPath } from "./search"; -export type ColumnId = - | "number" - | "app" - | "method" - | "host" - | "path" - | "status" - | "type" - | "size" - | "time" - | "remote" - | "version"; +/** The columns the list has built in; each sorts by the `SortColumn` of the same name */ +export type BuiltinColumnId = SortColumn; + +/** A column of a header or query parameter the person added, by its field's path */ +export type CustomColumnId = `field:${string}`; + +export type ColumnId = BuiltinColumnId | CustomColumnId; export interface ColumnDef { id: ColumnId; @@ -24,102 +21,81 @@ export interface ColumnDef { align: "left" | "right"; /** Shown until the person hides it */ defaultVisible: boolean; + /** Sorted by in the database; custom columns are not */ + sortable: boolean; + /** The header or query parameter a custom column shows: `header.x-request-id` */ + path?: string; } +const builtin = ( + id: BuiltinColumnId, + label: string, + width: number, + minWidth: number, + align: "left" | "right", + defaultVisible: boolean, +): ColumnDef => ({ id, label, width, minWidth, align, defaultVisible, sortable: true }); + export const COLUMNS: ColumnDef[] = [ - { - id: "number", - label: "#", - width: 46, - minWidth: 30, - align: "right", - defaultVisible: true, - }, - { - id: "app", - label: "App", - width: 18, - minWidth: 18, - align: "left", - defaultVisible: true, - }, - { - id: "method", - label: "Method", - width: 50, + builtin("number", "#", 46, 30, "left", true), + builtin("app", "App", 20, 20, "left", true), + builtin("method", "Method", 50, 40, "left", true), + builtin("host", "Host", 190, 60, "left", true), + builtin("path", "Path", 300, 60, "left", true), + builtin("query", "Query", 120, 40, "left", true), + builtin("status", "Status", 46, 34, "left", true), + builtin("type", "Type", 50, 34, "left", true), + builtin("requestSize", "Request Size", 80, 44, "right", false), + builtin("size", "Size", 72, 44, "right", true), + builtin("time", "Time", 64, 44, "right", true), + builtin("latency", "Latency", 64, 44, "right", false), + builtin("started", "Started", 92, 60, "left", false), + builtin("remote", "Remote Address", 150, 60, "left", false), + builtin("version", "HTTP Version", 70, 44, "left", false), + builtin("tls", "TLS Version", 70, 34, "left", false), + builtin("operation", "Operation", 160, 60, "left", false), +]; + +const DEFS = new Map(COLUMNS.map((c) => [c.id, c])); + +/** Map fields a custom column can show, as the query language names them */ +const CUSTOM_MAPS = ["req.header.", "res.header.", "header.", "query."]; + +/** `header.x-request-id` from what was typed, `@header.X-Request-Id:` included, or null */ +export function customPath(typed: string): string | null { + const path = typed.trim().replace(/^@/, "").replace(/:$/, "").trim(); + const map = CUSTOM_MAPS.find((m) => path.toLowerCase().startsWith(m)); + if (map == null) return null; + const key = path.slice(map.length); + // As a field path takes it, so the column's filter items make a term the search can run + if (!/^[A-Za-z0-9_-]+$/.test(key)) return null; + return map + key.toLowerCase(); +} + +export function customColumnId(path: string): CustomColumnId { + return `field:${path}`; +} + +export function isCustomColumn(id: ColumnId): id is CustomColumnId { + return id.startsWith("field:"); +} + +function customDef(id: CustomColumnId): ColumnDef { + const path = id.slice("field:".length); + return { + id, + label: path, + width: 140, minWidth: 40, align: "left", defaultVisible: true, - }, - { - id: "host", - label: "Host", - width: 190, - minWidth: 60, - align: "left", - defaultVisible: true, - }, - { - id: "path", - label: "Path", - width: 360, - minWidth: 80, - align: "left", - defaultVisible: true, - }, - { - id: "status", - label: "Status", - width: 46, - minWidth: 34, - align: "left", - defaultVisible: true, - }, - { - id: "type", - label: "Type", - width: 50, - minWidth: 34, - align: "left", - defaultVisible: true, - }, - { - id: "size", - label: "Size", - width: 72, - minWidth: 44, - align: "right", - defaultVisible: true, - }, - { - id: "time", - label: "Time", - width: 64, - minWidth: 44, - align: "right", - defaultVisible: true, - }, - { - id: "remote", - label: "Remote Address", - width: 150, - minWidth: 60, - align: "left", - defaultVisible: false, - }, - { - id: "version", - label: "HTTP Version", - width: 70, - minWidth: 44, - align: "left", - defaultVisible: false, - }, -]; - -const DEFS = new Map(COLUMNS.map((c) => [c.id, c])); + sortable: false, + path, + }; +} export function columnDef(id: ColumnId): ColumnDef { + if (isCustomColumn(id)) return customDef(id); const def = DEFS.get(id); if (def == null) throw new Error(`No column ${id}`); return def; @@ -131,27 +107,48 @@ export const MARK_WIDTH = 24; /** Each cell's padding, both sides together, added to its column's width; `px-1.5` per side */ export const CELL_PADDING = "0.75rem"; -/** What the person chose: which columns show, and the widths they dragged, by id */ +/** + * What the person chose: which columns show, the widths they dragged, by id, the header and + * query parameter columns they added, in order, and the list's sort + */ export interface ColumnPrefs { visible: Partial>; widths: Partial>; + custom: CustomColumnId[]; + /** The keys in order; empty for the order flows arrive in */ + sort: FlowSortKey[]; } -export const defaultColumnPrefs: ColumnPrefs = { visible: {}, widths: {} }; +export const defaultColumnPrefs: ColumnPrefs = { visible: {}, widths: {}, custom: [], sort: [] }; export interface Column extends ColumnDef { /** Its width now, in pixels of content */ current: number; } +/** Every column the picker lists: the built-in ones, then those the person added */ +export function allColumns(prefs: ColumnPrefs): ColumnDef[] { + return [...COLUMNS, ...prefs.custom.map(customDef)]; +} + +function isVisible(prefs: ColumnPrefs, def: ColumnDef): boolean { + return prefs.visible[def.id] ?? def.defaultVisible; +} + /** - * The columns shown, in order: the visible ones, the path always. Each keeps its width however - * narrow the list is; the list scrolls sideways instead. + * The columns shown, in order. Each keeps its width however narrow the list is; the list scrolls + * sideways instead. At least one always shows. */ -export function shownColumns(prefs: ColumnPrefs): Column[] { - return COLUMNS.filter((c) => c.id === "path" || (prefs.visible[c.id] ?? c.defaultVisible)).map( - (c) => ({ ...c, current: columnWidth(prefs, c.id) }), - ); +export function shownColumns(prefs: ColumnPrefs, unavailable: ColumnId[] = []): Column[] { + const all = allColumns(prefs).filter((c) => !unavailable.includes(c.id)); + const shown = all.filter((c) => isVisible(prefs, c)); + const columns = shown.length > 0 ? shown : all.filter((c) => c.id === "path"); + return columns.map((c) => ({ ...c, current: columnWidth(prefs, c.id) })); +} + +/** Columns the platform has nothing for: App, where it cannot tell which app sent a flow */ +export function unavailableColumns(hasApps: boolean): ColumnId[] { + return hasApps ? [] : ["app"]; } export function columnWidth(prefs: ColumnPrefs, id: ColumnId): number { @@ -171,12 +168,42 @@ export function gridWidth(columns: Column[]): string { return `calc(${px}px + ${columns.length} * ${CELL_PADDING})`; } -export function toggleColumn(prefs: ColumnPrefs, id: ColumnId): ColumnPrefs { - if (id === "path") return prefs; - const shown = prefs.visible[id] ?? columnDef(id).defaultVisible; +/** Whether the picker may uncheck `id`: any column but the last one shown */ +export function canHide(prefs: ColumnPrefs, id: ColumnId, unavailable: ColumnId[] = []): boolean { + const shown = allColumns(prefs).filter((c) => !unavailable.includes(c.id) && isVisible(prefs, c)); + return !shown.every((c) => c.id === id); +} + +export function toggleColumn( + prefs: ColumnPrefs, + id: ColumnId, + unavailable: ColumnId[] = [], +): ColumnPrefs { + const shown = isVisible(prefs, columnDef(id)); + if (shown && !canHide(prefs, id, unavailable)) return prefs; return { ...prefs, visible: { ...prefs.visible, [id]: !shown } }; } +/** Shows `id` alone */ +export function hideOtherColumns(prefs: ColumnPrefs, id: ColumnId): ColumnPrefs { + const visible: ColumnPrefs["visible"] = {}; + for (const c of allColumns(prefs)) visible[c.id] = c.id === id; + return { ...prefs, visible }; +} + +/** Adds a column for a header or query parameter's path, shown at the end; already there, shown */ +export function addCustomColumn(prefs: ColumnPrefs, path: string): ColumnPrefs { + const id = customColumnId(path); + const custom = prefs.custom.includes(id) ? prefs.custom : [...prefs.custom, id]; + return { ...prefs, custom, visible: { ...prefs.visible, [id]: true } }; +} + +export function removeCustomColumn(prefs: ColumnPrefs, id: CustomColumnId): ColumnPrefs { + const { [id]: _v, ...visible } = prefs.visible; + const { [id]: _w, ...widths } = prefs.widths; + return { ...prefs, custom: prefs.custom.filter((c) => c !== id), visible, widths }; +} + export function resizeColumn(prefs: ColumnPrefs, id: ColumnId, width: number): ColumnPrefs { const w = Math.max(columnDef(id).minWidth, Math.round(width)); return { ...prefs, widths: { ...prefs.widths, [id]: w } }; @@ -192,109 +219,285 @@ export function resetWidth(prefs: ColumnPrefs, id: ColumnId): ColumnPrefs { return { ...prefs, widths }; } +/** Shows Operation the first time a flow has one, unless the person ever set it themselves */ +export function showOperationFor(prefs: ColumnPrefs, flows: Flow[]): ColumnPrefs { + if (prefs.visible.operation !== undefined) return prefs; + if (!flows.some((f) => (operationOf(f) ?? "") !== "")) return prefs; + return { ...prefs, visible: { ...prefs.visible, operation: true } }; +} + /** * Reads stored preferences, dropping what no longer applies: unknown ids, and values of the * wrong type, so a change to the columns never breaks the list. */ export function parseColumnPrefs(value: unknown): ColumnPrefs { - const prefs: ColumnPrefs = { visible: {}, widths: {} }; + const prefs: ColumnPrefs = { visible: {}, widths: {}, custom: [], sort: [] }; if (value == null || typeof value !== "object") return prefs; - const { visible, widths } = value as Record; - for (const def of COLUMNS) { + const { visible, widths, custom, sort } = value as Record; + if (Array.isArray(custom)) { + for (const id of custom) { + if (typeof id !== "string" || !id.startsWith("field:")) continue; + const path = customPath(id.slice("field:".length)); + if (path != null && !prefs.custom.includes(customColumnId(path))) { + prefs.custom.push(customColumnId(path)); + } + } + } + for (const def of allColumns(prefs)) { const v = (visible as Record | undefined)?.[def.id]; if (typeof v === "boolean") prefs.visible[def.id] = v; const w = (widths as Record | undefined)?.[def.id]; if (typeof w === "number" && Number.isFinite(w) && w > 0) prefs.widths[def.id] = w; } + if (Array.isArray(sort)) { + for (const key of sort) { + if (key == null || typeof key !== "object") continue; + const { column, descending } = key as Record; + if (typeof column !== "string" || typeof descending !== "boolean") continue; + const def = DEFS.get(column as ColumnId); + if (def == null || prefs.sort.some((k) => k.column === column)) continue; + prefs.sort.push({ column: def.id as SortColumn, descending }); + } + } return prefs; } -/** A sort by a column, or by start time when null */ -export interface FlowSort { - column: ColumnId; - descending: boolean; -} +// -- Sorting -- /** Numbers sort largest first on the first click, names from A */ -export function firstDirection(column: ColumnId): boolean { - return column === "number" || column === "size" || column === "time" || column === "status"; -} - -/** - * A click on a column's label: sort by it, reverse the sort it has, then, from the reversed sort, - * go back to start time - */ -export function nextSort(sort: FlowSort | null, column: ColumnId): FlowSort | null { - const first = firstDirection(column); - if (sort?.column !== column) return { column, descending: first }; - if (sort.descending === first) return { column, descending: !first }; - return null; -} - -const collator = new Intl.Collator(undefined, { numeric: true, sensitivity: "base" }); - -type Key = string | number | null; - -function sortKey(flow: Flow, column: ColumnId): Key { +export function firstDirection(column: SortColumn): boolean { switch (column) { case "number": - return flow.number; - case "app": - return appKeyOf(flow); - case "method": - return flow.method; - case "host": - return hostOf(flow.url); - case "path": - return pathOf(flow.url); case "status": - return flow.error != null ? -1 : flow.status; - case "type": - return flow.contentType; case "size": - return flow.error != null && flow.status == null ? null : flow.responseSize; + case "requestSize": case "time": - return flow.elapsedMs; - case "remote": - return flow.remoteAddr; - case "version": - return flow.upstreamHttpVersion ?? flow.clientHttpVersion; + case "latency": + return true; + default: + return false; } } /** - * The flows in `sort`'s order, those without a value (none, empty or zero) lowest, so first - * ascending and last descending, then by start so equal values keep arrival order. Sorts the - * flows given, which are the ones loaded, not the whole database: the database only ever reads - * by start time, so no `ORDER BY` follows the column sort. + * A click on a column's label: sort by it alone, reverse the sort it has alone, then, from the + * reversed sort, go back to the order flows arrive in */ -export function sortFlows(flows: Flow[], sort: FlowSort | null): Flow[] { - if (sort == null) return flows; - const dir = sort.descending ? -1 : 1; - const keyed = flows.map((flow, i) => ({ flow, i, key: sortKey(flow, sort.column) || null })); - keyed.sort((a, b) => { - let c: number; - if (a.key == null || b.key == null) c = Number(b.key == null) - Number(a.key == null); - else if (typeof a.key === "number" && typeof b.key === "number") c = a.key - b.key; - else c = collator.compare(String(a.key), String(b.key)); - return c === 0 ? a.i - b.i : c * dir; - }); - return keyed.map((k) => k.flow); +export function nextSort(sort: FlowSortKey[], column: SortColumn): FlowSortKey[] { + const first = firstDirection(column); + const only = sort.length === 1 ? sort[0] : null; + if (only?.column !== column) return [{ column, descending: first }]; + if (only.descending === first) return [{ column, descending: !first }]; + return []; +} + +/** + * An Option-click on a column's label: adds it as the last key, ascending; again reverses it, and + * a third time takes it out + */ +export function toggleSortKey(sort: FlowSortKey[], column: SortColumn): FlowSortKey[] { + const key = sort.find((k) => k.column === column); + if (key == null) return [...sort, { column, descending: false }]; + if (!key.descending) + return sort.map((k) => (k.column === column ? { column, descending: true } : k)); + return sort.filter((k) => k.column !== column); +} + +/** Adds `column` as a key in `descending` order: the last key, or in its place if it is one */ +export function addSortKey( + sort: FlowSortKey[], + column: SortColumn, + descending: boolean, +): FlowSortKey[] { + if (sort.some((k) => k.column === column)) { + return sort.map((k) => (k.column === column ? { column, descending } : k)); + } + return [...sort, { column, descending }]; +} + +/** What an empty number sorts as, below every value; `EMPTY_NUMBER` in the lib's `flows.rs` */ +const EMPTY_NUMBER = -1e18; + +/** Text as SQLite's `NOCASE` compares it: ASCII letters lowercased, nothing else changed */ +function noCase(text: string | null | undefined): string { + return (text ?? "").replace(/[A-Z]/g, (c) => c.toLowerCase()); +} + +function emptyNumber(n: number | null | undefined): number { + return n == null || n === 0 ? EMPTY_NUMBER : n; +} + +/** + * The value `column` sorts `flow` by, as `SortColumn::sql` in the lib's `flows.rs` makes it in + * the database, so a flow placed here lands where a page read from there would have it + */ +export function columnSortKey(flow: Flow, column: SortColumn): string | number { + switch (column) { + case "number": + return flow.number; + case "app": + return noCase(flow.clientName ?? flow.clientBundleId ?? flow.clientPath); + case "method": + return noCase(flow.method); + case "host": + return noCase(urlParts(flow.url).authority); + case "path": + return noCase(urlParts(flow.url).path); + case "query": + return noCase(urlParts(flow.url).query); + case "status": + return flow.error != null ? -1 : emptyNumber(flow.status); + case "type": + return noCase(flow.contentType); + case "size": + return flow.error != null && flow.status == null + ? EMPTY_NUMBER + : emptyNumber(flow.responseSize); + case "requestSize": + return emptyNumber(flow.requestSize); + case "time": + return emptyNumber(flow.elapsedMs); + case "latency": + return emptyNumber(flow.elapsedHeadersMs); + case "started": + return flow.startedAt; + case "remote": + return noCase(flow.remoteAddr); + case "version": + return noCase(flow.upstreamHttpVersion ?? flow.clientHttpVersion); + case "tls": + return noCase(flow.upstreamTlsVersion); + case "operation": + return noCase( + flow.grpcService != null && flow.grpcMethod != null + ? `${flow.grpcService}/${flow.grpcMethod}` + : flow.graphqlOperation, + ); + } +} + +function compareKeys(a: string | number, b: string | number): number { + if (a === b) return 0; + return a < b ? -1 : 1; +} + +/** + * Orders flows as `list_flows` orders them for `sort`: each key in turn, then by start and id, + * oldest first + */ +export function sortComparator(sort: FlowSortKey[]): (a: Flow, b: Flow) => number { + return (a, b) => { + for (const { column, descending } of sort) { + const c = compareKeys(columnSortKey(a, column), columnSortKey(b, column)); + if (c !== 0) return descending ? -c : c; + } + return compareFlows(a, b); + }; +} + +/** The flows in `sort`'s order, as the database orders them */ +export function sortFlows(flows: Flow[], sort: FlowSortKey[]): Flow[] { + if (sort.length === 0) return flows; + return [...flows].sort(sortComparator(sort)); +} + +// -- Values -- + +/** + * A URL's parts as the lib's `url_parts` reads them: no scheme, all host; otherwise the + * authority up to the first `/`, `?` or `#`, then the path and the query without its `?` + */ +export function urlParts(url: string): { authority: string; path: string; query: string | null } { + const start = url.indexOf("://"); + if (start === -1) return { authority: url, path: "", query: null }; + const rest = url.slice(start + 3); + const end = rest.search(/[/?#]/); + const authority = end === -1 ? rest : rest.slice(0, end); + const tail = (end === -1 ? "" : rest.slice(end)).split("#")[0] ?? ""; + const q = tail.indexOf("?"); + return q === -1 + ? { authority, path: tail, query: null } + : { authority, path: tail.slice(0, q), query: tail.slice(q + 1) }; } /** The URL's host, with its port */ export function hostOf(url: string): string { - const start = url.indexOf("://"); - if (start === -1) return ""; - const rest = url.slice(start + 3); - const end = rest.search(/[/?#]/); - return end === -1 ? rest : rest.slice(0, end); + return url.includes("://") ? urlParts(url).authority : ""; } -function pathOf(url: string): string { - const start = url.indexOf("://"); - if (start === -1) return url; - const rest = url.slice(start + 3); - const end = rest.search(/[/?#]/); - return end === -1 ? "/" : rest.slice(end); +/** `Service/Method` for a gRPC call, the operation's name for GraphQL, or null */ +export function operationOf(flow: Flow): string | null { + if (flow.grpcService != null && flow.grpcMethod != null) { + return `${flow.grpcService}/${flow.grpcMethod}`; + } + return flow.graphqlOperation ?? null; +} + +/** `1.3` from `TLSv1.3` */ +export function tlsVersionOf(flow: Flow): string { + return (flow.upstreamTlsVersion ?? "").replace(/^TLSv?/i, ""); +} + +/** + * The value of a custom column's path for `flow`: a header's first value, the response's before + * the request's for `header.`, or a query parameter's first value + */ +export function customValue(flow: Flow, path: string): string | null { + const header = (headers: Flow["requestHeaders"], name: string) => + headers.find((h) => h.name.toLowerCase() === name)?.value ?? null; + if (path.startsWith("req.header.")) { + return header(flow.requestHeaders, path.slice("req.header.".length)); + } + if (path.startsWith("res.header.")) { + return header(flow.responseHeaders, path.slice("res.header.".length)); + } + if (path.startsWith("header.")) { + const name = path.slice("header.".length); + return header(flow.responseHeaders, name) ?? header(flow.requestHeaders, name); + } + if (path.startsWith("query.")) { + const query = urlParts(flow.url).query; + if (query == null) return null; + const name = path.slice("query.".length); + for (const [key, value] of new URLSearchParams(query)) { + if (key.toLowerCase() === name) return value; + } + } + return null; +} + +/** + * The search term that finds flows with `flow`'s value in `column`, or excludes them, for the + * cell's menu; null for columns with no field to match their value exactly + */ +export function cellFilterTerm(flow: Flow, column: ColumnId, negated: boolean): string | null { + const term = (field: string, value: string | null | undefined) => + value == null || value === "" ? null : fieldTerm(field, value, negated); + if (isCustomColumn(column)) { + const path = column.slice("field:".length); + return term(path, customValue(flow, path)); + } + switch (column) { + case "app": + return term("app", flow.clientName ?? flow.clientBundleId ?? flow.clientPath); + case "method": + return term("method", flow.method); + case "host": + return term("host", sidebarPath(flow.url).split("/")[0]); + case "path": + return flow.tunnelled ? null : term("path", urlParts(flow.url).path); + case "status": + return flow.status == null ? null : term("status", String(flow.status)); + case "type": + return term("type", flow.contentType?.split(";")[0]?.trim()); + case "operation": { + if (flow.grpcService != null && flow.grpcMethod != null) { + const both = `${fieldTerm("grpc.service", flow.grpcService)} ${fieldTerm("grpc.method", flow.grpcMethod)}`; + return negated ? `-(${both})` : both; + } + return term("graphql.operation", flow.graphqlOperation); + } + default: + return null; + } } diff --git a/apps/yaak-proxy/lib/flowActions.ts b/apps/yaak-proxy/lib/flowActions.ts index 985d306fd..a8e4e138b 100644 --- a/apps/yaak-proxy/lib/flowActions.ts +++ b/apps/yaak-proxy/lib/flowActions.ts @@ -3,11 +3,13 @@ import type { DropdownItem } from "@yaakapp-internal/ui"; import { save } from "@tauri-apps/plugin-dialog"; import { formatSize } from "@yaakapp-internal/lib/formatSize"; import { atom, useAtomValue, useSetAtom } from "jotai"; -import { useCallback } from "react"; +import { useCallback, useEffect } from "react"; import { base64ToBytes, bodyFileName } from "./body"; import { curlBodyText, toCurl } from "./curl"; +import type { FlowQuery } from "./query"; import { rpc } from "./rpc"; import { activeQueryAtom, lockedAtom } from "./store"; +import { subscribe } from "./tauri"; import { useShowToast } from "./toast"; /** The flow the save dialog is open for */ @@ -61,12 +63,32 @@ export async function exportHar( return rpc("export_har", { ...req, path }); } +/** Exports every flow `query` covers, or all of them, as HAR, saying how many it wrote */ +function useExportHarOf(): (query: FlowQuery | null) => Promise { + const showToast = useShowToast(); + return useCallback( + async (query: FlowQuery | null) => { + try { + const written = await exportHar(query == null ? {} : { filter: query }, "flows.har"); + if (written != null) { + showToast({ + message: written === 1 ? "Exported 1 flow as HAR" : `Exported ${written} flows as HAR`, + }); + } + } catch (err) { + showToast({ message: `Failed to export HAR: ${String(err)}`, color: "danger" }); + } + }, + [showToast], + ); +} + /** - * Exporting every flow the list covers as HAR, all of them or the search's, for the list's menus: - * it acts on flows, so it lives where they are chosen. + * Exporting every flow the list covers as HAR, all of them or the search's, for the list's and + * the search box's menus: it acts on flows, so it lives where they are chosen. */ export function useExportShownHar(): () => DropdownItem { - const showToast = useShowToast(); + const exportOf = useExportHarOf(); const query = useAtomValue(activeQueryAtom); // Nothing can be read to export while captured traffic is locked. const locked = useAtomValue(lockedAtom) != null; @@ -75,24 +97,28 @@ export function useExportShownHar(): () => DropdownItem { label: query == null ? "Export All Flows as HAR" : "Export Matching Flows as HAR", icon: "download", disabled: locked, - onSelect: async () => { - try { - const written = await exportHar(query == null ? {} : { filter: query }, "flows.har"); - if (written != null) { - showToast({ - message: - written === 1 ? "Exported 1 flow as HAR" : `Exported ${written} flows as HAR`, - }); - } - } catch (err) { - showToast({ message: `Failed to export HAR: ${String(err)}`, color: "danger" }); - } - }, + onSelect: () => exportOf(query), }), - [query, locked, showToast], + [query, locked, exportOf], ); } +/** File's Export All Flows as HAR, which the app menu sends as an event */ +export function useMenuExportHar() { + const exportOf = useExportHarOf(); + const locked = useAtomValue(lockedAtom) != null; + useEffect( + () => + subscribe(MENU_EXPORT_HAR_EVENT, () => { + if (!locked) void exportOf(null); + }), + [exportOf, locked], + ); +} + +/** `EXPORT_HAR_EVENT` in the app's `window_menu.rs` */ +const MENU_EXPORT_HAR_EVENT = "menu_export_har"; + /** What a flow's context menu and the detail pane's menu offer */ export function useFlowActions(): (flow: Flow) => DropdownItem[] { const showToast = useShowToast(); diff --git a/apps/yaak-proxy/lib/flowViews.test.ts b/apps/yaak-proxy/lib/flowViews.test.ts index 6ef8d323c..b8107835b 100644 --- a/apps/yaak-proxy/lib/flowViews.test.ts +++ b/apps/yaak-proxy/lib/flowViews.test.ts @@ -58,6 +58,7 @@ const flow: Flow = { grpcMethod: null, grpcStatus: null, grpcMessage: null, + graphqlOperation: null, tunnelled: false, agentRead: false, clientAddr: null, diff --git a/apps/yaak-proxy/lib/follow.test.ts b/apps/yaak-proxy/lib/follow.test.ts index 9db5bf3a7..30f97236b 100644 --- a/apps/yaak-proxy/lib/follow.test.ts +++ b/apps/yaak-proxy/lib/follow.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "vite-plus/test"; -import { countNewFlows } from "./follow"; +import { countNewFlows, isAtEnd, resumesFollowing } from "./follow"; const at = (...times: string[]) => times.map((t) => ({ startedAt: `2026-10-26T10:00:${t}` })); @@ -26,3 +26,28 @@ describe("countNewFlows", () => { expect(countNewFlows(at("03", "03.25"), "2026-10-26T10:00:03.2")).toBe(1); }); }); + +describe("resumesFollowing", () => { + const fits = { scrollHeight: 300, scrollTop: 0, clientHeight: 300 }; + const scrolledUp = { scrollHeight: 3000, scrollTop: 1200, clientHeight: 300 }; + const atBottom = { scrollHeight: 3000, scrollTop: 2700, clientHeight: 300 }; + + test("follows again when the list fits its viewport, so it never shows new flows as unseen", () => { + expect(isAtEnd(fits)).toBe(true); + expect(resumesFollowing({ following: false, sorted: false, el: fits })).toBe(true); + }); + + test("follows again at the end of a list that overflows", () => { + expect(resumesFollowing({ following: false, sorted: false, el: atBottom })).toBe(true); + }); + + test("stays held while scrolled up in a list that overflows", () => { + expect(isAtEnd(scrolledUp)).toBe(false); + expect(resumesFollowing({ following: false, sorted: false, el: scrolledUp })).toBe(false); + }); + + test("never follows while sorted, and has nothing to do while following", () => { + expect(resumesFollowing({ following: false, sorted: true, el: fits })).toBe(false); + expect(resumesFollowing({ following: true, sorted: false, el: fits })).toBe(false); + }); +}); diff --git a/apps/yaak-proxy/lib/follow.ts b/apps/yaak-proxy/lib/follow.ts index 354e1abb6..4b2c0e9b8 100644 --- a/apps/yaak-proxy/lib/follow.ts +++ b/apps/yaak-proxy/lib/follow.ts @@ -1,5 +1,30 @@ import type { Flow } from "@yaakapp-internal/proxy-lib"; +/** Where a list's scroll is, as its element reports it */ +export interface ScrollPosition { + scrollHeight: number; + scrollTop: number; + clientHeight: number; +} + +/** The list is scrolled to its end, or fits its viewport and has no scroll at all */ +export function isAtEnd(el: ScrollPosition): boolean { + return el.scrollHeight - el.scrollTop - el.clientHeight < 2; +} + +/** + * A held list follows new traffic again once it is at its end, unless it is sorted: after a + * scroll there, or after its flows, filter or size change leave it there, as a list that fits its + * viewport always is. One scrolled up in a list that overflows stays held. + */ +export function resumesFollowing(state: { + following: boolean; + sorted: boolean; + el: ScrollPosition; +}): boolean { + return !state.following && !state.sorted && isAtEnd(state.el); +} + /** * Flows in the list that started after the list was held. `heldAt` is the start of the newest * flow across all traffic at that moment, or null when there was none, so it still holds when diff --git a/apps/yaak-proxy/lib/sortedWindow.test.ts b/apps/yaak-proxy/lib/sortedWindow.test.ts new file mode 100644 index 000000000..d203083ff --- /dev/null +++ b/apps/yaak-proxy/lib/sortedWindow.test.ts @@ -0,0 +1,89 @@ +import type { Flow, ModelPayload } from "@yaakapp-internal/proxy-lib"; +import { describe, expect, test } from "vite-plus/test"; +import { sortComparator } from "./columns"; +import { appendPage, applySortedChanges, type SortedWindow } from "./sortedWindow"; + +function flow(n: number, status: number | null): Flow { + return { + id: `fl_${n}`, + number: n, + startedAt: `2026-11-09T12:00:0${n}`, + status, + error: null, + } as Flow; +} + +const bySize = sortComparator([{ column: "status", descending: true }]); +const upsert = (model: Flow, search?: ModelPayload["search"]): ModelPayload => ({ + model, + change: { type: "upsert", created: true }, + ...(search == null ? {} : { search }), +}); +const remove = (model: Flow): ModelPayload => ({ model, change: { type: "delete" } }); +const numbers = (flows: Flow[]) => flows.map((f) => f.number); + +describe("applySortedChanges", () => { + const held = [flow(1, 500), flow(2, 404), flow(3, 200)]; + + test("places a new or changed flow by the sort, and drops a deleted one", () => { + let flows = applySortedChanges(held, null, [upsert(flow(4, 300))], bySize, null); + expect(numbers(flows)).toEqual([1, 2, 4, 3]); + flows = applySortedChanges(flows, null, [upsert(flow(3, 503))], bySize, null); + expect(numbers(flows)).toEqual([3, 1, 2, 4]); + flows = applySortedChanges(flows, null, [remove(flow(1, 500))], bySize, null); + expect(numbers(flows)).toEqual([3, 2, 4]); + }); + + test("leaves a flow past the last one read for paging to bring", () => { + const boundary = held[2] ?? null; + const flows = applySortedChanges(held, boundary, [upsert(flow(4, 100))], bySize, null); + expect(numbers(flows)).toEqual([1, 2, 3]); + // The last flow read, written again unchanged, stays. + const same = applySortedChanges(held, boundary, [upsert(flow(3, 200))], bySize, null); + expect(numbers(same)).toEqual([1, 2, 3]); + // A held flow that moves past the end leaves until paging reaches it. + const moved = applySortedChanges(held, boundary, [upsert(flow(1, 100))], bySize, null); + expect(numbers(moved)).toEqual([2, 3]); + }); + + test("searching, takes only flows the search matched, and keeps ones it can't tell", () => { + const changes = [ + upsert(flow(4, 300), { key: "k", matches: true }), + upsert(flow(5, 300), { key: "k", matches: false }), + upsert(flow(2, 404), { key: "k", matches: false }), + upsert(flow(3, 999), { key: "other", matches: true }), + ]; + const flows = applySortedChanges(held, null, changes, bySize, "k"); + expect(numbers(flows)).toEqual([1, 4, 3]); + }); +}); + +test("a page adds what it does not already hold", () => { + const win: SortedWindow = { + key: "1", + sort: [], + query: null, + flows: [flow(1, 500), flow(2, 404)], + boundary: flow(2, 404), + cursor: [404, "fl_2"], + loading: false, + buffered: [], + }; + const next = appendPage(win, [flow(2, 404), flow(3, 200)], [200, "fl_3"], 2); + expect(numbers(next.flows)).toEqual([1, 2, 3]); + expect(next.boundary?.id).toBe("fl_3"); + expect(next.cursor).toEqual([200, "fl_3"]); + // A short page is the last: nothing is past it. + const last = appendPage(win, [flow(3, 200)], [200, "fl_3"], 2); + expect([last.boundary, last.cursor]).toEqual([null, null]); + // One the sweep deleted meanwhile is left out, but still marks where the page ended. + const swept = appendPage( + win, + [flow(3, 200), flow(4, 100)], + [100, "fl_4"], + 2, + (f) => f.id !== "fl_4", + ); + expect(numbers(swept.flows)).toEqual([1, 2, 3]); + expect(swept.boundary?.id).toBe("fl_4"); +}); diff --git a/apps/yaak-proxy/lib/sortedWindow.ts b/apps/yaak-proxy/lib/sortedWindow.ts new file mode 100644 index 000000000..f5fcd3d0e --- /dev/null +++ b/apps/yaak-proxy/lib/sortedWindow.ts @@ -0,0 +1,97 @@ +import type { Flow, FlowSortKey, ModelPayload, SortValue } from "@yaakapp-internal/proxy-lib"; +import type { FlowQuery } from "./query"; + +/** + * The flows the list shows while it is sorted: read from the database in the sort's order, a + * page at a time from the top, as `list_flows` orders them for `sort`. Flows that change are + * placed by the same comparison, and only up to where the pages read end, so paging on never + * skips or repeats one. + */ +export interface SortedWindow { + /** Unique to this load, so a page read for an earlier sort or search is dropped */ + key: string; + sort: FlowSortKey[]; + query: FlowQuery | null; + /** In the sort's order */ + flows: Flow[]; + /** + * The last flow of the last page, as it was read, while the database has more after it: where + * the next page starts. Null once every flow is read. + */ + boundary: Flow | null; + /** The next page's `cursor`, from the last page: the boundary's sort values as read */ + cursor: SortValue[] | null; + /** The first page has not arrived yet */ + loading: boolean; + /** Changes that came while the first page loaded, applied once it is in */ + buffered: ModelPayload[]; +} + +/** Where `flow` goes in `flows`, which `compare` orders */ +function insertionIndex(flows: Flow[], flow: Flow, compare: (a: Flow, b: Flow) => number): number { + let lo = 0; + let hi = flows.length; + while (lo < hi) { + const mid = (lo + hi) >>> 1; + const item = flows[mid]; + if (item != null && compare(item, flow) < 0) lo = mid + 1; + else hi = mid; + } + return lo; +} + +/** + * Applies flow events to a sorted window's flows: a changed flow moves to where `compare` puts + * it, a deleted one leaves, and one that sorts after `boundary`, where the pages read end, is + * left for paging. With `searchKey`, only flows the backend matched against that search join or + * stay; a flow matched against another search is left as it is, to be matched again. + */ +export function applySortedChanges( + flows: Flow[], + boundary: Flow | null, + changes: ModelPayload[], + compare: (a: Flow, b: Flow) => number, + searchKey: string | null, +): Flow[] { + if (changes.length === 0) return flows; + const next = flows.slice(); + for (const c of changes) { + const deleted = c.change.type === "delete"; + if (!deleted && searchKey != null && c.search?.key !== searchKey) continue; + const existing = next.findIndex((f) => f.id === c.model.id); + if (existing !== -1) next.splice(existing, 1); + if (deleted || (searchKey != null && c.search?.matches !== true)) continue; + if (boundary != null && compare(c.model, boundary) > 0) continue; + next.splice(insertionIndex(next, c.model, compare), 0, c.model); + } + return next; +} + +/** A page from `list_flows` as a sorted window takes it: where it ends, and where the next starts */ +export function pageBoundary( + page: Flow[], + cursor: SortValue[] | null, + limit: number, +): Pick { + const more = page.length >= limit; + return { boundary: more ? (page.at(-1) ?? null) : null, cursor: more ? cursor : null }; +} + +/** + * Adds the next page from `list_flows`, read after the window's boundary, less any it has and + * any `keep` leaves out + */ +export function appendPage( + win: SortedWindow, + page: Flow[], + cursor: SortValue[] | null, + limit: number, + keep: (flow: Flow) => boolean = () => true, +): SortedWindow { + const held = new Set(win.flows.map((f) => f.id)); + return { + ...win, + flows: [...win.flows, ...page.filter((f) => !held.has(f.id) && keep(f))], + ...pageBoundary(page, cursor, limit), + }; +} diff --git a/apps/yaak-proxy/lib/store.ts b/apps/yaak-proxy/lib/store.ts index fc9655c4d..997372718 100644 --- a/apps/yaak-proxy/lib/store.ts +++ b/apps/yaak-proxy/lib/store.ts @@ -1,8 +1,10 @@ import type { Flow, + FlowSortKey, GetProxyStateResponse, ModelPayload, SavedQuery, + SortValue, } from "@yaakapp-internal/proxy-lib"; import { selectedIdsFamily } from "@yaakapp-internal/ui"; import { atom, type createStore } from "jotai"; @@ -11,10 +13,11 @@ import { followingAtom, SIDEBAR_TREE_ID, selectedFlowAtom } from "./atoms"; import { type ColumnPrefs, defaultColumnPrefs, - type FlowSort, parseColumnPrefs, + sortComparator, sortFlows, } from "./columns"; +import { appendPage, applySortedChanges, pageBoundary, type SortedWindow } from "./sortedWindow"; import { applyChanges, emptyWindow, @@ -158,7 +161,7 @@ const storedColumnPrefsAtom = atomWithStorage( }, ); -/** Which columns the list shows and how wide, as the person left them */ +/** Which columns the list shows, how wide, and its sort, as the person left them */ export const columnPrefsAtom = atom( (get) => parseColumnPrefs(get(storedColumnPrefsAtom)), (get, set, update: ColumnPrefs | ((prev: ColumnPrefs) => ColumnPrefs)) => { @@ -167,8 +170,14 @@ export const columnPrefsAtom = atom( }, ); -/** The list's sort, or null for start time, the order flows arrive in */ -export const flowSortAtom = atom(null); +/** The list's sort keys, in order, kept with the columns; empty for the order flows arrive in */ +export const flowSortAtom = atom( + (get) => get(columnPrefsAtom).sort, + (_get, set, sort: FlowSortKey[]) => set(columnPrefsAtom, (p) => ({ ...p, sort })), +); + +/** The flows the list shows while sorted, read from the database in the sort's order */ +export const sortedWindowAtom = atom(null); /** A window onto the flows a search covers, loaded from the database as the main one is */ export interface SearchWindow { @@ -189,12 +198,26 @@ const shownWindowAtom = atom((get) => get(searchWindowAtom)?.win ?? get(flowWind /** The flows loaded for the list, oldest first */ export const windowFlowsAtom = atom((get) => get(shownWindowAtom).flows); -/** The flows the list shows: oldest first, or in the sort's order, over those loaded */ -export const shownFlowsAtom = atom((get) => sortFlows(get(windowFlowsAtom), get(flowSortAtom))); +/** + * The flows the list shows: oldest first, or in the sort's order from the database. Until a + * sort's first page is in, the flows loaded, sorted here, stand in for it. + */ +export const shownFlowsAtom = atom((get) => { + const sort = get(flowSortAtom); + if (sort.length === 0) return get(windowFlowsAtom); + const sorted = get(sortedWindowAtom); + if (sorted == null || (sorted.loading && sorted.flows.length === 0)) { + return sortFlows(get(windowFlowsAtom), sort); + } + return sorted.flows; +}); /** Flows in the database the list covers: every one, or those the search matches */ export const shownTotalAtom = atom((get) => get(shownWindowAtom).total); +/** The search's first page has not arrived, so the list shows none of its flows yet */ +export const searchLoadingAtom = atom((get) => get(searchWindowAtom)?.loading ?? false); + /** Start and id of the oldest flow shown, which loading older flows moves back */ export const oldestFlowAtom = atom((get) => get(windowFlowsAtom)[0] ?? null); @@ -288,6 +311,54 @@ export const loadOlderFlowsAtom = atom(null, async (get, set) => { } }); +/** Changes applied while a page of a sort loads, applied again over it once it is in */ +let changesDuringSortedPage: ModelPayload[] | null = null; + +/** The key flow events mark matches of the search shown with, or null for all traffic */ +function searchKeyOf(search: SearchWindow | null): string | null { + return search?.key ?? null; +} + +/** Loads the sort's next page, after the last flow shown, on scrolling to the bottom */ +export const loadMoreSortedAtom = atom(null, async (get, set) => { + const win = get(sortedWindowAtom); + const cursor = win?.cursor; + if (win == null || win.loading || win.boundary == null || cursor == null) return; + if (changesDuringSortedPage != null) return; + const changes: ModelPayload[] = []; + changesDuringSortedPage = changes; + pageStarted(); + try { + const res = await rpc("list_flows", { + ...win.query, + sort: win.sort, + cursor, + limit: PAGE_SIZE, + }); + set(sortedWindowAtom, (s) => { + if (s?.key !== win.key || s.cursor !== cursor) return s; + const kept = new Set(withoutRemoved(res.flows).map((f) => f.id)); + const merged = appendPage(s, res.flows, res.cursor, PAGE_SIZE, (f) => kept.has(f.id)); + const compare = sortComparator(s.sort); + const key = searchKeyOf(get(searchWindowAtom)); + return { + ...merged, + flows: applySortedChanges(merged.flows, merged.boundary, changes, compare, key), + }; + }); + } catch (err) { + // Read the sort again from its top. + console.warn("Failed to load more sorted flows", err); + reloadSorted?.(); + } finally { + changesDuringSortedPage = null; + pageSettled(); + } +}); + +/** Reads the sort's first page again, once `syncFlows` has started */ +let reloadSorted: (() => void) | null = null; + /** `changesFor`, with flows matched against another search sent to be matched again */ function changesOf( search: Pick, @@ -320,6 +391,7 @@ function matchLater(search: Pick, changes: ModelP search: { key: search.key, matches: covered.has(c.model.id) }, })); if (marked.length === 0) return; + applySortedChangesTo(store, marked); const following = store.get(followingAtom); store.set(searchWindowAtom, (s) => { if (s?.key !== search.key) return s; @@ -331,6 +403,19 @@ function matchLater(search: Pick, changes: ModelP .catch((err) => console.warn("Failed to match flows", err)); } +/** Applies flow events to the sorted window, or holds them while its first page loads */ +function applySortedChangesTo(store: Store, changes: ModelPayload[]) { + if (store.get(sortedWindowAtom) == null) return; + changesDuringSortedPage?.push(...changes); + const key = searchKeyOf(store.get(searchWindowAtom)); + store.set(sortedWindowAtom, (s) => { + if (s == null) return s; + if (s.loading) return { ...s, buffered: [...s.buffered, ...changes] }; + const flows = applySortedChanges(s.flows, s.boundary, changes, sortComparator(s.sort), key); + return flows === s.flows ? s : { ...s, flows }; + }); +} + /** * Loads the newest flows, then keeps them current with the engine's events, or with reads of the * database while another process owns the engine. Events gather for `FLUSH_MS` and apply @@ -366,6 +451,7 @@ export function syncFlows(store: Store) { const next = applyChanges(s.win, changesOf(s, s.win, changes)); return { ...s, win: following ? trimWindow(next) : next }; }); + applySortedChangesTo(store, changes); const selected = store.get(selectedFlowAtom); const change = changes.findLast((c) => c.model.id === selected?.id); if (change != null) { @@ -477,6 +563,16 @@ export function syncFlows(store: Store) { dropRemovedChanges(buffered, removed); return { ...s, win: drop(s.win), buffered }; }); + if (changesDuringSortedPage != null) dropRemovedChanges(changesDuringSortedPage, removed); + store.set(sortedWindowAtom, (s) => { + if (s == null) return s; + const buffered = [...s.buffered]; + dropRemovedChanges(buffered, removed); + const flows = s.flows.some((f) => removed.has(f.id)) + ? s.flows.filter((f) => !removed.has(f.id)) + : s.flows; + return { ...s, flows, buffered }; + }); if (removed.has(store.get(selectedFlowAtom)?.id ?? "")) store.set(selectedFlowAtom, null); if (recountTimer == null) recountTimer = setTimeout(recount, RECOUNT_MS); }); @@ -503,9 +599,62 @@ export function syncFlows(store: Store) { }) .finally(pageSettled); loadSearch(store.get(activeQueryAtom)); + loadSorted(); } reloadFlows = load; + /** + * Reads the first page of the list's sort from the database, over the search shown, or drops + * the sorted window when there is no sort. Until the page is in, the flows loaded, sorted here, + * stand in for it. + */ + let sortedLoads = 0; + function loadSorted() { + const sort = store.get(flowSortAtom); + if (sort.length === 0) { + store.set(sortedWindowAtom, null); + return; + } + const query = store.get(activeQueryAtom); + const key = `${++sortedLoads}:${JSON.stringify({ sort, query })}`; + store.set(sortedWindowAtom, { + key, + sort, + query, + flows: [], + boundary: null, + cursor: null, + loading: true, + buffered: [], + }); + const settle = (page: Flow[] | null, cursor: SortValue[] | null) => + store.set(sortedWindowAtom, (s) => { + if (s?.key !== key) return s; + const ends = pageBoundary(page ?? [], cursor, PAGE_SIZE); + const base = page == null ? s.flows : withoutRemoved(page); + const search = searchKeyOf(store.get(searchWindowAtom)); + const compare = sortComparator(sort); + const flows = applySortedChanges(base, ends.boundary, s.buffered, compare, search); + return { ...s, ...ends, flows, loading: false, buffered: [] }; + }); + pageStarted(); + void rpc("list_flows", { ...query, sort, limit: PAGE_SIZE }) + .then((res) => settle(res.flows, res.cursor)) + .catch((err) => { + console.warn("Failed to read the sorted flows", err); + settle(null, null); + }) + .finally(pageSettled); + } + reloadSorted = loadSorted; + let sortJson = JSON.stringify(store.get(flowSortAtom)); + store.sub(flowSortAtom, () => { + const json = JSON.stringify(store.get(flowSortAtom)); + if (json === sortJson) return; + sortJson = json; + loadSorted(); + }); + /** * Loads the first page of `query`'s flows into a window of its own, or drops it for null. The * backend matches the flows written from now on against it, and their events say so. @@ -590,6 +739,7 @@ export function syncFlows(store: Store) { if (key === activeKey) return; activeKey = key; loadSearch(query); + loadSorted(); }); // While another process owns the engine no events come, so new flows are read from the @@ -652,9 +802,11 @@ const emptyFlowsAtom = atom(null, (_get, set) => { set(flowWindowAtom, emptyWindow); set(nonSystemTotalAtom, 0); set(searchWindowAtom, (s) => (s == null ? s : { ...s, win: emptyWindow, buffered: [] })); + set(sortedWindowAtom, (s) => + s == null ? s : { ...s, flows: [], boundary: null, cursor: null, buffered: [] }, + ); set(selectedFlowAtom, null); set(followingAtom, true); - set(flowSortAtom, null); }); /** diff --git a/apps/yaak-proxy/lib/tauri.test.ts b/apps/yaak-proxy/lib/tauri.test.ts new file mode 100644 index 000000000..57ea38398 --- /dev/null +++ b/apps/yaak-proxy/lib/tauri.test.ts @@ -0,0 +1,65 @@ +import { beforeEach, describe, expect, test, vi } from "vite-plus/test"; + +type Handler = (e: { payload: unknown }) => void; + +/** Listeners Tauri holds, and the registrations not yet resolved */ +const listeners = new Set(); +let pending: (() => void)[] = []; + +vi.mock("@tauri-apps/api/event", () => ({ + // Registers asynchronously, as Tauri does, resolving only when the test lets it. + listen: (_event: string, handler: Handler) => + new Promise<() => void>((resolve) => { + pending.push(() => { + listeners.add(handler); + resolve(() => listeners.delete(handler)); + }); + }), +})); +vi.mock("@tauri-apps/api/core", () => ({ invoke: vi.fn() })); +vi.mock("@tauri-apps/api/webviewWindow", () => ({ getCurrentWebviewWindow: vi.fn() })); +vi.mock("@tauri-apps/plugin-os", () => ({ type: vi.fn() })); + +const { subscribe } = await import("./tauri"); + +async function register() { + for (const resolve of pending) resolve(); + pending = []; + await Promise.resolve(); + await Promise.resolve(); +} + +function emit(payload: unknown) { + for (const handler of listeners) handler({ payload }); +} + +describe("subscribe", () => { + beforeEach(() => { + listeners.clear(); + pending = []; + }); + + test("an effect cleaned up before its listener registers hears nothing, as in strict mode", async () => { + const heard = vi.fn(); + // Strict mode: subscribe, clean up at once, subscribe again + const first = subscribe("menu_export_har", heard); + first(); + const second = subscribe("menu_export_har", heard); + await register(); + emit(null); + expect(heard).toHaveBeenCalledTimes(1); + expect(listeners.size).toBe(1); + second(); + expect(listeners.size).toBe(0); + }); + + test("delivers events until unsubscribed", async () => { + const heard = vi.fn(); + const unsubscribe = subscribe("menu_export_har", heard); + await register(); + emit(1); + unsubscribe(); + emit(2); + expect(heard.mock.calls).toEqual([[1]]); + }); +}); diff --git a/apps/yaak-proxy/lib/tauri.ts b/apps/yaak-proxy/lib/tauri.ts index e4371eb89..344d0ad3b 100644 --- a/apps/yaak-proxy/lib/tauri.ts +++ b/apps/yaak-proxy/lib/tauri.ts @@ -9,15 +9,19 @@ export function command(cmd: string, args?: Record): Promise return invoke(cmd, args) as Promise; } -/** Subscribe to a Tauri event. Returns an unsubscribe function. */ +/** + * Subscribe to a Tauri event. Returns an unsubscribe function, which also removes a listener + * still being registered once it is, and stops it delivering meanwhile: an effect cleaned up + * that fast, as strict mode does to every one, would otherwise leave its listener attached and + * hear each event twice. + */ export function subscribe(event: string, callback: (payload: T) => void): () => void { - let unsub: (() => void) | null = null; - tauriListen(event, (e) => callback(e.payload)) - .then((fn) => { - unsub = fn; - }) - .catch(console.error); - return () => unsub?.(); + const subscription = unsubscribeWhenReady( + tauriListen(event, (e) => { + if (subscription.active()) callback(e.payload); + }), + ); + return subscription.unsubscribe; } /** Calls `callback` each time the window regains focus. Returns an unsubscribe function. */ diff --git a/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts b/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts index 98ca43b83..eeb878d1c 100644 --- a/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts +++ b/crates-proxy/yaak-proxy-lib/bindings/gen_models.ts @@ -126,6 +126,11 @@ grpcStatus: number | null, * A gRPC call's `grpc-message`, decoded. */ grpcMessage: string | null, +/** + * A GraphQL request's operation, from its body or URL: `GetUser`, `GetUser +2` for a + * batch, empty for an anonymous one. None when the request is not GraphQL. + */ +graphqlOperation: string | null, /** * An HTTPS connection passed through undecrypted, as the decrypt list leaves it out or its * client rejected the proxy's certificate: method `CONNECT`, its URL the host, its sizes diff --git a/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts b/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts index c97e3c210..de0b1e6b3 100644 --- a/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts +++ b/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts @@ -178,7 +178,24 @@ after?: string, * Only flows older than the flow with this id, by start and then id, for paging back * through older traffic. */ -before?: string, }; +before?: string, +/** + * The order to list in, the list's column sort: each key in turn, then by start and id, + * oldest first. Empty lists newest first, as without it. With keys, `before` does not + * apply; `cursor` pages instead. + */ +sort?: Array, +/** + * Only flows after these values in `sort`'s order, for paging through a sort: the `cursor` + * of the page before, the sort values of its last flow as it was read, so a change to that + * flow since does not move where the next page starts. + */ +cursor?: Array, }; + +/** + * One key of a sort. + */ +export type FlowSortKey = { column: SortColumn, descending: boolean, }; export type FlowsClearedPayload = Record; @@ -328,7 +345,12 @@ total: number, /** * Flows the filter covers, whatever the limit, when it narrows to fewer than all of them. */ -matching: number | null, }; +matching: number | null, +/** + * Sorted, the next page's `cursor`: the last flow's sort values, as read. None without a + * sort or a flow. + */ +cursor: Array | null, }; export type ListGrpcSchemasRequest = Record; @@ -574,6 +596,16 @@ export type SetSystemProxyAtLaunchRequest = { enabled: boolean, }; export type SetTrafficRequest = { traffic: TrafficRetention, }; +/** + * A column the list sorts by. + */ +export type SortColumn = "number" | "app" | "method" | "host" | "path" | "query" | "status" | "type" | "size" | "requestSize" | "time" | "latency" | "started" | "remote" | "version" | "tls" | "operation"; + +/** + * One of a flow's sort values, as the database sorted it. + */ +export type SortValue = number | string; + /** * How a change gets administrator rights. */ diff --git a/crates-proxy/yaak-proxy-lib/migrations/00000013_flow_graphql_operation.sql b/crates-proxy/yaak-proxy-lib/migrations/00000013_flow_graphql_operation.sql new file mode 100644 index 000000000..7b463c0aa --- /dev/null +++ b/crates-proxy/yaak-proxy-lib/migrations/00000013_flow_graphql_operation.sql @@ -0,0 +1,3 @@ +-- A GraphQL request's operation name, read from its body or URL as it was captured: empty for an +-- anonymous operation, NULL for a flow that is not GraphQL. +ALTER TABLE flows ADD COLUMN graphql_operation TEXT; diff --git a/crates-proxy/yaak-proxy-lib/src/convert.rs b/crates-proxy/yaak-proxy-lib/src/convert.rs index 12c122ffd..bb3a6e6c5 100644 --- a/crates-proxy/yaak-proxy-lib/src/convert.rs +++ b/crates-proxy/yaak-proxy-lib/src/convert.rs @@ -177,7 +177,7 @@ fn request_body( }; if mime == "application/json" || mime.ends_with("+json") { - if let Some(graphql) = graphql_body(flow, text) { + if let Some(graphql) = graphql_body(flow, content_type, text) { return (Some(BODY_TYPE_GRAPHQL.into()), graphql); } return text_body(BODY_TYPE_JSON, text); @@ -206,26 +206,23 @@ fn text_body(body_type: &str, text: &str) -> (Option, BTreeMap Option> { - let path = flow.url.split('?').next().unwrap_or_default(); - if !path.to_ascii_lowercase().ends_with("/graphql") { - return None; +/// A JSON body that is a GraphQL request, as the client's GraphQL body. Detected as the proxy +/// detects it at capture (`yaak_http::graphql`), which leaves out bodies with fields a GraphQL +/// body cannot hold, such as persisted-query `extensions`. +fn graphql_body(flow: &Flow, content_type: &str, text: &str) -> Option> { + let body = yaak_http::graphql::parse_json_body(Some(content_type), text, &flow.url)?; + let mut fields = BTreeMap::from([("query".to_string(), json!(body.query))]); + match body.variables { + Some(Value::String(variables)) => { + fields.insert("variables".into(), json!(variables)); + } + Some(variables) => { + let pretty = serde_json::to_string_pretty(&variables).ok()?; + fields.insert("variables".into(), json!(pretty)); + } + None => {} } - let Value::Object(body) = serde_json::from_str::(text).ok()? else { - return None; - }; - // Anything else, such as persisted-query `extensions`, would be lost in a GraphQL body. - if body.keys().any(|k| !matches!(k.as_str(), "query" | "variables" | "operationName")) { - return None; - } - let query = body.get("query")?.as_str()?; - let mut fields = BTreeMap::from([("query".to_string(), json!(query))]); - if let Some(variables) = body.get("variables").filter(|v| !v.is_null()) { - let pretty = serde_json::to_string_pretty(variables).ok()?; - fields.insert("variables".into(), json!(pretty)); - } - if let Some(name) = body.get("operationName").and_then(Value::as_str) { + if let Some(name) = body.operation_name { fields.insert("operationName".into(), json!(name)); } Some(fields) @@ -355,10 +352,14 @@ mod tests { let c = flow_to_request(&f, &body(persisted)); assert_eq!(c.request.body_type.as_deref(), Some("application/json")); - // The same body anywhere else is plain JSON. + // A `query` that is no GraphQL document is plain JSON. let f = flow("POST", "https://x.test/api", &[("Content-Type", "application/json")]); let c = flow_to_request(&f, &body(r#"{"query":"q"}"#)); assert_eq!(c.request.body_type.as_deref(), Some("application/json")); + + // A GraphQL document is GraphQL on any path, as capture reads it. + let c = flow_to_request(&f, &body(r#"{"query":"mutation Save { save { id } }"}"#)); + assert_eq!(c.request.body_type.as_deref(), Some("graphql")); } #[test] diff --git a/crates-proxy/yaak-proxy-lib/src/db.rs b/crates-proxy/yaak-proxy-lib/src/db.rs index 4a7c81ac4..cbb0d171f 100644 --- a/crates-proxy/yaak-proxy-lib/src/db.rs +++ b/crates-proxy/yaak-proxy-lib/src/db.rs @@ -543,6 +543,7 @@ mod tests { "grpc_method", "grpc_status", "grpc_message", + "graphql_operation", "agent_read", ] { assert!(flow_columns.contains(&column.to_string()), "{column} in {flow_columns:?}"); @@ -614,7 +615,7 @@ mod tests { assert_eq!( &applied[6..], [ - "00000007", "00000008", "00000009", "00000010", "00000011", "00000012" + "00000007", "00000008", "00000009", "00000010", "00000011", "00000012", "00000013" ] ); let mark_columns = columns(&conn, "flow_clear_mark"); diff --git a/crates-proxy/yaak-proxy-lib/src/flows.rs b/crates-proxy/yaak-proxy-lib/src/flows.rs index a01c4c97d..2885b77e0 100644 --- a/crates-proxy/yaak-proxy-lib/src/flows.rs +++ b/crates-proxy/yaak-proxy-lib/src/flows.rs @@ -34,6 +34,152 @@ pub struct FlowFilter { /// through older traffic. #[ts(optional)] pub before: Option, + /// The order to list in, the list's column sort: each key in turn, then by start and id, + /// oldest first. Empty lists newest first, as without it. With keys, `before` does not + /// apply; `cursor` pages instead. + #[ts(optional, as = "Option>")] + pub sort: Vec, + /// Only flows after these values in `sort`'s order, for paging through a sort: the `cursor` + /// of the page before, the sort values of its last flow as it was read, so a change to that + /// flow since does not move where the next page starts. + #[ts(optional, as = "Option>")] + pub cursor: Option>, +} + +/// One of a flow's sort values, as the database sorted it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(untagged)] +#[ts(export, export_to = "gen_rpc.ts")] +pub enum SortValue { + Int(#[ts(type = "number")] i64), + Text(String), +} + +impl SortValue { + fn from_sql(value: Value) -> Self { + match value { + Value::Integer(i) => Self::Int(i), + Value::Real(f) => Self::Int(f as i64), + Value::Text(t) => Self::Text(t), + Value::Null | Value::Blob(_) => Self::Text(String::new()), + } + } + + fn to_sql(&self) -> Value { + match self { + Self::Int(i) => Value::Integer(*i), + Self::Text(t) => Value::Text(t.clone()), + } + } +} + +/// A column the list sorts by. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_rpc.ts")] +pub enum SortColumn { + Number, + App, + Method, + Host, + Path, + Query, + Status, + Type, + Size, + RequestSize, + Time, + Latency, + Started, + Remote, + Version, + Tls, + Operation, +} + +/// One key of a sort. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct FlowSortKey { + pub column: SortColumn, + pub descending: bool, +} + +/// What an empty number sorts as: below every value, as an empty text's `''` is. +const EMPTY_NUMBER: &str = "-1000000000000000000"; + +impl SortColumn { + /// The value the column sorts by, in SQL, never NULL: empty values (none, `''` or `0`) + /// lowest, text without regard to ASCII case. `columnSortKey` in the app's + /// `lib/columns.ts` makes the same values for flows it places itself. + fn sql(self, columns: &FlowColumns) -> String { + let c = |n: &str| columns.col(n); + let number = |v: &str| format!("coalesce(nullif({v}, 0), {EMPTY_NUMBER})"); + let text = |v: &str| format!("(coalesce({v}, '') COLLATE NOCASE)"); + match self { + Self::Number => "flows.rowid".into(), + Self::App => text(&format!( + "coalesce({}, {}, {})", + c("client_name"), + c("client_bundle_id"), + c("client_path") + )), + Self::Method => text("method"), + Self::Host => text("url_authority(url)"), + Self::Path => text("url_path(url)"), + Self::Query => text("url_query_string(url)"), + Self::Status => { + format!("CASE WHEN error IS NOT NULL THEN -1 ELSE {} END", number("status")) + } + Self::Type => text("content_type"), + Self::Size => format!( + "CASE WHEN error IS NOT NULL AND status IS NULL THEN {EMPTY_NUMBER} ELSE {} END", + number("response_size") + ), + Self::RequestSize => number("request_size"), + Self::Time => number("elapsed_ms"), + Self::Latency => number("elapsed_headers_ms"), + Self::Started => "started_at".into(), + Self::Remote => text("remote_addr"), + Self::Version => text("coalesce(upstream_http_version, client_http_version)"), + Self::Tls => text(&c("upstream_tls_version")), + Self::Operation => { + let (service, method) = (c("grpc_service"), c("grpc_method")); + text(&format!( + "CASE WHEN {service} IS NOT NULL AND {method} IS NOT NULL \ + THEN {service} || '/' || {method} ELSE {} END", + c("graphql_operation") + )) + } + } + } +} + +/// A sort's keys in SQL, each with whether it descends, then start and id, ascending. +fn sort_terms(sort: &[FlowSortKey], columns: &FlowColumns) -> Vec<(String, bool)> { + sort.iter() + .map(|k| (k.column.sql(columns), k.descending)) + .chain([ + ("started_at".to_string(), false), + ("flows.id".to_string(), false), + ]) + .collect() +} + +/// The `ORDER BY` for a sort, and the condition for the flows after `cursor`'s values in it: +/// each key equal up to one that is past the cursor's. +fn sort_after(terms: &[(String, bool)], cursor: &[Value]) -> (String, Vec) { + let mut params = Vec::new(); + let mut alternatives = Vec::new(); + for (i, (term, descending)) in terms.iter().enumerate() { + let mut parts: Vec = terms[..i].iter().map(|(t, _)| format!("{t} = ?")).collect(); + params.extend(cursor[..i].iter().cloned()); + parts.push(format!("{term} {} ?", if *descending { "<" } else { ">" })); + params.push(cursor[i].clone()); + alternatives.push(format!("({})", parts.join(" AND "))); + } + (format!("({})", alternatives.join(" OR ")), params) } impl FlowFilter { @@ -197,20 +343,50 @@ pub fn page( db: &ProxyQueryManager, filter: &FlowFilter, ) -> Result<(Vec, usize, Option), String> { + let page = sorted_page(db, filter)?; + Ok((page.flows, page.total, page.matching)) +} + +/// A page of flows, as [`page`] reads it, with where a sort's next page starts. +pub struct Page { + pub flows: Vec, + pub total: usize, + pub matching: Option, + /// The sort values of the last flow, for the next page's `cursor`. None without a sort. + pub cursor: Option>, +} + +/// [`page`], with the cursor for a sort's next page. +pub fn sorted_page(db: &ProxyQueryManager, filter: &FlowFilter) -> Result { db.with_conn(|c| { let tx = c.conn().resolve().unchecked_transaction().map_err(|e| e.to_string())?; - let flows = list_in(&tx, filter)?; + let (flows, cursor) = list_and_cursor(&tx, filter)?; let total = count_in(&tx)?; let matching = if filter.narrows() { Some(count_matching(&tx, filter)?) } else { None }; tx.commit().map_err(|e| e.to_string())?; - Ok((flows, total, matching)) + Ok(Page { flows, total, matching, cursor }) }) } fn list_in(conn: &rusqlite::Connection, filter: &FlowFilter) -> Result, String> { + list_and_cursor(conn, filter).map(|(flows, _)| flows) +} + +/// The flows `filter` covers and, sorted, the sort values of the last one. +fn list_and_cursor( + conn: &rusqlite::Connection, + filter: &FlowFilter, +) -> Result<(Vec, Option>), String> { let (condition, mut params) = filter.condition_in(conn); - // The rowid, for the flow's number. - let mut sql = format!("SELECT rowid, * FROM flows WHERE {condition}"); + let terms = if filter.sort.is_empty() { + Vec::new() + } else { + sort_terms(&filter.sort, &FlowColumns::of(conn)) + }; + // The rowid, for the flow's number, and a sort's values, for its cursor. + let keys: String = + terms.iter().enumerate().map(|(i, (t, _))| format!(", {t} AS sort_{i}")).collect(); + let mut sql = format!("SELECT rowid, *{keys} FROM flows WHERE {condition}"); push_clear_mark(conn, &mut sql, &mut params); if let Some(id) = &filter.after { // By rowid, which only grows as flows are written. @@ -220,24 +396,53 @@ fn list_in(conn: &rusqlite::Connection, filter: &FlowFilter) -> Result sql.push_str(" AND rowid > ?"); params.push(rowid.into()); } - if let Some(id) = &filter.before { - let started_at: Value = conn - .query_row("SELECT started_at FROM flows WHERE id = ?", [id], |r| r.get(0)) - .map_err(|_| format!("No flow {id}"))?; - sql.push_str(" AND (started_at, id) < (?, ?)"); - params.push(started_at); - params.push(id.clone().into()); + if !terms.is_empty() { + if let Some(cursor) = &filter.cursor { + if cursor.len() != terms.len() { + return Err("The cursor is not for this sort".into()); + } + let cursor: Vec = cursor.iter().map(SortValue::to_sql).collect(); + let (after, after_params) = sort_after(&terms, &cursor); + sql.push_str(" AND "); + sql.push_str(&after); + params.extend(after_params); + } + let order: Vec = terms + .iter() + .map(|(t, descending)| format!("{t} {}", if *descending { "DESC" } else { "ASC" })) + .collect(); + sql.push_str(" ORDER BY "); + sql.push_str(&order.join(", ")); + } else { + if let Some(id) = &filter.before { + let started_at: Value = conn + .query_row("SELECT started_at FROM flows WHERE id = ?", [id], |r| r.get(0)) + .map_err(|_| format!("No flow {id}"))?; + sql.push_str(" AND (started_at, id) < (?, ?)"); + params.push(started_at); + params.push(id.clone().into()); + } + sql.push_str(" ORDER BY started_at DESC, id DESC"); } - sql.push_str(" ORDER BY started_at DESC, id DESC"); if let Some(limit) = filter.limit { sql.push_str(" LIMIT ?"); params.push((limit.min(i64::MAX as usize) as i64).into()); } let mut stmt = conn.prepare(&sql).map_err(|e| e.to_string())?; - let rows = stmt - .query_map(rusqlite::params_from_iter(params), Flow::from_row) - .map_err(|e| e.to_string())?; - rows.collect::, _>>().map_err(|e| e.to_string()) + let mut cursor = None; + let mut flows = Vec::new(); + let mut rows = stmt.query(rusqlite::params_from_iter(params)).map_err(|e| e.to_string())?; + while let Some(row) = rows.next().map_err(|e| e.to_string())? { + flows.push(Flow::from_row(row).map_err(|e| e.to_string())?); + if !terms.is_empty() { + let values = (0..terms.len()) + .map(|i| row.get::<_, Value>(format!("sort_{i}").as_str()).map(SortValue::from_sql)) + .collect::, _>>() + .map_err(|e| e.to_string())?; + cursor = Some(values); + } + } + Ok((flows, cursor)) } fn count_matching(conn: &rusqlite::Connection, filter: &FlowFilter) -> Result { @@ -1035,6 +1240,66 @@ mod tests { assert_eq!(matching, None); } + /// A sort on several columns orders by each in turn, text without regard to case, empty + /// values lowest and start as the last tiebreak, and pages through that order from a cursor. + #[test] + fn a_sort_on_several_columns_pages_in_its_order() { + let (_dir, db) = db_of(vec![ + flow("https://B.com/x", "GET", Some(200), None), + flow("https://a.com/y", "GET", Some(404), None), + flow("https://a.com/z", "GET", None, None), + flow("https://c.com/", "GET", Some(200), None), + flow("https://a.com/", "GET", None, Some("boom")), + flow("https://b.com/q", "GET", Some(200), None), + flow("https://A.com/", "GET", Some(404), None), + flow("https://c.com/", "GET", Some(200), None), + ]); + let key = |column, descending| FlowSortKey { column, descending }; + let read = |sort: &[FlowSortKey], cursor: Option>, limit| { + let filter = FlowFilter { + sort: sort.to_vec(), + cursor, + limit: Some(limit), + ..Default::default() + }; + let page = sorted_page(&db, &filter).unwrap(); + let ids: Vec = page.flows.iter().map(|f| f.id[3..].to_string()).collect(); + (ids, page.cursor) + }; + let sorted = |sort: &[FlowSortKey]| read(sort, None, 100).0; + let by_status = [key(SortColumn::Status, true), key(SortColumn::Host, false)]; + let all = sorted(&by_status); + assert_eq!(all, ["001", "006", "000", "005", "003", "007", "004", "002"]); + let mut paged: Vec = Vec::new(); + let mut cursor = None; + loop { + let (page, next) = read(&by_status, cursor.clone(), 3); + if page.is_empty() { + break; + } + cursor = next; + paged.extend(page); + } + assert_eq!(paged, all); + + // The last flow of a page changing does not move where the next page starts. + let (first, cursor) = read(&by_status, None, 3); + assert_eq!(first, ["001", "006", "000"]); + db.with_conn(|c| { + c.conn().resolve().execute("UPDATE flows SET status = 100 WHERE id = 'fl_000'", []) + }) + .unwrap(); + let (next, _) = read(&by_status, cursor, 3); + assert_eq!(next, ["005", "003", "007"]); + + let ascending = [key(SortColumn::Status, false), key(SortColumn::Host, false)]; + assert_eq!(sorted(&ascending), ["002", "004", "000", "005", "003", "007", "001", "006"]); + + // Query alone: the flows without one first, in start order. + let all = sorted(&[key(SortColumn::Query, true)]); + assert_eq!(all, ["000", "001", "002", "003", "004", "005", "006", "007"]); + } + #[test] fn a_search_reads_the_newest_page_through_the_index() { let (_dir, db) = db_of(vec![]); diff --git a/crates-proxy/yaak-proxy-lib/src/lib.rs b/crates-proxy/yaak-proxy-lib/src/lib.rs index cbec53467..d5cb19938 100644 --- a/crates-proxy/yaak-proxy-lib/src/lib.rs +++ b/crates-proxy/yaak-proxy-lib/src/lib.rs @@ -1655,6 +1655,9 @@ pub struct ListFlowsResponse { /// Flows the filter covers, whatever the limit, when it narrows to fewer than all of them. #[ts(type = "number | null")] pub matching: Option, + /// Sorted, the next page's `cursor`: the last flow's sort values, as read. None without a + /// sort or a flow. + pub cursor: Option>, } #[derive(Deserialize, TS)] @@ -2689,8 +2692,9 @@ fn list_actions( } fn list_flows(ctx: &ProxyCtx, req: flows::FlowFilter) -> Result { - let (flows, total, matching) = flows::page(&ctx.db()?, &req).map_err(rpc_error)?; - Ok(ListFlowsResponse { flows, total, matching }) + let flows::Page { flows, total, matching, cursor } = + flows::sorted_page(&ctx.db()?, &req).map_err(rpc_error)?; + Ok(ListFlowsResponse { flows, total, matching, cursor }) } async fn clear_flows(ctx: ProxyCtx, _req: ClearFlowsRequest) -> Result { @@ -2827,9 +2831,52 @@ struct InFlight { /// Changed since it was last written, by a message or a status. dirty: bool, last_write: Option, + /// The request body's first bytes, while it may be a GraphQL request whose operation is + /// read once it is whole. None once it can't be: not JSON, encoded, or too long. + graphql_scan: Option>, } +/// Most request body bytes read for a GraphQL operation; a longer body is not looked at. +const GRAPHQL_SCAN_LIMIT: usize = 64 * 1024; + impl InFlight { + /// Keeps a request body chunk for reading a GraphQL operation from, when the body may be + /// one: JSON, not encoded, and short. `len` is the chunk's size on the wire, `data` what of + /// it was captured. + fn scan_graphql(&mut self, first: bool, len: usize, data: &[u8]) { + if first { + let headers = &self.flow.request_headers; + let json = find_header(headers, "content-type") + .and_then(|t| t.split(';').next()) + .is_some_and(|t| t.trim().eq_ignore_ascii_case("application/json")); + let encoded = find_header(headers, "content-encoding") + .is_some_and(|e| !e.trim().eq_ignore_ascii_case("identity")); + let stream = self.flow.stream_kind.is_some(); + self.graphql_scan = (json && !encoded && !stream).then(Vec::new); + } + let Some(buf) = &mut self.graphql_scan else { + return; + }; + if data.len() < len || buf.len() + len > GRAPHQL_SCAN_LIMIT { + self.graphql_scan = None; + return; + } + buf.extend_from_slice(data); + } + + /// Reads the GraphQL operation from the request body kept by [`Self::scan_graphql`]. + fn finish_graphql_scan(&mut self) { + let Some(body) = self.graphql_scan.take() else { + return; + }; + let content_type = find_header(&self.flow.request_headers, "content-type"); + if let Some(op) = + yaak_http::graphql::detect(&self.flow.method, &self.flow.url, content_type, Some(&body)) + { + self.flow.graphql_operation = Some(op.label()); + } + } + /// Stops writing the bodies its stream's messages carry, which are read back from them. /// A request's bytes are kept while it uploads, as a call can end before its upload does, /// with a message still partway in: they are dropped once the upload ends, or kept as its @@ -3079,6 +3126,12 @@ impl FlowSink { let flow_id = Flow::generate_id(); let request_body = Flow::body_id(&flow_id, FlowBody::Request); let response_body = Flow::body_id(&flow_id, FlowBody::Response); + // GraphQL over GET is all in its URL; a body's operation is read once it is whole. + let graphql_operation = method + .eq_ignore_ascii_case("GET") + .then(|| yaak_http::graphql::detect(&method, &url, None, None)) + .flatten() + .map(|op| op.label()); let flow = Flow { id: flow_id, started_at: DateTime::::from(started_at).naive_utc(), @@ -3086,6 +3139,7 @@ impl FlowSink { url, client_http_version: http_version, client_addr: Some(client_addr.to_string()), + graphql_operation, ..Default::default() }; in_flight.insert( @@ -3100,6 +3154,7 @@ impl FlowSink { next_seq: 0, dirty: false, last_write: None, + graphql_scan: None, }, ); } @@ -3128,6 +3183,7 @@ impl FlowSink { next_seq: 0, dirty: false, last_write: None, + graphql_scan: None, }; // Shown while it is open, as tunnels often stay open for minutes. self.write_stream_flow(&mut f); @@ -3179,6 +3235,7 @@ impl FlowSink { // upload does. ProxyEvent::RequestBodyChunk { id, len, data } => { if let Some(f) = in_flight.get_mut(&id) { + f.scan_graphql(f.flow.request_size == 0, len, &data); f.flow.request_size += len as i64; if f.keeps_bytes(FlowBody::Request) { f.flow.request_truncated |= data.len() < len; @@ -3191,6 +3248,7 @@ impl FlowSink { if let Some(f) = in_flight.get_mut(&id) { f.flow.request_size = size as i64; f.flow.elapsed_request_ms = Some(clamp_ms(elapsed_ms)); + f.finish_graphql_scan(); if f.keeps_bytes(FlowBody::Request) { f.flow.request_truncated = truncated; finish_body(f.request_body.take()); @@ -4487,6 +4545,96 @@ mod tests { .unwrap(); } + /// A GraphQL request's operation is stored with the flow: from a JSON body sent in chunks, + /// and from a GET's URL. A JSON body that is not GraphQL, and one over the scan limit, store + /// none. + #[test] + fn graphql_operations_are_stored_at_capture() { + let dir = tempfile::tempdir().unwrap(); + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + let (tx, event_loop) = event_loop(&ctx); + let send = |event| tx.send(event).unwrap(); + let post = |id: u64, url: &str, chunks: &[&[u8]]| { + send(ProxyEvent::RequestStart { + id, + started_at: SystemTime::now(), + method: "POST".into(), + url: url.into(), + http_version: "HTTP/1.1".into(), + connection: 1, + client_addr: "127.0.0.1:1".parse().unwrap(), + }); + send(ProxyEvent::RequestHeader { + id, + name: "Content-Type".into(), + value: "application/json; charset=utf-8".into(), + }); + let mut size = 0; + for chunk in chunks { + size += chunk.len(); + send(ProxyEvent::RequestBodyChunk { + id, + len: chunk.len(), + data: bytes::Bytes::copy_from_slice(chunk), + }); + } + send(ProxyEvent::RequestBodyComplete { + id, + size: size as u64, + truncated: false, + elapsed_ms: 1, + }); + send(ProxyEvent::ResponseStart { + id, + status: 200, + http_version: "HTTP/1.1".into(), + remote_addr: None, + elapsed_ms: 2, + }); + send(ProxyEvent::ResponseBodyComplete { + id, + size: 0, + truncated: false, + elapsed_ms: 3, + error: None, + cancelled: false, + }); + }; + post( + 1, + "https://api.example.com/graphql", + &[ + br#"{"query":"query GetUser($id: ID!) "#, + br#"{ user(id: $id) { id } }"}"#, + ], + ); + post(2, "https://api.example.com/users", &[br#"{"query":"red shoes"}"#]); + let long = format!( + r#"{{"query":"query Big {{ a }}","variables":{{"pad":"{}"}}}}"#, + "x".repeat(GRAPHQL_SCAN_LIMIT) + ); + post(3, "https://api.example.com/big/graphql", &[long.as_bytes()]); + send_flow(&tx, 4, "https://api.example.com/graphql?query=query%20Feed%20%7B%20a%20%7D"); + drop(tx); + event_loop.join().unwrap(); + + let flows = flows::list(&ctx.db().unwrap(), &Default::default()).unwrap(); + assert_eq!(flows.len(), 4); + let operation = |url: &str| { + let flow = flows.iter().find(|f| f.url == url).unwrap(); + flow.graphql_operation.clone() + }; + assert_eq!(operation("https://api.example.com/graphql").as_deref(), Some("GetUser")); + assert_eq!(operation("https://api.example.com/users"), None); + assert_eq!(operation("https://api.example.com/big/graphql"), None); + assert_eq!( + operation("https://api.example.com/graphql?query=query%20Feed%20%7B%20a%20%7D") + .as_deref(), + Some("Feed") + ); + } + /// Flows that end together are written together, and every one the engine sent is written /// when it stops, mid-batch or not, each with its body. #[test] @@ -4545,6 +4693,7 @@ mod tests { next_seq: 0, dirty: false, last_write: None, + graphql_scan: None, }; sink.finish_flow(f); assert!(!sink.finished_due(), "the clock stands still"); diff --git a/crates-proxy/yaak-proxy-lib/src/models.rs b/crates-proxy/yaak-proxy-lib/src/models.rs index 2d922fd07..8c03f4aa9 100644 --- a/crates-proxy/yaak-proxy-lib/src/models.rs +++ b/crates-proxy/yaak-proxy-lib/src/models.rs @@ -103,6 +103,9 @@ pub struct Flow { pub grpc_status: Option, /// A gRPC call's `grpc-message`, decoded. pub grpc_message: Option, + /// A GraphQL request's operation, from its body or URL: `GetUser`, `GetUser +2` for a + /// batch, empty for an anonymous one. None when the request is not GraphQL. + pub graphql_operation: Option, /// An HTTPS connection passed through undecrypted, as the decrypt list leaves it out or its /// client rejected the proxy's certificate: method `CONNECT`, its URL the host, its sizes /// the bytes each way, and no headers or bodies. Open until `elapsedMs` or `error` is set. @@ -273,6 +276,7 @@ impl UpsertModelInfo for Flow { (GrpcMethod, self.grpc_method.into()), (GrpcStatus, self.grpc_status.into()), (GrpcMessage, self.grpc_message.into()), + (GraphqlOperation, self.graphql_operation.into()), (Tunnelled, self.tunnelled.into()), (ElapsedConnectMs, self.elapsed_connect_ms.into()), (ElapsedTlsMs, self.elapsed_tls_ms.into()), @@ -316,6 +320,7 @@ impl UpsertModelInfo for Flow { FlowIden::GrpcMethod, FlowIden::GrpcStatus, FlowIden::GrpcMessage, + FlowIden::GraphqlOperation, FlowIden::Tunnelled, FlowIden::ElapsedConnectMs, FlowIden::ElapsedTlsMs, @@ -374,6 +379,7 @@ impl UpsertModelInfo for Flow { grpc_method: added_column(r, "grpc_method")?, grpc_status: added_column(r, "grpc_status")?, grpc_message: added_column(r, "grpc_message")?, + graphql_operation: added_column(r, "graphql_operation")?, tunnelled: added_column(r, "tunnelled")?.unwrap_or_default(), agent_read: added_column(r, "agent_read")?.unwrap_or_default(), }) diff --git a/crates-proxy/yaak-proxy-lib/src/query.rs b/crates-proxy/yaak-proxy-lib/src/query.rs index a91185092..6359c3d0d 100644 --- a/crates-proxy/yaak-proxy-lib/src/query.rs +++ b/crates-proxy/yaak-proxy-lib/src/query.rs @@ -37,6 +37,7 @@ pub const FLAGS: &[&str] = &[ "ws", "sse", "grpc", + "graphql", ]; /// Flags `@is:` will take once flows have what they need. Valid, matching nothing. @@ -100,6 +101,14 @@ pub fn schema() -> &'static Schema { field("id", number(), "Flow number", "@id:>1200"), field("cwd", text(), "Working folder", "@cwd:myapp"), field("pid", number(), "Process id", "@pid:4121"), + field( + "graphql.operation", + text(), + "GraphQL operation name", + "@graphql.operation:GetUser", + ), + field("grpc.service", text(), "gRPC service", "@grpc.service:pkg.Greeter"), + field("grpc.method", text(), "gRPC method", "@grpc.method:SayHello"), field("body", text(), "Not yet", ""), field("req.body", text(), "Not yet", ""), field("res.body", text(), "Not yet", ""), @@ -471,9 +480,10 @@ const SEEN_VALUES: i64 = 200; const HEADER_VALUES: i64 = 20; /// The values seen for a field path, for completing it, most recent first: hosts, methods, -/// statuses, content types, protocols, schemes, apps, working folders, `@is` flags, and header -/// names, or a header's values when it has had few. Nothing for fields whose values are -/// near-unique (URLs, paths, ids, sizes, times, query parameters). +/// statuses, content types, protocols, schemes, apps, working folders, GraphQL operations, gRPC +/// services and methods, `@is` flags, header names, or a header's values when it has had few, +/// and query parameter names. Nothing for fields whose values are near-unique (URLs, paths, ids, +/// sizes, times, query parameter values). pub fn seen_values(db: &crate::db::ProxyQueryManager, path: &str) -> Result, String> { let path = path.to_lowercase(); let column = match path.as_str() { @@ -486,6 +496,10 @@ pub fn seen_values(db: &crate::db::ProxyQueryManager, path: &str) -> Result "url_scheme(url)", "app" => "client_name", "cwd" => "folder_name(client_cwd)", + "graphql.operation" => "graphql_operation", + "grpc.service" => "grpc_service", + "grpc.method" => "grpc_method", + "query" => return seen_query_names(db), "header" | "req.header" | "res.header" => { return seen_headers(db, &path, None); } @@ -524,6 +538,17 @@ fn newest_flows() -> String { ) } +/// The names of the query parameters seen, for completing `@query.`. +fn seen_query_names(db: &crate::db::ProxyQueryManager) -> Result, String> { + let sql = format!( + "SELECT q_lower(json_extract(kv.value, '$.name')) AS name FROM ({}) AS f, \ + json_each(url_query_json(f.url)) AS kv WHERE name IS NOT NULL AND name != '' \ + GROUP BY name ORDER BY max(f.rid) DESC LIMIT {SEEN_VALUES}", + newest_flows() + ); + select_strings(db, &sql, None) +} + /// Header names under a map, or one header's values when it has had few. fn seen_headers( db: &crate::db::ProxyQueryManager, @@ -723,6 +748,16 @@ pub fn url_sidebar_path(url: &str) -> Option { (normalized != path && !path.is_empty()).then_some(normalized) } +/// The authority as written, user info and port included: the list's Host column. +pub fn url_authority(url: &str) -> &str { + url_parts(url).authority +} + +/// The query string as written, without its `?`: the list's Query column. +pub fn url_query_string(url: &str) -> Option<&str> { + url_parts(url).query +} + /// The query's parameters, decoded, their names lowercased. pub fn url_query(url: &str) -> Vec<(String, String)> { let Some(query) = url_parts(url).query else { @@ -784,6 +819,7 @@ fn flags(flow: &Flow) -> Vec<&'static str> { "ws" => stream == Some("websocket"), "sse" => stream == Some("sse"), "grpc" => matches!(stream, Some("grpc" | "grpc_web")), + "graphql" => flow.graphql_operation.is_some(), _ => false, }; FLAGS.iter().copied().filter(|f| set(f)).collect() @@ -862,6 +898,16 @@ pub fn document(flow: &Flow) -> Document { if let Some(pid) = flow.client_pid { add("pid", num(f64::from(pid))); } + // An anonymous operation is GraphQL with no name: `@is:graphql`, not `@graphql.operation:*`. + if let Some(op) = flow.graphql_operation.as_deref().filter(|op| !op.is_empty()) { + add("graphql.operation", text(op)); + } + if let Some(service) = &flow.grpc_service { + add("grpc.service", text(service)); + } + if let Some(method) = &flow.grpc_method { + add("grpc.method", text(method)); + } for (side, headers) in [ ("req", &flow.request_headers), ("res", &flow.response_headers), @@ -919,7 +965,7 @@ impl FlowColumns { } /// `name`, or what the model reads in its place when the table lacks it. - fn col(&self, name: &str) -> String { + pub(crate) fn col(&self, name: &str) -> String { if self.0.contains(name) { return name.to_string(); } @@ -1005,6 +1051,9 @@ impl Sql<'_> { "req.size" => (vec!["request_size".into()], true), "duration" => (vec!["elapsed_ms".into()], true), "pid" => (vec![c("client_pid")], true), + "graphql.operation" => (vec![format!("nullif({}, '')", c("graphql_operation"))], false), + "grpc.service" => (vec![c("grpc_service")], false), + "grpc.method" => (vec![c("grpc_method")], false), "app" => { let (bundle, path, name) = (c("client_bundle_id"), c("client_path"), c("client_name")); @@ -1103,6 +1152,7 @@ impl Sql<'_> { "ws" => format!("{} = 'websocket'", c("stream_kind")), "sse" => format!("{} = 'sse'", c("stream_kind")), "grpc" => format!("{} IN ('grpc', 'grpc_web')", c("stream_kind")), + "graphql" => format!("{} IS NOT NULL", c("graphql_operation")), _ => "0".into(), } } @@ -1214,6 +1264,12 @@ pub(crate) fn add_functions(conn: &rusqlite::Connection) -> rusqlite::Result<()> conn.create_scalar_function("url_path", 1, flags, move |ctx| { Ok(url_path(&text_arg(ctx, 0)?.unwrap_or_default()).map(str::to_string)) })?; + conn.create_scalar_function("url_authority", 1, flags, move |ctx| { + Ok(url_authority(&text_arg(ctx, 0)?.unwrap_or_default()).to_string()) + })?; + conn.create_scalar_function("url_query_string", 1, flags, move |ctx| { + Ok(url_query_string(&text_arg(ctx, 0)?.unwrap_or_default()).map(str::to_string)) + })?; conn.create_scalar_function("url_sidebar_path", 1, flags, move |ctx| { Ok(url_sidebar_path(&text_arg(ctx, 0)?.unwrap_or_default())) })?; diff --git a/crates-proxy/yaak-proxy-lib/src/query/tests.rs b/crates-proxy/yaak-proxy-lib/src/query/tests.rs index 499ecf52a..6e88d6329 100644 --- a/crates-proxy/yaak-proxy-lib/src/query/tests.rs +++ b/crates-proxy/yaak-proxy-lib/src/query/tests.rs @@ -690,6 +690,13 @@ fn sample(count: usize) -> Vec { client_pid: rng.chance(50).then(|| (rng.next() % 1000) as i32), stream_kind: *rng.pick(&streams), grpc_status: *rng.pick(&[None, Some(0), Some(5)]), + grpc_service: rng + .chance(20) + .then(|| rng.pick(&["pkg.Greeter", "Users"]).to_string()), + grpc_method: rng.chance(20).then(|| rng.pick(&["SayHello", "getUser"]).to_string()), + graphql_operation: rng + .chance(25) + .then(|| rng.pick(&["GetUser", "", "Feed +2", "createÉtat"]).to_string()), error: rng.chance(15).then(|| "connection reset".to_string()), cancelled: rng.chance(10), saved_request_id: rng.chance(15).then(|| "rq_x".to_string()), @@ -859,6 +866,18 @@ const BATTERY: &[&str] = &[ "@is:ws", "@is:sse", "@is:grpc", + "@is:graphql", + "-@is:graphql", + "@graphql.operation:getuser", + "@graphql.operation:feed*", + "@graphql.operation:*", + "-@graphql.operation:*", + "@graphql.operation:createétat", + "@grpc.service:pkg.greeter", + "@grpc.service:*", + "@grpc.method:SayHello", + "-@grpc.method:*", + "@grpc.service:users @grpc.method:get*", "@is:*", "-@is:*", "@is:s*", @@ -961,11 +980,58 @@ fn seen_values_for_completion() { "size", "duration", "time", - "query", "query.page", ] { assert!(values(path).is_empty(), "{path}"); } + // A query parameter's names, for `@query.`, but not its values + let mut params = values("query"); + params.sort(); + assert_eq!(params, vec!["page", "sort"]); +} + +/// GraphQL operations and gRPC calls, by name and by flag, and offered for completion. +#[test] +fn graphql_and_grpc_fields() { + let base = |id: &str, url: &str| Flow { + id: id.into(), + url: url.into(), + method: "POST".into(), + started_at: now().naive_utc(), + ..Default::default() + }; + let flows = vec![ + Flow { + graphql_operation: Some("GetUser".into()), + ..base("fl_named", "https://api.example.com/graphql") + }, + Flow { + graphql_operation: Some(String::new()), + ..base("fl_anonymous", "https://api.example.com/graphql") + }, + Flow { + grpc_service: Some("pkg.Greeter".into()), + grpc_method: Some("SayHello".into()), + ..base("fl_call", "https://grpc.example.com/pkg.Greeter/SayHello") + }, + base("fl_plain", "https://example.com/"), + ]; + let (_dir, db, all) = db_of(flows); + let finds = |query: &str| { + let prepared = FlowQuery::at(query, false, now()); + let sql = sql_ids(&db, &prepared); + assert_eq!(sql, memory_ids(&all, &prepared), "the two disagree on {query:?}"); + sql + }; + assert_eq!(finds("@is:graphql"), ids(&["fl_anonymous", "fl_named"])); + assert_eq!(finds("@graphql.operation:getuser"), ids(&["fl_named"])); + assert_eq!(finds("@graphql.operation:*"), ids(&["fl_named"])); + assert_eq!(finds("@grpc.service:pkg.Greeter"), ids(&["fl_call"])); + assert_eq!(finds("@grpc.method:say*"), ids(&["fl_call"])); + assert_eq!(finds("-@grpc.method:*"), ids(&["fl_anonymous", "fl_named", "fl_plain"])); + assert_eq!(seen_values(&db, "graphql.operation").unwrap(), vec!["GetUser"]); + assert_eq!(seen_values(&db, "grpc.service").unwrap(), vec!["pkg.Greeter"]); + assert_eq!(seen_values(&db, "grpc.method").unwrap(), vec!["SayHello"]); } #[test] diff --git a/crates-tauri/yaak-app-proxy/src/lib.rs b/crates-tauri/yaak-app-proxy/src/lib.rs index dc5360ce1..fcebddd1e 100644 --- a/crates-tauri/yaak-app-proxy/src/lib.rs +++ b/crates-tauri/yaak-app-proxy/src/lib.rs @@ -36,6 +36,11 @@ fn setup_window_menu(win: &WebviewWindow) { let _ = w.close(); }); } + window_menu::EXPORT_HAR_ID => { + if let Err(e) = webview_window.emit(window_menu::EXPORT_HAR_EVENT, ()) { + error!("Failed to emit the HAR export: {e}"); + } + } "dev.refresh" => webview_window.eval("location.reload()").unwrap(), "dev.toggle_devtools" => { if webview_window.is_devtools_open() { diff --git a/crates-tauri/yaak-app-proxy/src/window_menu.rs b/crates-tauri/yaak-app-proxy/src/window_menu.rs index b71433af0..c09faed14 100644 --- a/crates-tauri/yaak-app-proxy/src/window_menu.rs +++ b/crates-tauri/yaak-app-proxy/src/window_menu.rs @@ -5,6 +5,12 @@ use tauri::menu::{ WINDOW_SUBMENU_ID, }; +/// File's Export All Flows as HAR, which the window carries out, as the list's menus do. +pub const EXPORT_HAR_ID: &str = "file.export_har"; + +/// The event the window hears File's Export All Flows as HAR by. +pub const EXPORT_HAR_EVENT: &str = "menu_export_har"; + pub fn app_menu(app_handle: &AppHandle) -> tauri::Result> { let pkg_info = app_handle.package_info(); let config = app_handle.config(); @@ -87,6 +93,9 @@ pub fn app_menu(app_handle: &AppHandle) -> tauri::Result> "File", true, &[ + &MenuItemBuilder::with_id(EXPORT_HAR_ID, "Export All Flows as HAR") + .build(app_handle)?, + &PredefinedMenuItem::separator(app_handle)?, &PredefinedMenuItem::close_window(app_handle, None)?, #[cfg(not(target_os = "macos"))] &PredefinedMenuItem::quit(app_handle, None)?, diff --git a/crates/yaak-http/src/graphql.rs b/crates/yaak-http/src/graphql.rs new file mode 100644 index 000000000..e764b15cc --- /dev/null +++ b/crates/yaak-http/src/graphql.rs @@ -0,0 +1,612 @@ +//! GraphQL request detection, ported from the curl importer +//! (`plugins/importer-curl/src/graphql.ts`) so the proxy and the importer agree. + +use serde_json::{Map, Value}; +use url::Url; + +/// A GraphQL request's JSON envelope. +#[derive(Debug, Clone, PartialEq)] +pub struct GraphQlBody { + pub query: String, + /// The `variables` value as sent (object, string, or other), None when absent or null. + pub variables: Option, + pub operation_name: Option, +} + +impl GraphQlBody { + /// `operationName` when present and non-blank, else the first named operation in `query`. + pub fn operation(&self) -> Option { + match self.operation_name.as_deref().map(str::trim) { + Some(name) if !name.is_empty() => Some(name.to_string()), + _ => first_operation_name(&self.query), + } + } +} + +/// A detected GraphQL request, as the proxy labels it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GraphQlOperation { + /// The operation's name; for a batch, the first named operation in it. None when anonymous. + pub name: Option, + /// How many operations the request carries: 1, or the batch's length. + pub count: usize, +} + +impl GraphQlOperation { + /// The name, with ` +N` for the rest of a batch (`GetUser +2`); empty when anonymous. + pub fn label(&self) -> String { + match &self.name { + None => String::new(), + Some(name) if self.count > 1 => format!("{name} +{}", self.count - 1), + Some(name) => name.clone(), + } + } +} + +/// A JSON GraphQL envelope that passes the importer's scored check. +pub fn parse_json_body(content_type: Option<&str>, text: &str, url: &str) -> Option { + if !is_json(content_type) { + return None; + } + match serde_json::from_str::(text).ok()? { + Value::Object(body) => check_envelope(&body, &url_path(url)), + _ => None, + } +} + +/// A non-empty JSON array whose every element passes `parse_json_body`'s check. +pub fn parse_json_batch( + content_type: Option<&str>, + text: &str, + url: &str, +) -> Option> { + if !is_json(content_type) { + return None; + } + let Value::Array(items) = serde_json::from_str::(text).ok()? else { + return None; + }; + if items.is_empty() { + return None; + } + let path = url_path(url); + items + .iter() + .map(|item| match item { + Value::Object(body) => check_envelope(body, &path), + _ => None, + }) + .collect() +} + +/// GraphQL over GET: the URL's `query`, `operationName` and `variables` parameters, scored like a +/// body. +pub fn parse_get(url: &str) -> Option { + let parsed = Url::parse(url).ok()?; + let mut query = None; + let mut operation_name = None; + let mut variables = None; + for (key, value) in parsed.query_pairs() { + match key.as_ref() { + "query" if query.is_none() => query = Some(value.into_owned()), + "operationName" if operation_name.is_none() => { + operation_name = Some(value.into_owned()) + } + "variables" if variables.is_none() && !value.is_empty() => { + variables = Some(Value::String(value.into_owned())) + } + _ => {} + } + } + let body = GraphQlBody { query: query?, variables, operation_name }; + passes(&body, parsed.path()).then_some(body) +} + +/// The proxy's check: GET reads the URL, anything else the body as one envelope or a batch. +pub fn detect( + method: &str, + url: &str, + content_type: Option<&str>, + body: Option<&[u8]>, +) -> Option { + if method.eq_ignore_ascii_case("GET") { + let body = parse_get(url)?; + return Some(GraphQlOperation { name: body.operation(), count: 1 }); + } + let text = std::str::from_utf8(body?).ok()?; + if let Some(body) = parse_json_body(content_type, text, url) { + return Some(GraphQlOperation { name: body.operation(), count: 1 }); + } + let batch = parse_json_batch(content_type, text, url)?; + Some(GraphQlOperation { + name: batch.iter().find_map(GraphQlBody::operation), + count: batch.len(), + }) +} + +fn is_json(content_type: Option<&str>) -> bool { + content_type + .and_then(|ct| ct.split(';').next()) + .is_some_and(|essence| essence.trim().eq_ignore_ascii_case("application/json")) +} + +fn url_path(url: &str) -> String { + match Url::parse(url) { + Ok(parsed) => parsed.path().to_string(), + Err(_) => url.to_string(), + } +} + +fn check_envelope(body: &Map, path: &str) -> Option { + let query = body.get("query")?.as_str()?; + if body.keys().any(|k| !matches!(k.as_str(), "query" | "variables" | "operationName")) { + return None; + } + let parsed = GraphQlBody { + query: query.to_string(), + variables: body.get("variables").filter(|v| !v.is_null()).cloned(), + operation_name: body.get("operationName").and_then(Value::as_str).map(str::to_string), + }; + passes(&parsed, path).then_some(parsed) +} + +/// The importer's scored signals: a document signal and a score of at least 4. +fn passes(body: &GraphQlBody, path: &str) -> bool { + let mut score = 0; + let mut has_document = false; + let query = body.query.as_str(); + // From the document's first token: comments and whitespace before it don't count. + let trimmed = skip_leading_ignored(query).trim_end(); + + if has_graphql_path_word(&path.to_lowercase()) { + score += 2; + } + + if ["query", "mutation", "subscription", "fragment"] + .iter() + .any(|keyword| starts_with_word(trimmed, keyword)) + { + score += 3; + } else if trimmed.len() >= 2 && trimmed.starts_with('{') && trimmed.ends_with('}') { + score += 3; + has_document = true; + } + + if let Some(open) = query.find('{') + && query[open..].contains('}') + { + score += 1; + has_document = true; + } + + if body.operation_name.as_deref().is_some_and(|name| !name.trim().is_empty()) { + score += 1; + } + + if matches!(body.variables, Some(Value::Object(_) | Value::Array(_) | Value::String(_))) { + score += 1; + } + + has_document && score >= 4 +} + +/// `document` from its first token: GraphQL's ignored tokens before it (whitespace, commas, a +/// byte order mark and `#` comments to the end of their line) left out. +fn skip_leading_ignored(document: &str) -> &str { + let mut rest = document; + loop { + let trimmed = + rest.trim_start_matches(|c: char| c.is_whitespace() || c == ',' || c == '\u{feff}'); + match trimmed.strip_prefix('#') { + Some(comment) => rest = comment.find(['\n', '\r']).map_or("", |end| &comment[end..]), + None => return trimmed, + } + } +} + +fn is_word_byte(b: u8) -> bool { + b.is_ascii_alphanumeric() || b == b'_' +} + +/// `^keyword\b` +fn starts_with_word(text: &str, keyword: &str) -> bool { + text.starts_with(keyword) + && !text.as_bytes().get(keyword.len()).copied().is_some_and(is_word_byte) +} + +/// `\b(graphql|gql)\b` +fn has_graphql_path_word(path: &str) -> bool { + let bytes = path.as_bytes(); + ["graphql", "gql"].iter().any(|word| { + path.match_indices(word).any(|(start, _)| { + let end = start + word.len(); + let before = start > 0 && is_word_byte(bytes[start - 1]); + let after = bytes.get(end).copied().is_some_and(is_word_byte); + !before && !after + }) + }) +} + +/// The first named `query`, `mutation` or `subscription` at the top level of a document, by a +/// token scan that skips comments and strings. Fragments are not operations. +pub fn first_operation_name(document: &str) -> Option { + let bytes = document.as_bytes(); + let mut i = 0; + let mut braces = 0usize; + let mut parens = 0usize; + let mut definition_start = true; + // Set after an operation keyword at a definition's start: the next token may be its name. + let mut after_keyword = false; + + while i < bytes.len() { + let b = bytes[i]; + match b { + b'#' => { + while i < bytes.len() && bytes[i] != b'\n' && bytes[i] != b'\r' { + i += 1; + } + continue; + } + b'"' if bytes[i..].starts_with(b"\"\"\"") => { + after_keyword = false; + i += 3; + while i < bytes.len() && !bytes[i..].starts_with(b"\"\"\"") { + i += if bytes[i..].starts_with(b"\\\"\"\"") { 4 } else { 1 }; + } + i += 3; + continue; + } + b'"' => { + after_keyword = false; + i += 1; + while i < bytes.len() && !matches!(bytes[i], b'"' | b'\n' | b'\r') { + i += if bytes[i] == b'\\' { 2 } else { 1 }; + } + i += 1; + continue; + } + b'_' | b'A'..=b'Z' | b'a'..=b'z' => { + let start = i; + while i < bytes.len() && is_word_byte(bytes[i]) { + i += 1; + } + let name = &document[start..i]; + if after_keyword { + return Some(name.to_string()); + } + if definition_start && braces == 0 && parens == 0 { + after_keyword = matches!(name, "query" | "mutation" | "subscription"); + definition_start = false; + } + continue; + } + b' ' | b'\t' | b'\n' | b'\r' | b',' => {} + _ => { + after_keyword = false; + match b { + b'{' => { + braces += 1; + definition_start = false; + } + b'}' => { + braces = braces.saturating_sub(1); + if braces == 0 && parens == 0 { + definition_start = true; + } + } + b'(' => parens += 1, + b')' => parens = parens.saturating_sub(1), + _ => {} + } + } + } + i += 1; + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + const JSON: Option<&str> = Some("application/json"); + + fn is_body(content_type: Option<&str>, body: Value, url: &str) -> bool { + parse_json_body(content_type, &body.to_string(), url).is_some() + } + + // Carried over from plugins/importer-curl/tests/graphql.test.ts. + + #[test] + fn detects_named_query_documents_without_a_graphql_url() { + let text = json!({ + "query": "query Search($id: ID!) { node(id: $id) { id } }", + "variables": { "id": "123" }, + "operationName": "Search", + }) + .to_string(); + assert_eq!( + parse_json_body(JSON, &text, "https://api.example.com/search"), + Some(GraphQlBody { + query: "query Search($id: ID!) { node(id: $id) { id } }".into(), + variables: Some(json!({ "id": "123" })), + operation_name: Some("Search".into()), + }) + ); + } + + #[test] + fn detects_mutation_documents() { + let body = json!({ "query": "mutation Save { saveThing { id } }" }); + assert!(is_body(JSON, body, "https://api.example.com")); + } + + #[test] + fn detects_anonymous_selection_set_documents() { + let body = json!({ "query": "{ viewer { id email } }" }); + assert!(is_body(JSON, body, "https://api.example.com")); + } + + #[test] + fn detects_document_bodies_on_graphql_looking_paths() { + let body = json!({ "query": "query Search { viewer { id } }", "operationName": "Search" }); + assert!(is_body(JSON, body, "https://api.example.com/v1/graphql")); + } + + #[test] + fn rejects_incomplete_operation_documents_on_graphql_paths() { + let body = json!({ "query": "query Search", "operationName": "Search" }); + assert!(!is_body(JSON, body, "https://api.example.com/graphql")); + } + + #[test] + fn rejects_plain_json_query_fields_on_graphql_paths() { + let body = json!({ "query": "SearchQueryInput!" }); + assert!(!is_body(JSON, body, "https://api.example.com/graphql")); + } + + #[test] + fn rejects_variables_and_operation_name_alone() { + let body = json!({ + "query": "SearchQueryInput!", + "variables": { "id": "123" }, + "operationName": "Search", + }); + assert!(!is_body(JSON, body, "https://api.example.com")); + } + + #[test] + fn detects_bodies_with_string_variables_without_parsing_them() { + let text = json!({ + "query": "query Search($id: ID!) { node(id: $id) { id } }", + "variables": "{ \"id\": \"123\" }", + }) + .to_string(); + assert_eq!( + parse_json_body(JSON, &text, "https://api.example.com"), + Some(GraphQlBody { + query: "query Search($id: ID!) { node(id: $id) { id } }".into(), + variables: Some(json!("{ \"id\": \"123\" }")), + operation_name: None, + }) + ); + } + + #[test] + fn rejects_envelopes_with_extra_fields() { + let body = json!({ + "query": "query Search($id: ID!) { node(id: $id) { id } }", + "variables": { "id": "123" }, + "extensions": { "persistedQuery": { "version": 1, "sha256Hash": "abc123" } }, + }); + assert!(!is_body(JSON, body, "https://api.example.com/graphql")); + } + + #[test] + fn ignores_invalid_json_and_non_object_json() { + assert_eq!(parse_json_body(JSON, "not json", "https://api.example.com/graphql"), None); + assert_eq!(parse_json_body(JSON, "[]", "https://api.example.com/graphql"), None); + } + + #[test] + fn ignores_non_json_mime_types() { + let body = json!({ "query": "query Search { viewer { id } }" }); + assert!(!is_body(Some("text/plain"), body, "https://api.example.com/graphql")); + } + + // Additions for the proxy. + + #[test] + fn content_type_essence_ignores_parameters_and_case() { + let body = json!({ "query": "query Search { viewer { id } }" }); + let ct = Some(" Application/JSON ; charset=utf-8"); + assert!(is_body(ct, body.clone(), "https://api.example.com")); + assert!(!is_body(None, body, "https://api.example.com")); + } + + #[test] + fn first_operation_name_reads_named_operations() { + assert_eq!(first_operation_name("query GetUser { user { id } }"), Some("GetUser".into())); + assert_eq!( + first_operation_name("mutation SaveUser($u: In!) { save(u: $u) { id } }"), + Some("SaveUser".into()) + ); + assert_eq!( + first_operation_name("subscription OnMessage { message { body } }"), + Some("OnMessage".into()) + ); + } + + #[test] + fn first_operation_name_skips_anonymous_operations() { + assert_eq!(first_operation_name("{ viewer { id } }"), None); + assert_eq!(first_operation_name("query { viewer { id } }"), None); + assert_eq!(first_operation_name("query($id: ID) { node(id: $id) { id } }"), None); + assert_eq!( + first_operation_name("{ viewer { id } } query Named { viewer { id } }"), + Some("Named".into()) + ); + } + + #[test] + fn first_operation_name_handles_variables_and_directives() { + let doc = r#"query GetUser($id: ID! = "x", $f: In = {a: 1}) @cached(ttl: 60) { + user(id: $id) @include(if: true) { name } + }"#; + assert_eq!(first_operation_name(doc), Some("GetUser".into())); + } + + #[test] + fn first_operation_name_ignores_keywords_in_strings_and_comments() { + let doc = r#" + # query Commented { x } + { search(text: "query Quoted { x }") { id } } + query Real { id } + "#; + assert_eq!(first_operation_name(doc), Some("Real".into())); + } + + /// Comments and whitespace before a document's first token don't hide its keyword or its + /// selection set, from detection or the name scan. + #[test] + fn detects_documents_after_leading_comments() { + let commented = "# viewer query\nquery GetViewer { viewer { id } }"; + let body = json!({ "query": commented }); + assert!(is_body(JSON, body.clone(), "https://api.example.com/graphql")); + let op = detect( + "POST", + "https://api.example.com/graphql", + JSON, + Some(body.to_string().as_bytes()), + ); + assert_eq!(op.map(|o| o.label()).as_deref(), Some("GetViewer")); + + for document in [ + "\n\n # one\r\n# two\n mutation Save { save { id } }", + "#only a comment line\nsubscription OnEvent { event { id } }", + "\u{feff} , # list\n{ viewer { id } }", + ] { + let body = json!({ "query": document }); + assert!(is_body(JSON, body, "https://api.example.com/graphql"), "{document:?}"); + } + // A selection set alone after a comment needs no GraphQL path. + let bare = json!({ "query": "# who\n{ viewer { id } }", "variables": {} }); + assert!(is_body(JSON, bare, "https://api.example.com/v1")); + + assert_eq!( + first_operation_name("# c\n # query Hidden { x }\n query Shown { a }"), + Some("Shown".into()) + ); + // A comment and nothing else is no document. + assert!(!is_body( + JSON, + json!({ "query": "# query X { a }" }), + "https://api.example.com/graphql" + )); + } + + #[test] + fn first_operation_name_skips_fragments() { + let doc = + "fragment UserFields on User { id name } query GetUser { user { ...UserFields } }"; + assert_eq!(first_operation_name(doc), Some("GetUser".into())); + assert_eq!(first_operation_name("fragment query on User { id }"), None); + } + + #[test] + fn first_operation_name_skips_block_strings() { + let doc = r#"{ a(text: """ query Hidden { "x" } \""" still """) } mutation Shown { b }"#; + assert_eq!(first_operation_name(doc), Some("Shown".into())); + } + + #[test] + fn operation_prefers_operation_name() { + let body = GraphQlBody { + query: "query FromDoc { x }".into(), + variables: None, + operation_name: Some("FromParam".into()), + }; + assert_eq!(body.operation(), Some("FromParam".into())); + let body = GraphQlBody { operation_name: Some(" ".into()), ..body }; + assert_eq!(body.operation(), Some("FromDoc".into())); + } + + #[test] + fn parse_get_reads_operation_name_param() { + let url = "https://api.example.com/graphql?query=%7B%20viewer%20%7B%20id%20%7D%20%7D\ + &operationName=Viewer&variables=%7B%22a%22%3A1%7D"; + let body = parse_get(url).unwrap(); + assert_eq!(body.query, "{ viewer { id } }"); + assert_eq!(body.variables, Some(json!("{\"a\":1}"))); + assert_eq!(body.operation(), Some("Viewer".into())); + } + + #[test] + fn parse_get_reads_name_from_document() { + let url = "https://api.example.com/api?query=query+GetUser+%7B+user+%7B+id+%7D+%7D"; + let op = detect("get", url, None, None).unwrap(); + assert_eq!(op, GraphQlOperation { name: Some("GetUser".into()), count: 1 }); + } + + #[test] + fn parse_get_rejects_plain_search_queries() { + assert_eq!(parse_get("https://shop.example.com/search?query=red+shoes"), None); + assert_eq!(parse_get("https://shop.example.com/graphql?q=shoes"), None); + assert_eq!( + detect("GET", "https://shop.example.com/search?query=red+shoes", None, None), + None + ); + } + + #[test] + fn detect_labels_batches() { + let text = json!([ + { "query": "query A { a }" }, + { "query": "query B { b }" }, + { "query": "query C { c }" }, + ]) + .to_string(); + let url = "https://api.example.com/graphql"; + assert_eq!(parse_json_batch(JSON, &text, url).map(|b| b.len()), Some(3)); + let op = detect("POST", url, JSON, Some(text.as_bytes())).unwrap(); + assert_eq!(op, GraphQlOperation { name: Some("A".into()), count: 3 }); + assert_eq!(op.label(), "A +2"); + } + + #[test] + fn detect_labels_batches_by_first_named_operation() { + let text = json!([{ "query": "{ a }" }, { "query": "query B { b }" }]).to_string(); + let url = "https://api.example.com/graphql"; + let op = detect("POST", url, JSON, Some(text.as_bytes())).unwrap(); + assert_eq!(op.label(), "B +1"); + } + + #[test] + fn batch_rejects_empty_or_mixed_arrays() { + let url = "https://api.example.com/graphql"; + assert_eq!(parse_json_batch(JSON, "[]", url), None); + let text = json!([{ "query": "query A { a }" }, { "foo": 1 }]).to_string(); + assert_eq!(parse_json_batch(JSON, &text, url), None); + } + + #[test] + fn detect_ignores_non_json_content_types_and_bad_bodies() { + let url = "https://api.example.com/graphql"; + let text = json!({ "query": "query A { a }" }).to_string(); + assert_eq!(detect("POST", url, Some("text/plain"), Some(text.as_bytes())), None); + assert_eq!(detect("POST", url, JSON, None), None); + assert_eq!(detect("POST", url, JSON, Some(&[0xff, 0xfe])), None); + } + + #[test] + fn label_is_empty_for_anonymous_operations() { + let url = "https://api.example.com/graphql"; + let text = json!({ "query": "{ viewer { id } }" }).to_string(); + let op = detect("POST", url, JSON, Some(text.as_bytes())).unwrap(); + assert_eq!(op, GraphQlOperation { name: None, count: 1 }); + assert_eq!(op.label(), ""); + assert_eq!(GraphQlOperation { name: None, count: 3 }.label(), ""); + assert_eq!(GraphQlOperation { name: Some("One".into()), count: 1 }.label(), "One"); + } +} diff --git a/crates/yaak-http/src/lib.rs b/crates/yaak-http/src/lib.rs index 2a4155c9e..dfbf03d9d 100644 --- a/crates/yaak-http/src/lib.rs +++ b/crates/yaak-http/src/lib.rs @@ -4,6 +4,7 @@ pub mod cookies; pub mod decompress; pub mod dns; pub mod error; +pub mod graphql; pub mod manager; mod proto; pub mod sender; From e870604607b9f22f3f7b6232e69575b74f809c59 Mon Sep 17 00:00:00 2001 From: Gregory Schier Date: Tue, 10 Nov 2026 00:46:27 -0700 Subject: [PATCH 19/45] Sort Yaak Proxy's custom columns, and complete Add Column's field alone A sort key can now name a header or query parameter column (`field:header.x-cache`). The database sorts it by the value the column shows, read from the stored headers or URL with the name bound as a parameter, and the app's in-memory sort places live flows the same way. Custom column headers sort on click and Option-click and their menus' sort items work; the sort is kept with the columns and removing a column drops it from the sort. The Add Column field completes `@` and the paths a column accepts, with no colon and no values, opens its menu on focus, normalises a pasted term, and says why a path cannot be added. Co-Authored-By: Claude Opus 5.5 --- .../yaak-proxy/components/AddColumnDialog.tsx | 47 ++- apps/yaak-proxy/components/FlowList.tsx | 4 +- apps/yaak-proxy/components/FlowListHeader.tsx | 58 ++-- apps/yaak-proxy/harness/backend.ts | 23 +- apps/yaak-proxy/lib/columns.test.ts | 71 +++++ apps/yaak-proxy/lib/columns.ts | 73 ++++- .../yaak-proxy-lib/bindings/gen_rpc.ts | 2 +- crates-proxy/yaak-proxy-lib/src/flows.rs | 275 ++++++++++++++++-- .../tests/fixtures/custom_column_sort.json | 101 +++++++ .../components/editor/query/extension.test.ts | 30 +- .../src/components/editor/query/extension.ts | 31 +- 11 files changed, 629 insertions(+), 86 deletions(-) create mode 100644 crates-proxy/yaak-proxy-lib/tests/fixtures/custom_column_sort.json diff --git a/apps/yaak-proxy/components/AddColumnDialog.tsx b/apps/yaak-proxy/components/AddColumnDialog.tsx index 2cba433ad..c2f3cea22 100644 --- a/apps/yaak-proxy/components/AddColumnDialog.tsx +++ b/apps/yaak-proxy/components/AddColumnDialog.tsx @@ -1,4 +1,6 @@ -import { Compartment, type Extension } from "@codemirror/state"; +import { startCompletion } from "@codemirror/autocomplete"; +import { Compartment, EditorState, type Extension } from "@codemirror/state"; +import { EditorView, ViewPlugin } from "@codemirror/view"; import type { Schema } from "@yaakapp-internal/query"; import { Dialog, @@ -11,7 +13,12 @@ import { queryFilter } from "@yaakapp-internal/ui/editor"; import { atom, useAtom, useSetAtom } from "jotai"; import { useEffect, useMemo, useRef, useState } from "react"; import { useRpcQuery } from "../hooks/useRpcQuery"; -import { addCustomColumn, customPath } from "../lib/columns"; +import { + addCustomColumn, + customPath, + customPathProblem, + fieldReferenceChanges, +} from "../lib/columns"; import { loadSeenValues } from "../lib/searchBox"; import { columnPrefsAtom } from "../lib/store"; @@ -20,9 +27,31 @@ export const addColumnDialogAtom = atom(false); const FORM_ID = "add-column"; +/** Keeps the field a reference to a field, `@header.x-cache`, whatever was typed or pasted */ +const asFieldReference = EditorState.transactionFilter.of((tr) => { + if (!tr.docChanged) return tr; + const changes = fieldReferenceChanges(tr.newDoc.toString()); + return changes.length === 0 ? tr : [tr, { changes, sequential: true }]; +}); + +/** Opens completion on the fields while the field is focused and empty */ +const completeWhenEmpty = [ + ViewPlugin.define((view) => { + if (view.hasFocus && view.state.doc.length === 0) { + queueMicrotask(() => startCompletion(view)); + } + return {}; + }), + EditorView.updateListener.of((update) => { + if (update.focusChanged && update.view.hasFocus && update.state.doc.length === 0) { + startCompletion(update.view); + } + }), +]; + /** - * Adds a column for a header or query parameter, named by its field as the search names it, - * `header.x-request-id` or `query.page`, completed as the search box completes it + * Adds a column for a header or query parameter, named by a reference to its field as the search + * names it, `@header.x-request-id` or `@query.page`, completed from the fields and the names seen */ export function AddColumnDialog() { const [open, setOpen] = useAtom(addColumnDialogAtom); @@ -41,13 +70,17 @@ function AddColumnForm({ onClose }: { onClose: () => void }) { const filter = useMemo(() => { if (schema == null) return []; const maps: Schema = { ...schema, fields: schema.fields.filter((f) => f.map) }; - return queryFilter({ schema: maps, loadValues: loadSeenValues }); + return [ + queryFilter({ schema: maps, loadValues: loadSeenValues, fieldsOnly: true }), + completeWhenEmpty, + ]; }, [schema]); - const extensions = useMemo(() => [compartment.current.of(filter)], [filter]); + const extensions = useMemo(() => [asFieldReference, compartment.current.of(filter)], [filter]); useEffect(() => { handleRef.current?.dispatch({ effects: compartment.current.reconfigure(filter) }); }, [filter]); const path = customPath(text); + const problem = customPathProblem(text); const submit = () => { if (path == null) return; @@ -77,6 +110,7 @@ function AddColumnForm({ onClose }: { onClose: () => void }) { placeholder="@header.x-request-id" defaultValue="" onChange={setText} + validate={problem == null} extensions={extensions} onKeyDown={(e) => { if (e.key === "Enter" && !e.defaultPrevented) { @@ -85,6 +119,7 @@ function AddColumnForm({ onClose }: { onClose: () => void }) { } }} /> +
{problem}
{ + (column: ColumnId, add: boolean) => { const current = latest.current.sort; const next = add ? toggleSortKey(current, column) : nextSort(current, column); if (next.length === 0) { diff --git a/apps/yaak-proxy/components/FlowListHeader.tsx b/apps/yaak-proxy/components/FlowListHeader.tsx index 65033f37c..19a483752 100644 --- a/apps/yaak-proxy/components/FlowListHeader.tsx +++ b/apps/yaak-proxy/components/FlowListHeader.tsx @@ -1,4 +1,4 @@ -import type { FlowSortKey, SortColumn } from "@yaakapp-internal/proxy-lib"; +import type { FlowSortKey } from "@yaakapp-internal/proxy-lib"; import { ContextMenu, Dropdown, @@ -17,6 +17,7 @@ import { type ColumnDef, type ColumnId, type ColumnPrefs, + type CustomColumnId, canHide, hideOtherColumns, isCustomColumn, @@ -38,7 +39,7 @@ interface Props { prefs: ColumnPrefs; sort: FlowSortKey[]; /** A click on a label, or with `add`, an Option-click */ - onSort: (column: SortColumn, add: boolean) => void; + onSort: (column: ColumnId, add: boolean) => void; /** Sets the sort from the menus; empty goes back to arrival order */ onSetSort: (sort: FlowSortKey[]) => void; onResetSort: () => void; @@ -67,6 +68,12 @@ export const FlowListHeader = memo(function FlowListHeader({ // Widths when a drag began, so each move sets the width from where it started. const startWidth = useRef(0); + // Taking out the sort's only key goes back to arrival order, as Reset Sort does + const removeColumn = (id: CustomColumnId) => { + if (sort.length > 0 && sort.every((k) => k.column === id)) onResetSort(); + setPrefs((p) => removeCustomColumn(p, id)); + }; + const pickerItems: DropdownItem[] = [ { type: "content", @@ -92,7 +99,7 @@ export const FlowListHeader = memo(function FlowListHeader({ label: "Remove Column", icon: "trash" as const, color: "danger" as const, - onSelect: () => setPrefs((p) => removeCustomColumn(p, c.id as `field:${string}`)), + onSelect: () => removeColumn(c.id as CustomColumnId), }, ], } @@ -106,22 +113,13 @@ export const FlowListHeader = memo(function FlowListHeader({ ]; const columnItems = (c: ColumnDef): DropdownItem[] => { - const column = c.sortable ? (c.id as SortColumn) : null; - const sortBy = (descending: boolean) => { - if (column != null) setSort([{ column, descending }]); - }; - const addTo = (descending: boolean) => { - if (column != null) setSort(addSortKey(sort, column, descending)); - }; + const sortBy = (descending: boolean) => setSort([{ column: c.id, descending }]); + const addTo = (descending: boolean) => setSort(addSortKey(sort, c.id, descending)); return [ - ...(column == null - ? [] - : ([ - { label: "Sort Ascending", onSelect: () => sortBy(false) }, - { label: "Sort Descending", onSelect: () => sortBy(true) }, - { label: "Add to Sort Ascending", onSelect: () => addTo(false) }, - { label: "Add to Sort Descending", onSelect: () => addTo(true) }, - ] satisfies DropdownItem[])), + { label: "Sort Ascending", onSelect: () => sortBy(false) }, + { label: "Sort Descending", onSelect: () => sortBy(true) }, + { label: "Add to Sort Ascending", onSelect: () => addTo(false) }, + { label: "Add to Sort Descending", onSelect: () => addTo(true) }, { label: "Clear Sorting", disabled: sort.length === 0, onSelect: onResetSort }, { type: "separator" }, { @@ -139,7 +137,7 @@ export const FlowListHeader = memo(function FlowListHeader({ { label: "Remove Column", color: "danger" as const, - onSelect: () => setPrefs((p) => removeCustomColumn(p, c.id as `field:${string}`)), + onSelect: () => removeColumn(c.id as CustomColumnId), }, ] : []), @@ -213,19 +211,15 @@ export const FlowListHeader = memo(function FlowListHeader({ data-column={c.id} className="relative h-full flex items-center min-w-0 px-1.5" > - {c.sortable ? ( - - ) : ( - {label} - )} + {/* On the column's right edge, the boundary with the next, resizing this one */} 0 ? manyFlows(many) : fixtures.map((f) => f.flow)) - .map((f, i) => ({ ...f, number: 1401 + i })) - .reverse(); - return { flows, total: flows.length, matching: null }; + const all = (many > 0 ? manyFlows(many) : fixtures.map((f) => f.flow)).map((f, i) => ({ + ...f, + number: 1401 + i, + })); + // In a sort's order, as the database lists it, all in one page + const sort = (req.sort ?? []) as FlowSortKey[]; + const flows = sort.length > 0 ? sortFlows(all, sort) : all.reverse(); + return { flows, total: flows.length, matching: null, cursor: null }; } case "list_seen": return { hosts: [], apps: [] }; @@ -179,6 +184,14 @@ function manyFlows(count: number): Flow[] { responseSize: pending ? 0 : base.responseSize, elapsedMs: pending ? null : base.elapsedMs, clientCwd: base.clientBundleId == null ? (folders[i % folders.length] ?? null) : null, + // A cache header with a few values and some without, for a column of it to sort + responseHeaders: + i % 5 === 4 + ? base.responseHeaders + : [ + ...base.responseHeaders, + { name: "X-Cache", value: ["HIT", "miss", "Hit", "EXPIRED"][i % 4] ?? "" }, + ], }; }); } diff --git a/apps/yaak-proxy/lib/columns.test.ts b/apps/yaak-proxy/lib/columns.test.ts index 5f4120a10..95b3dcce9 100644 --- a/apps/yaak-proxy/lib/columns.test.ts +++ b/apps/yaak-proxy/lib/columns.test.ts @@ -1,13 +1,17 @@ import type { Flow, FlowSortKey } from "@yaakapp-internal/proxy-lib"; import { describe, expect, test } from "vite-plus/test"; +import customSortFixture from "../../../crates-proxy/yaak-proxy-lib/tests/fixtures/custom_column_sort.json"; import { addCustomColumn, addSortKey, cellFilterTerm, type ColumnPrefs, customPath, + customPathProblem, customValue, defaultColumnPrefs, + fieldReference, + fieldReferenceChanges, gridTemplate, gridWidth, hideOtherColumns, @@ -154,6 +158,19 @@ describe("column preferences", () => { expect(prefs).toEqual(defaultColumnPrefs); }); + test("a custom column's sort persists with it and goes when it does", () => { + let prefs = addCustomColumn(defaultColumnPrefs, "header.x-cache"); + const sort: FlowSortKey[] = [ + { column: "field:header.x-cache", descending: true }, + { column: "status", descending: false }, + ]; + prefs = { ...prefs, sort }; + expect(parseColumnPrefs(JSON.parse(JSON.stringify(prefs))).sort).toEqual(sort); + // A key for a column that is no longer there is dropped + expect(parseColumnPrefs({ ...prefs, custom: [] }).sort).toEqual([sort[1]]); + expect(removeCustomColumn(prefs, "field:header.x-cache").sort).toEqual([sort[1]]); + }); + test("Operation shows itself once, unless the person set it", () => { const grpc = flow(1, { grpcService: "pkg.Greeter", grpcMethod: "SayHello" }); const anonymous = flow(2, { graphqlOperation: "" }); @@ -193,6 +210,33 @@ describe("sorting", () => { expect(addSortKey([], "time", true)).toEqual(one("time", true)); }); + test("a custom column sorts on click and Option-click as the others do", () => { + const cache = "field:header.x-cache"; + expect(nextSort([], cache)).toEqual(one(cache, false)); + expect(nextSort(one(cache, false), cache)).toEqual(one(cache, true)); + expect(toggleSortKey(one("status", true), cache)).toEqual([ + ...one("status", true), + ...one(cache, false), + ]); + }); + + // Flows and the order each sort lists them in, shared with + // `a_custom_column_sorts_by_its_value_and_pages` in the lib's `flows.rs`, which checks the + // database lists them in the same order. + test("a custom column sorts as the database does", () => { + const fixture = customSortFixture as { + flows: Partial[]; + sorts: { sort: FlowSortKey[]; order: number[] }[]; + }; + const flows = fixture.flows.map((f, i) => flow(i, f)); + for (const { sort, order } of fixture.sorts) { + expect( + sortFlows(flows, sort).map((f) => f.number), + JSON.stringify(sort), + ).toEqual(order); + } + }); + // The same flows and orders as `a_sort_on_several_columns_pages_in_its_order` in the lib's // `flows.rs`, so the app places a flow where the database would. const flows = [ @@ -261,6 +305,33 @@ describe("values", () => { expect(customPath("query.filter[status]")).toBeNull(); }); + test("says tersely why what was typed names no column", () => { + expect(customPathProblem("")).toBeNull(); + expect(customPathProblem("@")).toBeNull(); + expect(customPathProblem("@req.header.user-agent")).toBeNull(); + expect(customPathProblem("@hea")).toBe("Incomplete"); + expect(customPathProblem("@res.header")).toBe("Incomplete"); + expect(customPathProblem("@header.")).toBe("Missing name"); + expect(customPathProblem("@query.a b")).toBe("Invalid name"); + expect(customPathProblem("@host")).toBe("Unknown field"); + }); + + test("keeps the Add Column field a reference to a field", () => { + expect(fieldReference("")).toBe(""); + expect(fieldReference("@")).toBe("@"); + expect(fieldReference("h")).toBe("@h"); + expect(fieldReference("header.x-cache")).toBe("@header.x-cache"); + expect(fieldReference("@req.header.user-agent:")).toBe("@req.header.user-agent"); + expect(fieldReference(" @@Header.X-Cache: ")).toBe("@Header.X-Cache"); + expect(fieldReference(":")).toBe(""); + expect(fieldReference("@:")).toBe("@"); + expect(fieldReferenceChanges("@header.x")).toEqual([]); + expect(fieldReferenceChanges("query.page:")).toEqual([ + { from: 0, to: 0, insert: "@" }, + { from: 10, to: 11, insert: "" }, + ]); + }); + test("a custom column shows the first value, the response's header before the request's", () => { const f = flow(1, { url: "https://a.com/?Page=2&page=3", diff --git a/apps/yaak-proxy/lib/columns.ts b/apps/yaak-proxy/lib/columns.ts index b7f0e4492..f94655389 100644 --- a/apps/yaak-proxy/lib/columns.ts +++ b/apps/yaak-proxy/lib/columns.ts @@ -21,8 +21,6 @@ export interface ColumnDef { align: "left" | "right"; /** Shown until the person hides it */ defaultVisible: boolean; - /** Sorted by in the database; custom columns are not */ - sortable: boolean; /** The header or query parameter a custom column shows: `header.x-request-id` */ path?: string; } @@ -34,7 +32,7 @@ const builtin = ( minWidth: number, align: "left" | "right", defaultVisible: boolean, -): ColumnDef => ({ id, label, width, minWidth, align, defaultVisible, sortable: true }); +): ColumnDef => ({ id, label, width, minWidth, align, defaultVisible }); export const COLUMNS: ColumnDef[] = [ builtin("number", "#", 46, 30, "left", true), @@ -72,6 +70,45 @@ export function customPath(typed: string): string | null { return map + key.toLowerCase(); } +/** Why what was typed names no custom column, in a few words; null when it does or is empty */ +export function customPathProblem(typed: string): string | null { + if (customPath(typed) != null) return null; + const path = typed.trim().replace(/^@/, "").replace(/:$/, "").trim().toLowerCase(); + if (path === "") return null; + const map = CUSTOM_MAPS.find((m) => path.startsWith(m)); + if (map != null) return path === map ? "Missing name" : "Invalid name"; + // On the way to a map: `@hea`, `@req.header` + if (CUSTOM_MAPS.some((m) => m.startsWith(path))) return "Incomplete"; + return "Unknown field"; +} + +/** + * The edits that turn what the Add Column field holds into a reference to a field, `@` and its + * path: an `@` in front when it lacks one, and no colon or space after, as a pasted + * `@header.x-cache:` or a typed `header.x-cache` has. A lone `@` stays, to complete from. + */ +export function fieldReferenceChanges( + text: string, +): { from: number; to: number; insert: string }[] { + const lead = /^[\s@]*/.exec(text)?.[0].length ?? 0; + const trail = /[\s:]*$/.exec(text.slice(lead))?.[0].length ?? 0; + const end = text.length - trail; + const at = end > lead || text.includes("@") ? "@" : ""; + const changes = []; + if (text.slice(0, lead) !== at) changes.push({ from: 0, to: lead, insert: at }); + if (trail > 0) changes.push({ from: end, to: text.length, insert: "" }); + return changes; +} + +/** What the Add Column field shows for `text`, `fieldReferenceChanges` made */ +export function fieldReference(text: string): string { + let result = text; + for (const { from, to, insert } of [...fieldReferenceChanges(text)].reverse()) { + result = result.slice(0, from) + insert + result.slice(to); + } + return result; +} + export function customColumnId(path: string): CustomColumnId { return `field:${path}`; } @@ -89,7 +126,6 @@ function customDef(id: CustomColumnId): ColumnDef { minWidth: 40, align: "left", defaultVisible: true, - sortable: false, path, }; } @@ -198,10 +234,13 @@ export function addCustomColumn(prefs: ColumnPrefs, path: string): ColumnPrefs { return { ...prefs, custom, visible: { ...prefs.visible, [id]: true } }; } +/** Takes the column out, and out of the sort */ export function removeCustomColumn(prefs: ColumnPrefs, id: CustomColumnId): ColumnPrefs { const { [id]: _v, ...visible } = prefs.visible; const { [id]: _w, ...widths } = prefs.widths; - return { ...prefs, custom: prefs.custom.filter((c) => c !== id), visible, widths }; + const custom = prefs.custom.filter((c) => c !== id); + const sort = prefs.sort.filter((k) => k.column !== id); + return { ...prefs, custom, visible, widths, sort }; } export function resizeColumn(prefs: ColumnPrefs, id: ColumnId, width: number): ColumnPrefs { @@ -254,9 +293,11 @@ export function parseColumnPrefs(value: unknown): ColumnPrefs { if (key == null || typeof key !== "object") continue; const { column, descending } = key as Record; if (typeof column !== "string" || typeof descending !== "boolean") continue; - const def = DEFS.get(column as ColumnId); - if (def == null || prefs.sort.some((k) => k.column === column)) continue; - prefs.sort.push({ column: def.id as SortColumn, descending }); + // A built-in column, or one the person added that is still there + const id = column as ColumnId; + const known = isCustomColumn(id) ? prefs.custom.includes(id) : DEFS.has(id); + if (!known || prefs.sort.some((k) => k.column === id)) continue; + prefs.sort.push({ column: id, descending }); } } return prefs; @@ -265,7 +306,7 @@ export function parseColumnPrefs(value: unknown): ColumnPrefs { // -- Sorting -- /** Numbers sort largest first on the first click, names from A */ -export function firstDirection(column: SortColumn): boolean { +export function firstDirection(column: ColumnId): boolean { switch (column) { case "number": case "status": @@ -283,7 +324,7 @@ export function firstDirection(column: SortColumn): boolean { * A click on a column's label: sort by it alone, reverse the sort it has alone, then, from the * reversed sort, go back to the order flows arrive in */ -export function nextSort(sort: FlowSortKey[], column: SortColumn): FlowSortKey[] { +export function nextSort(sort: FlowSortKey[], column: ColumnId): FlowSortKey[] { const first = firstDirection(column); const only = sort.length === 1 ? sort[0] : null; if (only?.column !== column) return [{ column, descending: first }]; @@ -295,7 +336,7 @@ export function nextSort(sort: FlowSortKey[], column: SortColumn): FlowSortKey[] * An Option-click on a column's label: adds it as the last key, ascending; again reverses it, and * a third time takes it out */ -export function toggleSortKey(sort: FlowSortKey[], column: SortColumn): FlowSortKey[] { +export function toggleSortKey(sort: FlowSortKey[], column: ColumnId): FlowSortKey[] { const key = sort.find((k) => k.column === column); if (key == null) return [...sort, { column, descending: false }]; if (!key.descending) @@ -306,7 +347,7 @@ export function toggleSortKey(sort: FlowSortKey[], column: SortColumn): FlowSort /** Adds `column` as a key in `descending` order: the last key, or in its place if it is one */ export function addSortKey( sort: FlowSortKey[], - column: SortColumn, + column: ColumnId, descending: boolean, ): FlowSortKey[] { if (sort.some((k) => k.column === column)) { @@ -328,10 +369,12 @@ function emptyNumber(n: number | null | undefined): number { } /** - * The value `column` sorts `flow` by, as `SortColumn::sql` in the lib's `flows.rs` makes it in - * the database, so a flow placed here lands where a page read from there would have it + * The value `column` sorts `flow` by, as `SortBy::sql` in the lib's `flows.rs` makes it in the + * database, so a flow placed here lands where a page read from there would have it. A custom + * column's is the value it shows, as text. */ -export function columnSortKey(flow: Flow, column: SortColumn): string | number { +export function columnSortKey(flow: Flow, column: ColumnId): string | number { + if (isCustomColumn(column)) return noCase(customValue(flow, column.slice("field:".length))); switch (column) { case "number": return flow.number; diff --git a/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts b/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts index de0b1e6b3..266575b86 100644 --- a/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts +++ b/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts @@ -195,7 +195,7 @@ cursor?: Array, }; /** * One key of a sort. */ -export type FlowSortKey = { column: SortColumn, descending: boolean, }; +export type FlowSortKey = { column: SortColumn | `field:${string}`, descending: boolean, }; export type FlowsClearedPayload = Record; diff --git a/crates-proxy/yaak-proxy-lib/src/flows.rs b/crates-proxy/yaak-proxy-lib/src/flows.rs index 2885b77e0..d19962eed 100644 --- a/crates-proxy/yaak-proxy-lib/src/flows.rs +++ b/crates-proxy/yaak-proxy-lib/src/flows.rs @@ -98,14 +98,135 @@ pub enum SortColumn { } /// One key of a sort. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, TS)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, TS)] #[serde(rename_all = "camelCase")] #[ts(export, export_to = "gen_rpc.ts")] pub struct FlowSortKey { - pub column: SortColumn, + #[ts(type = "SortColumn | `field:${string}`")] + pub column: SortBy, pub descending: bool, } +/// What a sort key sorts by: a column the list has built in, or a column the person added for +/// a header or query parameter, written `field:header.x-cache` as the app names the column. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SortBy { + Column(SortColumn), + Field(FieldPath), +} + +impl From for SortBy { + fn from(column: SortColumn) -> Self { + Self::Column(column) + } +} + +const FIELD_PREFIX: &str = "field:"; + +impl Serialize for SortBy { + fn serialize(&self, serializer: S) -> Result { + match self { + Self::Column(column) => column.serialize(serializer), + Self::Field(field) => serializer.serialize_str(&format!("{FIELD_PREFIX}{field}")), + } + } +} + +impl<'de> Deserialize<'de> for SortBy { + fn deserialize>(deserializer: D) -> Result { + use serde::de::{Error, IntoDeserializer}; + let text = String::deserialize(deserializer)?; + if let Some(path) = text.strip_prefix(FIELD_PREFIX) { + return FieldPath::parse(path) + .map(Self::Field) + .ok_or_else(|| D::Error::custom(format!("Invalid sort field {path:?}"))); + } + let column: serde::de::value::StrDeserializer<'_, D::Error> = + text.as_str().into_deserializer(); + SortColumn::deserialize(column).map(Self::Column) + } +} + +/// The map a custom column's field is under, as the query language names it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FieldMap { + /// `header.`: the response's header, or the request's when the response has none. + Header, + RequestHeader, + ResponseHeader, + Query, +} + +impl FieldMap { + // `req.header.` and `res.header.` before `header.`, which neither starts with, all the same + const ALL: [Self; 4] = [ + Self::RequestHeader, + Self::ResponseHeader, + Self::Header, + Self::Query, + ]; + + fn prefix(self) -> &'static str { + match self { + Self::Header => "header.", + Self::RequestHeader => "req.header.", + Self::ResponseHeader => "res.header.", + Self::Query => "query.", + } + } +} + +/// A custom column's field: a header or query parameter by name, `header.x-cache`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FieldPath { + map: FieldMap, + /// Lowercased: names compare without regard to case + name: String, +} + +impl FieldPath { + /// The field `path` names, by the rule the app's `customPath` in `lib/columns.ts` keeps for + /// a custom column: a map, then a name of the characters the query language takes in a + /// field path, so the column's value makes a term a search can run. + pub fn parse(path: &str) -> Option { + let lower = path.to_ascii_lowercase(); + let map = FieldMap::ALL.into_iter().find(|m| lower.starts_with(m.prefix()))?; + let name = &lower[map.prefix().len()..]; + if name.contains('.') || !yaak_query::is_valid_path(name) { + return None; + } + Some(Self { map, name: name.to_string() }) + } + + /// The value the column shows, as text in SQL, NULL when the flow has none: the first value + /// under the name, as `customValue` in the app's `lib/columns.ts` reads it. Each `?` is + /// bound to the name, one parameter for each. + fn sql(&self) -> (String, Vec) { + let first = |json: &str| { + format!( + "(SELECT json_extract(kv.value, '$.value') FROM json_each({json}) AS kv \ + WHERE q_lower(json_extract(kv.value, '$.name')) = ? ORDER BY kv.key LIMIT 1)" + ) + }; + let (sql, sides) = match self.map { + FieldMap::Header => ( + format!("coalesce({}, {})", first("response_headers"), first("request_headers")), + 2, + ), + FieldMap::RequestHeader => (first("request_headers"), 1), + FieldMap::ResponseHeader => (first("response_headers"), 1), + FieldMap::Query => (first("url_query_json(url)"), 1), + }; + (sql, vec![Value::Text(self.name.clone()); sides]) + } +} + +impl std::fmt::Display for FieldPath { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}{}", self.map.prefix(), self.name) + } +} + /// What an empty number sorts as: below every value, as an empty text's `''` is. const EMPTY_NUMBER: &str = "-1000000000000000000"; @@ -156,26 +277,55 @@ impl SortColumn { } } -/// A sort's keys in SQL, each with whether it descends, then start and id, ascending. -fn sort_terms(sort: &[FlowSortKey], columns: &FlowColumns) -> Vec<(String, bool)> { +impl SortBy { + /// The value the key sorts by, in SQL, as [`SortColumn::sql`] makes it, and the parameters + /// its `?`s are bound to, in order. + fn sql(&self, columns: &FlowColumns) -> (String, Vec) { + match self { + Self::Column(column) => (column.sql(columns), Vec::new()), + // Text, as the other text columns sort + Self::Field(field) => { + let (value, params) = field.sql(); + (format!("(coalesce({value}, '') COLLATE NOCASE)"), params) + } + } + } +} + +/// One term of a sort in SQL: its value, the parameters the value's `?`s are bound to, and +/// whether it descends. +struct SortTerm { + sql: String, + params: Vec, + descending: bool, +} + +/// A sort's keys in SQL, then start and id, ascending. +fn sort_terms(sort: &[FlowSortKey], columns: &FlowColumns) -> Vec { + let tiebreak = |sql: &str| SortTerm { sql: sql.into(), params: Vec::new(), descending: false }; sort.iter() - .map(|k| (k.column.sql(columns), k.descending)) - .chain([ - ("started_at".to_string(), false), - ("flows.id".to_string(), false), - ]) + .map(|k| { + let (sql, params) = k.column.sql(columns); + SortTerm { sql, params, descending: k.descending } + }) + .chain([tiebreak("started_at"), tiebreak("flows.id")]) .collect() } -/// The `ORDER BY` for a sort, and the condition for the flows after `cursor`'s values in it: -/// each key equal up to one that is past the cursor's. -fn sort_after(terms: &[(String, bool)], cursor: &[Value]) -> (String, Vec) { +/// The condition for the flows after `cursor`'s values in a sort, and its parameters: each key +/// equal up to one that is past the cursor's. +fn sort_after(terms: &[SortTerm], cursor: &[Value]) -> (String, Vec) { let mut params = Vec::new(); let mut alternatives = Vec::new(); - for (i, (term, descending)) in terms.iter().enumerate() { - let mut parts: Vec = terms[..i].iter().map(|(t, _)| format!("{t} = ?")).collect(); - params.extend(cursor[..i].iter().cloned()); - parts.push(format!("{term} {} ?", if *descending { "<" } else { ">" })); + for (i, term) in terms.iter().enumerate() { + let mut parts = Vec::new(); + for (before, value) in terms[..i].iter().zip(cursor) { + parts.push(format!("{} = ?", before.sql)); + params.extend(before.params.iter().cloned()); + params.push(value.clone()); + } + parts.push(format!("{} {} ?", term.sql, if term.descending { "<" } else { ">" })); + params.extend(term.params.iter().cloned()); params.push(cursor[i].clone()); alternatives.push(format!("({})", parts.join(" AND "))); } @@ -377,15 +527,17 @@ fn list_and_cursor( conn: &rusqlite::Connection, filter: &FlowFilter, ) -> Result<(Vec, Option>), String> { - let (condition, mut params) = filter.condition_in(conn); let terms = if filter.sort.is_empty() { Vec::new() } else { sort_terms(&filter.sort, &FlowColumns::of(conn)) }; - // The rowid, for the flow's number, and a sort's values, for its cursor. + // The rowid, for the flow's number, and a sort's values, for its cursor and its order. let keys: String = - terms.iter().enumerate().map(|(i, (t, _))| format!(", {t} AS sort_{i}")).collect(); + terms.iter().enumerate().map(|(i, t)| format!(", {} AS sort_{i}", t.sql)).collect(); + let mut params: Vec = terms.iter().flat_map(|t| t.params.iter().cloned()).collect(); + let (condition, condition_params) = filter.condition_in(conn); + params.extend(condition_params); let mut sql = format!("SELECT rowid, *{keys} FROM flows WHERE {condition}"); push_clear_mark(conn, &mut sql, &mut params); if let Some(id) = &filter.after { @@ -407,9 +559,11 @@ fn list_and_cursor( sql.push_str(&after); params.extend(after_params); } + // By the values selected above, so a key with parameters is computed once a row let order: Vec = terms .iter() - .map(|(t, descending)| format!("{t} {}", if *descending { "DESC" } else { "ASC" })) + .enumerate() + .map(|(i, t)| format!("sort_{i} {}", if t.descending { "DESC" } else { "ASC" })) .collect(); sql.push_str(" ORDER BY "); sql.push_str(&order.join(", ")); @@ -1254,7 +1408,8 @@ mod tests { flow("https://A.com/", "GET", Some(404), None), flow("https://c.com/", "GET", Some(200), None), ]); - let key = |column, descending| FlowSortKey { column, descending }; + let key = + |column: SortColumn, descending| FlowSortKey { column: column.into(), descending }; let read = |sort: &[FlowSortKey], cursor: Option>, limit| { let filter = FlowFilter { sort: sort.to_vec(), @@ -1300,6 +1455,84 @@ mod tests { assert_eq!(all, ["000", "001", "002", "003", "004", "005", "006", "007"]); } + /// Flows and the order each sort lists them in, shared with the app's `lib/columns.test.ts`, + /// whose in-memory sort must agree with the database's. + #[derive(Deserialize)] + struct SortFixture { + flows: Vec, + sorts: Vec, + } + + #[derive(Deserialize)] + struct SortCase { + sort: Vec, + order: Vec, + } + + /// A custom column sorts by the value it shows: a header's first value, the response's over + /// the request's for `header.`, or a query parameter's, as text without regard to case and + /// empty lowest; alone, after a built-in key or before another, and paged from a cursor. + #[test] + fn a_custom_column_sorts_by_its_value_and_pages() { + let fixture: SortFixture = + serde_json::from_str(include_str!("../tests/fixtures/custom_column_sort.json")) + .unwrap(); + let (_dir, db) = db_of(fixture.flows); + let read = |sort: &[FlowSortKey], cursor: Option>, limit| { + let filter = FlowFilter { + sort: sort.to_vec(), + cursor, + limit: Some(limit), + ..Default::default() + }; + let page = sorted_page(&db, &filter).unwrap(); + let order: Vec = page.flows.iter().map(|f| f.id[3..].parse().unwrap()).collect(); + (order, page.cursor) + }; + for case in &fixture.sorts { + let (all, _) = read(&case.sort, None, 100); + assert_eq!(all, case.order, "{:?}", case.sort); + let mut paged = Vec::new(); + let mut cursor = None; + loop { + let (page, next) = read(&case.sort, cursor.clone(), 3); + if page.is_empty() { + break; + } + cursor = next; + paged.extend(page); + } + assert_eq!(paged, case.order, "paged {:?}", case.sort); + } + } + + /// A custom column's key names it as the app does, `field:` and its path, and takes only + /// the paths a custom column can have. + #[test] + fn a_sort_key_names_a_custom_column_by_its_path() { + let parse = |column: &str| { + serde_json::from_value::( + serde_json::json!({ "column": column, "descending": false }), + ) + .map(|k| k.column) + }; + assert_eq!(parse("status").unwrap(), SortBy::Column(SortColumn::Status)); + let field = parse("field:Res.Header.X-Cache").unwrap(); + assert_eq!(serde_json::to_value(&field).unwrap(), "field:res.header.x-cache"); + assert_eq!(serde_json::to_value(parse("tls").unwrap()).unwrap(), "tls"); + for path in [ + "field:host", + "field:header.", + "field:header.a b", + "field:header.a.b", + "field:query.x'); DROP TABLE flows; --", + "field:query.filter[status]", + "nope", + ] { + assert!(parse(path).is_err(), "{path}"); + } + } + #[test] fn a_search_reads_the_newest_page_through_the_index() { let (_dir, db) = db_of(vec![]); diff --git a/crates-proxy/yaak-proxy-lib/tests/fixtures/custom_column_sort.json b/crates-proxy/yaak-proxy-lib/tests/fixtures/custom_column_sort.json new file mode 100644 index 000000000..ec15c4c83 --- /dev/null +++ b/crates-proxy/yaak-proxy-lib/tests/fixtures/custom_column_sort.json @@ -0,0 +1,101 @@ +{ + "flows": [ + { + "method": "GET", + "url": "https://x.com/a?page=2&Page=9", + "requestHeaders": [{ "name": "x-cache", "value": "req0" }], + "responseHeaders": [{ "name": "X-Cache", "value": "HIT" }] + }, + { + "method": "POST", + "url": "https://x.com/b", + "responseHeaders": [{ "name": "x-cache", "value": "miss" }] + }, + { + "method": "GET", + "url": "https://x.com/c?page=10" + }, + { + "method": "POST", + "url": "https://x.com/d?q=a+b", + "requestHeaders": [{ "name": "X-CACHE", "value": "zeta" }] + }, + { + "method": "GET", + "url": "https://x.com/e?page=", + "responseHeaders": [ + { "name": "x-cache", "value": "hit" }, + { "name": "x-cache", "value": "AAA" } + ] + }, + { + "method": "GET", + "url": "https://x.com/f?PAGE=1", + "requestHeaders": [{ "name": "x-cache", "value": "zzz" }], + "responseHeaders": [{ "name": "x-cache", "value": "Hit" }] + }, + { + "method": "GET", + "url": "https://x.com/g?page=%41b", + "requestHeaders": [{ "name": "x-cache", "value": "early" }], + "responseHeaders": [{ "name": "x-cache", "value": "" }] + }, + { + "method": "GET", + "url": "https://x.com/h", + "requestHeaders": [{ "name": "x-cache", "value": "Beta" }] + } + ], + "sorts": [ + { + "sort": [{ "column": "field:res.header.x-cache", "descending": false }], + "order": [2, 3, 6, 7, 0, 4, 5, 1] + }, + { + "sort": [{ "column": "field:res.header.x-cache", "descending": true }], + "order": [1, 0, 4, 5, 2, 3, 6, 7] + }, + { + "sort": [{ "column": "field:req.header.x-cache", "descending": false }], + "order": [1, 2, 4, 7, 6, 0, 3, 5] + }, + { + "sort": [{ "column": "field:req.header.x-cache", "descending": true }], + "order": [5, 3, 0, 6, 7, 1, 2, 4] + }, + { + "sort": [{ "column": "field:header.x-cache", "descending": false }], + "order": [2, 6, 7, 0, 4, 5, 1, 3] + }, + { + "sort": [{ "column": "field:header.x-cache", "descending": true }], + "order": [3, 1, 0, 4, 5, 7, 2, 6] + }, + { + "sort": [{ "column": "field:query.page", "descending": false }], + "order": [1, 3, 4, 7, 5, 2, 0, 6] + }, + { + "sort": [{ "column": "field:query.page", "descending": true }], + "order": [6, 0, 2, 5, 1, 3, 4, 7] + }, + { + "sort": [{ "column": "field:query.q", "descending": true }], + "order": [3, 0, 1, 2, 4, 5, 6, 7] + }, + { + "sort": [ + { "column": "method", "descending": false }, + { "column": "field:header.x-cache", "descending": true } + ], + "order": [0, 4, 5, 7, 2, 6, 3, 1] + }, + { + "sort": [ + { "column": "field:header.x-cache", "descending": false }, + { "column": "field:query.page", "descending": true } + ], + "order": [6, 2, 7, 0, 5, 4, 1, 3] + } + ] +} diff --git a/packages/ui/src/components/editor/query/extension.test.ts b/packages/ui/src/components/editor/query/extension.test.ts index 16a4be1d1..86947fa7f 100644 --- a/packages/ui/src/components/editor/query/extension.test.ts +++ b/packages/ui/src/components/editor/query/extension.test.ts @@ -1,5 +1,33 @@ import { describe, expect, test } from "vite-plus/test"; -import { pathsAt } from "./extension"; +import type { Completion } from "@yaakapp-internal/query"; +import { fieldReferences, pathsAt } from "./extension"; + +describe("fieldReferences", () => { + const item = (insert: string, kind: Completion["kind"] = "field"): Completion => ({ + label: insert, + insert, + kind, + info: null, + example: null, + reopen: true, + }); + const refs = (items: Completion[]) => fieldReferences(items).map((i) => [i.insert, i.reopen]); + + test("ends a field or a map's key with no colon, and does not open again", () => { + expect(refs([item("@req.header.user-agent:"), item("@host:")])).toEqual([ + ["@req.header.user-agent", false], + ["@host", false], + ]); + }); + + test("opens again after a map, for its key", () => { + expect(refs([item("@header.")])).toEqual([["@header.", true]]); + }); + + test("never offers values", () => { + expect(refs([item("Mozilla/5.0", "value"), item("@query.")])).toEqual([["@query.", true]]); + }); +}); describe("pathsAt", () => { const at = (text: string) => pathsAt(text, text.length); diff --git a/packages/ui/src/components/editor/query/extension.ts b/packages/ui/src/components/editor/query/extension.ts index aed6e0f52..e273af6a9 100644 --- a/packages/ui/src/components/editor/query/extension.ts +++ b/packages/ui/src/components/editor/query/extension.ts @@ -7,7 +7,7 @@ import type { DecorationSet, ViewUpdate } from "@codemirror/view"; import { Decoration, EditorView, ViewPlugin } from "@codemirror/view"; import type { Tag } from "@lezer/highlight"; import { tags as t } from "@lezer/highlight"; -import type { Schema, TokenKind } from "@yaakapp-internal/query"; +import type { Completion as QueryCompletion, Schema, TokenKind } from "@yaakapp-internal/query"; import { queryCompletions, queryTokens } from "@yaakapp-internal/query"; /** A term the search cannot run, to underline: `from` and `to` count UTF-16 units */ @@ -36,6 +36,11 @@ export interface QueryFilterOptions { moreCompletions?: (typed: string) => Completion[]; /** The terms of a query the search cannot run, underlined as typing settles */ problems?: (text: string) => Promise; + /** + * Completes a reference to a field, `@header.x-request-id`, rather than a term: no colon + * after the field and no values, so choosing a field ends it, as a map's key does + */ + fieldsOnly?: boolean; } const tokenTags: Record = { @@ -116,7 +121,26 @@ function infoOf(info: string | null, example: string | null): Completion["info"] }; } -function completionSource({ schema, values, loadValues, moreCompletions }: QueryFilterOptions) { +/** + * The engine's completions as references to fields: fields alone, inserted without the colon a + * term's value follows, opening again only after a map, for its key + */ +export function fieldReferences(items: QueryCompletion[]): QueryCompletion[] { + return items + .filter((item) => item.kind === "field") + .map((item) => { + const insert = item.insert.replace(/:$/, ""); + return { ...item, insert, reopen: insert.endsWith(".") }; + }); +} + +function completionSource({ + schema, + values, + loadValues, + moreCompletions, + fieldsOnly, +}: QueryFilterOptions) { return async (ctx: CompletionContext): Promise => { const text = ctx.state.doc.toString(); let known = values ?? {}; @@ -126,8 +150,9 @@ function completionSource({ schema, values, loadValues, moreCompletions }: Query if (ctx.aborted) return null; } const result = queryCompletions(text, ctx.pos, schema, known); + const items = fieldsOnly ? fieldReferences(result?.items ?? []) : (result?.items ?? []); - const options: Completion[] = (result?.items ?? []).map((item) => ({ + const options: Completion[] = items.map((item) => ({ label: item.label, // Not a CodeMirror type: it names the kind for `onlyFields` below, and draws no icon type: item.kind === "field" ? "field" : "value", From 4e63ba2ce75888e45d521e55186ef55f5979a46c Mon Sep 17 00:00:00 2001 From: Gregory Schier Date: Tue, 10 Nov 2026 01:10:35 -0700 Subject: [PATCH 20/45] Drag Yaak Proxy's flow list columns to reorder them A column's label drags it: a lifted ghost follows the pointer, a line marks the boundary it would drop at, down the header and the rows in view, and the list scrolls sideways near its edges. A press that doesn't move still sorts, the resize handles still resize, and Escape cancels. The order is kept with the other column preferences as a list of ids, and the header menu gains Move Left and Move Right, the columns menu Reset Column Order. Co-Authored-By: Claude Opus 5.5 --- apps/yaak-proxy/components/FlowList.tsx | 27 +- apps/yaak-proxy/components/FlowListHeader.tsx | 368 +++++++++++++++--- apps/yaak-proxy/lib/columns.test.ts | 150 +++++++ apps/yaak-proxy/lib/columns.ts | 113 +++++- apps/yaak-proxy/package.json | 1 + package-lock.json | 1 + 6 files changed, 592 insertions(+), 68 deletions(-) diff --git a/apps/yaak-proxy/components/FlowList.tsx b/apps/yaak-proxy/components/FlowList.tsx index edf49d933..75b4ae566 100644 --- a/apps/yaak-proxy/components/FlowList.tsx +++ b/apps/yaak-proxy/components/FlowList.tsx @@ -301,6 +301,15 @@ export function FlowList({ style, className }: Props) { el.scrollLeft += e.deltaX; }, []); + // A dragged column's drop line, from the window's coordinates to the list's + const rootRef = useRef(null); + const [dropLine, setDropLine] = useState(null); + const getScrollElement = useCallback(() => scrollRef.current, []); + const handleDropLine = useCallback((x: number | null) => { + const root = rootRef.current; + setDropLine(x == null || root == null ? null : x - root.getBoundingClientRect().left); + }, []); + // A list too short to scroll still loads older flows on scrolling up. const handleWheel = useCallback( (e: WheelEvent) => { @@ -407,10 +416,11 @@ export function FlowList({ style, className }: Props) { return (
{/* oxlint-disable-next-line jsx-a11y/no-static-element-interactions -- scrolls the list sideways */} @@ -430,8 +440,23 @@ export function FlowList({ style, className }: Props) { onSort={handleSort} onSetSort={handleSetSort} onResetSort={follow} + scrollElement={getScrollElement} + onDropLine={handleDropLine} />
+ {/* Where a dragged column would drop, down the header and the rows in view */} + {dropLine != null && ( +
+
+
+ )}
void; onResetSort: () => void; + /** The rows' scrolling element, which a column dragged to its edge scrolls sideways */ + scrollElement: () => HTMLElement | null; + /** Where a dragged column would drop, in the window's coordinates, or null for nowhere */ + onDropLine: (x: number | null) => void; } /** * The list's only control surface: a click on a label sorts by it alone, a second reverses it * and a third goes back to arrival order; Option-click adds it to the sort as a further key. - * Dragging the edge after a label resizes its column and a double-click on it resets it. A - * right-click on a label opens the column's menu; the picker sits in the empty corner on hover. + * Dragging a label moves its column, and Escape puts it back. Dragging the edge after a label + * resizes its column and a double-click on it resets it. A right-click on a label opens the + * column's menu; the picker sits in the empty corner on hover. */ export const FlowListHeader = memo(function FlowListHeader({ columns, @@ -60,6 +94,8 @@ export const FlowListHeader = memo(function FlowListHeader({ onSort, onSetSort: setSort, onResetSort, + scrollElement, + onDropLine, }: Props) { const setPrefs = useSetAtom(columnPrefsAtom); const unavailable = unavailableColumns(useAtomValue(hasAppsAtom)); @@ -67,6 +103,12 @@ export const FlowListHeader = memo(function FlowListHeader({ const [menu, setMenu] = useState<{ x: number; y: number; column: ColumnDef | null } | null>(null); // Widths when a drag began, so each move sets the width from where it started. const startWidth = useRef(0); + const { dragging, sensors, dndHandlers, gridRef } = useColumnDrag({ + columns, + scrollElement, + onDropLine, + onDrop: (id, to) => setPrefs((p) => moveColumn(p, id, to, unavailable)), + }); // Taking out the sort's only key goes back to arrival order, as Reset Sort does const removeColumn = (id: CustomColumnId) => { @@ -110,11 +152,17 @@ export const FlowListHeader = memo(function FlowListHeader({ { type: "separator" }, { label: "Reset Sort", disabled: sort.length === 0, onSelect: onResetSort }, { label: "Reset Column Widths", onSelect: () => setPrefs((p) => resetWidths(p)) }, + { + label: "Reset Column Order", + disabled: prefs.order.length === 0, + onSelect: () => setPrefs((p) => resetOrder(p)), + }, ]; const columnItems = (c: ColumnDef): DropdownItem[] => { const sortBy = (descending: boolean) => setSort([{ column: c.id, descending }]); const addTo = (descending: boolean) => setSort(addSortKey(sort, c.id, descending)); + const at = columns.findIndex((col) => col.id === c.id); return [ { label: "Sort Ascending", onSelect: () => sortBy(false) }, { label: "Sort Descending", onSelect: () => sortBy(true) }, @@ -122,6 +170,17 @@ export const FlowListHeader = memo(function FlowListHeader({ { label: "Add to Sort Descending", onSelect: () => addTo(true) }, { label: "Clear Sorting", disabled: sort.length === 0, onSelect: onResetSort }, { type: "separator" }, + { + label: "Move Left", + disabled: at <= 0, + onSelect: () => setPrefs((p) => moveColumnBy(p, c.id, -1, unavailable)), + }, + { + label: "Move Right", + disabled: at === -1 || at >= columns.length - 1, + onSelect: () => setPrefs((p) => moveColumnBy(p, c.id, 1, unavailable)), + }, + { type: "separator" }, { label: "Hide Column", disabled: !canHide(prefs, c.id, unavailable), @@ -163,6 +222,7 @@ export const FlowListHeader = memo(function FlowListHeader({
{/* oxlint-disable-next-line jsx-a11y/no-static-element-interactions -- a right-click opens the column's menu */}
- {columns.map((c) => { - const at = sort.findIndex((k) => k.column === c.id); - const key = sort[at]; - const label = ( - <> - {key != null && ( - - )} - {key != null && sort.length > 1 && ( - {at + 1} - )} - {c.id === "app" ? "" : c.label} - - ); - const labelClass = classNames( - "flex items-center gap-0.5 min-w-0 w-full h-full whitespace-nowrap", - c.align === "right" && "justify-end", - key != null && "text-text-subtle", - ); - return ( - + {columns.map((c) => ( + - - {/* On the column's right edge, the boundary with the next, resizing this one */} - { - startWidth.current = c.current; - }} - onResizeMove={(e) => - setPrefs((p) => resizeColumn(p, c.id, startWidth.current + e.x - e.xStart)) - } - onReset={() => setPrefs((p) => resetWidth(p, c.id))} - /> - - ); - })} + column={c} + sort={sort} + dragging={dragging?.id === c.id} + onSort={onSort} + onResizeStart={() => { + startWidth.current = c.current; + }} + onResizeMove={(dx) => setPrefs((p) => resizeColumn(p, c.id, startWidth.current + dx))} + onResetWidth={() => setPrefs((p) => resetWidth(p, c.id))} + /> + ))} + {/* Over everything, as the header clips what overflows it */} + {createPortal( + + {dragging != null && ( +
+ +
+ )} +
, + document.body, + )} + ); }); + +interface HeaderCellProps { + column: Column; + sort: FlowSortKey[]; + /** Being dragged, so it dims where it was */ + dragging: boolean; + onSort: (column: ColumnId, add: boolean) => void; + onResizeStart: () => void; + /** How far the edge moved since the resize began */ + onResizeMove: (dx: number) => void; + onResetWidth: () => void; +} + +function HeaderCell({ + column: c, + sort, + dragging, + onSort, + onResizeStart, + onResizeMove, + onResetWidth, +}: HeaderCellProps) { + // The label drags the column; the resize handle beside it is outside it, so it still resizes. + const { listeners, setNodeRef } = useDraggable({ id: c.id }); + const sorted = sort.some((k) => k.column === c.id); + return ( + + + {/* On the column's right edge, the boundary with the next, resizing this one */} + onResizeMove(e.x - e.xStart)} + onReset={onResetWidth} + /> + + ); +} + +/** + * A column's name, after its sort direction and, sorted by several, its place in the sort. App's + * column is as narrow as its icons, so only its dragged ghost names it. + */ +function ColumnLabel({ + column: c, + sort, + ghost = false, +}: { + column: Column; + sort: FlowSortKey[]; + ghost?: boolean; +}) { + const at = sort.findIndex((k) => k.column === c.id); + const key = sort[at]; + return ( + <> + {key != null && ( + + )} + {key != null && sort.length > 1 && ( + {at + 1} + )} + {c.id === "app" && !ghost ? "" : c.label} + + ); +} + +/** + * Dragging a column's label: where it would drop, the boundary between the labels nearest the + * pointer, and the list scrolling sideways while the pointer is near its edge. The columns only + * move on the drop, so the rows lay out once. + */ +function useColumnDrag({ + columns, + scrollElement, + onDropLine, + onDrop, +}: { + columns: Column[]; + scrollElement: () => HTMLElement | null; + onDropLine: (x: number | null) => void; + /** Moves the column to the boundary `to` between those shown */ + onDrop: (id: ColumnId, to: number) => void; +}) { + const sensors = useSensors( + useSensor(PointerSensor, { activationConstraint: { distance: DRAG_DISTANCE } }), + ); + const gridRef = useRef(null); + const [dragging, setDragging] = useState(null); + // Live while dragging, read each frame + const state = useRef<{ + /** The column dragged */ + id: ColumnId; + /** The pointer when the press began, and now */ + start: number; + x: number; + /** The boundary it would drop at, and that boundary's line, or null where it would not move */ + to: number | null; + line: number | null; + frame: number; + } | null>(null); + const latest = useRef({ columns, scrollElement, onDropLine, onDrop }); + useLayoutEffect(() => { + latest.current = { columns, scrollElement, onDropLine, onDrop }; + }); + + const stop = useCallback(() => { + const s = state.current; + state.current = null; + if (s != null) cancelAnimationFrame(s.frame); + setDragging(null); + latest.current.onDropLine(null); + return s; + }, []); + + // A drag never outlives the header + useEffect( + () => () => { + stop(); + }, + [stop], + ); + + const onDragStart = useCallback((e: DragStartEvent) => { + const column = latest.current.columns.find((c) => c.id === e.active.id); + if (column == null) return; + const x = pointerX(e); + state.current = { id: column.id, start: x, x, to: null, line: null, frame: 0 }; + setDragging(column); + + const tick = () => { + const s = state.current; + if (s == null) return; + // Near an edge, the list scrolls that way, faster the nearer; the header follows it. + const list = latest.current.scrollElement(); + if (list != null) { + const { left, right } = list.getBoundingClientRect(); + const near = + s.x < left + EDGE ? s.x - left - EDGE : s.x > right - EDGE ? s.x - right + EDGE : 0; + if (near !== 0) { + const speed = Math.min(1, Math.abs(near) / EDGE) * EDGE_SPEED; + list.scrollLeft += Math.sign(near) * Math.max(1, speed); + } + } + // The boundary before the first label whose middle is right of the pointer + const cells = Array.from( + gridRef.current?.querySelectorAll(":scope > [data-column]") ?? [], + ).map((el) => el.getBoundingClientRect()); + let to = cells.findIndex((r) => s.x < r.left + r.width / 2); + if (to === -1) to = cells.length; + // Where it already is, it would not move, so no line. Its place is read each frame, as + // the columns can change under a drag: Operation shows itself when a flow has one. + const from = latest.current.columns.findIndex((c) => c.id === s.id); + const moves = from !== -1 && to !== from && to !== from + 1; + s.to = moves ? to : null; + const line = moves ? (cells[to]?.left ?? cells.at(-1)?.right ?? null) : null; + // Told again only when it changes: another boundary, or the same one scrolled + if (line !== s.line) latest.current.onDropLine(line); + s.line = line; + s.frame = requestAnimationFrame(tick); + }; + state.current.frame = requestAnimationFrame(tick); + }, []); + + // The pointer, from where the press began; the frames above read it, scrolling or not. + const onDragMove = useCallback((e: DragMoveEvent) => { + const s = state.current; + if (s != null) s.x = s.start + e.delta.x; + }, []); + + const onDragEnd = useCallback(() => { + const s = stop(); + if (s?.to != null) latest.current.onDrop(s.id, s.to); + }, [stop]); + + return { + dragging, + sensors, + gridRef, + dndHandlers: { onDragStart, onDragMove, onDragEnd, onDragCancel: stop }, + }; +} + +/** Where the press that began a drag was, across the window */ +function pointerX(e: DragStartEvent): number { + const event = e.activatorEvent; + return event instanceof globalThis.MouseEvent ? event.clientX : 0; +} diff --git a/apps/yaak-proxy/lib/columns.test.ts b/apps/yaak-proxy/lib/columns.test.ts index 95b3dcce9..446b3c71c 100644 --- a/apps/yaak-proxy/lib/columns.test.ts +++ b/apps/yaak-proxy/lib/columns.test.ts @@ -4,6 +4,7 @@ import customSortFixture from "../../../crates-proxy/yaak-proxy-lib/tests/fixtur import { addCustomColumn, addSortKey, + allColumns, cellFilterTerm, type ColumnPrefs, customPath, @@ -15,10 +16,13 @@ import { gridTemplate, gridWidth, hideOtherColumns, + moveColumn, + moveColumnBy, nextSort, operationOf, parseColumnPrefs, removeCustomColumn, + resetOrder, resetWidth, resetWidths, resizeColumn, @@ -121,6 +125,7 @@ describe("column preferences", () => { visible: { number: false, "field:header.x-cache": true }, widths: { host: 60 }, custom: ["field:header.x-cache"], + order: [], sort: [{ column: "status", descending: true }], }); expect( @@ -138,6 +143,7 @@ describe("column preferences", () => { visible: {}, widths: { size: 90 }, custom: ["field:query.page"], + order: [], sort: [{ column: "time", descending: false }], }); expect(parseColumnPrefs(null)).toEqual(defaultColumnPrefs); @@ -181,6 +187,150 @@ describe("column preferences", () => { }); }); +describe("column order", () => { + const defaults = ids(defaultColumnPrefs); + + test("a column moves to the boundary it was dropped at", () => { + expect(ids(moveColumn(defaultColumnPrefs, "status", 2))).toEqual([ + "number", + "app", + "status", + "method", + "host", + "path", + "query", + "type", + "size", + "time", + ]); + // Rightwards, before the column at the boundary + expect(ids(moveColumn(defaultColumnPrefs, "method", 5)).slice(2, 6)).toEqual([ + "host", + "path", + "method", + "query", + ]); + // To either end + expect(ids(moveColumn(defaultColumnPrefs, "time", 0))[0]).toBe("time"); + expect(ids(moveColumn(defaultColumnPrefs, "number", defaults.length)).at(-1)).toBe("number"); + // The picker lists them in the same order + const moved = moveColumn(defaultColumnPrefs, "time", 0); + expect(allColumns(moved)[0]?.id).toBe("time"); + }); + + test("a drop where the column already is changes nothing", () => { + expect(moveColumn(defaultColumnPrefs, "host", 3)).toBe(defaultColumnPrefs); + expect(moveColumn(defaultColumnPrefs, "host", 4)).toBe(defaultColumnPrefs); + expect(moveColumn(defaultColumnPrefs, "host", -1)).toBe(defaultColumnPrefs); + expect(moveColumn(defaultColumnPrefs, "host", defaults.length + 1)).toBe(defaultColumnPrefs); + // A hidden column has no place among those shown + expect(moveColumn(defaultColumnPrefs, "latency", 0)).toBe(defaultColumnPrefs); + }); + + test("Move Left and Move Right go one place, and not past the ends", () => { + expect(ids(moveColumnBy(defaultColumnPrefs, "host", -1)).slice(2, 5)).toEqual([ + "host", + "method", + "path", + ]); + expect(ids(moveColumnBy(defaultColumnPrefs, "host", 1)).slice(3, 6)).toEqual([ + "path", + "host", + "query", + ]); + expect(moveColumnBy(defaultColumnPrefs, "number", -1)).toBe(defaultColumnPrefs); + expect(moveColumnBy(defaultColumnPrefs, "time", 1)).toBe(defaultColumnPrefs); + // Where App has nothing to show, Number is next to Method + expect( + shownColumns(moveColumnBy(defaultColumnPrefs, "method", -1, ["app"]), ["app"]).map( + (c) => c.id, + )[0], + ).toBe("method"); + }); + + test("a hidden column keeps its place and comes back to it", () => { + let prefs = moveColumn(defaultColumnPrefs, "time", 0); + prefs = toggleColumn(prefs, "status"); + prefs = moveColumn(prefs, "host", 0); + expect(ids(prefs)).not.toContain("status"); + prefs = toggleColumn(prefs, "status"); + expect(ids(prefs)).toEqual([ + "host", + "time", + "number", + "app", + "method", + "path", + "query", + "status", + "type", + "size", + ]); + }); + + test("a column turned on goes at the end, unless it had a place", () => { + // Request Size's default place is before Size + let prefs = toggleColumn(defaultColumnPrefs, "requestSize"); + expect(ids(prefs).at(-1)).toBe("requestSize"); + expect(ids(prefs).slice(0, -1)).toEqual(defaults); + // Moved, hidden and shown again, it goes back where it was moved to + prefs = moveColumn(prefs, "requestSize", defaults.indexOf("size")); + prefs = toggleColumn(toggleColumn(prefs, "requestSize"), "requestSize"); + expect(ids(prefs).slice(-3)).toEqual(["requestSize", "size", "time"]); + // Already last of those shown, nothing needs placing + expect(toggleColumn(defaultColumnPrefs, "remote").order).toEqual([]); + }); + + test("a custom column is added at the end and leaves the order when removed", () => { + const moved = moveColumn(defaultColumnPrefs, "size", 0); + let prefs = addCustomColumn(moved, "header.x-cache"); + expect(ids(prefs).at(-1)).toBe("field:header.x-cache"); + prefs = moveColumn(prefs, "field:header.x-cache", 1); + expect(ids(prefs).slice(0, 2)).toEqual(["size", "field:header.x-cache"]); + // Hidden and added again, it keeps its place + prefs = addCustomColumn(toggleColumn(prefs, "field:header.x-cache"), "header.x-cache"); + expect(ids(prefs)[1]).toBe("field:header.x-cache"); + prefs = removeCustomColumn(prefs, "field:header.x-cache"); + expect(prefs.order).not.toContain("field:header.x-cache"); + expect(ids(prefs)).toEqual(ids(moved)); + // Added again, it starts over at the end + expect(ids(addCustomColumn(prefs, "header.x-cache")).at(-1)).toBe("field:header.x-cache"); + }); + + test("the order persists, and columns it doesn't list follow it in their default order", () => { + let prefs = addCustomColumn(defaultColumnPrefs, "query.page"); + prefs = moveColumn(prefs, "field:query.page", 0); + expect(parseColumnPrefs(JSON.parse(JSON.stringify(prefs))).order).toEqual(prefs.order); + const parsed = parseColumnPrefs({ + custom: ["field:query.page"], + order: ["time", "gone", 3, "time", "field:header.x-gone", "field:query.page", "host"], + }); + expect(parsed.order).toEqual(["time", "field:query.page", "host"]); + expect(ids(parsed)).toEqual([ + "time", + "field:query.page", + "host", + "number", + "app", + "method", + "path", + "query", + "status", + "type", + "size", + ]); + expect(parseColumnPrefs({ order: "time" }).order).toEqual([]); + }); + + test("Reset Column Order puts every column back", () => { + let prefs = moveColumn(defaultColumnPrefs, "time", 0); + prefs = toggleColumn(prefs, "latency"); + prefs = resetOrder(prefs); + expect(prefs.order).toEqual([]); + expect(ids(prefs)).toEqual([...defaults.slice(0, -1), "time", "latency"]); + }); +}); + describe("sorting", () => { const one = (column: FlowSortKey["column"], descending: boolean) => [{ column, descending }]; diff --git a/apps/yaak-proxy/lib/columns.ts b/apps/yaak-proxy/lib/columns.ts index f94655389..e55e2b4a2 100644 --- a/apps/yaak-proxy/lib/columns.ts +++ b/apps/yaak-proxy/lib/columns.ts @@ -145,26 +145,45 @@ export const CELL_PADDING = "0.75rem"; /** * What the person chose: which columns show, the widths they dragged, by id, the header and - * query parameter columns they added, in order, and the list's sort + * query parameter columns they added, in order, the order they dragged the columns into, and the + * list's sort */ export interface ColumnPrefs { visible: Partial>; widths: Partial>; custom: CustomColumnId[]; + /** + * The columns' order, by id. Columns not listed follow the listed ones, in their default order; + * empty for the default order. + */ + order: ColumnId[]; /** The keys in order; empty for the order flows arrive in */ sort: FlowSortKey[]; } -export const defaultColumnPrefs: ColumnPrefs = { visible: {}, widths: {}, custom: [], sort: [] }; +export const defaultColumnPrefs: ColumnPrefs = { + visible: {}, + widths: {}, + custom: [], + order: [], + sort: [], +}; export interface Column extends ColumnDef { /** Its width now, in pixels of content */ current: number; } -/** Every column the picker lists: the built-in ones, then those the person added */ +/** + * Every column, in the order the person left them: those they placed, then the rest in their + * default order, the built-in ones before those added. The picker lists them so. + */ export function allColumns(prefs: ColumnPrefs): ColumnDef[] { - return [...COLUMNS, ...prefs.custom.map(customDef)]; + const defaults = [...COLUMNS, ...prefs.custom.map(customDef)]; + if (prefs.order.length === 0) return defaults; + const byId = new Map(defaults.map((c) => [c.id, c])); + const placed = prefs.order.flatMap((id) => byId.get(id) ?? []); + return [...placed, ...defaults.filter((c) => !prefs.order.includes(c.id))]; } function isVisible(prefs: ColumnPrefs, def: ColumnDef): boolean { @@ -217,7 +236,66 @@ export function toggleColumn( ): ColumnPrefs { const shown = isVisible(prefs, columnDef(id)); if (shown && !canHide(prefs, id, unavailable)) return prefs; - return { ...prefs, visible: { ...prefs.visible, [id]: !shown } }; + const next = { ...prefs, visible: { ...prefs.visible, [id]: !shown } }; + return shown ? next : placeAtEnd(next, id); +} + +/** + * Puts a column just shown after the others shown, unless the person placed it before, so it + * goes back to that place. The columns shown before it are placed too, where they are, so the + * ones not listed don't jump ahead of it. + */ +function placeAtEnd(prefs: ColumnPrefs, id: ColumnId): ColumnPrefs { + if (prefs.order.includes(id)) return prefs; + const all = allColumns(prefs); + if (all.filter((c) => isVisible(prefs, c)).at(-1)?.id === id) return prefs; + const placed = all.filter( + (c) => c.id !== id && (prefs.order.includes(c.id) || isVisible(prefs, c)), + ); + return { ...prefs, order: [...placed.map((c) => c.id), id] }; +} + +/** + * Moves a shown column to the boundary `to` between the columns shown: 0 before the first, their + * count after the last. Hidden columns keep their places among the others. + */ +export function moveColumn( + prefs: ColumnPrefs, + id: ColumnId, + to: number, + unavailable: ColumnId[] = [], +): ColumnPrefs { + const shown = shownColumns(prefs, unavailable).map((c) => c.id); + const from = shown.indexOf(id); + if (from === -1 || to < 0 || to > shown.length || to === from || to === from + 1) return prefs; + // Every column shown, and those hidden that were placed, so they keep their places + const order = allColumns(prefs) + .filter((c) => c.id !== id && (prefs.order.includes(c.id) || shown.includes(c.id))) + .map((c) => c.id); + // Before the column at the boundary, or after the last one shown + const before = shown[to]; + const last = shown.at(-1); + const at = before != null ? order.indexOf(before) : last == null ? -1 : order.indexOf(last) + 1; + if (at === -1) return prefs; + order.splice(at, 0, id); + return { ...prefs, order }; +} + +/** One place left, or with `1`, right, among the columns shown */ +export function moveColumnBy( + prefs: ColumnPrefs, + id: ColumnId, + by: -1 | 1, + unavailable: ColumnId[] = [], +): ColumnPrefs { + const from = shownColumns(prefs, unavailable).findIndex((c) => c.id === id); + if (from === -1) return prefs; + return moveColumn(prefs, id, by === -1 ? from - 1 : from + 2, unavailable); +} + +/** Every column back to its default place */ +export function resetOrder(prefs: ColumnPrefs): ColumnPrefs { + return { ...prefs, order: [] }; } /** Shows `id` alone */ @@ -227,20 +305,24 @@ export function hideOtherColumns(prefs: ColumnPrefs, id: ColumnId): ColumnPrefs return { ...prefs, visible }; } -/** Adds a column for a header or query parameter's path, shown at the end; already there, shown */ +/** + * Adds a column for a header or query parameter's path, shown at the end; already there, shown, + * in its place + */ export function addCustomColumn(prefs: ColumnPrefs, path: string): ColumnPrefs { const id = customColumnId(path); const custom = prefs.custom.includes(id) ? prefs.custom : [...prefs.custom, id]; - return { ...prefs, custom, visible: { ...prefs.visible, [id]: true } }; + return placeAtEnd({ ...prefs, custom, visible: { ...prefs.visible, [id]: true } }, id); } -/** Takes the column out, and out of the sort */ +/** Takes the column out, and out of the order and the sort */ export function removeCustomColumn(prefs: ColumnPrefs, id: CustomColumnId): ColumnPrefs { const { [id]: _v, ...visible } = prefs.visible; const { [id]: _w, ...widths } = prefs.widths; const custom = prefs.custom.filter((c) => c !== id); + const order = prefs.order.filter((c) => c !== id); const sort = prefs.sort.filter((k) => k.column !== id); - return { ...prefs, custom, visible, widths, sort }; + return { ...prefs, custom, visible, widths, order, sort }; } export function resizeColumn(prefs: ColumnPrefs, id: ColumnId, width: number): ColumnPrefs { @@ -270,9 +352,9 @@ export function showOperationFor(prefs: ColumnPrefs, flows: Flow[]): ColumnPrefs * wrong type, so a change to the columns never breaks the list. */ export function parseColumnPrefs(value: unknown): ColumnPrefs { - const prefs: ColumnPrefs = { visible: {}, widths: {}, custom: [], sort: [] }; + const prefs: ColumnPrefs = { visible: {}, widths: {}, custom: [], order: [], sort: [] }; if (value == null || typeof value !== "object") return prefs; - const { visible, widths, custom, sort } = value as Record; + const { visible, widths, custom, order, sort } = value as Record; if (Array.isArray(custom)) { for (const id of custom) { if (typeof id !== "string" || !id.startsWith("field:")) continue; @@ -288,6 +370,15 @@ export function parseColumnPrefs(value: unknown): ColumnPrefs { const w = (widths as Record | undefined)?.[def.id]; if (typeof w === "number" && Number.isFinite(w) && w > 0) prefs.widths[def.id] = w; } + // Columns no longer there are dropped; those not listed follow the rest in their default order + if (Array.isArray(order)) { + for (const column of order) { + if (typeof column !== "string") continue; + const id = column as ColumnId; + const known = isCustomColumn(id) ? prefs.custom.includes(id) : DEFS.has(id); + if (known && !prefs.order.includes(id)) prefs.order.push(id); + } + } if (Array.isArray(sort)) { for (const key of sort) { if (key == null || typeof key !== "object") continue; diff --git a/apps/yaak-proxy/package.json b/apps/yaak-proxy/package.json index 7e160c0b9..dc2baf4ba 100644 --- a/apps/yaak-proxy/package.json +++ b/apps/yaak-proxy/package.json @@ -9,6 +9,7 @@ "lint": "tsc --noEmit" }, "dependencies": { + "@dnd-kit/core": "^6.3.1", "@tanstack/react-query": "^5.105.0", "@tanstack/react-virtual": "^3.14.24", "@tauri-apps/api": "^2.12.4", diff --git a/package-lock.json b/package-lock.json index 1238d6bc1..754dd06ca 100644 --- a/package-lock.json +++ b/package-lock.json @@ -246,6 +246,7 @@ "name": "@yaakapp/yaak-proxy", "version": "1.0.0", "dependencies": { + "@dnd-kit/core": "^6.3.1", "@tanstack/react-query": "^5.105.0", "@tanstack/react-virtual": "^3.14.24", "@tauri-apps/api": "^2.12.4", From ba403307d8883e6035fad87b5a10a0d8d51a99b8 Mon Sep 17 00:00:00 2001 From: Gregory Schier Date: Tue, 10 Nov 2026 01:18:44 -0700 Subject: [PATCH 21/45] Drop a dragged column where the pointer is when released A release right after a move used the target from the last frame, or none, since dnd-kit reports a move only once it renders. The drag now tracks the pointer itself and works out the target again on release. Co-Authored-By: Claude Opus 5.5 --- apps/yaak-proxy/components/FlowListHeader.tsx | 107 ++++++++++-------- 1 file changed, 59 insertions(+), 48 deletions(-) diff --git a/apps/yaak-proxy/components/FlowListHeader.tsx b/apps/yaak-proxy/components/FlowListHeader.tsx index 44998936e..223ced700 100644 --- a/apps/yaak-proxy/components/FlowListHeader.tsx +++ b/apps/yaak-proxy/components/FlowListHeader.tsx @@ -1,7 +1,6 @@ import { DndContext, DragOverlay, - type DragMoveEvent, type DragStartEvent, PointerSensor, useDraggable, @@ -406,8 +405,7 @@ function useColumnDrag({ const state = useRef<{ /** The column dragged */ id: ColumnId; - /** The pointer when the press began, and now */ - start: number; + /** The pointer, across the window */ x: number; /** The boundary it would drop at, and that boundary's line, or null where it would not move */ to: number | null; @@ -419,9 +417,16 @@ function useColumnDrag({ latest.current = { columns, scrollElement, onDropLine, onDrop }; }); + // The pointer, read as it moves rather than when dnd-kit next renders, so a release right + // after a move drops where the pointer is + const track = useCallback((e: PointerEvent) => { + if (state.current != null) state.current.x = e.clientX; + }, []); + const stop = useCallback(() => { const s = state.current; state.current = null; + window.removeEventListener("pointermove", track, true); if (s != null) cancelAnimationFrame(s.frame); setDragging(null); latest.current.onDropLine(null); @@ -436,63 +441,69 @@ function useColumnDrag({ [stop], ); - const onDragStart = useCallback((e: DragStartEvent) => { - const column = latest.current.columns.find((c) => c.id === e.active.id); - if (column == null) return; - const x = pointerX(e); - state.current = { id: column.id, start: x, x, to: null, line: null, frame: 0 }; - setDragging(column); + const onDragStart = useCallback( + (e: DragStartEvent) => { + const column = latest.current.columns.find((c) => c.id === e.active.id); + if (column == null) return; + state.current = { id: column.id, x: pointerX(e), to: null, line: null, frame: 0 }; + window.addEventListener("pointermove", track, true); + setDragging(column); - const tick = () => { - const s = state.current; - if (s == null) return; - // Near an edge, the list scrolls that way, faster the nearer; the header follows it. - const list = latest.current.scrollElement(); - if (list != null) { - const { left, right } = list.getBoundingClientRect(); - const near = - s.x < left + EDGE ? s.x - left - EDGE : s.x > right - EDGE ? s.x - right + EDGE : 0; - if (near !== 0) { - const speed = Math.min(1, Math.abs(near) / EDGE) * EDGE_SPEED; - list.scrollLeft += Math.sign(near) * Math.max(1, speed); + const tick = () => { + const s = state.current; + if (s == null) return; + // Near an edge, the list scrolls that way, faster the nearer; the header follows it. + const list = latest.current.scrollElement(); + if (list != null) { + const { left, right } = list.getBoundingClientRect(); + const near = + s.x < left + EDGE ? s.x - left - EDGE : s.x > right - EDGE ? s.x - right + EDGE : 0; + if (near !== 0) { + const speed = Math.min(1, Math.abs(near) / EDGE) * EDGE_SPEED; + list.scrollLeft += Math.sign(near) * Math.max(1, speed); + } } - } - // The boundary before the first label whose middle is right of the pointer - const cells = Array.from( - gridRef.current?.querySelectorAll(":scope > [data-column]") ?? [], - ).map((el) => el.getBoundingClientRect()); - let to = cells.findIndex((r) => s.x < r.left + r.width / 2); - if (to === -1) to = cells.length; - // Where it already is, it would not move, so no line. Its place is read each frame, as - // the columns can change under a drag: Operation shows itself when a flow has one. - const from = latest.current.columns.findIndex((c) => c.id === s.id); - const moves = from !== -1 && to !== from && to !== from + 1; - s.to = moves ? to : null; - const line = moves ? (cells[to]?.left ?? cells.at(-1)?.right ?? null) : null; - // Told again only when it changes: another boundary, or the same one scrolled - if (line !== s.line) latest.current.onDropLine(line); - s.line = line; - s.frame = requestAnimationFrame(tick); - }; - state.current.frame = requestAnimationFrame(tick); - }, []); - - // The pointer, from where the press began; the frames above read it, scrolling or not. - const onDragMove = useCallback((e: DragMoveEvent) => { - const s = state.current; - if (s != null) s.x = s.start + e.delta.x; - }, []); + target(s); + s.frame = requestAnimationFrame(tick); + }; + state.current.frame = requestAnimationFrame(tick); + // oxlint-disable-next-line react-hooks/exhaustive-deps -- `target` reads only refs + }, + [track], + ); const onDragEnd = useCallback(() => { + // Where the pointer is now, as a release can come before the next frame reads it + if (state.current != null) target(state.current); const s = stop(); if (s?.to != null) latest.current.onDrop(s.id, s.to); + // oxlint-disable-next-line react-hooks/exhaustive-deps -- `target` reads only refs }, [stop]); + /** Sets where the drag would drop, from the pointer and the labels where they are now */ + function target(s: NonNullable) { + // The boundary before the first label whose middle is right of the pointer + const cells = Array.from( + gridRef.current?.querySelectorAll(":scope > [data-column]") ?? [], + ).map((el) => el.getBoundingClientRect()); + let to = cells.findIndex((r) => s.x < r.left + r.width / 2); + if (to === -1) to = cells.length; + // Where it already is, it would not move, so no line. Its place is read each time, as the + // columns can change under a drag: Operation shows itself when a flow has one. + const from = latest.current.columns.findIndex((c) => c.id === s.id); + const moves = from !== -1 && to !== from && to !== from + 1; + s.to = moves ? to : null; + const line = moves ? (cells[to]?.left ?? cells.at(-1)?.right ?? null) : null; + // Told again only when it changes: another boundary, or the same one scrolled + if (line !== s.line) latest.current.onDropLine(line); + s.line = line; + } + return { dragging, sensors, gridRef, - dndHandlers: { onDragStart, onDragMove, onDragEnd, onDragCancel: stop }, + dndHandlers: { onDragStart, onDragEnd, onDragCancel: stop }, }; } From 78351f620e05107d8ab8df62db7ff3a2beb208e4 Mon Sep 17 00:00:00 2001 From: Gregory Schier Date: Tue, 10 Nov 2026 01:28:16 -0700 Subject: [PATCH 22/45] Drop a column dragged and released in one move dnd-kit starts a drag on the move that crosses its threshold and reports the release only once it has rendered the drag, so a quick drag could miss its last position or not drop at all. The header now follows the pointer from the press and drops the column on its release. Co-Authored-By: Claude Opus 5.5 --- apps/yaak-proxy/components/FlowListHeader.tsx | 128 +++++++++++------- 1 file changed, 76 insertions(+), 52 deletions(-) diff --git a/apps/yaak-proxy/components/FlowListHeader.tsx b/apps/yaak-proxy/components/FlowListHeader.tsx index 223ced700..1e002ab48 100644 --- a/apps/yaak-proxy/components/FlowListHeader.tsx +++ b/apps/yaak-proxy/components/FlowListHeader.tsx @@ -21,6 +21,7 @@ import { useAtomValue, useSetAtom } from "jotai"; import { memo, type MouseEvent, + type PointerEvent as ReactPointerEvent, useCallback, useEffect, useLayoutEffect, @@ -102,7 +103,7 @@ export const FlowListHeader = memo(function FlowListHeader({ const [menu, setMenu] = useState<{ x: number; y: number; column: ColumnDef | null } | null>(null); // Widths when a drag began, so each move sets the width from where it started. const startWidth = useRef(0); - const { dragging, sensors, dndHandlers, gridRef } = useColumnDrag({ + const { dragging, sensors, dndHandlers, gridRef, onPointerDownCapture } = useColumnDrag({ columns, scrollElement, onDropLine, @@ -222,6 +223,7 @@ export const FlowListHeader = memo(function FlowListHeader({ {/* oxlint-disable-next-line jsx-a11y/no-static-element-interactions -- a right-click opens the column's menu */}
{ - if (state.current != null) state.current.x = e.clientX; - }, []); - const stop = useCallback(() => { const s = state.current; state.current = null; - window.removeEventListener("pointermove", track, true); if (s != null) cancelAnimationFrame(s.frame); setDragging(null); latest.current.onDropLine(null); return s; }, []); - // A drag never outlives the header - useEffect( - () => () => { - stop(); - }, - [stop], - ); + const onDragStart = useCallback((e: DragStartEvent) => { + const column = latest.current.columns.find((c) => c.id === e.active.id); + if (column == null) return; + state.current = { id: column.id, x: pointer.current, to: null, line: null, frame: 0 }; + setDragging(column); - const onDragStart = useCallback( - (e: DragStartEvent) => { - const column = latest.current.columns.find((c) => c.id === e.active.id); - if (column == null) return; - state.current = { id: column.id, x: pointerX(e), to: null, line: null, frame: 0 }; - window.addEventListener("pointermove", track, true); - setDragging(column); - - const tick = () => { - const s = state.current; - if (s == null) return; - // Near an edge, the list scrolls that way, faster the nearer; the header follows it. - const list = latest.current.scrollElement(); - if (list != null) { - const { left, right } = list.getBoundingClientRect(); - const near = - s.x < left + EDGE ? s.x - left - EDGE : s.x > right - EDGE ? s.x - right + EDGE : 0; - if (near !== 0) { - const speed = Math.min(1, Math.abs(near) / EDGE) * EDGE_SPEED; - list.scrollLeft += Math.sign(near) * Math.max(1, speed); - } + const tick = () => { + const s = state.current; + if (s == null) return; + // Near an edge, the list scrolls that way, faster the nearer; the header follows it. + const list = latest.current.scrollElement(); + if (list != null) { + const { left, right } = list.getBoundingClientRect(); + const near = + s.x < left + EDGE ? s.x - left - EDGE : s.x > right - EDGE ? s.x - right + EDGE : 0; + if (near !== 0) { + const speed = Math.min(1, Math.abs(near) / EDGE) * EDGE_SPEED; + list.scrollLeft += Math.sign(near) * Math.max(1, speed); } - target(s); - s.frame = requestAnimationFrame(tick); - }; - state.current.frame = requestAnimationFrame(tick); - // oxlint-disable-next-line react-hooks/exhaustive-deps -- `target` reads only refs - }, - [track], - ); + } + target(s); + s.frame = requestAnimationFrame(tick); + }; + state.current.frame = requestAnimationFrame(tick); + // oxlint-disable-next-line react-hooks/exhaustive-deps -- `target` reads only refs + }, []); - const onDragEnd = useCallback(() => { + /** Drops the column where the pointer is, once; a second call finds no drag */ + const drop = useCallback(() => { // Where the pointer is now, as a release can come before the next frame reads it if (state.current != null) target(state.current); const s = stop(); @@ -480,6 +464,51 @@ function useColumnDrag({ // oxlint-disable-next-line react-hooks/exhaustive-deps -- `target` reads only refs }, [stop]); + // The pointer from a press on the header until its release, read before dnd-kit sees it. + // dnd-kit starts a drag on the move that crosses `DRAG_DISTANCE` and reports moves and the + // release only once it has rendered the drag, so a release right after a move would drop + // where the pointer was a frame before, or not at all. The release drops it here instead. + const pointer = useRef(0); + const track = useCallback((e: PointerEvent) => { + pointer.current = e.clientX; + if (state.current != null) state.current.x = e.clientX; + }, []); + const unlisten = useRef<(() => void) | null>(null); + const onPointerDownCapture = useCallback( + (e: ReactPointerEvent) => { + unlisten.current?.(); + pointer.current = e.clientX; + const up = (e: PointerEvent) => { + unlisten.current?.(); + track(e); + drop(); + }; + const cancel = () => { + unlisten.current?.(); + stop(); + }; + window.addEventListener("pointermove", track, true); + window.addEventListener("pointerup", up, true); + window.addEventListener("pointercancel", cancel, true); + unlisten.current = () => { + unlisten.current = null; + window.removeEventListener("pointermove", track, true); + window.removeEventListener("pointerup", up, true); + window.removeEventListener("pointercancel", cancel, true); + }; + }, + [track, drop, stop], + ); + + // A drag never outlives the header + useEffect( + () => () => { + unlisten.current?.(); + stop(); + }, + [stop], + ); + /** Sets where the drag would drop, from the pointer and the labels where they are now */ function target(s: NonNullable) { // The boundary before the first label whose middle is right of the pointer @@ -503,12 +532,7 @@ function useColumnDrag({ dragging, sensors, gridRef, - dndHandlers: { onDragStart, onDragEnd, onDragCancel: stop }, + onPointerDownCapture, + dndHandlers: { onDragStart, onDragEnd: drop, onDragCancel: stop }, }; } - -/** Where the press that began a drag was, across the window */ -function pointerX(e: DragStartEvent): number { - const event = e.activatorEvent; - return event instanceof globalThis.MouseEvent ? event.clientX : 0; -} From 9ee37e4015002d7842c2ad714e32099c98be5156 Mon Sep 17 00:00:00 2001 From: Gregory Schier Date: Tue, 10 Nov 2026 09:03:19 -0700 Subject: [PATCH 23/45] Show a dragged column's drop line at every boundary The line was hidden while a drop would leave the column where it is, so a drag showed nothing until it crossed a neighbour's middle. It now shows from the first movement, on the nearer edge of the column's own slot when the drop would change nothing, and at the first boundary over the mark column. The choice of boundary moves to a tested function in the columns module. Co-Authored-By: Claude Opus 5.5 --- apps/yaak-proxy/components/FlowListHeader.tsx | 17 +++++++------- apps/yaak-proxy/lib/columns.test.ts | 22 +++++++++++++++++++ apps/yaak-proxy/lib/columns.ts | 19 ++++++++++++++++ 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/apps/yaak-proxy/components/FlowListHeader.tsx b/apps/yaak-proxy/components/FlowListHeader.tsx index 1e002ab48..a232080f8 100644 --- a/apps/yaak-proxy/components/FlowListHeader.tsx +++ b/apps/yaak-proxy/components/FlowListHeader.tsx @@ -38,6 +38,7 @@ import { type ColumnPrefs, type CustomColumnId, canHide, + dropBoundary, hideOtherColumns, isCustomColumn, moveColumn, @@ -409,7 +410,7 @@ function useColumnDrag({ id: ColumnId; /** The pointer, across the window */ x: number; - /** The boundary it would drop at, and that boundary's line, or null where it would not move */ + /** The boundary it would move to, null where it would stay, and the line drawn on it */ to: number | null; line: number | null; frame: number; @@ -433,6 +434,8 @@ function useColumnDrag({ if (column == null) return; state.current = { id: column.id, x: pointer.current, to: null, line: null, frame: 0 }; setDragging(column); + // The line shows from the drag's first movement, not a frame later + target(state.current); const tick = () => { const s = state.current; @@ -511,18 +514,14 @@ function useColumnDrag({ /** Sets where the drag would drop, from the pointer and the labels where they are now */ function target(s: NonNullable) { - // The boundary before the first label whose middle is right of the pointer const cells = Array.from( gridRef.current?.querySelectorAll(":scope > [data-column]") ?? [], ).map((el) => el.getBoundingClientRect()); - let to = cells.findIndex((r) => s.x < r.left + r.width / 2); - if (to === -1) to = cells.length; - // Where it already is, it would not move, so no line. Its place is read each time, as the - // columns can change under a drag: Operation shows itself when a flow has one. + // Its place is read each time, as the columns can change under a drag: Operation shows + // itself when a flow has one. const from = latest.current.columns.findIndex((c) => c.id === s.id); - const moves = from !== -1 && to !== from && to !== from + 1; - s.to = moves ? to : null; - const line = moves ? (cells[to]?.left ?? cells.at(-1)?.right ?? null) : null; + const { to, line } = dropBoundary(s.x, cells, from); + s.to = to; // Told again only when it changes: another boundary, or the same one scrolled if (line !== s.line) latest.current.onDropLine(line); s.line = line; diff --git a/apps/yaak-proxy/lib/columns.test.ts b/apps/yaak-proxy/lib/columns.test.ts index 446b3c71c..cafcb4b3c 100644 --- a/apps/yaak-proxy/lib/columns.test.ts +++ b/apps/yaak-proxy/lib/columns.test.ts @@ -11,6 +11,7 @@ import { customPathProblem, customValue, defaultColumnPrefs, + dropBoundary, fieldReference, fieldReferenceChanges, gridTemplate, @@ -322,6 +323,27 @@ describe("column order", () => { expect(parseColumnPrefs({ order: "time" }).order).toEqual([]); }); + test("the drop line shows at every boundary, its own slot's edges included", () => { + // Four columns 100px wide from x 24, after the mark column; the third is dragged + const cells = [24, 124, 224, 324].map((left) => ({ left, right: left + 100 })); + const at = (x: number) => dropBoundary(x, cells, 2); + // Over the mark column or the first's left half: the first boundary + expect(at(5)).toEqual({ to: 0, line: 24 }); + expect(at(60)).toEqual({ to: 0, line: 24 }); + expect(at(80)).toEqual({ to: 1, line: 124 }); + // The left neighbour's near half and its own left half: its left edge, staying + expect(at(190)).toEqual({ to: null, line: 224 }); + expect(at(250)).toEqual({ to: null, line: 224 }); + // Its own right half and the right neighbour's near half: its right edge, staying + expect(at(290)).toEqual({ to: null, line: 324 }); + expect(at(350)).toEqual({ to: null, line: 324 }); + // Past the right neighbour's middle, and past the end + expect(at(380)).toEqual({ to: 4, line: 424 }); + expect(at(900)).toEqual({ to: 4, line: 424 }); + // The last column dragged to the end stays + expect(dropBoundary(900, cells, 3)).toEqual({ to: null, line: 424 }); + }); + test("Reset Column Order puts every column back", () => { let prefs = moveColumn(defaultColumnPrefs, "time", 0); prefs = toggleColumn(prefs, "latency"); diff --git a/apps/yaak-proxy/lib/columns.ts b/apps/yaak-proxy/lib/columns.ts index e55e2b4a2..693cd9ee4 100644 --- a/apps/yaak-proxy/lib/columns.ts +++ b/apps/yaak-proxy/lib/columns.ts @@ -298,6 +298,25 @@ export function resetOrder(prefs: ColumnPrefs): ColumnPrefs { return { ...prefs, order: [] }; } +/** + * Where a dragged column would drop, from the pointer and the shown columns' edges, left to + * right: the boundary before the first column whose middle is right of the pointer, and the + * line's place on it. Over its own column or the near half of a neighbour, that is an edge of + * its own slot, where it would stay, so `to` is null; the line still shows there. Left of the + * first column, over the fixed mark column, it is the first boundary. + */ +export function dropBoundary( + x: number, + cells: { left: number; right: number }[], + from: number, +): { to: number | null; line: number | null } { + let at = cells.findIndex((r) => x < (r.left + r.right) / 2); + if (at === -1) at = cells.length; + const line = cells[at]?.left ?? cells.at(-1)?.right ?? null; + const stays = from === -1 || at === from || at === from + 1; + return { to: stays ? null : at, line }; +} + /** Shows `id` alone */ export function hideOtherColumns(prefs: ColumnPrefs, id: ColumnId): ColumnPrefs { const visible: ColumnPrefs["visible"] = {}; From 185a4e30dfbc69da79904a044b199ea09c5c42dc Mon Sep 17 00:00:00 2001 From: Gregory Schier Date: Tue, 10 Nov 2026 09:22:17 -0700 Subject: [PATCH 24/45] Drag a whole column's ghost, and keep default places until reordered The ghost of a dragged column is now its header and the cells of the rows in view, copied once when the drag starts and moved sideways by a transform, kept within the list; the column dims in place, its cells by one style rule so no row renders. A built-in column turned on before any reordering takes its default place again; once the person has an order, a newly shown column goes at the end. Co-Authored-By: Claude Opus 5.5 --- apps/yaak-proxy/components/FlowList.tsx | 151 +++++++++++++++++- apps/yaak-proxy/components/FlowListHeader.tsx | 65 ++++---- apps/yaak-proxy/lib/columns.test.ts | 27 +++- apps/yaak-proxy/lib/columns.ts | 5 +- 4 files changed, 198 insertions(+), 50 deletions(-) diff --git a/apps/yaak-proxy/components/FlowList.tsx b/apps/yaak-proxy/components/FlowList.tsx index 75b4ae566..fc196d318 100644 --- a/apps/yaak-proxy/components/FlowList.tsx +++ b/apps/yaak-proxy/components/FlowList.tsx @@ -14,7 +14,14 @@ import { } from "@yaakapp-internal/ui"; import classNames from "classnames"; import { useAtom, useAtomValue, useSetAtom, useStore } from "jotai"; -import type { CSSProperties, KeyboardEvent, MouseEvent, ReactElement, WheelEvent } from "react"; +import type { + CSSProperties, + KeyboardEvent, + MouseEvent, + ReactElement, + RefObject, + WheelEvent, +} from "react"; import { memo, useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from "react"; import { selectedFlowAtom, selectedFlowIdAtom } from "../lib/atoms"; import { shortContentType } from "../lib/body"; @@ -61,7 +68,7 @@ import { import { useShowToast } from "../lib/toast"; import { AppIcon } from "./AppIcon"; import { FlowListEmpty, PausedRow } from "./FlowListEmpty"; -import { FlowListHeader } from "./FlowListHeader"; +import { type ColumnDragEvent, FlowListHeader } from "./FlowListHeader"; const BOTTOM_PADDING = 6; /** Rows from the top at which older flows start loading, or from the bottom, sorted, more */ @@ -305,6 +312,7 @@ export function FlowList({ style, className }: Props) { const rootRef = useRef(null); const [dropLine, setDropLine] = useState(null); const getScrollElement = useCallback(() => scrollRef.current, []); + const { ghostColumn, onColumnDrag } = useColumnGhost(rootRef, headerScrollRef, scrollRef); const handleDropLine = useCallback((x: number | null) => { const root = rootRef.current; setDropLine(x == null || root == null ? null : x - root.getBoundingClientRect().left); @@ -442,14 +450,19 @@ export function FlowList({ style, className }: Props) { onResetSort={follow} scrollElement={getScrollElement} onDropLine={handleDropLine} + onColumnDrag={onColumnDrag} />
- {/* Where a dragged column would drop, down the header and the rows in view */} + {/* A dragged column's cells dim where they are, in one rule rather than a render a row */} + {ghostColumn != null && ( + + )} + {/* Where a dragged column would drop, down the header and the rows in view, over its ghost */} {dropLine != null && (
{ }, [flows, following]); } +/** + * The ghost of a dragged column: its header and the cells of the rows in view, copied once when + * the drag starts, at the column's width and the list's height. It follows the pointer sideways, + * kept within the list, by a transform, so the rows don't render while it moves; flows arriving + * or the list scrolling sideways meanwhile leave the copy as it was. + */ +function useColumnGhost( + rootRef: RefObject, + headerRef: RefObject, + listRef: RefObject, +): { ghostColumn: ColumnId | null; onColumnDrag: (event: ColumnDragEvent) => void } { + const [ghostColumn, setGhostColumn] = useState(null); + const ghost = useRef(null); + + const onColumnDrag = useCallback( + (event: ColumnDragEvent) => { + if (event.type === "move") { + ghost.current?.move(event.x); + return; + } + ghost.current?.remove(); + ghost.current = null; + if (event.type === "end") { + setGhostColumn(null); + return; + } + const root = rootRef.current; + const header = headerRef.current; + const list = listRef.current; + if (root == null || header == null || list == null) return; + ghost.current = snapshotColumn(event.id, event.x, root, header, list); + setGhostColumn(event.id); + }, + [rootRef, headerRef, listRef], + ); + + useEffect(() => () => ghost.current?.remove(), []); + return { ghostColumn, onColumnDrag }; +} + +interface ColumnGhost { + /** Follows the pointer, across the window */ + move: (x: number) => void; + remove: () => void; +} + +function snapshotColumn( + id: ColumnId, + startX: number, + root: HTMLElement, + header: HTMLElement, + list: HTMLElement, +): ColumnGhost | null { + const cell = header.querySelector(`[data-column="${id}"]`); + if (cell == null) return null; + const rootBox = root.getBoundingClientRect(); + const cellBox = cell.getBoundingClientRect(); + const listBox = list.getBoundingClientRect(); + const listHeight = list.clientHeight; + + const el = document.createElement("div"); + el.setAttribute("aria-hidden", "true"); + el.className = classNames( + "pointer-events-none absolute top-0 z-20 overflow-hidden rounded-sm", + // Lifted: a shadow, and just enough opacity to read as held, not enough to show text through + "bg-surface border border-border-subtle shadow-lg opacity-95", + ); + Object.assign(el.style, { + left: `${cellBox.left - rootBox.left}px`, + width: `${cellBox.width}px`, + height: `${rootBox.height}px`, + willChange: "transform", + }); + + // The header's label, without its resize handle + const head = place(cellBox.top - rootBox.top, cellBox.height, "text-xs text-text-subtlest"); + const label = cell.cloneNode(true) as HTMLElement; + for (const handle of label.querySelectorAll(".cursor-col-resize")) handle.remove(); + label.style.width = "100%"; + head.append(label); + el.append(head); + + // The rows in view, clipped to the list as it shows them + const rows = document.createElement("div"); + rows.className = "absolute left-0 right-0 overflow-hidden"; + Object.assign(rows.style, { top: `${listBox.top - rootBox.top}px`, height: `${listHeight}px` }); + for (const row of list.querySelectorAll('[role="option"]')) { + const rowBox = row.getBoundingClientRect(); + if (rowBox.bottom <= listBox.top || rowBox.top >= listBox.top + listHeight) continue; + const source = row.querySelector(`[data-column="${id}"]`); + if (source == null) continue; + const copy = place( + rowBox.top - listBox.top, + rowBox.height, + classNames( + "whitespace-nowrap tabular-nums", + row.getAttribute("aria-selected") === "true" && "text-text bg-surface-highlight", + row.classList.contains("text-text-subtlest") && "text-text-subtlest", + ), + ); + for (const child of source.children) { + const clone = child.cloneNode(true) as HTMLElement; + clone.style.width = "100%"; + copy.append(clone); + } + rows.append(copy); + } + el.append(rows); + root.append(el); + + // Kept within the list's visible width + const min = listBox.left; + const max = Math.max(min, listBox.left + list.clientWidth - cellBox.width); + const move = (x: number) => { + const left = Math.min(max, Math.max(min, cellBox.left + x - startX)); + el.style.transform = `translateX(${left - cellBox.left}px)`; + }; + move(startX); + return { move, remove: () => el.remove() }; +} + +/** A line of the ghost, `top` pixels down its box */ +function place(top: number, height: number, className: string): HTMLElement { + const el = document.createElement("div"); + el.className = classNames("absolute left-0 right-0 flex items-center", className); + Object.assign(el.style, { top: `${top}px`, height: `${height}px` }); + return el; +} + interface RowProps { flow: Flow; index: number; diff --git a/apps/yaak-proxy/components/FlowListHeader.tsx b/apps/yaak-proxy/components/FlowListHeader.tsx index a232080f8..952ca6cc6 100644 --- a/apps/yaak-proxy/components/FlowListHeader.tsx +++ b/apps/yaak-proxy/components/FlowListHeader.tsx @@ -1,6 +1,5 @@ import { DndContext, - DragOverlay, type DragStartEvent, PointerSensor, useDraggable, @@ -28,7 +27,6 @@ import { useRef, useState, } from "react"; -import { createPortal } from "react-dom"; import { addSortKey, allColumns, @@ -77,8 +75,16 @@ interface Props { scrollElement: () => HTMLElement | null; /** Where a dragged column would drop, in the window's coordinates, or null for nowhere */ onDropLine: (x: number | null) => void; + /** A column's drag: its start, the pointer's moves, and its end, for the list's ghost */ + onColumnDrag: (event: ColumnDragEvent) => void; } +/** A column's drag, for the ghost the list draws of it; `x` is the pointer, across the window */ +export type ColumnDragEvent = + | { type: "start"; id: ColumnId; x: number } + | { type: "move"; x: number } + | { type: "end" }; + /** * The list's only control surface: a click on a label sorts by it alone, a second reverses it * and a third goes back to arrival order; Option-click adds it to the sort as a further key. @@ -97,6 +103,7 @@ export const FlowListHeader = memo(function FlowListHeader({ onResetSort, scrollElement, onDropLine, + onColumnDrag, }: Props) { const setPrefs = useSetAtom(columnPrefsAtom); const unavailable = unavailableColumns(useAtomValue(hasAppsAtom)); @@ -108,6 +115,7 @@ export const FlowListHeader = memo(function FlowListHeader({ columns, scrollElement, onDropLine, + onColumnDrag, onDrop: (id, to) => setPrefs((p) => moveColumn(p, id, to, unavailable)), }); @@ -259,24 +267,6 @@ export const FlowListHeader = memo(function FlowListHeader({ onResetWidth={() => setPrefs((p) => resetWidth(p, c.id))} /> ))} - {/* Over everything, as the header clips what overflows it */} - {createPortal( - - {dragging != null && ( -
- -
- )} -
, - document.body, - )} k.column === c.id); const key = sort[at]; return ( @@ -377,7 +356,7 @@ function ColumnLabel({ {key != null && sort.length > 1 && ( {at + 1} )} - {c.id === "app" && !ghost ? "" : c.label} + {c.id === "app" ? "" : c.label} ); } @@ -391,11 +370,13 @@ function useColumnDrag({ columns, scrollElement, onDropLine, + onColumnDrag, onDrop, }: { columns: Column[]; scrollElement: () => HTMLElement | null; onDropLine: (x: number | null) => void; + onColumnDrag: (event: ColumnDragEvent) => void; /** Moves the column to the boundary `to` between those shown */ onDrop: (id: ColumnId, to: number) => void; }) { @@ -413,11 +394,13 @@ function useColumnDrag({ /** The boundary it would move to, null where it would stay, and the line drawn on it */ to: number | null; line: number | null; + /** The pointer the ghost was last moved to */ + ghostX: number; frame: number; } | null>(null); - const latest = useRef({ columns, scrollElement, onDropLine, onDrop }); + const latest = useRef({ columns, scrollElement, onDropLine, onColumnDrag, onDrop }); useLayoutEffect(() => { - latest.current = { columns, scrollElement, onDropLine, onDrop }; + latest.current = { columns, scrollElement, onDropLine, onColumnDrag, onDrop }; }); const stop = useCallback(() => { @@ -426,13 +409,17 @@ function useColumnDrag({ if (s != null) cancelAnimationFrame(s.frame); setDragging(null); latest.current.onDropLine(null); + if (s != null) latest.current.onColumnDrag({ type: "end" }); return s; }, []); const onDragStart = useCallback((e: DragStartEvent) => { const column = latest.current.columns.find((c) => c.id === e.active.id); if (column == null) return; - state.current = { id: column.id, x: pointer.current, to: null, line: null, frame: 0 }; + const x = pointer.current; + state.current = { id: column.id, x, to: null, line: null, ghostX: x, frame: 0 }; + // Before the column dims, so the ghost is of it as it shows + latest.current.onColumnDrag({ type: "start", id: column.id, x }); setDragging(column); // The line shows from the drag's first movement, not a frame later target(state.current); @@ -452,6 +439,10 @@ function useColumnDrag({ } } target(s); + if (s.x !== s.ghostX) { + s.ghostX = s.x; + latest.current.onColumnDrag({ type: "move", x: s.x }); + } s.frame = requestAnimationFrame(tick); }; state.current.frame = requestAnimationFrame(tick); diff --git a/apps/yaak-proxy/lib/columns.test.ts b/apps/yaak-proxy/lib/columns.test.ts index cafcb4b3c..6e01d5a2d 100644 --- a/apps/yaak-proxy/lib/columns.test.ts +++ b/apps/yaak-proxy/lib/columns.test.ts @@ -269,17 +269,30 @@ describe("column order", () => { ]); }); - test("a column turned on goes at the end, unless it had a place", () => { + test("never reordered, a built-in column turned on takes its default place", () => { // Request Size's default place is before Size - let prefs = toggleColumn(defaultColumnPrefs, "requestSize"); + const prefs = toggleColumn(defaultColumnPrefs, "requestSize"); + expect(prefs.order).toEqual([]); + expect(ids(prefs).slice(-4)).toEqual(["type", "requestSize", "size", "time"]); + // Among the built-in ones, so a custom column stays at the end + const custom = addCustomColumn(defaultColumnPrefs, "header.x-cache"); + expect(ids(custom).at(-1)).toBe("field:header.x-cache"); + const both = toggleColumn(custom, "requestSize"); + expect(both.order).toEqual([]); + expect(ids(both).slice(-4)).toEqual(["requestSize", "size", "time", "field:header.x-cache"]); + }); + + test("once reordered, a column turned on goes at the end, unless it had a place", () => { + let prefs = moveColumn(defaultColumnPrefs, "time", 0); + prefs = toggleColumn(prefs, "requestSize"); expect(ids(prefs).at(-1)).toBe("requestSize"); - expect(ids(prefs).slice(0, -1)).toEqual(defaults); // Moved, hidden and shown again, it goes back where it was moved to - prefs = moveColumn(prefs, "requestSize", defaults.indexOf("size")); + const before = ids(prefs).indexOf("size"); + prefs = moveColumn(prefs, "requestSize", before); prefs = toggleColumn(toggleColumn(prefs, "requestSize"), "requestSize"); - expect(ids(prefs).slice(-3)).toEqual(["requestSize", "size", "time"]); - // Already last of those shown, nothing needs placing - expect(toggleColumn(defaultColumnPrefs, "remote").order).toEqual([]); + expect(ids(prefs).slice(-3)).toEqual(["type", "requestSize", "size"]); + // A custom column added goes at the end too + expect(ids(addCustomColumn(prefs, "query.page")).at(-1)).toBe("field:query.page"); }); test("a custom column is added at the end and leaves the order when removed", () => { diff --git a/apps/yaak-proxy/lib/columns.ts b/apps/yaak-proxy/lib/columns.ts index 693cd9ee4..7c2cc339f 100644 --- a/apps/yaak-proxy/lib/columns.ts +++ b/apps/yaak-proxy/lib/columns.ts @@ -243,10 +243,11 @@ export function toggleColumn( /** * Puts a column just shown after the others shown, unless the person placed it before, so it * goes back to that place. The columns shown before it are placed too, where they are, so the - * ones not listed don't jump ahead of it. + * ones not listed don't jump ahead of it. Until the person has ordered the columns, each keeps + * its default place: a built-in one among the others, an added one at the end. */ function placeAtEnd(prefs: ColumnPrefs, id: ColumnId): ColumnPrefs { - if (prefs.order.includes(id)) return prefs; + if (prefs.order.length === 0 || prefs.order.includes(id)) return prefs; const all = allColumns(prefs); if (all.filter((c) => isVisible(prefs, c)).at(-1)?.id === id) return prefs; const placed = all.filter( From 80b587f507b19608bc7ec3be59b590b964c762aa Mon Sep 17 00:00:00 2001 From: Gregory Schier Date: Tue, 10 Nov 2026 10:09:39 -0700 Subject: [PATCH 25/45] Slide columns aside for a dragged one instead of drawing a drop line While a column is dragged it hides where it is and the columns between it and where it would land slide by its width, header and cells together, so the gap shows the drop. One style element carries a rule a column and changes only when the target does; the rows don't render. On drop the rules go as the order changes, in the same task, so nothing animates; on Escape the columns slide home and the column shows again. The drop line and its code are gone. Co-Authored-By: Claude Opus 5.5 --- apps/yaak-proxy/components/FlowList.tsx | 165 ++++++++++++------ apps/yaak-proxy/components/FlowListHeader.tsx | 120 ++++++------- apps/yaak-proxy/lib/columns.test.ts | 39 +++-- apps/yaak-proxy/lib/columns.ts | 32 +++- 4 files changed, 223 insertions(+), 133 deletions(-) diff --git a/apps/yaak-proxy/components/FlowList.tsx b/apps/yaak-proxy/components/FlowList.tsx index fc196d318..df6a8af35 100644 --- a/apps/yaak-proxy/components/FlowList.tsx +++ b/apps/yaak-proxy/components/FlowList.tsx @@ -30,6 +30,7 @@ import { type Column, type ColumnId, cellFilterTerm, + columnShifts, customValue, gridTemplate, gridWidth, @@ -308,15 +309,10 @@ export function FlowList({ style, className }: Props) { el.scrollLeft += e.deltaX; }, []); - // A dragged column's drop line, from the window's coordinates to the list's + // A dragged column: its ghost, and the others sliding aside to make room for it const rootRef = useRef(null); - const [dropLine, setDropLine] = useState(null); const getScrollElement = useCallback(() => scrollRef.current, []); - const { ghostColumn, onColumnDrag } = useColumnGhost(rootRef, headerScrollRef, scrollRef); - const handleDropLine = useCallback((x: number | null) => { - const root = rootRef.current; - setDropLine(x == null || root == null ? null : x - root.getBoundingClientRect().left); - }, []); + const onColumnDrag = useColumnDragView(rootRef, headerScrollRef, scrollRef); // A list too short to scroll still loads older flows on scrolling up. const handleWheel = useCallback( @@ -449,27 +445,9 @@ export function FlowList({ style, className }: Props) { onSetSort={handleSetSort} onResetSort={follow} scrollElement={getScrollElement} - onDropLine={handleDropLine} onColumnDrag={onColumnDrag} />
- {/* A dragged column's cells dim where they are, in one rule rather than a render a row */} - {ghostColumn != null && ( - - )} - {/* Where a dragged column would drop, down the header and the rows in view, over its ghost */} - {dropLine != null && ( -
-
-
- )}
{ }, [flows, following]); } +/** How long the columns take to slide aside for a dragged column, or back */ +const SLIDE_MS = 150; + /** - * The ghost of a dragged column: its header and the cells of the rows in view, copied once when - * the drag starts, at the column's width and the list's height. It follows the pointer sideways, - * kept within the list, by a transform, so the rows don't render while it moves; flows arriving - * or the list scrolling sideways meanwhile leave the copy as it was. + * Shows a column's drag without rendering a row. Its ghost is its header and the cells of the + * rows in view, copied once when the drag starts and following the pointer sideways. The column + * hides where it is, and the columns between it and where it would drop slide aside by its width + * to leave a gap there, all by one style element: a rule a column, header and cells together, + * which rows arriving meanwhile follow too. Dropped, the rules go as the order changes, in the + * same task, so nothing animates or jumps; put back, the columns slide home first. */ -function useColumnGhost( +function useColumnDragView( rootRef: RefObject, headerRef: RefObject, listRef: RefObject, -): { ghostColumn: ColumnId | null; onColumnDrag: (event: ColumnDragEvent) => void } { - const [ghostColumn, setGhostColumn] = useState(null); - const ghost = useRef(null); +): (event: ColumnDragEvent) => void { + const drag = useRef(null); + // Columns sliding home after a drag was put back, until they are there + const settling = useRef<{ style: HTMLStyleElement; timer: number } | null>(null); + + const settle = useCallback(() => { + const s = settling.current; + settling.current = null; + if (s == null) return; + clearTimeout(s.timer); + s.style.remove(); + }, []); const onColumnDrag = useCallback( (event: ColumnDragEvent) => { - if (event.type === "move") { - ghost.current?.move(event.x); - return; + const d = drag.current; + switch (event.type) { + case "move": + d?.ghost?.move(event.x); + return; + case "target": + if (d == null) return; + d.to = event.to; + d.style.textContent = columnDragCss(d); + return; + case "end": { + drag.current = null; + if (d == null) return; + d.ghost?.remove(); + if (event.dropped || d.to == null) { + d.style.remove(); + return; + } + d.to = null; + d.style.textContent = columnDragCss(d); + settling.current = { style: d.style, timer: window.setTimeout(settle, SLIDE_MS) }; + return; + } + case "start": { + settle(); + d?.ghost?.remove(); + d?.style.remove(); + drag.current = null; + const root = rootRef.current; + const header = headerRef.current; + const list = listRef.current; + if (root == null || header == null || list == null) return; + const cells = Array.from( + header.querySelectorAll("[data-flow-header] > [data-column]"), + ); + const ids = cells.map((c) => c.dataset.column as ColumnId); + const from = ids.indexOf(event.id); + if (from === -1) return; + // Copied before it hides + const ghost = snapshotColumn(event.id, event.x, root, header, list); + const style = document.createElement("style"); + document.head.append(style); + const width = cells[from]?.getBoundingClientRect().width ?? 0; + drag.current = { ghost, ids, from, width, to: null, style }; + style.textContent = columnDragCss(drag.current); + } } - ghost.current?.remove(); - ghost.current = null; - if (event.type === "end") { - setGhostColumn(null); - return; - } - const root = rootRef.current; - const header = headerRef.current; - const list = listRef.current; - if (root == null || header == null || list == null) return; - ghost.current = snapshotColumn(event.id, event.x, root, header, list); - setGhostColumn(event.id); }, - [rootRef, headerRef, listRef], + [rootRef, headerRef, listRef, settle], ); - useEffect(() => () => ghost.current?.remove(), []); - return { ghostColumn, onColumnDrag }; + useEffect( + () => () => { + settle(); + drag.current?.ghost?.remove(); + drag.current?.style.remove(); + drag.current = null; + }, + [settle], + ); + return onColumnDrag; +} + +interface ColumnDragView { + ghost: ColumnGhost | null; + /** The columns shown, the dragged one's place among them and its width, and its target */ + ids: ColumnId[]; + from: number; + width: number; + to: number | null; + style: HTMLStyleElement; +} + +/** The rules for a drag: the column hidden, and those it passes slid aside */ +function columnDragCss({ ids, from, width, to }: ColumnDragView): string { + // A row's cell is the box in its grid: the column's child, or inside the `contents` span a + // tooltip wraps it in, as a box that is only its contents can't move + const cells = (is: string) => + `[data-flow-rows] ${is} > :not(.contents), [data-flow-rows] ${is} > .contents > *`; + const column = (id: ColumnId) => { + const is = `[data-column="${CSS.escape(id)}"]`; + return `[data-flow-header] > ${is}, ${cells(is)}`; + }; + const rules = [ + `[data-flow-header] > [data-column], ${cells("[data-column]")} { transition: transform ${SLIDE_MS}ms ease; }`, + ]; + const dragged = ids[from]; + if (dragged != null) rules.push(`${column(dragged)} { visibility: hidden; }`); + columnShifts(ids.length, from, width, to).forEach((dx, i) => { + const id = ids[i]; + if (dx !== 0 && id != null) rules.push(`${column(id)} { transform: translateX(${dx}px); }`); + }); + return rules.join("\n"); } interface ColumnGhost { diff --git a/apps/yaak-proxy/components/FlowListHeader.tsx b/apps/yaak-proxy/components/FlowListHeader.tsx index 952ca6cc6..837eec069 100644 --- a/apps/yaak-proxy/components/FlowListHeader.tsx +++ b/apps/yaak-proxy/components/FlowListHeader.tsx @@ -73,17 +73,20 @@ interface Props { onResetSort: () => void; /** The rows' scrolling element, which a column dragged to its edge scrolls sideways */ scrollElement: () => HTMLElement | null; - /** Where a dragged column would drop, in the window's coordinates, or null for nowhere */ - onDropLine: (x: number | null) => void; - /** A column's drag: its start, the pointer's moves, and its end, for the list's ghost */ + /** A column's drag, for the list to show: its ghost, and the columns making room for it */ onColumnDrag: (event: ColumnDragEvent) => void; } -/** A column's drag, for the ghost the list draws of it; `x` is the pointer, across the window */ +/** + * A column's drag: its start, the pointer's moves (`x`, across the window), the boundary it + * would drop at when that changes (`to`, null where it would stay), and its end, dropped there + * or put back + */ export type ColumnDragEvent = | { type: "start"; id: ColumnId; x: number } | { type: "move"; x: number } - | { type: "end" }; + | { type: "target"; to: number | null } + | { type: "end"; dropped: boolean }; /** * The list's only control surface: a click on a label sorts by it alone, a second reverses it @@ -102,7 +105,6 @@ export const FlowListHeader = memo(function FlowListHeader({ onSetSort: setSort, onResetSort, scrollElement, - onDropLine, onColumnDrag, }: Props) { const setPrefs = useSetAtom(columnPrefsAtom); @@ -111,10 +113,9 @@ export const FlowListHeader = memo(function FlowListHeader({ const [menu, setMenu] = useState<{ x: number; y: number; column: ColumnDef | null } | null>(null); // Widths when a drag began, so each move sets the width from where it started. const startWidth = useRef(0); - const { dragging, sensors, dndHandlers, gridRef, onPointerDownCapture } = useColumnDrag({ + const { sensors, dndHandlers, gridRef, onPointerDownCapture } = useColumnDrag({ columns, scrollElement, - onDropLine, onColumnDrag, onDrop: (id, to) => setPrefs((p) => moveColumn(p, id, to, unavailable)), }); @@ -232,6 +233,7 @@ export const FlowListHeader = memo(function FlowListHeader({ {/* oxlint-disable-next-line jsx-a11y/no-static-element-interactions -- a right-click opens the column's menu */}
{ startWidth.current = c.current; @@ -281,8 +282,6 @@ export const FlowListHeader = memo(function FlowListHeader({ interface HeaderCellProps { column: Column; sort: FlowSortKey[]; - /** Being dragged, so it dims where it was */ - dragging: boolean; onSort: (column: ColumnId, add: boolean) => void; onResizeStart: () => void; /** How far the edge moved since the resize began */ @@ -293,7 +292,6 @@ interface HeaderCellProps { function HeaderCell({ column: c, sort, - dragging, onSort, onResizeStart, onResizeMove, @@ -303,13 +301,7 @@ function HeaderCell({ const { listeners, setNodeRef } = useDraggable({ id: c.id }); const sorted = sort.some((k) => k.column === c.id); return ( - + ); - if (!hasButtonSubmenu) { + if (overlaid === 0) { return button; } - // The submenu trigger overlays the row as a sibling (not a child) because the row is - // itself a button and buttons cannot nest. Hover handling lives on this wrapper so the - // row keeps its focus highlight while the mouse is over the trigger. + // The actions and submenu trigger overlay the row as siblings (not children) because the row + // is itself a button and buttons cannot nest. Hover handling lives on this wrapper so the + // row keeps its focus highlight while the mouse is over them. return (
- { - // Prevent the trigger from stealing focus, which would unhighlight the row - e.preventDefault(); - }} - onClick={(e) => { - e.stopPropagation(); - onOpenSubmenu(item, e.currentTarget); - }} - /> + {actions.map((action) => ( + e.preventDefault()} + onClick={(e) => { + e.stopPropagation(); + action.onSelect(); + if (action.closeMenu) onCloseAll?.(); + }} + /> + ))} + {hasButtonSubmenu && ( + { + // Prevent the trigger from stealing focus, which would unhighlight the row + e.preventDefault(); + }} + onClick={(e) => { + e.stopPropagation(); + onOpenSubmenu(item, e.currentTarget); + }} + /> + )}
); diff --git a/packages/ui/src/components/editor/query/extension.ts b/packages/ui/src/components/editor/query/extension.ts index e273af6a9..ba8adc394 100644 --- a/packages/ui/src/components/editor/query/extension.ts +++ b/packages/ui/src/components/editor/query/extension.ts @@ -34,6 +34,11 @@ export interface QueryFilterOptions { * searches. `typed` is the word, and the completions replace it. */ moreCompletions?: (typed: string) => Completion[]; + /** + * Completions above the engine's while the box is empty, such as recent searches, each + * replacing the whole text + */ + leadCompletions?: () => Completion[]; /** The terms of a query the search cannot run, underlined as typing settles */ problems?: (text: string) => Promise; /** @@ -139,6 +144,7 @@ function completionSource({ values, loadValues, moreCompletions, + leadCompletions, fieldsOnly, }: QueryFilterOptions) { return async (ctx: CompletionContext): Promise => { @@ -179,11 +185,14 @@ function completionSource({ if (from === word.from) options.push(...more); } } - if (options.length === 0) return null; - // Field names pop up for something typed; on an empty spot, only when asked for const onlyFields = options.every((o) => o.type === "field"); - if (onlyFields && from === to && !ctx.explicit) return null; + if (onlyFields && from === to && !ctx.explicit) options.length = 0; + + // In an empty box, both they and the leading ones start at its start + const lead = text === "" ? (leadCompletions?.() ?? []) : []; + options.unshift(...lead); + if (options.length === 0) return null; // The engine has already filtered and ranked the options return { from, to, options, filter: false }; diff --git a/packages/ui/src/components/http/HttpRequestPaneView.tsx b/packages/ui/src/components/http/HttpRequestPaneView.tsx index 56a6fd619..0121f66ff 100644 --- a/packages/ui/src/components/http/HttpRequestPaneView.tsx +++ b/packages/ui/src/components/http/HttpRequestPaneView.tsx @@ -116,6 +116,8 @@ export interface HttpRequestPaneViewProps { * request doesn't carry to the next */ defaultTab?: string; + /** The tab to show over any remembered, while this request offers it */ + activeTab?: string; /** A tab was chosen */ onTabChange?: (tab: string) => void; } @@ -168,6 +170,7 @@ export function HttpRequestPaneView({ urlLine, dimEmptyTabs = false, defaultTab, + activeTab, onTabChange, }: HttpRequestPaneViewProps) { const { ErrorBoundary } = useEditorHost(); @@ -291,6 +294,7 @@ export function HttpRequestPaneView({ storageKey={tabsStorageKey} activeTabKey={request.id} defaultValue={defaultTab} + value={activeTab} keyedDefault={defaultTab != null} onChangeValue={onTabChange} > diff --git a/packages/ui/src/components/http/HttpResponsePaneView.tsx b/packages/ui/src/components/http/HttpResponsePaneView.tsx index 7d0d10537..8ff7b77a1 100644 --- a/packages/ui/src/components/http/HttpResponsePaneView.tsx +++ b/packages/ui/src/components/http/HttpResponsePaneView.tsx @@ -83,6 +83,8 @@ export interface HttpResponsePaneViewProps { * doesn't carry to the next */ defaultTab?: string; + /** The tab to show over any remembered, while this response offers it */ + activeTab?: string; /** A tab was chosen */ onTabChange?: (tab: string) => void; } @@ -115,6 +117,7 @@ export function HttpResponsePaneView({ bare = false, dimEmptyTabs = false, defaultTab, + activeTab, onTabChange, }: HttpResponsePaneViewProps) { const { ErrorBoundary } = useEditorHost(); @@ -269,6 +272,7 @@ export function HttpResponsePaneView({ storageKey={tabsStorageKey} activeTabKey={activeTabKey} defaultValue={defaultTab} + value={activeTab} keyedDefault={defaultTab != null} onChangeValue={onTabChange} > diff --git a/packages/ui/src/components/tabs/Tabs.tsx b/packages/ui/src/components/tabs/Tabs.tsx index 541117c8b..72381ea69 100644 --- a/packages/ui/src/components/tabs/Tabs.tsx +++ b/packages/ui/src/components/tabs/Tabs.tsx @@ -79,6 +79,11 @@ export interface TabsProps { label: string; /** Default tab value. If not provided, defaults to first tab. */ defaultValue?: string; + /** + * The tab shown, over the one stored or last chosen, while it is offered. Choosing another + * still calls `onChangeValue`, for the owner to pass it back. + */ + value?: string; /** Called when active tab changes */ onChangeValue?: (value: string) => void; tabs: TabItem[]; @@ -103,6 +108,7 @@ export interface TabsProps { export const Tabs = forwardRef(function Tabs( { defaultValue, + value: controlledValue, onChangeValue: onChangeValueProp, label, children, @@ -146,7 +152,12 @@ export const Tabs = forwardRef(function Tabs( keyedDefault && internal?.key !== activeTabKey ? undefined : internal?.value; const offered = (v: string | undefined) => v != null && (!keyedDefault || originalTabs.some((t) => t.value === v)); - const value = [storedActiveTab, internalValue, defaultValue].find(offered) ?? originalTabs[0]?.value; + const controlled = originalTabs.some((t) => t.value === controlledValue) + ? controlledValue + : undefined; + const value = + [controlled, storedActiveTab, internalValue, defaultValue].find(offered) ?? + originalTabs[0]?.value; // Helper to normalize storage (handle migration from old format) const normalizeStorage = useCallback( diff --git a/packages/ui/src/index.ts b/packages/ui/src/index.ts index a004464d3..57be17b70 100644 --- a/packages/ui/src/index.ts +++ b/packages/ui/src/index.ts @@ -15,6 +15,7 @@ export { DropMarker } from "./components/DropMarker"; export type { ContextMenuProps, DropdownItem, + DropdownItemAction, DropdownItemContent, DropdownItemDefault, DropdownItemSeparator, From cc83c7cd1e3e29d25d1ceef9ede315b01a597afd Mon Sep 17 00:00:00 2001 From: Gregory Schier Date: Tue, 10 Nov 2026 10:48:34 -0700 Subject: [PATCH 29/45] Keep filters out of the web view's storage, and let a newer selection win over a late read Filters can hold captured values, so neither the reload's places nor the launch's last filter go into web view storage: the engine holds the places in memory while the app runs, and proxy.db's last_filter row keeps the last filter. A read of a selected flow that answers after the window shows a newer copy no longer replaces it. The pane and stream harnesses select through goToPlace, so switching a tab no longer empties their card. Co-Authored-By: Claude Opus 5.5 --- apps/yaak-proxy/harness/FlowPanesHarness.tsx | 9 ++- apps/yaak-proxy/harness/LockedHarness.tsx | 9 ++- apps/yaak-proxy/harness/MessagesHarness.tsx | 11 +-- apps/yaak-proxy/harness/backend.ts | 12 +++ apps/yaak-proxy/lib/navigation.test.ts | 67 +++++++++++++++- apps/yaak-proxy/lib/navigation.ts | 80 +++++++++++-------- apps/yaak-proxy/main.tsx | 2 +- .../yaak-proxy-lib/bindings/gen_rpc.ts | 16 +++- .../migrations/00000014_filter_history.sql | 7 ++ crates-proxy/yaak-proxy-lib/src/db.rs | 1 + .../yaak-proxy-lib/src/filter_history.rs | 46 +++++++++++ crates-proxy/yaak-proxy-lib/src/lib.rs | 71 ++++++++++++++++ 12 files changed, 281 insertions(+), 50 deletions(-) diff --git a/apps/yaak-proxy/harness/FlowPanesHarness.tsx b/apps/yaak-proxy/harness/FlowPanesHarness.tsx index e24dcada3..75889c098 100644 --- a/apps/yaak-proxy/harness/FlowPanesHarness.tsx +++ b/apps/yaak-proxy/harness/FlowPanesHarness.tsx @@ -1,10 +1,10 @@ -import { useSetAtom } from "jotai"; +import { useSetAtom, useStore } from "jotai"; import { useEffect, useLayoutEffect, useState } from "react"; import { AddColumnDialog } from "../components/AddColumnDialog"; import { FlowDetail } from "../components/FlowDetail"; import { FlowList } from "../components/FlowList"; -import { selectedFlowAtom } from "../lib/atoms"; import { browserTabMemory, learnedTabKey, flowShape } from "../lib/defaultTab"; +import { goToPlace } from "../lib/navigation"; import { flowWindowAtom } from "../lib/store"; import { type FlowFixture, fixtures } from "./fixtures"; @@ -42,7 +42,7 @@ export function FlowPanesHarness() { function FixtureCard({ fixture }: { fixture: FlowFixture }) { const params = new URLSearchParams(location.search); - const setSelected = useSetAtom(selectedFlowAtom); + const store = useStore(); const setWindow = useSetAtom(flowWindowAtom); const [ready, setReady] = useState(false); useLayoutEffect(() => { @@ -52,7 +52,8 @@ function FixtureCard({ fixture }: { fixture: FlowFixture }) { browserTabMemory()?.setItem(learnedTabKey("response", flowShape(fixture.flow)), tab); } setWindow({ flows, hasOlder: false, total: flows.length }); - setSelected(flows.find((f) => f.id === fixture.flow.id) ?? fixture.flow); + const shown = flows.find((f) => f.id === fixture.flow.id) ?? fixture.flow; + goToPlace(store, "update", { selection: [shown.id], flow: shown }); setReady(true); // oxlint-disable-next-line react-hooks/exhaustive-deps -- once per fixture }, [fixture]); diff --git a/apps/yaak-proxy/harness/LockedHarness.tsx b/apps/yaak-proxy/harness/LockedHarness.tsx index cf948763e..3612813c5 100644 --- a/apps/yaak-proxy/harness/LockedHarness.tsx +++ b/apps/yaak-proxy/harness/LockedHarness.tsx @@ -82,10 +82,11 @@ export function LockedHarness() { const [synced, setSynced] = useState(false); useEffect(() => { syncProxyState(store); - syncNavigation(store); - syncFlows(store); - syncSidebarSelection(store); - setSynced(true); + void syncNavigation(store).then(() => { + syncFlows(store); + syncSidebarSelection(store); + setSynced(true); + }); }, [store]); if (!synced) return null; return ( diff --git a/apps/yaak-proxy/harness/MessagesHarness.tsx b/apps/yaak-proxy/harness/MessagesHarness.tsx index 1d980d0c6..84f1148fe 100644 --- a/apps/yaak-proxy/harness/MessagesHarness.tsx +++ b/apps/yaak-proxy/harness/MessagesHarness.tsx @@ -1,11 +1,11 @@ import type { Flow } from "@yaakapp-internal/proxy-lib"; -import { useSetAtom } from "jotai"; +import { useSetAtom, useStore } from "jotai"; import { useLayoutEffect, useMemo, useState } from "react"; import { FlowDetail } from "../components/FlowDetail"; import { FlowList } from "../components/FlowList"; import { ProxyEditorHost } from "../components/ProxyEditorHost"; import { MessageSourceContext } from "../hooks/useFlowMessages"; -import { selectedFlowAtom } from "../lib/atoms"; +import { goToPlace } from "../lib/navigation"; import { flowWindowAtom } from "../lib/store"; import { fixtureSource, type StreamFixture, streamFixtures } from "./streamFixtures"; @@ -51,15 +51,16 @@ function listedFlows(shown: Flow): Flow[] { function StreamDetail({ fixture }: { fixture: StreamFixture }) { const source = useMemo(() => fixtureSource(fixture), [fixture]); - const setSelected = useSetAtom(selectedFlowAtom); + const store = useStore(); const setWindow = useSetAtom(flowWindowAtom); const [flow, setFlow] = useState(fixture.flow); const withList = new URLSearchParams(location.search).has("list"); useLayoutEffect(() => { const flows = listedFlows(flow); setWindow({ flows, hasOlder: false, total: flows.length }); - setSelected(flows.find((f) => f.id === flow.id) ?? flow); - }, [flow, setSelected, setWindow]); + const shown = flows.find((f) => f.id === flow.id) ?? flow; + goToPlace(store, "update", { selection: [shown.id], flow: shown }); + }, [flow, store, setWindow]); // The count on the flow follows its messages, as the engine writes it useLayoutEffect( () => source.subscribe(() => setFlow((f) => ({ ...f, messageCount: f.messageCount + 1 }))), diff --git a/apps/yaak-proxy/harness/backend.ts b/apps/yaak-proxy/harness/backend.ts index 48c5d3dbc..c87286c53 100644 --- a/apps/yaak-proxy/harness/backend.ts +++ b/apps/yaak-proxy/harness/backend.ts @@ -72,6 +72,7 @@ function emit(event: string, payload: unknown = {}) { /** Filters applied, newest first, and saved searches, kept across reloads as the database is */ const HISTORY_KEY = "harness_filter_history"; const SAVED_KEY = "harness_saved_queries"; +const LAST_FILTER_KEY = "harness_last_filter"; function stored(key: string): T[] { try { @@ -146,6 +147,17 @@ function answer(cmd: string, payload: unknown): unknown { store(HISTORY_KEY, []); emit("filter_history_changed"); return 0; + // Held by the engine for as long as the app runs; the tab's session here + case "get_window_places": + return sessionStorage.getItem("harness_window_places"); + case "set_window_places": + sessionStorage.setItem("harness_window_places", String(req.places)); + return true; + case "get_last_filter": + return localStorage.getItem(LAST_FILTER_KEY) ?? ""; + case "set_last_filter": + localStorage.setItem(LAST_FILTER_KEY, String(req.filter)); + return true; case "list_saved_queries": return stored(SAVED_KEY); case "create_saved_query": { diff --git a/apps/yaak-proxy/lib/navigation.test.ts b/apps/yaak-proxy/lib/navigation.test.ts index 39d295b07..591e30f66 100644 --- a/apps/yaak-proxy/lib/navigation.test.ts +++ b/apps/yaak-proxy/lib/navigation.test.ts @@ -14,6 +14,8 @@ import { EMPTY_PLACE, stackOf } from "./places"; const existing = new Set(); const recorded: string[] = []; +let lastFilter = ""; +let windowPlaces: string | null = null; vi.mock("./rpc", () => ({ rpc: vi.fn(async (cmd: string, payload: { ids?: string[]; filter?: string }) => { @@ -21,6 +23,10 @@ vi.mock("./rpc", () => ({ return (payload.ids ?? []).filter((id) => existing.has(id)).map((id) => flow(id)); } if (cmd === "record_filter") recorded.push(payload.filter ?? ""); + if (cmd === "get_last_filter") return lastFilter; + if (cmd === "get_window_places") return windowPlaces; + if (cmd === "set_window_places") windowPlaces = (payload as { places: string }).places; + if (cmd === "set_last_filter") lastFilter = payload.filter ?? ""; return true; }), listen: vi.fn(() => () => {}), @@ -29,7 +35,7 @@ vi.mock("./tauri", () => ({ command: vi.fn(async () => {}), subscribe: vi.fn(() // The package's index reaches for the window as it loads; the sidebar's selection is all used. vi.mock("@yaakapp-internal/ui", () => ({ selectedIdsFamily: () => atom([]) })); -const { applyFilter, goToPlace, searchDraftAtom, searchDraftVersionAtom } = +const { applyFilter, goToPlace, searchDraftAtom, searchDraftVersionAtom, syncNavigation } = await import("./navigation"); function flow(id: string): Flow { @@ -47,6 +53,65 @@ const settle = () => new Promise((resolve) => setTimeout(resolve, 0)); beforeEach(() => { existing.clear(); recorded.length = 0; + lastFilter = ""; + windowPlaces = null; +}); + +describe("syncNavigation", () => { + test("a launch restores the last filter applied, and keeps each one applied after", async () => { + lastFilter = "@host:*.github.com"; + const store = createStore(); + await syncNavigation(store); + await settle(); + expect(store.get(appliedFilterAtom)).toBe("@host:*.github.com"); + expect(store.get(searchDraftAtom)).toBe("@host:*.github.com"); + expect(store.get(canGoBackAtom)).toBe(false); + + applyFilter(store, "stripe"); + await settle(); + expect(lastFilter).toBe("stripe"); + }); + + test("a launch the person has already moved on from keeps where they are", async () => { + lastFilter = "old"; + const store = createStore(); + await syncNavigation(store); + applyFilter(store, "new"); + await settle(); + expect(store.get(appliedFilterAtom)).toBe("new"); + }); + + test("a reload takes back the places, the tabs and the selection, not the last filter", async () => { + existing.add("B"); + lastFilter = "elsewhere"; + const before = createStore(); + await syncNavigation(before); + applyFilter(before, "x"); + goToPlace(before, "add", { selection: ["B"], flow: flow("B") }); + goToPlace(before, "update", { tabs: { response: "headers" } }); + await settle(); + + const after = createStore(); + await syncNavigation(after); + await settle(); + expect(after.get(appliedFilterAtom)).toBe("x"); + expect(after.get(placeAtom).tabs).toEqual({ response: "headers" }); + expect(after.get(selectedFlowAtom)?.id).toBe("B"); + expect(goToPlace(after, "back")).toBe(true); + }); + + test("a late read of the selected flow leaves a newer one alone", async () => { + existing.add("A"); + const store = newStore(); + goToPlace(store, "add", { selection: ["A"], flow: flow("A") }); + goToPlace(store, "add", { selection: ["B"], flow: flow("B") }); + store.set(selectedFlowAtom, null); + goToPlace(store, "back"); + const newer = { ...flow("A"), status: 200 } as Flow; + goToPlace(store, "update", { selection: ["A"], flow: newer }); + await settle(); + expect(store.get(selectedFlowAtom)).toBe(newer); + }); }); describe("goToPlace", () => { diff --git a/apps/yaak-proxy/lib/navigation.ts b/apps/yaak-proxy/lib/navigation.ts index d6b2b15c8..bd7fdbb9d 100644 --- a/apps/yaak-proxy/lib/navigation.ts +++ b/apps/yaak-proxy/lib/navigation.ts @@ -163,49 +163,61 @@ function readSelected(store: Store, id: string) { store.set(placeStackAtom, (s) => stackWithout(s, new Set([id]))); return; } - // The window has moved on meanwhile. - if (currentPlace(store.get(placeStackAtom)).selection.has(id)) { - store.set(selectedFlowAtom, found); - } + // The window has moved on meanwhile, or shows the flow already, as it was since. + const current = currentPlace(store.get(placeStackAtom)).selection.has(id); + if (current && store.get(selectedFlowAtom) == null) store.set(selectedFlowAtom, found); }) .catch((err) => console.warn("Failed to read the selected flow", err)); } -/** Where the window's places are kept for a reload, for this session */ -const PLACES_KEY = "proxy_places"; -/** Where the filter last applied is kept for the next launch */ -const LAST_FILTER_KEY = "proxy_last_filter"; - -function read(storage: () => Storage, key: string): string | null { - try { - return storage().getItem(key); - } catch { - return null; - } -} - -function write(storage: () => Storage, key: string, value: string) { - try { - storage().setItem(key, value); - } catch { - // Not kept: the next reload starts from the last filter. - } -} - /** - * Restores where the window was, before anything reads the filter: its places after a reload, - * or the last filter applied at launch. Keeps them as they change. + * Restores where the window was, before the flows load for its filter: its places after a + * reload, which the engine holds in memory while the app runs, or at launch the last filter + * applied, which the encrypted database keeps. Filters can hold captured values, so neither is + * kept in the web view's storage. Keeps them as they change. */ -export function syncNavigation(store: Store) { - const kept = stackFromJson(read(() => sessionStorage, PLACES_KEY)); - const lastFilter = read(() => localStorage, LAST_FILTER_KEY) ?? ""; - const stack = kept ?? stackOf({ ...EMPTY_PLACE, filter: lastFilter }); +export async function syncNavigation(store: Store) { + const kept = stackFromJson(await rpc("get_window_places", {}).catch(() => null)); + const stack = kept ?? stackOf(EMPTY_PLACE); store.set(placeStackAtom, stack); showPlace(store, currentPlace(stack), { filter: true, reveal: true }); + if (kept == null) { + // Unread while traffic is locked, as the database is. + rpc("get_last_filter", {}) + .then((filter) => { + // Only over the window as it opened, not where it has been taken since. + const now = store.get(placeStackAtom); + if (filter === "" || now.places.length > 1 || currentPlace(now).filter !== "") return; + goToPlace(store, "update", { filter }); + }) + .catch(() => {}); + } + + // One write at a time, the latest places after it, so they arrive in order. + let pending: string | null = null; + let writing = false; + const write = () => { + if (writing || pending == null) return; + const places = pending; + pending = null; + writing = true; + rpc("set_window_places", { places }) + .catch(() => {}) + .finally(() => { + writing = false; + write(); + }); + }; store.sub(placeStackAtom, () => { - const now = store.get(placeStackAtom); - write(() => sessionStorage, PLACES_KEY, stackToJson(now)); - write(() => localStorage, LAST_FILTER_KEY, currentPlace(now).filter); + pending = stackToJson(store.get(placeStackAtom)); + write(); + }); + let lastFilter = store.get(appliedFilterAtom); + store.sub(appliedFilterAtom, () => { + const filter = store.get(appliedFilterAtom); + if (filter === lastFilter) return; + lastFilter = filter; + rpc("set_last_filter", { filter }).catch(() => {}); }); } diff --git a/apps/yaak-proxy/main.tsx b/apps/yaak-proxy/main.tsx index 1d4c7cd9b..f4a86dd70 100644 --- a/apps/yaak-proxy/main.tsx +++ b/apps/yaak-proxy/main.tsx @@ -15,7 +15,7 @@ const jotaiStore = createStore(); syncProxyState(jotaiStore); // Before the flows load, so they load for the filter it restores -syncNavigation(jotaiStore); +await syncNavigation(jotaiStore); syncFlows(jotaiStore); syncSidebarSelection(jotaiStore); diff --git a/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts b/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts index d5226a8b4..41f542d20 100644 --- a/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts +++ b/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts @@ -250,6 +250,8 @@ note?: string, }; */ export type GetFlowsRequest = { ids: Array, }; +export type GetLastFilterRequest = Record; + export type GetMessagePayloadRequest = { flowId: string, seq: number, }; export type GetMessagePayloadResponse = { @@ -326,6 +328,8 @@ export type GetStorageSizeResponse = { */ bytes: number, }; +export type GetWindowPlacesRequest = Record; + export type GlobalAction = "recording_toggle" | "flows_clear"; export type HelperRegisterRequest = Record; @@ -492,7 +496,7 @@ export type ResetCapturedDataRequest = Record; export type RpcEventSchema = { decrypt_list_changed: DecryptListChangedPayload, filter_history_changed: FilterHistoryChangedPayload, flows_cleared: FlowsClearedPayload, flows_removed: FlowsRemovedPayload, message_write: MessagePayload, model_write: ModelPayload, proxy_state_changed: ProxyStatePayload, saved_queries_changed: SavedQueriesChangedPayload, }; -export type RpcSchema = { execute_action: [ActionInvocation, boolean], format_body: [FormatBodyRequest, string], get_decrypt_list: [GetDecryptListRequest, DecryptList], get_message_payload: [GetMessagePayloadRequest, GetMessagePayloadResponse], list_actions: [ListActionsRequest, ListActionsResponse], list_flows: [FlowFilter, ListFlowsResponse], query_schema: [QuerySchemaRequest, QuerySchemaResponse], validate_query: [ValidateQueryRequest, Array], watch_query: [WatchQueryRequest, boolean], list_saved_queries: [ListSavedQueriesRequest, Array], create_saved_query: [CreateSavedQueryRequest, SavedQuery], rename_saved_query: [RenameSavedQueryRequest, SavedQuery], delete_saved_query: [DeleteSavedQueryRequest, boolean], list_filter_history: [ListFilterHistoryRequest, Array], record_filter: [RecordFilterRequest, boolean], delete_filter_history: [DeleteFilterHistoryRequest, boolean], clear_filter_history: [ClearFilterHistoryRequest, number], list_grpc_schemas: [ListGrpcSchemasRequest, Array], list_messages: [ListMessagesRequest, ListMessagesResponse], remove_grpc_schema: [RemoveGrpcSchemaRequest, boolean], set_recording: [SetRecordingRequest, boolean], set_system_proxy_at_launch: [SetSystemProxyAtLaunchRequest, boolean], set_http_version: [SetHttpVersionRequest, HttpVersion], set_capture_body_limit: [SetCaptureBodyLimitRequest, number], set_traffic: [SetTrafficRequest, TrafficRetention], set_keep_traffic: [SetKeepTrafficRequest, number | null], add_grpc_schema: [AddGrpcSchemaRequest, GrpcSchema], app_icon: [AppIconRequest, string | null], ca_status: [CaStatusRequest, CaStatus], ca_trust: [CaTrustRequest, CaStatus], clear_flows: [ClearFlowsRequest, number], decode_grpc_message: [DecodeGrpcMessageRequest, DecodedGrpcMessage], export_har: [ExportHarRequest, number], filter_body: [FilterBodyRequest, FilterBodyResponse], get_flow_body: [GetFlowBodyRequest, GetFlowBodyResponse], get_proxy_state: [GetProxyStateRequest, GetProxyStateResponse], helper_register: [HelperRegisterRequest, SystemProxyStatus], helper_unregister: [HelperUnregisterRequest, SystemProxyStatus], edit_decrypt_list: [EditDecryptListRequest, DecryptList], list_seen: [ListSeenRequest, SeenTargets], set_decrypt_list: [SetDecryptListRequest, DecryptList], list_workspaces: [ListWorkspacesRequest, ListWorkspacesResponse], preview_save: [PreviewSaveRequest, SavePreview], reflect_grpc_host: [ReflectGrpcHostRequest, GrpcSchema], save_flow: [SaveFlowToYaakRequest, SavedFlow], save_flow_body: [SaveFlowBodyRequest, SaveFlowBodyResponse], system_proxy_set: [SystemProxySetRequest, SystemProxyStatus], system_proxy_status: [SystemProxyStatusRequest, SystemProxyStatus], take_over: [TakeOverRequest, boolean], set_port: [SetPortRequest, number], get_storage_size: [GetStorageSizeRequest, GetStorageSizeResponse], reset_captured_data: [ResetCapturedDataRequest, boolean], open_terminal: [OpenTerminalRequest, boolean], query_values: [QueryValuesRequest, Array], match_flows: [MatchFlowsRequest, Array], get_flows: [GetFlowsRequest, Array], count_saved_queries: [CountSavedQueriesRequest, Array], }; +export type RpcSchema = { execute_action: [ActionInvocation, boolean], format_body: [FormatBodyRequest, string], get_decrypt_list: [GetDecryptListRequest, DecryptList], get_message_payload: [GetMessagePayloadRequest, GetMessagePayloadResponse], list_actions: [ListActionsRequest, ListActionsResponse], list_flows: [FlowFilter, ListFlowsResponse], query_schema: [QuerySchemaRequest, QuerySchemaResponse], validate_query: [ValidateQueryRequest, Array], watch_query: [WatchQueryRequest, boolean], list_saved_queries: [ListSavedQueriesRequest, Array], create_saved_query: [CreateSavedQueryRequest, SavedQuery], rename_saved_query: [RenameSavedQueryRequest, SavedQuery], delete_saved_query: [DeleteSavedQueryRequest, boolean], list_filter_history: [ListFilterHistoryRequest, Array], record_filter: [RecordFilterRequest, boolean], delete_filter_history: [DeleteFilterHistoryRequest, boolean], clear_filter_history: [ClearFilterHistoryRequest, number], get_last_filter: [GetLastFilterRequest, string], set_last_filter: [SetLastFilterRequest, boolean], get_window_places: [GetWindowPlacesRequest, string | null], set_window_places: [SetWindowPlacesRequest, boolean], list_grpc_schemas: [ListGrpcSchemasRequest, Array], list_messages: [ListMessagesRequest, ListMessagesResponse], remove_grpc_schema: [RemoveGrpcSchemaRequest, boolean], set_recording: [SetRecordingRequest, boolean], set_system_proxy_at_launch: [SetSystemProxyAtLaunchRequest, boolean], set_http_version: [SetHttpVersionRequest, HttpVersion], set_capture_body_limit: [SetCaptureBodyLimitRequest, number], set_traffic: [SetTrafficRequest, TrafficRetention], set_keep_traffic: [SetKeepTrafficRequest, number | null], add_grpc_schema: [AddGrpcSchemaRequest, GrpcSchema], app_icon: [AppIconRequest, string | null], ca_status: [CaStatusRequest, CaStatus], ca_trust: [CaTrustRequest, CaStatus], clear_flows: [ClearFlowsRequest, number], decode_grpc_message: [DecodeGrpcMessageRequest, DecodedGrpcMessage], export_har: [ExportHarRequest, number], filter_body: [FilterBodyRequest, FilterBodyResponse], get_flow_body: [GetFlowBodyRequest, GetFlowBodyResponse], get_proxy_state: [GetProxyStateRequest, GetProxyStateResponse], helper_register: [HelperRegisterRequest, SystemProxyStatus], helper_unregister: [HelperUnregisterRequest, SystemProxyStatus], edit_decrypt_list: [EditDecryptListRequest, DecryptList], list_seen: [ListSeenRequest, SeenTargets], set_decrypt_list: [SetDecryptListRequest, DecryptList], list_workspaces: [ListWorkspacesRequest, ListWorkspacesResponse], preview_save: [PreviewSaveRequest, SavePreview], reflect_grpc_host: [ReflectGrpcHostRequest, GrpcSchema], save_flow: [SaveFlowToYaakRequest, SavedFlow], save_flow_body: [SaveFlowBodyRequest, SaveFlowBodyResponse], system_proxy_set: [SystemProxySetRequest, SystemProxyStatus], system_proxy_status: [SystemProxyStatusRequest, SystemProxyStatus], take_over: [TakeOverRequest, boolean], set_port: [SetPortRequest, number], get_storage_size: [GetStorageSizeRequest, GetStorageSizeResponse], reset_captured_data: [ResetCapturedDataRequest, boolean], open_terminal: [OpenTerminalRequest, boolean], query_values: [QueryValuesRequest, Array], match_flows: [MatchFlowsRequest, Array], get_flows: [GetFlowsRequest, Array], count_saved_queries: [CountSavedQueriesRequest, Array], }; export type SaveFlowBodyRequest = { flowId: string, which: FlowBody, /** @@ -606,6 +610,11 @@ export type SetKeepTrafficRequest = { */ bytes: number | null, }; +/** + * The filter the window applied, kept in the encrypted database for the next launch. + */ +export type SetLastFilterRequest = { filter: string, }; + export type SetPortRequest = { port: number, }; export type SetRecordingRequest = { enabled: boolean, }; @@ -614,6 +623,11 @@ export type SetSystemProxyAtLaunchRequest = { enabled: boolean, }; export type SetTrafficRequest = { traffic: TrafficRetention, }; +/** + * The window's places, opaque to the engine, kept in memory for a reload of the window. + */ +export type SetWindowPlacesRequest = { places: string, }; + /** * A column the list sorts by. */ diff --git a/crates-proxy/yaak-proxy-lib/migrations/00000014_filter_history.sql b/crates-proxy/yaak-proxy-lib/migrations/00000014_filter_history.sql index 62d18ea27..00b369bb9 100644 --- a/crates-proxy/yaak-proxy-lib/migrations/00000014_filter_history.sql +++ b/crates-proxy/yaak-proxy-lib/migrations/00000014_filter_history.sql @@ -4,3 +4,10 @@ CREATE TABLE filter_history filter TEXT NOT NULL PRIMARY KEY, applied_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL ); + +-- The filter the window last applied, however it was applied, which the next launch restores. +CREATE TABLE last_filter +( + id INTEGER NOT NULL PRIMARY KEY CHECK (id = 1), + filter TEXT NOT NULL +); diff --git a/crates-proxy/yaak-proxy-lib/src/db.rs b/crates-proxy/yaak-proxy-lib/src/db.rs index 088d7113d..545cb1e84 100644 --- a/crates-proxy/yaak-proxy-lib/src/db.rs +++ b/crates-proxy/yaak-proxy-lib/src/db.rs @@ -525,6 +525,7 @@ mod tests { "flow_sweep_pending", "flows", "grpc_schemas", + "last_filter", "saved_queries" ] ); diff --git a/crates-proxy/yaak-proxy-lib/src/filter_history.rs b/crates-proxy/yaak-proxy-lib/src/filter_history.rs index 4c4f473b9..08d7c8c79 100644 --- a/crates-proxy/yaak-proxy-lib/src/filter_history.rs +++ b/crates-proxy/yaak-proxy-lib/src/filter_history.rs @@ -84,6 +84,38 @@ pub fn clear(db: &ProxyQueryManager) -> Result { .map_err(|e| failed("clear", e)) } +/// The filter the window last applied, in any way, or empty for none. Empty for a database from +/// before it, read without migrating. +pub fn last(db: &ProxyQueryManager) -> Result { + db.with_conn(|c| { + let conn = c.conn().resolve(); + let Ok(mut stmt) = conn.prepare("SELECT filter FROM last_filter WHERE id = 1") else { + return Ok(String::new()); + }; + match stmt.query_row([], |row| row.get(0)) { + Ok(filter) => Ok(filter), + Err(rusqlite::Error::QueryReturnedNoRows) => Ok(String::new()), + Err(e) => Err(failed("read the last filter from", e)), + } + }) +} + +/// Keeps the filter the window applied, for the next launch to restore. +pub fn set_last(db: &ProxyQueryManager, filter: &str) -> Result<(), String> { + db.with_tx(|c| { + c.conn() + .resolve() + .execute( + "INSERT INTO last_filter (id, filter) VALUES (1, ?) \ + ON CONFLICT (id) DO UPDATE SET filter = excluded.filter", + [filter], + ) + .map(|_| ()) + .map_err(yaak_database::Error::from) + }) + .map_err(|e| failed("keep the last filter in", e)) +} + #[cfg(test)] mod tests { use super::*; @@ -136,6 +168,20 @@ mod tests { assert!(kept.contains(&"f6".to_string())); } + #[test] + fn keeps_the_last_filter_applied() { + let dir = tempfile::tempdir().unwrap(); + let db = open(&dir); + assert_eq!(last(&db).unwrap(), ""); + set_last(&db, "@host:*.github.com").unwrap(); + set_last(&db, "stripe").unwrap(); + assert_eq!(last(&db).unwrap(), "stripe"); + set_last(&db, "").unwrap(); + assert_eq!(last(&db).unwrap(), ""); + // Apart from the history, which only Enter writes. + assert_eq!(filters(&db), Vec::::new()); + } + #[test] fn removes_one_or_all() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates-proxy/yaak-proxy-lib/src/lib.rs b/crates-proxy/yaak-proxy-lib/src/lib.rs index 8d34ec8e1..8a6298ec9 100644 --- a/crates-proxy/yaak-proxy-lib/src/lib.rs +++ b/crates-proxy/yaak-proxy-lib/src/lib.rs @@ -110,6 +110,9 @@ pub struct ProxyCtx { /// Captured traffic is to be emptied as the databases are first unlocked, this launch /// being set to clear it. Taken once it is. clear_at_launch: Arc, + /// The window's places, as it last sent them, for a reload of it to take back. Held in + /// memory only, as filters can hold captured values, and gone when the app quits. + window_places: Arc>>, /// `traffic.lock`, locked shared for as long as this process has the databases open, so /// another launch can't delete them from under it: clearing them takes it alone. traffic_lock: Arc>>, @@ -167,6 +170,7 @@ impl ProxyCtx { resets: Arc::new(AtomicU64::new(0)), sweeper: Default::default(), clear_at_launch: Arc::new(AtomicBool::new(false)), + window_places: Arc::new(Mutex::new(None)), traffic_lock: Arc::new(Mutex::new(None)), system_proxy_intent: Arc::new(Mutex::new(())), watching_system_proxy: Arc::new(AtomicBool::new(false)), @@ -1778,6 +1782,28 @@ pub struct ClearFilterHistoryRequest {} #[ts(export, export_to = "gen_rpc.ts")] pub struct FilterHistoryChangedPayload {} +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct GetLastFilterRequest {} + +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct GetWindowPlacesRequest {} + +/// The window's places, opaque to the engine, kept in memory for a reload of the window. +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct SetWindowPlacesRequest { + pub places: String, +} + +/// The filter the window applied, kept in the encrypted database for the next launch. +#[derive(Deserialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +pub struct SetLastFilterRequest { + pub filter: String, +} + #[derive(Deserialize, TS)] #[ts(export, export_to = "gen_rpc.ts")] pub struct ClearFlowsRequest {} @@ -2809,6 +2835,27 @@ fn list_filter_history( filter_history::list(&ctx.db()?).map_err(rpc_error) } +fn get_window_places( + ctx: &ProxyCtx, + _req: GetWindowPlacesRequest, +) -> Result, RpcError> { + Ok(ctx.window_places.lock().unwrap_or_else(PoisonError::into_inner).clone()) +} + +fn set_window_places(ctx: &ProxyCtx, req: SetWindowPlacesRequest) -> Result { + *ctx.window_places.lock().unwrap_or_else(PoisonError::into_inner) = Some(req.places); + Ok(true) +} + +fn get_last_filter(ctx: &ProxyCtx, _req: GetLastFilterRequest) -> Result { + filter_history::last(&ctx.db()?).map_err(rpc_error) +} + +fn set_last_filter(ctx: &ProxyCtx, req: SetLastFilterRequest) -> Result { + filter_history::set_last(&ctx.db()?, &req.filter).map_err(rpc_error)?; + Ok(true) +} + fn record_filter(ctx: &ProxyCtx, req: RecordFilterRequest) -> Result { let recorded = filter_history::record(&ctx.db()?, &req.filter).map_err(rpc_error)?; if recorded { @@ -3722,6 +3769,10 @@ define_rpc! { record_filter(RecordFilterRequest) -> bool, delete_filter_history(DeleteFilterHistoryRequest) -> bool, clear_filter_history(ClearFilterHistoryRequest) -> usize, + get_last_filter(GetLastFilterRequest) -> String, + set_last_filter(SetLastFilterRequest) -> bool, + get_window_places(GetWindowPlacesRequest) -> Option, + set_window_places(SetWindowPlacesRequest) -> bool, list_grpc_schemas(ListGrpcSchemasRequest) -> Vec, list_messages(ListMessagesRequest) -> ListMessagesResponse, remove_grpc_schema(RemoveGrpcSchemaRequest) -> bool, @@ -4562,6 +4613,24 @@ mod tests { assert!(get_flows(ctx.clone(), req).await.unwrap().is_empty()); } + /// The window's places are held for a reload of it, in memory, and none at launch. + #[test] + fn window_places_are_held_in_memory_for_a_reload() { + let dir = tempfile::tempdir().unwrap(); + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + assert_eq!(get_window_places(&ctx, GetWindowPlacesRequest {}).unwrap(), None); + let places = r#"{"places":[{"filter":"stripe","selection":[],"tabs":{}}],"index":0}"#; + set_window_places(&ctx, SetWindowPlacesRequest { places: places.into() }).unwrap(); + let held = get_window_places(&ctx.clone(), GetWindowPlacesRequest {}).unwrap(); + assert_eq!(held.as_deref(), Some(places)); + drop(ctx); + + let (events, _rx) = RpcEventEmitter::new(); + let next = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + assert_eq!(get_window_places(&next, GetWindowPlacesRequest {}).unwrap(), None); + } + /// Filters applied are listed newest first, each once, and the app hears of each change. #[test] fn filter_history_commands_record_list_and_forget() { @@ -5191,6 +5260,7 @@ mod tests { flow_with_response(&ctx, "fl_b", "text/plain", b"two"); saved_queries::create(&ctx.db().unwrap(), "Errors", "@status:>=400").unwrap(); filter_history::record(&ctx.db().unwrap(), "@host:api.stripe.com").unwrap(); + filter_history::set_last(&ctx.db().unwrap(), "@status:>=400").unwrap(); assert_eq!(flows::count(&ctx.db().unwrap()).unwrap(), 2); } if let Some(settings) = settings { @@ -5217,6 +5287,7 @@ mod tests { history.iter().map(|h| h.filter.as_str()).collect::>(), ["@host:api.stripe.com"] ); + assert_eq!(filter_history::last(&db).unwrap(), "@status:>=400"); // New traffic is written as before. flow_with_response(&ctx, "fl_c", "text/plain", b"three"); assert_eq!(flows::count(&db).unwrap(), 1); From 7ece5fa5dfe4f1130d15a37df2adb1c7bbee2c57 Mon Sep 17 00:00:00 2001 From: Gregory Schier Date: Tue, 10 Nov 2026 01:14:45 -0700 Subject: [PATCH 30/45] Add a shared actions bar and a Pretty/Raw switch for Yaak Proxy's bodies The actions row moves out of the code editor into ActionsBar, a plain component a wrapper places over its content. FormatAction reformats an editor through its ref, and BodyViewModeSwitch names its first state for the body's format (Pretty, Table, Parts, Preview, Form) next to Raw. Yaak Proxy's request and response body tabs now share one viewer choice, with the switch in each tab's bar beside the text viewer's filter. The choice is remembered for both panes; bodies over the large size open raw. "Response (Raw)" leaves the proxy's tab menu, while Yaak's response pane keeps it until the remaining sites move. SegmentedControl moves from the client into packages/ui. Co-Authored-By: Claude Opus 5.5 --- .../components/HttpAuthenticationEditor.tsx | 3 +- .../components/ImportDataDialog.tsx | 11 +- .../yaak-client/components/MarkdownEditor.tsx | 2 +- .../components/Settings/SettingsAgents.tsx | 3 +- .../components/FlowBodyViewer.test.tsx | 80 +++++++++ apps/yaak-proxy/components/FlowBodyViewer.tsx | 170 ++++++++++++++++-- apps/yaak-proxy/components/FlowCard.tsx | 27 ++- .../components/FlowRequestBody.test.tsx | 15 +- .../yaak-proxy/components/FlowRequestBody.tsx | 15 +- apps/yaak-proxy/harness/fixtures.ts | 56 ++++++ apps/yaak-proxy/lib/atoms.ts | 10 +- packages/ui/src/components/ActionsBar.tsx | 36 ++++ .../ui/src/components/FormatAction.test.tsx | 62 +++++++ packages/ui/src/components/FormatAction.tsx | 49 +++++ .../ui/src/components}/SegmentedControl.tsx | 51 ++++-- .../http/BodyViewModeSwitch.test.tsx | 75 ++++++++ .../components/http/BodyViewModeSwitch.tsx | 77 ++++++++ .../components/http/ConfirmLargeResponse.tsx | 7 +- .../components/http/HttpResponsePaneView.tsx | 42 +++-- .../ui/src/components/viewers/TextViewer.tsx | 24 ++- packages/ui/src/index.ts | 13 +- 21 files changed, 729 insertions(+), 99 deletions(-) create mode 100644 apps/yaak-proxy/components/FlowBodyViewer.test.tsx create mode 100644 packages/ui/src/components/ActionsBar.tsx create mode 100644 packages/ui/src/components/FormatAction.test.tsx create mode 100644 packages/ui/src/components/FormatAction.tsx rename {apps/yaak-client/components/core => packages/ui/src/components}/SegmentedControl.tsx (73%) create mode 100644 packages/ui/src/components/http/BodyViewModeSwitch.test.tsx create mode 100644 packages/ui/src/components/http/BodyViewModeSwitch.tsx diff --git a/apps/yaak-client/components/HttpAuthenticationEditor.tsx b/apps/yaak-client/components/HttpAuthenticationEditor.tsx index 0da989bcc..159ceb16b 100644 --- a/apps/yaak-client/components/HttpAuthenticationEditor.tsx +++ b/apps/yaak-client/components/HttpAuthenticationEditor.tsx @@ -6,7 +6,7 @@ import type { Workspace, } from "@yaakapp-internal/models"; import { patchModel } from "@yaakapp-internal/models"; -import { HStack, Icon, InlineCode } from "@yaakapp-internal/ui"; +import { HStack, Icon, InlineCode, SegmentedControl } from "@yaakapp-internal/ui"; import { useCallback } from "react"; import { openFolderSettings } from "../commands/openFolderSettings"; import { openWorkspaceSettings } from "../commands/openWorkspaceSettings"; @@ -21,7 +21,6 @@ import { IconButton } from "./core/IconButton"; import { Input, type InputProps } from "./core/Input"; import { Link } from "./core/Link"; import { RadioDropdown } from "./core/RadioDropdown"; -import { SegmentedControl } from "./core/SegmentedControl"; import { DynamicForm } from "./DynamicForm"; import { EmptyStateText } from "./EmptyStateText"; diff --git a/apps/yaak-client/components/ImportDataDialog.tsx b/apps/yaak-client/components/ImportDataDialog.tsx index 43e698e06..50d1f6e9e 100644 --- a/apps/yaak-client/components/ImportDataDialog.tsx +++ b/apps/yaak-client/components/ImportDataDialog.tsx @@ -5,7 +5,15 @@ import { type ImportSource, type Workspace, } from "@yaakapp-internal/models"; -import { Banner, FormattedError, HStack, Icon, VStack, IconTooltip } from "@yaakapp-internal/ui"; +import { + Banner, + FormattedError, + HStack, + Icon, + VStack, + IconTooltip, + SegmentedControl, +} from "@yaakapp-internal/ui"; import classNames from "classnames"; import { type ComponentProps, useCallback, useEffect, useMemo, useRef, useState } from "react"; import { errorMessage } from "../lib/errorMessage"; @@ -22,7 +30,6 @@ import { CheckboxTree } from "./core/CheckboxTree"; import { Chip } from "./core/Chip"; import { DialogFooter } from "./core/Dialog"; import { Select } from "./core/Select"; -import { SegmentedControl } from "./core/SegmentedControl"; import { pluralize } from "@yaakapp-internal/lib/pluralize"; interface Props { diff --git a/apps/yaak-client/components/MarkdownEditor.tsx b/apps/yaak-client/components/MarkdownEditor.tsx index be31570f2..80f540c55 100644 --- a/apps/yaak-client/components/MarkdownEditor.tsx +++ b/apps/yaak-client/components/MarkdownEditor.tsx @@ -1,8 +1,8 @@ +import { SegmentedControl } from "@yaakapp-internal/ui"; import classNames from "classnames"; import { useRef, useState } from "react"; import type { EditorProps } from "./core/Editor/Editor"; import { Editor } from "./core/Editor/LazyEditor"; -import { SegmentedControl } from "./core/SegmentedControl"; import { Markdown } from "./Markdown"; type ViewMode = "edit" | "preview"; diff --git a/apps/yaak-client/components/Settings/SettingsAgents.tsx b/apps/yaak-client/components/Settings/SettingsAgents.tsx index 4ba04091c..bdc0f629f 100644 --- a/apps/yaak-client/components/Settings/SettingsAgents.tsx +++ b/apps/yaak-client/components/Settings/SettingsAgents.tsx @@ -1,7 +1,7 @@ import { useQuery, useQueryClient } from "@tanstack/react-query"; import { platform } from "@yaakapp-internal/platform"; import type { AgentSetupStatus, AppMetaData } from "@yaakapp-internal/rpc-schema"; -import { Banner, Heading, Icon, InlineCode, VStack } from "@yaakapp-internal/ui"; +import { Banner, Heading, Icon, InlineCode, SegmentedControl, VStack } from "@yaakapp-internal/ui"; import { useEffect } from "react"; import { useKeyValue } from "../../hooks/useKeyValue"; import { usePluginInfos } from "../../hooks/usePluginInfo"; @@ -14,7 +14,6 @@ import { import { appInfo } from "../../lib/appInfo"; import { rpc } from "../../lib/rpc"; import { Link } from "../core/Link"; -import { SegmentedControl } from "../core/SegmentedControl"; import { SettingCommand, SettingSnippet, diff --git a/apps/yaak-proxy/components/FlowBodyViewer.test.tsx b/apps/yaak-proxy/components/FlowBodyViewer.test.tsx new file mode 100644 index 000000000..f7b9f63da --- /dev/null +++ b/apps/yaak-proxy/components/FlowBodyViewer.test.tsx @@ -0,0 +1,80 @@ +import { describe, expect, test, vi } from "vite-plus/test"; +import { flowPrettyView, shownViewMode } from "./FlowBodyViewer"; + +// The package's index reaches for the window as it loads; the view rule is all this needs. +vi.mock("@yaakapp-internal/ui", async () => { + const { prettyViewFor } = await vi.importActual< + typeof import("../../../packages/ui/src/components/http/BodyViewModeSwitch") + >("../../../packages/ui/src/components/http/BodyViewModeSwitch"); + return { prettyViewFor, LARGE_BODY_BYTES: 2_000_000 }; +}); + +const text = (mime: string, language: string | null, which: "request" | "response" = "response") => + flowPrettyView({ + which, + mime, + text: "x", + binary: false, + language: language as never, + hasForm: false, + }); + +describe("flowPrettyView", () => { + test("names the formatted view per format, the same on both halves", () => { + for (const which of ["request", "response"] as const) { + expect(text("application/json", "json", which)).toBe("formatted"); + expect(text("text/csv", "text", which)).toBe("table"); + expect(text("image/svg+xml", "xml", which)).toBe("preview"); + } + expect(text("text/plain", "text")).toBeNull(); + }); + + test("HTML previews as a response and formats as a request", () => { + expect(text("text/html", "html", "response")).toBe("preview"); + expect(text("text/html", "html", "request")).toBe("formatted"); + }); + + test("multipart has parts whether or not it reads as text", () => { + const base = { which: "request" as const, mime: "multipart/form-data", language: null }; + expect(flowPrettyView({ ...base, text: null, binary: false, hasForm: false })).toBe("parts"); + }); + + test("a URL-encoded form's fields are its formatted view, where the tab has them", () => { + const base = { + which: "request" as const, + mime: "application/x-www-form-urlencoded", + text: "a=1", + binary: false, + language: "text" as const, + }; + expect(flowPrettyView({ ...base, hasForm: true })).toBe("form"); + expect(flowPrettyView({ ...base, hasForm: false })).toBeNull(); + }); + + test("images, binary and undecodable bodies have one view", () => { + const base = { which: "response" as const, language: null, hasForm: false }; + expect(flowPrettyView({ ...base, mime: "image/png", text: null, binary: true })).toBeNull(); + expect( + flowPrettyView({ ...base, mime: "application/json", text: null, binary: true }), + ).toBeNull(); + expect( + flowPrettyView({ ...base, mime: "application/octet-stream", text: null, binary: false }), + ).toBeNull(); + }); +}); + +describe("shownViewMode", () => { + test("follows the preference", () => { + expect(shownViewMode({ key: "a", large: false, preferred: "pretty", chosen: null })).toBe( + "pretty", + ); + expect(shownViewMode({ key: "a", large: false, preferred: "raw", chosen: null })).toBe("raw"); + }); + + test("a large body opens raw, and formats once asked, for that body alone", () => { + expect(shownViewMode({ key: "a", large: true, preferred: "pretty", chosen: null })).toBe("raw"); + const chosen = { key: "a", mode: "pretty" as const }; + expect(shownViewMode({ key: "a", large: true, preferred: "pretty", chosen })).toBe("pretty"); + expect(shownViewMode({ key: "b", large: true, preferred: "pretty", chosen })).toBe("raw"); + }); +}); diff --git a/apps/yaak-proxy/components/FlowBodyViewer.tsx b/apps/yaak-proxy/components/FlowBodyViewer.tsx index e63850af9..656eb3673 100644 --- a/apps/yaak-proxy/components/FlowBodyViewer.tsx +++ b/apps/yaak-proxy/components/FlowBodyViewer.tsx @@ -2,22 +2,32 @@ import { useQuery } from "@tanstack/react-query"; import { formatSize } from "@yaakapp-internal/lib/formatSize"; import type { Flow, FlowBody } from "@yaakapp-internal/proxy-lib"; import { + ACTIONS_BAR_CONTAINER, + ActionsBar, Banner, + BodyViewModeSwitch, Button, CsvViewer, HexdumpViewer, Icon, ImageViewer, + LARGE_BODY_BYTES, languageFromContentType, LoadingIcon, MultipartViewer, + type PrettyView, + prettyViewFor, + type ResponseViewMode, SvgViewer, TextViewer, WebPageViewer, } from "@yaakapp-internal/ui"; import type { CodeEditorProps } from "@yaakapp-internal/ui/editor"; +import classNames from "classnames"; +import { useAtom } from "jotai"; import type { ReactNode } from "react"; import { useCallback, useEffect, useMemo, useState } from "react"; +import { bodyViewModeAtom } from "../lib/atoms"; import { bodyAsText, bodyImageType, @@ -33,15 +43,97 @@ import { useSaveFlowBody } from "../lib/flowActions"; interface Props { flow: Flow; which: FlowBody; - pretty: boolean; + /** A URL-encoded body's fields, shown as its formatted view */ + form?: ReactNode; + /** Leaves saying the body was cut short to the tab, which says it its own way */ + hideCutNote?: boolean; } export function useFlowBody(flow: Flow, which: FlowBody, enabled = true) { return useQuery({ ...flowBodyQueryOptions(flow, which), enabled }); } -export function FlowBodyViewer({ flow, which, pretty }: Props) { +/** A mode chosen for one large body, which holds for it alone */ +interface ChosenViewMode { + key: string; + mode: ResponseViewMode; +} + +/** The mode a body shows in: the preference, except raw for a large one until chosen for it */ +export function shownViewMode({ + key, + large, + preferred, + chosen, +}: { + key: string; + large: boolean; + preferred: ResponseViewMode; + chosen: ChosenViewMode | null; +}): ResponseViewMode { + if (chosen?.key === key) return chosen.mode; + return large ? "raw" : preferred; +} + +/** + * Formatted or raw, as last chosen, except that a body big enough to slow the card opens raw, + * and choosing for it holds for that body alone + */ +export function useBodyViewMode( + key: string, + large: boolean, +): [ResponseViewMode, (mode: ResponseViewMode) => void] { + const [preferred, setPreferred] = useAtom(bodyViewModeAtom); + const [chosen, setChosen] = useState(null); + const mode = shownViewMode({ key, large, preferred, chosen }); + const setMode = useCallback( + (next: ResponseViewMode) => { + if (large) { + setChosen({ key, mode: next }); + } else { + setChosen(null); + setPreferred(next); + } + }, + [key, large, setPreferred], + ); + return [mode, setMode]; +} + +/** + * The formatted view a body has, or null when it has only the one: parts for multipart, the + * fields for a URL-encoded form, or else what the shared rule gives its text. Images and binary + * bodies have only their one view. + */ +export function flowPrettyView({ + which, + mime, + text, + binary, + language, + hasForm, +}: { + which: FlowBody; + mime: string; + text: string | null; + /** An image, or what failed to decode */ + binary: boolean; + language: CodeEditorProps["language"] | null; + hasForm: boolean; +}): PrettyView | null { + if (binary) return null; + if (mime.startsWith("multipart/")) return "parts"; + if (text == null) return null; + if (hasForm && mime === "application/x-www-form-urlencoded") return "form"; + return prettyViewFor({ mimeType: mime, language, previewHtml: which === "response" }); +} + +export function FlowBodyViewer({ flow, which, form, hideCutNote = false }: Props) { const body = useFlowBody(flow, which); + const [mode, setMode] = useBodyViewMode( + `${flow.id}.${which}`, + (body.data?.bytes.length ?? 0) > LARGE_BODY_BYTES, + ); if (body.isPending) { return ( @@ -69,12 +161,28 @@ export function FlowBodyViewer({ flow, which, pretty }: Props) { const imageType = decodeError == null ? bodyImageType(bytes, truncated, contentType) : null; const text = decodeError == null && imageType == null ? bodyAsText(bytes, truncated, contentType) : null; + const language = text == null ? null : languageFromContentType(contentType, text); + const prettyView = flowPrettyView({ + which, + mime, + text, + binary: decodeError != null || imageType != null, + language, + hasForm: form != null, + }); + // The one view a body without a formatted one has is its raw one + const pretty = prettyView != null && mode === "pretty"; + const modeSwitch = + prettyView == null ? null : ( + + ); let viewer: ReactNode; + let textViewer = false; if (decodeError != null) { // What failed to decode is as it was on the wire, compressed, so never text. viewer = ; - } else if (mime.startsWith("multipart/") && pretty) { + } else if (prettyView === "parts" && pretty) { const boundary = contentType?.match(/;\s*boundary="?([^";]+)"?/i)?.[1] ?? "unknown"; viewer = ( ; } else if (text == null) { viewer = ; - } else if (mime.startsWith("image/svg")) { - viewer = ; - } else if (/csv|tab-separated/.test(mime) && pretty) { + } else if (prettyView === "form" && pretty) { + viewer = form; + } else if (prettyView === "table" && pretty) { viewer = ; + } else if (prettyView === "preview" && pretty) { + viewer = mime.startsWith("image/svg") ? ( + + ) : ( + + ); } else { - const language = languageFromContentType(contentType, text); - if (language === "html" && pretty && which === "response") { - viewer = ; - } else { - viewer = ( - - ); - } + textViewer = true; + viewer = ( + + ); } - if (!truncated && decodeError == null) return viewer; + // The text viewer draws the switch in its own bar, beside its filter; the rest get a bar here + if (modeSwitch != null && !textViewer) { + viewer = ( +
+ {viewer} + {modeSwitch} +
+ ); + } + + if ((!truncated || hideCutNote) && decodeError == null) return viewer; return (
@@ -168,7 +295,16 @@ function FlowTextViewer({ text, language, pretty, -}: Props & { text: string; language: CodeEditorProps["language"] }) { + modeSwitch, +}: { + flow: Flow; + which: FlowBody; + text: string; + language: CodeEditorProps["language"]; + pretty: boolean; + /** Formatted or raw, in the viewer's bar after its filter; null for a body with one view */ + modeSwitch: ReactNode; +}) { const runFilter = useCallback( (expression: string) => filterText(flow.id, which, expression, language), [flow.id, which, language], @@ -183,6 +319,8 @@ function FlowTextViewer({ className="-mr-2 bg-surface" formatText={formatText} filterText={runFilter} + ownActionsBar + pinnedActions={modeSwitch} /> ); } diff --git a/apps/yaak-proxy/components/FlowCard.tsx b/apps/yaak-proxy/components/FlowCard.tsx index 6fd906c9c..78abde9bd 100644 --- a/apps/yaak-proxy/components/FlowCard.tsx +++ b/apps/yaak-proxy/components/FlowCard.tsx @@ -13,14 +13,13 @@ import { IconButton, parseCookieHeaders, ResponseCookies, - type ResponseViewMode, SplitLayout, Tooltip, } from "@yaakapp-internal/ui"; import classNames from "classnames"; import { useSetAtom } from "jotai"; -import { useCallback, useMemo, useState } from "react"; -import { BODY_TYPE_BINARY } from "@yaakapp-internal/ui/httpTypes"; +import { useCallback, useMemo } from "react"; +import { BODY_TYPE_FORM_URLENCODED } from "@yaakapp-internal/ui/httpTypes"; import { findHeader } from "../lib/body"; import { browserTabMemory, @@ -166,12 +165,15 @@ function FlowRequestHalf({ flow }: { flow: Flow }) { renderBody={(editor) => ( - ) : null + // The response's viewer choice, with the form's fields as a URL-encoded body's + // formatted view + viewer={ + } requestSize={flow.requestSize} /> @@ -199,7 +201,6 @@ function UrlLine({ flow }: { flow: Flow }) { } function FlowResponseHalf({ flow, className }: { flow: Flow; className: string }) { - const [viewMode, setViewMode] = useState("pretty"); // A stream's body is its messages, read by the Messages tab; reading it whole again on every // message would only hold copies of it const loadBody = flow.streamKind == null; @@ -268,16 +269,12 @@ function FlowResponseHalf({ flow, className }: { flow: Flow; className: string } body={loader} requestBody={null} emptyState={Waiting for a response} - viewMode={viewMode} - onViewModeChange={setViewMode} tabsStorageKey="proxy_response_tabs" activeTabKey={flow.id} defaultTab={defaultTab} onTabChange={onTabChange} extraTabs={extraTabs} - renderBodyViewer={() => ( - - )} + renderBodyViewer={() => } statusLineCenter={ flow.responseTruncated ? ( ({ storedSize: 7, }); -function render( - body: Parameters[0]["body"], - binaryViewer: string | null = null, -) { +function render(body: Parameters[0]["body"]) { return renderToStaticMarkup( - EDITOR
} - binaryViewer={binaryViewer == null ? null :
{binaryViewer}
} - requestSize={2048} - />, + EDITOR
} requestSize={2048} />, ); } @@ -49,9 +41,8 @@ describe("FlowRequestBody", () => { expect(html).toContain("LOADING"); }); - test("the editor once loaded, the proxy's viewer for a binary body", () => { + test("the viewer once loaded", () => { expect(render({ data: data(), isLoading: false, error: null })).toContain("EDITOR"); - expect(render({ data: data(), isLoading: false, error: null }, "HEXDUMP")).toContain("HEXDUMP"); }); test("a cut body says how much was kept", () => { diff --git a/apps/yaak-proxy/components/FlowRequestBody.tsx b/apps/yaak-proxy/components/FlowRequestBody.tsx index 162a0adcd..11d6cf69d 100644 --- a/apps/yaak-proxy/components/FlowRequestBody.tsx +++ b/apps/yaak-proxy/components/FlowRequestBody.tsx @@ -13,19 +13,17 @@ import type { FlowBodyData } from "../lib/flowViews"; interface Props { /** The request body as the engine's body RPC returned it */ body: HttpBodyResult; - /** The shared editor for the body's type */ - editor: ReactNode; - /** The proxy's own viewer, for a body that isn't text; null for one that is */ - binaryViewer: ReactNode | null; + /** The body in the viewer for its type, formatted or raw */ + viewer: ReactNode; /** Bytes of the request body on the wire */ requestSize: number; } /** - * The request half's Body tab: the body in its editor, or why it isn't there yet. A body that + * The request half's Body tab: the body in its viewer, or why it isn't there yet. A body that * failed to load says why, as the response's viewer does, rather than reading as no body. */ -export function FlowRequestBody({ body, editor, binaryViewer, requestSize }: Props) { +export function FlowRequestBody({ body, viewer, requestSize }: Props) { if (body.error != null) { return ( @@ -40,7 +38,6 @@ export function FlowRequestBody({ body, editor, binaryViewer, requestSize }: Pro ); } - if (binaryViewer != null) return binaryViewer; if (body.data?.truncated) { return (
@@ -51,9 +48,9 @@ export function FlowRequestBody({ body, editor, binaryViewer, requestSize }: Pro -
{editor}
+
{viewer}
); } - return editor; + return viewer; } diff --git a/apps/yaak-proxy/harness/fixtures.ts b/apps/yaak-proxy/harness/fixtures.ts index 7368f528c..0bca85476 100644 --- a/apps/yaak-proxy/harness/fixtures.ts +++ b/apps/yaak-proxy/harness/fixtures.ts @@ -75,6 +75,27 @@ const graphqlRequest = JSON.stringify({ operationName: "GetViewer", }); const formRequest = "grant_type=client_credentials&client_id=yaak-proxy&scope=read+write"; +const csvResponse = [ + "id,email,plan,created_at", + "1,ada@example.com,pro,2026-10-01", + "2,grace@example.com,team,2026-10-14", + "3,linus@example.com,free,2026-11-02", +].join("\n"); +const multipartBoundary = "yaakBoundary7MA4YWxk"; +const multipartRequest = [ + `--${multipartBoundary}`, + 'Content-Disposition: form-data; name="title"', + "", + "Quarterly report", + `--${multipartBoundary}`, + 'Content-Disposition: form-data; name="meta"; filename="meta.json"', + "Content-Type: application/json", + "", + '{"pages":12,"draft":false}', + `--${multipartBoundary}--`, + "", +].join("\r\n"); +const textResponse = "Uploaded 2 parts"; const htmlResponse = "

Signed in

You can close this window.

"; // A 24x24 PNG gradient @@ -198,6 +219,41 @@ export const fixtures: FlowFixture[] = [ requestBody: body(graphqlRequest), responseBody: body('{"data":{"viewer":{"login":"yaak"}}}'), }, + { + label: "CSV GET", + flow: { + ...base, + id: "fl_csv", + url: "https://api.example.com/users/export.csv", + contentType: "text/csv", + requestHeaders: [{ name: "Accept", value: "text/csv" }], + responseHeaders: [{ name: "Content-Type", value: "text/csv; charset=utf-8" }], + responseSize: csvResponse.length, + }, + requestBody: body(""), + responseBody: body(csvResponse), + }, + { + label: "Multipart POST", + flow: { + ...base, + id: "fl_multipart", + method: "POST", + url: "https://api.example.com/reports", + contentType: "text/plain", + requestHeaders: [ + { + name: "Content-Type", + value: `multipart/form-data; boundary=${multipartBoundary}`, + }, + ], + responseHeaders: [{ name: "Content-Type", value: "text/plain; charset=utf-8" }], + requestSize: multipartRequest.length, + responseSize: textResponse.length, + }, + requestBody: body(multipartRequest), + responseBody: body(textResponse), + }, { label: "gRPC call", flow: { diff --git a/apps/yaak-proxy/lib/atoms.ts b/apps/yaak-proxy/lib/atoms.ts index 291ddef95..e68132615 100644 --- a/apps/yaak-proxy/lib/atoms.ts +++ b/apps/yaak-proxy/lib/atoms.ts @@ -1,5 +1,5 @@ import type { Flow } from "@yaakapp-internal/proxy-lib"; -import type { SplitLayoutLayout } from "@yaakapp-internal/ui"; +import type { ResponseViewMode, SplitLayoutLayout } from "@yaakapp-internal/ui"; import { atom } from "jotai"; import { atomWithStorage } from "jotai/utils"; @@ -22,3 +22,11 @@ export const helperDialogOpenAtom = atom(false); /** Stacked by default; the header's toggle switches to side by side */ export const flowLayoutAtom = atomWithStorage("flow_layout", "vertical"); + +/** Bodies show formatted or raw, one choice for both halves of every flow */ +export const bodyViewModeAtom = atomWithStorage( + "body_view_mode", + "pretty", + undefined, + { getOnInit: true }, +); diff --git a/packages/ui/src/components/ActionsBar.tsx b/packages/ui/src/components/ActionsBar.tsx new file mode 100644 index 000000000..891d18395 --- /dev/null +++ b/packages/ui/src/components/ActionsBar.tsx @@ -0,0 +1,36 @@ +import classNames from "classnames"; +import type { ReactNode } from "react"; + +/** For the positioned element an `ActionsBar` sits in: hovering it brings the bar's actions in */ +export const ACTIONS_BAR_CONTAINER = "relative group/actions"; + +/** Set on an action's outermost element to keep it in view, as a control that shows state does */ +export const PINNED_ACTION = { "data-pinned": true } as const; + +interface Props { + children: ReactNode; + className?: string; +} + +/** + * A row of actions over the bottom-right corner of the content in an `ACTIONS_BAR_CONTAINER`: + * small bordered buttons on the surface colour, so they read over text. Each fades in while the + * container is hovered or the action has focus, unless it carries `PINNED_ACTION`. The row lets + * clicks through to the content between its actions. + */ +export function ActionsBar({ children, className }: Props) { + return ( +
:not([data-pinned])]:opacity-0", + "group-hover/actions:*:opacity-100! *:focus-within:opacity-100! *:hover:opacity-100!", + )} + > + {children} +
+ ); +} diff --git a/packages/ui/src/components/FormatAction.test.tsx b/packages/ui/src/components/FormatAction.test.tsx new file mode 100644 index 000000000..c58ff4c11 --- /dev/null +++ b/packages/ui/src/components/FormatAction.test.tsx @@ -0,0 +1,62 @@ +import { EditorState } from "@codemirror/state"; +import { renderToStaticMarkup } from "react-dom/server"; +import { describe, expect, test, vi } from "vite-plus/test"; +import { FormatAction, type FormattableEditor, formatEditor } from "./FormatAction"; + +/** An editor as far as formatting reaches into one, recording its changes and blurs */ +function fakeEditor(text: string) { + const changes: string[] = []; + const blur = vi.fn(); + let state = EditorState.create({ doc: text }); + const editor: FormattableEditor & { changes: string[]; blur: typeof blur } = { + get state() { + return state; + }, + dispatch: ((spec: { changes: { from: number; to: number; insert: string } }) => { + state = state.update(spec).state; + changes.push(state.doc.toString()); + }) as FormattableEditor["dispatch"], + contentDOM: { blur } as unknown as HTMLElement, + changes, + blur, + }; + return editor; +} + +describe("formatEditor", () => { + test("replaces the document with its formatted version and blurs", async () => { + const editor = fakeEditor('{"a":1}'); + await formatEditor(editor, async (t) => JSON.stringify(JSON.parse(t), null, 2)); + expect(editor.state.doc.toString()).toBe('{\n "a": 1\n}'); + expect(editor.changes).toHaveLength(1); + expect(editor.blur).toHaveBeenCalledOnce(); + }); + + test("leaves a document that is already formatted alone", async () => { + const editor = fakeEditor("same"); + await formatEditor(editor, async (t) => t); + expect(editor.changes).toHaveLength(0); + }); + + test("drops a result for text that changed while it was formatting", async () => { + const editor = fakeEditor("before"); + await formatEditor(editor, async () => { + editor.dispatch({ changes: { from: 0, to: 6, insert: "typed" } }); + return "formatted"; + }); + expect(editor.state.doc.toString()).toBe("typed"); + }); +}); + +describe("FormatAction", () => { + test("is offered for an editable editor only", () => { + const editor = { current: null }; + const format = async (t: string) => t; + expect(renderToStaticMarkup()).toContain( + "Reformat contents", + ); + expect(renderToStaticMarkup()).toBe( + "", + ); + }); +}); diff --git a/packages/ui/src/components/FormatAction.tsx b/packages/ui/src/components/FormatAction.tsx new file mode 100644 index 000000000..b79f44498 --- /dev/null +++ b/packages/ui/src/components/FormatAction.tsx @@ -0,0 +1,49 @@ +import type { EditorView } from "@codemirror/view"; +import type { RefObject } from "react"; +import { IconButton } from "./IconButton"; + +/** What reformatting needs of an editor; the `EditorView` an editor's `setRef` hands over has it */ +export type FormattableEditor = Pick; + +/** + * Replaces the editor's document with `format`'s version of it, and blurs it, since the cursor + * resets anyway. The change goes through the editor's own `onChange`, as typing does. + */ +export async function formatEditor( + editor: FormattableEditor, + format: (text: string) => Promise, +): Promise { + const before = editor.state.doc.toString(); + const formatted = await format(before); + // Typed into while formatting: the result is out of date + if (editor.state.doc.toString() !== before) return; + if (formatted === before) return; + editor.dispatch({ changes: { from: 0, to: editor.state.doc.length, insert: formatted } }); + editor.contentDOM.blur(); +} + +interface Props { + /** The editor to reformat, as its `setRef` set it */ + editor: RefObject; + format: (text: string) => Promise; + /** A read-only editor can't be reformatted, so offers nothing */ + readOnly?: boolean; +} + +/** Reformats an editable editor's contents, for an `ActionsBar` */ +export function FormatAction({ editor, format, readOnly }: Props) { + if (readOnly) return null; + return ( + { + if (editor.current == null) return; + await formatEditor(editor.current, format); + }} + /> + ); +} diff --git a/apps/yaak-client/components/core/SegmentedControl.tsx b/packages/ui/src/components/SegmentedControl.tsx similarity index 73% rename from apps/yaak-client/components/core/SegmentedControl.tsx rename to packages/ui/src/components/SegmentedControl.tsx index b2ab147f5..74092c9f2 100644 --- a/apps/yaak-client/components/core/SegmentedControl.tsx +++ b/packages/ui/src/components/SegmentedControl.tsx @@ -1,19 +1,33 @@ -import type { IconProps } from "@yaakapp-internal/ui"; -import { HStack, useStateWithDeps, IconTooltip, Label } from "@yaakapp-internal/ui"; +import { generateId } from "@yaakapp-internal/lib/generateId"; import classNames from "classnames"; import { type ReactNode, useRef } from "react"; -import { Button } from "./Button"; +import { useStateWithDeps } from "../hooks/useStateWithDeps"; +import { Button, CONTROL_TOOLTIP_DELAY } from "./Button"; +import type { IconProps } from "./Icon"; import { IconButton, type IconButtonProps } from "./IconButton"; -import { generateId } from "@yaakapp-internal/lib/generateId"; +import { IconTooltip } from "./IconTooltip"; +import { Label } from "./Label"; +import { HStack } from "./Stacks"; +import { Tooltip } from "./Tooltip"; -interface Props { - options: { value: T; label: string; icon?: IconProps["icon"] }[]; +export interface SegmentedControlOption { + value: T; + label: string; + icon?: IconProps["icon"]; + /** Shown on hover over a text option, leaving its label as its name */ + tooltip?: string; +} + +export interface SegmentedControlProps { + options: SegmentedControlOption[]; onChange: (value: T) => void; value: T; name: string; size?: IconButtonProps["size"]; label: string; className?: string; + /** Replaces the full width the control takes by default */ + containerClassName?: string; hideLabel?: boolean; labelClassName?: string; help?: ReactNode; @@ -29,7 +43,8 @@ export function SegmentedControl({ labelClassName, help, className, -}: Props) { + containerClassName = "w-full", +}: SegmentedControlProps) { const [selectedValue, setSelectedValue] = useStateWithDeps(value, [value]); const containerRef = useRef(null); const id = useRef(`input-${generateId()}`); @@ -38,8 +53,13 @@ export function SegmentedControl({ const inlineHelp = hideLabel && help ? : null; + const focusOption = (index: number) => { + const option = containerRef.current?.querySelectorAll('[role="radio"]')[index]; + option?.focus(); + }; + return ( -
+