Merge main into feat/request-versioning

Conflicts were all at registration points; both sides are kept.

Main now routes every write through one connection (#642): ClientDb only
reads and writes go through `with_tx` on a WriteDb. Versioning follows:
snapshot, restore, prune and the cascade delete move to WriteDb, while the
lookups and the "has it changed" check stay on ClientDb. Callers take a
write transaction the way their neighbours do:

- The HTTP send snapshots in its own transaction from
  `resolve_send_inputs`, and a failure there (including not getting the
  writer) is still logged and swallowed, so the response just has no
  version.
- gRPC and WebSocket connect snapshot inside the transaction that upserts
  the connection.
- The snapshot and restore RPCs, on the desktop and in the browser host,
  each run in a transaction.
- The request and workspace cascades live on main's WriteDb deletes.

The Request Changed dropdown moves into the response pane's
`statusLineRight` slot, since main moved the status line into the shared
pane view.

The plugin runtime's copy of the model bindings picks up `versionId`, and
`gen_rpc.ts` is regenerated.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Gregory SchierandClaude Opus 5.5 authored and Stetta task tk_mw563e1dcqtc4sqh committed 2026-11-06 09:23:32 -07:00
commit 1b0a196c50
794 files changed
+76953 -18061

No files matched your search

+2 -2
View File
@@ -8,7 +8,7 @@ publish = false
chrono = { version = "0.4.38", features = ["serde"] }
include_dir = "0.7"
log = { workspace = true }
nanoid = "0.4.0"
nanoid = "0.5.0"
rusqlite = { version = "0.38", features = ["bundled", "chrono"] }
sea-query-rusqlite = { version = "0.8.0", features = ["with-chrono"] }
sea-query = { version = "1.0", features = ["with-chrono", "attr"] }
@@ -24,5 +24,5 @@ r2d2_sqlite = { version = "0.32" }
# nanoid pulls getrandom, which needs to be told how to reach the browser's
# CSPRNG on wasm32-unknown-unknown. Native targets are unaffected.
[target.'cfg(target_arch = "wasm32")'.dependencies]
getrandom = { version = "0.2", features = ["js"] }
getrandom = { version = "0.3.4", features = ["wasm_js"] }
uuid = { version = "1", features = ["js"] }
+2 -1
View File
@@ -21,7 +21,8 @@
//! the second ask. Sharing the handle instead makes nested *reads* work the way
//! they do on the desktop; nested *write transactions* fail on both, only
//! differently (here SQLite refuses the inner `BEGIN`; natively the inner
//! connection blocks on `busy_timeout` and then fails).
//! call waits for the one writer connection, which the outer call holds, and
//! times out).
#[cfg(not(target_arch = "wasm32"))]
mod imp {
@@ -20,7 +20,9 @@ impl UpdateSource {
#[derive(Debug, Clone, Serialize, Deserialize, TS)]
#[serde(rename_all = "snake_case", tag = "type")]
pub enum ModelChangeEvent {
Upsert { created: bool },
Upsert {
created: bool,
},
/// A delete for a workspace implies deletion of every model in that
/// workspace — children are bulk-deleted without their own change rows or
/// events, and consumers must prune the subtree themselves (the frontend
+1
View File
@@ -445,6 +445,7 @@ export type Settings = {
updateChannel: string;
hideLicenseBadge: boolean;
promptFeedback: boolean;
mcpServerEnabled: boolean;
autoupdate: boolean;
autoDownloadUpdates: boolean;
checkNotifications: boolean;
+21 -6
View File
@@ -10,11 +10,24 @@ import type { BatchUpsertResult, ImportDestination, ImportPlan } from "./gen_uti
import type { Tokens } from "./parser";
import type { ServerSentEvent } from "./sse";
/**
* A coding agent the Agents settings pane can set up.
*/
export type AgentClient = "claude_code" | "codex" | "cursor" | "vscode" | "other";
/**
* Which setup steps an agent already has done on this machine. `None` means
* the pane cannot tell for that agent.
*/
export type AgentSetupStatus = { mcpServer: boolean | null, skill: boolean | null, };
/**
* What the frontend learns about the host it is talking to.
*/
export type AppMetaData = { isDev: boolean, version: string, cliVersion: string | null, name: string, appDataDir: string, appLogDir: string, vendoredPluginDir: string, defaultProjectDir: string, featureUpdater: boolean, featureLicense: boolean, };
export type CmdAgentSetupStatusReq = { agent: AgentClient, };
export type CmdCallFolderActionReq = { req: CallFolderActionRequest, };
export type CmdCallGrpcRequestActionReq = { req: CallGrpcRequestActionRequest, };
@@ -31,6 +44,8 @@ export type CmdCheckForUpdatesReq = Record<string, never>;
export type CmdCommitImportReq = { plan: ImportPlan, };
export type CmdCreateExampleWorkspaceReq = Record<string, never>;
export type CmdCurlToRequestReq = { command: string, workspaceId: string, };
export type CmdDecryptTemplateReq = { template: string, };
@@ -43,13 +58,15 @@ export type CmdDeleteAllHttpResponsesReq = { requestId: string, };
export type CmdDeleteSendHistoryReq = { workspaceId: string, };
export type CmdDetectImportSourceReq = { filePath?: string, url?: string, };
export type CmdDisableEncryptionReq = { workspaceId: string, };
export type CmdDismissNotificationReq = { notificationId: string, };
export type CmdEnableEncryptionReq = { workspaceId: string, };
export type CmdExportDataReq = { exportPath: string, workspaceIds: Array<string>, includePrivateEnvironments: boolean, };
export type CmdExportDataReq = { workspaceIds: Array<string>, includePrivateEnvironments: boolean, };
export type CmdFolderActionsReq = Record<string, never>;
@@ -143,12 +160,10 @@ export type CmdHttpResponseBodyPathReq = { responseId: string, };
export type CmdHttpResponseBodyReq = { responseId: string, filter: string | null, };
export type CmdImportDataReq = { filePath: string, destination: ImportDestination, };
export type CmdImportDataReq = { filePaths: Array<string>, urls: Array<string>, destination: ImportDestination, };
export type CmdImportSourcesForOriginReq = { filePath?: string, url?: string, };
export type CmdImportUrlReq = { url: string, destination: ImportDestination, };
export type CmdListImportSourcesReq = { workspaceId: string, };
export type CmdMetadataReq = Record<string, never>;
@@ -195,7 +210,7 @@ export type CmdSendHttpRequestReq = { environmentId: string | null, cookieJarId:
export type CmdSetWorkspaceKeyReq = { workspaceId: string, key: string, };
export type CmdSyncApplyReq = { syncOps: Array<SyncOp>, syncDir: string, workspaceId: string, };
export type CmdSyncApplyReq = { syncOps: Array<SyncOp>, syncDir: string, workspaceId: string, fsOnly: boolean, };
export type CmdSyncCalculateFsReq = { dir: string, };
@@ -260,6 +275,6 @@ export type ModelsWebsocketEventsReq = { connectionId: string, };
export type ModelsWorkspaceModelsReq = { workspaceId: string | null, };
export type RpcSchema = { cmd_metadata: [CmdMetadataReq, AppMetaData], cmd_template_tokens_to_string: [CmdTemplateTokensToStringReq, string], cmd_render_template: [CmdRenderTemplateReq, string], cmd_send_feedback: [CmdSendFeedbackReq, null], cmd_dismiss_notification: [CmdDismissNotificationReq, null], cmd_grpc_reflect: [CmdGrpcReflectReq, Array<ServiceDefinition>], cmd_grpc_go: [CmdGrpcGoReq, string], cmd_restart: [CmdRestartReq, null], cmd_send_ephemeral_request: [CmdSendEphemeralRequestReq, EphemeralHttpResponse], cmd_format_json: [CmdFormatJsonReq, string], cmd_format_graphql: [CmdFormatGraphqlReq, string], cmd_http_response_body: [CmdHttpResponseBodyReq, FilterResponse], cmd_http_response_body_path: [CmdHttpResponseBodyPathReq, string | null], cmd_http_request_body: [CmdHttpRequestBodyReq, Array<number> | null], cmd_get_sse_events: [CmdGetSseEventsReq, Array<ServerSentEvent>], cmd_get_http_response_events: [CmdGetHttpResponseEventsReq, Array<HttpResponseEvent>], cmd_import_data: [CmdImportDataReq, ImportPlan], cmd_import_url: [CmdImportUrlReq, ImportPlan], cmd_commit_import: [CmdCommitImportReq, BatchUpsertResult], cmd_list_import_sources: [CmdListImportSourcesReq, Array<ImportSource>], cmd_import_sources_for_origin: [CmdImportSourcesForOriginReq, Array<ImportSource>], cmd_http_request_actions: [CmdHttpRequestActionsReq, Array<GetHttpRequestActionsResponse>], cmd_websocket_request_actions: [CmdWebsocketRequestActionsReq, Array<GetWebsocketRequestActionsResponse>], cmd_call_websocket_request_action: [CmdCallWebsocketRequestActionReq, null], cmd_workspace_actions: [CmdWorkspaceActionsReq, Array<GetWorkspaceActionsResponse>], cmd_call_workspace_action: [CmdCallWorkspaceActionReq, null], cmd_folder_actions: [CmdFolderActionsReq, Array<GetFolderActionsResponse>], cmd_call_folder_action: [CmdCallFolderActionReq, null], cmd_grpc_request_actions: [CmdGrpcRequestActionsReq, Array<GetGrpcRequestActionsResponse>], cmd_template_function_summaries: [CmdTemplateFunctionSummariesReq, Array<GetTemplateFunctionSummaryResponse>], cmd_template_function_config: [CmdTemplateFunctionConfigReq, GetTemplateFunctionConfigResponse], cmd_get_http_authentication_summaries: [CmdGetHttpAuthenticationSummariesReq, Array<GetHttpAuthenticationSummaryResponse>], cmd_get_http_authentication_config: [CmdGetHttpAuthenticationConfigReq, GetHttpAuthenticationConfigResponse], cmd_call_http_request_action: [CmdCallHttpRequestActionReq, null], cmd_call_grpc_request_action: [CmdCallGrpcRequestActionReq, null], cmd_call_http_authentication_action: [CmdCallHttpAuthenticationActionReq, null], cmd_curl_to_request: [CmdCurlToRequestReq, HttpRequest], cmd_export_data: [CmdExportDataReq, null], cmd_save_base64_to_binary: [CmdSaveBase64ToBinaryReq, null], cmd_save_response: [CmdSaveResponseReq, null], cmd_send_http_request: [CmdSendHttpRequestReq, HttpResponse], cmd_reload_plugins: [CmdReloadPluginsReq, Array<[string, string]>], cmd_plugin_info: [CmdPluginInfoReq, PluginMetadata], cmd_delete_all_grpc_connections: [CmdDeleteAllGrpcConnectionsReq, null], cmd_delete_send_history: [CmdDeleteSendHistoryReq, null], cmd_delete_all_http_responses: [CmdDeleteAllHttpResponsesReq, null], cmd_get_workspace_meta: [CmdGetWorkspaceMetaReq, WorkspaceMeta], cmd_new_child_window: [CmdNewChildWindowReq, null], cmd_new_main_window: [CmdNewMainWindowReq, null], cmd_check_for_updates: [CmdCheckForUpdatesReq, boolean], cmd_decrypt_template: [CmdDecryptTemplateReq, string], cmd_secure_template: [CmdSecureTemplateReq, string], cmd_get_themes: [CmdGetThemesReq, Array<GetThemesResponse>], cmd_enable_encryption: [CmdEnableEncryptionReq, null], cmd_reveal_workspace_key: [CmdRevealWorkspaceKeyReq, string], cmd_set_workspace_key: [CmdSetWorkspaceKeyReq, null], cmd_disable_encryption: [CmdDisableEncryptionReq, null], cmd_default_headers: [CmdDefaultHeadersReq, Array<HttpRequestHeader>], models_upsert: [ModelsUpsertReq, string], models_delete: [ModelsDeleteReq, string], models_duplicate: [ModelsDuplicateReq, string], models_snapshot_request: [ModelsSnapshotRequestReq, ModelVersion], models_request_version: [ModelsRequestVersionReq, RequestVersionComparison], models_restore_request_version: [ModelsRestoreRequestVersionReq, string], models_websocket_events: [ModelsWebsocketEventsReq, Array<WebsocketEvent>], models_grpc_events: [ModelsGrpcEventsReq, Array<GrpcEvent>], models_get_settings: [ModelsGetSettingsReq, Settings], models_get_graphql_introspection: [ModelsGetGraphqlIntrospectionReq, GraphQlIntrospection | null], models_upsert_graphql_introspection: [ModelsUpsertGraphqlIntrospectionReq, GraphQlIntrospection], models_workspace_models: [ModelsWorkspaceModelsReq, string], cmd_git_checkout: [CmdGitCheckoutReq, string], cmd_git_branch: [CmdGitBranchReq, null], cmd_git_delete_branch: [CmdGitDeleteBranchReq, BranchDeleteResult], cmd_git_delete_remote_branch: [CmdGitDeleteRemoteBranchReq, null], cmd_git_merge_branch: [CmdGitMergeBranchReq, null], cmd_git_rename_branch: [CmdGitRenameBranchRLine truncated
export type RpcSchema = { cmd_metadata: [CmdMetadataReq, AppMetaData], cmd_agent_setup_status: [CmdAgentSetupStatusReq, AgentSetupStatus], cmd_template_tokens_to_string: [CmdTemplateTokensToStringReq, string], cmd_render_template: [CmdRenderTemplateReq, string], cmd_send_feedback: [CmdSendFeedbackReq, null], cmd_dismiss_notification: [CmdDismissNotificationReq, null], cmd_grpc_reflect: [CmdGrpcReflectReq, Array<ServiceDefinition>], cmd_grpc_go: [CmdGrpcGoReq, string], cmd_restart: [CmdRestartReq, null], cmd_send_ephemeral_request: [CmdSendEphemeralRequestReq, EphemeralHttpResponse], cmd_format_json: [CmdFormatJsonReq, string], cmd_format_graphql: [CmdFormatGraphqlReq, string], cmd_http_response_body: [CmdHttpResponseBodyReq, FilterResponse], cmd_http_response_body_path: [CmdHttpResponseBodyPathReq, string | null], cmd_http_request_body: [CmdHttpRequestBodyReq, Array<number> | null], cmd_get_sse_events: [CmdGetSseEventsReq, Array<ServerSentEvent>], cmd_get_http_response_events: [CmdGetHttpResponseEventsReq, Array<HttpResponseEvent>], cmd_import_data: [CmdImportDataReq, ImportPlan], cmd_commit_import: [CmdCommitImportReq, BatchUpsertResult], cmd_list_import_sources: [CmdListImportSourcesReq, Array<ImportSource>], cmd_import_sources_for_origin: [CmdImportSourcesForOriginReq, Array<ImportSource>], cmd_detect_import_source: [CmdDetectImportSourceReq, string], cmd_http_request_actions: [CmdHttpRequestActionsReq, Array<GetHttpRequestActionsResponse>], cmd_websocket_request_actions: [CmdWebsocketRequestActionsReq, Array<GetWebsocketRequestActionsResponse>], cmd_call_websocket_request_action: [CmdCallWebsocketRequestActionReq, null], cmd_workspace_actions: [CmdWorkspaceActionsReq, Array<GetWorkspaceActionsResponse>], cmd_call_workspace_action: [CmdCallWorkspaceActionReq, null], cmd_folder_actions: [CmdFolderActionsReq, Array<GetFolderActionsResponse>], cmd_call_folder_action: [CmdCallFolderActionReq, null], cmd_grpc_request_actions: [CmdGrpcRequestActionsReq, Array<GetGrpcRequestActionsResponse>], cmd_template_function_summaries: [CmdTemplateFunctionSummariesReq, Array<GetTemplateFunctionSummaryResponse>], cmd_template_function_config: [CmdTemplateFunctionConfigReq, GetTemplateFunctionConfigResponse], cmd_get_http_authentication_summaries: [CmdGetHttpAuthenticationSummariesReq, Array<GetHttpAuthenticationSummaryResponse>], cmd_get_http_authentication_config: [CmdGetHttpAuthenticationConfigReq, GetHttpAuthenticationConfigResponse], cmd_call_http_request_action: [CmdCallHttpRequestActionReq, null], cmd_call_grpc_request_action: [CmdCallGrpcRequestActionReq, null], cmd_call_http_authentication_action: [CmdCallHttpAuthenticationActionReq, null], cmd_curl_to_request: [CmdCurlToRequestReq, HttpRequest], cmd_export_data: [CmdExportDataReq, string], cmd_create_example_workspace: [CmdCreateExampleWorkspaceReq, BatchUpsertResult], cmd_save_base64_to_binary: [CmdSaveBase64ToBinaryReq, null], cmd_save_response: [CmdSaveResponseReq, null], cmd_send_http_request: [CmdSendHttpRequestReq, HttpResponse], cmd_reload_plugins: [CmdReloadPluginsReq, Array<[string, string]>], cmd_plugin_info: [CmdPluginInfoReq, PluginMetadata], cmd_delete_all_grpc_connections: [CmdDeleteAllGrpcConnectionsReq, null], cmd_delete_send_history: [CmdDeleteSendHistoryReq, null], cmd_delete_all_http_responses: [CmdDeleteAllHttpResponsesReq, null], cmd_get_workspace_meta: [CmdGetWorkspaceMetaReq, WorkspaceMeta], cmd_new_child_window: [CmdNewChildWindowReq, null], cmd_new_main_window: [CmdNewMainWindowReq, null], cmd_check_for_updates: [CmdCheckForUpdatesReq, boolean], cmd_decrypt_template: [CmdDecryptTemplateReq, string], cmd_secure_template: [CmdSecureTemplateReq, string], cmd_get_themes: [CmdGetThemesReq, Array<GetThemesResponse>], cmd_enable_encryption: [CmdEnableEncryptionReq, null], cmd_reveal_workspace_key: [CmdRevealWorkspaceKeyReq, string], cmd_set_workspace_key: [CmdSetWorkspaceKeyReq, null], cmd_disable_encryption: [CmdDisableEncryptionReq, null], cmd_default_headers: [CmdDefaultHeadersReq, Array<HttpRequestHeader>], models_upsert: [ModelsUpsertReq, string], models_delete: [ModelsDeleteReq, string], models_duplicate: [ModelsDuplicateReq, string], models_snapshot_request: [ModelsSnapshotRequestReq, ModelVersion], models_request_version: [ModelsRequestVersionReq, RequestVersionComparison], models_restore_request_version: [ModelsRestoreRequestVersionReq, string], models_websocket_events: [ModelsWebsocketEventsReq, Array<WebsocketEvent>], models_grpc_events: [ModelsGrpcEventsReq, Array<GrpcEvent>], models_get_settings: [ModelsGetSettingsReq, Settings], models_get_graphql_introspection: [ModelsGetGraphqlIntrospectionReq, GraphQlIntrospection | null], models_upsert_graphql_introspection: [ModelsUpsertGraphqlIntrospectionReq, GraphQlIntrospection], models_workspace_models: [ModelsWorkspaceModelsReq, string], cmd_git_checkout: [CmdGitCheckoutReq, string], cmd_git_branch: [CmdGitBranchReq, null], cmd_git_delete_branch: [CmdGitDeleteBranchReq, BranchDeleteResulLine truncated
export type WatchResult = { unlistenEvent: string, };
+6 -2
View File
@@ -1,5 +1,9 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type PluginMetadata = { version: string, name: string, displayName: string, description: string | null, homepageUrl: string | null, repositoryUrl: string | null, };
export type PluginMetadata = { version: string, name: string, displayName: string, description: string | null, homepageUrl: string | null, repositoryUrl: string | null,
/**
* Why the plugin should no longer be used, from `deprecated` in its package.json
*/
deprecated: string | null, };
export type PluginVersion = { id: string, version: string, url: string, description: string | null, name: string, displayName: string, homepageUrl: string | null, repositoryUrl: string | null, checksum: string, readme: string | null, yanked: boolean, };
export type PluginVersion = { id: string, version: string, url: string, description: string | null, name: string, displayName: string, homepageUrl: string | null, repositoryUrl: string | null, checksum: string, readme: string | null, yanked: boolean, deprecated: string | null, };
+4 -1
View File
@@ -3,4 +3,7 @@ import type { SyncModel, SyncState } from "./gen_models";
export type FsCandidate = { "type": "FsCandidate", model: SyncModel, relPath: string, checksum: string, };
export type SyncOp = { "type": "fsCreate", model: SyncModel, } | { "type": "fsUpdate", model: SyncModel, state: SyncState, } | { "type": "fsDelete", state: SyncState, fs: FsCandidate | null, } | { "type": "dbCreate", fs: FsCandidate, } | { "type": "dbUpdate", state: SyncState, fs: FsCandidate, } | { "type": "dbDelete", model: SyncModel, state: SyncState, } | { "type": "ignorePrivate", model: SyncModel, };
/**
* Outbound operations record the file candidate checksum, or `None` if it was absent.
*/
export type SyncOp = { "type": "fsCreate", model: SyncModel, fsChecksum: string | null, } | { "type": "fsUpdate", model: SyncModel, state: SyncState, fsChecksum: string | null, } | { "type": "fsDelete", state: SyncState, fs: FsCandidate | null, } | { "type": "dbCreate", fs: FsCandidate, } | { "type": "dbUpdate", model: SyncModel, state: SyncState, fs: FsCandidate, } | { "type": "dbDelete", model: SyncModel, state: SyncState, } | { "type": "ignorePrivate", model: SyncModel, };
+94 -23
View File
@@ -1,7 +1,21 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { Environment, Folder, GrpcRequest, HttpRequest, WebsocketRequest, Workspace } from "./gen_models";
import type {
Environment,
Folder,
GrpcRequest,
HttpRequest,
WebsocketRequest,
Workspace,
} from "./gen_models";
export type BatchUpsertResult = { workspaces: Array<Workspace>, environments: Array<Environment>, folders: Array<Folder>, httpRequests: Array<HttpRequest>, grpcRequests: Array<GrpcRequest>, websocketRequests: Array<WebsocketRequest>, };
export type BatchUpsertResult = {
workspaces: Array<Workspace>;
environments: Array<Environment>;
folders: Array<Folder>;
httpRequests: Array<HttpRequest>;
grpcRequests: Array<GrpcRequest>;
websocketRequests: Array<WebsocketRequest>;
};
export type ImportConflictResolution = "keep_mine" | "take_source";
@@ -11,38 +25,95 @@ export type ImportConflictResolution = "keep_mine" | "take_source";
* The destination workspace and optional folder IDs are captured in the plan so the preview describes
* the exact destination that confirmation will use.
*/
export type ImportDestination = { "type": "new_workspace" } | { "type": "existing_workspace", workspaceId: string, folderId?: string, };
export type ImportDestination =
| { type: "new_workspace" }
| { type: "existing_workspace"; workspaceId: string; folderId?: string };
/**
* Where an import's contents came from, used to link the committed workspace back to it.
*/
export type ImportOrigin = {
/**
* The absolute file path or URL the contents were read from.
*/
origin: string, label: string, };
export type ImportOrigin = {
/**
* The absolute file path or URL the contents were read from.
*/
origin: string;
label: string;
};
export type ImportPlan = { importer: string, destination: ImportDestination, resources: BatchUpsertResult, warnings: Array<ImportPlanWarning>,
/**
* Stable source key for every model in `resources`, keyed by its planned ID.
*/
sourceKeys: { [key in string]?: string },
/**
* One entry per plannable resource; commit applies only the selected ones.
*/
items: Array<ImportPlanItem>, origin?: ImportOrigin, };
export type ImportPlan = {
importer: string;
destination: ImportDestination;
resources: BatchUpsertResult;
warnings: Array<ImportPlanWarning>;
/**
* Stable source key for every model in `resources`, keyed by its planned ID.
*/
sourceKeys: { [key in string]?: string };
/**
* One entry per plannable resource; commit applies only the selected ones.
*/
items: Array<ImportPlanItem>;
origin?: ImportOrigin;
/**
* Individual origins in a combined import, each retaining its own reimport history.
*/
sources: Array<ImportPlanSource>;
};
export type ImportPlanAction = "create" | "update" | "delete" | "unchanged" | "keep_local" | "conflict";
export type ImportPlanAction =
| "create"
| "update"
| "delete"
| "unchanged"
| "keep_local"
| "conflict"
| "ignored";
export type ImportPlanItem = {
action: ImportPlanAction;
model: ImportResourceType;
modelId: string;
name: string;
/**
* Planned parent folder ID for incoming resources; current parent for deletions.
*/
parentId?: string;
selected: boolean;
resolution?: ImportConflictResolution;
reason?: ImportPlanReason;
/**
* Fields where the source and the local copy disagree, so the preview can say why
*/
changedFields: Array<string>;
};
export type ImportPlanItem = { action: ImportPlanAction, model: ImportResourceType, modelId: string, name: string,
/**
* Planned parent folder ID for incoming resources; current parent for deletions.
* Extra context for an action that would otherwise be indistinguishable from its plain form.
*/
parentId?: string, selected: boolean, resolution?: ImportConflictResolution, };
export type ImportPlanReason = "moved_into_ignored_folder";
export type ImportPlanWarning = { title: string, detail: string, };
export type ImportPlanSource = {
importer: string;
origin: ImportOrigin;
workspaceId: string;
sourceKeys: { [key in string]?: string };
linkedSourceId?: string;
};
export type ImportPlanWarning = { title: string; detail: string; level: ImportPlanWarningLevel };
/**
* Whether a plan's note is something to know or something to think twice about.
*/
export type ImportPlanWarningLevel = "info" | "warning";
/**
* The model types an import plan can contain.
*/
export type ImportResourceType = "environment" | "folder" | "grpc_request" | "http_request" | "websocket_request" | "workspace";
export type ImportResourceType =
| "environment"
| "folder"
| "grpc_request"
| "http_request"
| "websocket_request"
| "workspace";
+56 -18
View File
@@ -29,11 +29,10 @@ use yaak_plugins::api::{PluginNameVersion, PluginSearchResponse, PluginUpdatesRe
use yaak_plugins::events::{
CallFolderActionRequest, CallGrpcRequestActionRequest, CallHttpRequestActionRequest,
CallWebsocketRequestActionRequest, CallWorkspaceActionRequest, FilterResponse,
GetFolderActionsResponse, GetGrpcRequestActionsResponse,
GetHttpAuthenticationConfigResponse, GetHttpAuthenticationSummaryResponse,
GetHttpRequestActionsResponse, GetTemplateFunctionConfigResponse,
GetTemplateFunctionSummaryResponse, GetThemesResponse, GetWebsocketRequestActionsResponse,
GetWorkspaceActionsResponse, JsonPrimitive, RenderPurpose,
GetFolderActionsResponse, GetGrpcRequestActionsResponse, GetHttpAuthenticationConfigResponse,
GetHttpAuthenticationSummaryResponse, GetHttpRequestActionsResponse,
GetTemplateFunctionConfigResponse, GetTemplateFunctionSummaryResponse, GetThemesResponse,
GetWebsocketRequestActionsResponse, GetWorkspaceActionsResponse, JsonPrimitive, RenderPurpose,
};
use yaak_plugins::plugin_meta::PluginMetadata;
use yaak_sse::sse::ServerSentEvent;
@@ -98,6 +97,35 @@ pub struct CmdSyncWatchReq {
#[ts(export, export_to = "gen_rpc.ts")]
pub struct CmdMetadataReq {}
/// A coding agent the Agents settings pane can set up.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, TS)]
#[serde(rename_all = "snake_case")]
#[ts(export, export_to = "gen_rpc.ts")]
pub enum AgentClient {
ClaudeCode,
Codex,
Cursor,
Vscode,
Other,
}
#[derive(Debug, Deserialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export, export_to = "gen_rpc.ts")]
pub struct CmdAgentSetupStatusReq {
pub agent: AgentClient,
}
/// Which setup steps an agent already has done on this machine. `None` means
/// the pane cannot tell for that agent.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export, export_to = "gen_rpc.ts")]
pub struct AgentSetupStatus {
pub mcp_server: Option<bool>,
pub skill: Option<bool>,
}
#[derive(Debug, Deserialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export, export_to = "gen_rpc.ts")]
@@ -229,15 +257,8 @@ pub struct CmdGetHttpResponseEventsReq {
#[serde(rename_all = "camelCase")]
#[ts(export, export_to = "gen_rpc.ts")]
pub struct CmdImportDataReq {
pub file_path: String,
pub destination: ImportDestination,
}
#[derive(Debug, Deserialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export, export_to = "gen_rpc.ts")]
pub struct CmdImportUrlReq {
pub url: String,
pub file_paths: Vec<String>,
pub urls: Vec<String>,
pub destination: ImportDestination,
}
@@ -265,6 +286,16 @@ pub struct CmdImportSourcesForOriginReq {
pub url: Option<String>,
}
#[derive(Debug, Deserialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export, export_to = "gen_rpc.ts")]
pub struct CmdDetectImportSourceReq {
#[ts(optional)]
pub file_path: Option<String>,
#[ts(optional)]
pub url: Option<String>,
}
#[derive(Debug, Deserialize, TS)]
#[ts(export, export_to = "gen_rpc.ts")]
pub struct CmdHttpRequestActionsReq {}
@@ -367,11 +398,14 @@ pub struct CmdCurlToRequestReq {
pub workspace_id: String,
}
#[derive(Debug, Deserialize, TS)]
#[ts(export, export_to = "gen_rpc.ts")]
pub struct CmdCreateExampleWorkspaceReq {}
#[derive(Debug, Deserialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export, export_to = "gen_rpc.ts")]
pub struct CmdExportDataReq {
pub export_path: String,
pub workspace_ids: Vec<String>,
pub include_private_environments: bool,
}
@@ -851,6 +885,8 @@ pub struct CmdSyncApplyReq {
pub sync_ops: Vec<SyncOp>,
pub sync_dir: PathBuf,
pub workspace_id: String,
#[serde(default)]
pub fs_only: bool,
}
#[derive(Debug, Deserialize, TS)]
@@ -943,6 +979,7 @@ macro_rules! with_commands {
($callback:ident) => {
$callback! {
cmd_metadata(CmdMetadataReq) -> AppMetaData,
cmd_agent_setup_status(CmdAgentSetupStatusReq) -> AgentSetupStatus,
cmd_template_tokens_to_string(CmdTemplateTokensToStringReq) -> String,
cmd_render_template(CmdRenderTemplateReq) -> String,
cmd_send_feedback(CmdSendFeedbackReq) -> (),
@@ -959,10 +996,10 @@ macro_rules! with_commands {
cmd_get_sse_events(CmdGetSseEventsReq) -> Vec<ServerSentEvent>,
cmd_get_http_response_events(CmdGetHttpResponseEventsReq) -> Vec<HttpResponseEvent>,
cmd_import_data(CmdImportDataReq) -> ImportPlan,
cmd_import_url(CmdImportUrlReq) -> ImportPlan,
cmd_commit_import(CmdCommitImportReq) -> BatchUpsertResult,
cmd_list_import_sources(CmdListImportSourcesReq) -> Vec<ImportSource>,
cmd_import_sources_for_origin(CmdImportSourcesForOriginReq) -> Vec<ImportSource>,
cmd_detect_import_source(CmdDetectImportSourceReq) -> String,
cmd_http_request_actions(CmdHttpRequestActionsReq) -> Vec<GetHttpRequestActionsResponse>,
cmd_websocket_request_actions(CmdWebsocketRequestActionsReq) -> Vec<GetWebsocketRequestActionsResponse>,
cmd_call_websocket_request_action(CmdCallWebsocketRequestActionReq) -> (),
@@ -979,7 +1016,8 @@ macro_rules! with_commands {
cmd_call_grpc_request_action(CmdCallGrpcRequestActionReq) -> (),
cmd_call_http_authentication_action(CmdCallHttpAuthenticationActionReq) -> (),
cmd_curl_to_request(CmdCurlToRequestReq) -> HttpRequest,
cmd_export_data(CmdExportDataReq) -> (),
cmd_export_data(CmdExportDataReq) -> String,
cmd_create_example_workspace(CmdCreateExampleWorkspaceReq) -> BatchUpsertResult,
cmd_save_base64_to_binary(CmdSaveBase64ToBinaryReq) -> (),
cmd_save_response(CmdSaveResponseReq) -> (),
cmd_send_http_request(CmdSendHttpRequestReq) -> HttpResponse,
@@ -1043,7 +1081,7 @@ macro_rules! with_commands {
cmd_git_rm_remote(CmdGitRmRemoteReq) -> (),
cmd_sync_calculate(CmdSyncCalculateReq) -> Vec<SyncOp>,
cmd_sync_calculate_fs(CmdSyncCalculateFsReq) -> Vec<SyncOp>,
cmd_sync_apply(CmdSyncApplyReq) -> (),
cmd_sync_apply(CmdSyncApplyReq) -> bool,
cmd_ws_delete_connections(CmdWsDeleteConnectionsReq) -> (),
cmd_ws_send(CmdWsSendReq) -> WebsocketConnection,
cmd_ws_close(CmdWsCloseReq) -> WebsocketConnection,
+18 -5
View File
@@ -13,8 +13,9 @@ pub type BoxFuture<T> = Pin<Box<dyn Future<Output = T> + Send + 'static>>;
/// `dispatch` call frame, so it cannot borrow, and contexts are cheap clones
/// (handles and `Arc`s). Synchronous handlers wrap into this via `rpc_handler!`
/// with no visible change.
type HandlerFn<Ctx> =
Box<dyn Fn(Ctx, serde_json::Value) -> BoxFuture<Result<serde_json::Value, RpcError>> + Send + Sync>;
type HandlerFn<Ctx> = Box<
dyn Fn(Ctx, serde_json::Value) -> BoxFuture<Result<serde_json::Value, RpcError>> + Send + Sync,
>;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RpcError {
@@ -138,6 +139,9 @@ impl RpcEventEmitter {
/// - `RpcSchema` — a struct with ts-rs derives for TypeScript type generation
/// - `RpcEventSchema` — (if events declared) a struct mapping event names to payload types
///
/// Commands under the optional `async_commands` block are `async fn(Ctx, Req)` handlers,
/// wrapped with `rpc_handler_async!`.
///
/// The wire name for each command/event is derived from `stringify!($ident)`.
///
/// # Example
@@ -161,6 +165,9 @@ macro_rules! define_rpc {
commands {
$( $handler:ident ( $req:ty ) -> $res:ty ),* $(,)?
}
$( async_commands {
$( $async_handler:ident ( $async_req:ty ) -> $async_res:ty ),* $(,)?
} )?
events {
$( $evt_ident:ident ( $evt_payload:ty ) ),* $(,)?
}
@@ -168,6 +175,7 @@ macro_rules! define_rpc {
pub fn build_router() -> $crate::RpcRouter<$ctx> {
let mut router = $crate::RpcRouter::new();
$( router.register(stringify!($handler), $crate::rpc_handler!($handler)); )*
$( $( router.register(stringify!($async_handler), $crate::rpc_handler_async!($async_handler)); )* )?
router
}
@@ -175,6 +183,7 @@ macro_rules! define_rpc {
#[ts(export, export_to = "gen_rpc.ts")]
pub struct RpcSchema {
$( pub $handler: ($req, $res), )*
$( $( pub $async_handler: ($async_req, $async_res), )* )?
}
#[derive(ts_rs::TS)]
@@ -190,10 +199,14 @@ macro_rules! define_rpc {
commands {
$( $handler:ident ( $req:ty ) -> $res:ty ),* $(,)?
}
$( async_commands {
$( $async_handler:ident ( $async_req:ty ) -> $async_res:ty ),* $(,)?
} )?
) => {
pub fn build_router() -> $crate::RpcRouter<$ctx> {
let mut router = $crate::RpcRouter::new();
$( router.register(stringify!($handler), $crate::rpc_handler!($handler)); )*
$( $( router.register(stringify!($async_handler), $crate::rpc_handler_async!($async_handler)); )* )?
router
}
@@ -201,6 +214,7 @@ macro_rules! define_rpc {
#[ts(export, export_to = "gen_rpc.ts")]
pub struct RpcSchema {
$( pub $handler: ($req, $res), )*
$( $( pub $async_handler: ($async_req, $async_res), )* )?
}
};
}
@@ -249,9 +263,8 @@ macro_rules! rpc_handler_async {
Box::new(|ctx, payload| {
Box::pin(async move {
let req = serde_json::from_value(payload).map_err($crate::RpcError::from)?;
let res = $f(ctx, req)
.await
.map_err(|e| $crate::RpcError { message: e.to_string() })?;
let res =
$f(ctx, req).await.map_err(|e| $crate::RpcError { message: e.to_string() })?;
serde_json::to_value(res).map_err($crate::RpcError::from)
})
})
+317
View File
@@ -0,0 +1,317 @@
//! Which of the Agents pane's setup steps are already done on this machine.
//!
//! Detection reads the files the agents keep their MCP servers in and the
//! skills directory `yaak agent install` writes to. A missing or unparsable
//! file is "not detected", never an error: the pane shows a done mark when it
//! can, and the command when it cannot.
use serde_json::Value;
use std::path::{Path, PathBuf};
use yaak_rpc_schema::{AgentClient, AgentSetupStatus};
const SKILL_NAME: &str = "use-yaak";
const SERVER_NAME: &str = "yaak";
pub fn agent_setup_status(home: &Path, agent: AgentClient) -> AgentSetupStatus {
AgentSetupStatus {
mcp_server: mcp_server_added(home, agent),
skill: skill_installed(home, agent),
}
}
fn skill_installed(home: &Path, agent: AgentClient) -> Option<bool> {
let skills_dir = match agent {
AgentClient::ClaudeCode => home.join(".claude").join("skills"),
AgentClient::Codex => home.join(".codex").join("skills"),
AgentClient::Cursor => home.join(".cursor").join("skills"),
AgentClient::Vscode | AgentClient::Other => home.join(".agents").join("skills"),
};
Some(skills_dir.join(SKILL_NAME).is_dir())
}
fn mcp_server_added(home: &Path, agent: AgentClient) -> Option<bool> {
match agent {
AgentClient::ClaudeCode => Some(claude_code_has_server(&home.join(".claude.json"))),
AgentClient::Codex => Some(codex_has_server(&home.join(".codex").join("config.toml"))),
AgentClient::Cursor => {
Some(json_has_server(&home.join(".cursor").join("mcp.json"), "mcpServers"))
}
AgentClient::Vscode => Some(json_has_server(&vscode_user_mcp_json(home), "servers")),
AgentClient::Other => None,
}
}
fn vscode_user_mcp_json(home: &Path) -> PathBuf {
let user_dir = if cfg!(target_os = "macos") {
home.join("Library").join("Application Support").join("Code").join("User")
} else if cfg!(target_os = "windows") {
std::env::var_os("APPDATA")
.map(PathBuf::from)
.unwrap_or_else(|| home.join("AppData").join("Roaming"))
.join("Code")
.join("User")
} else {
home.join(".config").join("Code").join("User")
};
user_dir.join("mcp.json")
}
fn read_json(path: &Path) -> Option<Value> {
let text = std::fs::read_to_string(path).ok()?;
serde_json::from_str(&text).ok()
}
/// Only the CLI's own entry counts. An entry named `yaak` that points at a
/// URL is the deprecated plugin's server and still needs replacing.
fn has_server(servers: Option<&Value>) -> bool {
servers.and_then(|s| s.get(SERVER_NAME)).is_some_and(is_cli_server)
}
fn is_cli_server(entry: &Value) -> bool {
let command = entry.get("command").and_then(Value::as_str);
let args: Vec<&str> = entry
.get("args")
.and_then(Value::as_array)
.map(|a| a.iter().filter_map(Value::as_str).collect())
.unwrap_or_default();
is_cli_launch(command, &args)
}
/// `yaak mcp`, or `npx -y @yaakapp/cli mcp`, which runs the same binary.
fn is_cli_launch(command: Option<&str>, args: &[&str]) -> bool {
let has_mcp = args.contains(&"mcp");
let via_npx = command_stem(command) == Some("npx")
&& args.iter().any(|a| a.trim_start_matches("@yaakapp/cli").len() < a.len());
has_mcp && (command_stem(command) == Some("yaak") || via_npx)
}
fn command_stem(command: Option<&str>) -> Option<&str> {
command.and_then(|c| Path::new(c).file_stem().and_then(|s| s.to_str()))
}
fn json_has_server(path: &Path, key: &str) -> bool {
read_json(path).is_some_and(|root| has_server(root.get(key)))
}
/// `claude mcp add` writes to the top level with `-s user` and under the
/// current project's entry without it, so both count.
fn claude_code_has_server(path: &Path) -> bool {
let Some(root) = read_json(path) else {
return false;
};
if has_server(root.get("mcpServers")) {
return true;
}
root.get("projects")
.and_then(|p| p.as_object())
.is_some_and(|projects| projects.values().any(|p| has_server(p.get("mcpServers"))))
}
/// A `[mcp_servers.yaak]` table, key bare or quoted, whose `command` is yaak
/// and whose `args` include mcp. The file is scanned line by line, not parsed.
fn codex_has_server(path: &Path) -> bool {
let Ok(text) = std::fs::read_to_string(path) else {
return false;
};
let mut in_table = false;
let mut command = None;
let mut args = Vec::new();
let mut in_args = false;
for line in text.lines().map(str::trim) {
if line.starts_with('[') {
in_table = is_codex_server_header(line);
in_args = false;
} else if in_table {
let rest = if in_args {
Some(line)
} else if let Some((key, value)) = line.split_once('=') {
match key.trim() {
"command" => {
command = Some(value.trim().trim_matches(QUOTES).to_string());
None
}
"args" => Some(value),
_other => None,
}
} else {
None
};
if let Some(rest) = rest {
args.extend(
rest.split(',')
.map(|a| a.trim().trim_matches(QUOTES_OR_BRACKETS))
.filter(|a| !a.is_empty())
.map(str::to_string),
);
in_args = !rest.contains(']');
}
}
}
let args: Vec<&str> = args.iter().map(String::as_str).collect();
is_cli_launch(command.as_deref(), &args)
}
const QUOTES: [char; 2] = ['"', '\''];
const QUOTES_OR_BRACKETS: [char; 4] = ['"', '\'', '[', ']'];
fn is_codex_server_header(line: &str) -> bool {
let Some(rest) = line.strip_prefix('[') else {
return false;
};
let Some(inner) = rest.split(']').next() else {
return false;
};
let mut parts = inner.split('.').map(|p| p.trim().trim_matches(|c| c == '"' || c == '\''));
parts.next() == Some("mcp_servers") && parts.next() == Some(SERVER_NAME)
}
#[cfg(test)]
mod tests {
use super::*;
use std::fs;
fn write(path: &Path, contents: &str) {
fs::create_dir_all(path.parent().unwrap()).unwrap();
fs::write(path, contents).unwrap();
}
#[test]
fn nothing_installed() {
let home = tempfile::tempdir().unwrap();
for agent in [
AgentClient::ClaudeCode,
AgentClient::Codex,
AgentClient::Cursor,
AgentClient::Vscode,
] {
let status = agent_setup_status(home.path(), agent);
assert_eq!(status.mcp_server, Some(false), "{agent:?}");
assert_eq!(status.skill, Some(false), "{agent:?}");
}
let other = agent_setup_status(home.path(), AgentClient::Other);
assert_eq!(other.mcp_server, None);
assert_eq!(other.skill, Some(false));
}
#[test]
fn skill_directories_per_agent() {
let home = tempfile::tempdir().unwrap();
fs::create_dir_all(home.path().join(".claude/skills/use-yaak")).unwrap();
fs::create_dir_all(home.path().join(".agents/skills/use-yaak")).unwrap();
assert_eq!(agent_setup_status(home.path(), AgentClient::ClaudeCode).skill, Some(true));
assert_eq!(agent_setup_status(home.path(), AgentClient::Codex).skill, Some(false));
assert_eq!(agent_setup_status(home.path(), AgentClient::Cursor).skill, Some(false));
assert_eq!(agent_setup_status(home.path(), AgentClient::Vscode).skill, Some(true));
assert_eq!(agent_setup_status(home.path(), AgentClient::Other).skill, Some(true));
}
#[test]
fn claude_code_top_level_or_project_scope() {
let home = tempfile::tempdir().unwrap();
let path = home.path().join(".claude.json");
write(&path, r#"{"mcpServers":{"yaak":{"command":"yaak","args":["mcp"]}}}"#);
assert_eq!(agent_setup_status(home.path(), AgentClient::ClaudeCode).mcp_server, Some(true));
write(
&path,
r#"{"projects":{"/Users/me/app":{"mcpServers":{"yaak":{"command":"/usr/local/bin/yaak","args":["mcp"]}}}}}"#,
);
assert_eq!(agent_setup_status(home.path(), AgentClient::ClaudeCode).mcp_server, Some(true));
write(
&path,
r#"{"mcpServers":{"yaak":{"type":"http","url":"http://127.0.0.1:64343/mcp"}}}"#,
);
assert_eq!(
agent_setup_status(home.path(), AgentClient::ClaudeCode).mcp_server,
Some(false)
);
write(&path, r#"{"mcpServers":{"other":{}},"projects":{"/x":{"mcpServers":{}}}}"#);
assert_eq!(
agent_setup_status(home.path(), AgentClient::ClaudeCode).mcp_server,
Some(false)
);
write(&path, "{ not json");
assert_eq!(
agent_setup_status(home.path(), AgentClient::ClaudeCode).mcp_server,
Some(false)
);
}
#[test]
fn codex_table_header() {
let home = tempfile::tempdir().unwrap();
let path = home.path().join(".codex/config.toml");
write(
&path,
"model = \"o3\"\n\n[mcp_servers.yaak]\ncommand = \"yaak\"\nargs = [\"mcp\"]\n",
);
assert_eq!(agent_setup_status(home.path(), AgentClient::Codex).mcp_server, Some(true));
write(&path, " [ mcp_servers . \"yaak\" ]\ncommand = 'yaak'\nargs = ['mcp']\n[other]\n");
assert_eq!(agent_setup_status(home.path(), AgentClient::Codex).mcp_server, Some(true));
write(&path, "[mcp_servers.yaak]\nurl = \"http://127.0.0.1:64343/mcp\"\n");
assert_eq!(agent_setup_status(home.path(), AgentClient::Codex).mcp_server, Some(false));
write(&path, "[mcp_servers.yaak]\ncommand = \"yaak\"\nargs = [\n \"mcp\",\n]\n");
assert_eq!(agent_setup_status(home.path(), AgentClient::Codex).mcp_server, Some(true));
write(&path, "[mcp_servers.yaak.env]\nFOO = \"bar\"\n");
assert_eq!(agent_setup_status(home.path(), AgentClient::Codex).mcp_server, Some(false));
write(
&path,
"[mcp_servers.yaak]\n[mcp_servers.other]\ncommand = \"yaak\"\nargs = [\"mcp\"]\n",
);
assert_eq!(agent_setup_status(home.path(), AgentClient::Codex).mcp_server, Some(false));
write(&path, "[mcp_servers.yaakish]\ncommand = \"x\"\n[mcp_servers]\n");
assert_eq!(agent_setup_status(home.path(), AgentClient::Codex).mcp_server, Some(false));
}
#[test]
fn cursor_and_vscode_json() {
let home = tempfile::tempdir().unwrap();
write(
&home.path().join(".cursor/mcp.json"),
r#"{"mcpServers":{"yaak":{"command":"yaak","args":["mcp"]}}}"#,
);
assert_eq!(agent_setup_status(home.path(), AgentClient::Cursor).mcp_server, Some(true));
write(
&home.path().join(".cursor/mcp.json"),
r#"{"mcpServers":{"yaak":{"url":"http://127.0.0.1:64343/mcp"}}}"#,
);
assert_eq!(agent_setup_status(home.path(), AgentClient::Cursor).mcp_server, Some(false));
write(
&home.path().join(".cursor/mcp.json"),
r#"{"mcpServers":{"yaak":{"command":"npx","args":["-y","@yaakapp/cli","mcp"]}}}"#,
);
assert_eq!(agent_setup_status(home.path(), AgentClient::Cursor).mcp_server, Some(true));
write(
&home.path().join(".cursor/mcp.json"),
r#"{"mcpServers":{"yaak":{"command":"npx","args":["-y","mcp-remote","http://127.0.0.1:64343/mcp"]}}}"#,
);
assert_eq!(agent_setup_status(home.path(), AgentClient::Cursor).mcp_server, Some(false));
write(&home.path().join(".cursor/mcp.json"), r#"{"servers":{"yaak":{}}}"#);
assert_eq!(agent_setup_status(home.path(), AgentClient::Cursor).mcp_server, Some(false));
if !cfg!(target_os = "windows") {
write(
&vscode_user_mcp_json(home.path()),
r#"{"servers":{"yaak":{"type":"stdio","command":"yaak","args":["mcp"]}}}"#,
);
assert_eq!(agent_setup_status(home.path(), AgentClient::Vscode).mcp_server, Some(true));
}
}
}
+11 -4
View File
@@ -2,22 +2,29 @@
use crate::error::Result;
use crate::host::Host;
use std::path::Path;
use yaak::export::{self, ExportDataParams};
use yaak_models::example::create_example_workspace;
use yaak_models::export::{self, ExportDataParams};
use yaak_models::util::BatchUpsertResult;
use yaak_rpc_schema::*;
use yaak_templates::format_json::format_json;
pub async fn cmd_export_data<H: Host>(host: H, req: CmdExportDataReq) -> Result<()> {
pub async fn cmd_export_data<H: Host>(host: H, req: CmdExportDataReq) -> Result<String> {
let version = host.app_version();
Ok(export::export_data(ExportDataParams {
query_manager: host.query_manager(),
yaak_version: &version,
export_path: Path::new(&req.export_path),
workspace_ids: req.workspace_ids.iter().map(|s| s.as_str()).collect(),
include_private_environments: req.include_private_environments,
})?)
}
pub async fn cmd_create_example_workspace<H: Host>(
host: H,
_req: CmdCreateExampleWorkspaceReq,
) -> Result<BatchUpsertResult> {
Ok(create_example_workspace(host.query_manager(), &host.update_source())?)
}
pub async fn cmd_format_json<H: Host>(_host: H, req: CmdFormatJsonReq) -> Result<String> {
Ok(format_json(&req.text, " "))
}
+1 -1
View File
@@ -72,7 +72,7 @@ pub trait Host: Clone {
self.query_manager().connect()
}
fn blobs(&self) -> BlobContext {
fn blobs(&self) -> BlobContext<'_> {
self.blob_manager().connect()
}
}
+1
View File
@@ -12,6 +12,7 @@
//! can serve (native windows, the updater, dialogs) or that still lean on it.
pub mod actions;
pub mod agents;
pub mod auth;
pub mod data;
pub mod encryption;
+31 -19
View File
@@ -12,10 +12,11 @@ use yaak_models::queries::workspaces::default_headers;
use yaak_rpc_schema::*;
pub async fn models_upsert<H: Host>(host: H, req: ModelsUpsertReq) -> Result<String> {
let db = host.db();
let blobs = host.blob_manager();
let source = host.update_source();
Ok(yaak_models::models_ops::upsert_model(&db, blobs, req.model, &source)?)
Ok(host
.query_manager()
.with_tx(|tx| yaak_models::models_ops::upsert_model(tx, blobs, req.model, &source))?)
}
/// Deletes cascade — a workspace can hold thousands of requests — and run in a
@@ -56,7 +57,9 @@ pub async fn models_snapshot_request<H: Host>(
host: H,
req: ModelsSnapshotRequestReq,
) -> Result<ModelVersion> {
Ok(host.db().snapshot_request_by_id(&req.request_id, req.reason)?)
Ok(host
.query_manager()
.with_tx(|tx| tx.snapshot_request_by_id(&req.request_id, req.reason))?)
}
/// A version and the live request side by side, for the diff and for deciding
@@ -78,7 +81,9 @@ pub async fn models_restore_request_version<H: Host>(
req: ModelsRestoreRequestVersionReq,
) -> Result<String> {
let source = host.update_source();
let restored = host.db().restore_request_version(&req.version_id, &source)?;
let restored = host
.query_manager()
.with_tx(|tx| tx.restore_request_version(&req.version_id, &source))?;
Ok(restored.id().to_string())
}
@@ -112,12 +117,9 @@ pub async fn models_upsert_graphql_introspection<H: Host>(
req: ModelsUpsertGraphqlIntrospectionReq,
) -> Result<GraphQlIntrospection> {
let source = host.update_source();
Ok(host.db().upsert_graphql_introspection(
&req.workspace_id,
&req.request_id,
req.content,
&source,
)?)
Ok(host.query_manager().with_tx(|tx| {
tx.upsert_graphql_introspection(&req.workspace_id, &req.request_id, req.content, &source)
})?)
}
/// Everything the frontend's model store needs to boot, as one JSON string.
@@ -149,9 +151,16 @@ pub async fn models_workspace_models<H: PluginHost>(
// Add the workspace children
if let Some(wid) = req.workspace_id.as_deref() {
// Opening a workspace is where the rows it is assumed to have get created
host.query_manager().with_tx(|tx| {
tx.ensure_base_environment(wid)?;
tx.ensure_default_cookie_jar(wid)?;
tx.ensure_workspace_meta(wid)?;
Ok::<(), yaak_models::error::Error>(())
})?;
let db = host.db();
l.append(&mut db.list_cookie_jars(wid)?.into_iter().map(Into::into).collect());
l.append(&mut db.list_environments_ensure_base(wid)?.into_iter().map(Into::into).collect());
l.append(&mut db.list_environments(wid)?.into_iter().map(Into::into).collect());
l.append(&mut db.list_folders(wid)?.into_iter().map(Into::into).collect());
l.append(&mut db.list_grpc_connections(wid)?.into_iter().map(Into::into).collect());
l.append(&mut db.list_grpc_requests(wid)?.into_iter().map(Into::into).collect());
@@ -169,23 +178,26 @@ pub async fn cmd_get_workspace_meta<H: Host>(
host: H,
req: CmdGetWorkspaceMetaReq,
) -> Result<WorkspaceMeta> {
let db = host.db();
let workspace = db.get_workspace(&req.workspace_id)?;
Ok(db.get_or_create_workspace_meta(&workspace.id)?)
let workspace = host.db().get_workspace(&req.workspace_id)?;
Ok(host.query_manager().with_tx(|tx| tx.ensure_workspace_meta(&workspace.id))?)
}
pub async fn cmd_delete_all_grpc_connections<H: Host>(
host: H,
req: CmdDeleteAllGrpcConnectionsReq,
) -> Result<()> {
Ok(host.db().delete_all_grpc_connections_for_request(&req.request_id, &host.update_source())?)
Ok(host.query_manager().with_tx(|tx| {
tx.delete_all_grpc_connections_for_request(&req.request_id, &host.update_source())
})?)
}
pub async fn cmd_delete_all_http_responses<H: Host>(
host: H,
req: CmdDeleteAllHttpResponsesReq,
) -> Result<()> {
host.db().delete_all_http_responses_for_request(&req.request_id, &host.update_source())?;
host.query_manager().with_tx(|tx| {
tx.delete_all_http_responses_for_request(&req.request_id, &host.update_source())
})?;
Ok(())
}
@@ -193,9 +205,9 @@ pub async fn cmd_ws_delete_connections<H: Host>(
host: H,
req: CmdWsDeleteConnectionsReq,
) -> Result<()> {
Ok(host
.db()
.delete_all_websocket_connections_for_request(&req.request_id, &host.update_source())?)
Ok(host.query_manager().with_tx(|tx| {
tx.delete_all_websocket_connections_for_request(&req.request_id, &host.update_source())
})?)
}
pub async fn cmd_delete_send_history<H: Host>(host: H, req: CmdDeleteSendHistoryReq) -> Result<()> {
+1
View File
@@ -37,6 +37,7 @@ fn fallback_plugin_metadata(directory: &str) -> PluginMetadata {
description: Some(format!("Plugin metadata could not be loaded from {directory}")),
homepage_url: None,
repository_url: None,
deprecated: None,
}
}
+68 -59
View File
@@ -118,6 +118,7 @@ impl Host for TestHost {
#[tokio::test(flavor = "multi_thread")]
async fn writes_carry_the_client_id() {
let host = TestHost::new();
host.drain_writes(); // the rows startup creates
let workspace = Workspace { name: "From a test".to_string(), ..Default::default() };
let id = models_upsert(host.clone(), ModelsUpsertReq { model: AnyModel::Workspace(workspace) })
@@ -406,21 +407,23 @@ async fn a_single_threaded_host_can_implement_the_trait() {
// shared code; only the callback came from the host. Rendering a real
// variable is what proves the chain was resolved rather than skipped.
let environment = host
.db()
.upsert_environment(
&Environment {
workspace_id: id.clone(),
name: "Test env".to_string(),
variables: vec![EnvironmentVariable {
enabled: true,
name: "greeting".to_string(),
value: "hello".to_string(),
id: None,
}],
..Default::default()
},
&host.update_source(),
)
.query_manager()
.with_tx(|tx| {
tx.upsert_environment(
&Environment {
workspace_id: id.clone(),
name: "Test env".to_string(),
variables: vec![EnvironmentVariable {
enabled: true,
name: "greeting".to_string(),
value: "hello".to_string(),
id: None,
}],
..Default::default()
},
&host.update_source(),
)
})
.expect("seed environment");
let rendered = cmd_render_template(
@@ -459,30 +462,33 @@ async fn auth_values_are_rendered_before_the_host_sees_them() {
};
let workspace = host
.db()
.upsert_workspace(
&Workspace { name: "Auth".to_string(), ..Default::default() },
&host.update_source(),
)
.query_manager()
.with_tx(|tx| {
tx.upsert_workspace(
&Workspace { name: "Auth".to_string(), ..Default::default() },
&host.update_source(),
)
})
.expect("workspace");
host.db()
.upsert_environment(
&Environment {
workspace_id: workspace.id.clone(),
name: "Env".to_string(),
variables: vec![EnvironmentVariable {
enabled: true,
name: "token".to_string(),
value: "s3cret".to_string(),
id: None,
}],
..Default::default()
},
&host.update_source(),
)
host.query_manager()
.with_tx(|tx| {
tx.upsert_environment(
&Environment {
workspace_id: workspace.id.clone(),
name: "Env".to_string(),
variables: vec![EnvironmentVariable {
enabled: true,
name: "token".to_string(),
value: "s3cret".to_string(),
id: None,
}],
..Default::default()
},
&host.update_source(),
)
})
.expect("environment");
let environment =
host.db().list_environments_ensure_base(&workspace.id).expect("list").remove(0);
let environment = host.db().list_environments(&workspace.id).expect("list").remove(0);
let mut values = HashMap::new();
values.insert("password".to_string(), JsonPrimitive::String("${[ token ]}".to_string()));
@@ -521,30 +527,33 @@ async fn template_function_values_are_rendered_before_the_host_sees_them() {
};
let workspace = host
.db()
.upsert_workspace(
&Workspace { name: "Functions".to_string(), ..Default::default() },
&host.update_source(),
)
.query_manager()
.with_tx(|tx| {
tx.upsert_workspace(
&Workspace { name: "Functions".to_string(), ..Default::default() },
&host.update_source(),
)
})
.expect("workspace");
host.db()
.upsert_environment(
&Environment {
workspace_id: workspace.id.clone(),
name: "Env".to_string(),
variables: vec![EnvironmentVariable {
enabled: true,
name: "1PASSWORD_TOKEN".to_string(),
value: "ops_abc123".to_string(),
id: None,
}],
..Default::default()
},
&host.update_source(),
)
host.query_manager()
.with_tx(|tx| {
tx.upsert_environment(
&Environment {
workspace_id: workspace.id.clone(),
name: "Env".to_string(),
variables: vec![EnvironmentVariable {
enabled: true,
name: "1PASSWORD_TOKEN".to_string(),
value: "ops_abc123".to_string(),
id: None,
}],
..Default::default()
},
&host.update_source(),
)
})
.expect("environment");
let environment =
host.db().list_environments_ensure_base(&workspace.id).expect("list").remove(0);
let environment = host.db().list_environments(&workspace.id).expect("list").remove(0);
let mut values = HashMap::new();
values.insert("token".to_string(), JsonPrimitive::String("${[1PASSWORD_TOKEN]}".to_string()));
+1
View File
@@ -5,5 +5,6 @@ edition = "2024"
publish = false
[dependencies]
dirs = "7"
serde_json = { workspace = true }
tokio = { workspace = true, features = ["process"] }
+46
View File
@@ -0,0 +1,46 @@
use std::path::PathBuf;
/// The data directory of the app with identifier `app_id`, as Tauri resolves it. Under WSL,
/// the Windows app's directory when it exists.
pub fn resolve_data_dir(app_id: &str) -> PathBuf {
if let Some(dir) = wsl_data_dir(app_id) {
return dir;
}
dirs::data_dir().expect("Could not determine data directory").join(app_id)
}
/// Detect WSL and resolve the Windows AppData\Roaming path for the app's data directory.
fn wsl_data_dir(app_id: &str) -> Option<PathBuf> {
if !cfg!(target_os = "linux") {
return None;
}
let proc_version = std::fs::read_to_string("/proc/version").ok()?;
let is_wsl = proc_version.to_lowercase().contains("microsoft");
if !is_wsl {
return None;
}
// We're in WSL, so try to resolve the app's data directory in Windows
// Get the Windows %APPDATA% path via cmd.exe
let appdata_output =
std::process::Command::new("cmd.exe").args(["/C", "echo", "%APPDATA%"]).output().ok()?;
let win_path = String::from_utf8(appdata_output.stdout).ok()?.trim().to_string();
if win_path.is_empty() || win_path == "%APPDATA%" {
return None;
}
// Convert Windows path to WSL path using wslpath (handles custom mount points)
let wslpath_output = std::process::Command::new("wslpath").arg(&win_path).output().ok()?;
let wsl_appdata = String::from_utf8(wslpath_output.stdout).ok()?.trim().to_string();
if wsl_appdata.is_empty() {
return None;
}
let wsl_path = PathBuf::from(wsl_appdata).join(app_id);
if wsl_path.exists() { Some(wsl_path) } else { None }
}
+1
View File
@@ -1,3 +1,4 @@
pub mod command;
pub mod data_dir;
pub mod platform;
pub mod serde;
+2 -2
View File
@@ -6,8 +6,8 @@ publish = false
[dependencies]
base32 = "0.5.1" # For encoding human-readable key
base64 = "0.22.1" # For encoding in the database
chacha20poly1305 = "0.10.1"
base64 = "0.23.1" # For encoding in the database
chacha20poly1305 = "0.11.0"
keyring = { workspace = true, features = ["apple-native", "windows-native", "sync-secret-service"] }
log = { workspace = true }
serde = { workspace = true, features = ["derive"] }
+34 -13
View File
@@ -1,14 +1,14 @@
use crate::error::Error::{DecryptionError, EncryptionError, InvalidEncryptedData};
use crate::error::Result;
use chacha20poly1305::aead::generic_array::typenum::Unsigned;
use chacha20poly1305::aead::{Aead, AeadCore, Key, KeyInit, OsRng};
use chacha20poly1305::XChaCha20Poly1305;
use chacha20poly1305::aead::array::typenum::Unsigned;
use chacha20poly1305::aead::{Aead, AeadCore, Generate, Key, KeyInit};
use chacha20poly1305::{XChaCha20Poly1305, XNonce};
const ENCRYPTION_TAG: &str = "yA4k3nC";
const ENCRYPTION_VERSION: u8 = 1;
pub(crate) fn encrypt_data(data: &[u8], key: &Key<XChaCha20Poly1305>) -> Result<Vec<u8>> {
let nonce = XChaCha20Poly1305::generate_nonce(&mut OsRng);
let nonce = XNonce::generate();
let cipher = XChaCha20Poly1305::new(&key);
let ciphered_data = cipher.encrypt(&nonce, data).map_err(|_| EncryptionError)?;
@@ -34,11 +34,12 @@ pub(crate) fn decrypt_data(cipher_data: &[u8], key: &Key<XChaCha20Poly1305>) ->
return Err(InvalidEncryptedData);
}
let nonce_bytes = <XChaCha20Poly1305 as AeadCore>::NonceSize::to_usize();
let nonce_bytes = <XChaCha20Poly1305 as AeadCore>::NonceSize::USIZE;
let (nonce, ciphered_data) = rest.split_at_checked(nonce_bytes).ok_or(InvalidEncryptedData)?;
let nonce: &XNonce = nonce.try_into().map_err(|_| InvalidEncryptedData)?;
let cipher = XChaCha20Poly1305::new(&key);
cipher.decrypt(nonce.into(), ciphered_data).map_err(|_e| DecryptionError)
cipher.decrypt(nonce, ciphered_data).map_err(|_e| DecryptionError)
}
#[cfg(test)]
@@ -46,12 +47,12 @@ mod test {
use crate::encryption::{decrypt_data, encrypt_data};
use crate::error::Error::InvalidEncryptedData;
use crate::error::Result;
use chacha20poly1305::aead::OsRng;
use chacha20poly1305::{KeyInit, XChaCha20Poly1305};
use chacha20poly1305::aead::{Generate, Key};
use chacha20poly1305::XChaCha20Poly1305;
#[test]
fn test_encrypt_decrypt() -> Result<()> {
let key = XChaCha20Poly1305::generate_key(OsRng);
let key = Key::<XChaCha20Poly1305>::generate();
let encrypted = encrypt_data("hello world".as_bytes(), &key)?;
let decrypted = decrypt_data(encrypted.as_slice(), &key)?;
assert_eq!(String::from_utf8(decrypted).unwrap(), "hello world");
@@ -60,7 +61,7 @@ mod test {
#[test]
fn test_decrypt_empty() -> Result<()> {
let key = XChaCha20Poly1305::generate_key(OsRng);
let key = Key::<XChaCha20Poly1305>::generate();
let encrypted = encrypt_data(&[], &key)?;
assert_eq!(encrypted.len(), 48);
let decrypted = decrypt_data(encrypted.as_slice(), &key)?;
@@ -70,7 +71,7 @@ mod test {
#[test]
fn test_decrypt_bad_version() -> Result<()> {
let key = XChaCha20Poly1305::generate_key(OsRng);
let key = Key::<XChaCha20Poly1305>::generate();
let mut encrypted = encrypt_data("hello world".as_bytes(), &key)?;
encrypted[7] = 0;
let decrypted = decrypt_data(encrypted.as_slice(), &key);
@@ -80,7 +81,7 @@ mod test {
#[test]
fn test_decrypt_bad_tag() -> Result<()> {
let key = XChaCha20Poly1305::generate_key(OsRng);
let key = Key::<XChaCha20Poly1305>::generate();
let mut encrypted = encrypt_data("hello world".as_bytes(), &key)?;
encrypted[0] = 2;
let decrypted = decrypt_data(encrypted.as_slice(), &key);
@@ -90,9 +91,29 @@ mod test {
#[test]
fn test_decrypt_unencrypted_data() -> Result<()> {
let key = XChaCha20Poly1305::generate_key(OsRng);
let key = Key::<XChaCha20Poly1305>::generate();
let decrypted = decrypt_data("123".as_bytes(), &key);
assert!(matches!(decrypted, Err(InvalidEncryptedData)));
Ok(())
}
}
#[cfg(test)]
mod compatibility {
use super::*;
/// Produced by chacha20poly1305 0.10.1. Decrypting it here proves a crate
/// upgrade hasn't changed the on-disk format, which users already have
/// rows of in their databases.
const V1_FROM_0_10: &str = "7941346b336e43017c7cb13467eecaa963b11734be636f9cc4152de348584fb27e95d1a70973e557cd335cf29e12d0d305d63ca0aa168f1b17003b1690a9d49140f0";
#[test]
fn decrypts_data_written_by_the_previous_release() {
let bytes: Vec<u8> = (0..V1_FROM_0_10.len())
.step_by(2)
.map(|i| u8::from_str_radix(&V1_FROM_0_10[i..i + 2], 16).unwrap())
.collect();
let key = Key::<XChaCha20Poly1305>::from([7u8; 32]);
assert_eq!(decrypt_data(&bytes, &key).unwrap(), b"yaak golden vector");
}
}
+9 -7
View File
@@ -83,7 +83,7 @@ impl EncryptionManager {
let workspace_meta = self.query_manager.with_tx::<WorkspaceMeta, Error>(|tx| {
let workspace = tx.get_workspace(workspace_id)?;
let workspace_meta = tx.get_or_create_workspace_meta(workspace_id)?;
let workspace_meta = tx.ensure_workspace_meta(workspace_id)?;
tx.upsert_workspace(
&Workspace { encryption_key_challenge, ..workspace },
&UpdateSource::Background,
@@ -103,7 +103,7 @@ impl EncryptionManager {
pub fn ensure_workspace_key(&self, workspace_id: &str) -> Result<WorkspaceMeta> {
let workspace_meta =
self.query_manager.connect().get_or_create_workspace_meta(workspace_id)?;
self.query_manager.with_tx(|tx| tx.ensure_workspace_meta(workspace_id))?;
// Already exists
if let Some(_) = workspace_meta.encryption_key {
@@ -120,7 +120,7 @@ impl EncryptionManager {
self.query_manager.with_tx::<(), Error>(|tx| {
let workspace = tx.get_workspace(workspace_id)?;
let workspace_meta = tx.get_or_create_workspace_meta(workspace_id)?;
let workspace_meta = tx.ensure_workspace_meta(workspace_id)?;
// Clear encryption challenge on workspace
tx.upsert_workspace(
@@ -152,10 +152,12 @@ impl EncryptionManager {
}
};
let db = self.query_manager.connect();
let workspace_meta = db.get_or_create_workspace_meta(workspace_id)?;
let key = match workspace_meta.encryption_key {
let key = match self
.query_manager
.connect()
.get_workspace_meta(workspace_id)
.and_then(|m| m.encryption_key)
{
None => return Err(MissingWorkspaceKey),
Some(k) => k,
};
+5 -6
View File
@@ -2,7 +2,7 @@ use crate::encryption::{decrypt_data, encrypt_data};
use crate::error::Error::GenericError;
use crate::error::Result;
use base32::Alphabet;
use chacha20poly1305::aead::{Key, KeyInit, OsRng};
use chacha20poly1305::aead::{Generate, Key};
use chacha20poly1305::XChaCha20Poly1305;
use keyring::{Entry, Error};
use log::info;
@@ -24,11 +24,12 @@ impl MasterKey {
let without_prefix = encoded.strip_prefix(HUMAN_PREFIX).unwrap_or(&encoded);
let key_bytes = base32::decode(Alphabet::Crockford {}, &without_prefix)
.ok_or(GenericError("Failed to decode master key".to_string()))?;
Key::<XChaCha20Poly1305>::clone_from_slice(key_bytes.as_slice())
Key::<XChaCha20Poly1305>::try_from(key_bytes.as_slice())
.map_err(|_| GenericError("Master key is the wrong length".to_string()))?
}
Err(Error::NoEntry) => {
info!("Creating new master key");
let key = XChaCha20Poly1305::generate_key(OsRng);
let key = Key::<XChaCha20Poly1305>::generate();
let encoded = base32::encode(Alphabet::Crockford {}, key.as_slice());
let with_prefix = format!("{HUMAN_PREFIX}{encoded}");
entry.set_password(&with_prefix)?;
@@ -50,9 +51,7 @@ impl MasterKey {
#[cfg(test)]
pub(crate) fn test_key() -> Self {
let key: Key<XChaCha20Poly1305> = Key::<XChaCha20Poly1305>::clone_from_slice(
"00000000000000000000000000000000".as_bytes(),
);
let key = Key::<XChaCha20Poly1305>::from(*b"00000000000000000000000000000000");
Self { key }
}
}
+5 -3
View File
@@ -2,7 +2,7 @@ use crate::encryption::{decrypt_data, encrypt_data};
use crate::error::Error::InvalidHumanKey;
use crate::error::Result;
use base32::Alphabet;
use chacha20poly1305::aead::{Key, KeyInit, OsRng};
use chacha20poly1305::aead::{Generate, Key};
use chacha20poly1305::{KeySizeUser, XChaCha20Poly1305};
#[derive(Debug, Clone)]
@@ -39,7 +39,9 @@ impl WorkspaceKey {
}
pub(crate) fn from_raw_key(key: &[u8]) -> Self {
Self { key: Key::<XChaCha20Poly1305>::clone_from_slice(key) }
Self {
key: Key::<XChaCha20Poly1305>::try_from(key).expect("workspace key must be 32 bytes"),
}
}
pub(crate) fn raw_key(&self) -> &[u8] {
@@ -47,7 +49,7 @@ impl WorkspaceKey {
}
pub(crate) fn create() -> Result<Self> {
let key = XChaCha20Poly1305::generate_key(OsRng);
let key = Key::<XChaCha20Poly1305>::generate();
Ok(Self::from_raw_key(key.as_slice()))
}
+14
View File
@@ -0,0 +1,14 @@
[package]
name = "yaak-filter"
version = "0.1.0"
edition = "2024"
publish = false
[dependencies]
fancy-regex = "0.19"
indexmap = "2"
sxd-document = "0.3.2"
sxd-xpath = "0.4.2"
[dev-dependencies]
serde_json = { workspace = true, features = ["preserve_order"] }
+423
View File
@@ -0,0 +1,423 @@
//! The response document as `JSON.parse` builds it, and `JSON.stringify(_, null, 2)`.
use super::value::JsValue;
use indexmap::IndexMap;
const MAX_DEPTH: usize = 128;
#[derive(Debug)]
pub enum Json {
Null,
Bool(bool),
Number(f64),
String(String),
Array(Vec<Json>),
Object(IndexMap<String, Json>),
}
pub fn parse(text: &str) -> Result<Json, String> {
let mut parser = Parser { bytes: text.as_bytes(), text, pos: 0 };
parser.skip_ws();
let value = parser.value(0)?;
parser.skip_ws();
if parser.pos < parser.bytes.len() {
return Err(parser.error("trailing characters"));
}
Ok(value)
}
struct Parser<'t> {
bytes: &'t [u8],
text: &'t str,
pos: usize,
}
impl Parser<'_> {
fn error(&self, message: &str) -> String {
let mut end = self.pos.min(self.text.len());
while !self.text.is_char_boundary(end) {
end -= 1;
}
let before = &self.text[..end];
let line = before.matches('\n').count() + 1;
let column = before.rsplit('\n').next().map_or(0, |l| l.chars().count()) + 1;
format!("{message} at line {line} column {column}")
}
fn peek(&self) -> Option<u8> {
self.bytes.get(self.pos).copied()
}
fn skip_ws(&mut self) {
while let Some(b' ' | b'\t' | b'\n' | b'\r') = self.peek() {
self.pos += 1;
}
}
fn expect_word(&mut self, word: &str, value: Json) -> Result<Json, String> {
if self.text[self.pos..].starts_with(word) {
self.pos += word.len();
Ok(value)
} else {
Err(self.error("expected value"))
}
}
fn value(&mut self, depth: usize) -> Result<Json, String> {
if depth > MAX_DEPTH {
return Err(self.error("recursion limit exceeded"));
}
match self.peek() {
None => Err(self.error("EOF while parsing a value")),
Some(b'n') => self.expect_word("null", Json::Null),
Some(b't') => self.expect_word("true", Json::Bool(true)),
Some(b'f') => self.expect_word("false", Json::Bool(false)),
Some(b'"') => Ok(Json::String(self.string()?)),
Some(b'[') => self.array(depth),
Some(b'{') => self.object(depth),
Some(b'-' | b'0'..=b'9') => self.number(),
Some(_) => Err(self.error("expected value")),
}
}
fn array(&mut self, depth: usize) -> Result<Json, String> {
self.pos += 1;
let mut items = Vec::new();
self.skip_ws();
if self.peek() == Some(b']') {
self.pos += 1;
return Ok(Json::Array(items));
}
loop {
self.skip_ws();
items.push(self.value(depth + 1)?);
self.skip_ws();
match self.peek() {
Some(b',') => self.pos += 1,
Some(b']') => {
self.pos += 1;
return Ok(Json::Array(items));
}
_ => return Err(self.error("expected `,` or `]`")),
}
}
}
fn object(&mut self, depth: usize) -> Result<Json, String> {
self.pos += 1;
let mut map = IndexMap::new();
self.skip_ws();
if self.peek() == Some(b'}') {
self.pos += 1;
return Ok(Json::Object(map));
}
loop {
self.skip_ws();
if self.peek() != Some(b'"') {
return Err(self.error("key must be a string"));
}
let key = self.string()?;
self.skip_ws();
if self.peek() != Some(b':') {
return Err(self.error("expected `:`"));
}
self.pos += 1;
self.skip_ws();
let value = self.value(depth + 1)?;
map.insert(key, value);
self.skip_ws();
match self.peek() {
Some(b',') => self.pos += 1,
Some(b'}') => {
self.pos += 1;
return Ok(Json::Object(js_key_order(map)));
}
_ => return Err(self.error("expected `,` or `}`")),
}
}
}
fn number(&mut self) -> Result<Json, String> {
let start = self.pos;
if self.peek() == Some(b'-') {
self.pos += 1;
}
match self.peek() {
Some(b'0') => self.pos += 1,
Some(b'1'..=b'9') => self.digits(),
_ => return Err(self.error("invalid number")),
}
if self.peek() == Some(b'.') {
self.pos += 1;
if !matches!(self.peek(), Some(b'0'..=b'9')) {
return Err(self.error("invalid number"));
}
self.digits();
}
if let Some(b'e' | b'E') = self.peek() {
self.pos += 1;
if let Some(b'+' | b'-') = self.peek() {
self.pos += 1;
}
if !matches!(self.peek(), Some(b'0'..=b'9')) {
return Err(self.error("invalid number"));
}
self.digits();
}
let number =
self.text[start..self.pos].parse::<f64>().map_err(|e| self.error(&e.to_string()))?;
Ok(Json::Number(number))
}
fn digits(&mut self) {
while let Some(b'0'..=b'9') = self.peek() {
self.pos += 1;
}
}
fn hex4(&mut self) -> Result<u16, String> {
let hex =
self.text.get(self.pos..self.pos + 4).ok_or_else(|| self.error("invalid escape"))?;
let unit = u16::from_str_radix(hex, 16).map_err(|_| self.error("invalid escape"))?;
if !hex.bytes().all(|b| b.is_ascii_hexdigit()) {
return Err(self.error("invalid escape"));
}
self.pos += 4;
Ok(unit)
}
fn string(&mut self) -> Result<String, String> {
self.pos += 1;
let mut out = String::new();
loop {
let run = self.pos;
while let Some(b) = self.peek() {
if b == b'"' || b == b'\\' || b < 0x20 {
break;
}
self.pos += 1;
}
out.push_str(&self.text[run..self.pos]);
match self.peek() {
None => return Err(self.error("EOF while parsing a string")),
Some(b'"') => {
self.pos += 1;
return Ok(out);
}
Some(b'\\') => {
self.pos += 1;
let escape =
self.peek().ok_or_else(|| self.error("EOF while parsing a string"))?;
self.pos += 1;
match escape {
b'"' => out.push('"'),
b'\\' => out.push('\\'),
b'/' => out.push('/'),
b'b' => out.push('\u{8}'),
b'f' => out.push('\u{c}'),
b'n' => out.push('\n'),
b'r' => out.push('\r'),
b't' => out.push('\t'),
b'u' => {
let unit = self.hex4()?;
let mut units = vec![unit];
if (0xD800..0xDC00).contains(&unit)
&& self.text[self.pos..].starts_with("\\u")
{
let save = self.pos;
self.pos += 2;
let low = self.hex4()?;
if (0xDC00..0xE000).contains(&low) {
units.push(low);
} else {
self.pos = save;
}
}
out.push_str(&String::from_utf16_lossy(&units));
}
_ => return Err(self.error("invalid escape")),
}
}
Some(_) => return Err(self.error("control character in string")),
}
}
}
}
/// `JSON.parse` gives array-index keys first, ascending, then the rest in document order.
fn js_key_order(map: IndexMap<String, Json>) -> IndexMap<String, Json> {
if !map.keys().any(|k| array_index(k).is_some()) {
return map;
}
let (mut indexes, rest): (Vec<_>, Vec<_>) =
map.into_iter().partition(|(k, _)| array_index(k).is_some());
indexes.sort_by_key(|(k, _)| array_index(k));
indexes.into_iter().chain(rest).collect()
}
/// The value of a canonical array-index string ("0", "17", never "07" or "-1").
pub fn array_index(key: &str) -> Option<u32> {
if key.is_empty()
|| (key.len() > 1 && key.starts_with('0'))
|| !key.bytes().all(|b| b.is_ascii_digit())
{
return None;
}
key.parse::<u32>().ok().filter(|i| *i < u32::MAX)
}
/// `JSON.stringify(values, null, 2)`.
pub fn stringify_results(values: &[JsValue<'_>]) -> String {
let mut out = String::new();
write_array(&mut out, values.iter(), values.is_empty(), 0);
out
}
fn indent(out: &mut String, depth: usize) {
out.push('\n');
for _ in 0..depth {
out.push_str(" ");
}
}
fn write_array<'v, 'a: 'v>(
out: &mut String,
items: impl Iterator<Item = &'v JsValue<'a>>,
empty: bool,
depth: usize,
) {
if empty {
out.push_str("[]");
return;
}
out.push('[');
for (i, item) in items.enumerate() {
if i > 0 {
out.push(',');
}
indent(out, depth + 1);
if !write_js(out, item, depth + 1) {
out.push_str("null");
}
}
indent(out, depth);
out.push(']');
}
/// Writes the value and returns true, or false for what `JSON.stringify` drops.
fn write_js(out: &mut String, value: &JsValue<'_>, depth: usize) -> bool {
match value {
JsValue::Undefined | JsValue::Function(_) => return false,
JsValue::Null => out.push_str("null"),
JsValue::Bool(b) => out.push_str(if *b { "true" } else { "false" }),
JsValue::Number(n) => write_number(out, *n),
JsValue::String(s) => write_string(out, s),
JsValue::Doc(json) => write_json(out, json, depth),
JsValue::Array(array) => {
write_array(out, array.items.iter(), array.items.is_empty(), depth)
}
JsValue::Regex(_) => out.push_str("{}"),
}
true
}
fn write_json(out: &mut String, json: &Json, depth: usize) {
match json {
Json::Null => out.push_str("null"),
Json::Bool(b) => out.push_str(if *b { "true" } else { "false" }),
Json::Number(n) => write_number(out, *n),
Json::String(s) => write_string(out, s),
Json::Array(items) => {
if items.is_empty() {
out.push_str("[]");
return;
}
out.push('[');
for (i, item) in items.iter().enumerate() {
if i > 0 {
out.push(',');
}
indent(out, depth + 1);
write_json(out, item, depth + 1);
}
indent(out, depth);
out.push(']');
}
Json::Object(map) => {
if map.is_empty() {
out.push_str("{}");
return;
}
out.push('{');
for (i, (key, item)) in map.iter().enumerate() {
if i > 0 {
out.push(',');
}
indent(out, depth + 1);
write_string(out, key);
out.push_str(": ");
write_json(out, item, depth + 1);
}
indent(out, depth);
out.push('}');
}
}
}
fn write_number(out: &mut String, n: f64) {
if n.is_finite() {
out.push_str(&number_to_string(n));
} else {
out.push_str("null");
}
}
fn write_string(out: &mut String, s: &str) {
out.push('"');
for c in s.chars() {
match c {
'"' => out.push_str("\\\""),
'\\' => out.push_str("\\\\"),
'\u{8}' => out.push_str("\\b"),
'\u{c}' => out.push_str("\\f"),
'\n' => out.push_str("\\n"),
'\r' => out.push_str("\\r"),
'\t' => out.push_str("\\t"),
c if (c as u32) < 0x20 => out.push_str(&format!("\\u{:04x}", c as u32)),
c => out.push(c),
}
}
out.push('"');
}
/// `Number.prototype.toString()` for radix 10.
pub fn number_to_string(n: f64) -> String {
if n.is_nan() {
return "NaN".to_string();
}
if n == 0.0 {
return "0".to_string();
}
if n.is_infinite() {
return if n > 0.0 { "Infinity" } else { "-Infinity" }.to_string();
}
let sign = if n < 0.0 { "-" } else { "" };
let sci = format!("{:e}", n.abs());
let (mantissa, exponent) = sci.split_once('e').unwrap_or((&sci, "0"));
let digits: String = mantissa.chars().filter(|c| *c != '.').collect();
let k = digits.len() as i32;
let n_exp = exponent.parse::<i32>().unwrap_or(0) + 1;
let body = if k <= n_exp && n_exp <= 21 {
format!("{digits}{}", "0".repeat((n_exp - k) as usize))
} else if 0 < n_exp && n_exp <= 21 {
format!("{}.{}", &digits[..n_exp as usize], &digits[n_exp as usize..])
} else if -6 < n_exp && n_exp <= 0 {
format!("0.{}{digits}", "0".repeat((-n_exp) as usize))
} else {
let e = n_exp - 1;
let e_sign = if e < 0 { "-" } else { "+" };
let rest = if k > 1 { format!(".{}", &digits[1..]) } else { String::new() };
format!("{}{rest}e{e_sign}{}", &digits[..1], e.abs())
};
format!("{sign}{body}")
}
+798
View File
@@ -0,0 +1,798 @@
//! A port of jsonpath-plus 11.0.0 (`JSONPath({ path, json })` with its default
//! options and safe evaluator), so expressions written for the JavaScript
//! filter plugin keep their results. Where it differs on purpose: quoted
//! bracket names are always literal keys, and some RFC 9535 syntax that
//! jsonpath-plus finds nothing for works (`[-1]`, `[::-1]`, `[?@.a > 1]`,
//! `length()`, `count()`, `match()`, `search()`, `value()`).
mod json;
mod path;
mod regex;
mod rfc;
mod script;
mod value;
use json::Json;
pub(crate) use json::number_to_string;
use path::Loc;
use regex::JsRegex;
use rfc::Query;
use std::cell::{Cell, RefCell};
use std::collections::HashMap;
use std::rc::Rc;
use std::sync::LazyLock;
use value::{JsValue, own_property};
/// Nested `_trace` calls; a debug build spends about 8 KB of stack on each.
const MAX_DEPTH: usize = 180;
/// Steps a single evaluation may take; `..` after `..` grows exponentially with nesting.
const MAX_STEPS: usize = 20_000_000;
pub fn evaluate(text: &str, query: &str) -> Result<String, String> {
let root = json::parse(text).map_err(|e| format!("Invalid JSON: {e}"))?;
if let Some(keys) = path::literal_keys(query) {
let found = keys.iter().try_fold(JsValue::from_doc(&root), |v, k| own_property(&v, k));
return Ok(json::stringify_results(found.as_slice()));
}
let engine = Engine::new(&root);
let values =
engine.run(query, &JsValue::from_doc(&root)).map_err(|e| format!("Invalid filter: {e}"))?;
Ok(json::stringify_results(&values))
}
#[derive(Clone, Debug)]
enum PathElem {
Str(Rc<str>),
Num(f64),
}
impl PathElem {
fn to_js(&self) -> JsValue<'static> {
match self {
PathElem::Str(s) => JsValue::string(s.to_string()),
PathElem::Num(n) => JsValue::Number(*n),
}
}
fn text(&self) -> String {
match self {
PathElem::Str(s) => s.to_string(),
PathElem::Num(n) => number_to_string(*n),
}
}
}
struct PathNode {
elem: PathElem,
prev: Path,
}
/// The steps from the root, newest first and shared, as jsonpath-plus copies its arrays.
type Path = Option<Rc<PathNode>>;
fn push(path: &Path, elem: PathElem) -> Path {
Some(Rc::new(PathNode { elem, prev: path.clone() }))
}
/// `JSONPath.toPathString`.
fn path_string(path: &Path) -> String {
let mut elems = Vec::new();
let mut node = path.as_ref();
while let Some(n) = node {
elems.push(n.elem.text());
node = n.prev.as_ref();
}
elems.reverse();
let mut out = "$".to_string();
for elem in elems.iter().skip(1) {
let operator =
elem == "~" || elem == "^" || (elem.starts_with('@') && elem.ends_with("()"));
if operator {
continue;
}
if !elem.is_empty() && elem.bytes().all(|b| b.is_ascii_digit() || b == b'*') {
out.push_str(&format!("[{elem}]"));
} else {
out.push_str(&format!("['{elem}']"));
}
}
out
}
struct Ret<'a> {
path: Path,
value: JsValue<'a>,
/// The steps after a `^`, to trace again from the parent.
parent_selector: Option<Rc<[Loc]>>,
}
enum Traced<'a> {
One(Ret<'a>),
Many(Vec<Option<Ret<'a>>>),
}
fn add<'a>(ret: &mut Vec<Option<Ret<'a>>>, traced: Traced<'a>) {
match traced {
Traced::One(r) => ret.push(Some(r)),
Traced::Many(items) => ret.extend(items),
}
}
enum FilterPlan {
/// jsonpath-plus spots `@.key[?(...)]` in a filter and keeps the items
/// whose `key` the inner filter selects anything from, ignoring the rest.
Nested {
key: String,
inner: Loc,
},
Script(Rc<Script>),
}
struct Script {
expr: script::Expr,
code: String,
uses_path: bool,
}
#[allow(clippy::type_complexity)]
pub(crate) struct Engine<'a> {
root: &'a Json,
paths: RefCell<HashMap<String, Rc<[Loc]>>>,
scripts: RefCell<HashMap<String, Rc<Script>>>,
filters: RefCell<HashMap<String, Rc<FilterPlan>>>,
rfc_filters: RefCell<HashMap<String, Rc<rfc::Logical>>>,
/// RFC 9535 `match()` and `search()` patterns, `None` when invalid.
regexes: RefCell<HashMap<(String, bool), Option<Rc<JsRegex>>>>,
has_parent_selector: Cell<bool>,
depth: Cell<usize>,
steps: Cell<usize>,
}
static SLICE: LazyLock<fancy_regex::Regex> = LazyLock::new(|| {
fancy_regex::Regex::new(r"^(-?[0-9]*):(-?[0-9]*):?([0-9]*)$")
.unwrap_or_else(|e| unreachable!("{e}"))
});
static NESTED: LazyLock<fancy_regex::Regex> = LazyLock::new(|| {
fancy_regex::Regex::new(r"@[^\n\r\x{2028}\x{2029}]?([^?]*)[\['](\??\([^\n\r\x{2028}\x{2029}]*?\))(?![^\n\r\x{2028}\x{2029}]\)\])[\]']")
.unwrap_or_else(|e| unreachable!("{e}"))
});
fn is_line_terminator(c: char) -> bool {
matches!(c, '\n' | '\r' | '\u{2028}' | '\u{2029}')
}
impl<'a> Engine<'a> {
fn new(root: &'a Json) -> Self {
Engine {
root,
paths: RefCell::default(),
scripts: RefCell::default(),
filters: RefCell::default(),
rfc_filters: RefCell::default(),
regexes: RefCell::default(),
has_parent_selector: Cell::new(false),
depth: Cell::new(0),
steps: Cell::new(0),
}
}
/// `JSONPath.evaluate` from `start`, as values.
fn run(&self, query: &str, start: &JsValue<'a>) -> Result<Vec<JsValue<'a>>, String> {
let locs = self.path(query)?;
let locs: &[Loc] = match &locs[..] {
[Loc::Str(first), rest @ ..] if &**first == "$" && !rest.is_empty() => rest,
all => all,
};
let root_path = push(&None, PathElem::Str("$".into()));
let frame = Frame {
expr: locs,
val: start,
path: &root_path,
parent: &JsValue::Null,
parent_prop: &JsValue::Null,
literal_priority: false,
};
let traced = self.trace(&frame)?;
let rets = match traced {
Traced::One(r) => vec![Some(r)],
Traced::Many(items) => items,
};
Ok(rets
.into_iter()
.flatten()
.filter(|r| r.parent_selector.is_none())
.map(|r| r.value)
.collect())
}
pub(crate) fn query(
&self,
query: &Query,
current: &JsValue<'a>,
) -> Result<Vec<JsValue<'a>>, String> {
let start = if query.root { JsValue::from_doc(self.root) } else { current.clone() };
self.run(&query.path, &start)
}
pub(crate) fn i_regexp(&self, pattern: &str, whole: bool) -> Option<Rc<JsRegex>> {
self.regexes
.borrow_mut()
.entry((pattern.to_string(), whole))
.or_insert_with(|| JsRegex::i_regexp(pattern, whole).map(Rc::new))
.clone()
}
fn path(&self, query: &str) -> Result<Rc<[Loc]>, String> {
if let Some(locs) = self.paths.borrow().get(query) {
return Ok(locs.clone());
}
let locs: Rc<[Loc]> = path::parse(query)?.into();
for loc in locs.iter() {
self.check(loc)?;
}
self.paths.borrow_mut().insert(query.to_string(), locs.clone());
Ok(locs)
}
/// Compiles what a step will evaluate, so a malformed or unsupported
/// expression is an error even when no node reaches it.
fn check(&self, loc: &Loc) -> Result<(), String> {
match loc {
Loc::Str(s) | Loc::Part(s) => {
if s.starts_with("?(") {
self.filter(s).map(drop)
} else if s.starts_with('(') {
self.script(s).map(drop)
} else if s.contains(',') {
s.split(',').try_for_each(|part| self.check(&Loc::Part(part.into())))
} else {
Ok(())
}
}
Loc::Filter(text) => self.rfc_filter(text).map(drop),
Loc::Num(_) | Loc::Other(_) | Loc::ArrayKey(_) | Loc::Names(_) => Ok(()),
}
}
fn script(&self, code: &str) -> Result<Rc<Script>, String> {
if let Some(script) = self.scripts.borrow().get(code) {
return Ok(script.clone());
}
let expr = script::parse(&script::substitute(code)).map_err(|e| format!("{e}: {code}"))?;
let script =
Rc::new(Script { expr, code: code.to_string(), uses_path: code.contains("@path") });
self.scripts.borrow_mut().insert(code.to_string(), script.clone());
Ok(script)
}
fn filter(&self, loc: &str) -> Result<Rc<FilterPlan>, String> {
if let Some(plan) = self.filters.borrow().get(loc) {
return Ok(plan.clone());
}
let code = match loc.strip_prefix("?(").and_then(|s| s.strip_suffix(')')) {
Some(inner) if !loc.contains(is_line_terminator) => inner,
_ => loc,
};
let nested = NESTED.captures(code).map_err(|e| e.to_string())?;
let plan = match nested {
Some(c) => {
let inner = Loc::Str(c[2].into());
self.check(&inner)?;
FilterPlan::Nested { key: c[1].to_string(), inner }
}
None => FilterPlan::Script(self.script(code)?),
};
let plan = Rc::new(plan);
self.filters.borrow_mut().insert(loc.to_string(), plan.clone());
Ok(plan)
}
fn rfc_filter(&self, text: &str) -> Result<Rc<rfc::Logical>, String> {
if let Some(filter) = self.rfc_filters.borrow().get(text) {
return Ok(filter.clone());
}
let filter = Rc::new(rfc::parse(text).map_err(|e| format!("{e}: ?{text}"))?);
self.rfc_filters.borrow_mut().insert(text.to_string(), filter.clone());
Ok(filter)
}
fn run_script(
&self,
script: &Script,
value: &JsValue<'a>,
property: JsValue<'a>,
path: impl FnOnce() -> String,
parent: &JsValue<'a>,
parent_property: &JsValue<'a>,
) -> Result<JsValue<'a>, String> {
let scope = script::Scope {
value: value.clone(),
property,
parent: parent.clone(),
parent_property: parent_property.clone(),
path: script.uses_path.then(path),
};
script::eval(&script.expr, &scope, self).map_err(|e| format!("{e}: {}", script.code))
}
fn trace(&self, f: &Frame<'_, 'a>) -> Result<Traced<'a>, String> {
let depth = self.depth.get() + 1;
if depth > MAX_DEPTH {
return Err("the path goes too deep".to_string());
}
self.steps.set(self.steps.get() + 1);
if self.steps.get() > MAX_STEPS {
return Err("the path takes too many steps to evaluate".to_string());
}
self.depth.set(depth);
let traced = self.trace_inner(f);
self.depth.set(depth - 1);
traced
}
/// Traces the rest of the path from a child of the frame's value.
fn child(
&self,
f: &Frame<'_, 'a>,
child: &JsValue<'a>,
elem: PathElem,
literal_priority: bool,
ret: &mut Vec<Option<Ret<'a>>>,
) -> Result<(), String> {
let path = push(f.path, elem.clone());
let parent_prop = elem.to_js();
let next = Frame {
expr: f.rest(),
val: child,
path: &path,
parent: f.val,
parent_prop: &parent_prop,
literal_priority,
};
add(ret, self.trace(&next)?);
Ok(())
}
/// Traces other steps from the frame's own value.
fn again(
&self,
f: &Frame<'_, 'a>,
expr: &[Loc],
ret: &mut Vec<Option<Ret<'a>>>,
) -> Result<(), String> {
add(ret, self.trace(&Frame { expr, literal_priority: false, ..*f })?);
Ok(())
}
fn with_first(first: Loc, rest: &[Loc]) -> Vec<Loc> {
std::iter::once(first).chain(rest.iter().cloned()).collect()
}
fn trace_inner(&self, f: &Frame<'_, 'a>) -> Result<Traced<'a>, String> {
let Some(loc) = f.expr.first() else {
return Ok(Traced::One(Ret {
path: f.path.clone(),
value: f.val.clone(),
parent_selector: None,
}));
};
let mut ret = Vec::new();
match loc {
Loc::Names(names) => {
let child_priority = names.len() > 1 || !f.literal_priority;
for name in names.iter() {
if let Some(child) = f.own(name) {
self.child(
f,
&child,
PathElem::Str(name.as_str().into()),
child_priority,
&mut ret,
)?;
}
}
}
Loc::Filter(text) => {
let filter = self.rfc_filter(text)?;
for (m, child) in walk(f.val) {
if filter.test(&child, self)? {
self.child(f, &child, m, false, &mut ret)?;
}
}
}
Loc::Num(_) | Loc::Other(_) | Loc::ArrayKey(_) => {
let elem = match loc {
Loc::Num(n) => PathElem::Num(*n),
_ => PathElem::Str(loc_text(loc).into()),
};
match f.own(&elem.text()) {
Some(child) => self.child(f, &child, elem, false, &mut ret)?,
None if matches!(loc, Loc::ArrayKey(_)) => {}
None => {
return Err(format!(
"{} is not a property name the path can follow",
elem.text()
));
}
}
}
Loc::Str(s) | Loc::Part(s) => {
if let Some(early) = self.trace_str(f, s, matches!(loc, Loc::Str(_)), &mut ret)? {
return Ok(early);
}
}
}
if self.has_parent_selector.get() {
self.resolve_parent_selectors(f, &mut ret)?;
}
Ok(Traced::Many(ret))
}
/// A `^` below this frame resolves here, by tracing what follows it from this frame's value.
fn resolve_parent_selectors(
&self,
f: &Frame<'_, 'a>,
ret: &mut Vec<Option<Ret<'a>>>,
) -> Result<(), String> {
let mut t = 0;
while t < ret.len() {
let selector = ret[t]
.as_ref()
.and_then(|r| r.parent_selector.clone().map(|s| (s, r.path.clone())));
if let Some((rest, path)) = selector {
let frame = Frame { expr: &rest, path: &path, literal_priority: false, ..*f };
match self.trace(&frame)? {
Traced::One(r) => ret[t] = Some(r),
Traced::Many(items) => {
let mut items = items.into_iter();
ret[t] = items.next().flatten();
for item in items {
t += 1;
ret.insert(t, item);
}
}
}
}
t += 1;
}
Ok(())
}
/// `_trace` for a text step, in jsonpath-plus's order of tests. Returns
/// the result itself for the steps that end the trace (`^`, `~`, a type).
fn trace_str(
&self,
f: &Frame<'_, 'a>,
s: &str,
rfc_additions: bool,
ret: &mut Vec<Option<Ret<'a>>>,
) -> Result<Option<Traced<'a>>, String> {
let named = |key: &str| PathElem::Str(key.into());
if let Some(child) = f.own(s).filter(|_| f.literal_priority) {
self.child(f, &child, named(s), false, ret)?;
} else if s == "*" {
for (m, child) in walk(f.val) {
self.child(f, &child, m, true, ret)?;
}
} else if s == ".." {
self.again(f, f.rest(), ret)?;
for (m, child) in walk(f.val) {
if matches!(child, JsValue::Doc(_) | JsValue::Null) {
let path = push(f.path, m.clone());
let parent_prop = m.to_js();
let next = Frame {
val: &child,
path: &path,
parent: f.val,
parent_prop: &parent_prop,
literal_priority: false,
..*f
};
add(ret, self.trace(&next)?);
}
}
} else if s == "^" {
self.has_parent_selector.set(true);
let parent_path = f.path.as_ref().and_then(|n| n.prev.clone());
return Ok(Some(Traced::One(Ret {
path: parent_path,
value: JsValue::Undefined,
parent_selector: Some(f.rest().into()),
})));
} else if s == "~" {
return Ok(Some(Traced::One(Ret {
path: push(f.path, named("~")),
value: f.parent_prop.clone(),
parent_selector: None,
})));
} else if s == "$" {
let frame = Frame {
expr: f.rest(),
parent: &JsValue::Null,
parent_prop: &JsValue::Null,
literal_priority: false,
..*f
};
add(ret, self.trace(&frame)?);
} else if SLICE.is_match(s).map_err(|e| e.to_string())? {
if let JsValue::Doc(Json::Array(items)) = f.val {
self.trace_indexes(f, slice_indexes(s, items.len()), ret)?;
}
} else if s.starts_with("?(") {
self.trace_filter(f, s, ret)?;
} else if s.starts_with('(') {
let script = self.script(s)?;
let property = f.path.as_ref().map_or(JsValue::Undefined, |n| n.elem.to_js());
let result = self.run_script(
&script,
f.val,
property,
|| path_string(f.path),
f.parent,
f.parent_prop,
)?;
let next = match result {
JsValue::Undefined => Loc::Str("".into()),
JsValue::String(s) => Loc::Part(s.as_ref().into()),
JsValue::Number(n) => Loc::Num(n),
array @ (JsValue::Doc(Json::Array(_)) | JsValue::Array(_)) => {
Loc::ArrayKey(array.to_js_string().into())
}
other => Loc::Other(other.to_js_string().into()),
};
self.again(f, &Self::with_first(next, f.rest()), ret)?;
} else if s.starts_with('@') {
if value_type(s)?.matches(f.val) {
return Ok(Some(Traced::One(Ret {
path: f.path.clone(),
value: f.val.clone(),
parent_selector: None,
})));
}
} else if let Some((key, child)) =
s.strip_prefix('`').and_then(|key| f.own(key).map(|child| (key, child)))
{
self.child(f, &child, named(key), true, ret)?;
} else if s.contains(',') {
for part in s.split(',') {
self.again(f, &Self::with_first(Loc::Part(part.into()), f.rest()), ret)?;
}
} else if let Some(child) = f.own(s).filter(|_| !f.literal_priority) {
self.child(f, &child, named(s), true, ret)?;
} else if rfc_additions && let JsValue::Doc(Json::Array(items)) = f.val {
if let Some(index) = negative_index(s, items.len()) {
self.trace_indexes(f, vec![index], ret)?;
} else if let Some(indexes) = reverse_slice_indexes(s, items.len()) {
self.trace_indexes(f, indexes, ret)?;
}
}
Ok(None)
}
fn trace_filter(
&self,
f: &Frame<'_, 'a>,
loc: &str,
ret: &mut Vec<Option<Ret<'a>>>,
) -> Result<(), String> {
match &*self.filter(loc)? {
FilterPlan::Nested { key, inner } => {
for (m, child) in walk(f.val) {
let nested = if key.is_empty() {
child.clone()
} else if matches!(child, JsValue::Null) {
return Err(format!("Cannot read properties of null (reading '{key}')"));
} else {
own_property(&child, key).unwrap_or(JsValue::Undefined)
};
let frame = Frame {
expr: std::slice::from_ref(inner),
val: &nested,
literal_priority: false,
..*f
};
let found = match self.trace(&frame)? {
Traced::One(_) => true,
Traced::Many(items) => !items.is_empty(),
};
if found {
self.child(f, &child, m, false, ret)?;
}
}
}
FilterPlan::Script(script) => {
for (m, child) in walk(f.val) {
let at_path = || path_string(&push(f.path, m.clone()));
if self
.run_script(script, &child, m.to_js(), at_path, f.parent, f.parent_prop)?
.truthy()
{
self.child(f, &child, m, false, ret)?;
}
}
}
}
Ok(())
}
fn trace_indexes(
&self,
f: &Frame<'_, 'a>,
indexes: Vec<f64>,
ret: &mut Vec<Option<Ret<'a>>>,
) -> Result<(), String> {
for i in indexes {
self.again(f, &Self::with_first(Loc::Num(i), f.rest()), ret)?;
}
Ok(())
}
}
/// One `_trace` call's arguments.
#[derive(Clone, Copy)]
struct Frame<'f, 'a> {
expr: &'f [Loc],
val: &'f JsValue<'a>,
path: &'f Path,
parent: &'f JsValue<'a>,
parent_prop: &'f JsValue<'a>,
literal_priority: bool,
}
impl<'a> Frame<'_, 'a> {
fn rest(&self) -> &[Loc] {
self.expr.get(1..).unwrap_or_default()
}
/// `val && Object.hasOwn(val, key)`, and the property.
fn own(&self, key: &str) -> Option<JsValue<'a>> {
self.val.truthy().then(|| own_property(self.val, key)).flatten()
}
}
fn loc_text(loc: &Loc) -> String {
match loc {
Loc::Str(s) | Loc::Part(s) | Loc::Other(s) | Loc::ArrayKey(s) | Loc::Filter(s) => {
s.to_string()
}
Loc::Num(n) => number_to_string(*n),
Loc::Names(names) => names.join(","),
}
}
/// `_walk`: array indexes, or object keys in `JSON.parse` order.
fn walk<'a>(val: &JsValue<'a>) -> Vec<(PathElem, JsValue<'a>)> {
match val {
JsValue::Doc(Json::Array(items)) => items
.iter()
.enumerate()
.map(|(i, item)| (PathElem::Num(i as f64), JsValue::from_doc(item)))
.collect(),
JsValue::Doc(Json::Object(map)) => map
.iter()
.map(|(k, v)| (PathElem::Str(k.as_str().into()), JsValue::from_doc(v)))
.collect(),
_ => Vec::new(),
}
}
/// `_slice`: Python-style `start:end:step` with jsonpath-plus's defaults.
fn slice_indexes(loc: &str, len: usize) -> Vec<f64> {
let len = len as f64;
let parts: Vec<&str> = loc.split(':').collect();
let number = |s: &str| value::string_to_number(s);
let or_default = |n: f64, default: f64| if n == 0.0 || n.is_nan() { default } else { n };
let step = parts.get(2).filter(|s| !s.is_empty()).map_or(1.0, |s| or_default(number(s), 1.0));
let start = parts.first().filter(|s| !s.is_empty()).map_or(0.0, |s| or_default(number(s), 0.0));
let end = parts.get(1).filter(|s| !s.is_empty()).map_or(len, |s| number(s));
let mut indexes = Vec::new();
if end.is_nan() {
return indexes;
}
let start = if start < 0.0 { (start + len).max(0.0) } else { start.min(len) };
let end = if end < 0.0 { (end + len).max(0.0) } else { end.min(len) };
let mut i = start;
while i < end {
indexes.push(i);
i += step;
}
indexes
}
fn parse_int(s: &str) -> Option<i64> {
let digits = s.strip_prefix('-').unwrap_or(s);
if digits.is_empty() || !digits.bytes().all(|b| b.is_ascii_digit()) {
return None;
}
Some(s.parse::<i64>().unwrap_or(if s.starts_with('-') { i64::MIN } else { i64::MAX }))
}
/// RFC 9535 `[-n]`, which jsonpath-plus looks up as the key "-n".
fn negative_index(loc: &str, len: usize) -> Option<f64> {
let n = parse_int(loc).filter(|n| *n < 0 && !loc.starts_with("-0"))?;
let index = len as i64 + n;
(index >= 0).then_some(index as f64)
}
/// An RFC 9535 slice with a negative step, which jsonpath-plus does not read as a slice.
fn reverse_slice_indexes(loc: &str, len: usize) -> Option<Vec<f64>> {
let parts: Vec<&str> = loc.split(':').collect();
let [start, end, step] = parts[..] else {
return None;
};
let step = parse_int(step).filter(|s| *s < 0)?;
let bound = |s: &str| if s.is_empty() { Some(None) } else { parse_int(s).map(Some) };
let (start, end) = (bound(start)?, bound(end)?);
let len = len as i64;
let normalize = |i: i64| if i >= 0 { i } else { len.saturating_add(i) };
let upper = normalize(start.unwrap_or(len - 1)).clamp(-1, len - 1);
let lower = end.map_or(-1, |e| normalize(e).clamp(-1, len - 1));
let mut indexes = Vec::new();
let mut i = upper;
while lower < i {
indexes.push(i as f64);
i = i.saturating_add(step);
}
Some(indexes)
}
#[derive(Clone, Copy)]
enum ValueType {
Scalar,
Boolean,
String,
Undefined,
Function,
Integer,
Number,
NonFinite,
Object,
Array,
Null,
}
/// The `@string()`-style step, by jsonpath-plus's `loc.slice(1, -2)`.
fn value_type(loc: &str) -> Result<ValueType, String> {
let chars: Vec<char> = loc.chars().collect();
let name: String =
chars.get(1..chars.len().saturating_sub(2)).map(|c| c.iter().collect()).unwrap_or_default();
Ok(match name.as_str() {
"scalar" => ValueType::Scalar,
"boolean" => ValueType::Boolean,
"string" => ValueType::String,
"undefined" => ValueType::Undefined,
"function" => ValueType::Function,
"integer" => ValueType::Integer,
"number" => ValueType::Number,
"nonFinite" => ValueType::NonFinite,
"object" => ValueType::Object,
"array" => ValueType::Array,
"null" => ValueType::Null,
"other" => {
return Err(
"@other() needs an otherTypeCallback, which filters cannot supply".to_string()
);
}
name => return Err(format!("Unknown value type {name}")),
})
}
impl ValueType {
fn matches(self, val: &JsValue<'_>) -> bool {
match self {
ValueType::Scalar => !val.truthy() || !val.is_object(),
ValueType::Boolean => matches!(val, JsValue::Bool(_)),
ValueType::String => matches!(val, JsValue::String(_)),
ValueType::Undefined => matches!(val, JsValue::Undefined),
ValueType::Function => matches!(val, JsValue::Function(_)),
ValueType::Integer => {
matches!(val, JsValue::Number(n) if n.is_finite() && n.fract() == 0.0)
}
ValueType::Number => matches!(val, JsValue::Number(n) if n.is_finite()),
ValueType::NonFinite => matches!(val, JsValue::Number(n) if !n.is_finite()),
ValueType::Object => val.truthy() && val.type_of() == "object",
ValueType::Array => matches!(val, JsValue::Doc(Json::Array(_)) | JsValue::Array(_)),
ValueType::Null => matches!(val, JsValue::Null),
}
}
}
+213
View File
@@ -0,0 +1,213 @@
//! Path strings into the steps the engine traces: jsonpath-plus's
//! `toPathArray`, after lifting out what this engine reads differently.
use super::json::{self, Json};
use super::rfc::{closing_bracket, quoted_string};
use fancy_regex::{Captures, Regex};
use std::rc::Rc;
use std::sync::LazyLock;
/// One step of a path. jsonpath-plus decides what a step means while tracing,
/// so most steps stay text until then.
#[derive(Debug, Clone)]
pub enum Loc {
Str(Rc<str>),
/// A part of a `,` union or a script's result, which take none of the RFC 9535 additions.
Part(Rc<str>),
Num(f64),
/// A script result that is neither a string nor a number, by its string form.
Other(Rc<str>),
/// A script's array result, which jsonpath-plus follows only if its string form is a key.
ArrayKey(Rc<str>),
/// Quoted names: always literal keys.
Names(Rc<[String]>),
/// An RFC 9535 filter written without parentheses.
Filter(Rc<str>),
}
const PLACEHOLDER_OPEN: char = '\u{E000}';
const PLACEHOLDER_CLOSE: char = '\u{E001}';
static PROPERTY_TYPE: LazyLock<Regex> = LazyLock::new(|| regex(r"@[A-Za-z0-9_$\-]+\(\)"));
static SCRIPT: LazyLock<Regex> = LazyLock::new(|| {
regex(r"[\['](\??\([^\n\r\x{2028}\x{2029}]*?\))[\]'](?![^\n\r\x{2028}\x{2029}]\])")
});
static QUOTED: LazyLock<Regex> = LazyLock::new(|| regex(r#"\[['"]([^'\]]*)['"]\]"#));
static BOUNDARY: LazyLock<Regex> = LazyLock::new(|| regex(r#"['"]?\.['"]?(?![^\[]*\])|\[['"]?"#));
static PARENT: LazyLock<Regex> = LazyLock::new(|| regex(r"(?:;)?(\^+)(?:;)?"));
static DESCENDANT: LazyLock<Regex> = LazyLock::new(|| regex(r";;;|;;"));
static TRAILING: LazyLock<Regex> = LazyLock::new(|| regex(r";$|'?\]|'$"));
static SUBX: LazyLock<Regex> = LazyLock::new(|| regex(r"#([0-9]+)"));
fn regex(pattern: &str) -> Regex {
Regex::new(pattern).unwrap_or_else(|e| unreachable!("{pattern}: {e}"))
}
fn replace(
re: &Regex,
text: &str,
rep: impl FnMut(&Captures<'_, str>) -> String,
) -> Result<String, String> {
re.try_replacen(text, 0, rep)
.map(|s| s.into_owned())
.map_err(|e| format!("path is too complex: {e}"))
}
pub fn parse(path: &str) -> Result<Vec<Loc>, String> {
let (text, lifted) = if path.contains([PLACEHOLDER_OPEN, PLACEHOLDER_CLOSE]) {
(path.to_string(), Vec::new())
} else {
lift(path)
};
let mut subx: Vec<String> = Vec::new();
let normalized = replace(&PROPERTY_TYPE, &text, |c| format!(";{};", &c[0]))?;
let normalized = replace(&SCRIPT, &normalized, |c| {
subx.push(c[1].to_string());
format!("[#{}]", subx.len() - 1)
})?;
let normalized = replace(&QUOTED, &normalized, |c| {
format!("['{}']", c[1].replace('.', "%@%").replace('~', "%%@@%%"))
})?;
let normalized = normalized.replace('~', ";~;");
let normalized = replace(&BOUNDARY, &normalized, |_| ";".to_string())?;
let normalized = normalized.replace("%@%", ".").replace("%%@@%%", "~");
let normalized = replace(&PARENT, &normalized, |c| {
let ups: Vec<&str> = c[1].split("").filter(|s| !s.is_empty()).collect();
format!(";{};", ups.join(";"))
})?;
let normalized = replace(&DESCENDANT, &normalized, |_| ";..;".to_string())?;
let normalized = replace(&TRAILING, &normalized, |_| String::new())?;
normalized
.split(';')
.map(|part| {
if let Some(index) = placeholder_index(part) {
return Ok(lifted.get(index).cloned().unwrap_or_else(|| Loc::Str(part.into())));
}
let subx_index =
SUBX.captures(part).map_err(|e| e.to_string())?.map(|c| c[1].parse::<usize>().ok());
Ok(match subx_index {
None => Loc::Str(part.into()),
Some(index) => match index.and_then(|i| subx.get(i)) {
Some(script) => Loc::Str(script.as_str().into()),
None => Loc::Other("undefined".into()),
},
})
})
.collect()
}
/// The old plugin walked complete literal paths containing `["` by own
/// property instead of handing them to jsonpath-plus (so `$["s"].length` is
/// `0` for an empty string). These are their keys.
pub fn literal_keys(path: &str) -> Option<Vec<String>> {
if !path.contains("[\"") {
return None;
}
let mut rest = path.trim().strip_prefix('$')?;
let mut keys = Vec::new();
while !rest.is_empty() {
if let Some(after) = rest.strip_prefix('.') {
let len = after
.find(|c: char| !(c.is_ascii_alphanumeric() || c == '_' || c == '$'))
.unwrap_or(after.len());
let name = &after[..len];
if name.is_empty() || name == "$" || name.starts_with(|c: char| c.is_ascii_digit()) {
return None;
}
keys.push(name.to_string());
rest = &after[len..];
} else if let Some(after) = rest.strip_prefix("[\"") {
let mut escaped = false;
let end = after.char_indices().find_map(|(i, c)| {
let closes = c == '"' && !escaped;
escaped = c == '\\' && !escaped;
closes.then_some(i)
})?;
let Json::String(key) = json::parse(&format!("\"{}", &after[..=end])).ok()? else {
return None;
};
keys.push(key);
rest = after[end + 1..].strip_prefix(']')?;
} else if let Some(after) = rest.strip_prefix('[') {
let len = after.find(|c: char| !c.is_ascii_digit()).unwrap_or(after.len());
let digits = &after[..len];
let index = digits.parse::<u64>().ok().filter(|i| *i <= 9_007_199_254_740_991)?;
if digits.len() > 1 && digits.starts_with('0') {
return None;
}
keys.push(index.to_string());
rest = after[len..].strip_prefix(']')?;
} else {
return None;
}
}
Some(keys)
}
fn placeholder_index(part: &str) -> Option<usize> {
part.strip_prefix(PLACEHOLDER_OPEN)?.strip_suffix(PLACEHOLDER_CLOSE)?.parse().ok()
}
/// Lifts out quoted names (`["a.b"]`, `['a', 'b']`) and RFC 9535 filters
/// without parentheses (`[?@.a > 1]`), leaving placeholders jsonpath-plus's
/// normalization passes through as single steps.
fn lift(path: &str) -> (String, Vec<Loc>) {
let chars: Vec<char> = path.chars().collect();
let mut out = String::new();
let mut lifted = Vec::new();
let mut i = 0;
while i < chars.len() {
if chars[i] == '[' {
let next = chars.get(i + 1).copied();
let after = chars.get(i + 2).copied();
let replacement = match next {
Some('?') if after != Some('(') => closing_bracket(&chars, i).map(|close| {
let text: String = chars[i + 2..close - 1].iter().collect();
(Loc::Filter(text.into()), close)
}),
Some('?' | '(') => {
if let Some(close) = closing_bracket(&chars, i) {
out.extend(&chars[i..close]);
i = close;
continue;
}
None
}
_ => quoted_names(&chars, i + 1)
.map(|(names, close)| (Loc::Names(names.into()), close)),
};
if let Some((loc, close)) = replacement {
out.push_str(&format!("[{PLACEHOLDER_OPEN}{}{PLACEHOLDER_CLOSE}]", lifted.len()));
lifted.push(loc);
i = close;
continue;
}
}
out.push(chars[i]);
i += 1;
}
(out, lifted)
}
/// `"a"` or `'a', "b"` up to the closing `]`, and the index after it.
fn quoted_names(chars: &[char], mut i: usize) -> Option<(Vec<String>, usize)> {
let blank = |c: &char| matches!(c, ' ' | '\t' | '\n' | '\r');
let mut names = Vec::new();
loop {
while chars.get(i).is_some_and(blank) {
i += 1;
}
let (name, end) = quoted_string(chars, i)?;
names.push(name);
i = end;
while chars.get(i).is_some_and(blank) {
i += 1;
}
match chars.get(i) {
Some(']') => return Some((names, i + 1)),
Some(',') => i += 1,
_ => return None,
}
}
}
+367
View File
@@ -0,0 +1,367 @@
//! JavaScript regular expressions on `fancy-regex`.
use super::value::{JsArray, JsValue, MatchInfo};
use fancy_regex::{Captures, Regex, RegexBuilder};
use std::cell::Cell;
use std::fmt;
use std::rc::Rc;
const JS_WHITESPACE: &str =
r"\t\n\x0B\x0C\r \u00A0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F\u3000\uFEFF";
/// JavaScript's `\b` and `\B` only count ASCII word characters.
const WORD_BOUNDARY: &str =
r"(?:(?<=[0-9A-Za-z_])(?![0-9A-Za-z_])|(?<![0-9A-Za-z_])(?=[0-9A-Za-z_]))";
const NOT_WORD_BOUNDARY: &str =
r"(?:(?<=[0-9A-Za-z_])(?=[0-9A-Za-z_])|(?<![0-9A-Za-z_])(?![0-9A-Za-z_]))";
const NOT_LINE_TERMINATOR: &str = r"[^\n\r\u2028\u2029]";
pub struct JsRegex {
pub source: String,
pub flags: String,
regex: Regex,
global: bool,
/// In UTF-16 code units, as JavaScript counts it.
pub last_index: Cell<usize>,
}
impl fmt::Debug for JsRegex {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "/{}/{}", self.source, self.flags)
}
}
impl JsRegex {
pub fn new(source: &str, flags: &str) -> Result<Self, String> {
let mut seen = String::new();
for flag in flags.chars() {
if seen.contains(flag) || !"dgimsuvy".contains(flag) {
return Err(format!("Invalid flags supplied to RegExp constructor '{flags}'"));
}
if flag == 'v' || flag == 'y' {
return Err(format!("the regular expression flag {flag} is not supported"));
}
seen.push(flag);
}
let dot_all = flags.contains('s');
let translated = translate(source, dot_all)?;
let regex = RegexBuilder::new(&translated)
.case_insensitive(flags.contains('i'))
.multi_line(flags.contains('m'))
.build()
.map_err(|e| format!("Invalid regular expression: /{source}/{flags}: {e}"))?;
Ok(JsRegex {
source: if source.is_empty() { "(?:)".to_string() } else { source.to_string() },
flags: flags.to_string(),
regex,
global: flags.contains('g'),
last_index: Cell::new(0),
})
}
/// An RFC 9535 `match()` or `search()` pattern; `None` when it is not one this engine reads.
pub fn i_regexp(pattern: &str, whole: bool) -> Option<Self> {
let source = if whole { format!("^(?:{pattern})$") } else { pattern.to_string() };
JsRegex::new(&source, "u").ok()
}
pub fn is_match(&self, text: &str) -> Result<bool, String> {
self.regex.is_match(text).map_err(runtime_error)
}
pub fn test(&self, text: &str) -> Result<bool, String> {
if !self.global {
return self.is_match(text);
}
let Some(start) = utf16_to_byte(text, self.last_index.get()) else {
self.last_index.set(0);
return Ok(false);
};
match self.regex.find_from_pos(text, start).map_err(runtime_error)? {
Some(m) => {
self.last_index.set(byte_to_utf16(text, m.end()));
Ok(true)
}
None => {
self.last_index.set(0);
Ok(false)
}
}
}
pub fn match_string<'a>(&self, text: &str) -> Result<JsValue<'a>, String> {
if self.global {
self.last_index.set(0);
let mut matches = Vec::new();
let mut pos = 0;
while pos <= text.len() {
let Some(m) = self.regex.find_from_pos(text, pos).map_err(runtime_error)? else {
break;
};
matches.push(JsValue::string(m.as_str()));
pos = if m.end() == m.start() { next_boundary(text, m.end()) } else { m.end() };
}
return Ok(if matches.is_empty() { JsValue::Null } else { JsValue::array(matches) });
}
let Some(captures) = self.regex.captures(text).map_err(runtime_error)? else {
return Ok(JsValue::Null);
};
let index = captures.get(0).map_or(0, |m| byte_to_utf16(text, m.start()));
Ok(JsValue::Array(Rc::new(JsArray {
items: capture_values(&captures, 0),
match_info: Some(MatchInfo { index: index as f64, input: text.to_string() }),
})))
}
/// `String.prototype.split` with a regular expression separator, captures included.
pub fn split<'a>(&self, text: &str, limit: usize) -> Result<Vec<JsValue<'a>>, String> {
let mut parts = Vec::new();
if limit == 0 {
return Ok(parts);
}
if text.is_empty() {
if self.regex.find(text).map_err(runtime_error)?.is_none() {
parts.push(JsValue::string(""));
}
return Ok(parts);
}
let (mut p, mut q) = (0, 0);
while q < text.len() {
let Some(captures) = self.regex.captures_from_pos(text, q).map_err(runtime_error)?
else {
break;
};
let Some(whole) = captures.get(0) else { break };
q = whole.start();
if q >= text.len() {
break;
}
let e = whole.end().min(text.len());
if e == p {
q = next_boundary(text, q);
continue;
}
parts.push(JsValue::string(&text[p..q]));
if parts.len() == limit {
return Ok(parts);
}
p = e;
for capture in capture_values(&captures, 1) {
parts.push(capture);
if parts.len() == limit {
return Ok(parts);
}
}
q = p;
}
parts.push(JsValue::string(&text[p..]));
Ok(parts)
}
}
fn capture_values<'a>(captures: &Captures<'_, str>, from: usize) -> Vec<JsValue<'a>> {
(from..captures.len())
.map(|i| captures.get(i).map_or(JsValue::Undefined, |m| JsValue::string(m.as_str())))
.collect()
}
fn runtime_error(e: fancy_regex::Error) -> String {
format!("regular expression failed: {e}")
}
fn next_boundary(text: &str, pos: usize) -> usize {
text[pos..].chars().next().map_or(pos + 1, |c| pos + c.len_utf8())
}
fn utf16_to_byte(text: &str, index: usize) -> Option<usize> {
let mut units = 0;
for (byte, c) in text.char_indices() {
if units >= index {
return Some(byte);
}
units += c.len_utf16();
}
(units >= index).then_some(text.len())
}
fn byte_to_utf16(text: &str, byte: usize) -> usize {
text[..byte].encode_utf16().count()
}
/// Rewrites a JavaScript pattern into `fancy-regex` syntax with JavaScript's
/// meaning for `.`, `\d`, `\w`, `\s`, `\b` and the characters JavaScript
/// takes literally (`]`, `{` and `}` outside a quantifier, `[` inside a class).
fn translate(source: &str, dot_all: bool) -> Result<String, String> {
let chars: Vec<char> = source.chars().collect();
let mut out = String::new();
let mut in_class = false;
let mut i = 0;
while i < chars.len() {
let c = chars[i];
i += 1;
match c {
'\\' => {
let Some(&e) = chars.get(i) else {
return Err("Invalid regular expression: \\ at end of pattern".to_string());
};
i += 1;
match e {
'd' => out.push_str(if in_class { "0-9" } else { "[0-9]" }),
'D' => out.push_str("[^0-9]"),
'w' => out.push_str(if in_class { "0-9A-Za-z_" } else { "[0-9A-Za-z_]" }),
'W' => out.push_str("[^0-9A-Za-z_]"),
's' if in_class => out.push_str(JS_WHITESPACE),
's' => out.push_str(&format!("[{JS_WHITESPACE}]")),
'S' => out.push_str(&format!("[^{JS_WHITESPACE}]")),
'b' if in_class => out.push_str(r"\x08"),
'b' => out.push_str(WORD_BOUNDARY),
'B' => out.push_str(NOT_WORD_BOUNDARY),
'n' => out.push_str(r"\n"),
'r' => out.push_str(r"\r"),
't' => out.push_str(r"\t"),
'f' => out.push_str(r"\x0C"),
'v' => out.push_str(r"\x0B"),
'0' if !chars.get(i).is_some_and(char::is_ascii_digit) => out.push_str(r"\x00"),
'1'..='9' if !in_class => {
out.push('\\');
out.push(e);
while let Some(d) = chars.get(i).filter(|d| d.is_ascii_digit()) {
out.push(*d);
i += 1;
}
}
'x' if hex_run(&chars, i, 2) => {
out.push_str(&format!(
r"\x{{{}}}",
chars[i..i + 2].iter().collect::<String>()
));
i += 2;
}
'u' if chars.get(i) == Some(&'{') => {
let end = chars[i..].iter().position(|c| *c == '}').map(|p| p + i);
let Some(end) = end else {
return Err(
"Invalid regular expression: invalid Unicode escape".to_string()
);
};
out.push_str(&format!(
r"\x{{{}}}",
chars[i + 1..end].iter().collect::<String>()
));
i = end + 1;
}
'u' if hex_run(&chars, i, 4) => {
let high =
u32::from_str_radix(&chars[i..i + 4].iter().collect::<String>(), 16)
.unwrap_or(0);
i += 4;
let mut code = high;
if (0xD800..0xDC00).contains(&high)
&& chars.get(i) == Some(&'\\')
&& chars.get(i + 1) == Some(&'u')
&& hex_run(&chars, i + 2, 4)
{
let low = u32::from_str_radix(
&chars[i + 2..i + 6].iter().collect::<String>(),
16,
)
.unwrap_or(0);
if (0xDC00..0xE000).contains(&low) {
code = 0x10000 + ((high - 0xD800) << 10) + (low - 0xDC00);
i += 6;
}
}
out.push_str(&format!(r"\x{{{code:X}}}"));
}
'c' if chars.get(i).is_some_and(char::is_ascii_alphabetic) => {
out.push_str(&format!(r"\x{{{:X}}}", (chars[i] as u32) % 32));
i += 1;
}
'k' if chars.get(i) == Some(&'<') => out.push_str(r"\k"),
'p' | 'P' if chars.get(i) == Some(&'{') => {
let end = chars[i..]
.iter()
.position(|c| *c == '}')
.map_or(chars.len() - 1, |p| p + i);
out.push('\\');
out.push(e);
out.extend(&chars[i..=end]);
i = end + 1;
}
e => push_literal(&mut out, e),
}
}
'[' if in_class => out.push_str(r"\["),
'[' => {
let negated = chars.get(i) == Some(&'^');
let body = if negated { i + 1 } else { i };
if chars.get(body) == Some(&']') {
out.push_str(if negated { r"[\s\S]" } else { r"[^\s\S]" });
i = body + 1;
} else {
in_class = true;
out.push('[');
if negated {
out.push('^');
i += 1;
}
}
}
']' if in_class => {
in_class = false;
out.push(']');
}
'&' | '~' if in_class => {
out.push('\\');
out.push(c);
}
']' | '}' => push_literal(&mut out, c),
'{' if !in_class => {
let quantifier = quantifier_len(&chars[i..]);
if quantifier > 0 && !out.is_empty() {
out.push('{');
out.extend(&chars[i..i + quantifier]);
i += quantifier;
} else {
push_literal(&mut out, '{');
}
}
'.' if !in_class && !dot_all => out.push_str(NOT_LINE_TERMINATOR),
'.' if !in_class => out.push_str(r"[\s\S]"),
'(' if !in_class && chars.get(i) == Some(&'?') && chars.get(i + 1) == Some(&'<') => {
if matches!(chars.get(i + 2), Some('=' | '!')) {
out.push_str("(?<");
} else {
out.push_str("(?P<");
}
i += 2;
}
c => out.push(c),
}
}
Ok(out)
}
fn push_literal(out: &mut String, c: char) {
if c.is_ascii_punctuation() || c == ' ' {
out.push('\\');
}
out.push(c);
}
fn hex_run(chars: &[char], from: usize, len: usize) -> bool {
chars.get(from..from + len).is_some_and(|run| run.iter().all(char::is_ascii_hexdigit))
}
/// The length of `n}`, `n,}` or `n,m}` after a `{`, or 0 when JavaScript reads the brace literally.
fn quantifier_len(rest: &[char]) -> usize {
let digits = |from: usize| rest[from..].iter().take_while(|c| c.is_ascii_digit()).count();
let mut i = digits(0);
if i == 0 {
return 0;
}
if rest.get(i) == Some(&',') {
i += 1;
i += digits(i);
}
if rest.get(i) == Some(&'}') { i + 1 } else { 0 }
}
+711
View File
@@ -0,0 +1,711 @@
//! RFC 9535 filter expressions, for `[?expr]` written without parentheses,
//! and its function extensions, which `?(...)` filters may call too.
use super::Engine;
use super::json::Json;
use super::value::JsValue;
const MAX_DEPTH: usize = 100;
/// An embedded query: `@` or `$` and the segments after it, run by the engine.
#[derive(Debug, Clone)]
pub struct Query {
pub root: bool,
pub path: String,
pub singular: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RfcFunction {
Length,
Count,
Match,
Search,
Value,
}
impl RfcFunction {
pub fn from_name(name: &str) -> Option<Self> {
match name {
"length" => Some(RfcFunction::Length),
"count" => Some(RfcFunction::Count),
"match" => Some(RfcFunction::Match),
"search" => Some(RfcFunction::Search),
"value" => Some(RfcFunction::Value),
_ => None,
}
}
pub fn name(self) -> &'static str {
match self {
RfcFunction::Length => "length",
RfcFunction::Count => "count",
RfcFunction::Match => "match",
RfcFunction::Search => "search",
RfcFunction::Value => "value",
}
}
pub fn takes_nodes(self) -> bool {
match self {
RfcFunction::Count | RfcFunction::Value => true,
RfcFunction::Length | RfcFunction::Match | RfcFunction::Search => false,
}
}
pub fn arity(self) -> usize {
match self {
RfcFunction::Length | RfcFunction::Count | RfcFunction::Value => 1,
RfcFunction::Match | RfcFunction::Search => 2,
}
}
fn returns_logical(self) -> bool {
match self {
RfcFunction::Match | RfcFunction::Search => true,
RfcFunction::Length | RfcFunction::Count | RfcFunction::Value => false,
}
}
/// The function's result; `None` is RFC 9535's Nothing.
fn call<'a>(
self,
args: &[Option<JsValue<'a>>],
nodes: &[JsValue<'a>],
engine: &Engine<'a>,
) -> Option<JsValue<'a>> {
let arg = |i: usize| args.get(i).cloned().flatten();
match self {
RfcFunction::Length => match arg(0)? {
JsValue::String(s) => Some(JsValue::Number(s.chars().count() as f64)),
JsValue::Doc(Json::Array(items)) => Some(JsValue::Number(items.len() as f64)),
JsValue::Doc(Json::Object(map)) => Some(JsValue::Number(map.len() as f64)),
JsValue::Array(array) => Some(JsValue::Number(array.items.len() as f64)),
_ => None,
},
RfcFunction::Count => Some(JsValue::Number(nodes.len() as f64)),
RfcFunction::Value => match nodes {
[one] => Some(one.clone()),
_ => None,
},
RfcFunction::Match | RfcFunction::Search => {
let matched = match (arg(0), arg(1)) {
(Some(JsValue::String(text)), Some(JsValue::String(pattern))) => engine
.i_regexp(&pattern, self == RfcFunction::Match)
.is_some_and(|re| re.is_match(&text).unwrap_or(false)),
_ => false,
};
Some(JsValue::Bool(matched))
}
}
}
/// A call from a `?(...)` filter, where Nothing is `undefined`.
pub fn call_js<'a>(self, args: &[Vec<JsValue<'a>>], engine: &Engine<'a>) -> JsValue<'a> {
let nodes =
if self.takes_nodes() { args.first().cloned().unwrap_or_default() } else { Vec::new() };
let values: Vec<Option<JsValue<'a>>> = if self.takes_nodes() {
Vec::new()
} else {
args.iter()
.map(|arg| arg.first().cloned().filter(|v| !matches!(v, JsValue::Undefined)))
.collect()
};
self.call(&values, &nodes, engine).unwrap_or(JsValue::Undefined)
}
}
fn is_name_first(c: char) -> bool {
c.is_ascii_alphabetic() || c == '_' || (c as u32) >= 0x80
}
fn is_name_char(c: char) -> bool {
is_name_first(c) || c.is_ascii_digit()
}
fn is_blank(c: char) -> bool {
matches!(c, ' ' | '\t' | '\n' | '\r')
}
/// The index after the `]` that closes the bracket opened at `open`, skipping quoted text.
pub fn closing_bracket(chars: &[char], open: usize) -> Option<usize> {
let mut depth = 0;
let mut i = open;
while i < chars.len() {
match chars[i] {
'[' => depth += 1,
']' => {
depth -= 1;
if depth == 0 {
return Some(i + 1);
}
}
quote @ ('\'' | '"') => {
i += 1;
while i < chars.len() && chars[i] != quote {
i += if chars[i] == '\\' { 2 } else { 1 };
}
}
_ => {}
}
i += 1;
}
None
}
/// Reads the segments after `@` or `$` from `start`, returning where they end.
pub fn scan_query(chars: &[char], start: usize, root: bool) -> (usize, Query) {
let mut end = start;
let mut singular = true;
loop {
let mut i = end;
while chars.get(i).copied().is_some_and(is_blank) {
i += 1;
}
if chars.get(i) == Some(&'.') && chars.get(i + 1) == Some(&'.') {
i += 2;
singular = false;
match chars.get(i) {
Some('*') => i += 1,
Some('[') => match closing_bracket(chars, i) {
Some(close) => i = close,
None => break,
},
Some(c) if is_name_first(*c) => i = name_end(chars, i),
_ => break,
}
} else if chars.get(i) == Some(&'.') {
i += 1;
match chars.get(i) {
Some('*') => {
singular = false;
i += 1;
}
Some(c) if is_name_first(*c) => i = name_end(chars, i),
_ => break,
}
} else if chars.get(i) == Some(&'[') {
let Some(close) = closing_bracket(chars, i) else {
break;
};
let selector: String = chars[i + 1..close - 1].iter().collect();
singular &= is_singular_selector(selector.trim_matches(is_blank));
i = close;
} else {
break;
}
end = i;
}
let path = format!("${}", chars[start..end].iter().collect::<String>());
(end, Query { root, path, singular })
}
fn name_end(chars: &[char], mut i: usize) -> usize {
while chars.get(i).copied().is_some_and(is_name_char) {
i += 1;
}
i
}
fn is_singular_selector(selector: &str) -> bool {
let digits = selector.strip_prefix('-').unwrap_or(selector);
let index = !digits.is_empty() && digits.bytes().all(|b| b.is_ascii_digit());
let chars: Vec<char> = selector.chars().collect();
let quoted = quoted_string(&chars, 0).is_some_and(|(_, end)| end == chars.len());
index || quoted
}
#[derive(Debug)]
pub enum Logical {
Or(Vec<Logical>),
And(Vec<Logical>),
Not(Box<Logical>),
Compare(Comparable, CompareOp, Comparable),
Exists(Query),
Test(Call),
}
#[derive(Debug, Clone, Copy)]
pub enum CompareOp {
Eq,
Ne,
Lt,
Le,
Gt,
Ge,
}
#[derive(Debug)]
pub enum Comparable {
Literal(Literal),
Query(Query),
Call(Call),
}
#[derive(Debug)]
pub enum Literal {
Null,
Bool(bool),
Number(f64),
String(String),
}
#[derive(Debug)]
pub struct Call {
function: RfcFunction,
args: Vec<Arg>,
}
#[derive(Debug)]
enum Arg {
Value(Comparable),
Nodes(Query),
}
pub fn parse(text: &str) -> Result<Logical, String> {
let mut parser = Parser { chars: text.chars().collect(), index: 0, depth: 0 };
parser.blank();
let logical = parser.or()?;
parser.blank();
if parser.index < parser.chars.len() {
return Err(parser.error("unexpected"));
}
Ok(logical)
}
struct Parser {
chars: Vec<char>,
index: usize,
depth: usize,
}
impl Parser {
fn peek(&self) -> Option<char> {
self.chars.get(self.index).copied()
}
fn eat(&mut self, s: &str) -> bool {
let matches =
s.chars().enumerate().all(|(i, c)| self.chars.get(self.index + i) == Some(&c));
if matches {
self.index += s.chars().count();
}
matches
}
fn blank(&mut self) {
while self.peek().is_some_and(is_blank) {
self.index += 1;
}
}
fn error(&self, message: &str) -> String {
let rest: String = self.chars[self.index.min(self.chars.len())..].iter().collect();
if rest.is_empty() {
format!("{message} end of filter expression")
} else {
format!("{message} \"{rest}\" in filter expression")
}
}
fn or(&mut self) -> Result<Logical, String> {
self.depth += 1;
if self.depth > MAX_DEPTH {
return Err("filter expression is nested too deeply".to_string());
}
let mut items = vec![self.and()?];
loop {
self.blank();
if !self.eat("||") {
break;
}
self.blank();
items.push(self.and()?);
}
self.depth -= 1;
Ok(if items.len() == 1 { items.remove(0) } else { Logical::Or(items) })
}
fn and(&mut self) -> Result<Logical, String> {
let mut items = vec![self.basic()?];
loop {
self.blank();
if !self.eat("&&") {
break;
}
self.blank();
items.push(self.basic()?);
}
Ok(if items.len() == 1 { items.remove(0) } else { Logical::And(items) })
}
fn basic(&mut self) -> Result<Logical, String> {
if self.peek() == Some('!') && self.chars.get(self.index + 1) != Some(&'=') {
self.index += 1;
self.blank();
let inner = if self.peek() == Some('(') { self.paren()? } else { self.test()? };
return Ok(Logical::Not(Box::new(inner)));
}
if self.peek() == Some('(') {
return self.paren();
}
let left = self.comparable()?;
let save = self.index;
self.blank();
let op = [
("==", CompareOp::Eq),
("!=", CompareOp::Ne),
("<=", CompareOp::Le),
(">=", CompareOp::Ge),
("<", CompareOp::Lt),
(">", CompareOp::Gt),
]
.into_iter()
.find(|(text, _)| self.eat(text));
let Some((_, op)) = op else {
self.index = save;
return self.as_test(left);
};
self.blank();
let right = self.comparable()?;
for side in [&left, &right] {
check_comparable(side)?;
}
Ok(Logical::Compare(left, op, right))
}
fn paren(&mut self) -> Result<Logical, String> {
self.index += 1;
self.blank();
let inner = self.or()?;
self.blank();
if !self.eat(")") {
return Err(self.error("expected `)` before"));
}
Ok(inner)
}
fn test(&mut self) -> Result<Logical, String> {
let comparable = self.comparable()?;
self.as_test(comparable)
}
fn as_test(&self, comparable: Comparable) -> Result<Logical, String> {
match comparable {
Comparable::Query(query) => Ok(Logical::Exists(query)),
Comparable::Call(call) if call.function.returns_logical() => Ok(Logical::Test(call)),
Comparable::Call(call) => Err(format!(
"{}() gives a value; compare it rather than test it",
call.function.name()
)),
Comparable::Literal(_) => {
Err(self.error("a literal is not a test; expected a comparison before"))
}
}
}
fn comparable(&mut self) -> Result<Comparable, String> {
match self.peek() {
Some(c @ ('@' | '$')) => {
let (end, query) = scan_query(&self.chars, self.index + 1, c == '$');
self.index = end;
Ok(Comparable::Query(query))
}
Some('\'' | '"') => Ok(Comparable::Literal(Literal::String(self.string()?))),
Some(c) if c == '-' || c.is_ascii_digit() => {
Ok(Comparable::Literal(Literal::Number(self.number()?)))
}
Some(c) if c.is_ascii_lowercase() => {
let start = self.index;
while self
.peek()
.is_some_and(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_')
{
self.index += 1;
}
let name: String = self.chars[start..self.index].iter().collect();
if self.peek() == Some('(') {
return self.call(&name).map(Comparable::Call);
}
match name.as_str() {
"true" => Ok(Comparable::Literal(Literal::Bool(true))),
"false" => Ok(Comparable::Literal(Literal::Bool(false))),
"null" => Ok(Comparable::Literal(Literal::Null)),
_ => {
self.index = start;
Err(self.error("unexpected"))
}
}
}
_ => Err(self.error("expected a query, literal or function at")),
}
}
fn call(&mut self, name: &str) -> Result<Call, String> {
let function =
RfcFunction::from_name(name).ok_or_else(|| format!("unknown function {name}()"))?;
self.depth += 1;
if self.depth > MAX_DEPTH {
return Err("filter expression is nested too deeply".to_string());
}
self.index += 1;
let mut args = Vec::new();
loop {
self.blank();
if args.is_empty() && self.eat(")") {
break;
}
let arg = self.comparable()?;
args.push(if function.takes_nodes() {
match arg {
Comparable::Query(query) => Arg::Nodes(query),
Comparable::Literal(_) | Comparable::Call(_) => {
return Err(format!("{}() takes a query", function.name()));
}
}
} else {
check_comparable(&arg)?;
Arg::Value(arg)
});
self.blank();
if self.eat(")") {
break;
}
if !self.eat(",") {
return Err(self.error("expected `,` or `)` before"));
}
}
if args.len() != function.arity() {
return Err(format!("{}() takes {} argument(s)", function.name(), function.arity()));
}
self.depth -= 1;
Ok(Call { function, args })
}
fn number(&mut self) -> Result<f64, String> {
let start = self.index;
self.eat("-");
let digits = |p: &mut Parser| {
let from = p.index;
while p.peek().is_some_and(|c| c.is_ascii_digit()) {
p.index += 1;
}
p.index - from
};
let int_start = self.index;
if digits(self) == 0 || (self.chars[int_start] == '0' && self.index - int_start > 1) {
return Err(self.error("invalid number at"));
}
if self.eat(".") && digits(self) == 0 {
return Err(self.error("invalid number at"));
}
if self.peek().is_some_and(|c| c == 'e' || c == 'E') {
self.index += 1;
if !self.eat("+") {
self.eat("-");
}
if digits(self) == 0 {
return Err(self.error("invalid number at"));
}
}
let text: String = self.chars[start..self.index].iter().collect();
text.parse::<f64>().map_err(|_| self.error("invalid number at"))
}
fn string(&mut self) -> Result<String, String> {
quoted_string(&self.chars, self.index)
.map(|(value, end)| {
self.index = end;
value
})
.ok_or_else(|| self.error("invalid string literal at"))
}
}
/// A quoted name with JSON escapes (and `\'`), and the index after its closing quote.
pub fn quoted_string(chars: &[char], start: usize) -> Option<(String, usize)> {
let quote = *chars.get(start).filter(|c| matches!(c, '\'' | '"'))?;
let mut out = String::new();
let mut i = start + 1;
loop {
let c = *chars.get(i)?;
i += 1;
if c == quote {
return Some((out, i));
}
if c != '\\' {
out.push(c);
continue;
}
let escaped = *chars.get(i)?;
i += 1;
match escaped {
'"' | '\'' | '\\' | '/' => out.push(escaped),
'b' => out.push('\u{8}'),
'f' => out.push('\u{c}'),
'n' => out.push('\n'),
'r' => out.push('\r'),
't' => out.push('\t'),
'u' => {
let hex = |at: usize| -> Option<u16> {
let digits: String = chars.get(at..at + 4)?.iter().collect();
if !digits.chars().all(|c| c.is_ascii_hexdigit()) {
return None;
}
u16::from_str_radix(&digits, 16).ok()
};
let mut units = vec![hex(i)?];
i += 4;
let paired = (0xD800..0xDC00).contains(&units[0])
&& chars.get(i) == Some(&'\\')
&& chars.get(i + 1) == Some(&'u');
if let Some(low) = hex(i + 2).filter(|low| paired && (0xDC00..0xE000).contains(low))
{
units.push(low);
i += 6;
}
out.push_str(&String::from_utf16_lossy(&units));
}
_ => return None,
}
}
}
fn check_comparable(comparable: &Comparable) -> Result<(), String> {
match comparable {
Comparable::Query(query) if !query.singular => Err(format!(
"{}{} can select several nodes, so it cannot be compared",
if query.root { "$" } else { "@" },
&query.path[1..]
)),
Comparable::Call(call) if call.function.returns_logical() => {
Err(format!("{}() gives a test result, not a value to compare", call.function.name()))
}
Comparable::Query(_) | Comparable::Call(_) | Comparable::Literal(_) => Ok(()),
}
}
impl Logical {
pub fn test<'a>(&self, current: &JsValue<'a>, engine: &Engine<'a>) -> Result<bool, String> {
Ok(match self {
Logical::Or(items) => {
for item in items {
if item.test(current, engine)? {
return Ok(true);
}
}
false
}
Logical::And(items) => {
for item in items {
if !item.test(current, engine)? {
return Ok(false);
}
}
true
}
Logical::Not(inner) => !inner.test(current, engine)?,
Logical::Exists(query) => !engine.query(query, current)?.is_empty(),
Logical::Test(call) => call.eval(current, engine)?.is_some_and(|v| v.truthy()),
Logical::Compare(left, op, right) => {
let left = left.eval(current, engine)?;
let right = right.eval(current, engine)?;
let (l, r) = (left.as_ref(), right.as_ref());
match op {
CompareOp::Eq => equal(l, r),
CompareOp::Ne => !equal(l, r),
CompareOp::Lt => less(l, r),
CompareOp::Le => less(l, r) || equal(l, r),
CompareOp::Gt => less(r, l),
CompareOp::Ge => less(r, l) || equal(l, r),
}
}
})
}
}
impl Comparable {
fn eval<'a>(
&self,
current: &JsValue<'a>,
engine: &Engine<'a>,
) -> Result<Option<JsValue<'a>>, String> {
Ok(match self {
Comparable::Literal(Literal::Null) => Some(JsValue::Null),
Comparable::Literal(Literal::Bool(b)) => Some(JsValue::Bool(*b)),
Comparable::Literal(Literal::Number(n)) => Some(JsValue::Number(*n)),
Comparable::Literal(Literal::String(s)) => Some(JsValue::string(s.clone())),
Comparable::Query(query) => match engine.query(query, current)?.as_slice() {
[one] => Some(one.clone()),
_ => None,
},
Comparable::Call(call) => call.eval(current, engine)?,
})
}
}
impl Call {
fn eval<'a>(
&self,
current: &JsValue<'a>,
engine: &Engine<'a>,
) -> Result<Option<JsValue<'a>>, String> {
let mut values = Vec::new();
let mut nodes = Vec::new();
for arg in &self.args {
match arg {
Arg::Value(comparable) => values.push(comparable.eval(current, engine)?),
Arg::Nodes(query) => nodes = engine.query(query, current)?,
}
}
Ok(self.function.call(&values, &nodes, engine))
}
}
fn equal(a: Option<&JsValue<'_>>, b: Option<&JsValue<'_>>) -> bool {
match (a, b) {
(None, None) => true,
(Some(a), Some(b)) => deep_equal(a, b),
_ => false,
}
}
fn deep_equal(a: &JsValue<'_>, b: &JsValue<'_>) -> bool {
match (a, b) {
(JsValue::Doc(a), JsValue::Doc(b)) => json_equal(a, b),
(JsValue::Null, JsValue::Null) => true,
(JsValue::Bool(a), JsValue::Bool(b)) => a == b,
(JsValue::Number(a), JsValue::Number(b)) => a == b,
(JsValue::String(a), JsValue::String(b)) => a == b,
_ => false,
}
}
fn json_equal(a: &Json, b: &Json) -> bool {
match (a, b) {
(Json::Array(a), Json::Array(b)) => {
a.len() == b.len() && a.iter().zip(b).all(|(a, b)| json_equal(a, b))
}
(Json::Object(a), Json::Object(b)) => {
a.len() == b.len() && a.iter().all(|(k, v)| b.get(k).is_some_and(|w| json_equal(v, w)))
}
(Json::Null, Json::Null) => true,
(Json::Bool(a), Json::Bool(b)) => a == b,
(Json::Number(a), Json::Number(b)) => a == b,
(Json::String(a), Json::String(b)) => a == b,
(Json::Null, _)
| (Json::Bool(_), _)
| (Json::Number(_), _)
| (Json::String(_), _)
| (Json::Array(_), _)
| (Json::Object(_), _) => false,
}
}
fn less(a: Option<&JsValue<'_>>, b: Option<&JsValue<'_>>) -> bool {
match (a, b) {
(Some(JsValue::Number(a)), Some(JsValue::Number(b))) => a < b,
(Some(JsValue::String(a)), Some(JsValue::String(b))) => a < b,
_ => false,
}
}
+803
View File
@@ -0,0 +1,803 @@
//! jsonpath-plus's safe evaluator: jsep's grammar (with its regex plugin,
//! `typeof`, `void`, `undefined`) and Safe-Script's semantics, minus what it
//! would let a filter do beyond reading the document.
use super::Engine;
use super::regex::JsRegex;
use super::rfc::{Query, RfcFunction, scan_query};
use super::value::{JsValue, call_method, get_property, is_allowed_method_name, is_js_whitespace};
use std::rc::Rc;
const MAX_DEPTH: usize = 100;
#[derive(Debug)]
pub enum Expr {
Literal(Literal),
Ident(String),
Member(Box<Expr>, Property),
Call(Box<Expr>, Vec<Expr>),
Rfc(RfcFunction, Vec<RfcArg>),
Array(Vec<Expr>),
Unary(UnaryOp, Box<Expr>),
Binary(BinaryOp, Box<Expr>, Box<Expr>),
Conditional(Box<Expr>, Box<Expr>, Box<Expr>),
}
#[derive(Debug)]
pub enum Literal {
Undefined,
Null,
Bool(bool),
Number(f64),
String(String),
Regex(Rc<JsRegex>),
}
#[derive(Debug)]
pub enum Property {
Name(String),
Computed(Box<Expr>),
}
#[derive(Debug)]
pub enum RfcArg {
Value(Expr),
Nodes(Query),
}
#[derive(Clone, Copy, Debug)]
pub enum UnaryOp {
Neg,
Plus,
Not,
BitNot,
TypeOf,
Void,
}
#[derive(Clone, Copy, Debug)]
pub enum BinaryOp {
Or,
And,
BitOr,
BitXor,
BitAnd,
Eq,
Ne,
StrictEq,
StrictNe,
Lt,
Gt,
Le,
Ge,
Shl,
Shr,
UShr,
Add,
Sub,
Mul,
Div,
Rem,
}
impl BinaryOp {
fn precedence(self) -> u8 {
match self {
BinaryOp::Or => 1,
BinaryOp::And => 2,
BinaryOp::BitOr => 3,
BinaryOp::BitXor => 4,
BinaryOp::BitAnd => 5,
BinaryOp::Eq | BinaryOp::Ne | BinaryOp::StrictEq | BinaryOp::StrictNe => 6,
BinaryOp::Lt | BinaryOp::Gt | BinaryOp::Le | BinaryOp::Ge => 7,
BinaryOp::Shl | BinaryOp::Shr | BinaryOp::UShr => 8,
BinaryOp::Add | BinaryOp::Sub => 9,
BinaryOp::Mul | BinaryOp::Div | BinaryOp::Rem => 10,
}
}
}
const BINARY_OPS: &[(&str, Option<BinaryOp>)] = &[
(">>>=", None),
("===", Some(BinaryOp::StrictEq)),
("!==", Some(BinaryOp::StrictNe)),
(">>>", Some(BinaryOp::UShr)),
("**=", None),
("<<=", None),
(">>=", None),
("||=", None),
("&&=", None),
("??=", None),
("||", Some(BinaryOp::Or)),
("&&", Some(BinaryOp::And)),
("==", Some(BinaryOp::Eq)),
("!=", Some(BinaryOp::Ne)),
("<=", Some(BinaryOp::Le)),
(">=", Some(BinaryOp::Ge)),
("<<", Some(BinaryOp::Shl)),
(">>", Some(BinaryOp::Shr)),
("**", None),
("??", None),
("*=", None),
("/=", None),
("%=", None),
("+=", None),
("-=", None),
("&=", None),
("^=", None),
("|=", None),
("|", Some(BinaryOp::BitOr)),
("^", Some(BinaryOp::BitXor)),
("&", Some(BinaryOp::BitAnd)),
("<", Some(BinaryOp::Lt)),
(">", Some(BinaryOp::Gt)),
("+", Some(BinaryOp::Add)),
("-", Some(BinaryOp::Sub)),
("*", Some(BinaryOp::Mul)),
("/", Some(BinaryOp::Div)),
("%", Some(BinaryOp::Rem)),
("=", None),
];
const UNARY_OPS: &[(&str, UnaryOp)] = &[
("typeof", UnaryOp::TypeOf),
("void", UnaryOp::Void),
("-", UnaryOp::Neg),
("!", UnaryOp::Not),
("~", UnaryOp::BitNot),
("+", UnaryOp::Plus),
];
pub fn parse(code: &str) -> Result<Expr, String> {
let mut parser = Parser { chars: code.chars().collect(), index: 0, depth: 0 };
let mut nodes = parser.expressions(None)?;
match nodes.len() {
0 => Ok(Expr::Literal(Literal::Undefined)),
1 => Ok(nodes.remove(0)),
_ => match &nodes[0] {
Expr::Ident(name) if matches!(name.as_str(), "var" | "let" | "const") => {
Err(format!("variable declarations ({name}) are not supported"))
}
_ => {
Err("multiple expressions (separated by `;` or `,`) are not supported".to_string())
}
},
}
}
fn is_identifier_start(c: char) -> bool {
c.is_ascii_alphabetic() || c == '$' || c == '_' || (c as u32) >= 128
}
fn is_identifier_part(c: char) -> bool {
is_identifier_start(c) || c.is_ascii_digit()
}
struct Parser {
chars: Vec<char>,
index: usize,
depth: usize,
}
impl Parser {
fn peek(&self) -> Option<char> {
self.chars.get(self.index).copied()
}
fn peek_at(&self, offset: usize) -> Option<char> {
self.chars.get(self.index + offset).copied()
}
fn rest_starts_with(&self, s: &str) -> bool {
s.chars().enumerate().all(|(i, c)| self.peek_at(i) == Some(c))
}
fn error(&self, message: &str) -> String {
format!("{message} at character {}", self.index)
}
fn unexpected(&self) -> String {
self.error(&format!("Unexpected \"{}\"", self.peek().map(String::from).unwrap_or_default()))
}
fn spaces(&mut self) {
while let Some(' ' | '\t' | '\n' | '\r') = self.peek() {
self.index += 1;
}
}
fn enter(&mut self) -> Result<(), String> {
self.depth += 1;
if self.depth > MAX_DEPTH {
Err("expression is nested too deeply".to_string())
} else {
Ok(())
}
}
fn expressions(&mut self, until: Option<char>) -> Result<Vec<Expr>, String> {
let mut nodes = Vec::new();
while let Some(c) = self.peek() {
if c == ';' || c == ',' {
self.index += 1;
continue;
}
match self.expression()? {
Some(node) => nodes.push(node),
None if self.index < self.chars.len() => {
if Some(c) == until {
break;
}
return Err(self.unexpected());
}
None => {}
}
}
Ok(nodes)
}
fn expression(&mut self) -> Result<Option<Expr>, String> {
self.enter()?;
let node = self.binary_expression()?;
self.spaces();
let node = match node {
Some(test) if self.peek() == Some('?') => {
self.index += 1;
let consequent =
self.expression()?.ok_or_else(|| self.error("Expected expression"))?;
self.spaces();
if self.peek() != Some(':') {
return Err(self.error("Expected :"));
}
self.index += 1;
let alternate =
self.expression()?.ok_or_else(|| self.error("Expected expression"))?;
Some(Expr::Conditional(Box::new(test), Box::new(consequent), Box::new(alternate)))
}
node => node,
};
self.depth -= 1;
Ok(node)
}
fn binary_op(&mut self) -> Result<Option<BinaryOp>, String> {
self.spaces();
let Some((text, op)) = BINARY_OPS.iter().find(|(text, _)| self.rest_starts_with(text))
else {
return Ok(None);
};
match op {
Some(op) => {
self.index += text.chars().count();
Ok(Some(*op))
}
None if text.ends_with('=') => Err(format!("assignment ({text}) is not supported")),
None => Err(format!("the {text} operator is not supported")),
}
}
fn binary_expression(&mut self) -> Result<Option<Expr>, String> {
enum Item {
Node(Expr),
Op(BinaryOp),
}
let Some(left) = self.token()? else {
return Ok(None);
};
let Some(op) = self.binary_op()? else {
return Ok(Some(left));
};
let right =
self.token()?.ok_or_else(|| self.error("Expected expression after operator"))?;
let mut stack = vec![Item::Node(left), Item::Op(op), Item::Node(right)];
let mut operators = 1;
while let Some(op) = self.binary_op()? {
operators += 1;
if operators > MAX_DEPTH {
return Err("expression has too many operators".to_string());
}
while stack.len() > 2
&& matches!(&stack[stack.len() - 2], Item::Op(prev) if op.precedence() <= prev.precedence())
{
let (Some(Item::Node(right)), Some(Item::Op(prev)), Some(Item::Node(left))) =
(stack.pop(), stack.pop(), stack.pop())
else {
return Err(self.error("malformed expression"));
};
stack.push(Item::Node(Expr::Binary(prev, Box::new(left), Box::new(right))));
}
let node =
self.token()?.ok_or_else(|| self.error("Expected expression after operator"))?;
stack.push(Item::Op(op));
stack.push(Item::Node(node));
}
let mut node = match stack.pop() {
Some(Item::Node(node)) => node,
_ => return Err(self.error("malformed expression")),
};
while let (Some(Item::Op(op)), Some(Item::Node(left))) = (stack.pop(), stack.pop()) {
node = Expr::Binary(op, Box::new(left), Box::new(node));
}
Ok(Some(node))
}
fn after_token(&self, node: Option<Expr>) -> Result<Option<Expr>, String> {
if node.is_some() && (self.rest_starts_with("++") || self.rest_starts_with("--")) {
return Err("increment and decrement (++, --) are not supported".to_string());
}
Ok(node)
}
fn token(&mut self) -> Result<Option<Expr>, String> {
self.spaces();
self.enter()?;
let node = self.token_inner();
self.depth -= 1;
node
}
fn token_inner(&mut self) -> Result<Option<Expr>, String> {
if self.peek() == Some('/') {
let regex = self.regex_literal()?;
let node = self.token_property(regex)?;
return self.after_token(Some(node));
}
if self.rest_starts_with("++") || self.rest_starts_with("--") {
return Err("increment and decrement (++, --) are not supported".to_string());
}
let Some(c) = self.peek() else {
return Ok(None);
};
if c.is_ascii_digit() || c == '.' {
return self.number().map(Some);
}
let node = if c == '\'' || c == '"' {
Expr::Literal(Literal::String(self.string_literal()?))
} else if c == '[' {
self.index += 1;
let items = self.arguments(']')?;
Expr::Array(items)
} else {
for (text, op) in UNARY_OPS {
let len = text.chars().count();
let word = is_identifier_start(c);
if self.rest_starts_with(text)
&& (!word || self.peek_at(len).is_some_and(|next| !is_identifier_part(next)))
{
self.index += len;
let argument =
self.token()?.ok_or_else(|| self.error("missing unaryOp argument"))?;
return self.after_token(Some(Expr::Unary(*op, Box::new(argument))));
}
}
if is_identifier_start(c) {
let name = self.identifier()?;
match name.as_str() {
"true" => Expr::Literal(Literal::Bool(true)),
"false" => Expr::Literal(Literal::Bool(false)),
"null" => Expr::Literal(Literal::Null),
"undefined" => Expr::Literal(Literal::Undefined),
"this" => return Err("`this` is not supported".to_string()),
_ => Expr::Ident(name),
}
} else if c == '(' {
match self.group()? {
Some(node) => node,
None => return Ok(None),
}
} else {
return Ok(None);
}
};
let node = self.token_property(node)?;
self.after_token(Some(node))
}
fn token_property(&mut self, mut node: Expr) -> Result<Expr, String> {
self.spaces();
let depth = self.depth;
while let Some(mut c) = self.peek() {
if !matches!(c, '.' | '[' | '(' | '?') {
break;
}
self.enter()?;
let mut optional = false;
if c == '?' {
if self.peek_at(1) != Some('.') {
break;
}
optional = true;
self.index += 2;
self.spaces();
c = self.peek().unwrap_or(' ');
}
self.index += 1;
if c == '[' {
let property = self.expression()?.ok_or_else(|| self.unexpected())?;
self.spaces();
if self.peek() != Some(']') {
return Err(self.error("Unclosed ["));
}
self.index += 1;
node = Expr::Member(Box::new(node), Property::Computed(Box::new(property)));
} else if c == '(' {
node = self.call(node)?;
} else if c == '.' || optional {
if optional {
self.index -= 1;
}
self.spaces();
let name = self.identifier()?;
node = Expr::Member(Box::new(node), Property::Name(name));
}
self.spaces();
}
self.depth = depth;
Ok(node)
}
fn call(&mut self, callee: Expr) -> Result<Expr, String> {
let function = match &callee {
Expr::Ident(name) => match RfcFunction::from_name(name) {
Some(function) => function,
None => return Err(format!("unknown function {name}()")),
},
Expr::Member(_, Property::Name(name)) if !is_allowed_method_name(name) => {
return Err(format!("unsupported method {name}()"));
}
_ => {
let args = self.arguments(')')?;
return Ok(Expr::Call(Box::new(callee), args));
}
};
if !function.takes_nodes() {
let args = self.arguments(')')?;
if args.len() != function.arity() {
return Err(format!(
"{}() takes {} argument(s)",
function.name(),
function.arity()
));
}
return Ok(Expr::Rfc(function, args.into_iter().map(RfcArg::Value).collect()));
}
self.spaces();
let start = self.index;
let (prefix_len, root) = if self.rest_starts_with("_$_root") {
(7, true)
} else if self.rest_starts_with("_$_v") {
(4, false)
} else if self.rest_starts_with("@") {
(1, false)
} else if self.rest_starts_with("$") {
(1, true)
} else {
return Err(self.error(&format!("{}() takes a query (@ or $)", function.name())));
};
if self.peek_at(prefix_len).is_some_and(|c| is_identifier_part(c) && c != '$') {
return Err(self.error(&format!("{}() takes a query (@ or $)", function.name())));
}
let (end, query) = scan_query(&self.chars, start + prefix_len, root);
self.index = end;
self.spaces();
if self.peek() != Some(')') {
return Err(self.error(&format!("{}() takes one query", function.name())));
}
self.index += 1;
Ok(Expr::Rfc(function, vec![RfcArg::Nodes(query)]))
}
fn number(&mut self) -> Result<Expr, String> {
let mut number = String::new();
while let Some(c) = self.peek().filter(char::is_ascii_digit) {
number.push(c);
self.index += 1;
}
if self.peek() == Some('.') {
number.push('.');
self.index += 1;
while let Some(c) = self.peek().filter(char::is_ascii_digit) {
number.push(c);
self.index += 1;
}
}
if let Some(e @ ('e' | 'E')) = self.peek() {
number.push(e);
self.index += 1;
if let Some(sign @ ('+' | '-')) = self.peek() {
number.push(sign);
self.index += 1;
}
while let Some(c) = self.peek().filter(char::is_ascii_digit) {
number.push(c);
self.index += 1;
}
if !number.ends_with(|c: char| c.is_ascii_digit()) {
return Err(self.error(&format!("Expected exponent ({number})")));
}
}
match self.peek() {
Some(c) if is_identifier_start(c) => {
return Err(
self.error(&format!("Variable names cannot start with a number ({number}{c})"))
);
}
Some('.') => return Err(self.error("Unexpected period")),
_ if number == "." => return Err(self.error("Unexpected period")),
_ => {}
}
let mantissa = number.split(['e', 'E']).next().unwrap_or("");
let value = if mantissa.chars().any(|c| c.is_ascii_digit()) {
number
.trim_end_matches('.')
.replace(".e", "e")
.replace(".E", "E")
.parse::<f64>()
.unwrap_or(f64::NAN)
} else {
f64::NAN
};
Ok(Expr::Literal(Literal::Number(value)))
}
fn string_literal(&mut self) -> Result<String, String> {
let quote = self.peek();
self.index += 1;
let mut out = String::new();
while let Some(c) = self.peek() {
self.index += 1;
if Some(c) == quote {
return Ok(out);
}
if c == '\\' {
let escaped = self.peek();
self.index += 1;
match escaped {
Some('n') => out.push('\n'),
Some('r') => out.push('\r'),
Some('t') => out.push('\t'),
Some('b') => out.push('\u{8}'),
Some('f') => out.push('\u{c}'),
Some('v') => out.push('\u{b}'),
Some(other) => out.push(other),
None => {}
}
} else {
out.push(c);
}
}
Err(self.error(&format!("Unclosed quote after \"{out}\"")))
}
fn regex_literal(&mut self) -> Result<Expr, String> {
self.index += 1;
let start = self.index;
let mut in_class = false;
while let Some(c) = self.peek() {
if c == '/' && !in_class {
let pattern: String = self.chars[start..self.index].iter().collect();
self.index += 1;
let mut flags = String::new();
while let Some(f) = self.peek().filter(char::is_ascii_alphanumeric) {
flags.push(f);
self.index += 1;
}
let regex = JsRegex::new(&pattern, &flags).map_err(|e| self.error(&e))?;
return Ok(Expr::Literal(Literal::Regex(Rc::new(regex))));
}
if c == '[' {
in_class = true;
} else if in_class && c == ']' {
in_class = false;
}
self.index += if c == '\\' { 2 } else { 1 };
}
Err(self.error("Unclosed Regex"))
}
fn identifier(&mut self) -> Result<String, String> {
match self.peek() {
Some(c) if is_identifier_start(c) => self.index += 1,
_ => {
return Err(self.error(&format!(
"Unexpected {}",
self.peek().map(String::from).unwrap_or_default()
)));
}
}
let start = self.index - 1;
while self.peek().is_some_and(is_identifier_part) {
self.index += 1;
}
Ok(self.chars[start..self.index].iter().collect())
}
fn arguments(&mut self, terminator: char) -> Result<Vec<Expr>, String> {
let mut args: Vec<Option<Expr>> = Vec::new();
let mut separators = 0;
loop {
self.spaces();
let Some(c) = self.peek() else {
return Err(self.error(&format!("Expected {terminator}")));
};
if c == terminator {
self.index += 1;
if terminator == ')' && separators > 0 && separators >= args.len() {
return Err(self.error("Unexpected token )"));
}
break;
} else if c == ',' {
self.index += 1;
separators += 1;
if separators != args.len() {
if terminator == ')' {
return Err(self.error("Unexpected token ,"));
}
while args.len() < separators {
args.push(None);
}
}
} else if args.len() != separators && separators != 0 {
return Err(self.error("Expected comma"));
} else {
let node = self.expression()?.ok_or_else(|| self.error("Expected comma"))?;
args.push(Some(node));
}
}
args.into_iter()
.map(|arg| arg.ok_or_else(|| "array holes are not supported".to_string()))
.collect()
}
fn group(&mut self) -> Result<Option<Expr>, String> {
self.index += 1;
let mut nodes = self.expressions(Some(')'))?;
if self.peek() != Some(')') {
return Err(self.error("Unclosed ("));
}
self.index += 1;
match nodes.len() {
0 => Ok(None),
1 => Ok(Some(nodes.remove(0))),
_ => Err("sequence expressions (a, b) are not supported".to_string()),
}
}
}
/// jsonpath-plus's textual substitution of `@` forms before jsep sees the code.
pub fn substitute(code: &str) -> String {
let code = code
.replace("@parentProperty", "_$_parentProperty")
.replace("@parent", "_$_parent")
.replace("@property", "_$_property")
.replace("@root", "_$_root");
let mut out = String::with_capacity(code.len());
let mut chars = code.chars().peekable();
while let Some(c) = chars.next() {
if c == '@'
&& chars.peek().is_some_and(|n| matches!(n, '.' | ')' | '[') || is_js_whitespace(*n))
{
out.push_str("_$_v");
} else {
out.push(c);
}
}
if code.contains("@path") { out.replace("@path", "_$_path") } else { out }
}
pub struct Scope<'a> {
pub value: JsValue<'a>,
pub property: JsValue<'a>,
pub parent: JsValue<'a>,
pub parent_property: JsValue<'a>,
pub path: Option<String>,
}
pub fn eval<'a>(
expr: &Expr,
scope: &Scope<'a>,
engine: &Engine<'a>,
) -> Result<JsValue<'a>, String> {
Ok(match expr {
Expr::Literal(literal) => match literal {
Literal::Undefined => JsValue::Undefined,
Literal::Null => JsValue::Null,
Literal::Bool(b) => JsValue::Bool(*b),
Literal::Number(n) => JsValue::Number(*n),
Literal::String(s) => JsValue::string(s.clone()),
Literal::Regex(re) => JsValue::Regex(re.clone()),
},
Expr::Ident(name) => match name.as_str() {
"_$_v" => scope.value.clone(),
"_$_property" => scope.property.clone(),
"_$_parent" => scope.parent.clone(),
"_$_parentProperty" => scope.parent_property.clone(),
"_$_root" => JsValue::from_doc(engine.root),
"_$_path" if scope.path.is_some() => {
JsValue::string(scope.path.clone().unwrap_or_default())
}
_ => return Err(format!("{name} is not defined")),
},
Expr::Member(object, property) => {
let key = match property {
Property::Name(name) => name.clone(),
Property::Computed(expr) => eval(expr, scope, engine)?.to_js_string(),
};
let object = eval(object, scope, engine)?;
get_property(&object, &key)?
}
Expr::Call(callee, args) => {
let args =
args.iter().map(|arg| eval(arg, scope, engine)).collect::<Result<Vec<_>, _>>()?;
match eval(callee, scope, engine)? {
JsValue::Function(bound) => call_method(&bound, &args)?,
other => return Err(format!("{} is not a function", other.type_of())),
}
}
Expr::Rfc(function, args) => {
let args = args
.iter()
.map(|arg| match arg {
RfcArg::Value(expr) => eval(expr, scope, engine).map(|v| vec![v]),
RfcArg::Nodes(query) => engine.query(query, &scope.value),
})
.collect::<Result<Vec<_>, _>>()?;
function.call_js(&args, engine)
}
Expr::Array(items) => JsValue::array(
items.iter().map(|item| eval(item, scope, engine)).collect::<Result<_, _>>()?,
),
Expr::Unary(op, argument) => {
let value = eval(argument, scope, engine)?;
match op {
UnaryOp::Neg => JsValue::Number(-value.to_number()),
UnaryOp::Plus => JsValue::Number(value.to_number()),
UnaryOp::Not => JsValue::Bool(!value.truthy()),
UnaryOp::BitNot => JsValue::Number(f64::from(!value.to_int32())),
UnaryOp::TypeOf => JsValue::string(value.type_of()),
UnaryOp::Void => JsValue::Undefined,
}
}
Expr::Binary(op, left, right) => {
let left = eval(left, scope, engine)?;
if matches!(op, BinaryOp::Or | BinaryOp::And) {
let short_circuits = left.truthy() == matches!(op, BinaryOp::Or);
return if short_circuits { Ok(left) } else { eval(right, scope, engine) };
}
let right = eval(right, scope, engine)?;
binary(*op, &left, &right)
}
Expr::Conditional(test, consequent, alternate) => {
if eval(test, scope, engine)?.truthy() {
eval(consequent, scope, engine)?
} else {
eval(alternate, scope, engine)?
}
}
})
}
fn binary<'a>(op: BinaryOp, a: &JsValue<'a>, b: &JsValue<'a>) -> JsValue<'a> {
match op {
BinaryOp::Or | BinaryOp::And => JsValue::Undefined,
BinaryOp::BitOr => JsValue::Number(f64::from(a.to_int32() | b.to_int32())),
BinaryOp::BitXor => JsValue::Number(f64::from(a.to_int32() ^ b.to_int32())),
BinaryOp::BitAnd => JsValue::Number(f64::from(a.to_int32() & b.to_int32())),
BinaryOp::Eq => JsValue::Bool(a.loose_eq(b)),
BinaryOp::Ne => JsValue::Bool(!a.loose_eq(b)),
BinaryOp::StrictEq => JsValue::Bool(a.strict_eq(b)),
BinaryOp::StrictNe => JsValue::Bool(!a.strict_eq(b)),
BinaryOp::Lt => JsValue::Bool(a.less_than(b) == Some(true)),
BinaryOp::Gt => JsValue::Bool(b.less_than(a) == Some(true)),
BinaryOp::Le => JsValue::Bool(b.less_than(a) == Some(false)),
BinaryOp::Ge => JsValue::Bool(a.less_than(b) == Some(false)),
BinaryOp::Shl => JsValue::Number(f64::from(a.to_int32().wrapping_shl(b.to_uint32() & 31))),
BinaryOp::Shr => JsValue::Number(f64::from(a.to_int32() >> (b.to_uint32() & 31))),
BinaryOp::UShr => JsValue::Number(f64::from(a.to_uint32() >> (b.to_uint32() & 31))),
BinaryOp::Add => a.add(b),
BinaryOp::Sub => JsValue::Number(a.to_number() - b.to_number()),
BinaryOp::Mul => JsValue::Number(a.to_number() * b.to_number()),
BinaryOp::Div => JsValue::Number(a.to_number() / b.to_number()),
BinaryOp::Rem => JsValue::Number(a.to_number() % b.to_number()),
}
}
+953
View File
@@ -0,0 +1,953 @@
//! The slice of JavaScript values and coercions the safe evaluator needs.
use super::json::{Json, array_index, number_to_string};
use super::regex::JsRegex;
use std::borrow::Cow;
use std::cmp::Ordering;
use std::rc::Rc;
#[derive(Clone, Debug)]
pub enum JsValue<'a> {
Undefined,
Null,
Bool(bool),
Number(f64),
String(Cow<'a, str>),
/// An array or object of the document, compared by identity.
Doc(&'a Json),
Array(Rc<JsArray<'a>>),
Regex(Rc<JsRegex>),
Function(Rc<BoundMethod<'a>>),
}
#[derive(Debug, Default)]
pub struct JsArray<'a> {
pub items: Vec<JsValue<'a>>,
/// Set on the array `String.prototype.match` returns.
pub match_info: Option<MatchInfo>,
}
#[derive(Debug)]
pub struct MatchInfo {
pub index: f64,
pub input: String,
}
#[derive(Debug)]
pub struct BoundMethod<'a> {
pub receiver: JsValue<'a>,
pub method: Method,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Method {
StringIncludes,
StringStartsWith,
StringEndsWith,
StringIndexOf,
StringLastIndexOf,
StringToLowerCase,
StringToUpperCase,
StringTrim,
StringTrimStart,
StringTrimEnd,
StringSlice,
StringSubstring,
StringSplit,
StringMatch,
StringToString,
ArrayIncludes,
ArrayIndexOf,
ArrayJoin,
ArrayToString,
NumberToString,
NumberToFixed,
RegexTest,
}
impl Method {
pub fn name(self) -> &'static str {
match self {
Method::StringIncludes | Method::ArrayIncludes => "includes",
Method::StringStartsWith => "startsWith",
Method::StringEndsWith => "endsWith",
Method::StringIndexOf | Method::ArrayIndexOf => "indexOf",
Method::StringLastIndexOf => "lastIndexOf",
Method::StringToLowerCase => "toLowerCase",
Method::StringToUpperCase => "toUpperCase",
Method::StringTrim => "trim",
Method::StringTrimStart => "trimStart",
Method::StringTrimEnd => "trimEnd",
Method::StringSlice => "slice",
Method::StringSubstring => "substring",
Method::StringSplit => "split",
Method::StringMatch => "match",
Method::StringToString | Method::ArrayToString | Method::NumberToString => "toString",
Method::ArrayJoin => "join",
Method::NumberToFixed => "toFixed",
Method::RegexTest => "test",
}
}
}
const STRING_METHODS: &[(&str, Method)] = &[
("includes", Method::StringIncludes),
("startsWith", Method::StringStartsWith),
("endsWith", Method::StringEndsWith),
("indexOf", Method::StringIndexOf),
("lastIndexOf", Method::StringLastIndexOf),
("toLowerCase", Method::StringToLowerCase),
("toUpperCase", Method::StringToUpperCase),
("trim", Method::StringTrim),
("trimStart", Method::StringTrimStart),
("trimEnd", Method::StringTrimEnd),
("slice", Method::StringSlice),
("substring", Method::StringSubstring),
("split", Method::StringSplit),
("match", Method::StringMatch),
("toString", Method::StringToString),
];
const ARRAY_METHODS: &[(&str, Method)] = &[
("includes", Method::ArrayIncludes),
("indexOf", Method::ArrayIndexOf),
("join", Method::ArrayJoin),
("toString", Method::ArrayToString),
];
const NUMBER_METHODS: &[(&str, Method)] = &[
("toString", Method::NumberToString),
("toFixed", Method::NumberToFixed),
];
const REGEX_METHODS: &[(&str, Method)] = &[("test", Method::RegexTest)];
/// Every method name a call may use; anything else is rejected before evaluation.
pub fn is_allowed_method_name(name: &str) -> bool {
[STRING_METHODS, ARRAY_METHODS, NUMBER_METHODS, REGEX_METHODS]
.iter()
.any(|t| t.iter().any(|(n, _)| *n == name))
}
const STRING_PROTO: &[&str] = &[
"anchor",
"at",
"big",
"blink",
"bold",
"charAt",
"charCodeAt",
"codePointAt",
"concat",
"fontcolor",
"fontsize",
"fixed",
"isWellFormed",
"italics",
"link",
"localeCompare",
"matchAll",
"normalize",
"padEnd",
"padStart",
"repeat",
"replace",
"replaceAll",
"search",
"small",
"strike",
"sub",
"substr",
"sup",
"toWellFormed",
"trimLeft",
"trimRight",
"toLocaleLowerCase",
"toLocaleUpperCase",
"valueOf",
];
const ARRAY_PROTO: &[&str] = &[
"at",
"concat",
"copyWithin",
"fill",
"find",
"findIndex",
"findLast",
"findLastIndex",
"lastIndexOf",
"pop",
"push",
"reverse",
"shift",
"unshift",
"slice",
"sort",
"splice",
"keys",
"entries",
"values",
"forEach",
"filter",
"flat",
"flatMap",
"map",
"every",
"some",
"reduce",
"reduceRight",
"toReversed",
"toSorted",
"toSpliced",
"with",
"toLocaleString",
];
const NUMBER_PROTO: &[&str] = &["toExponential", "toPrecision", "valueOf", "toLocaleString"];
const BOOLEAN_PROTO: &[&str] = &["toString", "valueOf"];
const REGEX_PROTO: &[&str] = &["exec", "compile", "toString"];
const FUNCTION_PROTO: &[&str] = &[
"length",
"name",
"arguments",
"caller",
"apply",
"bind",
"call",
"toString",
];
const OBJECT_PROTO: &[&str] = &[
"hasOwnProperty",
"isPrototypeOf",
"propertyIsEnumerable",
"toString",
"valueOf",
"toLocaleString",
];
/// Reading these off a prototype is refused, as jsonpath-plus's safe evaluator does.
const BLOCKED: &[&str] = &[
"constructor",
"__proto__",
"__defineGetter__",
"__defineSetter__",
"__lookupGetter__",
"__lookupSetter__",
];
pub fn is_js_whitespace(c: char) -> bool {
matches!(
c,
'\t' | '\n' | '\u{b}' | '\u{c}' | '\r' | ' ' | '\u{a0}' | '\u{1680}' | '\u{2000}'
..='\u{200a}'
| '\u{2028}'
| '\u{2029}'
| '\u{202f}'
| '\u{205f}'
| '\u{3000}'
| '\u{feff}'
)
}
impl<'a> JsValue<'a> {
pub fn from_doc(json: &'a Json) -> Self {
match json {
Json::Null => JsValue::Null,
Json::Bool(b) => JsValue::Bool(*b),
Json::Number(n) => JsValue::Number(*n),
Json::String(s) => JsValue::String(Cow::Borrowed(s)),
Json::Array(_) | Json::Object(_) => JsValue::Doc(json),
}
}
pub fn string(s: impl Into<String>) -> Self {
JsValue::String(Cow::Owned(s.into()))
}
pub fn array(items: Vec<JsValue<'a>>) -> Self {
JsValue::Array(Rc::new(JsArray { items, match_info: None }))
}
pub fn truthy(&self) -> bool {
match self {
JsValue::Undefined | JsValue::Null => false,
JsValue::Bool(b) => *b,
JsValue::Number(n) => *n != 0.0 && !n.is_nan(),
JsValue::String(s) => !s.is_empty(),
JsValue::Doc(_) | JsValue::Array(_) | JsValue::Regex(_) | JsValue::Function(_) => true,
}
}
pub fn is_object(&self) -> bool {
match self {
JsValue::Doc(_) | JsValue::Array(_) | JsValue::Regex(_) | JsValue::Function(_) => true,
JsValue::Undefined
| JsValue::Null
| JsValue::Bool(_)
| JsValue::Number(_)
| JsValue::String(_) => false,
}
}
pub fn type_of(&self) -> &'static str {
match self {
JsValue::Undefined => "undefined",
JsValue::Null | JsValue::Doc(_) | JsValue::Array(_) | JsValue::Regex(_) => "object",
JsValue::Bool(_) => "boolean",
JsValue::Number(_) => "number",
JsValue::String(_) => "string",
JsValue::Function(_) => "function",
}
}
/// `ToPrimitive`: every object here stringifies, whatever the hint.
pub fn to_primitive(&self) -> JsValue<'a> {
if self.is_object() { JsValue::string(self.to_js_string()) } else { self.clone() }
}
pub fn to_number(&self) -> f64 {
match self {
JsValue::Undefined => f64::NAN,
JsValue::Null => 0.0,
JsValue::Bool(b) => f64::from(u8::from(*b)),
JsValue::Number(n) => *n,
JsValue::String(s) => string_to_number(s),
JsValue::Doc(_) | JsValue::Array(_) | JsValue::Regex(_) | JsValue::Function(_) => {
string_to_number(&self.to_js_string())
}
}
}
pub fn to_js_string(&self) -> String {
match self {
JsValue::Undefined => "undefined".to_string(),
JsValue::Null => "null".to_string(),
JsValue::Bool(b) => b.to_string(),
JsValue::Number(n) => number_to_string(*n),
JsValue::String(s) => s.to_string(),
JsValue::Doc(Json::Array(items)) => join_doc(items, ","),
JsValue::Doc(_) => "[object Object]".to_string(),
JsValue::Array(array) => join_values(&array.items, ","),
JsValue::Regex(re) => format!("/{}/{}", re.source, re.flags),
JsValue::Function(_) => "function () { [native code] }".to_string(),
}
}
pub fn to_int32(&self) -> i32 {
self.to_uint32() as i32
}
pub fn to_uint32(&self) -> u32 {
let n = self.to_number();
if !n.is_finite() {
return 0;
}
n.trunc().rem_euclid(4_294_967_296.0) as u32
}
/// `ToIntegerOrInfinity`.
pub fn to_integer(&self) -> f64 {
let n = self.to_number();
if n.is_nan() { 0.0 } else { n.trunc() + 0.0 }
}
fn same_object(&self, other: &JsValue<'a>) -> bool {
match (self, other) {
(JsValue::Doc(a), JsValue::Doc(b)) => std::ptr::eq(*a, *b),
(JsValue::Array(a), JsValue::Array(b)) => Rc::ptr_eq(a, b),
(JsValue::Regex(a), JsValue::Regex(b)) => Rc::ptr_eq(a, b),
(JsValue::Function(a), JsValue::Function(b)) => Rc::ptr_eq(a, b),
_ => false,
}
}
pub fn strict_eq(&self, other: &JsValue<'a>) -> bool {
match (self, other) {
(JsValue::Undefined, JsValue::Undefined) | (JsValue::Null, JsValue::Null) => true,
(JsValue::Bool(a), JsValue::Bool(b)) => a == b,
(JsValue::Number(a), JsValue::Number(b)) => a == b,
(JsValue::String(a), JsValue::String(b)) => a == b,
(a, b) => a.same_object(b),
}
}
/// `SameValueZero`, for `Array.prototype.includes`.
fn same_value_zero(&self, other: &JsValue<'a>) -> bool {
match (self, other) {
(JsValue::Number(a), JsValue::Number(b)) if a.is_nan() && b.is_nan() => true,
(a, b) => a.strict_eq(b),
}
}
pub fn loose_eq(&self, other: &JsValue<'a>) -> bool {
use JsValue::{Bool, Null, Number, String, Undefined};
match (self, other) {
(Undefined | Null, Undefined | Null) => true,
(Undefined | Null, _) | (_, Undefined | Null) => false,
(Number(a), String(b)) => *a == string_to_number(b),
(String(a), Number(b)) => string_to_number(a) == *b,
(Bool(_), _) => JsValue::Number(self.to_number()).loose_eq(other),
(_, Bool(_)) => self.loose_eq(&JsValue::Number(other.to_number())),
(a, b) if a.is_object() && !b.is_object() => a.to_primitive().loose_eq(b),
(a, b) if !a.is_object() && b.is_object() => a.loose_eq(&b.to_primitive()),
(a, b) => a.strict_eq(b),
}
}
/// `IsLessThan`; `None` is JavaScript's undefined, from NaN.
pub fn less_than(&self, other: &JsValue<'a>) -> Option<bool> {
let (a, b) = (self.to_primitive(), other.to_primitive());
if let (JsValue::String(a), JsValue::String(b)) = (&a, &b) {
return Some(a.encode_utf16().cmp(b.encode_utf16()) == Ordering::Less);
}
let (a, b) = (a.to_number(), b.to_number());
if a.is_nan() || b.is_nan() { None } else { Some(a < b) }
}
pub fn add(&self, other: &JsValue<'a>) -> JsValue<'a> {
let (a, b) = (self.to_primitive(), other.to_primitive());
if matches!(a, JsValue::String(_)) || matches!(b, JsValue::String(_)) {
JsValue::string(a.to_js_string() + &b.to_js_string())
} else {
JsValue::Number(a.to_number() + b.to_number())
}
}
}
fn join_doc(items: &[Json], separator: &str) -> String {
let parts: Vec<String> = items
.iter()
.map(|item| match item {
Json::Null => String::new(),
item => JsValue::from_doc(item).to_js_string(),
})
.collect();
parts.join(separator)
}
fn join_values(items: &[JsValue<'_>], separator: &str) -> String {
let parts: Vec<String> = items
.iter()
.map(|item| match item {
JsValue::Undefined | JsValue::Null => String::new(),
item => item.to_js_string(),
})
.collect();
parts.join(separator)
}
/// `StringToNumber`.
pub fn string_to_number(s: &str) -> f64 {
let s = s.trim_matches(is_js_whitespace);
if s.is_empty() {
return 0.0;
}
for (prefix, radix) in [
("0x", 16),
("0X", 16),
("0o", 8),
("0O", 8),
("0b", 2),
("0B", 2),
] {
if let Some(digits) = s.strip_prefix(prefix) {
if digits.is_empty() || !digits.chars().all(|c| c.is_digit(radix)) {
return f64::NAN;
}
return digits.chars().fold(0.0, |n, c| {
n * f64::from(radix) + f64::from(c.to_digit(radix).unwrap_or(0))
});
}
}
let unsigned = s.strip_prefix(['+', '-']).unwrap_or(s);
if unsigned == "Infinity" {
return if s.starts_with('-') { f64::NEG_INFINITY } else { f64::INFINITY };
}
let bytes = unsigned.as_bytes();
let mut i = 0;
let int_digits = bytes.iter().take_while(|b| b.is_ascii_digit()).count();
i += int_digits;
let mut frac_digits = 0;
if bytes.get(i) == Some(&b'.') {
i += 1;
frac_digits = bytes[i..].iter().take_while(|b| b.is_ascii_digit()).count();
i += frac_digits;
}
if int_digits + frac_digits == 0 {
return f64::NAN;
}
if let Some(b'e' | b'E') = bytes.get(i) {
i += 1;
if let Some(b'+' | b'-') = bytes.get(i) {
i += 1;
}
let exp_digits = bytes[i..].iter().take_while(|b| b.is_ascii_digit()).count();
if exp_digits == 0 {
return f64::NAN;
}
i += exp_digits;
}
if i != bytes.len() {
return f64::NAN;
}
s.parse::<f64>().unwrap_or(f64::NAN)
}
/// An own property, as `Object.hasOwn` sees it.
pub fn own_property<'a>(value: &JsValue<'a>, key: &str) -> Option<JsValue<'a>> {
match value {
JsValue::Doc(Json::Object(map)) => map.get(key).map(JsValue::from_doc),
JsValue::Doc(Json::Array(items)) => {
if key == "length" {
return Some(JsValue::Number(items.len() as f64));
}
array_index(key).and_then(|i| items.get(i as usize)).map(JsValue::from_doc)
}
JsValue::Array(array) => {
if key == "length" {
return Some(JsValue::Number(array.items.len() as f64));
}
if let Some(info) = &array.match_info {
match key {
"index" => return Some(JsValue::Number(info.index)),
"input" => return Some(JsValue::string(info.input.clone())),
"groups" => return Some(JsValue::Undefined),
_ => {}
}
}
array_index(key).and_then(|i| array.items.get(i as usize)).cloned()
}
JsValue::String(s) => {
if key == "length" {
return Some(JsValue::Number(s.encode_utf16().count() as f64));
}
let i = array_index(key)? as usize;
let unit = s.encode_utf16().nth(i)?;
Some(JsValue::string(String::from_utf16_lossy(&[unit])))
}
JsValue::Regex(re) => match key {
"lastIndex" => Some(JsValue::Number(re.last_index.get() as f64)),
"source" => Some(JsValue::string(re.source.clone())),
"flags" => {
let mut flags: Vec<char> = re.flags.chars().collect();
flags.sort_by_key(|f| "dgimsuvy".find(*f));
Some(JsValue::string(flags.into_iter().collect::<String>()))
}
_ => [
"hasIndices",
"global",
"ignoreCase",
"multiline",
"dotAll",
"unicode",
"unicodeSets",
"sticky",
]
.iter()
.position(|name| *name == key)
.map(|i| JsValue::Bool(re.flags.contains("dgimsuvy".as_bytes()[i] as char))),
},
JsValue::Doc(_)
| JsValue::Undefined
| JsValue::Null
| JsValue::Bool(_)
| JsValue::Number(_)
| JsValue::Function(_) => None,
}
}
/// `object[key]` in the safe evaluator: own properties, then the allowlisted
/// methods; any other inherited member is refused rather than guessed at.
pub fn get_property<'a>(object: &JsValue<'a>, key: &str) -> Result<JsValue<'a>, String> {
if matches!(object, JsValue::Undefined | JsValue::Null) {
return Err(format!(
"Cannot read properties of {} (reading '{key}')",
object.to_js_string()
));
}
if let Some(own) = own_property(object, key) {
return Ok(own);
}
if BLOCKED.contains(&key) {
return Err(format!(
"Cannot read properties of {} (reading '{key}')",
object.to_js_string()
));
}
let (methods, proto, kind): (&[(&str, Method)], &[&str], &str) = match object {
JsValue::String(_) => (STRING_METHODS, STRING_PROTO, "a string"),
JsValue::Doc(Json::Array(_)) | JsValue::Array(_) => {
(ARRAY_METHODS, ARRAY_PROTO, "an array")
}
JsValue::Number(_) => (NUMBER_METHODS, NUMBER_PROTO, "a number"),
JsValue::Regex(_) => (REGEX_METHODS, REGEX_PROTO, "a regular expression"),
JsValue::Bool(_) => (&[], BOOLEAN_PROTO, "a boolean"),
JsValue::Function(_) => (&[], FUNCTION_PROTO, "a function"),
JsValue::Doc(_) | JsValue::Undefined | JsValue::Null => (&[], &[], "an object"),
};
if let Some((_, method)) = methods.iter().find(|(name, _)| *name == key) {
return Ok(JsValue::Function(Rc::new(BoundMethod {
receiver: object.clone(),
method: *method,
})));
}
if proto.contains(&key) || OBJECT_PROTO.contains(&key) {
return Err(format!("{key} is not supported on {kind}"));
}
Ok(JsValue::Undefined)
}
fn utf16(s: &str) -> Vec<u16> {
s.encode_utf16().collect()
}
fn from_utf16(units: &[u16]) -> String {
String::from_utf16_lossy(units)
}
fn find_units(haystack: &[u16], needle: &[u16], from: usize) -> Option<usize> {
if needle.is_empty() {
return (from <= haystack.len()).then_some(from);
}
(from..=haystack.len().checked_sub(needle.len())?).find(|&i| haystack[i..].starts_with(needle))
}
fn clamp_index(n: f64, len: usize) -> usize {
n.clamp(0.0, len as f64) as usize
}
/// A relative start or end index, as `slice` and `includes` take them.
fn relative_index(n: f64, len: usize) -> usize {
if n < 0.0 { clamp_index(len as f64 + n, len) } else { clamp_index(n, len) }
}
fn arg<'v, 'a>(args: &'v [JsValue<'a>], i: usize) -> &'v JsValue<'a> {
args.get(i).unwrap_or(&JsValue::Undefined)
}
pub fn call_method<'a>(
bound: &BoundMethod<'a>,
args: &[JsValue<'a>],
) -> Result<JsValue<'a>, String> {
let receiver = &bound.receiver;
match bound.method {
Method::StringIncludes
| Method::StringStartsWith
| Method::StringEndsWith
| Method::StringIndexOf
| Method::StringLastIndexOf
| Method::StringToLowerCase
| Method::StringToUpperCase
| Method::StringTrim
| Method::StringTrimStart
| Method::StringTrimEnd
| Method::StringSlice
| Method::StringSubstring
| Method::StringSplit
| Method::StringMatch
| Method::StringToString => string_method(bound.method, &receiver.to_js_string(), args),
Method::ArrayIncludes
| Method::ArrayIndexOf
| Method::ArrayJoin
| Method::ArrayToString => {
let items: Vec<JsValue<'a>> = match receiver {
JsValue::Doc(Json::Array(items)) => items.iter().map(JsValue::from_doc).collect(),
JsValue::Array(array) => array.items.clone(),
_ => Vec::new(),
};
array_method(bound.method, &items, args)
}
Method::NumberToString => {
let n = receiver.to_number();
let radix = arg(args, 0);
let radix = if matches!(radix, JsValue::Undefined) { 10.0 } else { radix.to_integer() };
if !(2.0..=36.0).contains(&radix) {
return Err("toString() radix must be between 2 and 36".to_string());
}
Ok(JsValue::string(number_to_radix(n, radix as u32)))
}
Method::NumberToFixed => {
let digits = arg(args, 0).to_integer();
if !(0.0..=100.0).contains(&digits) {
return Err("toFixed() digits argument must be between 0 and 100".to_string());
}
Ok(JsValue::string(to_fixed(receiver.to_number(), digits as usize)))
}
Method::RegexTest => match receiver {
JsValue::Regex(re) => re.test(&arg(args, 0).to_js_string()).map(JsValue::Bool),
_ => Ok(JsValue::Bool(false)),
},
}
}
fn string_method<'a>(method: Method, s: &str, args: &[JsValue<'a>]) -> Result<JsValue<'a>, String> {
let units = utf16(s);
let len = units.len();
let search_string = |i: usize| -> Result<Vec<u16>, String> {
if let JsValue::Regex(_) = arg(args, i) {
return Err(format!(
"First argument to String.prototype.{} must not be a regular expression",
method.name()
));
}
Ok(utf16(&arg(args, i).to_js_string()))
};
Ok(match method {
Method::StringIncludes => {
let needle = search_string(0)?;
let from = clamp_index(arg(args, 1).to_integer(), len);
JsValue::Bool(find_units(&units, &needle, from).is_some())
}
Method::StringStartsWith => {
let needle = search_string(0)?;
let from = clamp_index(arg(args, 1).to_integer(), len);
JsValue::Bool(units[from..].starts_with(&needle))
}
Method::StringEndsWith => {
let needle = search_string(0)?;
let end = match arg(args, 1) {
JsValue::Undefined => len,
end => clamp_index(end.to_integer(), len),
};
JsValue::Bool(units[..end].ends_with(&needle))
}
Method::StringIndexOf => {
let needle = utf16(&arg(args, 0).to_js_string());
let from = clamp_index(arg(args, 1).to_integer(), len);
JsValue::Number(find_units(&units, &needle, from).map_or(-1.0, |i| i as f64))
}
Method::StringLastIndexOf => {
let needle = utf16(&arg(args, 0).to_js_string());
let position = arg(args, 1).to_number();
let start = if position.is_nan() { len } else { clamp_index(position.trunc(), len) };
let last = len.checked_sub(needle.len()).map(|max| start.min(max));
let found =
last.and_then(|last| (0..=last).rev().find(|&i| units[i..].starts_with(&needle)));
JsValue::Number(found.map_or(-1.0, |i| i as f64))
}
Method::StringToLowerCase => JsValue::string(s.to_lowercase()),
Method::StringToUpperCase => JsValue::string(s.to_uppercase()),
Method::StringTrim => JsValue::string(s.trim_matches(is_js_whitespace)),
Method::StringTrimStart => JsValue::string(s.trim_start_matches(is_js_whitespace)),
Method::StringTrimEnd => JsValue::string(s.trim_end_matches(is_js_whitespace)),
Method::StringSlice => {
let start = relative_index(arg(args, 0).to_integer(), len);
let end = match arg(args, 1) {
JsValue::Undefined => len,
end => relative_index(end.to_integer(), len),
};
JsValue::string(if start < end {
from_utf16(&units[start..end])
} else {
String::new()
})
}
Method::StringSubstring => {
let start = clamp_index(arg(args, 0).to_integer(), len);
let end = match arg(args, 1) {
JsValue::Undefined => len,
end => clamp_index(end.to_integer(), len),
};
JsValue::string(from_utf16(&units[start.min(end)..start.max(end)]))
}
Method::StringSplit => split(s, &units, arg(args, 0), arg(args, 1))?,
Method::StringMatch => {
let regex = match arg(args, 0) {
JsValue::Regex(re) => re.clone(),
JsValue::Undefined => Rc::new(JsRegex::new("(?:)", "")?),
other => Rc::new(JsRegex::new(&other.to_js_string(), "")?),
};
regex.match_string(s)?
}
Method::StringToString => JsValue::string(s),
Method::ArrayIncludes
| Method::ArrayIndexOf
| Method::ArrayJoin
| Method::ArrayToString
| Method::NumberToString
| Method::NumberToFixed
| Method::RegexTest => JsValue::Undefined,
})
}
fn split<'a>(
s: &str,
units: &[u16],
separator: &JsValue<'a>,
limit: &JsValue<'a>,
) -> Result<JsValue<'a>, String> {
let limit = match limit {
JsValue::Undefined => u32::MAX,
limit => limit.to_uint32(),
} as usize;
if let JsValue::Regex(re) = separator {
let parts = re.split(s, limit)?;
return Ok(JsValue::array(parts));
}
if limit == 0 {
return Ok(JsValue::array(Vec::new()));
}
if let JsValue::Undefined = separator {
return Ok(JsValue::array(vec![JsValue::string(s)]));
}
let separator = utf16(&separator.to_js_string());
let parts: Vec<JsValue<'a>> = if separator.is_empty() {
units.iter().map(|u| JsValue::string(from_utf16(&[*u]))).collect()
} else {
let mut parts = Vec::new();
let mut start = 0;
while let Some(i) = find_units(units, &separator, start) {
parts.push(JsValue::string(from_utf16(&units[start..i])));
start = i + separator.len();
}
parts.push(JsValue::string(from_utf16(&units[start..])));
parts
};
Ok(JsValue::array(parts.into_iter().take(limit).collect()))
}
fn array_method<'a>(
method: Method,
items: &[JsValue<'a>],
args: &[JsValue<'a>],
) -> Result<JsValue<'a>, String> {
let len = items.len();
Ok(match method {
Method::ArrayIncludes => {
let from = relative_index(arg(args, 1).to_integer(), len);
JsValue::Bool(items[from..].iter().any(|item| item.same_value_zero(arg(args, 0))))
}
Method::ArrayIndexOf => {
let from = relative_index(arg(args, 1).to_integer(), len);
let found = items[from..].iter().position(|item| item.strict_eq(arg(args, 0)));
JsValue::Number(found.map_or(-1.0, |i| (i + from) as f64))
}
Method::ArrayJoin => {
let separator = match arg(args, 0) {
JsValue::Undefined => ",".to_string(),
separator => separator.to_js_string(),
};
JsValue::string(join_values(items, &separator))
}
Method::ArrayToString => JsValue::string(join_values(items, ",")),
Method::StringIncludes
| Method::StringStartsWith
| Method::StringEndsWith
| Method::StringIndexOf
| Method::StringLastIndexOf
| Method::StringToLowerCase
| Method::StringToUpperCase
| Method::StringTrim
| Method::StringTrimStart
| Method::StringTrimEnd
| Method::StringSlice
| Method::StringSubstring
| Method::StringSplit
| Method::StringMatch
| Method::StringToString
| Method::NumberToString
| Method::NumberToFixed
| Method::RegexTest => JsValue::Undefined,
})
}
/// V8's `DoubleToRadixCString`, digit for digit.
fn number_to_radix(value: f64, radix: u32) -> String {
if radix == 10 || !value.is_finite() {
return number_to_string(value);
}
let digit_char = |d: u32| std::char::from_digit(d, radix).unwrap_or('0');
let next_double = |x: f64| f64::from_bits(x.to_bits() + 1);
let exponent = |x: f64| {
let biased = ((x.to_bits() >> 52) & 0x7FF) as i32;
if biased == 0 { -1074 } else { biased - 1075 }
};
let radix_f = f64::from(radix);
let negative = value < 0.0;
let value = value.abs();
let mut integer = value.floor();
let mut fraction = value - integer;
let mut delta = (0.5 * (next_double(value) - value)).max(next_double(0.0));
let mut fraction_digits: Vec<u32> = Vec::new();
let mut has_point = false;
if fraction >= delta {
has_point = true;
loop {
fraction *= radix_f;
delta *= radix_f;
let digit = fraction as u32;
fraction_digits.push(digit);
fraction -= f64::from(digit);
if (fraction > 0.5 || (fraction == 0.5 && digit & 1 == 1)) && fraction + delta > 1.0 {
loop {
match fraction_digits.pop() {
None => {
integer += 1.0;
break;
}
Some(d) if d + 1 < radix => {
fraction_digits.push(d + 1);
break;
}
Some(_) => {}
}
}
break;
}
if fraction < delta {
break;
}
}
}
let mut integer_digits: Vec<char> = Vec::new();
while exponent(integer / radix_f) > 0 {
integer /= radix_f;
integer_digits.push('0');
}
loop {
let remainder = integer % radix_f;
integer_digits.push(digit_char(remainder as u32));
integer = (integer - remainder) / radix_f;
if integer <= 0.0 {
break;
}
}
let mut out = String::new();
if negative {
out.push('-');
}
out.extend(integer_digits.iter().rev());
if has_point && !fraction_digits.is_empty() {
out.push('.');
out.extend(fraction_digits.iter().map(|d| digit_char(*d)));
}
out
}
/// `Number.prototype.toFixed`: exact decimal value, ties away from zero.
fn to_fixed(x: f64, digits: usize) -> String {
if !x.is_finite() || x.abs() >= 1e21 {
return number_to_string(x);
}
let exact = format!("{:.1100}", x.abs());
let (int_part, frac_part) = exact.split_once('.').unwrap_or((&exact, ""));
let mut kept: Vec<u8> = int_part.bytes().chain(frac_part.bytes().take(digits)).collect();
if frac_part.as_bytes().get(digits).is_some_and(|d| *d >= b'5') {
let mut i = kept.len();
loop {
if i == 0 {
kept.insert(0, b'1');
break;
}
i -= 1;
if kept[i] == b'9' {
kept[i] = b'0';
} else {
kept[i] += 1;
break;
}
}
}
let int_len = kept.len() - digits;
let mut out = String::from_utf8_lossy(&kept[..int_len]).into_owned();
if digits > 0 {
out.push('.');
out.push_str(&String::from_utf8_lossy(&kept[int_len..]));
}
if x < 0.0 { format!("-{out}") } else { out }
}
+57
View File
@@ -0,0 +1,57 @@
//! JSONPath (jsonpath-plus, with RFC 9535 additions) and XPath 1.0 filtering
//! of response text.
mod jsonpath;
mod xml;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Language {
JsonPath,
XPath,
}
impl Language {
/// XML and HTML bodies take XPath; everything else is treated as JSON.
pub fn for_content_type(content_type: &str) -> Self {
let content_type = content_type.to_lowercase();
if content_type.contains("xml") || content_type.contains("html") {
Language::XPath
} else {
Language::JsonPath
}
}
}
/// The filtered text, or an empty string and the reason it could not be filtered.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct FilterResult {
pub content: String,
pub error: Option<String>,
}
impl FilterResult {
fn from_result(result: Result<String, String>) -> Self {
match result {
Ok(content) => FilterResult { content, error: None },
Err(error) => FilterResult { content: String::new(), error: Some(error) },
}
}
}
pub fn filter(language: Language, text: &str, query: &str) -> FilterResult {
match language {
Language::JsonPath => json_path(text, query),
Language::XPath => xpath(text, query),
}
}
/// Every match, as a pretty-printed JSON array.
pub fn json_path(text: &str, query: &str) -> FilterResult {
FilterResult::from_result(jsonpath::evaluate(text, query))
}
/// Matched nodes serialized as XML, one per line, or the string form of a
/// number, string, or boolean result.
pub fn xpath(text: &str, query: &str) -> FilterResult {
FilterResult::from_result(xml::evaluate(text, query))
}
+206
View File
@@ -0,0 +1,206 @@
use std::collections::BTreeMap;
use sxd_document::dom::{ChildOfElement, ChildOfRoot, Element, Text};
use sxd_document::parser;
use sxd_xpath::nodeset::Node;
use sxd_xpath::{Context, Factory, Value};
pub(crate) fn evaluate(text: &str, query: &str) -> Result<String, String> {
let xpath = Factory::new()
.build(query)
.map_err(|e| format!("Invalid filter: {e}"))?
.ok_or_else(|| "Invalid filter: empty expression".to_string())?;
// sxd only takes a DOCTYPE after some prolog token, and whitespace is one.
let text = if text.starts_with("<!DOCTYPE") { format!(" {text}") } else { text.to_string() };
let package = parser::parse(&text).map_err(|e| format!("Invalid XML: {e}"))?;
let document = package.as_document();
for child in document.root().children() {
if let ChildOfRoot::Element(e) = child {
merge_adjacent_text(e);
}
}
let value = xpath
.evaluate(&Context::new(), document.root())
.map_err(|e| format!("Invalid filter: {e}"))?;
Ok(match value {
Value::Nodeset(nodes) => {
nodes.document_order().into_iter().map(serialize_node).collect::<Vec<_>>().join("\n")
}
Value::String(s) => s,
Value::Boolean(b) => b.to_string(),
Value::Number(n) => crate::jsonpath::number_to_string(n),
})
}
/// The parser splits text at every entity and CDATA section; XPath sees one text node.
fn merge_adjacent_text(element: Element) {
let mut previous: Option<Text> = None;
for child in element.children() {
match child {
ChildOfElement::Text(text) => match previous {
Some(previous) => {
previous.set_text(&format!("{}{}", previous.text(), text.text()));
text.remove_from_parent();
}
None => previous = Some(text),
},
ChildOfElement::Element(e) => {
previous = None;
merge_adjacent_text(e);
}
ChildOfElement::Comment(_) | ChildOfElement::ProcessingInstruction(_) => {
previous = None
}
}
}
}
/// Namespace declarations already written, keyed by prefix (`None` is the default).
type Scope<'d> = BTreeMap<Option<&'d str>, &'d str>;
fn serialize_node(node: Node) -> String {
let mut out = String::new();
match node {
Node::Root(root) => {
for child in root.children() {
match child {
ChildOfRoot::Element(e) => write_element(&mut out, e, &Scope::new()),
ChildOfRoot::Comment(c) => write_comment(&mut out, c.text()),
ChildOfRoot::ProcessingInstruction(pi) => {
write_pi(&mut out, pi.target(), pi.value())
}
}
}
}
Node::Element(e) => write_element(&mut out, e, &Scope::new()),
Node::Attribute(a) => {
let name = node.prefixed_name().unwrap_or_else(|| a.name().local_part().to_string());
write_attribute(&mut out, &name, a.value());
}
Node::Text(t) => escape_into(&mut out, t.text(), false),
Node::Comment(c) => write_comment(&mut out, c.text()),
Node::ProcessingInstruction(pi) => write_pi(&mut out, pi.target(), pi.value()),
Node::Namespace(ns) => {
write_attribute(&mut out, &format!("xmlns:{}", ns.prefix()), ns.uri())
}
}
out
}
/// Elements declare the namespaces they use where they first need them, as xmldom does.
fn write_element<'d>(out: &mut String, element: Element<'d>, parent_scope: &Scope<'d>) {
let mut scope = parent_scope.clone();
let mut declarations = Vec::new();
let mut declare = |prefix: Option<&'d str>, uri: &'d str| {
if prefix != Some("xml") && scope.get(&prefix) != Some(&uri) {
scope.insert(prefix, uri);
declarations.push((prefix, uri));
}
};
let name = element.name();
let name = match name.namespace_uri() {
Some(uri) => match element.prefix_for_namespace_uri(uri, element.preferred_prefix()) {
Some(prefix) => {
declare(Some(prefix), uri);
format!("{prefix}:{}", name.local_part())
}
None => {
declare(None, uri);
name.local_part().to_string()
}
},
None => {
if parent_scope.get(&None).is_some_and(|uri| !uri.is_empty()) {
declare(None, "");
}
name.local_part().to_string()
}
};
let mut attributes = Vec::new();
for attribute in element.attributes() {
let attr_name = attribute.name();
let prefix = attr_name.namespace_uri().and_then(|uri| {
Some((element.prefix_for_namespace_uri(uri, attribute.preferred_prefix())?, uri))
});
let attr_name = match prefix {
Some((prefix, uri)) => {
declare(Some(prefix), uri);
format!("{prefix}:{}", attr_name.local_part())
}
None => attr_name.local_part().to_string(),
};
attributes.push((attr_name, attribute.value()));
}
out.push('<');
out.push_str(&name);
for (attr_name, value) in attributes {
out.push(' ');
write_attribute(out, &attr_name, value);
}
for (prefix, uri) in declarations {
out.push(' ');
match prefix {
Some(prefix) => write_attribute(out, &format!("xmlns:{prefix}"), uri),
None => write_attribute(out, "xmlns", uri),
}
}
let children = element.children();
if children.is_empty() {
out.push_str("/>");
return;
}
out.push('>');
for child in children {
match child {
ChildOfElement::Element(e) => write_element(out, e, &scope),
ChildOfElement::Text(t) => escape_into(out, t.text(), false),
ChildOfElement::Comment(c) => write_comment(out, c.text()),
ChildOfElement::ProcessingInstruction(pi) => write_pi(out, pi.target(), pi.value()),
}
}
out.push_str("</");
out.push_str(&name);
out.push('>');
}
fn write_attribute(out: &mut String, name: &str, value: &str) {
out.push_str(name);
out.push_str("=\"");
escape_into(out, value, true);
out.push('"');
}
fn write_comment(out: &mut String, text: &str) {
out.push_str("<!--");
out.push_str(text);
out.push_str("-->");
}
fn write_pi(out: &mut String, target: &str, value: Option<&str>) {
out.push_str("<?");
out.push_str(target);
if let Some(value) = value {
out.push(' ');
out.push_str(value);
}
out.push_str("?>");
}
fn escape_into(out: &mut String, text: &str, attribute: bool) {
for c in text.chars() {
match c {
'<' => out.push_str("&lt;"),
'>' => out.push_str("&gt;"),
'&' => out.push_str("&amp;"),
'"' if attribute => out.push_str("&quot;"),
'\t' if attribute => out.push_str("&#9;"),
'\n' if attribute => out.push_str("&#10;"),
'\r' if attribute => out.push_str("&#13;"),
c => out.push(c),
}
}
}
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+300
View File
@@ -0,0 +1,300 @@
//! Ported from the `@yaak/filter-jsonpath` plugin's tests.
use serde_json::{Value, json};
use yaak_filter::{FilterResult, Language, filter, json_path};
fn matches(json: &Value, path: &str) -> Value {
let result = json_path(&json.to_string(), path);
assert_eq!(result.error, None, "{path}");
serde_json::from_str(&result.content).unwrap()
}
/// The path the response breadcrumb builds for a key (`segmentsToJsonPath` in the client).
fn crumb(key: &str) -> String {
let bare = key != "$"
&& key.chars().next().is_some_and(|c| c.is_ascii_alphabetic() || c == '_' || c == '$')
&& key.chars().all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '$');
if bare { format!(".{key}") } else { format!("[{}]", serde_json::to_string(key).unwrap()) }
}
#[test]
fn breadcrumb_selects_the_literal_key_not_an_operator_or_expression() {
let keys = [
"identifier",
"a.b",
"space key",
"",
"quote\"key",
"single'key",
"back\\slash",
"bracket]key",
"combined\"\\]'key",
"*",
"..",
"$",
"$foo",
"^",
"~",
"a,b",
"0:2",
"?(true)",
"(@.length-1)",
"@number()",
"`key",
"x)]y",
"x)']y",
"semi;colon",
"percent%@%key",
"line\nbreak\t\u{0}",
"emoji 🤔",
"__proto__",
"constructor",
];
for key in keys {
let mut object = serde_json::Map::new();
object.insert(key.to_string(), json!("needle-value"));
object.insert("decoy".to_string(), json!("wrong-value"));
let path = format!("${}", crumb(key));
assert_eq!(matches(&Value::Object(object), &path), json!(["needle-value"]), "{key:?}");
}
}
#[test]
fn nested_unusual_keys_and_arrays_filter_at_every_crumb() {
let inner = json!({ "inner'\\]": { "": "needle-value" } });
let json = json!({ "outer": { "quote\"key": ["decoy", inner] } });
let crumbs = [
crumb("outer"),
crumb("quote\"key"),
"[1]".to_string(),
crumb("inner'\\]"),
crumb(""),
];
let path = |count: usize| format!("${}", crumbs[..count].concat());
assert_eq!(matches(&json, &path(5)), json!(["needle-value"]));
assert_eq!(matches(&json, &path(4)), json!([{ "": "needle-value" }]));
assert_eq!(matches(&json, &path(3)), json!([inner]));
assert_eq!(matches(&json, &path(2)), json!([["decoy", inner]]));
assert_eq!(matches(&json, &path(1)), json!([json["outer"]]));
assert_eq!(matches(&json, &path(0)), json!([json]));
}
#[test]
fn root_arrays_and_quoted_numeric_object_keys_remain_distinct() {
let json = json!(["decoy", { "01": { "1": "needle-value" } }]);
assert_eq!(matches(&json, r#"$[1]["01"]["1"]"#), json!(["needle-value"]));
assert_eq!(matches(&json, r#"$[1]["01"]["missing"]"#), json!([]));
}
#[test]
fn retains_the_matched_value() {
for value in [
json!(null),
json!(false),
json!(0),
json!(""),
json!([1, 2]),
json!({ "value": 3 }),
] {
let json = json!({ "quote\"key": value });
assert_eq!(matches(&json, r#"$["quote\"key"]"#), json!([value]));
}
}
#[test]
fn only_follows_own_properties_while_allowing_prototype_named_keys() {
let json: Value =
serde_json::from_str(r#"{"a.b":{"__proto__":{"constructor":"own-value"}}}"#).unwrap();
assert_eq!(matches(&json, r#"$["a.b"].__proto__.constructor"#), json!(["own-value"]));
assert_eq!(matches(&json, r#"$["a.b"].constructor"#), json!([]));
assert_eq!(matches(&json, r#"$["a.b"].__proto__.toString"#), json!([]));
assert_eq!(matches(&json, r#"$["a.b"].__proto__.constructor.__proto__"#), json!([]));
}
#[test]
fn existing_jsonpath_expressions() {
let json = json!({ "items": [{ "id": 1 }, { "id": 2 }, { "id": 3 }], "space key": 4 });
let cases = [
("$", json!([json])),
("$.items[1].id", json!([2])),
(r#"$["space key"]"#, json!([4])),
("$['space key']", json!([4])),
("$.items[*].id", json!([1, 2, 3])),
("$..id", json!([1, 2, 3])),
("$.items[0:2].id", json!([1, 2])),
("$.items[0,2].id", json!([1, 3])),
("$.items[-1].id", json!([3])),
("$.items[?(@.id > 1)].id", json!([2, 3])),
("$.items[?@.id > 1].id", json!([2, 3])),
(r#"$.items[?(@["id"] > 1)].id"#, json!([2, 3])),
(r#"$["items"][*].id"#, json!([1, 2, 3])),
("$.items[?length(@) == 1].id", json!([1, 2, 3])),
("$.missing", json!([])),
];
for (path, expected) in cases {
assert_eq!(matches(&json, path), expected, "{path}");
}
}
fn error(json: &Value, path: &str) -> String {
let result = json_path(&json.to_string(), path);
assert_eq!(result.content, "", "{path}");
let error = result.error.unwrap_or_else(|| panic!("{path} should fail"));
assert!(error.starts_with("Invalid filter: "), "{path}: {error}");
error
}
#[test]
fn jsonpath_plus_extensions_work() {
let json = json!({ "items": [{ "id": 1 }, { "id": 2 }, { "id": 3 }], "space key": 4 });
let cases = [
("$.items[(@.length-1)].id", json!([3])),
(r#"$["items"].$"#, json!([json["items"]])),
("$.items[?(@.id.toString() == '1')]", json!([{ "id": 1 }])),
("$.items[?(@property == 1)].id", json!([2])),
("$.items[?(@.id > 1)]^", json!([json["items"], json["items"]])),
("$.items[*]~", json!([0, 1, 2])),
("$..*@number()", json!([4, 1, 2, 3])),
("$.items[?(@.id == '2')].id", json!([2])),
("$.items[?(/^[13]$/.test(@.id))].id", json!([1, 3])),
("$.`space key", json!([4])),
];
for (path, expected) in cases {
assert_eq!(matches(&json, path), expected, "{path}");
}
}
#[test]
fn calls_outside_the_allowlist_and_statements_are_errors() {
let json = json!({ "items": [{ "id": 1, "name": "a" }], "none": [] });
for (path, named) in [
("$.items[?(@.name.padStart(2) == ' a')]", "padStart"),
("$.items[?(@.name['padStart'](2) == ' a')]", "padStart"),
("$.items[?(@.hasOwnProperty('id'))]", "hasOwnProperty"),
("$.items[?(@.id; true)]", "multiple expressions"),
("$.items[?(var x = 1; x)]", "var"),
("$.items[?(@.id = 2)]", "assignment"),
("$.items[?(@.id++)]", "++"),
("$.items[?(@.id ** 2)]", "**"),
("$.items[?(foo(@.id))]", "foo()"),
] {
let message = error(&json, path);
assert!(message.contains(named), "{path}: {message}");
}
}
#[test]
fn deviation_eager_parse_rejects_filters_no_node_reaches() {
let json = json!({ "none": [] });
assert!(error(&json, "$.none[?(@.a ==)]").contains("Expected expression"));
assert!(error(&json, "$.none[?(@.a.padStart(1))]").contains("padStart"));
}
#[test]
fn deviation_rfc_negative_index_and_step() {
let json = json!({ "a": [0, 1, 2, 3] });
assert_eq!(matches(&json, "$.a[-1]"), json!([3]));
assert_eq!(matches(&json, "$.a[-5]"), json!([]));
assert_eq!(matches(&json, "$.a[::-1]"), json!([3, 2, 1, 0]));
assert_eq!(matches(&json, "$.a[3:0:-2]"), json!([3, 1]));
// Parts of a union keep jsonpath-plus's reading, so its non-empty results stand.
assert_eq!(matches(&json, "$.a[0,-1]"), json!([0]));
}
#[test]
fn deviation_rfc_filter_without_parentheses() {
let json = json!({ "a": [{ "x": 1 }, { "x": 0 }, { "y": 2 }] });
assert_eq!(matches(&json, "$.a[?@.x]"), json!([{ "x": 1 }, { "x": 0 }]));
assert_eq!(matches(&json, "$.a[?@.x == '1']"), json!([]));
assert_eq!(matches(&json, "$.a[?@.x < 1 || @.y]"), json!([{ "x": 0 }, { "y": 2 }]));
assert!(error(&json, "$.a[?@.* == 1]").contains("cannot be compared"));
}
#[test]
fn deviation_rfc_functions() {
let json = json!({ "a": [{ "n": "ab", "t": [1, 2] }, { "n": "b", "t": [] }] });
assert_eq!(matches(&json, "$.a[?length(@.n) == 2].n"), json!(["ab"]));
assert_eq!(matches(&json, "$.a[?(count(@.t[*]) == 0)].n"), json!(["b"]));
assert_eq!(matches(&json, "$.a[?match(@.n, 'a.')].n"), json!(["ab"]));
assert_eq!(matches(&json, "$.a[?(search(@.n, 'b'))].n"), json!(["ab", "b"]));
assert_eq!(matches(&json, "$.a[?value(@.t[0]) == 1].n"), json!(["ab"]));
assert!(error(&json, "$.a[?length(@.n)]").contains("compare it"));
}
#[test]
fn deviation_falsy_root_is_a_document() {
for root in ["0", "\"\"", "false", "null"] {
assert_eq!(json_path(root, "$").content, format!("[\n {root}\n]"), "{root}");
}
}
#[test]
fn deviation_lone_surrogate_becomes_replacement_character() {
assert_eq!(matches(&json!({ "s": "🤔" }), "$.s[0]"), json!(["\u{fffd}"]));
}
#[test]
fn integer_like_keys_come_first_as_in_javascript() {
let result = json_path(r#"{"b":1,"10":2,"a":3,"2":4}"#, "$.*~");
assert_eq!(result.content, "[\n \"2\",\n \"10\",\n \"b\",\n \"a\"\n]");
}
#[test]
fn output_is_pretty_printed_and_keeps_key_order() {
let result = json_path(r#"{"b":{"z":1,"a":[true,null]}}"#, "$.b");
assert_eq!(
result,
FilterResult {
content: "[\n {\n \"z\": 1,\n \"a\": [\n true,\n null\n ]\n }\n]"
.to_string(),
error: None,
}
);
}
#[test]
fn invalid_json_is_an_error() {
let result = json_path("{not json", "$");
assert_eq!(result.content, "");
assert!(result.error.unwrap().starts_with("Invalid JSON: "));
}
#[test]
fn content_type_picks_the_language() {
assert_eq!(Language::for_content_type("application/json"), Language::JsonPath);
assert_eq!(Language::for_content_type("text/plain"), Language::JsonPath);
assert_eq!(Language::for_content_type(""), Language::JsonPath);
assert_eq!(Language::for_content_type("application/xml"), Language::XPath);
assert_eq!(Language::for_content_type("application/rss+XML"), Language::XPath);
assert_eq!(Language::for_content_type("text/html; charset=utf-8"), Language::XPath);
assert_eq!(filter(Language::JsonPath, "[1]", "$[0]").content, "[\n 1\n]");
}
#[test]
fn comparing_an_array_to_an_object_is_false() {
let result = json_path(r#"[{"a":[],"b":{}},{"a":[1],"b":[1]}]"#, "$[?@.a == @.b].a");
assert_eq!(result.content, "[\n [\n 1\n ]\n]");
}
#[test]
fn invalid_escape_before_a_multibyte_character_is_an_error() {
let result = json_path("{\"a\":\"\\é\"}", "$");
assert!(result.error.unwrap().starts_with("Invalid JSON: "));
}
#[test]
fn deeply_nested_filters_are_errors() {
let nested = |open: &str, inner: &str, close: &str| {
format!("{}{inner}{}", open.repeat(5000), close.repeat(5000))
};
for query in [
format!("$[?{} == 1]", nested("length(", "@", ")")),
format!("$[?{}]", nested("(", "@", ")")),
format!("$[?({} == 1)]", nested("(", "@", ")")),
nested("$[?@", "", "]"),
] {
let result = json_path("[[1]]", &query);
assert!(result.error.unwrap().starts_with("Invalid filter: "), "{}", &query[..20]);
}
}
+77
View File
@@ -0,0 +1,77 @@
//! Every case in `fixtures/jsonpath_plus.json`, which
//! `fixtures/generate-jsonpath-plus.mjs` records from jsonpath-plus 11.0.0 as
//! the removed plugin ran it.
use serde_json::Value;
use std::collections::BTreeMap;
use yaak_filter::json_path;
/// Where this engine means to differ from jsonpath-plus; the fixture holds the
/// result each such case should give instead.
const DEVIATIONS: &[&str] = &[
// `["a.b"]` and `['a.b']` are literal keys, never operators or expressions.
"literal-key",
// RFC 9535 `[-1]`, which jsonpath-plus looks up as the key "-1".
"rfc-negative-index",
// RFC 9535 `[::-1]`, which jsonpath-plus does not read as a slice.
"rfc-negative-step",
// RFC 9535 `[?@.a > 1]`, which jsonpath-plus looks up as a key.
"rfc-filter",
// RFC 9535 `length()`, `count()`, `match()`, `search()` and `value()` in `?(...)`.
"rfc-function",
// A call outside the allowlist, or several statements: an error, not JavaScript.
"unsupported",
// Filters are compiled up front, so a bad one fails even when no node reaches it.
"eager-parse",
// jsonpath-plus returns undefined for a root of 0, "", false or null.
"falsy-root",
// Indexing into a string can split a surrogate pair, which becomes U+FFFD here.
"lone-surrogate",
];
#[test]
fn matches_jsonpath_plus_except_named_deviations() {
let fixture: Value = serde_json::from_str(include_str!("fixtures/jsonpath_plus.json")).unwrap();
let docs = fixture["docs"].as_object().unwrap();
let cases = fixture["cases"].as_array().unwrap();
assert!(cases.len() >= 300, "{} cases", cases.len());
let mut failures = Vec::new();
let mut per_deviation: BTreeMap<&str, usize> = BTreeMap::new();
for case in cases {
let doc = docs[case["doc"].as_str().unwrap()].as_str().unwrap();
let path = case["path"].as_str().unwrap();
let expected = &case["expected"];
let label = match case["deviation"].as_str() {
Some(deviation) => {
assert!(
DEVIATIONS.contains(&deviation),
"unknown deviation {deviation} for {path}"
);
*per_deviation.entry(deviation).or_default() += 1;
format!("{} {path} [{deviation}]", case["doc"])
}
None => format!("{} {path}", case["doc"]),
};
let result = json_path(doc, path);
let ok = if expected["error"] == Value::Bool(true) {
result.content.is_empty()
&& result.error.as_ref().is_some_and(|e| e.starts_with("Invalid filter: "))
} else {
result.error.is_none() && Some(result.content.as_str()) == expected["content"].as_str()
};
if !ok {
failures.push(format!("{label}\n expected: {expected}\n got: {result:?}"));
}
}
for deviation in DEVIATIONS {
assert!(per_deviation.contains_key(deviation), "no case covers {deviation}");
}
assert!(
failures.is_empty(),
"{} of {} cases differ:\n{}",
failures.len(),
cases.len(),
failures.join("\n")
);
}
+105
View File
@@ -0,0 +1,105 @@
//! Expected output is what the `@yaak/filter-xpath` plugin (xmldom and xpath)
//! returned for the same input, except where a test says otherwise.
use yaak_filter::{Language, filter, xpath};
const ITEMS: &str = r#"<root><item id="1">A</item><item id="2">B &amp; C</item><empty/></root>"#;
const FEED: &str = r#"<feed xmlns="http://www.w3.org/2005/Atom" xmlns:m="urn:m"><entry><title>T</title><m:thumb url="x"/></entry></feed>"#;
const MIXED: &str =
r#"<?xml version="1.0"?><!-- c --><a x="1&quot;2"><!--note--><b><![CDATA[<raw>]]></b></a>"#;
fn content(xml: &str, query: &str) -> String {
let result = xpath(xml, query);
assert_eq!(result.error, None, "{query}");
result.content
}
#[test]
fn node_sets_serialize_one_node_per_line() {
assert_eq!(
content(ITEMS, "//item"),
"<item id=\"1\">A</item>\n<item id=\"2\">B &amp; C</item>"
);
assert_eq!(content(ITEMS, "//item/text()"), "A\nB &amp; C");
assert_eq!(content(ITEMS, "//empty"), "<empty/>");
assert_eq!(content(ITEMS, "//missing"), "");
assert_eq!(content(ITEMS, "//item[@id='2']/.."), ITEMS);
assert_eq!(content(MIXED, "//comment()"), "<!-- c -->\n<!--note-->");
}
/// The plugin printed a leading space before each attribute (` id="1"`).
#[test]
fn attributes_serialize_as_name_and_value() {
assert_eq!(content(ITEMS, "//item/@id"), "id=\"1\"\nid=\"2\"");
assert_eq!(content(MIXED, "/a/@x"), "x=\"1&quot;2\"");
}
#[test]
fn scalar_results_print_like_javascript() {
assert_eq!(content(ITEMS, "count(//item)"), "2");
assert_eq!(content(ITEMS, "string(//item[2])"), "B & C");
assert_eq!(content(ITEMS, "sum(//item/@id) div 4"), "0.75");
assert_eq!(content(ITEMS, "boolean(//empty)"), "true");
assert_eq!(content(ITEMS, "1 div 0"), "Infinity");
assert_eq!(content(ITEMS, "0 div 0"), "NaN");
assert_eq!(content(ITEMS, "-0"), "0");
}
#[test]
fn namespaces_are_declared_where_they_are_used() {
assert_eq!(
content(FEED, "//*[local-name()='entry']"),
r#"<entry xmlns="http://www.w3.org/2005/Atom"><title>T</title><m:thumb url="x" xmlns:m="urn:m"/></entry>"#
);
assert_eq!(content(FEED, "//*[local-name()='thumb']"), r#"<m:thumb url="x" xmlns:m="urn:m"/>"#);
// XPath 1.0 has no default namespace for unprefixed names.
assert_eq!(content(FEED, "//entry"), "");
}
/// The plugin kept the XML declaration and the CDATA section; this parser keeps neither.
#[test]
fn root_serializes_the_whole_document() {
assert_eq!(
content(MIXED, "/"),
r#"<!-- c --><a x="1&quot;2"><!--note--><b>&lt;raw&gt;</b></a>"#
);
assert_eq!(content(MIXED, "//b/text()"), "&lt;raw&gt;");
assert_eq!(content("<a>x<![CDATA[<y>]]>&amp;z</a>", "/a/text()"), "x&lt;y&gt;&amp;z");
}
#[test]
fn doctype_is_accepted() {
assert_eq!(content("<!DOCTYPE note><note>hi</note>", "/note"), "<note>hi</note>");
assert_eq!(content("<?xml version=\"1.0\"?><!DOCTYPE note><note/>", "/note"), "<note/>");
}
#[test]
fn invalid_queries_and_documents_are_errors() {
for (xml, query, prefix) in [
(ITEMS, "//item[", "Invalid filter: "),
(ITEMS, "", "Invalid filter: "),
(ITEMS, "unknown-fn()", "Invalid filter: "),
("<html><body><p>one<br></p></body></html>", "//p", "Invalid XML: "),
("<a><b>", "/a", "Invalid XML: "),
] {
let result = xpath(xml, query);
assert_eq!(result.content, "", "{query}");
assert!(
result.error.as_deref().unwrap_or_default().starts_with(prefix),
"{query}: {result:?}"
);
}
}
#[test]
fn filter_dispatches_to_xpath() {
assert_eq!(filter(Language::XPath, ITEMS, "count(//item)").content, "2");
}
#[test]
fn numbers_switch_to_exponents_where_javascript_does() {
assert_eq!(content("<a>0.0000001</a>", "number(/a)"), "1e-7");
assert_eq!(content("<a>0.000001</a>", "number(/a)"), "0.000001");
assert_eq!(content("<a>1000000000000000000000</a>", "number(/a)"), "1e+21");
assert_eq!(content("<a>100000000000000000000</a>", "number(/a)"), "100000000000000000000");
}
+1 -1
View File
@@ -6,7 +6,7 @@ publish = false
[dependencies]
chrono = { workspace = true, features = ["serde"] }
git2 = { version = "0.20.4", features = ["vendored-libgit2", "vendored-openssl"] }
git2 = { version = "0.21.0", features = ["vendored-libgit2", "vendored-openssl"] }
log = { workspace = true }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
+3 -3
View File
@@ -74,11 +74,11 @@ fn git_log_inner(dir: &Path, rela_path: Option<&Path>) -> crate::error::Result<V
Some(GitCommit {
oid: oid.to_string(),
author: GitAuthor {
name: author.name().map(|s| s.to_string()),
email: author.email().map(|s| s.to_string()),
name: author.name().ok().map(|s| s.to_string()),
email: author.email().ok().map(|s| s.to_string()),
},
when: convert_git_time_to_date(author.when()),
message: commit.message().map(|m| m.to_string()),
message: commit.message().ok().map(|m| m.to_string()),
})
})
.collect();
+5 -4
View File
@@ -40,10 +40,11 @@ pub async fn git_pull(dir: &Path) -> Result<PullResult> {
let repo = open_repo(dir)?;
let branch_name = get_current_branch_name(&repo)?;
let remote = get_default_remote_in_repo(&repo)?;
let remote_name =
remote.name().ok_or(GenericError("Failed to get remote name".to_string()))?.to_string();
let remote_url =
remote.url().ok_or(GenericError("Failed to get remote url".to_string()))?.to_string();
let remote_name = remote
.name()?
.ok_or(GenericError("Failed to get remote name".to_string()))?
.to_string();
let remote_url = remote.url()?.to_string();
(branch_name, remote_name, remote_url)
};
+5 -4
View File
@@ -23,10 +23,11 @@ pub async fn git_push(dir: &Path) -> Result<PushResult> {
let repo = open_repo(dir)?;
let branch_name = get_current_branch_name(&repo)?;
let remote = get_default_remote_for_push_in_repo(&repo)?;
let remote_name =
remote.name().ok_or(GenericError("Failed to get remote name".to_string()))?.to_string();
let remote_url =
remote.url().ok_or(GenericError("Failed to get remote url".to_string()))?.to_string();
let remote_name = remote
.name()?
.ok_or(GenericError("Failed to get remote name".to_string()))?
.to_string();
let remote_url = remote.url()?.to_string();
(branch_name, remote_name, remote_url)
};
+4 -3
View File
@@ -18,8 +18,8 @@ pub fn git_remotes(dir: &Path) -> Result<Vec<GitRemote>> {
for remote in repo.remotes()?.into_iter() {
let name = match remote {
None => continue,
Some(name) => name,
Ok(Some(name)) => name,
_ => continue,
};
let r = match repo.find_remote(name) {
Ok(r) => r,
@@ -28,7 +28,8 @@ pub fn git_remotes(dir: &Path) -> Result<Vec<GitRemote>> {
continue;
}
};
remotes.push(GitRemote { name: name.to_string(), url: r.url().map(|u| u.to_string()) });
remotes
.push(GitRemote { name: name.to_string(), url: r.url().ok().map(|u| u.to_string()) });
}
Ok(remotes)
+7 -7
View File
@@ -87,7 +87,7 @@ pub fn git_worktree_status(dir: &Path) -> crate::error::Result<GitWorktreeStatus
let mut entries = Vec::new();
for entry in repo.statuses(Some(&mut opts))?.into_iter() {
let Some(rela_path) = entry.path() else {
let Ok(rela_path) = entry.path() else {
continue;
};
let Some((status, staged)) = git_status_from_raw(entry.status()) else {
@@ -135,7 +135,7 @@ pub fn git_status(dir: &Path) -> crate::error::Result<GitStatusSummary> {
let obj = entry.to_object(&repo)?;
let content = obj.as_blob().unwrap().content();
let name = Path::new(entry.name().unwrap_or_default());
SyncModel::from_bytes(content.into(), name)?.map(|m| m.0)
SyncModel::from_bytes(content.into(), name).ok().flatten().map(|m| m.0)
}
Err(_) => None,
},
@@ -143,7 +143,7 @@ pub fn git_status(dir: &Path) -> crate::error::Result<GitStatusSummary> {
let next = {
let full_path = repo.workdir().unwrap().join(rela_path.clone());
SyncModel::from_file(full_path.as_path())?.map(|m| m.0)
SyncModel::from_file(full_path.as_path()).ok().flatten().map(|m| m.0)
};
entries.push(GitStatusEntry {
@@ -174,7 +174,7 @@ fn git_branch_info_for_repo(
dir: &Path,
) -> crate::error::Result<GitBranchInfo> {
let (head_ref, head_ref_shorthand) = git_head_refs(repo);
let origins = repo.remotes()?.into_iter().filter_map(|o| Some(o?.to_string())).collect();
let origins = repo.remotes()?.into_iter().filter_map(|o| Some(o.ok()??.to_string())).collect();
let local_branches = local_branch_names(repo)?;
let remote_branches = remote_branch_names(repo)?;
@@ -182,7 +182,7 @@ fn git_branch_info_for_repo(
let (ahead, behind) = (|| -> Option<(usize, usize)> {
let head = repo.head().ok()?;
let local_oid = head.target()?;
let branch_name = head.shorthand()?;
let branch_name = head.shorthand().ok()?;
let upstream_ref =
repo.find_branch(&format!("origin/{branch_name}"), git2::BranchType::Remote).ok()?;
let upstream_oid = upstream_ref.get().target()?;
@@ -205,8 +205,8 @@ fn git_branch_info_for_repo(
fn git_head_refs(repo: &git2::Repository) -> (Option<String>, Option<String>) {
match repo.head() {
Ok(head) => {
let head_ref = head.name().map(|s| s.to_string());
let head_ref_shorthand = head.shorthand().map(|s| s.to_string());
let head_ref = head.name().ok().map(|s| s.to_string());
let head_ref_shorthand = head.shorthand().ok().map(|s| s.to_string());
(head_ref, head_ref_shorthand)
}
Err(_) => {
+2 -1
View File
@@ -99,7 +99,7 @@ pub(crate) fn get_default_remote_name_in_repo(repo: &Repository) -> Result<Strin
}
// if `origin` exists return that
let found_origin = remotes.iter().any(|r| r.is_some_and(|r| r == DEFAULT_REMOTE_NAME));
let found_origin = remotes.iter().any(|r| matches!(r, Ok(Some(r)) if r == DEFAULT_REMOTE_NAME));
if found_origin {
return Ok(DEFAULT_REMOTE_NAME.into());
}
@@ -109,6 +109,7 @@ pub(crate) fn get_default_remote_name_in_repo(repo: &Repository) -> Result<Strin
let first_remote = remotes
.iter()
.next()
.and_then(|r| r.ok())
.flatten()
.map(String::from)
.ok_or_else(|| GenericError("no remote found".into()))?;
+6 -6
View File
@@ -12,16 +12,16 @@ dunce = "1.0.4"
hyper-rustls = { version = "0.27.7", default-features = false, features = ["http2"] }
hyper-util = { version = "0.1.13", default-features = false, features = ["client-legacy"] }
log = { workspace = true }
md5 = "0.7.0"
prost = "0.13.4"
prost-reflect = { version = "0.14.4", default-features = false, features = ["serde", "derive"] }
prost-types = "0.13.4"
md5 = "0.8.0"
prost = "0.14.4"
prost-reflect = { version = "0.16.5", default-features = false, features = ["serde", "derive"] }
prost-types = "0.14.4"
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
tokio = { workspace = true, features = ["macros", "rt-multi-thread", "fs", "process"] }
tokio-stream = "0.1.14"
tonic = { version = "0.12.3", default-features = false, features = ["transport"] }
tonic-reflection = "0.12.3"
tonic = { version = "0.14.6", default-features = false, features = ["transport"] }
tonic-reflection = "0.14.6"
uuid = { version = "1.7.0", features = ["v4"] }
yaak-common = { workspace = true }
yaak-tls = { workspace = true }
+5
View File
@@ -0,0 +1,5 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type MethodDefinition = { name: string, schema: string, clientStreaming: boolean, serverStreaming: boolean, };
export type ServiceDefinition = { name: string, methods: Array<MethodDefinition>, };
+2 -2
View File
@@ -10,7 +10,7 @@ use log::debug;
use std::collections::BTreeMap;
use tokio_stream::StreamExt;
use tonic::Request;
use tonic::body::BoxBody;
use tonic::body::Body;
use tonic::transport::Uri;
use tonic_reflection::pb::v1::server_reflection_request::MessageRequest;
use tonic_reflection::pb::v1::server_reflection_response::MessageResponse;
@@ -22,7 +22,7 @@ use tonic_reflection::pb::v1::{ExtensionRequest, FileDescriptorResponse};
use tonic_reflection::pb::{v1, v1alpha};
use yaak_tls::ClientCertificateConfig;
pub struct AutoReflectionClient<T = Client<HttpsConnector<HttpConnector>, BoxBody>> {
pub struct AutoReflectionClient<T = Client<HttpsConnector<HttpConnector>, Body>> {
use_v1alpha: bool,
client_v1: v1::server_reflection_client::ServerReflectionClient<T>,
client_v1alpha: v1alpha::server_reflection_client::ServerReflectionClient<T>,
+4 -4
View File
@@ -27,7 +27,7 @@ use std::sync::Arc;
use tokio::sync::RwLock;
use tokio_stream::StreamExt;
use tokio_stream::wrappers::ReceiverStream;
use tonic::body::BoxBody;
use tonic::body::Body;
use tonic::metadata::{MetadataKey, MetadataValue};
use tonic::transport::Uri;
use tonic::{IntoRequest, IntoStreamingRequest, Request, Response, Status, Streaming};
@@ -36,7 +36,7 @@ use yaak_tls::ClientCertificateConfig;
#[derive(Clone)]
pub struct GrpcConnection {
pool: Arc<RwLock<DescriptorPool>>,
conn: Client<HttpsConnector<HttpConnector>, BoxBody>,
conn: Client<HttpsConnector<HttpConnector>, Body>,
pub uri: Uri,
use_reflection: bool,
max_message_size: usize,
@@ -338,10 +338,10 @@ impl GrpcConnection {
}
fn grpc_client(
conn: Client<HttpsConnector<HttpConnector>, BoxBody>,
conn: Client<HttpsConnector<HttpConnector>, Body>,
uri: Uri,
max_message_size: usize,
) -> tonic::client::Grpc<Client<HttpsConnector<HttpConnector>, BoxBody>> {
) -> tonic::client::Grpc<Client<HttpsConnector<HttpConnector>, Body>> {
tonic::client::Grpc::with_origin(conn, uri)
.max_decoding_message_size(max_message_size)
.max_encoding_message_size(max_message_size)
+2 -2
View File
@@ -4,7 +4,7 @@ use hyper_util::client::legacy::Client;
use hyper_util::client::legacy::connect::HttpConnector;
use hyper_util::rt::TokioExecutor;
use log::info;
use tonic::body::BoxBody;
use tonic::body::Body;
use yaak_tls::{ClientCertificateConfig, get_tls_config};
// I think ALPN breaks this because we're specifying http2_only
@@ -13,7 +13,7 @@ const WITH_ALPN: bool = false;
pub(crate) fn get_transport(
validate_certificates: bool,
client_cert: Option<ClientCertificateConfig>,
) -> Result<Client<HttpsConnector<HttpConnector>, BoxBody>> {
) -> Result<Client<HttpsConnector<HttpConnector>, Body>> {
let tls_config = get_tls_config(validate_certificates, WITH_ALPN, client_cert.clone())?;
let mut http = HttpConnector::new();
+5 -5
View File
@@ -5,24 +5,23 @@ edition = "2024"
publish = false
[dependencies]
async-compression = { version = "0.4", features = ["tokio", "gzip", "deflate", "brotli", "zstd"] }
async-compression = { version = "0.4", features = ["tokio", "gzip", "deflate", "zlib", "brotli", "zstd"] }
async-trait = "0.1"
brotli = "7"
brotli = "9"
bytes = "1.11.1"
cookie = "0.18.1"
flate2 = "1"
futures-util = "0.3"
http-body = "1"
url = "2"
zstd = "0.13"
zstd = "0.14"
hyper-util = { version = "0.1.17", default-features = false, features = ["client-legacy"] }
log = { workspace = true }
mime_guess = "2.0.5"
native-tls = { version = "0.2", features = ["alpn"] }
reqwest = { workspace = true, features = [
"rustls-tls-manual-roots-no-provider",
"rustls-no-provider",
"native-tls",
"native-tls-alpn",
"socks",
"http2",
"stream",
@@ -41,3 +40,4 @@ yaak-tls = { workspace = true }
[dev-dependencies]
tempfile = "3"
tokio = { workspace = true, features = ["net"] }
+104 -1
View File
@@ -10,8 +10,14 @@ use yaak_tls::{
pub const HTTP2_MAX_RESPONSE_HEADER_LIST_SIZE: u32 = 1024 * 1024;
/// hyper caps HTTP/1 responses at 100 headers, which real servers exceed.
/// https://yaak.app/feedback/posts/when-response-headers-exceed-a-certain-count-hyper-throws-error
pub const HTTP1_MAX_RESPONSE_HEADERS: usize = 1024;
fn client_builder() -> ClientBuilder {
Client::builder().http2_max_header_list_size(HTTP2_MAX_RESPONSE_HEADER_LIST_SIZE)
Client::builder()
.http2_max_header_list_size(HTTP2_MAX_RESPONSE_HEADER_LIST_SIZE)
.http1_max_headers(HTTP1_MAX_RESPONSE_HEADERS)
}
#[derive(Clone)]
@@ -399,3 +405,100 @@ UFNAXPsoutUompC1Z57bKA6OKM2hRANCAATCYYKhzgHEaRaGsYVjJSoXvoroL8qe
assert!(build_native_tls_identity(Some(empty)).unwrap().is_none());
}
}
#[cfg(test)]
mod header_limit_tests {
use super::*;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
/// Accepts one connection and replies with a response head of exactly
/// `total_headers` fields. The trailing `Content-Length` is one of them,
/// because hyper counts every field against its limit.
async fn serve_response_with_header_count(total_headers: usize) -> String {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
// Read the request head so the client is not writing into a closed socket
let mut buf = [0u8; 4096];
loop {
let n = socket.read(&mut buf).await.unwrap();
if n == 0 || buf[..n].windows(4).any(|w| w == b"\r\n\r\n") {
break;
}
}
let mut response = String::from("HTTP/1.1 200 OK\r\n");
for i in 0..total_headers - 1 {
response.push_str(&format!("X-Test-{i}: value-{i}\r\n"));
}
response.push_str("Content-Length: 0\r\n\r\n");
// A response the client rejects mid-parse closes the socket under
// us, so a failed write here is an expected outcome, not a fault
let _ = socket.write_all(response.as_bytes()).await;
let _ = socket.flush().await;
});
format!("http://{addr}/")
}
fn options() -> HttpConnectionOptions {
HttpConnectionOptions {
id: "test".to_string(),
validate_certificates: true,
http_version: HttpVersion::Http1,
proxy: HttpConnectionProxySetting::Disabled,
client_certificate: None,
dns_overrides: Vec::new(),
address_filter: None,
}
}
/// hyper defaults to 100 header fields per HTTP/1 response and fails the
/// whole request past that, which is what users hit in the field.
#[tokio::test]
async fn responses_with_more_than_100_headers_are_accepted() {
let url = serve_response_with_header_count(150).await;
let (client, _resolver) = options().build_client().unwrap();
let response = client
.inner()
.get(&url)
.send()
.await
.expect("request with 150 response headers should succeed");
assert_eq!(response.status(), 200);
assert_eq!(response.headers().len(), 150);
for i in 0..149 {
assert_eq!(
response.headers().get(format!("x-test-{i}")).unwrap(),
format!("value-{i}").as_str(),
);
}
}
/// Pins both sides of the boundary. The sizes are written out rather than
/// derived from `HTTP1_MAX_RESPONSE_HEADERS`, so retuning the limit trips
/// this test instead of silently moving with it.
#[tokio::test]
async fn the_limit_is_exactly_1024_header_fields() {
let (client, _resolver) = options().build_client().unwrap();
let at_limit = serve_response_with_header_count(1024).await;
let response = client
.inner()
.get(&at_limit)
.send()
.await
.expect("a response with 1024 header fields should succeed");
assert_eq!(response.headers().len(), 1024);
let over_limit = serve_response_with_header_count(1025).await;
assert!(client.inner().get(&over_limit).send().await.is_err());
}
}
+601 -10
View File
@@ -1,11 +1,13 @@
use crate::error::{Error, Result};
use async_compression::tokio::bufread::{
BrotliDecoder, DeflateDecoder as AsyncDeflateDecoder, GzipDecoder,
ZstdDecoder as AsyncZstdDecoder,
ZlibDecoder as AsyncZlibDecoder, ZstdDecoder as AsyncZstdDecoder,
};
use flate2::read::{DeflateDecoder, GzDecoder};
use flate2::read::{DeflateDecoder, GzDecoder, ZlibDecoder};
use std::io::Read;
use tokio::io::{AsyncBufRead, AsyncRead};
use std::pin::Pin;
use std::task::{Context, Poll, ready};
use tokio::io::{AsyncBufRead, AsyncRead, BufReader, ReadBuf};
/// Supported compression encodings
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -66,13 +68,77 @@ fn decompress_gzip(data: &[u8]) -> Result<Vec<u8>> {
Ok(decompressed)
}
/// `deflate` names zlib-wrapped data (RFC 1950), but plenty of senders send it raw (RFC 1951).
fn decompress_deflate(data: &[u8]) -> Result<Vec<u8>> {
let mut decoder = DeflateDecoder::new(data);
let mut decompressed = Vec::new();
decoder
.read_to_end(&mut decompressed)
.map_err(|e| Error::DecompressionError(format!("deflate decompression failed: {}", e)))?;
Ok(decompressed)
let read = |zlib: bool| {
let mut decompressed = Vec::new();
let result = if zlib {
ZlibDecoder::new(data).read_to_end(&mut decompressed)
} else {
DeflateDecoder::new(data).read_to_end(&mut decompressed)
};
result.map(|_| decompressed)
};
match deflate_variant(data) {
DeflateVariant::Zlib => read(true),
DeflateVariant::Raw => read(false),
// Zlib only when it ends whole, its checksum verified, or else raw.
DeflateVariant::Either => inflate_zlib_whole(data).or_else(|_| read(false)),
}
.map_err(|e| Error::DecompressionError(format!("deflate decompression failed: {}", e)))
}
/// Zlib data decoded through to its end and checksum, failing on any less
fn inflate_zlib_whole(data: &[u8]) -> std::io::Result<Vec<u8>> {
let mut inflater = flate2::Decompress::new(true);
let mut out = Vec::new();
loop {
// Always room to write, so a call that makes no progress has run out of input.
out.reserve(64 * 1024);
let (read, written) = (inflater.total_in(), inflater.total_out());
let rest = &data[usize::try_from(read).unwrap_or(data.len()).min(data.len())..];
// Not `Finish`, which fails for good when the output outgrows its buffer.
let status = inflater
.decompress_vec(rest, &mut out, flate2::FlushDecompress::None)
.map_err(std::io::Error::other)?;
if status == flate2::Status::StreamEnd {
return Ok(out);
}
if inflater.total_in() == read && inflater.total_out() == written {
return Err(std::io::ErrorKind::UnexpectedEof.into());
}
}
}
/// Which `deflate` a body is, from its first five bytes
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum DeflateVariant {
Zlib,
Raw,
/// A zlib header that is also the start of a raw stored block: only decoding tells.
Either,
}
/// Zlib data opens with a header whose first byte names deflate with a window of at most 32 KB,
/// and whose two bytes are a multiple of 31. Raw deflate passes that only when it opens with a
/// stored block that isn't the last, whose length follows in the next two bytes and its
/// complement in the two after.
fn deflate_variant(data: &[u8]) -> DeflateVariant {
let zlib = match data {
[cmf, flg, ..] => {
cmf & 0x0f == 8 && cmf >> 4 <= 7 && (u16::from(*cmf) << 8 | u16::from(*flg)) % 31 == 0
}
_ => false,
};
match data {
_ if !zlib => DeflateVariant::Raw,
[_, l0, l1, n0, n1, ..]
if u16::from_le_bytes([*l0, *l1]) != !u16::from_le_bytes([*n0, *n1]) =>
{
DeflateVariant::Zlib
}
_ => DeflateVariant::Either,
}
}
fn decompress_brotli(data: &[u8]) -> Result<Vec<u8>> {
@@ -96,12 +162,262 @@ pub fn streaming_decoder<R: AsyncBufRead + Unpin + Send + 'static>(
match encoding {
ContentEncoding::Identity => Box::new(reader),
ContentEncoding::Gzip => Box::new(GzipDecoder::new(reader)),
ContentEncoding::Deflate => Box::new(AsyncDeflateDecoder::new(reader)),
ContentEncoding::Deflate => Box::new(StreamingDeflateDecoder::new(reader)),
ContentEncoding::Brotli => Box::new(BrotliDecoder::new(reader)),
ContentEncoding::Zstd => Box::new(AsyncZstdDecoder::new(reader)),
}
}
/// Decodes `deflate`, zlib-wrapped or raw, as its first five bytes say. When they could be
/// either, it decodes both ways at once until one fails, and goes on with the other.
struct StreamingDeflateDecoder<R> {
state: DeflateState<R>,
}
enum DeflateState<R> {
Sniffing(Replay<R>),
Zlib(AsyncZlibDecoder<BufReader<Replay<R>>>),
Raw(AsyncDeflateDecoder<BufReader<Replay<R>>>),
Either(EitherInflate<BufReader<Replay<R>>>),
/// Only while switching from sniffing to one of the others
Switching,
}
impl<R> StreamingDeflateDecoder<R> {
fn new(reader: R) -> Self {
let source = Replay { inner: reader, header: Vec::new(), pos: 0, sniffing: true };
Self { state: DeflateState::Sniffing(source) }
}
}
impl<R: AsyncRead + Unpin> AsyncRead for StreamingDeflateDecoder<R> {
fn poll_read(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &mut ReadBuf<'_>,
) -> Poll<std::io::Result<()>> {
let this = self.get_mut();
loop {
match &mut this.state {
DeflateState::Sniffing(source) => {
// Read until the header is in hand, or the input ends short of it.
while source.header.len() < 5 {
let mut header = [0u8; 5];
let mut header = ReadBuf::new(&mut header[source.header.len()..]);
ready!(Pin::new(&mut *source).poll_read(cx, &mut header))?;
if header.filled().is_empty() {
break;
}
}
let DeflateState::Sniffing(mut source) =
std::mem::replace(&mut this.state, DeflateState::Switching)
else {
unreachable!()
};
source.sniffing = false;
let variant = deflate_variant(&source.header);
let reader = BufReader::new(source);
this.state = match variant {
DeflateVariant::Zlib => DeflateState::Zlib(AsyncZlibDecoder::new(reader)),
DeflateVariant::Raw => DeflateState::Raw(AsyncDeflateDecoder::new(reader)),
DeflateVariant::Either => DeflateState::Either(EitherInflate::new(reader)),
};
}
DeflateState::Zlib(d) => return Pin::new(d).poll_read(cx, buf),
DeflateState::Raw(d) => return Pin::new(d).poll_read(cx, buf),
DeflateState::Either(d) => return Pin::new(d).poll_read(cx, buf),
DeflateState::Switching => {
return Poll::Ready(Err(std::io::Error::other("deflate decoder is broken")));
}
}
}
}
}
/// Reads `inner`, keeping what it read while sniffing, then reads that again before the rest.
struct Replay<R> {
inner: R,
header: Vec<u8>,
/// Where reading resumes in `header` once sniffing is over
pos: usize,
sniffing: bool,
}
impl<R: AsyncRead + Unpin> AsyncRead for Replay<R> {
fn poll_read(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &mut ReadBuf<'_>,
) -> Poll<std::io::Result<()>> {
let this = self.get_mut();
if this.sniffing {
let before = buf.filled().len();
ready!(Pin::new(&mut this.inner).poll_read(cx, buf))?;
this.header.extend_from_slice(&buf.filled()[before..]);
return Poll::Ready(Ok(()));
}
if this.pos < this.header.len() {
let n = buf.remaining().min(this.header.len() - this.pos);
buf.put_slice(&this.header[this.pos..this.pos + n]);
this.pos += n;
return Poll::Ready(Ok(()));
}
Pin::new(&mut this.inner).poll_read(cx, buf)
}
}
/// Input fed to the readings of an ambiguous deflate body at a time, which deflate grows to at
/// most about a megabyte
const DEFLATE_FEED: usize = 1024;
/// Output held while both readings of an ambiguous deflate body are still valid, past which it is
/// taken to be zlib, as its header says. A wrong reading fails within a few bytes in practice.
const DEFLATE_HELD_LIMIT: usize = 1024 * 1024;
/// Both readings of a deflate body that could be zlib or raw, fed the same input. Their output
/// is held back while both are still valid, and the one left streams on: a wrong reading fails
/// within a few bytes in practice, so little is held. If both end, zlib's checksum verified it.
struct EitherInflate<B> {
input: B,
/// Zlib, then raw
readings: [Option<Inflate>; 2],
/// Why the last reading to fail failed
error: Option<std::io::Error>,
}
struct Inflate {
inflater: flate2::Decompress,
out: Vec<u8>,
/// Where reading resumes in `out`
pos: usize,
ended: bool,
}
impl<B> EitherInflate<B> {
fn new(input: B) -> Self {
let reading = |zlib| Inflate {
inflater: flate2::Decompress::new(zlib),
out: Vec::new(),
pos: 0,
ended: false,
};
Self { input, readings: [Some(reading(true)), Some(reading(false))], error: None }
}
}
impl Inflate {
/// Decodes all of `input` onto `out`, or up to the end of the stream
fn feed(&mut self, input: &[u8]) -> std::io::Result<()> {
let mut consumed = 0;
while !self.ended {
if self.out.capacity() - self.out.len() < 32 * 1024 {
self.out.reserve(64 * 1024);
}
let (read, written) = (self.inflater.total_in(), self.inflater.total_out());
let status = self
.inflater
.decompress_vec(&input[consumed..], &mut self.out, flate2::FlushDecompress::None)
.map_err(std::io::Error::other)?;
let progress = (self.inflater.total_in() - read, self.inflater.total_out() - written);
consumed += usize::try_from(progress.0).unwrap_or(usize::MAX).min(input.len());
if status == flate2::Status::StreamEnd {
self.ended = true;
} else if consumed == input.len() && self.out.len() < self.out.capacity() {
// All of it in, and all that it decodes to out
break;
} else if progress == (0, 0) {
return Err(std::io::Error::other("deflate decoding stalled"));
}
}
Ok(())
}
}
impl<B: AsyncBufRead + Unpin> AsyncRead for EitherInflate<B> {
fn poll_read(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &mut ReadBuf<'_>,
) -> Poll<std::io::Result<()>> {
let this = self.get_mut();
loop {
let [zlib, raw] = &mut this.readings;
match (zlib.as_mut(), raw.as_mut()) {
(None, None) => {
let error = this.error.take();
return Poll::Ready(Err(
error.unwrap_or_else(|| std::io::Error::other("deflate decoding failed"))
));
}
// Both ended: zlib's checksum verified it.
(Some(z), Some(r)) if z.ended && r.ended => *raw = None,
(Some(only), None) | (None, Some(only)) => {
if only.pos < only.out.len() {
let n = buf.remaining().min(only.out.len() - only.pos);
buf.put_slice(&only.out[only.pos..only.pos + n]);
only.pos += n;
if only.pos == only.out.len() {
only.out.clear();
only.pos = 0;
}
return Poll::Ready(Ok(()));
}
if only.ended {
return Poll::Ready(Ok(()));
}
ready!(this.feed(cx))?;
}
(Some(_), Some(_)) => ready!(this.feed(cx))?,
}
}
}
}
impl<B: AsyncBufRead + Unpin> EitherInflate<B> {
/// Feeds the next input to each reading still going, dropping those that fail on it. Pending
/// until there is input; at its end, a reading that hasn't ended was cut short.
fn feed(&mut self, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
let chunk = ready!(Pin::new(&mut self.input).poll_fill_buf(cx))?;
// A little input at a time, since deflate can grow a thousandfold.
let len = chunk.len().min(DEFLATE_FEED);
let chunk = &chunk[..len];
let [zlib, raw] = &mut self.readings;
if let (Some(z), Some(r)) = (zlib.as_ref(), raw.as_ref()) {
let (z_held, r_held) = (z.out.len() - z.pos, r.out.len() - r.pos);
let cut_short = len == 0 && !z.ended && !r.ended;
if cut_short {
// Both were cut short: keep what decoded furthest, so a cut body shows its start.
if r_held > z_held {
*zlib = None
} else {
*raw = None
}
// The one left gives what it decoded, then fails on the end of input.
return Poll::Ready(Ok(()));
} else if z_held.max(r_held) > DEFLATE_HELD_LIMIT {
// Holding no more for a raw body zlib still reads validly: it is taken to be zlib.
*raw = None;
}
}
for slot in &mut self.readings {
let Some(reading) = slot.as_mut().filter(|r| !r.ended) else {
continue;
};
let result = if len == 0 {
Err(std::io::ErrorKind::UnexpectedEof.into())
} else {
reading.feed(chunk)
};
if let Err(e) = result {
*slot = None;
self.error = Some(e);
}
}
Pin::new(&mut self.input).consume(len);
Poll::Ready(Ok(()))
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -159,6 +475,281 @@ mod tests {
assert_eq!(result.decompressed_size, original.len() as u64);
}
fn zlib(data: &[u8]) -> Vec<u8> {
let mut encoder = flate2::write::ZlibEncoder::new(Vec::new(), Compression::default());
encoder.write_all(data).unwrap();
encoder.finish().unwrap()
}
fn raw_deflate(data: &[u8]) -> Vec<u8> {
let mut encoder = flate2::write::DeflateEncoder::new(Vec::new(), Compression::default());
encoder.write_all(data).unwrap();
encoder.finish().unwrap()
}
/// Raw deflate whose first two bytes also pass as a zlib header: a stored block of 29 bytes
/// (`08 1d`, with `0x081d` a multiple of 31), then an empty final one.
fn raw_deflate_like_zlib(data: &[u8; 29]) -> Vec<u8> {
let mut out = vec![0x08, 0x1d, 0x00, 0xe2, 0xff];
out.extend_from_slice(data);
out.extend_from_slice(&[0x01, 0x00, 0x00, 0xff, 0xff]);
assert_eq!(deflate_variant(&out), DeflateVariant::Either);
out
}
/// Zlib that opens as `raw_deflate_like_zlib` does: header `08 1d`, then a stored block of
/// 0xffe2 bytes, which `1d 00 e2 ff` also reads as the start of, and a final stored block of
/// `extra` bytes.
fn zlib_like_raw_deflate(extra: u16) -> (Vec<u8>, Vec<u8>) {
let data: Vec<u8> = (0..0xffe2 + u32::from(extra)).map(|i| (i % 251) as u8).collect();
let mut out = vec![0x08, 0x1d, 0x00, 0xe2, 0xff, 0x1d, 0x00];
out.extend_from_slice(&data[..0xffe2]);
out.push(0x01);
out.extend_from_slice(&extra.to_le_bytes());
out.extend_from_slice(&(!extra).to_le_bytes());
out.extend_from_slice(&data[0xffe2..]);
let (mut a, mut b) = (1u32, 0u32);
for byte in &data {
a = (a + u32::from(*byte)) % 65521;
b = (b + a) % 65521;
}
out.extend_from_slice(&((b << 16) | a).to_be_bytes());
assert_eq!(deflate_variant(&out), DeflateVariant::Either);
(out, data)
}
async fn stream_decode(data: Vec<u8>, encoding: ContentEncoding) -> std::io::Result<Vec<u8>> {
use tokio::io::AsyncReadExt;
let mut out = Vec::new();
// One byte at a time, to read the header across polls as a slow connection would.
let reader = BufReader::with_capacity(1, std::io::Cursor::new(data));
streaming_decoder(reader, encoding).read_to_end(&mut out).await?;
Ok(out)
}
/// What a body decodes to before it fails, and why it failed
async fn stream_decode_until_error(data: Vec<u8>) -> (Vec<u8>, Option<std::io::Error>) {
use tokio::io::AsyncReadExt;
let mut out = Vec::new();
let mut decoder =
streaming_decoder(BufReader::new(std::io::Cursor::new(data)), ContentEncoding::Deflate);
let mut buf = [0u8; 4096];
loop {
match decoder.read(&mut buf).await {
Ok(0) => return (out, None),
Ok(n) => out.extend_from_slice(&buf[..n]),
Err(e) => return (out, Some(e)),
}
}
}
#[tokio::test]
async fn streaming_deflate_keeps_what_decoded_of_a_cut_ambiguous_body() {
// Bytes both readings still accept when the input ends: zlib in a stored block too
let text = *b"\x1d\0xxxxxxxxxxxxxxxxxxxxxxxxxxx";
let mut cut = raw_deflate_like_zlib(&text);
// Without its final block, as a body cut at the capture limit is
cut.truncate(cut.len() - 5);
let (decoded, error) = stream_decode_until_error(cut).await;
assert_eq!(decoded, text);
assert_eq!(error.map(|e| e.kind()), Some(std::io::ErrorKind::UnexpectedEof));
}
/// Zlib of `megabytes` of zeros whose raw reading is a stored block of 29 bytes and an empty
/// final one, both inside zlib's first stored block: raw ends at once, zlib goes on.
fn zlib_whose_raw_reading_ends_early(megabytes: usize) -> (Vec<u8>, Vec<u8>) {
let mut stored = vec![b'z'; 0xffe2];
stored[27..32].copy_from_slice(&[0x01, 0x00, 0x00, 0xff, 0xff]);
let mut expected = stored.clone();
expected.resize(stored.len() + megabytes * 1024 * 1024, 0);
let mut data = vec![0x08, 0x1d, 0x00, 0xe2, 0xff, 0x1d, 0x00];
data.extend_from_slice(&stored);
data.extend_from_slice(&raw_deflate(&expected[stored.len()..]));
let (mut a, mut b) = (1u32, 0u32);
for byte in &expected {
a = (a + u32::from(*byte)) % 65521;
b = (b + a) % 65521;
}
data.extend_from_slice(&((b << 16) | a).to_be_bytes());
assert_eq!(deflate_variant(&data), DeflateVariant::Either);
(data, expected)
}
#[tokio::test]
async fn streaming_deflate_holds_little_while_both_readings_go() {
use tokio::io::AsyncReadExt;
let (data, expected) = zlib_whose_raw_reading_ends_early(8);
let mut decoder =
StreamingDeflateDecoder::new(BufReader::new(std::io::Cursor::new(data.clone())));
let mut first = [0u8; 1024];
decoder.read_exact(&mut first).await.unwrap();
assert_eq!(first[..], expected[..1024]);
let DeflateState::Either(either) = &decoder.state else {
panic!("not read both ways")
};
// The limit, what one feed of input adds past it (deflate grows at most about 1,032
// times), and the raw reading's 64 KB: far short of the 8 MB body.
let held: usize = either.readings.iter().flatten().map(|r| r.out.len() - r.pos).sum();
let bound = 3 * 1024 * 1024;
assert!(held <= bound, "held {held} bytes for a 1 KB read");
let decoded = stream_decode(data, ContentEncoding::Deflate).await.unwrap();
assert!(decoded == expected);
}
const DEFLATE_TEXT: &[u8] = b"{\"service\":\"browser\",\"message\":\"hello deflate\"}";
#[test]
fn test_decompress_deflate_zlib_wrapped() {
let result = decompress(zlib(DEFLATE_TEXT), ContentEncoding::Deflate).unwrap();
assert_eq!(result.data, DEFLATE_TEXT);
let like_zlib = raw_deflate_like_zlib(b"twenty-nine bytes of raw text");
let result = decompress(like_zlib, ContentEncoding::Deflate).unwrap();
assert_eq!(result.data, b"twenty-nine bytes of raw text");
let both = *b"\x1d\0xxxxxxxxxxxxxxxxxxxxxxxxxxx";
let result = decompress(raw_deflate_like_zlib(&both), ContentEncoding::Deflate).unwrap();
assert_eq!(result.data, both);
for extra in [0, 10_000] {
let (zlib_like_raw, data) = zlib_like_raw_deflate(extra);
let result = decompress(zlib_like_raw, ContentEncoding::Deflate).unwrap();
assert!(result.data == data, "{extra} extra bytes");
}
let (data, expected) = large_raw_like_zlib();
assert!(decompress(data, ContentEncoding::Deflate).unwrap().data == expected);
}
/// Ambiguous raw deflate of more than 64 KB: a stored block that reads as zlib's header and
/// the start of a stored block of 0xffe2 bytes, then incompressible data, which zlib reads
/// as a block header 7 + 0xffe2 bytes in and fails on.
fn large_raw_like_zlib() -> (Vec<u8>, Vec<u8>) {
let mut expected = b"\x1d\0xxxxxxxxxxxxxxxxxxxxxxxxxxx".to_vec();
let mut seed = 1u32;
expected.extend((0..70_000).map(|_| {
seed = seed.wrapping_mul(1_103_515_245).wrapping_add(12_345);
(seed >> 16) as u8
}));
let mut data = vec![0x08, 0x1d, 0x00, 0xe2, 0xff];
data.extend_from_slice(&expected[..29]);
data.extend_from_slice(&raw_deflate(&expected[29..]));
assert!(data.len() > 64 * 1024);
(data, expected)
}
/// Greptile's case on #812: raw deflate that zlib reads validly past 64 KB. After the first
/// stored block, a final stored block of 65,535 bytes, in which zlib finds a valid stored
/// block header at 65,513 (`00 28 00 d7 ff`) and an invalid block type only at 65,558.
fn raw_valid_as_zlib_past_64k() -> (Vec<u8>, Vec<u8>) {
let mut block: Vec<u8> = (0..65_535u32).map(|i| b'a' + (i % 26) as u8).collect();
let first = 5 + 29 + 5;
block[65_513 - first..65_518 - first].copy_from_slice(&[0x00, 0x28, 0x00, 0xd7, 0xff]);
block[65_558 - first] = 0x07;
let mut expected = b"\x1d\0xxxxxxxxxxxxxxxxxxxxxxxxxxx".to_vec();
let mut data = raw_deflate_like_zlib(&expected.clone().try_into().unwrap());
// Make the first block not the last, and follow it with the large final one.
data.truncate(data.len() - 5);
data.extend_from_slice(&[0x01, 0xff, 0xff, 0x00, 0x00]);
data.extend_from_slice(&block);
expected.extend_from_slice(&block);
(data, expected)
}
#[test]
fn test_decompress_deflate_raw_valid_as_zlib_past_64k() {
let (data, expected) = raw_valid_as_zlib_past_64k();
// Raw alone decodes it; zlib alone gets past 64 KB before failing.
let mut raw = Vec::new();
DeflateDecoder::new(data.as_slice()).read_to_end(&mut raw).unwrap();
assert!(raw == expected);
let mut zlib = flate2::Decompress::new(true);
let mut out = Vec::with_capacity(200 * 1024);
let failed = zlib.decompress_vec(&data, &mut out, flate2::FlushDecompress::None);
assert!(failed.is_err() && zlib.total_in() > 65_536, "zlib read {}", zlib.total_in());
assert!(decompress(data, ContentEncoding::Deflate).unwrap().data == expected);
}
#[tokio::test]
async fn streaming_deflate_decodes_raw_valid_as_zlib_past_64k() {
use tokio::io::AsyncReadExt;
let (data, expected) = raw_valid_as_zlib_past_64k();
for capacity in [1, 8 * 1024, 256 * 1024] {
let mut out = Vec::new();
let reader = BufReader::with_capacity(capacity, std::io::Cursor::new(data.clone()));
streaming_decoder(reader, ContentEncoding::Deflate)
.read_to_end(&mut out)
.await
.unwrap();
assert!(out == expected, "read {capacity} bytes at a time");
}
}
#[tokio::test]
async fn streaming_deflate_decodes_zlib_and_raw() {
let zlib_wrapped = stream_decode(zlib(DEFLATE_TEXT), ContentEncoding::Deflate).await;
assert_eq!(zlib_wrapped.unwrap(), DEFLATE_TEXT);
let raw = stream_decode(raw_deflate(DEFLATE_TEXT), ContentEncoding::Deflate).await;
assert_eq!(raw.unwrap(), DEFLATE_TEXT);
let large: Vec<u8> = (0..200_000u32).flat_map(|i| (i % 251).to_le_bytes()).collect();
let large_zlib = stream_decode(zlib(&large), ContentEncoding::Deflate).await;
assert_eq!(large_zlib.unwrap(), large);
let large_raw = stream_decode(raw_deflate(&large), ContentEncoding::Deflate).await;
assert_eq!(large_raw.unwrap(), large);
}
#[tokio::test]
async fn streaming_deflate_retries_raw_that_looks_like_zlib() {
let data = raw_deflate_like_zlib(b"twenty-nine bytes of raw text");
// A zlib decoder alone rejects it.
assert!(ZlibDecoder::new(data.as_slice()).read_to_end(&mut Vec::new()).is_err());
let decoded = stream_decode(data, ContentEncoding::Deflate).await.unwrap();
assert_eq!(decoded, b"twenty-nine bytes of raw text");
// Read as zlib, this one is a stored block that starts decoding, and only fails later.
let both = *b"\x1d\0xxxxxxxxxxxxxxxxxxxxxxxxxxx";
let data = raw_deflate_like_zlib(&both);
let decoded = stream_decode(data, ContentEncoding::Deflate).await.unwrap();
assert_eq!(decoded, both);
}
#[tokio::test]
async fn streaming_deflate_decodes_zlib_that_looks_like_raw() {
for extra in [0, 10_000] {
let (data, expected) = zlib_like_raw_deflate(extra);
let decoded = stream_decode(data, ContentEncoding::Deflate).await.unwrap();
assert!(decoded == expected, "{extra} extra bytes");
}
}
/// However the reader reads ahead, the wrong reading is dropped and the right one kept.
#[tokio::test]
async fn streaming_deflate_retries_large_raw_whatever_the_chunking() {
use tokio::io::AsyncReadExt;
let (data, expected) = large_raw_like_zlib();
for capacity in [1, 8 * 1024, 256 * 1024] {
let mut out = Vec::new();
let reader = BufReader::with_capacity(capacity, std::io::Cursor::new(data.clone()));
streaming_decoder(reader, ContentEncoding::Deflate)
.read_to_end(&mut out)
.await
.unwrap();
assert!(out == expected, "read {capacity} bytes at a time");
}
}
#[tokio::test]
async fn streaming_deflate_still_fails_on_garbage() {
let garbage = b"\x78\x9c this is not deflate in any form".to_vec();
assert!(stream_decode(garbage, ContentEncoding::Deflate).await.is_err());
assert!(stream_decode(b"\xff".to_vec(), ContentEncoding::Deflate).await.is_err());
}
#[tokio::test]
async fn streaming_gzip_is_unchanged() {
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
encoder.write_all(DEFLATE_TEXT).unwrap();
let gzip = encoder.finish().unwrap();
assert_eq!(stream_decode(gzip, ContentEncoding::Gzip).await.unwrap(), DEFLATE_TEXT);
}
#[test]
fn test_decompress_brotli() {
// Compress some data with brotli
+63 -1
View File
@@ -181,7 +181,11 @@ fn strip_query_params(url: &str, names: &[&str]) -> String {
}
fn build_url(r: &HttpRequest) -> String {
let (url_string, params) = apply_path_placeholders(&ensure_proto(&r.url), &r.url_parameters);
// Trim stray whitespace: a leading space breaks scheme
// detection in `ensure_proto` (IdnaError); a trailing space becomes `/path%20` -> 404.
let trimmed_url = r.url.trim();
let (url_string, params) =
apply_path_placeholders(&ensure_proto(trimmed_url), &r.url_parameters);
let mut url = append_query_params(
&url_string,
params
@@ -817,6 +821,64 @@ mod tests {
assert_eq!(result, "https://example.com/api?foo=bar#section");
}
#[test]
fn test_build_url_trims_leading_whitespace() {
// A leading space must not break scheme detection (it used to produce
// `http:// https://...`, which fails to parse with an IdnaError).
let r = HttpRequest {
url: " https://example.com/api".to_string(),
url_parameters: vec![],
..Default::default()
};
let result = build_url(&r);
assert_eq!(result, "https://example.com/api");
}
#[test]
fn test_build_url_trims_trailing_whitespace() {
let r = HttpRequest {
url: "https://example.com/api ".to_string(),
url_parameters: vec![],
..Default::default()
};
let result = build_url(&r);
assert_eq!(result, "https://example.com/api");
}
#[test]
fn test_build_url_trims_whitespace_with_params() {
// A trailing space must not end up percent-encoded in the path once query
// parameters are appended (e.g. `/api%20?foo=bar`, which 404s).
let r = HttpRequest {
url: " https://example.com/api ".to_string(),
url_parameters: vec![HttpUrlParameter {
enabled: true,
name: "foo".to_string(),
value: "bar".to_string(),
id: None,
}],
..Default::default()
};
let result = build_url(&r);
assert_eq!(result, "https://example.com/api?foo=bar");
}
#[test]
fn test_build_url_trims_whitespace_without_scheme() {
let r = HttpRequest {
url: " example.com/api\t".to_string(),
url_parameters: vec![],
..Default::default()
};
let result = build_url(&r);
// ensure_proto defaults to http:// for regular domains
assert_eq!(result, "http://example.com/api");
}
#[test]
fn test_build_url_with_existing_query_and_fragment() {
let r = HttpRequest {
+32 -36
View File
@@ -7,7 +7,7 @@
use log::info;
use std::path::PathBuf;
use yaak_models::blob_manager::BlobManager;
use yaak_models::client_db::ClientDb;
use yaak_models::client_db::WriteDb;
use yaak_models::error::Result;
const MODEL_CHANGES_RETENTION_HOURS: i64 = 1;
@@ -44,7 +44,8 @@ impl Host {
}
/// Run once after the database is open, before the host answers anything.
pub fn on_launch(host: &Host, db: &ClientDb, blobs: &BlobManager) -> Result<()> {
/// Takes the write handle: a launch closes what the last session left open.
pub fn on_launch(host: &Host, db: &WriteDb, blobs: &BlobManager) -> Result<()> {
db.prune_model_changes_older_than_hours(MODEL_CHANGES_RETENTION_HOURS)?;
if host.role == Role::Owner {
@@ -77,53 +78,48 @@ mod tests {
#[test]
fn only_the_owner_closes_what_the_last_session_left_open() {
let (query_manager, blob_manager, _rx) = init_in_memory().expect("Failed to init DB");
let db = query_manager.connect();
let source = &UpdateSource::Background;
let workspace = db
.upsert_workspace(
&Workspace { name: "Hooks".to_string(), ..Default::default() },
source,
)
.unwrap();
let request = db
.upsert_http_request(
&HttpRequest { workspace_id: workspace.id.clone(), ..Default::default() },
source,
)
.unwrap();
let pending = db
.upsert_http_response(
&HttpResponse {
request_id: request.id.clone(),
workspace_id: workspace.id.clone(),
state: HttpResponseState::Connected,
..Default::default()
},
source,
&blob_manager,
)
let pending = query_manager
.with_tx(|db| {
let workspace = db.upsert_workspace(
&Workspace { name: "Hooks".to_string(), ..Default::default() },
source,
)?;
let request = db.upsert_http_request(
&HttpRequest { workspace_id: workspace.id.clone(), ..Default::default() },
source,
)?;
db.upsert_http_response(
&HttpResponse {
request_id: request.id.clone(),
workspace_id: workspace.id.clone(),
state: HttpResponseState::Connected,
..Default::default()
},
source,
&blob_manager,
)
})
.unwrap();
on_launch(&Host::guest(), &db, &blob_manager).unwrap();
let response = db.get_http_response(&pending.id).unwrap();
query_manager.with_tx(|db| on_launch(&Host::guest(), db, &blob_manager)).unwrap();
let response = query_manager.connect().get_http_response(&pending.id).unwrap();
assert!(matches!(response.state, HttpResponseState::Connected));
on_launch(&Host::owner(), &db, &blob_manager).unwrap();
let response = db.get_http_response(&pending.id).unwrap();
query_manager.with_tx(|db| on_launch(&Host::owner(), db, &blob_manager)).unwrap();
let response = query_manager.connect().get_http_response(&pending.id).unwrap();
assert!(matches!(response.state, HttpResponseState::Closed));
}
#[test]
fn owner_without_a_filesystem_still_sweeps_blobs() {
let (query_manager, blob_manager, _rx) = init_in_memory().expect("Failed to init DB");
let db = query_manager.connect();
{
let blob_ctx = blob_manager.connect();
blob_ctx.insert_chunk(&BodyChunk::new("rs_gone", 0, b"dead".to_vec())).unwrap();
}
blob_manager
.with_tx(|b| b.insert_chunk(&BodyChunk::new("rs_gone", 0, b"dead".to_vec())))
.unwrap();
on_launch(&Host::owner(), &db, &blob_manager).unwrap();
query_manager.with_tx(|db| on_launch(&Host::owner(), db, &blob_manager)).unwrap();
assert!(!blob_manager.connect().body_exists("rs_gone").unwrap());
}
+28
View File
@@ -0,0 +1,28 @@
[package]
name = "yaak-mcp"
version = "0.1.0"
edition = "2024"
publish = false
[dependencies]
bytes = { version = "1" }
http-body-util = { version = "0.1" }
hyper = { version = "1", features = ["server", "http1"] }
hyper-util = { version = "0.1", features = ["tokio"] }
log = { workspace = true }
rmcp = { version = "3.5.1", default-features = false, features = [
"server",
"macros",
"transport-io",
"transport-streamable-http-server",
] }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
tokio = { workspace = true, features = ["net", "rt", "macros"] }
tokio-util = { version = "0.7" }
yaak-models = { workspace = true }
yaak-plugins = { workspace = true }
[dev-dependencies]
reqwest = { workspace = true }
tokio = { workspace = true, features = ["rt-multi-thread"] }
+124
View File
@@ -0,0 +1,124 @@
use crate::{McpHost, YaakMcpServer};
use bytes::Bytes;
use http_body_util::combinators::BoxBody;
use http_body_util::{BodyExt, Full};
use hyper::body::Incoming;
use hyper::server::conn::http1;
use hyper::service::service_fn;
use hyper::{Request, Response, StatusCode};
use hyper_util::rt::TokioIo;
use log::{debug, warn};
use rmcp::transport::streamable_http_server::session::local::LocalSessionManager;
use rmcp::transport::streamable_http_server::{StreamableHttpServerConfig, StreamableHttpService};
use std::convert::Infallible;
use std::net::SocketAddr;
use std::sync::Arc;
use tokio::net::TcpListener;
use tokio::task::JoinHandle;
use tokio_util::sync::CancellationToken;
pub const DEFAULT_PORT: u16 = 64343;
pub const PATH: &str = "/mcp";
/// A running server. Dropping it does not stop it; call [`HttpServer::stop`].
pub struct HttpServer {
pub addr: SocketAddr,
cancel: CancellationToken,
task: JoinHandle<()>,
}
impl HttpServer {
pub async fn stop(self) {
self.cancel.cancel();
let _ = self.task.await;
}
}
/// Serve the tools on `addr` at [`PATH`], answering only requests that carry
/// `Authorization: Bearer <token>`.
pub async fn serve<H: McpHost>(
host: Arc<H>,
addr: SocketAddr,
token: String,
) -> std::io::Result<HttpServer> {
let listener = TcpListener::bind(addr).await?;
let addr = listener.local_addr()?;
let cancel = CancellationToken::new();
let config =
StreamableHttpServerConfig::default().with_cancellation_token(cancel.child_token());
let mcp = StreamableHttpService::new(
move || Ok(YaakMcpServer::from_arc(host.clone())),
Arc::new(LocalSessionManager::default()),
config,
);
let expected = Arc::new(format!("Bearer {token}"));
let accept_cancel = cancel.clone();
let task = tokio::spawn(async move {
loop {
let stream = tokio::select! {
_ = accept_cancel.cancelled() => break,
accepted = listener.accept() => match accepted {
Ok((stream, _)) => stream,
Err(e) => {
warn!("MCP server failed to accept a connection: {e}");
continue;
}
},
};
let mcp = mcp.clone();
let expected = expected.clone();
let conn_cancel = accept_cancel.clone();
tokio::spawn(async move {
let service = service_fn(move |req: Request<Incoming>| {
let mcp = mcp.clone();
let expected = expected.clone();
async move { Ok::<_, Infallible>(route(&mcp, &expected, req).await) }
});
let conn = http1::Builder::new().serve_connection(TokioIo::new(stream), service);
tokio::select! {
_ = conn_cancel.cancelled() => {}
result = conn => if let Err(e) = result {
debug!("MCP connection ended: {e}");
},
}
});
}
});
Ok(HttpServer { addr, cancel, task })
}
async fn route<S>(
mcp: &StreamableHttpService<S, LocalSessionManager>,
expected: &str,
req: Request<Incoming>,
) -> Response<BoxBody<Bytes, Infallible>>
where
S: rmcp::ServerHandler + Send + 'static,
{
if req.uri().path() != PATH {
return plain(StatusCode::NOT_FOUND, "Not found");
}
let given = req.headers().get(hyper::header::AUTHORIZATION).map(|v| v.as_bytes());
if !given.is_some_and(|given| constant_time_eq(given, expected.as_bytes())) {
let mut response = plain(StatusCode::UNAUTHORIZED, "Missing or invalid bearer token");
response
.headers_mut()
.insert(hyper::header::WWW_AUTHENTICATE, "Bearer".parse().expect("valid header"));
return response;
}
mcp.handle(req).await
}
fn plain(status: StatusCode, message: &'static str) -> Response<BoxBody<Bytes, Infallible>> {
let mut response = Response::new(Full::new(Bytes::from_static(message.as_bytes())).boxed());
*response.status_mut() = status;
response
}
fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
a.len() == b.len() && a.iter().zip(b).fold(0u8, |acc, (x, y)| acc | (x ^ y)) == 0
}
+64
View File
@@ -0,0 +1,64 @@
//! Yaak's MCP server: the tools, written once against [`McpHost`], and the
//! transports the CLI serves them on: stdio by default, streamable HTTP with
//! `yaak mcp --http`.
pub mod http;
mod params;
mod server;
use serde::Serialize;
use std::future::Future;
use yaak_models::models::{Folder, HttpRequest, HttpResponse};
use yaak_plugins::events::ShowToastRequest;
pub use server::YaakMcpServer;
pub type HostResult<T> = Result<T, String>;
#[derive(Debug, Clone, Serialize)]
pub struct WorkspaceInfo {
pub id: String,
pub name: String,
}
/// What the tools need from whichever process hosts them. Errors are shown to
/// the MCP client as is.
pub trait McpHost: Send + Sync + 'static {
/// The workspaces a tool may act on: open windows in the app, every
/// workspace in the CLI.
fn list_workspaces(&self) -> HostResult<Vec<WorkspaceInfo>>;
/// The environment a send uses when the caller names none.
fn active_environment_id(&self, workspace_id: &str) -> HostResult<Option<String>>;
fn list_http_requests(&self, workspace_id: &str) -> HostResult<Vec<HttpRequest>>;
fn get_http_request(&self, id: &str) -> HostResult<Option<HttpRequest>>;
fn upsert_http_request(&self, request: &HttpRequest) -> HostResult<HttpRequest>;
fn delete_http_request(&self, id: &str) -> HostResult<HttpRequest>;
fn list_folders(&self, workspace_id: &str) -> HostResult<Vec<Folder>>;
fn upsert_folder(&self, folder: &Folder) -> HostResult<Folder>;
fn delete_folder(&self, id: &str) -> HostResult<Folder>;
/// Send `request` from `workspace_id`, with `environment_id` overriding the
/// active environment for this send only.
fn send_http_request(
&self,
workspace_id: &str,
request: HttpRequest,
environment_id: Option<String>,
) -> impl Future<Output = HostResult<HttpResponse>> + Send;
fn show_toast(&self, toast: ShowToastRequest) -> HostResult<()>;
}
/// Serve the tools on stdin and stdout until the client disconnects.
pub async fn serve_stdio<H: McpHost>(host: H) -> Result<(), String> {
use rmcp::ServiceExt;
let service = YaakMcpServer::new(host)
.serve(rmcp::transport::stdio())
.await
.map_err(|e| format!("Failed to start MCP server: {e}"))?;
service.waiting().await.map_err(|e| format!("MCP server stopped: {e}"))?;
Ok(())
}
+281
View File
@@ -0,0 +1,281 @@
//! Tool inputs, mirroring the zod schemas of the `@yaak/mcp-server` plugin
//! they replace. Optional fields skip serializing so that a tool's input,
//! serialized, holds only what the caller gave, ready to lay over a model.
use rmcp::schemars::{self, JsonSchema};
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
const WORKSPACE_ID: &str = "Workspace ID (required if multiple workspaces are open)";
const BODY_TYPE: &str = r#"Body type. Supported values: "binary", "graphql", "application/x-www-form-urlencoded", "multipart/form-data", or any text-based type (e.g., "application/json", "text/plain")"#;
const BODY: &str = r#"Body content object. Structure varies by bodyType:
- "binary": { filePath: "/path/to/file" }
- "graphql": { query: "{ users { name } }", variables: "{\"id\": \"123\"}" }
- "application/x-www-form-urlencoded": { form: [{ name: "key", value: "val", enabled: true }] }
- "multipart/form-data": { form: [{ name: "field", value: "text", file: "/path/to/file", enabled: true }] }
- text-based (application/json, etc.): { text: "raw body content" }"#;
const AUTHENTICATION_TYPE: &str = r#"Authentication type. Common values: "basic", "bearer", "oauth2", "apikey", "jwt", "awsv4", "oauth1", "ntlm", "none". Use null to inherit from parent."#;
const AUTHENTICATION: &str = r#"Authentication configuration object. Structure varies by authenticationType:
- "basic": { username: "user", password: "pass" }
- "bearer": { token: "abc123", prefix: "Bearer" }
- "oauth2": { clientId: "...", clientSecret: "...", grantType: "authorization_code", authorizationUrl: "...", accessTokenUrl: "...", scope: "...", ... }
- "apikey": { location: "header" | "query", key: "X-API-Key", value: "..." }
- "jwt": { algorithm: "HS256", secret: "...", payload: "{ ... }" }
- "awsv4": { accessKeyId: "...", secretAccessKey: "...", service: "sts", region: "us-east-1", sessionToken: "..." }
- "none": {}"#;
fn enabled_default() -> bool {
true
}
#[derive(Debug, Serialize, Deserialize, JsonSchema)]
pub struct Pair {
pub name: String,
pub value: String,
#[serde(default = "enabled_default")]
pub enabled: bool,
}
/// The given fields of a tool input, by their model names.
pub fn given_fields<T: Serialize>(input: &T) -> Map<String, Value> {
match serde_json::to_value(input) {
Ok(Value::Object(fields)) => fields,
_ => Map::new(),
}
}
#[derive(Debug, Deserialize, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct WorkspaceParams {
#[schemars(description = WORKSPACE_ID)]
pub workspace_id: Option<String>,
}
#[derive(Debug, Deserialize, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct GetHttpRequestParams {
#[schemars(description = "The HTTP request ID")]
pub id: String,
#[schemars(description = WORKSPACE_ID)]
pub workspace_id: Option<String>,
}
#[derive(Debug, Deserialize, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct SendHttpRequestParams {
#[schemars(description = "The HTTP request ID to send")]
pub id: String,
#[schemars(
description = "Environment ID for this send only; defaults to the active environment"
)]
pub environment_id: Option<String>,
#[schemars(description = WORKSPACE_ID)]
pub workspace_id: Option<String>,
}
#[derive(Debug, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct CreateHttpRequestParams {
#[serde(skip_serializing)]
#[schemars(description = WORKSPACE_ID)]
pub workspace_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = "Request name (empty string to auto-generate from URL)")]
pub name: Option<String>,
#[schemars(description = "Request URL")]
pub url: String,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = "HTTP method (defaults to GET)")]
pub method: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = "Parent folder ID")]
pub folder_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = "Request description")]
pub description: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = "Request headers")]
pub headers: Option<Vec<Pair>>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = "URL query parameters")]
pub url_parameters: Option<Vec<Pair>>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = BODY_TYPE)]
pub body_type: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = BODY)]
pub body: Option<Map<String, Value>>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = AUTHENTICATION_TYPE)]
pub authentication_type: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = AUTHENTICATION)]
pub authentication: Option<Map<String, Value>>,
}
#[derive(Debug, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct UpdateHttpRequestParams {
#[serde(skip_serializing)]
#[schemars(description = "HTTP request ID to update")]
pub id: String,
#[serde(skip_serializing)]
#[schemars(description = WORKSPACE_ID)]
pub workspace_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = "Request name")]
pub name: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = "Request URL")]
pub url: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = "HTTP method")]
pub method: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = "Parent folder ID")]
pub folder_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = "Request description")]
pub description: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = "Request headers")]
pub headers: Option<Vec<Pair>>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = "URL query parameters")]
pub url_parameters: Option<Vec<Pair>>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = BODY_TYPE)]
pub body_type: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = BODY)]
pub body: Option<Map<String, Value>>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = AUTHENTICATION_TYPE)]
pub authentication_type: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = AUTHENTICATION)]
pub authentication: Option<Map<String, Value>>,
}
#[derive(Debug, Deserialize, JsonSchema)]
pub struct DeleteHttpRequestParams {
#[schemars(description = "HTTP request ID to delete")]
pub id: String,
}
#[derive(Debug, Deserialize, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct GetFolderParams {
#[schemars(description = "The folder ID")]
pub id: String,
#[schemars(description = WORKSPACE_ID)]
pub workspace_id: Option<String>,
}
#[derive(Debug, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct CreateFolderParams {
#[serde(skip_serializing)]
#[schemars(description = WORKSPACE_ID)]
pub workspace_id: Option<String>,
#[schemars(description = "Folder name")]
pub name: String,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = "Parent folder ID (for nested folders)")]
pub folder_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = "Folder description")]
pub description: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = "Sort priority for ordering")]
pub sort_priority: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = "Default headers to apply to requests in this folder")]
pub headers: Option<Vec<Pair>>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = AUTHENTICATION_TYPE)]
pub authentication_type: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = AUTHENTICATION)]
pub authentication: Option<Map<String, Value>>,
}
#[derive(Debug, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct UpdateFolderParams {
#[serde(skip_serializing)]
#[schemars(description = "Folder ID to update")]
pub id: String,
#[serde(skip_serializing)]
#[schemars(description = WORKSPACE_ID)]
pub workspace_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = "Folder name")]
pub name: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = "Parent folder ID (for nested folders)")]
pub folder_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = "Folder description")]
pub description: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = "Sort priority for ordering")]
pub sort_priority: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = "Default headers to apply to requests in this folder")]
pub headers: Option<Vec<Pair>>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = AUTHENTICATION_TYPE)]
pub authentication_type: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(description = AUTHENTICATION)]
pub authentication: Option<Map<String, Value>>,
}
#[derive(Debug, Deserialize, JsonSchema)]
pub struct DeleteFolderParams {
#[schemars(description = "Folder ID to delete")]
pub id: String,
}
#[derive(Debug, Clone, Copy, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum ToastIcon {
AlertTriangle,
Check,
CheckCircle,
ChevronDown,
Copy,
Info,
Pin,
Search,
Trash,
}
#[derive(Debug, Clone, Copy, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum ToastColor {
Primary,
Secondary,
Info,
Success,
Notice,
Warning,
Danger,
}
#[derive(Debug, Deserialize, JsonSchema)]
pub struct ShowToastParams {
#[schemars(description = "The message to display")]
pub message: String,
#[schemars(description = "Icon name")]
pub icon: Option<ToastIcon>,
#[schemars(description = "Toast color")]
pub color: Option<ToastColor>,
#[schemars(description = "Timeout in milliseconds")]
pub timeout: Option<f64>,
}
+338
View File
@@ -0,0 +1,338 @@
use crate::params::*;
use crate::{HostResult, McpHost, WorkspaceInfo};
use rmcp::handler::server::router::tool::ToolRouter;
use rmcp::handler::server::wrapper::Parameters;
use rmcp::model::{Implementation, ServerCapabilities, ServerConfig};
use rmcp::{ServerHandler, tool, tool_handler, tool_router};
use serde::Serialize;
use serde::de::DeserializeOwned;
use serde_json::{Map, Value, json};
use std::sync::Arc;
use yaak_models::models::{Folder, HttpRequest};
use yaak_plugins::events::{Color, Icon, ShowToastRequest};
type ToolResult = Result<String, String>;
pub struct YaakMcpServer<H: McpHost> {
host: Arc<H>,
tool_router: ToolRouter<Self>,
}
impl<H: McpHost> Clone for YaakMcpServer<H> {
fn clone(&self) -> Self {
Self { host: self.host.clone(), tool_router: self.tool_router.clone() }
}
}
impl<H: McpHost> YaakMcpServer<H> {
pub fn new(host: H) -> Self {
Self::from_arc(Arc::new(host))
}
pub fn from_arc(host: Arc<H>) -> Self {
Self { host, tool_router: Self::tool_router() }
}
fn resolve_workspace(&self, workspace_id: Option<&str>) -> HostResult<WorkspaceInfo> {
let workspaces = self.host.list_workspaces()?;
let Some(workspace_id) = workspace_id else {
if workspaces.len() > 1 {
let list = workspaces
.iter()
.enumerate()
.map(|(i, w)| format!("{}. {} (ID: {})", i + 1, w.name, w.id))
.collect::<Vec<_>>()
.join("\n");
return Err(format!(
"Multiple workspaces are open. Please specify which workspace to use.\n\n\
Currently open workspaces:\n{list}\n\n\
You can use the list_workspaces tool to get workspace IDs, then use other tools \
with the workspace context. For example, ask the user which workspace they want \
to work with by presenting them with the numbered list above."
));
}
return workspaces.into_iter().next().ok_or_else(|| "No workspace is open".to_string());
};
if let Some(workspace) = workspaces.iter().find(|w| w.id == workspace_id) {
return Ok(workspace.clone());
}
let list =
workspaces.iter().map(|w| format!("- {} (ID: {})", w.name, w.id)).collect::<Vec<_>>();
Err(format!(
"Workspace with ID \"{workspace_id}\" not found.\n\nAvailable workspaces:\n{}",
list.join("\n")
))
}
fn find_folder(&self, workspace_id: &str, id: &str) -> HostResult<Option<Folder>> {
Ok(self.host.list_folders(workspace_id)?.into_iter().find(|f| f.id == id))
}
}
#[tool_router]
impl<H: McpHost> YaakMcpServer<H> {
#[tool(
name = "list_workspaces",
title = "List Workspaces",
description = "List all open workspaces in Yaak"
)]
async fn list_workspaces(&self) -> ToolResult {
to_json(&self.host.list_workspaces()?)
}
#[tool(
name = "get_workspace_id",
title = "Get Workspace ID",
description = "Get the current workspace ID"
)]
async fn get_workspace_id(&self) -> ToolResult {
Ok(self.resolve_workspace(None)?.id)
}
#[tool(
name = "get_environment_id",
title = "Get Environment ID",
description = "Get the current environment ID"
)]
async fn get_environment_id(&self) -> ToolResult {
let workspace = self.resolve_workspace(None)?;
Ok(self
.host
.active_environment_id(&workspace.id)?
.unwrap_or_else(|| "No environment selected".to_string()))
}
#[tool(
name = "list_http_requests",
title = "List HTTP Requests",
description = "List all HTTP requests in a workspace"
)]
async fn list_http_requests(&self, Parameters(p): Parameters<WorkspaceParams>) -> ToolResult {
let workspace = self.resolve_workspace(p.workspace_id.as_deref())?;
to_json(&self.host.list_http_requests(&workspace.id)?)
}
#[tool(
name = "get_http_request",
title = "Get HTTP Request",
description = "Get details of a specific HTTP request by ID"
)]
async fn get_http_request(
&self,
Parameters(p): Parameters<GetHttpRequestParams>,
) -> ToolResult {
self.resolve_workspace(p.workspace_id.as_deref())?;
to_json(&self.host.get_http_request(&p.id)?)
}
#[tool(
name = "send_http_request",
title = "Send HTTP Request",
description = "Send an HTTP request and get the response"
)]
async fn send_http_request(
&self,
Parameters(p): Parameters<SendHttpRequestParams>,
) -> ToolResult {
let workspace = self.resolve_workspace(p.workspace_id.as_deref())?;
let request = self
.host
.get_http_request(&p.id)?
.ok_or_else(|| format!("HTTP request with ID {} not found", p.id))?;
let response =
self.host.send_http_request(&workspace.id, request, p.environment_id).await?;
// Plugins never saw where the host keeps the body, so neither do MCP clients
let mut response = serde_json::to_value(response).map_err(|e| e.to_string())?;
if let Value::Object(fields) = &mut response {
fields.remove("bodyPath");
}
to_json(&response)
}
#[tool(
name = "create_http_request",
title = "Create HTTP Request",
description = "Create a new HTTP request"
)]
async fn create_http_request(
&self,
Parameters(p): Parameters<CreateHttpRequestParams>,
) -> ToolResult {
let workspace = self.resolve_workspace(p.workspace_id.as_deref())?;
let mut fields = Map::new();
fields.insert("name".into(), json!(""));
fields.insert("method".into(), json!("GET"));
fields.insert("workspaceId".into(), json!(workspace.id));
fields.extend(given_fields(&p));
fields.insert("id".into(), json!(""));
let request: HttpRequest = from_fields(fields)?;
to_json(&self.host.upsert_http_request(&request)?)
}
#[tool(
name = "update_http_request",
title = "Update HTTP Request",
description = "Update an existing HTTP request"
)]
async fn update_http_request(
&self,
Parameters(p): Parameters<UpdateHttpRequestParams>,
) -> ToolResult {
self.resolve_workspace(p.workspace_id.as_deref())?;
let existing = self
.host
.get_http_request(&p.id)?
.ok_or_else(|| format!("HTTP request with ID {} not found", p.id))?;
let request: HttpRequest = merge(&existing, given_fields(&p))?;
to_json(&self.host.upsert_http_request(&request)?)
}
#[tool(
name = "delete_http_request",
title = "Delete HTTP Request",
description = "Delete an HTTP request by ID"
)]
async fn delete_http_request(
&self,
Parameters(p): Parameters<DeleteHttpRequestParams>,
) -> ToolResult {
let request = self.host.delete_http_request(&p.id)?;
Ok(format!("Deleted: {} ({})", request.name, request.id))
}
#[tool(
name = "list_folders",
title = "List Folders",
description = "List all folders in a workspace"
)]
async fn list_folders(&self, Parameters(p): Parameters<WorkspaceParams>) -> ToolResult {
let workspace = self.resolve_workspace(p.workspace_id.as_deref())?;
to_json(&self.host.list_folders(&workspace.id)?)
}
#[tool(
name = "get_folder",
title = "Get Folder",
description = "Get details of a specific folder by ID"
)]
async fn get_folder(&self, Parameters(p): Parameters<GetFolderParams>) -> ToolResult {
let workspace = self.resolve_workspace(p.workspace_id.as_deref())?;
to_json(&self.find_folder(&workspace.id, &p.id)?)
}
#[tool(
name = "create_folder",
title = "Create Folder",
description = "Create a new folder in a workspace"
)]
async fn create_folder(&self, Parameters(p): Parameters<CreateFolderParams>) -> ToolResult {
let workspace = self.resolve_workspace(p.workspace_id.as_deref())?;
let mut fields = given_fields(&p);
fields.insert("workspaceId".into(), json!(workspace.id));
fields.insert("id".into(), json!(""));
let folder: Folder = from_fields(fields)?;
to_json(&self.host.upsert_folder(&folder)?)
}
#[tool(
name = "update_folder",
title = "Update Folder",
description = "Update an existing folder"
)]
async fn update_folder(&self, Parameters(p): Parameters<UpdateFolderParams>) -> ToolResult {
let workspace = self.resolve_workspace(p.workspace_id.as_deref())?;
let existing = self
.find_folder(&workspace.id, &p.id)?
.ok_or_else(|| format!("Folder with ID {} not found", p.id))?;
let folder: Folder = merge(&existing, given_fields(&p))?;
to_json(&self.host.upsert_folder(&folder)?)
}
#[tool(
name = "delete_folder",
title = "Delete Folder",
description = "Delete a folder by ID"
)]
async fn delete_folder(&self, Parameters(p): Parameters<DeleteFolderParams>) -> ToolResult {
let folder = self.host.delete_folder(&p.id)?;
Ok(format!("Deleted: {} ({})", folder.name, folder.id))
}
#[tool(
name = "show_toast",
title = "Show Toast",
description = "Show a toast notification in Yaak"
)]
async fn show_toast(&self, Parameters(p): Parameters<ShowToastParams>) -> ToolResult {
self.host.show_toast(ShowToastRequest {
message: p.message.clone(),
icon: p.icon.map(icon),
color: p.color.map(color),
timeout: Some(p.timeout.map_or(5000, |t| t as i32)),
})?;
Ok(format!("✓ Toast shown: \"{}\"", p.message))
}
}
#[tool_handler]
impl<H: McpHost> ServerHandler for YaakMcpServer<H> {
fn get_info(&self) -> ServerConfig {
let mut info = ServerConfig::new(ServerCapabilities::builder().enable_tools().build());
info.server_info = Implementation::new("yaak-mcp-server", env!("CARGO_PKG_VERSION"));
info
}
}
fn to_json<T: Serialize>(value: &T) -> ToolResult {
serde_json::to_string_pretty(value).map_err(|e| format!("Failed to serialize result: {e}"))
}
fn from_fields<T: DeserializeOwned>(fields: Map<String, Value>) -> HostResult<T> {
serde_json::from_value(Value::Object(fields)).map_err(|e| format!("Invalid input: {e}"))
}
/// The given fields laid over `existing`, keeping its id.
fn merge<T: Serialize + DeserializeOwned>(
existing: &T,
given: Map<String, Value>,
) -> HostResult<T> {
let Value::Object(mut fields) = serde_json::to_value(existing).map_err(|e| e.to_string())?
else {
return Err("Model did not serialize to an object".to_string());
};
let id = fields.get("id").cloned();
fields.extend(given);
if let Some(id) = id {
fields.insert("id".into(), id);
}
from_fields(fields)
}
fn icon(icon: ToastIcon) -> Icon {
match icon {
ToastIcon::AlertTriangle => Icon::AlertTriangle,
ToastIcon::Check => Icon::Check,
ToastIcon::CheckCircle => Icon::CheckCircle,
ToastIcon::ChevronDown => Icon::ChevronDown,
ToastIcon::Copy => Icon::Copy,
ToastIcon::Info => Icon::Info,
ToastIcon::Pin => Icon::Pin,
ToastIcon::Search => Icon::Search,
ToastIcon::Trash => Icon::Trash,
}
}
fn color(color: ToastColor) -> Color {
match color {
ToastColor::Primary => Color::Primary,
ToastColor::Secondary => Color::Secondary,
ToastColor::Info => Color::Info,
ToastColor::Success => Color::Success,
ToastColor::Notice => Color::Notice,
ToastColor::Warning => Color::Warning,
ToastColor::Danger => Color::Danger,
}
}
+98
View File
@@ -0,0 +1,98 @@
use reqwest::StatusCode;
use std::sync::Arc;
use yaak_mcp::{HostResult, McpHost, WorkspaceInfo};
use yaak_models::models::{Folder, HttpRequest, HttpResponse};
use yaak_plugins::events::ShowToastRequest;
struct EmptyHost;
impl McpHost for EmptyHost {
fn list_workspaces(&self) -> HostResult<Vec<WorkspaceInfo>> {
Ok(Vec::new())
}
fn active_environment_id(&self, _: &str) -> HostResult<Option<String>> {
Ok(None)
}
fn list_http_requests(&self, _: &str) -> HostResult<Vec<HttpRequest>> {
Ok(Vec::new())
}
fn get_http_request(&self, _: &str) -> HostResult<Option<HttpRequest>> {
Ok(None)
}
fn upsert_http_request(&self, r: &HttpRequest) -> HostResult<HttpRequest> {
Ok(r.clone())
}
fn delete_http_request(&self, _: &str) -> HostResult<HttpRequest> {
Err("missing".into())
}
fn list_folders(&self, _: &str) -> HostResult<Vec<Folder>> {
Ok(Vec::new())
}
fn upsert_folder(&self, f: &Folder) -> HostResult<Folder> {
Ok(f.clone())
}
fn delete_folder(&self, _: &str) -> HostResult<Folder> {
Err("missing".into())
}
async fn send_http_request(
&self,
_: &str,
_: HttpRequest,
_: Option<String>,
) -> HostResult<HttpResponse> {
Err("missing".into())
}
fn show_toast(&self, _: ShowToastRequest) -> HostResult<()> {
Ok(())
}
}
const INITIALIZE: &str = r#"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"test","version":"1.0"}}}"#;
#[tokio::test]
async fn requires_the_bearer_token() {
let server = yaak_mcp::http::serve(
Arc::new(EmptyHost),
"127.0.0.1:0".parse().unwrap(),
"secret".to_string(),
)
.await
.unwrap();
let url = format!("http://{}/mcp", server.addr);
let client = reqwest::Client::new();
let post = |url: &str, auth: Option<&str>| {
let mut request = client
.post(url)
.header("Content-Type", "application/json")
.header("Accept", "application/json, text/event-stream")
.body(INITIALIZE);
if let Some(auth) = auth {
request = request.header("Authorization", auth);
}
request.send()
};
for auth in [
None,
Some("Bearer wrong"),
Some("Bearer secre"),
Some("secret"),
] {
let response = post(&url, auth).await.unwrap();
assert_eq!(response.status(), StatusCode::UNAUTHORIZED, "{auth:?}");
assert_eq!(response.headers()["www-authenticate"], "Bearer");
}
let response = post(&url, Some("Bearer secret")).await.unwrap();
assert_eq!(response.status(), StatusCode::OK);
assert!(response.headers().contains_key("mcp-session-id"));
let body = response.text().await.unwrap();
assert!(body.contains("yaak-mcp-server"), "{body}");
let other = format!("http://{}/other", server.addr);
assert_eq!(post(&other, Some("Bearer secret")).await.unwrap().status(), StatusCode::NOT_FOUND);
let addr = server.addr;
server.stop().await;
assert!(tokio::net::TcpStream::connect(addr).await.is_err());
}
+1 -1
View File
@@ -10,7 +10,7 @@ yaak-database = { workspace = true }
hex = { workspace = true }
include_dir = "0.7"
log = { workspace = true }
nanoid = "0.4.0"
nanoid = "0.5.0"
rusqlite = { version = "0.38", features = ["bundled", "chrono"] }
sea-query = { version = "1.0", features = ["with-chrono", "attr"] }
sea-query-rusqlite = { version = "0.8.0", features = ["with-chrono"] }
+9 -2
View File
@@ -364,8 +364,14 @@ export type ImportSourceResource = {
importSourceId: string;
sourceKey: string;
modelType: string;
modelId: string;
snapshot: string;
/**
* `None` once the user has decided not to import this key
*/
modelId?: string;
/**
* Hash of the resource as last applied or decided from the source, if one was recorded
*/
contentHash?: string;
};
export type InheritedBoolSetting = { enabled?: boolean; value: boolean };
@@ -496,6 +502,7 @@ export type Settings = {
updateChannel: string;
hideLicenseBadge: boolean;
promptFeedback: boolean;
mcpServerEnabled: boolean;
autoupdate: boolean;
autoDownloadUpdates: boolean;
checkNotifications: boolean;
+94 -23
View File
@@ -1,7 +1,21 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { Environment, Folder, GrpcRequest, HttpRequest, WebsocketRequest, Workspace } from "./gen_models";
import type {
Environment,
Folder,
GrpcRequest,
HttpRequest,
WebsocketRequest,
Workspace,
} from "./gen_models";
export type BatchUpsertResult = { workspaces: Array<Workspace>, environments: Array<Environment>, folders: Array<Folder>, httpRequests: Array<HttpRequest>, grpcRequests: Array<GrpcRequest>, websocketRequests: Array<WebsocketRequest>, };
export type BatchUpsertResult = {
workspaces: Array<Workspace>;
environments: Array<Environment>;
folders: Array<Folder>;
httpRequests: Array<HttpRequest>;
grpcRequests: Array<GrpcRequest>;
websocketRequests: Array<WebsocketRequest>;
};
export type ImportConflictResolution = "keep_mine" | "take_source";
@@ -11,38 +25,95 @@ export type ImportConflictResolution = "keep_mine" | "take_source";
* The destination workspace and optional folder IDs are captured in the plan so the preview describes
* the exact destination that confirmation will use.
*/
export type ImportDestination = { "type": "new_workspace" } | { "type": "existing_workspace", workspaceId: string, folderId?: string, };
export type ImportDestination =
| { type: "new_workspace" }
| { type: "existing_workspace"; workspaceId: string; folderId?: string };
/**
* Where an import's contents came from, used to link the committed workspace back to it.
*/
export type ImportOrigin = {
/**
* The absolute file path or URL the contents were read from.
*/
origin: string, label: string, };
export type ImportOrigin = {
/**
* The absolute file path or URL the contents were read from.
*/
origin: string;
label: string;
};
export type ImportPlan = { importer: string, destination: ImportDestination, resources: BatchUpsertResult, warnings: Array<ImportPlanWarning>,
/**
* Stable source key for every model in `resources`, keyed by its planned ID.
*/
sourceKeys: { [key in string]?: string },
/**
* One entry per plannable resource; commit applies only the selected ones.
*/
items: Array<ImportPlanItem>, origin?: ImportOrigin, };
export type ImportPlan = {
importer: string;
destination: ImportDestination;
resources: BatchUpsertResult;
warnings: Array<ImportPlanWarning>;
/**
* Stable source key for every model in `resources`, keyed by its planned ID.
*/
sourceKeys: { [key in string]?: string };
/**
* One entry per plannable resource; commit applies only the selected ones.
*/
items: Array<ImportPlanItem>;
origin?: ImportOrigin;
/**
* Individual origins in a combined import, each retaining its own reimport history.
*/
sources: Array<ImportPlanSource>;
};
export type ImportPlanAction = "create" | "update" | "delete" | "unchanged" | "keep_local" | "conflict";
export type ImportPlanAction =
| "create"
| "update"
| "delete"
| "unchanged"
| "keep_local"
| "conflict"
| "ignored";
export type ImportPlanItem = {
action: ImportPlanAction;
model: ImportResourceType;
modelId: string;
name: string;
/**
* Planned parent folder ID for incoming resources; current parent for deletions.
*/
parentId?: string;
selected: boolean;
resolution?: ImportConflictResolution;
reason?: ImportPlanReason;
/**
* Fields where the source and the local copy disagree, so the preview can say why
*/
changedFields: Array<string>;
};
export type ImportPlanItem = { action: ImportPlanAction, model: ImportResourceType, modelId: string, name: string,
/**
* Planned parent folder ID for incoming resources; current parent for deletions.
* Extra context for an action that would otherwise be indistinguishable from its plain form.
*/
parentId?: string, selected: boolean, resolution?: ImportConflictResolution, };
export type ImportPlanReason = "moved_into_ignored_folder";
export type ImportPlanWarning = { title: string, detail: string, };
export type ImportPlanSource = {
importer: string;
origin: ImportOrigin;
workspaceId: string;
sourceKeys: { [key in string]?: string };
linkedSourceId?: string;
};
export type ImportPlanWarning = { title: string; detail: string; level: ImportPlanWarningLevel };
/**
* Whether a plan's note is something to know or something to think twice about.
*/
export type ImportPlanWarningLevel = "info" | "warning";
/**
* The model types an import plan can contain.
*/
export type ImportResourceType = "environment" | "folder" | "grpc_request" | "http_request" | "websocket_request" | "workspace";
export type ImportResourceType =
| "environment"
| "folder"
| "grpc_request"
| "http_request"
| "websocket_request"
| "workspace";
@@ -0,0 +1,24 @@
-- Replace the per-resource snapshot with a content hash, and let a row exist without a model
-- so a resource the user chose not to import can be remembered.
CREATE TABLE import_source_resources_new
(
model TEXT DEFAULT 'import_source_resource' NOT NULL,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL,
import_source_id TEXT NOT NULL,
source_key TEXT NOT NULL,
model_type TEXT NOT NULL,
model_id TEXT,
content_hash TEXT,
PRIMARY KEY (import_source_id, source_key)
);
INSERT INTO import_source_resources_new (model, created_at, updated_at, import_source_id,
source_key, model_type, model_id, content_hash)
SELECT model, created_at, updated_at, import_source_id, source_key, model_type, model_id, NULL
FROM import_source_resources;
DROP TABLE import_source_resources;
ALTER TABLE import_source_resources_new
RENAME TO import_source_resources;
@@ -0,0 +1,3 @@
-- Add a setting to run the built-in MCP server
ALTER TABLE settings
ADD COLUMN mcp_server_enabled BOOLEAN DEFAULT FALSE NOT NULL;
+245 -110
View File
@@ -1,9 +1,11 @@
use crate::error::Error::GenericError;
use crate::error::Result;
use crate::util::generate_prefixed_id;
use include_dir::{Dir, include_dir};
use log::{debug, info};
use rusqlite::{OptionalExtension, params};
use yaak_database::{SqliteConn, SqlitePool};
use rusqlite::{OptionalExtension, Transaction, TransactionBehavior, params};
use std::ops::Deref;
use yaak_database::{ConnectionOrTx, SqlitePool};
static BLOB_MIGRATIONS_DIR: Dir = include_dir!("$CARGO_MANIFEST_DIR/blob_migrations");
@@ -22,41 +24,64 @@ impl BodyChunk {
}
}
/// Manages the blob database connection pool.
// Pool is internally synchronized — don't wrap it in a Mutex. A Mutex held across the
/// Manages the blob database: a reader pool and a single writer, for the
/// same reason as [`crate::query_manager::QueryManager`].
// Pools are internally synchronized — don't wrap them in a Mutex. A Mutex held across the
// blocking `get()` serializes every blob access behind the slowest waiter, freezing the
// whole app whenever the pool is exhausted.
#[derive(Debug, Clone)]
pub struct BlobManager {
pool: SqlitePool,
readers: SqlitePool,
writer: SqlitePool,
}
impl BlobManager {
pub fn new(pool: SqlitePool) -> Self {
Self { pool }
/// `writer` must be a pool with a single connection.
pub fn new(readers: SqlitePool, writer: SqlitePool) -> Self {
Self { readers, writer }
}
pub fn connect(&self) -> BlobContext {
let conn = self.pool.get().expect("Failed to get blob DB connection from pool");
BlobContext { conn }
/// A read handle from the reader pool.
pub fn connect(&self) -> BlobContext<'_> {
let conn = self.readers.get().expect("Failed to get blob DB connection from pool");
BlobContext { conn: ConnectionOrTx::Connection(conn) }
}
/// Run `func` in a transaction on the writer connection.
pub fn with_tx<T, E>(
&self,
func: impl FnOnce(&BlobWriter) -> std::result::Result<T, E>,
) -> std::result::Result<T, E>
where
E: From<crate::error::Error>,
{
let conn = self.writer.get().map_err(crate::error::Error::SqlPoolError)?;
let tx = Transaction::new_unchecked(&conn, TransactionBehavior::Immediate)
.map_err(crate::error::Error::SqlError)?;
let writer = BlobWriter { ctx: BlobContext { conn: ConnectionOrTx::Transaction(&tx) } };
match func(&writer) {
Ok(val) => {
tx.commit().map_err(|e| {
GenericError(format!("Failed to commit blob transaction {e:?}"))
})?;
Ok(val)
}
Err(e) => {
tx.rollback().map_err(|e| {
GenericError(format!("Failed to rollback blob transaction {e:?}"))
})?;
Err(e)
}
}
}
}
/// Context for blob database operations.
pub struct BlobContext {
conn: SqliteConn,
/// Read handle for the blob database.
pub struct BlobContext<'a> {
conn: ConnectionOrTx<'a>,
}
impl BlobContext {
/// Insert a single chunk.
pub fn insert_chunk(&self, chunk: &BodyChunk) -> Result<()> {
self.conn.execute(
"INSERT INTO body_chunks (id, body_id, chunk_index, data) VALUES (?1, ?2, ?3, ?4)",
params![chunk.id, chunk.body_id, chunk.chunk_index, chunk.data],
)?;
Ok(())
}
impl<'a> BlobContext<'a> {
/// Get all chunks for a body, ordered by chunk_index.
pub fn get_chunks(&self, body_id: &str) -> Result<Vec<BodyChunk>> {
let mut stmt = self.conn.prepare(
@@ -87,25 +112,11 @@ impl BlobContext {
Ok(ids)
}
/// Delete all chunks for a body.
pub fn delete_chunks(&self, body_id: &str) -> Result<()> {
self.conn.execute("DELETE FROM body_chunks WHERE body_id = ?1", params![body_id])?;
Ok(())
}
/// Delete all chunks matching a body_id prefix (e.g., "rs_abc123.%" to delete all bodies for a response).
pub fn delete_chunks_like(&self, body_id_prefix: &str) -> Result<()> {
self.conn
.execute("DELETE FROM body_chunks WHERE body_id LIKE ?1", params![body_id_prefix])?;
Ok(())
}
}
/// Get total size of a body without loading data.
impl BlobContext {
/// Get total size of a body without loading data.
pub fn get_body_size(&self, body_id: &str) -> Result<usize> {
let size: i64 = self
.conn
.resolve()
.query_row(
"SELECT COALESCE(SUM(LENGTH(data)), 0) FROM body_chunks WHERE body_id = ?1",
params![body_id],
@@ -119,6 +130,7 @@ impl BlobContext {
pub fn body_exists(&self, body_id: &str) -> Result<bool> {
let count: i64 = self
.conn
.resolve()
.query_row(
"SELECT COUNT(*) FROM body_chunks WHERE body_id = ?1",
params![body_id],
@@ -129,6 +141,96 @@ impl BlobContext {
}
}
/// Write handle for the blob database. Derefs to [`BlobContext`] for reads.
pub struct BlobWriter<'a> {
ctx: BlobContext<'a>,
}
impl<'a> Deref for BlobWriter<'a> {
type Target = BlobContext<'a>;
fn deref(&self) -> &BlobContext<'a> {
&self.ctx
}
}
impl<'a> BlobWriter<'a> {
/// Insert a single chunk.
pub fn insert_chunk(&self, chunk: &BodyChunk) -> Result<()> {
self.conn.execute(
"INSERT INTO body_chunks (id, body_id, chunk_index, data) VALUES (?1, ?2, ?3, ?4)",
params![chunk.id, chunk.body_id, chunk.chunk_index, chunk.data],
)?;
Ok(())
}
/// Delete all chunks for a body.
pub fn delete_chunks(&self, body_id: &str) -> Result<()> {
self.conn.execute("DELETE FROM body_chunks WHERE body_id = ?1", params![body_id])?;
Ok(())
}
/// Delete all chunks matching a body_id prefix (e.g., "rs_abc123.%" to delete all
/// bodies for a response).
pub fn delete_chunks_like(&self, body_id_prefix: &str) -> Result<()> {
self.conn
.execute("DELETE FROM body_chunks WHERE body_id LIKE ?1", params![body_id_prefix])?;
Ok(())
}
}
/// Size of each stored chunk. Bodies arrive in small reads (8 to 16 KiB is typical), so they
/// are grouped into chunks of this size to avoid thousands of tiny inserts for large bodies.
pub const BODY_CHUNK_SIZE: usize = 1024 * 1024;
/// Writes one body as it streams, grouping its pieces into [`BODY_CHUNK_SIZE`] chunks. Each
/// full chunk is inserted as soon as it fills; [`BodyChunkWriter::finish`] inserts the rest.
pub struct BodyChunkWriter {
blob_manager: BlobManager,
body_id: String,
buf: Vec<u8>,
chunk_index: i32,
total_bytes: usize,
}
impl BodyChunkWriter {
pub fn new(blob_manager: BlobManager, body_id: impl Into<String>) -> Self {
Self {
blob_manager,
body_id: body_id.into(),
buf: Vec::new(),
chunk_index: 0,
total_bytes: 0,
}
}
pub fn write(&mut self, data: &[u8]) -> Result<()> {
self.total_bytes += data.len();
self.buf.extend_from_slice(data);
while self.buf.len() >= BODY_CHUNK_SIZE {
let data = self.buf.drain(..BODY_CHUNK_SIZE).collect();
self.insert(data)?;
}
Ok(())
}
/// Inserts the partial last chunk and returns the bytes written in all.
pub fn finish(mut self) -> Result<usize> {
if !self.buf.is_empty() {
let data = std::mem::take(&mut self.buf);
self.insert(data)?;
}
Ok(self.total_bytes)
}
fn insert(&mut self, data: Vec<u8>) -> Result<()> {
let chunk = BodyChunk::new(&self.body_id, self.chunk_index, data);
self.blob_manager.with_tx(|b| b.insert_chunk(&chunk))?;
self.chunk_index += 1;
Ok(())
}
}
/// Run migrations for the blob database.
pub fn migrate_blob_db(pool: &SqlitePool) -> Result<()> {
info!("Running blob database migrations");
@@ -196,28 +298,39 @@ pub fn migrate_blob_db(pool: &SqlitePool) -> Result<()> {
#[cfg(test)]
mod tests {
use super::*;
use crate::error::Error;
fn create_test_pool() -> SqlitePool {
fn create_test_manager() -> BlobManager {
let manager = r2d2_sqlite::SqliteConnectionManager::memory();
let pool = r2d2::Pool::builder().max_size(1).build(manager).unwrap();
migrate_blob_db(&pool).unwrap();
pool
BlobManager::new(pool.clone(), pool)
}
fn insert(manager: &BlobManager, chunks: &[BodyChunk]) {
manager
.with_tx(|b| {
for c in chunks {
b.insert_chunk(c)?;
}
Ok::<_, Error>(())
})
.unwrap();
}
#[test]
fn test_insert_and_get_chunks() {
let pool = create_test_pool();
let manager = BlobManager::new(pool);
let ctx = manager.connect();
let manager = create_test_manager();
let body_id = "rs_test123.request";
let chunk1 = BodyChunk::new(body_id, 0, b"Hello, ".to_vec());
let chunk2 = BodyChunk::new(body_id, 1, b"World!".to_vec());
insert(
&manager,
&[
BodyChunk::new(body_id, 0, b"Hello, ".to_vec()),
BodyChunk::new(body_id, 1, b"World!".to_vec()),
],
);
ctx.insert_chunk(&chunk1).unwrap();
ctx.insert_chunk(&chunk2).unwrap();
let chunks = ctx.get_chunks(body_id).unwrap();
let chunks = manager.connect().get_chunks(body_id).unwrap();
assert_eq!(chunks.len(), 2);
assert_eq!(chunks[0].chunk_index, 0);
assert_eq!(chunks[0].data, b"Hello, ");
@@ -227,18 +340,19 @@ mod tests {
#[test]
fn test_get_chunks_ordered_by_index() {
let pool = create_test_pool();
let manager = BlobManager::new(pool);
let ctx = manager.connect();
let manager = create_test_manager();
let body_id = "rs_test123.request";
// Insert out of order
ctx.insert_chunk(&BodyChunk::new(body_id, 2, b"C".to_vec())).unwrap();
ctx.insert_chunk(&BodyChunk::new(body_id, 0, b"A".to_vec())).unwrap();
ctx.insert_chunk(&BodyChunk::new(body_id, 1, b"B".to_vec())).unwrap();
insert(
&manager,
&[
BodyChunk::new(body_id, 2, b"C".to_vec()),
BodyChunk::new(body_id, 0, b"A".to_vec()),
BodyChunk::new(body_id, 1, b"B".to_vec()),
],
);
let chunks = ctx.get_chunks(body_id).unwrap();
let chunks = manager.connect().get_chunks(body_id).unwrap();
assert_eq!(chunks.len(), 3);
assert_eq!(chunks[0].data, b"A");
assert_eq!(chunks[1].data, b"B");
@@ -247,89 +361,85 @@ mod tests {
#[test]
fn test_delete_chunks() {
let pool = create_test_pool();
let manager = BlobManager::new(pool);
let ctx = manager.connect();
let manager = create_test_manager();
let body_id = "rs_test123.request";
ctx.insert_chunk(&BodyChunk::new(body_id, 0, b"data".to_vec())).unwrap();
insert(&manager, &[BodyChunk::new(body_id, 0, b"data".to_vec())]);
assert!(manager.connect().body_exists(body_id).unwrap());
assert!(ctx.body_exists(body_id).unwrap());
ctx.delete_chunks(body_id).unwrap();
manager.with_tx(|b| b.delete_chunks(body_id)).unwrap();
let ctx = manager.connect();
assert!(!ctx.body_exists(body_id).unwrap());
assert_eq!(ctx.get_chunks(body_id).unwrap().len(), 0);
}
#[test]
fn test_delete_chunks_like() {
let pool = create_test_pool();
let manager = BlobManager::new(pool);
let ctx = manager.connect();
let manager = create_test_manager();
// Insert chunks for same response but different body types
ctx.insert_chunk(&BodyChunk::new("rs_abc.request", 0, b"req".to_vec())).unwrap();
ctx.insert_chunk(&BodyChunk::new("rs_abc.response", 0, b"resp".to_vec())).unwrap();
ctx.insert_chunk(&BodyChunk::new("rs_other.request", 0, b"other".to_vec())).unwrap();
insert(
&manager,
&[
BodyChunk::new("rs_abc.request", 0, b"req".to_vec()),
BodyChunk::new("rs_abc.response", 0, b"resp".to_vec()),
BodyChunk::new("rs_other.request", 0, b"other".to_vec()),
],
);
// Delete all bodies for rs_abc
ctx.delete_chunks_like("rs_abc.%").unwrap();
manager.with_tx(|b| b.delete_chunks_like("rs_abc.%")).unwrap();
// rs_abc bodies should be gone
let ctx = manager.connect();
assert!(!ctx.body_exists("rs_abc.request").unwrap());
assert!(!ctx.body_exists("rs_abc.response").unwrap());
// rs_other should still exist
assert!(ctx.body_exists("rs_other.request").unwrap());
}
#[test]
fn test_get_body_size() {
let pool = create_test_pool();
let manager = BlobManager::new(pool);
let ctx = manager.connect();
let manager = create_test_manager();
let body_id = "rs_test123.request";
ctx.insert_chunk(&BodyChunk::new(body_id, 0, b"Hello".to_vec())).unwrap();
ctx.insert_chunk(&BodyChunk::new(body_id, 1, b"World".to_vec())).unwrap();
insert(
&manager,
&[
BodyChunk::new(body_id, 0, b"Hello".to_vec()),
BodyChunk::new(body_id, 1, b"World".to_vec()),
],
);
let size = ctx.get_body_size(body_id).unwrap();
let size = manager.connect().get_body_size(body_id).unwrap();
assert_eq!(size, 10); // "Hello" + "World" = 10 bytes
}
#[test]
fn test_get_body_size_empty() {
let pool = create_test_pool();
let manager = BlobManager::new(pool);
let ctx = manager.connect();
let size = ctx.get_body_size("nonexistent").unwrap();
let manager = create_test_manager();
let size = manager.connect().get_body_size("nonexistent").unwrap();
assert_eq!(size, 0);
}
#[test]
fn test_body_exists() {
let pool = create_test_pool();
let manager = BlobManager::new(pool);
let ctx = manager.connect();
let manager = create_test_manager();
assert!(!manager.connect().body_exists("rs_test.request").unwrap());
assert!(!ctx.body_exists("rs_test.request").unwrap());
insert(&manager, &[BodyChunk::new("rs_test.request", 0, b"data".to_vec())]);
ctx.insert_chunk(&BodyChunk::new("rs_test.request", 0, b"data".to_vec())).unwrap();
assert!(ctx.body_exists("rs_test.request").unwrap());
assert!(manager.connect().body_exists("rs_test.request").unwrap());
}
#[test]
fn test_multiple_bodies_isolated() {
let pool = create_test_pool();
let manager = BlobManager::new(pool);
let manager = create_test_manager();
insert(
&manager,
&[
BodyChunk::new("body1", 0, b"data1".to_vec()),
BodyChunk::new("body2", 0, b"data2".to_vec()),
],
);
let ctx = manager.connect();
ctx.insert_chunk(&BodyChunk::new("body1", 0, b"data1".to_vec())).unwrap();
ctx.insert_chunk(&BodyChunk::new("body2", 0, b"data2".to_vec())).unwrap();
let chunks1 = ctx.get_chunks("body1").unwrap();
let chunks2 = ctx.get_chunks("body2").unwrap();
@@ -340,17 +450,42 @@ mod tests {
}
#[test]
fn test_large_chunk() {
let pool = create_test_pool();
let manager = BlobManager::new(pool);
let ctx = manager.connect();
fn test_chunk_writer_groups_pieces() {
let manager = create_test_manager();
let body_id = "fl_test.response";
let piece = vec![7u8; 16 * 1024];
let pieces = BODY_CHUNK_SIZE * 2 / piece.len() + 3;
let mut writer = BodyChunkWriter::new(manager.clone(), body_id);
for _ in 0..pieces {
writer.write(&piece).unwrap();
}
assert_eq!(manager.connect().get_chunks(body_id).unwrap().len(), 2);
assert_eq!(writer.finish().unwrap(), pieces * piece.len());
let chunks = manager.connect().get_chunks(body_id).unwrap();
let sizes: Vec<_> = chunks.iter().map(|c| c.data.len()).collect();
assert_eq!(sizes, vec![BODY_CHUNK_SIZE, BODY_CHUNK_SIZE, 3 * piece.len()]);
assert_eq!(chunks.iter().map(|c| c.chunk_index).collect::<Vec<_>>(), vec![0, 1, 2]);
}
#[test]
fn test_chunk_writer_empty_body() {
let manager = create_test_manager();
let writer = BodyChunkWriter::new(manager.clone(), "fl_empty.request");
assert_eq!(writer.finish().unwrap(), 0);
assert!(!manager.connect().body_exists("fl_empty.request").unwrap());
}
#[test]
fn test_large_chunk() {
let manager = create_test_manager();
// 1MB chunk
let large_data: Vec<u8> = (0..1024 * 1024).map(|i| (i % 256) as u8).collect();
let body_id = "rs_large.request";
insert(&manager, &[BodyChunk::new(body_id, 0, large_data.clone())]);
ctx.insert_chunk(&BodyChunk::new(body_id, 0, large_data.clone())).unwrap();
let ctx = manager.connect();
let chunks = ctx.get_chunks(body_id).unwrap();
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].data, large_data);
+44 -10
View File
@@ -3,18 +3,24 @@ use crate::models::{AnyModel, UpsertModelInfo};
use crate::util::{ModelChangeEvent, ModelPayload, UpdateSource};
use rusqlite::params;
use sea_query::{IntoColumnRef, IntoIden, SimpleExpr};
use std::cell::RefCell;
use std::fmt::Debug;
use std::sync::mpsc;
use std::ops::Deref;
use yaak_database::DbContext;
/// A read handle. Comes from the reader pool and can only query.
///
/// Anything that changes a row lives on [`WriteDb`], which is only ever handed
/// out inside a transaction on the single writer connection. That split is
/// what keeps the pool from filling with writers waiting on each other: there
/// is one writer, so there is never a second one to wait for.
pub struct ClientDb<'a> {
pub(crate) ctx: DbContext<'a>,
pub(crate) events_tx: mpsc::Sender<ModelPayload>,
}
impl<'a> ClientDb<'a> {
pub fn new(ctx: DbContext<'a>, events_tx: mpsc::Sender<ModelPayload>) -> Self {
Self { ctx, events_tx }
pub fn new(ctx: DbContext<'a>) -> Self {
Self { ctx }
}
/// Access the underlying connection for custom queries.
@@ -22,8 +28,6 @@ impl<'a> ClientDb<'a> {
self.ctx.conn()
}
// --- Read delegates (thin wrappers over DbContext) ---
pub(crate) fn find_one<M>(
&self,
col: impl IntoColumnRef + IntoIden + Clone,
@@ -64,6 +68,38 @@ impl<'a> ClientDb<'a> {
{
Ok(self.ctx.find_many(col, value, limit)?)
}
}
/// A write handle: a [`ClientDb`] on the writer connection, inside a
/// transaction, that can also change rows. Derefs to [`ClientDb`] so every
/// query is available while writing, and reads inside the transaction see
/// its own uncommitted writes.
///
/// Model events are held back until the transaction commits; a rollback
/// discards them along with the rows.
pub struct WriteDb<'a> {
db: ClientDb<'a>,
pending_events: RefCell<Vec<ModelPayload>>,
}
impl<'a> Deref for WriteDb<'a> {
type Target = ClientDb<'a>;
fn deref(&self) -> &ClientDb<'a> {
&self.db
}
}
impl<'a> WriteDb<'a> {
pub fn new(ctx: DbContext<'a>) -> Self {
Self { db: ClientDb::new(ctx), pending_events: RefCell::new(Vec::new()) }
}
/// The events for everything written so far, to send once the
/// transaction has committed.
pub(crate) fn into_events(self) -> Vec<ModelPayload> {
self.pending_events.into_inner()
}
/// Bulk-delete all rows matching a column value WITHOUT recording model
/// changes or emitting events. Only use for cascades whose deletion is
@@ -80,8 +116,6 @@ impl<'a> ClientDb<'a> {
Ok(self.ctx.delete_many::<M>(col, value)?)
}
// --- Write operations (with event recording) ---
pub(crate) fn upsert<M>(&self, model: &M, source: &UpdateSource) -> Result<M>
where
M: Into<AnyModel> + UpsertModelInfo + Clone,
@@ -95,7 +129,7 @@ impl<'a> ClientDb<'a> {
};
self.record_model_change(&payload)?;
let _ = self.events_tx.send(payload);
self.pending_events.borrow_mut().push(payload);
Ok(m)
}
@@ -113,7 +147,7 @@ impl<'a> ClientDb<'a> {
};
self.record_model_change(&payload)?;
let _ = self.events_tx.send(payload);
self.pending_events.borrow_mut().push(payload);
Ok(m.clone())
}
+3
View File
@@ -33,6 +33,9 @@ pub enum Error {
#[error("No base environment for {0}")]
MissingBaseEnvironment(String),
#[error("Invalid environment selection: {0}")]
InvalidEnvironment(String),
#[error("Multiple base environments for {0}. Delete duplicates before continuing.")]
MultipleBaseEnvironments(String),
+104
View File
@@ -0,0 +1,104 @@
//! The example workspace offered during onboarding.
//!
//! Authored as JSON with `{{PLACEHOLDER}}` ids so requests can reference each
//! other (the chaining example names another request by id). Every placeholder
//! gets a fresh id per creation, so the example can be created more than once.
use crate::error::Result;
use crate::models::{Environment, Folder, HttpRequest, UpsertModelInfo, Workspace};
use crate::query_manager::QueryManager;
use crate::util::{BatchUpsertResult, UpdateSource};
const EXAMPLE_JSON: &str = include_str!("example_workspace.json");
const WORKSPACE_IDS: &[&str] = &["WORKSPACE"];
const ENVIRONMENT_IDS: &[&str] = &["ENV_BASE", "ENV_USER_2"];
const FOLDER_IDS: &[&str] = &[
"FOLDER_BASICS",
"FOLDER_VARIABLES",
"FOLDER_CHAINING",
"FOLDER_AUTH",
];
const REQUEST_IDS: &[&str] = &[
"RQ_LIST_POSTS",
"RQ_GET_POST",
"RQ_CREATE_POST",
"RQ_CURRENT_USER",
"RQ_LIST_USERS",
"RQ_POSTS_BY_FIRST_USER",
"RQ_TODOS",
"RQ_LOG_IN",
"RQ_ME",
];
pub fn create_example_workspace(
query_manager: &QueryManager,
source: &UpdateSource,
) -> Result<BatchUpsertResult> {
let resources = example_resources()?;
Ok(query_manager.with_tx(|tx| {
tx.batch_upsert(
resources.workspaces,
resources.environments,
resources.folders,
resources.http_requests,
resources.grpc_requests,
resources.websocket_requests,
source,
)
})?)
}
fn example_resources() -> Result<BatchUpsertResult> {
let mut json = EXAMPLE_JSON.to_string();
let placeholders = WORKSPACE_IDS
.iter()
.map(|p| (*p, Workspace::generate_id()))
.chain(ENVIRONMENT_IDS.iter().map(|p| (*p, Environment::generate_id())))
.chain(FOLDER_IDS.iter().map(|p| (*p, Folder::generate_id())))
.chain(REQUEST_IDS.iter().map(|p| (*p, HttpRequest::generate_id())));
for (placeholder, id) in placeholders {
json = json.replace(&format!("{{{{{placeholder}}}}}"), &id);
}
debug_assert!(!json.contains("{{"), "example workspace has an unmapped placeholder");
Ok(serde_json::from_str(&json)?)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::init_in_memory;
#[test]
fn every_placeholder_is_mapped() {
let json = serde_json::to_string(&example_resources().unwrap()).unwrap();
assert!(!json.contains("{{"), "unmapped placeholder in {json}");
}
#[test]
fn chained_request_references_a_created_request() {
let (query_manager, _blobs, _rx) = init_in_memory().unwrap();
let created = create_example_workspace(&query_manager, &UpdateSource::Background).unwrap();
assert_eq!(created.workspaces.len(), 1);
let workspace_id = &created.workspaces[0].id;
let db = query_manager.connect();
let requests = db.list_http_requests(workspace_id).unwrap();
assert_eq!(requests.len(), REQUEST_IDS.len());
assert_eq!(db.list_folders(workspace_id).unwrap().len(), FOLDER_IDS.len());
assert_eq!(db.list_environments(workspace_id).unwrap().len(), ENVIRONMENT_IDS.len());
let list_users = requests.iter().find(|r| r.name == "List users").unwrap();
let chained = requests.iter().find(|r| r.name == "Posts by the first user").unwrap();
let param = &chained.url_parameters[0].value;
assert!(param.contains(&format!("request='{}'", list_users.id)), "{param}");
}
#[test]
fn creating_twice_makes_two_workspaces() {
let (query_manager, _blobs, _rx) = init_in_memory().unwrap();
create_example_workspace(&query_manager, &UpdateSource::Background).unwrap();
create_example_workspace(&query_manager, &UpdateSource::Background).unwrap();
assert_eq!(query_manager.connect().list_workspaces().unwrap().len(), 2);
}
}
@@ -0,0 +1,181 @@
{
"workspaces": [
{
"id": "{{WORKSPACE}}",
"name": "Example Workspace",
"description": "A small tour of Yaak using the [Yaak Playground](https://yaak.run) API. Nothing here needs an account.\n\n1. Open **Getting Started / List posts** and press Send\n2. Open **Variables / Get the current user**, then switch the environment in the top-left to change which user it fetches\n3. Open **Chaining / Posts by the first user** to see how one request reads a value out of another request's response\n4. Open **Authentication / Get my profile** to see a login token used for you\n\nDelete this workspace whenever you're done with it."
}
],
"environments": [
{
"id": "{{ENV_BASE}}",
"workspaceId": "{{WORKSPACE}}",
"name": "Global Variables",
"parentModel": "workspace",
"public": true,
"variables": [
{ "name": "base_url", "value": "https://yaak.run", "enabled": true },
{ "name": "user_id", "value": "1", "enabled": true },
{ "name": "username", "value": "ada", "enabled": true },
{ "name": "password", "value": "yaak", "enabled": true }
]
},
{
"id": "{{ENV_USER_2}}",
"workspaceId": "{{WORKSPACE}}",
"name": "Another User",
"parentModel": "environment",
"public": true,
"color": "#c084fc",
"variables": [
{ "name": "user_id", "value": "2", "enabled": true },
{ "name": "username", "value": "grace", "enabled": true }
]
}
],
"folders": [
{
"id": "{{FOLDER_BASICS}}",
"workspaceId": "{{WORKSPACE}}",
"name": "Getting Started",
"sortPriority": 1,
"description": "Plain requests. Send one, then look through the response tabs: Body, Headers, Cookies, and Timeline."
},
{
"id": "{{FOLDER_VARIABLES}}",
"workspaceId": "{{WORKSPACE}}",
"name": "Variables",
"sortPriority": 2,
"description": "Anything wrapped in `${[ ... ]}` is a template. Variables come from the environment selected in the top-left, and sub-environments override the base values."
},
{
"id": "{{FOLDER_CHAINING}}",
"workspaceId": "{{WORKSPACE}}",
"name": "Chaining",
"sortPriority": 3,
"description": "The `response()` template function reads a value out of another request's response, sending that request first if it has to. Put the cursor in a URL and press Ctrl+Space to insert one."
},
{
"id": "{{FOLDER_AUTH}}",
"workspaceId": "{{WORKSPACE}}",
"name": "Authentication",
"sortPriority": 4,
"description": "This folder sets Bearer auth to the token from the **Log in** response. Every request inside inherits it, so the token lives in one place. When the last login is more than 50 minutes old, Yaak logs in again before sending.",
"authenticationType": "bearer",
"authentication": {
"token": "${[ response.body.path(request='{{RQ_LOG_IN}}', path='$.accessToken', behavior='ttl', ttl='3000') ]}"
}
}
],
"httpRequests": [
{
"id": "{{RQ_LIST_POSTS}}",
"workspaceId": "{{WORKSPACE}}",
"folderId": "{{FOLDER_BASICS}}",
"name": "List posts",
"method": "GET",
"url": "${[ base_url ]}/posts",
"sortPriority": 1,
"urlParameters": [{ "name": "limit", "value": "5", "enabled": true }],
"description": "Query parameters live in the Params tab. Try raising `limit`."
},
{
"id": "{{RQ_GET_POST}}",
"workspaceId": "{{WORKSPACE}}",
"folderId": "{{FOLDER_BASICS}}",
"name": "Get a post",
"method": "GET",
"url": "${[ base_url ]}/posts/1",
"sortPriority": 2,
"description": "Change the ID at the end of the URL and send again."
},
{
"id": "{{RQ_CREATE_POST}}",
"workspaceId": "{{WORKSPACE}}",
"folderId": "{{FOLDER_BASICS}}",
"name": "Create a post",
"method": "POST",
"url": "${[ base_url ]}/posts",
"sortPriority": 3,
"bodyType": "application/json",
"body": {
"text": "{\n \"title\": \"Hello from Yaak\",\n \"body\": \"Templates work inside bodies too\",\n \"userId\": ${[ user_id ]}\n}"
},
"headers": [{ "name": "Content-Type", "value": "application/json", "enabled": true }],
"description": "Posts you create are only visible to you, and go back to the sample data after an hour. Put the new ID into **Get a post** to read it back."
},
{
"id": "{{RQ_CURRENT_USER}}",
"workspaceId": "{{WORKSPACE}}",
"folderId": "{{FOLDER_VARIABLES}}",
"name": "Get the current user",
"method": "GET",
"url": "${[ base_url ]}/users/${[ user_id ]}",
"sortPriority": 1,
"description": "Switch to the **Another User** environment in the top-left and send again. Hover a template in the URL to see what it resolves to."
},
{
"id": "{{RQ_TODOS}}",
"workspaceId": "{{WORKSPACE}}",
"folderId": "{{FOLDER_VARIABLES}}",
"name": "List the user's todos",
"method": "GET",
"url": "${[ base_url ]}/users/${[ user_id ]}/todos",
"sortPriority": 2,
"urlParameters": [{ "name": "completed", "value": "false", "enabled": true }],
"description": "The same `user_id` variable, used in a different request. Disable the `completed` parameter to see finished todos too."
},
{
"id": "{{RQ_LIST_USERS}}",
"workspaceId": "{{WORKSPACE}}",
"folderId": "{{FOLDER_CHAINING}}",
"name": "List users",
"method": "GET",
"url": "${[ base_url ]}/users",
"sortPriority": 1
},
{
"id": "{{RQ_POSTS_BY_FIRST_USER}}",
"workspaceId": "{{WORKSPACE}}",
"folderId": "{{FOLDER_CHAINING}}",
"name": "Posts by the first user",
"method": "GET",
"url": "${[ base_url ]}/posts",
"sortPriority": 2,
"urlParameters": [
{
"name": "userId",
"value": "${[ response.body.path(request='{{RQ_LIST_USERS}}', path='$[0].id') ]}",
"enabled": true
}
],
"description": "The `userId` parameter is pulled from the **List users** response with the JSONPath `$[0].id`. If that request hasn't been sent yet, Yaak sends it first."
},
{
"id": "{{RQ_LOG_IN}}",
"workspaceId": "{{WORKSPACE}}",
"folderId": "{{FOLDER_AUTH}}",
"name": "Log in",
"method": "POST",
"url": "${[ base_url ]}/auth/token",
"sortPriority": 1,
"authenticationType": "none",
"bodyType": "application/json",
"body": {
"text": "{\n \"username\": \"${[ username ]}\",\n \"password\": \"${[ password ]}\"\n}"
},
"headers": [{ "name": "Content-Type", "value": "application/json", "enabled": true }],
"description": "Sends the username and password from the environment and returns a token. Its Auth tab is set to No Auth, so it doesn't try to use the token it's fetching."
},
{
"id": "{{RQ_ME}}",
"workspaceId": "{{WORKSPACE}}",
"folderId": "{{FOLDER_AUTH}}",
"name": "Get my profile",
"method": "GET",
"url": "${[ base_url ]}/auth/me",
"sortPriority": 2,
"description": "Open the Auth tab: it inherits Bearer auth from the folder, and there's nothing to copy and paste. Check the Timeline tab after sending to see the Authorization header that was sent. To try it as someone else, switch to **Another User** and send **Log in** again."
}
]
}
+33
View File
@@ -0,0 +1,33 @@
//! Building an export document.
//!
//! Here rather than in `yaak` because the whole of it is this crate: a connection, the
//! resources, and JSON. Putting it where the browser can reach it too is what keeps a
//! tab's export from being a second implementation that drifts.
use crate::error::Result;
use crate::query_manager::QueryManager;
use crate::util::get_workspace_export_resources;
pub struct ExportDataParams<'a> {
pub query_manager: &'a QueryManager,
pub yaak_version: &'a str,
pub workspace_ids: Vec<&'a str>,
pub include_private_environments: bool,
}
/// The export document, as JSON.
///
/// Returned rather than written: where an export goes is the host's to decide, and a browser
/// tab has no path to be handed. The desktop hands the bytes to its save dialog, the CLI
/// writes them, a tab downloads them. None of that is this function's business.
pub fn export_data(params: ExportDataParams<'_>) -> Result<String> {
let db = params.query_manager.connect();
let export_data = get_workspace_export_resources(
&db,
params.yaak_version,
params.workspace_ids,
params.include_private_environments,
)?;
Ok(serde_json::to_string_pretty(&export_data)?)
}
+60 -14
View File
@@ -10,10 +10,12 @@ use yaak_database::SqlitePool;
pub mod blob_manager;
pub mod client_db;
mod connection_or_tx;
pub mod content;
pub mod cookies;
mod connection_or_tx;
pub mod error;
pub mod example;
pub mod export;
pub mod migrate;
pub mod models;
pub mod models_ops;
@@ -66,6 +68,17 @@ mod open {
.map_err(|e| Error::Database(e.to_string()))
}
/// `(readers, writer)` over one file: a pool of `max_size` readers and a
/// pool of exactly one writer.
pub fn file_pools(
path: impl Into<PathBuf>,
max_size: u32,
min_idle: u32,
) -> Result<(SqlitePool, SqlitePool)> {
let path: PathBuf = path.into();
Ok((file_pool(&path, max_size, min_idle)?, file_pool(&path, 1, 1)?))
}
pub fn memory_pool() -> Result<SqlitePool> {
let manager = SqliteConnectionManager::memory().with_init(|c| init_connection(c));
// In-memory DB doesn't support multiple connections
@@ -92,6 +105,16 @@ mod open {
Ok(SqlitePool::single(conn))
}
/// One connection is all a browser VFS allows, so it reads and writes.
pub fn file_pools(
path: impl Into<PathBuf>,
max_size: u32,
min_idle: u32,
) -> Result<(SqlitePool, SqlitePool)> {
let pool = file_pool(path, max_size, min_idle)?;
Ok((pool.clone(), pool))
}
pub fn memory_pool() -> Result<SqlitePool> {
let conn = Connection::open_in_memory()?;
init_connection(&conn)?;
@@ -110,27 +133,40 @@ pub fn init_standalone(
let db_path = db_path.as_ref();
let blob_path = blob_path.as_ref();
// Main database pool. Sized for concurrent in-flight queries, not concurrent app
// features — connections are held per-statement, so even heavy fan-out (e.g. many
// gRPC streams) only needs a handful at once. Keep max_size modest: WAL connections
// hold ~3 file descriptors each, and macOS GUI apps get a 256 fd soft limit.
// Each database gets a reader pool and a one-connection writer pool; see
// `QueryManager` for why. Reader pools are sized for concurrent in-flight
// queries, not concurrent app features — connections are held per-statement,
// so even heavy fan-out (e.g. many gRPC streams) only needs a handful at once.
// Keep them modest: WAL connections hold ~3 file descriptors each, and macOS
// GUI apps get a 256 fd soft limit.
info!("Initializing app database {db_path:?}");
let pool = open::file_pool(db_path, 20, 2)?;
migrate_db(&pool)?;
let (readers, writer) = open::file_pools(db_path, 20, 2)?;
migrate_db(&writer)?;
info!("Initializing blobs database {blob_path:?}");
let blob_pool = open::file_pool(blob_path, 10, 1)?;
migrate_blob_db(&blob_pool)?;
let blob_manager = init_blob_store(blob_path)?;
let (tx, rx) = mpsc::channel();
let query_manager = QueryManager::new(pool, tx);
let blob_manager = BlobManager::new(blob_pool);
let query_manager = QueryManager::new(readers, writer, tx);
bootstrap(&query_manager)?;
Ok((query_manager, blob_manager, rx))
}
/// Open (creating and migrating as needed) a blobs database on its own, for a process that
/// stores bodies without the app database.
pub fn init_blob_store(blob_path: impl AsRef<Path>) -> Result<BlobManager> {
let blob_path = blob_path.as_ref();
info!("Initializing blobs database {blob_path:?}");
let (blob_readers, blob_writer) = open::file_pools(blob_path, 10, 1)?;
migrate_blob_db(&blob_writer)?;
Ok(BlobManager::new(blob_readers, blob_writer))
}
/// Initialize the database managers with in-memory SQLite databases.
/// Useful for testing and CI environments.
///
/// An in-memory database is private to its connection, so the one connection
/// is both the reader pool and the writer.
pub fn init_in_memory() -> Result<(QueryManager, BlobManager, mpsc::Receiver<ModelPayload>)> {
let pool = open::memory_pool()?;
migrate_db(&pool)?;
@@ -139,8 +175,18 @@ pub fn init_in_memory() -> Result<(QueryManager, BlobManager, mpsc::Receiver<Mod
migrate_blob_db(&blob_pool)?;
let (tx, rx) = mpsc::channel();
let query_manager = QueryManager::new(pool, tx);
let blob_manager = BlobManager::new(blob_pool);
let query_manager = QueryManager::new(pool.clone(), pool, tx);
let blob_manager = BlobManager::new(blob_pool.clone(), blob_pool);
bootstrap(&query_manager)?;
Ok((query_manager, blob_manager, rx))
}
/// The rows every client assumes exist. Workspaces are not among them: a fresh
/// install has none, and the client shows onboarding until the user makes one.
fn bootstrap(query_manager: &QueryManager) -> Result<()> {
query_manager.with_tx(|tx| {
tx.ensure_settings()?;
Ok(())
})
}
+12 -4
View File
@@ -377,6 +377,7 @@ pub struct Settings {
pub update_channel: String,
pub hide_license_badge: bool,
pub prompt_feedback: bool,
pub mcp_server_enabled: bool,
pub autoupdate: bool,
pub auto_download_updates: bool,
pub check_notifications: bool,
@@ -435,6 +436,7 @@ impl UpsertModelInfo for Settings {
(UpdateChannel, self.update_channel.into()),
(HideLicenseBadge, self.hide_license_badge.into()),
(PromptFeedback, self.prompt_feedback.into()),
(McpServerEnabled, self.mcp_server_enabled.into()),
(Autoupdate, self.autoupdate.into()),
(AutoDownloadUpdates, self.auto_download_updates.into()),
(ColoredMethods, self.colored_methods.into()),
@@ -465,6 +467,7 @@ impl UpsertModelInfo for Settings {
SettingsIden::UpdateChannel,
SettingsIden::HideLicenseBadge,
SettingsIden::PromptFeedback,
SettingsIden::McpServerEnabled,
SettingsIden::Autoupdate,
SettingsIden::AutoDownloadUpdates,
SettingsIden::ColoredMethods,
@@ -506,6 +509,7 @@ impl UpsertModelInfo for Settings {
auto_download_updates: row.get("auto_download_updates")?,
hide_license_badge: row.get("hide_license_badge")?,
prompt_feedback: row.get("prompt_feedback")?,
mcp_server_enabled: row.get("mcp_server_enabled")?,
colored_methods: row.get("colored_methods")?,
check_notifications: row.get("check_notifications")?,
hotkeys: serde_json::from_str(&hotkeys).unwrap_or_default(),
@@ -2850,7 +2854,7 @@ impl UpsertModelInfo for Plugin {
}
}
#[derive(Debug, Clone, Serialize, Deserialize, Default, TS)]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default, TS)]
#[serde(default, rename_all = "camelCase")]
#[ts(export, export_to = "gen_models.ts")]
#[enum_def(table_name = "sync_states")]
@@ -3133,8 +3137,12 @@ pub struct ImportSourceResource {
pub import_source_id: String,
pub source_key: String,
pub model_type: String,
pub model_id: String,
pub snapshot: String,
/// `None` once the user has decided not to import this key
#[ts(optional)]
pub model_id: Option<String>,
/// Hash of the resource as last applied or decided from the source, if one was recorded
#[ts(optional)]
pub content_hash: Option<String>,
}
impl<'s> TryFrom<&Row<'s>> for ImportSourceResource {
@@ -3149,7 +3157,7 @@ impl<'s> TryFrom<&Row<'s>> for ImportSourceResource {
source_key: r.get("source_key")?,
model_type: r.get("model_type")?,
model_id: r.get("model_id")?,
snapshot: r.get("snapshot")?,
content_hash: r.get("content_hash")?,
})
}
}
+4 -4
View File
@@ -8,14 +8,14 @@
//! caller is a desktop window or an HTTP request.
use crate::blob_manager::BlobManager;
use crate::client_db::ClientDb;
use crate::client_db::WriteDb;
use crate::error::Error::GenericError;
use crate::error::Result;
use crate::models::AnyModel;
use crate::util::UpdateSource;
pub fn upsert_model(
db: &ClientDb,
db: &WriteDb,
blobs: &BlobManager,
model: AnyModel,
source: &UpdateSource,
@@ -41,7 +41,7 @@ pub fn upsert_model(
/// Deletes cascade, so callers run this inside a transaction.
pub fn delete_model(
tx: &ClientDb,
tx: &WriteDb,
blobs: &BlobManager,
model: AnyModel,
source: &UpdateSource,
@@ -69,7 +69,7 @@ pub fn delete_model(
/// The model is re-read from the database rather than taken from the caller, so
/// a duplicate never comes from a stale frontend snapshot.
pub fn duplicate_model(
tx: &ClientDb,
tx: &WriteDb,
model_type: &str,
model_id: &str,
source: &UpdateSource,
+2 -2
View File
@@ -1,10 +1,10 @@
use crate::client_db::ClientDb;
use crate::client_db::WriteDb;
use crate::error::Result;
use crate::models::{Environment, Folder, GrpcRequest, HttpRequest, WebsocketRequest, Workspace};
use crate::util::{BatchUpsertResult, UpdateSource};
use log::info;
impl<'a> ClientDb<'a> {
impl<'a> WriteDb<'a> {
pub fn batch_upsert(
&self,
workspaces: Vec<Workspace>,
+10 -6
View File
@@ -1,4 +1,4 @@
use crate::client_db::ClientDb;
use crate::client_db::{ClientDb, WriteDb};
use crate::error::Result;
use crate::models::{CookieJar, CookieJarIden};
use crate::util::UpdateSource;
@@ -9,18 +9,22 @@ impl<'a> ClientDb<'a> {
}
pub fn list_cookie_jars(&self, workspace_id: &str) -> Result<Vec<CookieJar>> {
let mut cookie_jars = self.find_many(CookieJarIden::WorkspaceId, workspace_id, None)?;
self.find_many(CookieJarIden::WorkspaceId, workspace_id, None)
}
}
if cookie_jars.is_empty() {
impl<'a> WriteDb<'a> {
/// A workspace with no cookie jar gets a default one.
pub fn ensure_default_cookie_jar(&self, workspace_id: &str) -> Result<()> {
if self.list_cookie_jars(workspace_id)?.is_empty() {
let jar = CookieJar {
name: "Default".to_string(),
workspace_id: workspace_id.to_string(),
..Default::default()
};
cookie_jars.push(self.upsert_cookie_jar(&jar, &UpdateSource::Background)?);
self.upsert_cookie_jar(&jar, &UpdateSource::Background)?;
}
Ok(cookie_jars)
Ok(())
}
pub fn delete_cookie_jar(
+241 -75
View File
@@ -1,5 +1,5 @@
use super::conflict_free_name;
use crate::client_db::ClientDb;
use crate::client_db::{ClientDb, WriteDb};
use crate::error::Error::{MissingBaseEnvironment, MultipleBaseEnvironments};
use crate::error::Result;
use crate::models::{Environment, EnvironmentIden, EnvironmentVariable};
@@ -11,6 +11,27 @@ impl<'a> ClientDb<'a> {
self.find_one(EnvironmentIden::Id, id)
}
/// Resolve an explicit execution environment, rejecting foreign and folder environments.
/// Folder variables are inherited from the request's folder, never selected globally.
pub fn get_environment_for_workspace(
&self,
workspace_id: &str,
environment_id: &str,
) -> Result<Environment> {
let environment = self.get_environment(environment_id)?;
if environment.workspace_id != workspace_id {
return Err(crate::error::Error::InvalidEnvironment(format!(
"Environment {environment_id} does not belong to workspace {workspace_id}"
)));
}
if !matches!(environment.parent_model.as_str(), "workspace" | "environment") {
return Err(crate::error::Error::InvalidEnvironment(format!(
"Environment {environment_id} is not a workspace environment"
)));
}
Ok(environment)
}
pub fn get_environment_by_folder_id(&self, folder_id: &str) -> Result<Option<Environment>> {
let mut environments: Vec<Environment> =
self.find_many(EnvironmentIden::ParentId, folder_id, None)?;
@@ -20,7 +41,7 @@ impl<'a> ClientDb<'a> {
}
pub fn get_base_environment(&self, workspace_id: &str) -> Result<Environment> {
let environments = self.list_environments_ensure_base(workspace_id)?;
let environments = self.list_environments(workspace_id)?;
let base_environments = environments
.into_iter()
.filter(|e| e.parent_model == "workspace")
@@ -30,38 +51,96 @@ impl<'a> ClientDb<'a> {
return Err(MultipleBaseEnvironments(workspace_id.to_string()));
}
Ok(base_environments.first().cloned().ok_or(
// Should never happen because one should be created above if it does not exist
MissingBaseEnvironment(workspace_id.to_string()),
)?)
Ok(base_environments
.first()
.cloned()
.ok_or(MissingBaseEnvironment(workspace_id.to_string()))?)
}
/// Lists environments and will create a base environment if one doesn't exist
pub fn list_environments_ensure_base(&self, workspace_id: &str) -> Result<Vec<Environment>> {
let mut environments = self.list_environments_dangerous(workspace_id)?;
pub fn list_environments(&self, workspace_id: &str) -> Result<Vec<Environment>> {
Ok(self.find_many::<Environment>(EnvironmentIden::WorkspaceId, workspace_id, None)?)
}
let base_environment = environments.iter().find(|e| e.parent_model == "workspace");
/// Find other environments with the same parent folder
fn list_duplicate_folder_environments(&self, environment: &Environment) -> Vec<Environment> {
if environment.parent_model != "folder" {
return Vec::new();
}
if let None = base_environment {
let e = self.upsert_environment(
&Environment {
workspace_id: workspace_id.to_string(),
name: "Global Variables".to_string(),
parent_model: "workspace".to_string(),
..Default::default()
},
&UpdateSource::Background,
self.list_environments(&environment.workspace_id)
.unwrap_or_default()
.into_iter()
.filter(|e| {
e.id != environment.id
&& e.parent_model == "folder"
&& e.parent_id == environment.parent_id
})
.collect()
}
pub fn resolve_environments(
&self,
workspace_id: &str,
folder_id: Option<&str>,
active_environment_id: Option<&str>,
) -> Result<Vec<Environment>> {
let mut environments = Vec::new();
if let Some(folder_id) = folder_id {
let folder = self.get_folder(folder_id)?;
// Add current folder's environment
if let Some(e) = self.get_environment_by_folder_id(folder_id)? {
environments.push(e);
};
// Recurse up
let ancestors = self.resolve_environments(
workspace_id,
folder.folder_id.as_deref(),
active_environment_id,
)?;
info!("Created base environment {} for {workspace_id}", e.id);
environments.push(e);
environments.extend(ancestors);
} else {
// Add active and base environments
if let Some(id) = active_environment_id {
if let Ok(e) = self.get_environment(&id) {
// Add active sub environment
environments.push(e);
};
};
// Add the base environment. A workspace that has never been
// opened has none yet; it simply contributes no variables.
match self.get_base_environment(workspace_id) {
Ok(e) => environments.push(e),
Err(MissingBaseEnvironment(_)) => {}
Err(e) => return Err(e),
}
}
Ok(environments)
}
}
/// List environments for a workspace. Prefer list_environments_ensure_base()
fn list_environments_dangerous(&self, workspace_id: &str) -> Result<Vec<Environment>> {
Ok(self.find_many::<Environment>(EnvironmentIden::WorkspaceId, workspace_id, None)?)
impl<'a> WriteDb<'a> {
/// The workspace's base environment, created if it does not exist.
pub fn ensure_base_environment(&self, workspace_id: &str) -> Result<Environment> {
match self.get_base_environment(workspace_id) {
Err(MissingBaseEnvironment(_)) => {}
other => return other,
}
let e = self.upsert_environment(
&Environment {
workspace_id: workspace_id.to_string(),
name: "Global Variables".to_string(),
parent_model: "workspace".to_string(),
..Default::default()
},
&UpdateSource::Background,
)?;
info!("Created base environment {} for {workspace_id}", e.id);
Ok(e)
}
pub fn delete_environment(
@@ -72,7 +151,7 @@ impl<'a> ClientDb<'a> {
let deleted_environment = self.delete(environment, source)?;
// Recreate the base environment if we happened to delete it
self.list_environments_ensure_base(&environment.workspace_id)?;
self.ensure_base_environment(&environment.workspace_id)?;
Ok(deleted_environment)
}
@@ -90,7 +169,7 @@ impl<'a> ClientDb<'a> {
let mut environment = environment.clone();
environment.id = "".to_string();
let sibling_names = self
.list_environments_dangerous(&environment.workspace_id)?
.list_environments(&environment.workspace_id)?
.into_iter()
.map(|e| e.name)
.collect::<Vec<_>>();
@@ -98,23 +177,6 @@ impl<'a> ClientDb<'a> {
self.upsert_environment(&environment, source)
}
/// Find other environments with the same parent folder
fn list_duplicate_folder_environments(&self, environment: &Environment) -> Vec<Environment> {
if environment.parent_model != "folder" {
return Vec::new();
}
self.list_environments_dangerous(&environment.workspace_id)
.unwrap_or_default()
.into_iter()
.filter(|e| {
e.id != environment.id
&& e.parent_model == "folder"
&& e.parent_id == environment.parent_id
})
.collect()
}
pub fn upsert_environment(
&self,
environment: &Environment,
@@ -154,43 +216,147 @@ impl<'a> ClientDb<'a> {
source,
)
}
}
pub fn resolve_environments(
&self,
workspace_id: &str,
folder_id: Option<&str>,
active_environment_id: Option<&str>,
) -> Result<Vec<Environment>> {
let mut environments = Vec::new();
#[cfg(test)]
mod tests {
use crate::error::Error;
use crate::init_in_memory;
use crate::models::{Environment, EnvironmentVariable, Folder, Workspace};
use crate::query_manager::QueryManager;
use crate::render::make_vars_hashmap;
use crate::util::UpdateSource;
if let Some(folder_id) = folder_id {
let folder = self.get_folder(folder_id)?;
// Add current folder's environment
if let Some(e) = self.get_environment_by_folder_id(folder_id)? {
environments.push(e);
};
// Recurse up
let ancestors = self.resolve_environments(
workspace_id,
folder.folder_id.as_deref(),
active_environment_id,
)?;
environments.extend(ancestors);
} else {
// Add active and base environments
if let Some(id) = active_environment_id {
if let Ok(e) = self.get_environment(&id) {
// Add active sub environment
environments.push(e);
fn fixture() -> (QueryManager, Environment, Environment, Environment) {
let (manager, _blobs, _rx) = init_in_memory().unwrap();
let source = &UpdateSource::Background;
let (base, staging, folder) = manager
.with_tx(|db| {
// Fresh databases no longer come with a workspace, so make one
let workspace = db.upsert_workspace(
&Workspace { name: "Test".into(), ..Default::default() },
source,
)?;
let variable = |name: &str, value: &str| EnvironmentVariable {
enabled: true,
name: name.into(),
value: value.into(),
..Default::default()
};
};
let base = db.ensure_base_environment(&workspace.id)?;
let base = db.upsert_environment(
&Environment {
variables: vec![
variable("marker", "global"),
variable("global_only", "inherited"),
],
..base
},
source,
)?;
let staging = db.upsert_environment(
&Environment {
workspace_id: workspace.id.clone(),
parent_model: "environment".into(),
parent_id: Some(base.id.clone()),
name: "Staging".into(),
variables: vec![variable("marker", "staging")],
..Default::default()
},
source,
)?;
let folder = db.upsert_folder(
&Folder {
workspace_id: workspace.id.clone(),
name: "Folder".into(),
..Default::default()
},
source,
)?;
let folder = db.upsert_environment(
&Environment {
workspace_id: workspace.id,
parent_model: "folder".into(),
parent_id: Some(folder.id),
variables: vec![variable("marker", "folder")],
..Default::default()
},
source,
)?;
Ok::<_, Error>((base, staging, folder))
})
.unwrap();
(manager, base, staging, folder)
}
// Add the base environment
environments.push(self.get_base_environment(workspace_id)?);
#[test]
fn explicit_environment_preserves_global_and_folder_inheritance() {
let (manager, base, staging, folder) = fixture();
let db = manager.connect();
let selected = db.get_environment_for_workspace(&base.workspace_id, &staging.id).unwrap();
let vars = make_vars_hashmap(
db.resolve_environments(&base.workspace_id, None, Some(&selected.id)).unwrap(),
);
assert_eq!(vars["marker"], "staging");
assert_eq!(vars["global_only"], "inherited");
let vars = make_vars_hashmap(
db.resolve_environments(
&base.workspace_id,
folder.parent_id.as_deref(),
Some(&selected.id),
)
.unwrap(),
);
assert_eq!(vars["marker"], "folder");
assert_eq!(vars["global_only"], "inherited");
}
#[test]
fn base_environment_can_be_selected_explicitly() {
let (manager, base, _staging, _folder) = fixture();
let db = manager.connect();
let selected = db.get_environment_for_workspace(&base.workspace_id, &base.id).unwrap();
let vars = make_vars_hashmap(
db.resolve_environments(&base.workspace_id, None, Some(&selected.id)).unwrap(),
);
assert_eq!(vars["marker"], "global");
}
#[test]
fn explicit_environment_rejects_empty_and_missing_ids() {
let (manager, base, _staging, _folder) = fixture();
for id in ["", "ev_missing"] {
assert!(matches!(
manager.connect().get_environment_for_workspace(&base.workspace_id, id),
Err(Error::ModelNotFound(_))
));
}
}
Ok(environments)
#[test]
fn explicit_environment_rejects_another_workspace() {
let (manager, _base, staging, _folder) = fixture();
let other = manager
.with_tx(|db| {
db.upsert_workspace(
&Workspace { name: "Other".into(), ..Default::default() },
&UpdateSource::Background,
)
})
.unwrap();
assert!(matches!(
manager.connect().get_environment_for_workspace(&other.id, &staging.id),
Err(Error::InvalidEnvironment(_))
));
}
#[test]
fn explicit_environment_rejects_folder_variables() {
let (manager, base, _staging, folder) = fixture();
assert!(matches!(
manager.connect().get_environment_for_workspace(&base.workspace_id, &folder.id),
Err(Error::InvalidEnvironment(_))
));
}
}
+96 -94
View File
@@ -1,5 +1,5 @@
use super::{conflict_free_name, merge_headers};
use crate::client_db::ClientDb;
use crate::client_db::{ClientDb, WriteDb};
use crate::connection_or_tx::ConnectionOrTx;
use crate::error::Result;
use crate::models::{
@@ -20,99 +20,6 @@ impl<'a> ClientDb<'a> {
self.find_many(FolderIden::WorkspaceId, workspace_id, None)
}
pub fn delete_folder(&self, folder: &Folder, source: &UpdateSource) -> Result<Folder> {
match self.conn() {
ConnectionOrTx::Connection(_) => {}
ConnectionOrTx::Transaction(_) => {}
}
let fid = &folder.id;
for m in self.find_many::<HttpRequest>(HttpRequestIden::FolderId, fid, None)? {
self.delete_http_request(&m, source)?;
}
for m in self.find_many::<GrpcRequest>(GrpcRequestIden::FolderId, fid, None)? {
self.delete_grpc_request(&m, source)?;
}
for m in self.find_many::<WebsocketRequest>(WebsocketRequestIden::FolderId, fid, None)? {
self.delete_websocket_request(&m, source)?;
}
for e in self.find_many(EnvironmentIden::ParentId, fid, None)? {
self.delete_environment(&e, source)?;
}
// Recurse down into child folders
for folder in self.find_many::<Folder>(FolderIden::FolderId, fid, None)? {
self.delete_folder(&folder, source)?;
}
self.delete(folder, source)
}
pub fn delete_folder_by_id(&self, id: &str, source: &UpdateSource) -> Result<Folder> {
let folder = self.get_folder(id)?;
self.delete_folder(&folder, source)
}
pub fn upsert_folder(&self, folder: &Folder, source: &UpdateSource) -> Result<Folder> {
self.upsert(folder, source)
}
pub fn duplicate_folder(&self, src_folder: &Folder, source: &UpdateSource) -> Result<Folder> {
let fid = &src_folder.id;
let mut folder = Folder {
id: "".into(),
sort_priority: src_folder.sort_priority + 0.001,
..src_folder.clone()
};
let sibling_names = self
.list_folders(&folder.workspace_id)?
.into_iter()
.filter(|f| f.folder_id == folder.folder_id)
.map(|f| f.name)
.collect::<Vec<_>>();
folder.name = conflict_free_name(&folder.name, &sibling_names);
let new_folder = self.upsert_folder(&folder, source)?;
for m in self.find_many::<HttpRequest>(HttpRequestIden::FolderId, fid, None)? {
self.upsert_http_request(
&HttpRequest { id: "".into(), folder_id: Some(new_folder.id.clone()), ..m },
source,
)?;
}
for m in self.find_many::<WebsocketRequest>(WebsocketRequestIden::FolderId, fid, None)? {
self.upsert_websocket_request(
&WebsocketRequest { id: "".into(), folder_id: Some(new_folder.id.clone()), ..m },
source,
)?;
}
for m in self.find_many::<GrpcRequest>(GrpcRequestIden::FolderId, fid, None)? {
self.upsert_grpc_request(
&GrpcRequest { id: "".into(), folder_id: Some(new_folder.id.clone()), ..m },
source,
)?;
}
for m in self.find_many::<Environment>(EnvironmentIden::ParentId, fid, None)? {
self.upsert_environment(
&Environment { id: "".into(), parent_id: Some(new_folder.id.clone()), ..m },
source,
)?;
}
for m in self.find_many::<Folder>(FolderIden::FolderId, fid, None)? {
// Recurse down
self.duplicate_folder(&Folder { folder_id: Some(new_folder.id.clone()), ..m }, source)?;
}
Ok(new_folder)
}
pub fn resolve_auth_for_folder(
&self,
folder: &Folder,
@@ -219,3 +126,98 @@ impl<'a> ClientDb<'a> {
})
}
}
impl<'a> WriteDb<'a> {
pub fn delete_folder(&self, folder: &Folder, source: &UpdateSource) -> Result<Folder> {
match self.conn() {
ConnectionOrTx::Connection(_) => {}
ConnectionOrTx::Transaction(_) => {}
}
let fid = &folder.id;
for m in self.find_many::<HttpRequest>(HttpRequestIden::FolderId, fid, None)? {
self.delete_http_request(&m, source)?;
}
for m in self.find_many::<GrpcRequest>(GrpcRequestIden::FolderId, fid, None)? {
self.delete_grpc_request(&m, source)?;
}
for m in self.find_many::<WebsocketRequest>(WebsocketRequestIden::FolderId, fid, None)? {
self.delete_websocket_request(&m, source)?;
}
for e in self.find_many(EnvironmentIden::ParentId, fid, None)? {
self.delete_environment(&e, source)?;
}
// Recurse down into child folders
for folder in self.find_many::<Folder>(FolderIden::FolderId, fid, None)? {
self.delete_folder(&folder, source)?;
}
self.delete(folder, source)
}
pub fn delete_folder_by_id(&self, id: &str, source: &UpdateSource) -> Result<Folder> {
let folder = self.get_folder(id)?;
self.delete_folder(&folder, source)
}
pub fn upsert_folder(&self, folder: &Folder, source: &UpdateSource) -> Result<Folder> {
self.upsert(folder, source)
}
pub fn duplicate_folder(&self, src_folder: &Folder, source: &UpdateSource) -> Result<Folder> {
let fid = &src_folder.id;
let mut folder = Folder {
id: "".into(),
sort_priority: src_folder.sort_priority + 0.001,
..src_folder.clone()
};
let sibling_names = self
.list_folders(&folder.workspace_id)?
.into_iter()
.filter(|f| f.folder_id == folder.folder_id)
.map(|f| f.name)
.collect::<Vec<_>>();
folder.name = conflict_free_name(&folder.name, &sibling_names);
let new_folder = self.upsert_folder(&folder, source)?;
for m in self.find_many::<HttpRequest>(HttpRequestIden::FolderId, fid, None)? {
self.upsert_http_request(
&HttpRequest { id: "".into(), folder_id: Some(new_folder.id.clone()), ..m },
source,
)?;
}
for m in self.find_many::<WebsocketRequest>(WebsocketRequestIden::FolderId, fid, None)? {
self.upsert_websocket_request(
&WebsocketRequest { id: "".into(), folder_id: Some(new_folder.id.clone()), ..m },
source,
)?;
}
for m in self.find_many::<GrpcRequest>(GrpcRequestIden::FolderId, fid, None)? {
self.upsert_grpc_request(
&GrpcRequest { id: "".into(), folder_id: Some(new_folder.id.clone()), ..m },
source,
)?;
}
for m in self.find_many::<Environment>(EnvironmentIden::ParentId, fid, None)? {
self.upsert_environment(
&Environment { id: "".into(), parent_id: Some(new_folder.id.clone()), ..m },
source,
)?;
}
for m in self.find_many::<Folder>(FolderIden::FolderId, fid, None)? {
// Recurse down
self.duplicate_folder(&Folder { folder_id: Some(new_folder.id.clone()), ..m }, source)?;
}
Ok(new_folder)
}
}
@@ -1,4 +1,4 @@
use crate::client_db::ClientDb;
use crate::client_db::{ClientDb, WriteDb};
use crate::error::Result;
use crate::models::{GraphQlIntrospection, GraphQlIntrospectionIden};
use crate::util::UpdateSource;
@@ -11,7 +11,9 @@ impl<'a> ClientDb<'a> {
pub fn get_graphql_introspection(&self, request_id: &str) -> Option<GraphQlIntrospection> {
self.find_optional(GraphQlIntrospectionIden::RequestId, request_id)
}
}
impl<'a> WriteDb<'a> {
pub fn upsert_graphql_introspection(
&self,
workspace_id: &str,
@@ -1,4 +1,4 @@
use crate::client_db::ClientDb;
use crate::client_db::{ClientDb, WriteDb};
use crate::error::Result;
use crate::models::{GrpcConnection, GrpcConnectionIden, GrpcConnectionState};
use crate::queries::MAX_HISTORY_ITEMS;
@@ -13,6 +13,20 @@ impl<'a> ClientDb<'a> {
self.find_one(GrpcConnectionIden::Id, id)
}
pub fn list_grpc_connections_for_request(
&self,
request_id: &str,
limit: Option<u64>,
) -> Result<Vec<GrpcConnection>> {
self.find_many(GrpcConnectionIden::RequestId, request_id, limit)
}
pub fn list_grpc_connections(&self, workspace_id: &str) -> Result<Vec<GrpcConnection>> {
self.find_many(GrpcConnectionIden::WorkspaceId, workspace_id, None)
}
}
impl<'a> WriteDb<'a> {
pub fn delete_all_grpc_connections_for_request(
&self,
request_id: &str,
@@ -53,18 +67,6 @@ impl<'a> ClientDb<'a> {
self.delete_grpc_connection(&grpc_connection, source)
}
pub fn list_grpc_connections_for_request(
&self,
request_id: &str,
limit: Option<u64>,
) -> Result<Vec<GrpcConnection>> {
self.find_many(GrpcConnectionIden::RequestId, request_id, limit)
}
pub fn list_grpc_connections(&self, workspace_id: &str) -> Result<Vec<GrpcConnection>> {
self.find_many(GrpcConnectionIden::WorkspaceId, workspace_id, None)
}
pub fn cancel_pending_grpc_connections(&self) -> Result<()> {
let closed = serde_json::to_value(&GrpcConnectionState::Closed)?;
let (sql, params) = Query::update()
@@ -1,4 +1,4 @@
use crate::client_db::ClientDb;
use crate::client_db::{ClientDb, WriteDb};
use crate::error::Result;
use crate::models::{GrpcEvent, GrpcEventIden};
use crate::util::UpdateSource;
@@ -11,7 +11,9 @@ impl<'a> ClientDb<'a> {
pub fn list_grpc_events(&self, connection_id: &str) -> Result<Vec<GrpcEvent>> {
self.find_many(GrpcEventIden::ConnectionId, connection_id, None)
}
}
impl<'a> WriteDb<'a> {
pub fn upsert_grpc_event(
&self,
grpc_event: &GrpcEvent,
+48 -46
View File
@@ -1,5 +1,5 @@
use super::{conflict_free_name, merge_headers};
use crate::client_db::ClientDb;
use crate::client_db::{ClientDb, WriteDb};
use crate::error::Result;
use crate::models::{
AnyModel, Folder, FolderIden, GrpcRequest, GrpcRequestIden, HttpRequestHeader,
@@ -32,51 +32,6 @@ impl<'a> ClientDb<'a> {
Ok(children)
}
pub fn delete_grpc_request(
&self,
m: &GrpcRequest,
source: &UpdateSource,
) -> Result<GrpcRequest> {
self.delete_all_grpc_connections_for_request(m.id.as_str(), source)?;
self.delete_model_versions_for_model(m.id.as_str())?;
self.delete(m, source)
}
pub fn delete_grpc_request_by_id(
&self,
id: &str,
source: &UpdateSource,
) -> Result<GrpcRequest> {
let request = self.get_grpc_request(id)?;
self.delete_grpc_request(&request, source)
}
pub fn duplicate_grpc_request(
&self,
grpc_request: &GrpcRequest,
source: &UpdateSource,
) -> Result<GrpcRequest> {
let mut request = grpc_request.clone();
request.id = "".to_string();
request.sort_priority = request.sort_priority + 0.001;
let sibling_names = self
.list_grpc_requests(&request.workspace_id)?
.into_iter()
.filter(|m| m.folder_id == request.folder_id)
.map(|m| m.name)
.collect::<Vec<_>>();
request.name = conflict_free_name(&request.name, &sibling_names);
self.upsert(&request, source)
}
pub fn upsert_grpc_request(
&self,
grpc_request: &GrpcRequest,
source: &UpdateSource,
) -> Result<GrpcRequest> {
self.upsert(grpc_request, source)
}
pub fn resolve_auth_for_grpc_request(
&self,
grpc_request: &GrpcRequest,
@@ -147,3 +102,50 @@ impl<'a> ClientDb<'a> {
})
}
}
impl<'a> WriteDb<'a> {
pub fn delete_grpc_request(
&self,
m: &GrpcRequest,
source: &UpdateSource,
) -> Result<GrpcRequest> {
self.delete_all_grpc_connections_for_request(m.id.as_str(), source)?;
self.delete_model_versions_for_model(m.id.as_str())?;
self.delete(m, source)
}
pub fn delete_grpc_request_by_id(
&self,
id: &str,
source: &UpdateSource,
) -> Result<GrpcRequest> {
let request = self.get_grpc_request(id)?;
self.delete_grpc_request(&request, source)
}
pub fn duplicate_grpc_request(
&self,
grpc_request: &GrpcRequest,
source: &UpdateSource,
) -> Result<GrpcRequest> {
let mut request = grpc_request.clone();
request.id = "".to_string();
request.sort_priority = request.sort_priority + 0.001;
let sibling_names = self
.list_grpc_requests(&request.workspace_id)?
.into_iter()
.filter(|m| m.folder_id == request.folder_id)
.map(|m| m.name)
.collect::<Vec<_>>();
request.name = conflict_free_name(&request.name, &sibling_names);
self.upsert(&request, source)
}
pub fn upsert_grpc_request(
&self,
grpc_request: &GrpcRequest,
source: &UpdateSource,
) -> Result<GrpcRequest> {
self.upsert(grpc_request, source)
}
}
+115 -100
View File
@@ -1,5 +1,5 @@
use super::{conflict_free_name, merge_headers};
use crate::client_db::ClientDb;
use crate::client_db::{ClientDb, WriteDb};
use crate::error::Result;
use crate::models::{
AnyModel, Folder, FolderIden, HttpRequest, HttpRequestHeader, HttpRequestIden,
@@ -18,51 +18,6 @@ impl<'a> ClientDb<'a> {
self.find_many(HttpRequestIden::WorkspaceId, workspace_id, None)
}
pub fn delete_http_request(
&self,
m: &HttpRequest,
source: &UpdateSource,
) -> Result<HttpRequest> {
self.delete_all_http_responses_for_request(m.id.as_str(), source)?;
self.delete_model_versions_for_model(m.id.as_str())?;
self.delete(m, source)
}
pub fn delete_http_request_by_id(
&self,
id: &str,
source: &UpdateSource,
) -> Result<HttpRequest> {
let http_request = self.get_http_request(id)?;
self.delete_http_request(&http_request, source)
}
pub fn duplicate_http_request(
&self,
http_request: &HttpRequest,
source: &UpdateSource,
) -> Result<HttpRequest> {
let mut http_request = http_request.clone();
http_request.id = "".to_string();
http_request.sort_priority = http_request.sort_priority + 0.001;
let sibling_names = self
.list_http_requests(&http_request.workspace_id)?
.into_iter()
.filter(|m| m.folder_id == http_request.folder_id)
.map(|m| m.name)
.collect::<Vec<_>>();
http_request.name = conflict_free_name(&http_request.name, &sibling_names);
self.upsert(&http_request, source)
}
pub fn upsert_http_request(
&self,
http_request: &HttpRequest,
source: &UpdateSource,
) -> Result<HttpRequest> {
self.upsert(http_request, source)
}
pub fn resolve_auth_for_http_request(
&self,
http_request: &HttpRequest,
@@ -180,6 +135,53 @@ impl<'a> ClientDb<'a> {
}
}
impl<'a> WriteDb<'a> {
pub fn delete_http_request(
&self,
m: &HttpRequest,
source: &UpdateSource,
) -> Result<HttpRequest> {
self.delete_all_http_responses_for_request(m.id.as_str(), source)?;
self.delete_model_versions_for_model(m.id.as_str())?;
self.delete(m, source)
}
pub fn delete_http_request_by_id(
&self,
id: &str,
source: &UpdateSource,
) -> Result<HttpRequest> {
let http_request = self.get_http_request(id)?;
self.delete_http_request(&http_request, source)
}
pub fn duplicate_http_request(
&self,
http_request: &HttpRequest,
source: &UpdateSource,
) -> Result<HttpRequest> {
let mut http_request = http_request.clone();
http_request.id = "".to_string();
http_request.sort_priority = http_request.sort_priority + 0.001;
let sibling_names = self
.list_http_requests(&http_request.workspace_id)?
.into_iter()
.filter(|m| m.folder_id == http_request.folder_id)
.map(|m| m.name)
.collect::<Vec<_>>();
http_request.name = conflict_free_name(&http_request.name, &sibling_names);
self.upsert(&http_request, source)
}
pub fn upsert_http_request(
&self,
http_request: &HttpRequest,
source: &UpdateSource,
) -> Result<HttpRequest> {
self.upsert(http_request, source)
}
}
#[cfg(test)]
mod tests {
use crate::init_in_memory;
@@ -191,8 +193,10 @@ mod tests {
#[test]
fn request_resolution_preserves_duplicate_request_headers() {
let (query_manager, _blob_manager, _rx) = init_in_memory().expect("Failed to init DB");
let workspace = query_manager
.with_tx(|tx| tx.upsert_workspace(&Workspace::default(), &UpdateSource::Background))
.expect("Failed to create workspace");
let db = query_manager.connect();
let workspace = db.list_workspaces().expect("Failed to list workspaces").remove(0);
let request = HttpRequest {
workspace_id: workspace.id,
headers: vec![
@@ -226,72 +230,83 @@ mod tests {
#[test]
fn http_version_resolves_through_the_inheritance_chain() {
let (query_manager, _blob_manager, _rx) = init_in_memory().expect("Failed to init DB");
let db = query_manager.connect();
let source = &UpdateSource::Background;
let workspace = db
.upsert_workspace(
&Workspace {
name: "Test".to_string(),
setting_http_version: HttpVersion::Http2,
..Default::default()
},
&UpdateSource::Background,
)
.expect("Failed to upsert workspace");
let folder = db
.upsert_folder(
&Folder { workspace_id: workspace.id.clone(), ..Default::default() },
&UpdateSource::Background,
)
.expect("Failed to upsert folder");
let request = db
.upsert_http_request(
&HttpRequest {
workspace_id: workspace.id.clone(),
folder_id: Some(folder.id.clone()),
..Default::default()
},
&UpdateSource::Background,
)
.expect("Failed to upsert request");
let (folder, request) = query_manager
.with_tx(|db| {
let workspace = db.upsert_workspace(
&Workspace {
name: "Test".to_string(),
setting_http_version: HttpVersion::Http2,
..Default::default()
},
source,
)?;
let folder = db.upsert_folder(
&Folder { workspace_id: workspace.id.clone(), ..Default::default() },
source,
)?;
let request = db.upsert_http_request(
&HttpRequest {
workspace_id: workspace.id.clone(),
folder_id: Some(folder.id.clone()),
..Default::default()
},
source,
)?;
Ok::<_, crate::error::Error>((folder, request))
})
.expect("Failed to seed");
// No overrides, so the workspace base value applies
let resolved = db.resolve_settings_for_http_request(&request).expect("Failed to resolve");
let resolved = query_manager
.connect()
.resolve_settings_for_http_request(&request)
.expect("Failed to resolve");
assert_eq!(resolved.http_version.value, HttpVersion::Http2);
assert_eq!(resolved.http_version.source_model, "workspace");
// A folder override beats the workspace base
db.upsert_folder(
&Folder {
setting_http_version: InheritedHttpVersionSetting {
enabled: true,
value: HttpVersion::Http1,
},
..folder
},
&UpdateSource::Background,
)
.expect("Failed to update folder");
let resolved = db.resolve_settings_for_http_request(&request).expect("Failed to resolve");
query_manager
.with_tx(|db| {
db.upsert_folder(
&Folder {
setting_http_version: InheritedHttpVersionSetting {
enabled: true,
value: HttpVersion::Http1,
},
..folder
},
source,
)
})
.expect("Failed to update folder");
let resolved = query_manager
.connect()
.resolve_settings_for_http_request(&request)
.expect("Failed to resolve");
assert_eq!(resolved.http_version.value, HttpVersion::Http1);
assert_eq!(resolved.http_version.source_model, "folder");
// A request override beats them both
let request = db
.upsert_http_request(
&HttpRequest {
setting_http_version: InheritedHttpVersionSetting {
enabled: true,
value: HttpVersion::Auto,
let request = query_manager
.with_tx(|db| {
db.upsert_http_request(
&HttpRequest {
setting_http_version: InheritedHttpVersionSetting {
enabled: true,
value: HttpVersion::Auto,
},
..request
},
..request
},
&UpdateSource::Background,
)
source,
)
})
.expect("Failed to update request");
let resolved = db.resolve_settings_for_http_request(&request).expect("Failed to resolve");
let resolved = query_manager
.connect()
.resolve_settings_for_http_request(&request)
.expect("Failed to resolve");
assert_eq!(resolved.http_version.value, HttpVersion::Auto);
assert_eq!(resolved.http_version.source_model, "http_request");
}
@@ -1,4 +1,4 @@
use crate::client_db::ClientDb;
use crate::client_db::{ClientDb, WriteDb};
use crate::error::Result;
use crate::models::{HttpResponseEvent, HttpResponseEventIden};
use crate::util::UpdateSource;
@@ -7,7 +7,9 @@ impl<'a> ClientDb<'a> {
pub fn list_http_response_events(&self, response_id: &str) -> Result<Vec<HttpResponseEvent>> {
self.find_many(HttpResponseEventIden::ResponseId, response_id, None)
}
}
impl<'a> WriteDb<'a> {
pub fn upsert_http_response_event(
&self,
http_response_event: &HttpResponseEvent,
@@ -1,5 +1,5 @@
use crate::blob_manager::BlobManager;
use crate::client_db::ClientDb;
use crate::client_db::{ClientDb, WriteDb};
use crate::error::Result;
use crate::models::{HttpResponse, HttpResponseIden, HttpResponseState};
use crate::queries::MAX_HISTORY_ITEMS;
@@ -31,20 +31,6 @@ impl<'a> ClientDb<'a> {
self.find_many(HttpResponseIden::WorkspaceId, workspace_id, limit)
}
/// Returns the number of responses deleted.
pub fn delete_all_http_responses_for_request(
&self,
request_id: &str,
source: &UpdateSource,
) -> Result<usize> {
let responses = self.list_http_responses_for_request(request_id, None)?;
let count = responses.len();
for m in responses {
self.delete(&m, source)?;
}
Ok(count)
}
/// Delete blob-stored response bodies whose owning HTTP response row no
/// longer exists. Blob ids are keyed by the response that owns them —
/// "{response_id}" for a response body, "{response_id}.request" for the
@@ -55,19 +41,24 @@ impl<'a> ClientDb<'a> {
///
/// Returns the number of orphaned bodies deleted.
pub fn delete_orphaned_response_body_blobs(&self, blobs: &BlobManager) -> Result<usize> {
let mut deleted = 0;
let orphaned = blobs
.connect()
.list_body_ids()?
.into_iter()
.filter(|body_id| {
let response_id = body_id.split('.').next().unwrap_or_default();
self.find_optional::<HttpResponse>(HttpResponseIden::Id, response_id).is_none()
})
.collect::<Vec<_>>();
let blob_ctx = blobs.connect();
for body_id in blob_ctx.list_body_ids()? {
let response_id = body_id.split('.').next().unwrap_or_default();
if self.find_optional::<HttpResponse>(HttpResponseIden::Id, response_id).is_some() {
continue;
blobs.with_tx(|b| {
for body_id in &orphaned {
b.delete_chunks(body_id)?;
}
blob_ctx.delete_chunks(&body_id)?;
deleted += 1;
}
Ok::<_, crate::error::Error>(())
})?;
Ok(deleted)
Ok(orphaned.len())
}
/// Delete response body data (blob chunks and body files) whose owning HTTP
@@ -96,8 +87,7 @@ impl<'a> ClientDb<'a> {
let Some(response_id) = path.file_name().and_then(|n| n.to_str()) else {
continue;
};
if self.find_optional::<HttpResponse>(HttpResponseIden::Id, response_id).is_some()
{
if self.find_optional::<HttpResponse>(HttpResponseIden::Id, response_id).is_some() {
continue;
}
if fs::remove_file(&path).is_ok() {
@@ -108,6 +98,22 @@ impl<'a> ClientDb<'a> {
Ok(deleted)
}
}
impl<'a> WriteDb<'a> {
/// Returns the number of responses deleted.
pub fn delete_all_http_responses_for_request(
&self,
request_id: &str,
source: &UpdateSource,
) -> Result<usize> {
let responses = self.list_http_responses_for_request(request_id, None)?;
let count = responses.len();
for m in responses {
self.delete(&m, source)?;
}
Ok(count)
}
/// Returns the number of responses deleted.
pub fn delete_all_http_responses_for_workspace(
@@ -138,9 +144,8 @@ impl<'a> ClientDb<'a> {
}
// Delete request body blobs (pattern: {response_id}.request)
let blob_ctx = blob_manager.connect();
let body_id = format!("{}.request", http_response.id);
if let Err(e) = blob_ctx.delete_chunks(&body_id) {
if let Err(e) = blob_manager.with_tx(|b| b.delete_chunks(&body_id)) {
error!("Failed to delete request body blobs: {}", e);
}
@@ -187,36 +192,50 @@ impl<'a> ClientDb<'a> {
#[cfg(test)]
mod tests {
use crate::blob_manager::{BlobManager, BodyChunk};
use crate::client_db::ClientDb;
use crate::error::Error;
use crate::init_in_memory;
use crate::models::{HttpRequest, HttpResponse, Workspace};
use crate::query_manager::QueryManager;
use crate::util::UpdateSource;
/// A workspace, a request, and one response that still exists.
fn seed_live_response(db: &ClientDb, blob_manager: &BlobManager) -> HttpResponse {
fn seed_live_response(
query_manager: &QueryManager,
blob_manager: &BlobManager,
) -> HttpResponse {
let source = &UpdateSource::Background;
let workspace = db
.upsert_workspace(
&Workspace { name: "GC Test".to_string(), ..Default::default() },
source,
)
.expect("Failed to upsert workspace");
let request = db
.upsert_http_request(
&HttpRequest { workspace_id: workspace.id.clone(), ..Default::default() },
source,
)
.expect("Failed to upsert request");
db.upsert_http_response(
&HttpResponse {
request_id: request.id.clone(),
workspace_id: workspace.id.clone(),
..Default::default()
},
source,
blob_manager,
)
.expect("Failed to upsert response")
query_manager
.with_tx(|db| {
let workspace = db.upsert_workspace(
&Workspace { name: "GC Test".to_string(), ..Default::default() },
source,
)?;
let request = db.upsert_http_request(
&HttpRequest { workspace_id: workspace.id.clone(), ..Default::default() },
source,
)?;
db.upsert_http_response(
&HttpResponse {
request_id: request.id.clone(),
workspace_id: workspace.id.clone(),
..Default::default()
},
source,
blob_manager,
)
})
.expect("Failed to seed response")
}
fn insert_bodies(blob_manager: &BlobManager, body_ids: &[&str]) {
blob_manager
.with_tx(|b| {
for id in body_ids {
b.insert_chunk(&BodyChunk::new(*id, 0, b"data".to_vec()))?;
}
Ok::<_, Error>(())
})
.expect("Failed to insert chunks");
}
/// What a browser host runs: no filesystem, so bodies exist only as blob
@@ -224,23 +243,21 @@ mod tests {
#[test]
fn deletes_orphaned_response_body_blobs() {
let (query_manager, blob_manager, _rx) = init_in_memory().expect("Failed to init DB");
let db = query_manager.connect();
let live = seed_live_response(&db, &blob_manager);
let live = seed_live_response(&query_manager, &blob_manager);
let live_request_body_id = format!("{}.request", live.id);
{
// Scope the connection: the in-memory pool only has one, and the GC
// needs to take it
let blob_ctx = blob_manager.connect();
blob_ctx.insert_chunk(&BodyChunk::new(&live.id, 0, b"live".to_vec())).unwrap();
blob_ctx
.insert_chunk(&BodyChunk::new(&live_request_body_id, 0, b"live".to_vec()))
.unwrap();
blob_ctx.insert_chunk(&BodyChunk::new("rs_gone", 0, b"dead".to_vec())).unwrap();
blob_ctx.insert_chunk(&BodyChunk::new("rs_gone.request", 0, b"dead".to_vec())).unwrap();
}
insert_bodies(
&blob_manager,
&[
&live.id,
&live_request_body_id,
"rs_gone",
"rs_gone.request",
],
);
let deleted = db
let deleted = query_manager
.connect()
.delete_orphaned_response_body_blobs(&blob_manager)
.expect("Failed to GC response body blobs");
assert_eq!(deleted, 2);
@@ -255,24 +272,18 @@ mod tests {
#[test]
fn deletes_orphaned_response_bodies() {
let (query_manager, blob_manager, _rx) = init_in_memory().expect("Failed to init DB");
let db = query_manager.connect();
let live = seed_live_response(&db, &blob_manager);
let live = seed_live_response(&query_manager, &blob_manager);
let live_body_id = format!("{}.request", live.id);
{
// Scope the connection: the in-memory pool only has one, and the GC
// needs to take it
let blob_ctx = blob_manager.connect();
blob_ctx.insert_chunk(&BodyChunk::new(&live_body_id, 0, b"live".to_vec())).unwrap();
blob_ctx.insert_chunk(&BodyChunk::new("rs_gone.request", 0, b"dead".to_vec())).unwrap();
}
insert_bodies(&blob_manager, &[&live_body_id, "rs_gone.request"]);
let dir = std::env::temp_dir().join(format!("yaak-blob-gc-test-{}", live.id));
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(dir.join(&live.id), b"live").unwrap();
std::fs::write(dir.join("rs_gone"), b"dead").unwrap();
let deleted = db
let deleted = query_manager
.connect()
.delete_orphaned_response_bodies(&blob_manager, &dir)
.expect("Failed to GC response bodies");
assert_eq!(deleted, 2);
@@ -1,4 +1,4 @@
use crate::client_db::ClientDb;
use crate::client_db::{ClientDb, WriteDb};
use crate::error::Result;
use crate::models::{ImportSourceResource, ImportSourceResourceIden};
use sea_query::ExprTrait;
@@ -20,7 +20,9 @@ impl<'a> ClientDb<'a> {
let items = stmt.query_map(&*params.as_params(), |row| row.try_into())?;
Ok(items.filter_map(|v| v.ok()).collect())
}
}
impl<'a> WriteDb<'a> {
pub fn upsert_import_source_resource(
&self,
resource: &ImportSourceResource,
@@ -34,7 +36,7 @@ impl<'a> ClientDb<'a> {
ImportSourceResourceIden::SourceKey,
ImportSourceResourceIden::ModelType,
ImportSourceResourceIden::ModelId,
ImportSourceResourceIden::Snapshot,
ImportSourceResourceIden::ContentHash,
])
.values_panic([
CurrentTimestamp.into(),
@@ -42,8 +44,8 @@ impl<'a> ClientDb<'a> {
resource.import_source_id.as_str().into(),
resource.source_key.as_str().into(),
resource.model_type.as_str().into(),
resource.model_id.as_str().into(),
resource.snapshot.as_str().into(),
resource.model_id.clone().into(),
resource.content_hash.clone().into(),
])
.on_conflict(
OnConflict::columns([
@@ -54,7 +56,7 @@ impl<'a> ClientDb<'a> {
ImportSourceResourceIden::UpdatedAt,
ImportSourceResourceIden::ModelType,
ImportSourceResourceIden::ModelId,
ImportSourceResourceIden::Snapshot,
ImportSourceResourceIden::ContentHash,
])
.to_owned(),
)
@@ -1,4 +1,4 @@
use crate::client_db::ClientDb;
use crate::client_db::{ClientDb, WriteDb};
use crate::error::Result;
use crate::models::{ImportSource, ImportSourceIden};
use crate::util::UpdateSource;
@@ -25,7 +25,9 @@ impl<'a> ClientDb<'a> {
let sources = self.list_import_sources(workspace_id)?;
Ok(sources.into_iter().find(|s| s.importer == importer && s.origin == origin))
}
}
impl<'a> WriteDb<'a> {
pub fn upsert_import_source(
&self,
import_source: &ImportSource,
+3 -1
View File
@@ -1,4 +1,4 @@
use crate::client_db::ClientDb;
use crate::client_db::{ClientDb, WriteDb};
use crate::error::Result;
use crate::models::{KeyValue, KeyValueIden, UpsertModelInfo};
use crate::util::UpdateSource;
@@ -89,7 +89,9 @@ impl<'a> ClientDb<'a> {
.build_rusqlite(SqliteQueryBuilder);
self.conn().resolve().query_row(sql.as_str(), &*params.as_params(), KeyValue::from_row).ok()
}
}
impl<'a> WriteDb<'a> {
pub fn set_key_value_dte(
&self,
namespace: &str,
+1 -1
View File
@@ -25,8 +25,8 @@ mod websocket_events;
mod websocket_requests;
mod workspace_metas;
pub mod workspaces;
pub use model_changes::PersistedModelChange;
pub(crate) use duplicate_name::conflict_free_name;
pub use model_changes::PersistedModelChange;
const MAX_HISTORY_ITEMS: usize = 20;
+98 -57
View File
@@ -1,4 +1,4 @@
use crate::client_db::ClientDb;
use crate::client_db::{ClientDb, WriteDb};
use crate::error::Result;
use crate::util::ModelPayload;
use rusqlite::params;
@@ -69,7 +69,9 @@ impl<'a> ClientDb<'a> {
Ok(items.collect::<std::result::Result<Vec<_>, rusqlite::Error>>()?)
}
}
impl<'a> WriteDb<'a> {
pub fn prune_model_changes_older_than_days(&self, days: i64) -> Result<usize> {
let offset = format!("-{days} days");
Ok(self.conn().resolve().execute(
@@ -101,23 +103,36 @@ mod tests {
use crate::util::{ModelChangeEvent, UpdateSource};
use serde_json::json;
/// Startup bootstraps rows of its own; these tests count only their own.
fn clear_changes(query_manager: &crate::query_manager::QueryManager) {
query_manager
.with_tx(|db| {
db.conn().resolve().execute("DELETE FROM model_changes", [])?;
Ok::<_, crate::error::Error>(())
})
.expect("Failed to clear model changes");
}
#[test]
fn records_model_changes_for_upsert_and_delete() {
let (query_manager, blob_manager, _rx) = init_in_memory().expect("Failed to init DB");
let db = query_manager.connect();
clear_changes(&query_manager);
let workspace = db
.upsert_workspace(
&Workspace {
name: "Changes Test".to_string(),
setting_follow_redirects: true,
setting_validate_certificates: true,
..Default::default()
},
&UpdateSource::Sync,
)
let workspace = query_manager
.with_tx(|db| {
db.upsert_workspace(
&Workspace {
name: "Changes Test".to_string(),
setting_follow_redirects: true,
setting_validate_certificates: true,
..Default::default()
},
&UpdateSource::Sync,
)
})
.expect("Failed to upsert workspace");
let db = query_manager.connect();
let created_changes = db.list_model_changes_after(0, 10).expect("Failed to list changes");
assert_eq!(created_changes.len(), 1);
assert_eq!(created_changes[0].payload.model.id(), workspace.id);
@@ -128,9 +143,14 @@ mod tests {
));
assert!(matches!(created_changes[0].payload.update_source, UpdateSource::Sync));
db.delete_workspace_by_id(&workspace.id, &UpdateSource::Sync, &blob_manager)
drop(db);
query_manager
.with_tx(|db| {
db.delete_workspace_by_id(&workspace.id, &UpdateSource::Sync, &blob_manager)
})
.expect("Failed to delete workspace");
let db = query_manager.connect();
let all_changes = db.list_model_changes_after(0, 10).expect("Failed to list changes");
assert_eq!(all_changes.len(), 2);
assert!(matches!(all_changes[1].payload.change, ModelChangeEvent::Delete));
@@ -146,19 +166,23 @@ mod tests {
#[test]
fn prunes_old_model_changes() {
let (query_manager, _blob_manager, _rx) = init_in_memory().expect("Failed to init DB");
clear_changes(&query_manager);
query_manager
.with_tx(|db| {
db.upsert_workspace(
&Workspace {
name: "Prune Test".to_string(),
setting_follow_redirects: true,
setting_validate_certificates: true,
..Default::default()
},
&UpdateSource::Sync,
)
})
.expect("Failed to upsert workspace");
let db = query_manager.connect();
db.upsert_workspace(
&Workspace {
name: "Prune Test".to_string(),
setting_follow_redirects: true,
setting_validate_certificates: true,
..Default::default()
},
&UpdateSource::Sync,
)
.expect("Failed to upsert workspace");
let changes = db.list_model_changes_after(0, 10).expect("Failed to list changes");
assert_eq!(changes.len(), 1);
@@ -170,31 +194,41 @@ mod tests {
)
.expect("Failed to age model change row");
let pruned =
db.prune_model_changes_older_than_days(30).expect("Failed to prune model changes");
drop(db);
let pruned = query_manager
.with_tx(|db| db.prune_model_changes_older_than_days(30))
.expect("Failed to prune model changes");
assert_eq!(pruned, 1);
assert!(db.list_model_changes_after(0, 10).expect("Failed to list changes").is_empty());
assert!(
query_manager
.connect()
.list_model_changes_after(0, 10)
.expect("Failed to list changes")
.is_empty()
);
}
#[test]
fn list_model_changes_since_uses_timestamp_with_id_tiebreaker() {
let (query_manager, blob_manager, _rx) = init_in_memory().expect("Failed to init DB");
clear_changes(&query_manager);
query_manager
.with_tx(|db| {
let workspace = db.upsert_workspace(
&Workspace {
name: "Cursor Test".to_string(),
setting_follow_redirects: true,
setting_validate_certificates: true,
..Default::default()
},
&UpdateSource::Sync,
)?;
db.delete_workspace_by_id(&workspace.id, &UpdateSource::Sync, &blob_manager)
})
.expect("Failed to seed changes");
let db = query_manager.connect();
let workspace = db
.upsert_workspace(
&Workspace {
name: "Cursor Test".to_string(),
setting_follow_redirects: true,
setting_validate_certificates: true,
..Default::default()
},
&UpdateSource::Sync,
)
.expect("Failed to upsert workspace");
db.delete_workspace_by_id(&workspace.id, &UpdateSource::Sync, &blob_manager)
.expect("Failed to delete workspace");
let all = db.list_model_changes_after(0, 10).expect("Failed to list changes");
assert_eq!(all.len(), 2);
@@ -213,19 +247,23 @@ mod tests {
#[test]
fn prunes_old_model_changes_by_hours() {
let (query_manager, _blob_manager, _rx) = init_in_memory().expect("Failed to init DB");
clear_changes(&query_manager);
query_manager
.with_tx(|db| {
db.upsert_workspace(
&Workspace {
name: "Prune Hour Test".to_string(),
setting_follow_redirects: true,
setting_validate_certificates: true,
..Default::default()
},
&UpdateSource::Sync,
)
})
.expect("Failed to upsert workspace");
let db = query_manager.connect();
db.upsert_workspace(
&Workspace {
name: "Prune Hour Test".to_string(),
setting_follow_redirects: true,
setting_validate_certificates: true,
..Default::default()
},
&UpdateSource::Sync,
)
.expect("Failed to upsert workspace");
let changes = db.list_model_changes_after(0, 10).expect("Failed to list changes");
assert_eq!(changes.len(), 1);
@@ -237,14 +275,17 @@ mod tests {
)
.expect("Failed to age model change row");
let pruned =
db.prune_model_changes_older_than_hours(1).expect("Failed to prune model changes");
drop(db);
let pruned = query_manager
.with_tx(|db| db.prune_model_changes_older_than_hours(1))
.expect("Failed to prune model changes");
assert_eq!(pruned, 1);
}
#[test]
fn list_model_changes_deserializes_http_response_event_payload() {
let (query_manager, _blob_manager, _rx) = init_in_memory().expect("Failed to init DB");
clear_changes(&query_manager);
let db = query_manager.connect();
let payload = json!({
+223 -209
View File
@@ -1,4 +1,4 @@
use crate::client_db::ClientDb;
use crate::client_db::{ClientDb, WriteDb};
use crate::error::Result;
use crate::models::{
GrpcRequest, HttpRequest, ModelVersion, ModelVersionIden, ModelVersionReason, UpsertModelInfo,
@@ -28,6 +28,29 @@ impl<'a> ClientDb<'a> {
self.find_many(ModelVersionIden::ModelId, model_id, None)
}
/// Whether a request's content has moved on from a given version.
pub fn request_matches_version(&self, version: &ModelVersion) -> Result<bool> {
let live = self.get_any_request(&version.model_id)?;
let hash = content_hash(&version_document(&live.to_value()?)?)?;
Ok(hash == version.content_hash)
}
fn find_version_by_hash(&self, model_id: &str, content_hash: &str) -> Option<ModelVersion> {
let (sql, params) = Query::select()
.from(ModelVersionIden::Table)
.column(sea_query::Asterisk)
.cond_where(
Expr::col(ModelVersionIden::ModelId)
.eq(model_id)
.and(Expr::col(ModelVersionIden::ContentHash).eq(content_hash)),
)
.build_rusqlite(SqliteQueryBuilder);
let mut stmt = self.conn().prepare(sql.as_str()).ok()?;
stmt.query_row(&*params.as_params(), ModelVersion::from_row).ok()
}
}
impl<'a> WriteDb<'a> {
/// Capture a request's current content, or return the version that already
/// holds it.
///
@@ -131,13 +154,6 @@ impl<'a> ClientDb<'a> {
})
}
/// Whether a request's content has moved on from a given version.
pub fn request_matches_version(&self, version: &ModelVersion) -> Result<bool> {
let live = self.get_any_request(&version.model_id)?;
let hash = content_hash(&version_document(&live.to_value()?)?)?;
Ok(hash == version.content_hash)
}
pub fn delete_model_versions_for_model(&self, model_id: &str) -> Result<usize> {
self.delete_many_untracked::<ModelVersion>(ModelVersionIden::ModelId, model_id)
}
@@ -170,26 +186,12 @@ impl<'a> ClientDb<'a> {
"#;
Ok(self.conn().execute(sql, rusqlite::params![model_id, cutoff, RETENTION_COUNT])?)
}
fn find_version_by_hash(&self, model_id: &str, content_hash: &str) -> Option<ModelVersion> {
let (sql, params) = Query::select()
.from(ModelVersionIden::Table)
.column(sea_query::Asterisk)
.cond_where(
Expr::col(ModelVersionIden::ModelId)
.eq(model_id)
.and(Expr::col(ModelVersionIden::ContentHash).eq(content_hash)),
)
.build_rusqlite(SqliteQueryBuilder);
let mut stmt = self.conn().prepare(sql.as_str()).ok()?;
stmt.query_row(&*params.as_params(), ModelVersion::from_row).ok()
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::client_db::ClientDb;
use crate::blob_manager::BlobManager;
use crate::init_in_memory;
use crate::models::{HttpRequest, HttpResponse, Workspace};
@@ -197,7 +199,19 @@ mod tests {
UpdateSource::Background
}
fn seed(db: &ClientDb) -> (Workspace, HttpRequest) {
/// Run a test inside one write transaction on a fresh database. Reads
/// inside it see the transaction's own writes.
fn with_db(test: impl FnOnce(&WriteDb, &BlobManager)) {
let (query_manager, blobs, _rx) = init_in_memory().expect("Failed to init DB");
query_manager
.with_tx(|db| {
test(db, &blobs);
Ok::<_, crate::error::Error>(())
})
.expect("Failed to run test transaction");
}
fn seed(db: &WriteDb) -> (Workspace, HttpRequest) {
let workspace = db
.upsert_workspace(&Workspace { name: "Versions".to_string(), ..Default::default() }, &source())
.expect("Failed to upsert workspace");
@@ -215,56 +229,56 @@ mod tests {
(workspace, request)
}
fn snapshot(db: &ClientDb, request_id: &str, reason: ModelVersionReason) -> ModelVersion {
fn snapshot(db: &WriteDb, request_id: &str, reason: ModelVersionReason) -> ModelVersion {
db.snapshot_request_by_id(request_id, reason).expect("Failed to snapshot")
}
#[test]
fn snapshotting_unchanged_content_reuses_the_same_version() {
let (query_manager, _blobs, _rx) = init_in_memory().expect("Failed to init DB");
let db = query_manager.connect();
let (_workspace, request) = seed(&db);
with_db(|db, _blobs| {
let (_workspace, request) = seed(db);
let first = snapshot(&db, &request.id, ModelVersionReason::Send);
let second = snapshot(&db, &request.id, ModelVersionReason::Idle);
let third = snapshot(&db, &request.id, ModelVersionReason::Switch);
let first = snapshot(db, &request.id, ModelVersionReason::Send);
let second = snapshot(db, &request.id, ModelVersionReason::Idle);
let third = snapshot(db, &request.id, ModelVersionReason::Switch);
assert_eq!(first.id, second.id);
assert_eq!(first.id, third.id);
// The first capture's reason is the one that sticks; a version is its content
assert_eq!(second.reason, ModelVersionReason::Send);
assert_eq!(db.list_model_versions(&request.id).unwrap().len(), 1);
assert_eq!(first.id, second.id);
assert_eq!(first.id, third.id);
// The first capture's reason is the one that sticks; a version is its content
assert_eq!(second.reason, ModelVersionReason::Send);
assert_eq!(db.list_model_versions(&request.id).unwrap().len(), 1);
});
}
#[test]
fn bookkeeping_writes_do_not_mint_a_version() {
let (query_manager, _blobs, _rx) = init_in_memory().expect("Failed to init DB");
let db = query_manager.connect();
let (_workspace, request) = seed(&db);
with_db(|db, _blobs| {
let (_workspace, request) = seed(db);
let first = snapshot(&db, &request.id, ModelVersionReason::Send);
let first = snapshot(db, &request.id, ModelVersionReason::Send);
let folder = db
.upsert_folder(
&crate::models::Folder {
workspace_id: request.workspace_id.clone(),
..Default::default()
let folder = db
.upsert_folder(
&crate::models::Folder {
workspace_id: request.workspace_id.clone(),
..Default::default()
},
&source(),
)
.unwrap();
db.upsert_http_request(
&HttpRequest {
folder_id: Some(folder.id),
sort_priority: 42.0,
..db.get_http_request(&request.id).unwrap()
},
&source(),
)
.unwrap();
db.upsert_http_request(
&HttpRequest {
folder_id: Some(folder.id),
sort_priority: 42.0,
..db.get_http_request(&request.id).unwrap()
},
&source(),
)
.unwrap();
assert_eq!(snapshot(&db, &request.id, ModelVersionReason::Idle).id, first.id);
assert_eq!(db.list_model_versions(&request.id).unwrap().len(), 1);
assert_eq!(snapshot(db, &request.id, ModelVersionReason::Idle).id, first.id);
assert_eq!(db.list_model_versions(&request.id).unwrap().len(), 1);
});
}
/// The pair editor writes a generated `id` into every header row the first
@@ -273,139 +287,139 @@ mod tests {
/// a version whose diff is nothing but ids.
#[test]
fn row_ids_written_by_the_editor_do_not_mint_a_version() {
let (query_manager, _blobs, _rx) = init_in_memory().expect("Failed to init DB");
let db = query_manager.connect();
let (_workspace, request) = seed(&db);
with_db(|db, _blobs| {
let (_workspace, request) = seed(db);
let header = |id: Option<&str>| crate::models::HttpRequestHeader {
name: "Accept".to_string(),
value: "application/json".to_string(),
id: id.map(str::to_string),
..Default::default()
};
let header = |id: Option<&str>| crate::models::HttpRequestHeader {
name: "Accept".to_string(),
value: "application/json".to_string(),
id: id.map(str::to_string),
..Default::default()
};
let request = db
.upsert_http_request(&HttpRequest { headers: vec![header(None)], ..request }, &source())
let request = db
.upsert_http_request(&HttpRequest { headers: vec![header(None)], ..request }, &source())
.unwrap();
let first = snapshot(db, &request.id, ModelVersionReason::Send);
// Opening the request in the editor fills the row id in
db.upsert_http_request(
&HttpRequest { headers: vec![header(Some("row_generated"))], ..request.clone() },
&source(),
)
.unwrap();
let first = snapshot(&db, &request.id, ModelVersionReason::Send);
// Opening the request in the editor fills the row id in
db.upsert_http_request(
&HttpRequest { headers: vec![header(Some("row_generated"))], ..request.clone() },
&source(),
)
.unwrap();
assert_eq!(snapshot(db, &request.id, ModelVersionReason::Idle).id, first.id);
assert_eq!(db.list_model_versions(&request.id).unwrap().len(), 1);
assert_eq!(snapshot(&db, &request.id, ModelVersionReason::Idle).id, first.id);
assert_eq!(db.list_model_versions(&request.id).unwrap().len(), 1);
// A real edit to the same row still counts
db.upsert_http_request(
&HttpRequest {
headers: vec![crate::models::HttpRequestHeader {
value: "text/plain".to_string(),
..header(Some("row_generated"))
}],
..request.clone()
},
&source(),
)
.unwrap();
assert_ne!(snapshot(&db, &request.id, ModelVersionReason::Idle).id, first.id);
// A real edit to the same row still counts
db.upsert_http_request(
&HttpRequest {
headers: vec![crate::models::HttpRequestHeader {
value: "text/plain".to_string(),
..header(Some("row_generated"))
}],
..request.clone()
},
&source(),
)
.unwrap();
assert_ne!(snapshot(db, &request.id, ModelVersionReason::Idle).id, first.id);
});
}
#[test]
fn editing_content_mints_a_version() {
let (query_manager, _blobs, _rx) = init_in_memory().expect("Failed to init DB");
let db = query_manager.connect();
let (_workspace, request) = seed(&db);
with_db(|db, _blobs| {
let (_workspace, request) = seed(db);
snapshot(&db, &request.id, ModelVersionReason::Send);
db.upsert_http_request(
&HttpRequest { url: "https://example.com/two".to_string(), ..request.clone() },
&source(),
)
.unwrap();
snapshot(&db, &request.id, ModelVersionReason::Idle);
snapshot(db, &request.id, ModelVersionReason::Send);
db.upsert_http_request(
&HttpRequest { url: "https://example.com/two".to_string(), ..request.clone() },
&source(),
)
.unwrap();
snapshot(db, &request.id, ModelVersionReason::Idle);
assert_eq!(db.list_model_versions(&request.id).unwrap().len(), 2);
assert_eq!(db.list_model_versions(&request.id).unwrap().len(), 2);
});
}
#[test]
fn restoring_writes_the_old_content_back_without_a_new_version() {
let (query_manager, _blobs, _rx) = init_in_memory().expect("Failed to init DB");
let db = query_manager.connect();
let (_workspace, request) = seed(&db);
with_db(|db, _blobs| {
let (_workspace, request) = seed(db);
let original = snapshot(&db, &request.id, ModelVersionReason::Send);
db.upsert_http_request(
&HttpRequest {
url: "https://example.com/two".to_string(),
name: "Edited".to_string(),
..request.clone()
},
&source(),
)
.unwrap();
let edited = snapshot(&db, &request.id, ModelVersionReason::Idle);
let original = snapshot(db, &request.id, ModelVersionReason::Send);
db.upsert_http_request(
&HttpRequest {
url: "https://example.com/two".to_string(),
name: "Edited".to_string(),
..request.clone()
},
&source(),
)
.unwrap();
let edited = snapshot(db, &request.id, ModelVersionReason::Idle);
db.restore_request_version(&original.id, &source()).expect("Failed to restore");
db.restore_request_version(&original.id, &source()).expect("Failed to restore");
let live = db.get_http_request(&request.id).unwrap();
assert_eq!(live.url, "https://example.com/one");
assert_eq!(live.name, "Original");
assert_eq!(live.id, request.id);
let live = db.get_http_request(&request.id).unwrap();
assert_eq!(live.url, "https://example.com/one");
assert_eq!(live.name, "Original");
assert_eq!(live.id, request.id);
// The restored content already had a version, and the edit it replaced
// still has its own, so nothing new appears
let versions = db.list_model_versions(&request.id).unwrap();
assert_eq!(versions.len(), 2);
assert!(versions.iter().any(|v| v.id == original.id));
assert!(versions.iter().any(|v| v.id == edited.id));
// The restored content already had a version, and the edit it replaced
// still has its own, so nothing new appears
let versions = db.list_model_versions(&request.id).unwrap();
assert_eq!(versions.len(), 2);
assert!(versions.iter().any(|v| v.id == original.id));
assert!(versions.iter().any(|v| v.id == edited.id));
});
}
/// The case restore exists to be safe for: an edit that was never captured.
#[test]
fn restoring_captures_uncaptured_edits_first() {
let (query_manager, _blobs, _rx) = init_in_memory().expect("Failed to init DB");
let db = query_manager.connect();
let (_workspace, request) = seed(&db);
with_db(|db, _blobs| {
let (_workspace, request) = seed(db);
let original = snapshot(&db, &request.id, ModelVersionReason::Send);
db.upsert_http_request(
&HttpRequest { url: "https://example.com/unsaved".to_string(), ..request.clone() },
&source(),
)
.unwrap();
let original = snapshot(db, &request.id, ModelVersionReason::Send);
db.upsert_http_request(
&HttpRequest { url: "https://example.com/unsaved".to_string(), ..request.clone() },
&source(),
)
.unwrap();
db.restore_request_version(&original.id, &source()).expect("Failed to restore");
db.restore_request_version(&original.id, &source()).expect("Failed to restore");
let versions = db.list_model_versions(&request.id).unwrap();
assert_eq!(versions.len(), 2);
let rescued = versions.iter().find(|v| v.id != original.id).unwrap();
assert_eq!(rescued.reason, ModelVersionReason::Restore);
assert_eq!(rescued.document.get("url").unwrap(), "https://example.com/unsaved");
let versions = db.list_model_versions(&request.id).unwrap();
assert_eq!(versions.len(), 2);
let rescued = versions.iter().find(|v| v.id != original.id).unwrap();
assert_eq!(rescued.reason, ModelVersionReason::Restore);
assert_eq!(rescued.document.get("url").unwrap(), "https://example.com/unsaved");
});
}
#[test]
fn request_matches_version_tracks_the_live_content() {
let (query_manager, _blobs, _rx) = init_in_memory().expect("Failed to init DB");
let db = query_manager.connect();
let (_workspace, request) = seed(&db);
with_db(|db, _blobs| {
let (_workspace, request) = seed(db);
let version = snapshot(&db, &request.id, ModelVersionReason::Send);
assert!(db.request_matches_version(&version).unwrap());
let version = snapshot(db, &request.id, ModelVersionReason::Send);
assert!(db.request_matches_version(&version).unwrap());
db.upsert_http_request(
&HttpRequest { url: "https://example.com/two".to_string(), ..request.clone() },
&source(),
)
.unwrap();
assert!(!db.request_matches_version(&version).unwrap());
db.upsert_http_request(
&HttpRequest { url: "https://example.com/two".to_string(), ..request.clone() },
&source(),
)
.unwrap();
assert!(!db.request_matches_version(&version).unwrap());
});
}
/// Write `count` distinct versions by walking the request's URL forward.
fn make_versions(db: &ClientDb, request: &HttpRequest, count: usize) -> Vec<ModelVersion> {
fn make_versions(db: &WriteDb, request: &HttpRequest, count: usize) -> Vec<ModelVersion> {
(0..count)
.map(|i| {
db.upsert_http_request(
@@ -420,84 +434,84 @@ mod tests {
#[test]
fn unreferenced_versions_are_pruned_to_the_newest_fifty() {
let (query_manager, _blobs, _rx) = init_in_memory().expect("Failed to init DB");
let db = query_manager.connect();
let (_workspace, request) = seed(&db);
with_db(|db, _blobs| {
let (_workspace, request) = seed(db);
let versions = make_versions(&db, &request, RETENTION_COUNT as usize + 10);
let versions = make_versions(db, &request, RETENTION_COUNT as usize + 10);
let kept = db.list_model_versions(&request.id).unwrap();
assert_eq!(kept.len(), RETENTION_COUNT as usize);
// The oldest went first
assert!(!kept.iter().any(|v| v.id == versions[0].id));
assert!(kept.iter().any(|v| v.id == versions.last().unwrap().id));
let kept = db.list_model_versions(&request.id).unwrap();
assert_eq!(kept.len(), RETENTION_COUNT as usize);
// The oldest went first
assert!(!kept.iter().any(|v| v.id == versions[0].id));
assert!(kept.iter().any(|v| v.id == versions.last().unwrap().id));
});
}
#[test]
fn a_referenced_version_survives_retention() {
let (query_manager, blobs, _rx) = init_in_memory().expect("Failed to init DB");
let db = query_manager.connect();
let (workspace, request) = seed(&db);
with_db(|db, blobs| {
let (workspace, request) = seed(db);
let pinned = snapshot(&db, &request.id, ModelVersionReason::Send);
db.upsert_http_response(
&HttpResponse {
request_id: request.id.clone(),
workspace_id: workspace.id.clone(),
version_id: Some(pinned.id.clone()),
..Default::default()
},
&source(),
&blobs,
)
.unwrap();
let pinned = snapshot(db, &request.id, ModelVersionReason::Send);
db.upsert_http_response(
&HttpResponse {
request_id: request.id.clone(),
workspace_id: workspace.id.clone(),
version_id: Some(pinned.id.clone()),
..Default::default()
},
&source(),
blobs,
)
.unwrap();
make_versions(&db, &request, RETENTION_COUNT as usize + 10);
make_versions(db, &request, RETENTION_COUNT as usize + 10);
let kept = db.list_model_versions(&request.id).unwrap();
assert!(
kept.iter().any(|v| v.id == pinned.id),
"a version a response points at must outlive retention",
);
let kept = db.list_model_versions(&request.id).unwrap();
assert!(
kept.iter().any(|v| v.id == pinned.id),
"a version a response points at must outlive retention",
);
});
}
#[test]
fn unreferenced_versions_expire_after_thirty_days() {
let (query_manager, _blobs, _rx) = init_in_memory().expect("Failed to init DB");
let db = query_manager.connect();
let (_workspace, request) = seed(&db);
with_db(|db, _blobs| {
let (_workspace, request) = seed(db);
let old = snapshot(&db, &request.id, ModelVersionReason::Send);
db.conn()
.execute(
"UPDATE model_versions SET created_at = datetime('now', '-31 days') WHERE id = ?1",
rusqlite::params![old.id],
let old = snapshot(db, &request.id, ModelVersionReason::Send);
db.conn()
.execute(
"UPDATE model_versions SET created_at = datetime('now', '-31 days') WHERE id = ?1",
rusqlite::params![old.id],
)
.unwrap();
// Any later capture prunes
db.upsert_http_request(
&HttpRequest { url: "https://example.com/two".to_string(), ..request.clone() },
&source(),
)
.unwrap();
let fresh = snapshot(db, &request.id, ModelVersionReason::Idle);
// Any later capture prunes
db.upsert_http_request(
&HttpRequest { url: "https://example.com/two".to_string(), ..request.clone() },
&source(),
)
.unwrap();
let fresh = snapshot(&db, &request.id, ModelVersionReason::Idle);
let kept = db.list_model_versions(&request.id).unwrap();
assert_eq!(kept.len(), 1);
assert_eq!(kept[0].id, fresh.id);
let kept = db.list_model_versions(&request.id).unwrap();
assert_eq!(kept.len(), 1);
assert_eq!(kept[0].id, fresh.id);
});
}
#[test]
fn deleting_a_request_deletes_its_versions() {
let (query_manager, _blobs, _rx) = init_in_memory().expect("Failed to init DB");
let db = query_manager.connect();
let (_workspace, request) = seed(&db);
with_db(|db, _blobs| {
let (_workspace, request) = seed(db);
make_versions(&db, &request, 3);
assert!(!db.list_model_versions(&request.id).unwrap().is_empty());
make_versions(db, &request, 3);
assert!(!db.list_model_versions(&request.id).unwrap().is_empty());
db.delete_http_request_by_id(&request.id, &source()).unwrap();
assert!(db.list_model_versions(&request.id).unwrap().is_empty());
db.delete_http_request_by_id(&request.id, &source()).unwrap();
assert!(db.list_model_versions(&request.id).unwrap().is_empty());
});
}
}
@@ -1,4 +1,4 @@
use crate::client_db::ClientDb;
use crate::client_db::{ClientDb, WriteDb};
use crate::error::Result;
use crate::models::{PluginKeyValue, PluginKeyValueIden};
use sea_query::ExprTrait;
@@ -22,7 +22,9 @@ impl<'a> ClientDb<'a> {
.query_row(sql.as_str(), &*params.as_params(), |row| row.try_into())
.ok()
}
}
impl<'a> WriteDb<'a> {
pub fn set_plugin_key_value(
&self,
plugin_name: &str,
+3 -1
View File
@@ -1,4 +1,4 @@
use crate::client_db::ClientDb;
use crate::client_db::{ClientDb, WriteDb};
use crate::error::Result;
use crate::models::{Plugin, PluginIden};
use crate::util::UpdateSource;
@@ -15,7 +15,9 @@ impl<'a> ClientDb<'a> {
pub fn list_plugins(&self) -> Result<Vec<Plugin>> {
self.find_all()
}
}
impl<'a> WriteDb<'a> {
pub fn delete_plugin(&self, plugin: &Plugin, source: &UpdateSource) -> Result<Plugin> {
self.delete(plugin, source)
}
+47 -37
View File
@@ -1,52 +1,62 @@
use std::collections::HashMap;
use crate::client_db::ClientDb;
use crate::client_db::{ClientDb, WriteDb};
use crate::error::Result;
use crate::models::{EditorKeymap, Settings, SettingsIden};
use crate::util::UpdateSource;
impl<'a> ClientDb<'a> {
/// The settings row, or the defaults if it has not been written yet.
/// [`WriteDb::ensure_settings`] persists it at startup.
pub fn get_settings(&self) -> Settings {
let id = "default".to_string();
self.find_optional::<Settings>(SettingsIden::Id, "default").unwrap_or_else(default_settings)
}
}
if let Some(s) = self.find_optional::<Settings>(SettingsIden::Id, &id) {
return s;
};
let settings = Settings {
model: "settings".to_string(),
id,
created_at: Default::default(),
updated_at: Default::default(),
appearance: "system".to_string(),
client_certificates: Vec::new(),
editor_font_size: 12,
editor_font: None,
editor_keymap: EditorKeymap::Default,
editor_soft_wrap: true,
interface_font_size: 14,
interface_scale: 1.0,
interface_font: None,
hide_window_controls: false,
use_native_titlebar: false,
open_workspace_new_window: None,
proxy: None,
theme_dark: "yaak-dark".to_string(),
theme_light: "yaak-light".to_string(),
update_channel: "stable".to_string(),
autoupdate: true,
colored_methods: false,
hide_license_badge: false,
prompt_feedback: true,
auto_download_updates: true,
check_notifications: true,
hotkeys: HashMap::new(),
};
self.upsert(&settings, &UpdateSource::Background).expect("Failed to upsert settings")
impl<'a> WriteDb<'a> {
/// Create the settings row if it does not exist.
pub fn ensure_settings(&self) -> Result<Settings> {
if let Some(s) = self.find_optional::<Settings>(SettingsIden::Id, "default") {
return Ok(s);
}
self.upsert(&default_settings(), &UpdateSource::Background)
}
pub fn upsert_settings(&self, settings: &Settings, source: &UpdateSource) -> Result<Settings> {
self.upsert(settings, source)
}
}
fn default_settings() -> Settings {
Settings {
model: "settings".to_string(),
id: "default".to_string(),
created_at: Default::default(),
updated_at: Default::default(),
appearance: "system".to_string(),
client_certificates: Vec::new(),
editor_font_size: 12,
editor_font: None,
editor_keymap: EditorKeymap::Default,
editor_soft_wrap: true,
interface_font_size: 14,
interface_scale: 1.0,
interface_font: None,
hide_window_controls: false,
use_native_titlebar: false,
open_workspace_new_window: None,
proxy: None,
theme_dark: "yaak-dark".to_string(),
theme_light: "yaak-light".to_string(),
update_channel: "stable".to_string(),
autoupdate: true,
colored_methods: false,
hide_license_badge: false,
prompt_feedback: true,
mcp_server_enabled: false,
auto_download_updates: true,
check_notifications: true,
hotkeys: HashMap::new(),
}
}
@@ -1,4 +1,4 @@
use crate::client_db::ClientDb;
use crate::client_db::{ClientDb, WriteDb};
use crate::error::Result;
use crate::models::{SyncState, SyncStateIden, UpsertModelInfo};
use crate::util::UpdateSource;
@@ -12,10 +12,6 @@ impl<'a> ClientDb<'a> {
self.find_one(SyncStateIden::Id, id)
}
pub fn upsert_sync_state(&self, sync_state: &SyncState) -> Result<SyncState> {
self.upsert(sync_state, &UpdateSource::Sync)
}
pub fn list_sync_states_for_workspace(
&self,
workspace_id: &str,
@@ -34,6 +30,12 @@ impl<'a> ClientDb<'a> {
let items = stmt.query_map(&*params.as_params(), SyncState::from_row)?;
Ok(items.map(|v| v.unwrap()).collect())
}
}
impl<'a> WriteDb<'a> {
pub fn upsert_sync_state(&self, sync_state: &SyncState) -> Result<SyncState> {
self.upsert(sync_state, &UpdateSource::Sync)
}
pub fn delete_sync_state(&self, sync_state: &SyncState) -> Result<SyncState> {
self.delete(sync_state, &UpdateSource::Sync)
@@ -1,4 +1,4 @@
use crate::client_db::ClientDb;
use crate::client_db::{ClientDb, WriteDb};
use crate::error::Result;
use crate::models::{WebsocketConnection, WebsocketConnectionIden, WebsocketConnectionState};
use crate::queries::MAX_HISTORY_ITEMS;
@@ -13,6 +13,22 @@ impl<'a> ClientDb<'a> {
self.find_one(WebsocketConnectionIden::Id, id)
}
pub fn list_websocket_connections(
&self,
workspace_id: &str,
) -> Result<Vec<WebsocketConnection>> {
self.find_many(WebsocketConnectionIden::WorkspaceId, workspace_id, None)
}
pub fn list_websocket_connections_for_request(
&self,
request_id: &str,
) -> Result<Vec<WebsocketConnection>> {
self.find_many(WebsocketConnectionIden::RequestId, request_id, None)
}
}
impl<'a> WriteDb<'a> {
pub fn delete_all_websocket_connections_for_request(
&self,
request_id: &str,
@@ -37,20 +53,6 @@ impl<'a> ClientDb<'a> {
Ok(())
}
pub fn list_websocket_connections(
&self,
workspace_id: &str,
) -> Result<Vec<WebsocketConnection>> {
self.find_many(WebsocketConnectionIden::WorkspaceId, workspace_id, None)
}
pub fn list_websocket_connections_for_request(
&self,
request_id: &str,
) -> Result<Vec<WebsocketConnection>> {
self.find_many(WebsocketConnectionIden::RequestId, request_id, None)
}
pub fn delete_websocket_connection(
&self,
websocket_connection: &WebsocketConnection,
@@ -1,4 +1,4 @@
use crate::client_db::ClientDb;
use crate::client_db::{ClientDb, WriteDb};
use crate::error::Result;
use crate::models::{WebsocketEvent, WebsocketEventIden};
use crate::util::UpdateSource;
@@ -11,7 +11,9 @@ impl<'a> ClientDb<'a> {
pub fn list_websocket_events(&self, connection_id: &str) -> Result<Vec<WebsocketEvent>> {
self.find_many(WebsocketEventIden::ConnectionId, connection_id, None)
}
}
impl<'a> WriteDb<'a> {
pub fn upsert_websocket_event(
&self,
websocket_event: &WebsocketEvent,
Loaded 100 of 794 files, more files were not shown because too many files have changed in this diff. Show more