From 10e962a0e684d3c629bada67c27d37b6e1252260 Mon Sep 17 00:00:00 2001 From: Gregory Schier Date: Sun, 16 Aug 2026 11:10:14 -0700 Subject: [PATCH] Add a plugin API for reading HTTP response bodies (#560) --- Cargo.lock | 1 + crates-cli/yaak-cli/src/plugin_events.rs | 14 +- crates-tauri/yaak-app-client/src/lib.rs | 8 +- .../yaak-app-client/src/plugin_events.rs | 9 + .../yaak-rpc-schema/bindings/gen_models.ts | 2 +- crates/yaak-models/bindings/gen_models.ts | 1 - crates/yaak-models/src/models.rs | 7 + crates/yaak-plugins/bindings/gen_events.ts | 60 ++++- crates/yaak-plugins/bindings/gen_models.ts | 1 - crates/yaak-plugins/src/events.rs | 76 ++++++ crates/yaak/Cargo.toml | 1 + crates/yaak/src/lib.rs | 1 + crates/yaak/src/plugin_events.rs | 190 ++++++++++++-- crates/yaak/src/response_body.rs | 232 ++++++++++++++++++ crates/yaak/src/send.rs | 13 + package-lock.json | 2 +- packages/plugin-runtime-types/package.json | 2 +- .../src/bindings/gen_events.ts | 60 ++++- .../src/bindings/gen_models.ts | 1 - .../src/plugins/Context.ts | 99 +++++++- .../plugin-runtime-types/src/plugins/index.ts | 8 +- packages/plugin-runtime/src/PluginInstance.ts | 91 ++++++- packages/plugin-runtime/src/responseBody.ts | 194 +++++++++++++++ .../plugin-runtime/tests/responseBody.test.ts | 203 +++++++++++++++ .../mcp-server/src/tools/httpRequest.ts | 2 +- plugins/auth-ntlm/src/index.ts | 2 +- plugins/auth-oauth2/src/fetchAccessToken.ts | 8 +- .../src/getOrRefreshAccessToken.ts | 7 +- .../template-function-response/src/index.ts | 49 ++-- 29 files changed, 1274 insertions(+), 70 deletions(-) create mode 100644 crates/yaak/src/response_body.rs create mode 100644 packages/plugin-runtime/src/responseBody.ts create mode 100644 packages/plugin-runtime/tests/responseBody.test.ts diff --git a/Cargo.lock b/Cargo.lock index 57263446..c61d3d54 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11196,6 +11196,7 @@ name = "yaak" version = "0.1.0" dependencies = [ "async-trait", + "base64 0.22.1", "log 0.4.29", "md5 0.8.0", "serde_json", diff --git a/crates-cli/yaak-cli/src/plugin_events.rs b/crates-cli/yaak-cli/src/plugin_events.rs index d232a88a..146c1025 100644 --- a/crates-cli/yaak-cli/src/plugin_events.rs +++ b/crates-cli/yaak-cli/src/plugin_events.rs @@ -1,5 +1,7 @@ use crate::context::CliExecutionContext; use arboard::Clipboard; +use base64::Engine; +use base64::prelude::BASE64_STANDARD; use console::Term; use inquire::{Confirm, Editor, Password, PasswordDisplayMode, Select, Text}; use serde_json::Value; @@ -12,6 +14,7 @@ use yaak::plugin_events::{ GroupedPluginEvent, HostRequest, SharedPluginEventContext, handle_shared_plugin_event, }; use yaak::render::{render_grpc_request, render_http_request}; +use yaak::response_body::FileResponseBodyStore; use yaak::send::{SendHttpRequestWithPluginsParams, send_http_request_with_plugins}; use yaak_crypto::manager::EncryptionManager; use yaak_http::cookies::get_cookie_value_from_jar; @@ -131,6 +134,7 @@ async fn build_plugin_reply( match handle_shared_plugin_event( &host_context.query_manager, + &FileResponseBodyStore::new(&host_context.query_manager), &event.payload, SharedPluginEventContext { plugin_name, workspace_id: shared_workspace_id }, ) { @@ -223,7 +227,15 @@ async fn build_plugin_reply( .await { Ok(result) => Some(InternalEventPayload::SendHttpRequestResponse( - SendHttpRequestResponse { http_response: result.response }, + SendHttpRequestResponse { + http_response: result.response, + // Nothing saved this body, so the reply is the only + // place the plugin can get it. + body: result + .response_body + .returned_bytes() + .map(|b| BASE64_STANDARD.encode(b)), + }, )), Err(err) => Some(InternalEventPayload::ErrorResponse(ErrorResponse { error: format!("Failed to send HTTP request in CLI: {err}"), diff --git a/crates-tauri/yaak-app-client/src/lib.rs b/crates-tauri/yaak-app-client/src/lib.rs index 14136893..00e976c6 100644 --- a/crates-tauri/yaak-app-client/src/lib.rs +++ b/crates-tauri/yaak-app-client/src/lib.rs @@ -45,7 +45,7 @@ use yaak_plugins::events::{ CallFolderActionArgs, CallFolderActionRequest, CallGrpcRequestActionArgs, CallGrpcRequestActionRequest, CallHttpRequestActionArgs, CallHttpRequestActionRequest, CallWebsocketRequestActionArgs, CallWebsocketRequestActionRequest, CallWorkspaceActionArgs, - CallWorkspaceActionRequest, Color, FilterResponse, GetFolderActionsResponse, + CallWorkspaceActionRequest, Color, ErrorResponse, FilterResponse, GetFolderActionsResponse, GetGrpcRequestActionsResponse, GetHttpAuthenticationConfigResponse, GetHttpAuthenticationSummaryResponse, GetHttpRequestActionsResponse, GetWebsocketRequestActionsResponse, GetWorkspaceActionsResponse, InternalEvent, @@ -1709,6 +1709,7 @@ fn monitor_plugin_events(app_handle: &AppHandle) { let ev = match ev { Ok(Some(ev)) => ev, + // Nothing to say, or the reply comes later from somewhere else. Ok(None) => return, Err(e) => { warn!("Failed to handle plugin event: {e:?}"); @@ -1721,7 +1722,10 @@ fn monitor_plugin_events(app_handle: &AppHandle) { timeout: Some(30000), }), ); - return; + // Tell the plugin as well as the user. It is awaiting a + // reply, and a toast it cannot see would leave it + // waiting for one that never comes. + InternalEventPayload::ErrorResponse(ErrorResponse { error: e.to_string() }) } }; diff --git a/crates-tauri/yaak-app-client/src/plugin_events.rs b/crates-tauri/yaak-app-client/src/plugin_events.rs index 95fb3297..3049a2a2 100644 --- a/crates-tauri/yaak-app-client/src/plugin_events.rs +++ b/crates-tauri/yaak-app-client/src/plugin_events.rs @@ -7,6 +7,8 @@ use crate::{ call_frontend, cookie_jar_from_window, environment_from_window, get_window_from_plugin_context, workspace_from_window, }; +use base64::Engine; +use base64::prelude::BASE64_STANDARD; use chrono::Utc; use log::error; use std::sync::Arc; @@ -16,6 +18,7 @@ use tauri_plugin_opener::OpenerExt; use yaak::plugin_events::{ GroupedPluginEvent, HostRequest, SharedPluginEventContext, handle_shared_plugin_event, }; +use yaak::response_body::FileResponseBodyStore; use yaak_crypto::manager::EncryptionManager; use yaak_http::cookies::get_cookie_value_from_jar; use yaak_models::models::{HttpResponse, Plugin}; @@ -54,6 +57,7 @@ pub(crate) async fn handle_plugin_event( match handle_shared_plugin_event( app_handle.db_manager().inner(), + &FileResponseBodyStore::new(app_handle.db_manager().inner()), &event.payload, SharedPluginEventContext { plugin_name: &plugin_name, @@ -313,8 +317,13 @@ async fn handle_host_plugin_request( ) .await?; + // An ad-hoc request saves nothing, so the engine hands the body + // back and this reply is the only place the plugin can get it. + let body = http_response.body.returned_bytes().map(|b| BASE64_STANDARD.encode(b)); + Ok(Some(InternalEventPayload::SendHttpRequestResponse(SendHttpRequestResponse { http_response: http_response.response, + body, }))) } HostRequest::OpenWindow(req) => { diff --git a/crates/common/yaak-rpc-schema/bindings/gen_models.ts b/crates/common/yaak-rpc-schema/bindings/gen_models.ts index fdab3de6..5ffd2d13 100644 --- a/crates/common/yaak-rpc-schema/bindings/gen_models.ts +++ b/crates/common/yaak-rpc-schema/bindings/gen_models.ts @@ -55,7 +55,7 @@ urlParameters: Array, settingSendCookies: InheritedBoolSetting export type HttpRequestHeader = { enabled?: boolean, name: string, value: string, id?: string, }; -export type HttpResponse = { model: "http_response", id: string, createdAt: string, updatedAt: string, workspaceId: string, requestId: string, bodyPath: string | null, contentLength: number | null, contentLengthCompressed: number | null, elapsed: number, elapsedHeaders: number, elapsedDns: number, error: string | null, headers: Array, remoteAddr: string | null, requestContentLength: number | null, requestHeaders: Array, status: number, statusReason: string | null, state: HttpResponseState, url: string, version: string | null, }; +export type HttpResponse = { model: "http_response", id: string, createdAt: string, updatedAt: string, workspaceId: string, requestId: string, contentLength: number | null, contentLengthCompressed: number | null, elapsed: number, elapsedHeaders: number, elapsedDns: number, error: string | null, headers: Array, remoteAddr: string | null, requestContentLength: number | null, requestHeaders: Array, status: number, statusReason: string | null, state: HttpResponseState, url: string, version: string | null, }; export type HttpResponseEvent = { model: "http_response_event", id: string, createdAt: string, updatedAt: string, workspaceId: string, responseId: string, event: HttpResponseEventData, }; diff --git a/crates/yaak-models/bindings/gen_models.ts b/crates/yaak-models/bindings/gen_models.ts index 6a606587..7c8a71e0 100644 --- a/crates/yaak-models/bindings/gen_models.ts +++ b/crates/yaak-models/bindings/gen_models.ts @@ -225,7 +225,6 @@ export type HttpResponse = { updatedAt: string; workspaceId: string; requestId: string; - bodyPath: string | null; contentLength: number | null; contentLengthCompressed: number | null; elapsed: number; diff --git a/crates/yaak-models/src/models.rs b/crates/yaak-models/src/models.rs index 97f884ee..3a779c8c 100644 --- a/crates/yaak-models/src/models.rs +++ b/crates/yaak-models/src/models.rs @@ -1677,6 +1677,13 @@ pub struct HttpResponse { pub workspace_id: String, pub request_id: String, + /// Where the engine put the body, when it puts it in a file. + /// + /// Not exported to TypeScript: a path is only meaningful to a host that + /// has the filesystem it names, and bodies are moving off it. Read a body + /// by response id instead — the frontend through + /// `cmd_http_response_body_path`, plugins through `ctx.httpResponse.body`. + #[ts(skip)] pub body_path: Option, pub content_length: Option, pub content_length_compressed: Option, diff --git a/crates/yaak-plugins/bindings/gen_events.ts b/crates/yaak-plugins/bindings/gen_events.ts index 0dbd3212..2daff845 100644 --- a/crates/yaak-plugins/bindings/gen_events.ts +++ b/crates/yaak-plugins/bindings/gen_events.ts @@ -416,6 +416,32 @@ export type GetHttpRequestByIdRequest = { id: string, }; export type GetHttpRequestByIdResponse = { httpRequest: HttpRequest | null, }; +/** + * Ask what a response's body is, before deciding whether to pull it. + * + * Bodies are addressed by response id and never by path, so where the host + * keeps the bytes is its own business. + */ +export type GetHttpResponseBodyInfoRequest = { responseId: string, }; + +export type GetHttpResponseBodyInfoResponse = { +/** + * How many bytes are stored right now, which is not necessarily what the + * `Content-Length` header claimed. Zero when the response has no body. + */ +contentLength: number, +/** + * Whether the response has finished arriving. While it has not, the body + * keeps growing past `content_length`, and a reader that wants all of it + * asks again. + */ +complete: boolean, +/** + * The response's `Content-Type` header, verbatim, so the reader can pick a + * charset. + */ +contentType?: string | null, }; + export type GetKeyValueRequest = { key: string, }; export type GetKeyValueResponse = { value?: string, }; @@ -452,7 +478,7 @@ export type ImportResponse = { resources: ImportResources, }; export type InternalEvent = { id: string, pluginRefId: string, pluginName: string, replyId: string | null, context: PluginContext, payload: InternalEventPayload, }; -export type InternalEventPayload = { "type": "boot_request" } & BootRequest | { "type": "boot_response" } | { "type": "reload_response" } & ReloadResponse | { "type": "terminate_request" } | { "type": "terminate_response" } | { "type": "import_request" } & ImportRequest | { "type": "import_response" } & ImportResponse | { "type": "filter_request" } & FilterRequest | { "type": "filter_response" } & FilterResponse | { "type": "export_http_request_request" } & ExportHttpRequestRequest | { "type": "export_http_request_response" } & ExportHttpRequestResponse | { "type": "send_http_request_request" } & SendHttpRequestRequest | { "type": "send_http_request_response" } & SendHttpRequestResponse | { "type": "list_cookie_names_request" } & ListCookieNamesRequest | { "type": "list_cookie_names_response" } & ListCookieNamesResponse | { "type": "get_cookie_value_request" } & GetCookieValueRequest | { "type": "get_cookie_value_response" } & GetCookieValueResponse | { "type": "get_http_request_actions_request" } & EmptyPayload | { "type": "get_http_request_actions_response" } & GetHttpRequestActionsResponse | { "type": "call_http_request_action_request" } & CallHttpRequestActionRequest | { "type": "get_websocket_request_actions_request" } & EmptyPayload | { "type": "get_websocket_request_actions_response" } & GetWebsocketRequestActionsResponse | { "type": "call_websocket_request_action_request" } & CallWebsocketRequestActionRequest | { "type": "get_workspace_actions_request" } & EmptyPayload | { "type": "get_workspace_actions_response" } & GetWorkspaceActionsResponse | { "type": "call_workspace_action_request" } & CallWorkspaceActionRequest | { "type": "get_folder_actions_request" } & EmptyPayload | { "type": "get_folder_actions_response" } & GetFolderActionsResponse | { "type": "call_folder_action_request" } & CallFolderActionRequest | { "type": "get_grpc_request_actions_request" } & EmptyPayload | { "type": "get_grpc_request_actions_response" } & GetGrpcRequestActionsResponse | { "type": "call_grpc_request_action_request" } & CallGrpcRequestActionRequest | { "type": "get_template_function_summary_request" } & EmptyPayload | { "type": "get_template_function_summary_response" } & GetTemplateFunctionSummaryResponse | { "type": "get_template_function_config_request" } & GetTemplateFunctionConfigRequest | { "type": "get_template_function_config_response" } & GetTemplateFunctionConfigResponse | { "type": "call_template_function_request" } & CallTemplateFunctionRequest | { "type": "call_template_function_response" } & CallTemplateFunctionResponse | { "type": "get_http_authentication_summary_request" } & EmptyPayload | { "type": "get_http_authentication_summary_response" } & GetHttpAuthenticationSummaryResponse | { "type": "get_http_authentication_config_request" } & GetHttpAuthenticationConfigRequest | { "type": "get_http_authentication_config_response" } & GetHttpAuthenticationConfigResponse | { "type": "call_http_authentication_request" } & CallHttpAuthenticationRequest | { "type": "call_http_authentication_response" } & CallHttpAuthenticationResponse | { "type": "call_http_authentication_action_request" } & CallHttpAuthenticationActionRequest | { "type": "call_http_authentication_action_response" } & EmptyPayload | { "type": "copy_text_request" } & CopyTextRequest | { "type": "copy_text_response" } & EmptyPayload | { "type": "render_http_request_request" } & RenderHttpRequestRequest | { "type": "render_http_request_response" } & RenderHttpRequestResponse | { "type": "render_grpc_request_request" } & RenderGrpcRequestRequest | { "type": "render_grpc_request_response" } & RenderGrpcRequestResponse | { "type": "template_render_request" } & TemplateRenderRequest | { "type": "template_render_response" } & TemplateRenderResponse | { "type": "get_key_value_request" } & GetKeyValueRequest | { "type": "get_key_value_response" } & GetKeyValueResponse | { "type": "set_key_value_request" } & SetKeyValueRequest | { "type": "set_key_value_response" } & SetKeyValueResponse | { "type": "delete_key_value_request" } & DeleteKeyValueRequest | { "type": "delete_key_value_response" } & DeleteKeyValueResponse | { "type": "open_window_request" } & OpenWindowRequest | { "type": "window_navigate_event" } & WindowNavigateEvent | { "type": "window_close_event" } | { "type": "close_window_request" } & CloseWindowRequest | { "type": "open_external_url_request" } & OpenExternalUrlRequest | { "type": "open_external_url_response" } & EmptyPayload | { "type": "show_toast_request" } & ShowToastRequest | { "type": "show_toast_response" } & EmptyPayload | { "type": "prompt_text_request" } & PromptTextRequest | { "type": "prompt_text_response" } & PromptTextResponse | { "type": "prompt_form_request" } & PromptFormRequest | { "type": "prompt_form_response" } & PromptFormResponse | { "type": "window_info_request" } & WindowInfoRequest | { "type": "window_info_response" } & WindowInfoResponse | { "type": "list_open_workspaces_request" } & ListOpenWorkspacesRequest | { "type": "list_open_workspaces_response" } & ListOpenWorkspacesResponse | { "type": "get_http_request_by_id_request" } & GetHttpRequestByIdRequest | { "type": "get_http_request_by_id_response" } & GetHttpRequestByIdResponse | { "type": "find_http_responses_request" } & FindHttpResponsesRequest | { "type": "find_http_responses_response" } & FindHttpResponsesResponse | { "type": "list_http_requests_request" } & ListHttpRequestsRequest | { "type": "list_http_requests_response" } & ListHttpRequestsResponse | { "type": "list_folders_request" } & ListFoldersRequest | { "type": "list_folders_response" } & ListFoldersResponse | { "type": "upsert_model_request" } & UpsertModelRequest | { "type": "upsert_model_response" } & UpsertModelResponse | { "type": "delete_model_request" } & DeleteModelRequest | { "type": "delete_model_response" } & DeleteModelResponse | { "type": "get_themes_request" } & GetThemesRequest | { "type": "get_themes_response" } & GetThemesResponse | { "type": "empty_response" } & EmptyPayload | { "type": "error_response" } & ErrorResponse; +export type InternalEventPayload = { "type": "boot_request" } & BootRequest | { "type": "boot_response" } | { "type": "reload_response" } & ReloadResponse | { "type": "terminate_request" } | { "type": "terminate_response" } | { "type": "import_request" } & ImportRequest | { "type": "import_response" } & ImportResponse | { "type": "filter_request" } & FilterRequest | { "type": "filter_response" } & FilterResponse | { "type": "export_http_request_request" } & ExportHttpRequestRequest | { "type": "export_http_request_response" } & ExportHttpRequestResponse | { "type": "send_http_request_request" } & SendHttpRequestRequest | { "type": "send_http_request_response" } & SendHttpRequestResponse | { "type": "list_cookie_names_request" } & ListCookieNamesRequest | { "type": "list_cookie_names_response" } & ListCookieNamesResponse | { "type": "get_cookie_value_request" } & GetCookieValueRequest | { "type": "get_cookie_value_response" } & GetCookieValueResponse | { "type": "get_http_request_actions_request" } & EmptyPayload | { "type": "get_http_request_actions_response" } & GetHttpRequestActionsResponse | { "type": "call_http_request_action_request" } & CallHttpRequestActionRequest | { "type": "get_websocket_request_actions_request" } & EmptyPayload | { "type": "get_websocket_request_actions_response" } & GetWebsocketRequestActionsResponse | { "type": "call_websocket_request_action_request" } & CallWebsocketRequestActionRequest | { "type": "get_workspace_actions_request" } & EmptyPayload | { "type": "get_workspace_actions_response" } & GetWorkspaceActionsResponse | { "type": "call_workspace_action_request" } & CallWorkspaceActionRequest | { "type": "get_folder_actions_request" } & EmptyPayload | { "type": "get_folder_actions_response" } & GetFolderActionsResponse | { "type": "call_folder_action_request" } & CallFolderActionRequest | { "type": "get_grpc_request_actions_request" } & EmptyPayload | { "type": "get_grpc_request_actions_response" } & GetGrpcRequestActionsResponse | { "type": "call_grpc_request_action_request" } & CallGrpcRequestActionRequest | { "type": "get_template_function_summary_request" } & EmptyPayload | { "type": "get_template_function_summary_response" } & GetTemplateFunctionSummaryResponse | { "type": "get_template_function_config_request" } & GetTemplateFunctionConfigRequest | { "type": "get_template_function_config_response" } & GetTemplateFunctionConfigResponse | { "type": "call_template_function_request" } & CallTemplateFunctionRequest | { "type": "call_template_function_response" } & CallTemplateFunctionResponse | { "type": "get_http_authentication_summary_request" } & EmptyPayload | { "type": "get_http_authentication_summary_response" } & GetHttpAuthenticationSummaryResponse | { "type": "get_http_authentication_config_request" } & GetHttpAuthenticationConfigRequest | { "type": "get_http_authentication_config_response" } & GetHttpAuthenticationConfigResponse | { "type": "call_http_authentication_request" } & CallHttpAuthenticationRequest | { "type": "call_http_authentication_response" } & CallHttpAuthenticationResponse | { "type": "call_http_authentication_action_request" } & CallHttpAuthenticationActionRequest | { "type": "call_http_authentication_action_response" } & EmptyPayload | { "type": "copy_text_request" } & CopyTextRequest | { "type": "copy_text_response" } & EmptyPayload | { "type": "render_http_request_request" } & RenderHttpRequestRequest | { "type": "render_http_request_response" } & RenderHttpRequestResponse | { "type": "render_grpc_request_request" } & RenderGrpcRequestRequest | { "type": "render_grpc_request_response" } & RenderGrpcRequestResponse | { "type": "template_render_request" } & TemplateRenderRequest | { "type": "template_render_response" } & TemplateRenderResponse | { "type": "get_key_value_request" } & GetKeyValueRequest | { "type": "get_key_value_response" } & GetKeyValueResponse | { "type": "set_key_value_request" } & SetKeyValueRequest | { "type": "set_key_value_response" } & SetKeyValueResponse | { "type": "delete_key_value_request" } & DeleteKeyValueRequest | { "type": "delete_key_value_response" } & DeleteKeyValueResponse | { "type": "open_window_request" } & OpenWindowRequest | { "type": "window_navigate_event" } & WindowNavigateEvent | { "type": "window_close_event" } | { "type": "close_window_request" } & CloseWindowRequest | { "type": "open_external_url_request" } & OpenExternalUrlRequest | { "type": "open_external_url_response" } & EmptyPayload | { "type": "show_toast_request" } & ShowToastRequest | { "type": "show_toast_response" } & EmptyPayload | { "type": "prompt_text_request" } & PromptTextRequest | { "type": "prompt_text_response" } & PromptTextResponse | { "type": "prompt_form_request" } & PromptFormRequest | { "type": "prompt_form_response" } & PromptFormResponse | { "type": "window_info_request" } & WindowInfoRequest | { "type": "window_info_response" } & WindowInfoResponse | { "type": "list_open_workspaces_request" } & ListOpenWorkspacesRequest | { "type": "list_open_workspaces_response" } & ListOpenWorkspacesResponse | { "type": "get_http_request_by_id_request" } & GetHttpRequestByIdRequest | { "type": "get_http_request_by_id_response" } & GetHttpRequestByIdResponse | { "type": "find_http_responses_request" } & FindHttpResponsesRequest | { "type": "find_http_responses_response" } & FindHttpResponsesResponse | { "type": "get_http_response_body_info_request" } & GetHttpResponseBodyInfoRequest | { "type": "get_http_response_body_info_response" } & GetHttpResponseBodyInfoResponse | { "type": "read_http_response_body_chunk_request" } & ReadHttpResponseBodyChunkRequest | { "type": "read_http_response_body_chunk_response" } & ReadHttpResponseBodyChunkResponse | { "type": "list_http_requests_request" } & ListHttpRequestsRequest | { "type": "list_http_requests_response" } & ListHttpRequestsResponse | { "type": "list_folders_request" } & ListFoldersRequest | { "type": "list_folders_response" } & ListFoldersResponse | { "type": "upsert_model_request" } & UpsertModelRequest | { "type": "upsert_model_response" } & UpsertModelResponse | { "type": "delete_model_request" } & DeleteModelRequest | { "type": "delete_model_response" } & DeleteModelResponse | { "type": "get_themes_request" } & GetThemesRequest | { "type": "get_themes_response" } & GetThemesResponse | { "type": "empty_response" } & EmptyPayload | { "type": "error_response" } & ErrorResponse; export type JsonPrimitive = string | number | boolean | null; @@ -502,6 +528,27 @@ required?: boolean, }; export type PromptTextResponse = { value: string | null, }; +/** + * Pull one window of a response body. + * + * Reads are idempotent: the bytes live in durable storage, so the same window + * can be asked for as many times as the plugin likes. + */ +export type ReadHttpResponseBodyChunkRequest = { responseId: string, offset: number, length: number, }; + +export type ReadHttpResponseBodyChunkResponse = { +/** + * Base64, because the desktop transport is a WebSocket that only sends + * text frames today. A host that can carry binary sends the bytes as they + * are and fills this in from them. + */ +data: string, +/** + * Bytes decoded from `data`. Short of the requested length means the body + * ended here. + */ +length: number, }; + export type ReloadResponse = { silent: boolean, }; export type RenderGrpcRequestRequest = { grpcRequest: GrpcRequest, purpose: RenderPurpose, }; @@ -516,7 +563,16 @@ export type RenderPurpose = "send" | "preview"; export type SendHttpRequestRequest = { httpRequest: Partial, }; -export type SendHttpRequestResponse = { httpResponse: HttpResponse, }; +export type SendHttpRequestResponse = { httpResponse: HttpResponse, +/** + * The body, base64, when the send saved nothing. + * + * A request with no id behind it produces a response the model store never + * sees, so it cannot be read back by id later the way a saved one can. + * This is the only copy of it. `None` means the body was stored and should + * be read with `read_http_response_body_chunk_request`. + */ +body?: string | null, }; export type SetKeyValueRequest = { key: string, value: string, }; diff --git a/crates/yaak-plugins/bindings/gen_models.ts b/crates/yaak-plugins/bindings/gen_models.ts index d0ba2f1d..2f3542af 100644 --- a/crates/yaak-plugins/bindings/gen_models.ts +++ b/crates/yaak-plugins/bindings/gen_models.ts @@ -224,7 +224,6 @@ export type HttpResponse = { updatedAt: string; workspaceId: string; requestId: string; - bodyPath: string | null; contentLength: number | null; contentLengthCompressed: number | null; elapsed: number; diff --git a/crates/yaak-plugins/src/events.rs b/crates/yaak-plugins/src/events.rs index 5fd480e5..5e97e8e1 100644 --- a/crates/yaak-plugins/src/events.rs +++ b/crates/yaak-plugins/src/events.rs @@ -171,6 +171,12 @@ pub enum InternalEventPayload { FindHttpResponsesRequest(FindHttpResponsesRequest), FindHttpResponsesResponse(FindHttpResponsesResponse), + + GetHttpResponseBodyInfoRequest(GetHttpResponseBodyInfoRequest), + GetHttpResponseBodyInfoResponse(GetHttpResponseBodyInfoResponse), + ReadHttpResponseBodyChunkRequest(ReadHttpResponseBodyChunkRequest), + ReadHttpResponseBodyChunkResponse(ReadHttpResponseBodyChunkResponse), + ListHttpRequestsRequest(ListHttpRequestsRequest), ListHttpRequestsResponse(ListHttpRequestsResponse), ListFoldersRequest(ListFoldersRequest), @@ -288,6 +294,15 @@ pub struct SendHttpRequestRequest { #[ts(export, export_to = "gen_events.ts")] pub struct SendHttpRequestResponse { pub http_response: HttpResponse, + + /// The body, base64, when the send saved nothing. + /// + /// A request with no id behind it produces a response the model store never + /// sees, so it cannot be read back by id later the way a saved one can. + /// This is the only copy of it. `None` means the body was stored and should + /// be read with `read_http_response_body_chunk_request`. + #[ts(optional = nullable)] + pub body: Option, } #[derive(Debug, Clone, Default, Serialize, Deserialize, TS)] @@ -1413,6 +1428,67 @@ pub struct FindHttpResponsesResponse { pub http_responses: Vec, } +/// Ask what a response's body is, before deciding whether to pull it. +/// +/// Bodies are addressed by response id and never by path, so where the host +/// keeps the bytes is its own business. +#[derive(Debug, Clone, Default, Serialize, Deserialize, TS)] +#[serde(default, rename_all = "camelCase")] +#[ts(export, export_to = "gen_events.ts")] +pub struct GetHttpResponseBodyInfoRequest { + pub response_id: String, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize, TS)] +#[serde(default, rename_all = "camelCase")] +#[ts(export, export_to = "gen_events.ts")] +pub struct GetHttpResponseBodyInfoResponse { + /// How many bytes are stored right now, which is not necessarily what the + /// `Content-Length` header claimed. Zero when the response has no body. + #[ts(type = "number")] + pub content_length: u64, + + /// Whether the response has finished arriving. While it has not, the body + /// keeps growing past `content_length`, and a reader that wants all of it + /// asks again. + pub complete: bool, + + /// The response's `Content-Type` header, verbatim, so the reader can pick a + /// charset. + #[ts(optional = nullable)] + pub content_type: Option, +} + +/// Pull one window of a response body. +/// +/// Reads are idempotent: the bytes live in durable storage, so the same window +/// can be asked for as many times as the plugin likes. +#[derive(Debug, Clone, Default, Serialize, Deserialize, TS)] +#[serde(default, rename_all = "camelCase")] +#[ts(export, export_to = "gen_events.ts")] +pub struct ReadHttpResponseBodyChunkRequest { + pub response_id: String, + #[ts(type = "number")] + pub offset: u64, + #[ts(type = "number")] + pub length: u64, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize, TS)] +#[serde(default, rename_all = "camelCase")] +#[ts(export, export_to = "gen_events.ts")] +pub struct ReadHttpResponseBodyChunkResponse { + /// Base64, because the desktop transport is a WebSocket that only sends + /// text frames today. A host that can carry binary sends the bytes as they + /// are and fills this in from them. + pub data: String, + + /// Bytes decoded from `data`. Short of the requested length means the body + /// ended here. + #[ts(type = "number")] + pub length: u64, +} + #[derive(Debug, Clone, Default, Serialize, Deserialize, TS)] #[serde(default, rename_all = "camelCase")] #[ts(export, export_to = "gen_events.ts")] diff --git a/crates/yaak/Cargo.toml b/crates/yaak/Cargo.toml index b0af1b57..03b8e161 100644 --- a/crates/yaak/Cargo.toml +++ b/crates/yaak/Cargo.toml @@ -6,6 +6,7 @@ publish = false [dependencies] async-trait = "0.1" +base64 = "0.22.1" # For carrying body chunks over a text-only plugin transport log = { workspace = true } md5 = "0.8.0" serde_json = { workspace = true } diff --git a/crates/yaak/src/lib.rs b/crates/yaak/src/lib.rs index 7bc790e8..e21be7e3 100644 --- a/crates/yaak/src/lib.rs +++ b/crates/yaak/src/lib.rs @@ -3,6 +3,7 @@ pub mod export; pub mod import; pub mod plugin_events; pub mod render; +pub mod response_body; pub mod send; pub use error::Error; diff --git a/crates/yaak/src/plugin_events.rs b/crates/yaak/src/plugin_events.rs index 96e2e4df..1bb8bba9 100644 --- a/crates/yaak/src/plugin_events.rs +++ b/crates/yaak/src/plugin_events.rs @@ -1,3 +1,6 @@ +use crate::response_body::ResponseBodyStore; +use base64::Engine; +use base64::prelude::BASE64_STANDARD; use yaak_models::models::AnyModel; use yaak_models::query_manager::QueryManager; use yaak_models::util::UpdateSource; @@ -5,12 +8,14 @@ use yaak_plugins::events::{ CloseWindowRequest, CopyTextRequest, DeleteKeyValueRequest, DeleteKeyValueResponse, DeleteModelRequest, DeleteModelResponse, ErrorResponse, FindHttpResponsesRequest, FindHttpResponsesResponse, GetCookieValueRequest, GetHttpRequestByIdRequest, - GetHttpRequestByIdResponse, GetKeyValueRequest, GetKeyValueResponse, InternalEventPayload, - ListCookieNamesRequest, ListFoldersRequest, ListFoldersResponse, ListHttpRequestsRequest, - ListHttpRequestsResponse, ListOpenWorkspacesRequest, OpenExternalUrlRequest, OpenWindowRequest, - PromptFormRequest, PromptTextRequest, ReloadResponse, RenderGrpcRequestRequest, - RenderHttpRequestRequest, SendHttpRequestRequest, SetKeyValueRequest, ShowToastRequest, - TemplateRenderRequest, UpsertModelRequest, UpsertModelResponse, WindowInfoRequest, + GetHttpRequestByIdResponse, GetHttpResponseBodyInfoRequest, GetHttpResponseBodyInfoResponse, + GetKeyValueRequest, GetKeyValueResponse, InternalEventPayload, ListCookieNamesRequest, + ListFoldersRequest, ListFoldersResponse, ListHttpRequestsRequest, ListHttpRequestsResponse, + ListOpenWorkspacesRequest, OpenExternalUrlRequest, OpenWindowRequest, PromptFormRequest, + PromptTextRequest, ReadHttpResponseBodyChunkRequest, ReadHttpResponseBodyChunkResponse, + ReloadResponse, RenderGrpcRequestRequest, RenderHttpRequestRequest, SendHttpRequestRequest, + SetKeyValueRequest, ShowToastRequest, TemplateRenderRequest, UpsertModelRequest, + UpsertModelResponse, WindowInfoRequest, }; pub struct SharedPluginEventContext<'a> { @@ -40,6 +45,8 @@ pub enum SharedRequest<'a> { ListFolders(&'a ListFoldersRequest), ListHttpRequests(&'a ListHttpRequestsRequest), FindHttpResponses(&'a FindHttpResponsesRequest), + GetHttpResponseBodyInfo(&'a GetHttpResponseBodyInfoRequest), + ReadHttpResponseBodyChunk(&'a ReadHttpResponseBodyChunkRequest), UpsertModel(&'a UpsertModelRequest), DeleteModel(&'a DeleteModelRequest), } @@ -136,6 +143,12 @@ impl<'a> From<&'a InternalEventPayload> for GroupedPluginRequest<'a> { InternalEventPayload::FindHttpResponsesRequest(req) => { GroupedPluginRequest::Shared(SharedRequest::FindHttpResponses(req)) } + InternalEventPayload::GetHttpResponseBodyInfoRequest(req) => { + GroupedPluginRequest::Shared(SharedRequest::GetHttpResponseBodyInfo(req)) + } + InternalEventPayload::ReadHttpResponseBodyChunkRequest(req) => { + GroupedPluginRequest::Shared(SharedRequest::ReadHttpResponseBodyChunk(req)) + } InternalEventPayload::UpsertModelRequest(req) => { GroupedPluginRequest::Shared(SharedRequest::UpsertModel(req)) } @@ -182,13 +195,17 @@ impl<'a> From<&'a InternalEventPayload> for GroupedPluginRequest<'a> { pub fn handle_shared_plugin_event<'a>( query_manager: &QueryManager, + body_store: &dyn ResponseBodyStore, payload: &'a InternalEventPayload, context: SharedPluginEventContext<'_>, ) -> GroupedPluginEvent<'a> { match GroupedPluginRequest::from(payload) { - GroupedPluginRequest::Shared(req) => { - GroupedPluginEvent::Handled(Some(build_shared_reply(query_manager, req, context))) - } + GroupedPluginRequest::Shared(req) => GroupedPluginEvent::Handled(Some(build_shared_reply( + query_manager, + body_store, + req, + context, + ))), GroupedPluginRequest::Host(req) => GroupedPluginEvent::ToHandle(req), GroupedPluginRequest::Ignore => GroupedPluginEvent::Handled(None), } @@ -196,6 +213,7 @@ pub fn handle_shared_plugin_event<'a>( fn build_shared_reply( query_manager: &QueryManager, + body_store: &dyn ResponseBodyStore, request: SharedRequest<'_>, context: SharedPluginEventContext<'_>, ) -> InternalEventPayload { @@ -283,6 +301,31 @@ fn build_shared_reply( http_responses, }) } + SharedRequest::GetHttpResponseBodyInfo(req) => match body_store.info(&req.response_id) { + Ok(info) => InternalEventPayload::GetHttpResponseBodyInfoResponse( + GetHttpResponseBodyInfoResponse { + content_length: info.content_length, + content_type: info.content_type, + complete: info.complete, + }, + ), + Err(err) => InternalEventPayload::ErrorResponse(ErrorResponse { + error: format!("Failed to read body of response {}: {err}", req.response_id), + }), + }, + SharedRequest::ReadHttpResponseBodyChunk(req) => { + match body_store.read_chunk(&req.response_id, req.offset, req.length) { + Ok(bytes) => InternalEventPayload::ReadHttpResponseBodyChunkResponse( + ReadHttpResponseBodyChunkResponse { + length: bytes.len() as u64, + data: BASE64_STANDARD.encode(bytes), + }, + ), + Err(err) => InternalEventPayload::ErrorResponse(ErrorResponse { + error: format!("Failed to read body of response {}: {err}", req.response_id), + }), + } + } SharedRequest::UpsertModel(req) => { use AnyModel::*; @@ -437,10 +480,26 @@ fn build_shared_reply( #[cfg(test)] mod tests { use super::*; + use crate::response_body::{FileResponseBodyStore, ResponseBodyInfo}; + use std::cell::RefCell; use tempfile::TempDir; use yaak_models::models::{AnyModel, Folder, HttpRequest, Workspace}; use yaak_models::util::UpdateSource; + /// The real dispatch, with the store the desktop and CLI hand it. + fn dispatch<'a>( + query_manager: &QueryManager, + payload: &'a InternalEventPayload, + context: SharedPluginEventContext<'_>, + ) -> GroupedPluginEvent<'a> { + handle_shared_plugin_event( + query_manager, + &FileResponseBodyStore::new(query_manager), + payload, + context, + ) + } + fn seed_query_manager() -> (QueryManager, TempDir) { let temp_dir = TempDir::new().expect("Failed to create temp dir"); let db_path = temp_dir.path().join("db.sqlite"); @@ -498,7 +557,7 @@ mod tests { let payload = InternalEventPayload::ListHttpRequestsRequest( yaak_plugins::events::ListHttpRequestsRequest { folder_id: None }, ); - let result = handle_shared_plugin_event( + let result = dispatch( &query_manager, &payload, SharedPluginEventContext { plugin_name: "@yaak/test", workspace_id: None }, @@ -517,7 +576,7 @@ mod tests { let by_workspace_payload = InternalEventPayload::ListHttpRequestsRequest( yaak_plugins::events::ListHttpRequestsRequest { folder_id: None }, ); - let by_workspace = handle_shared_plugin_event( + let by_workspace = dispatch( &query_manager, &by_workspace_payload, SharedPluginEventContext { plugin_name: "@yaak/test", workspace_id: Some("wk_test") }, @@ -536,7 +595,7 @@ mod tests { folder_id: Some("fl_test".to_string()), }, ); - let by_folder = handle_shared_plugin_event( + let by_folder = dispatch( &query_manager, &by_folder_payload, SharedPluginEventContext { plugin_name: "@yaak/test", workspace_id: None }, @@ -559,7 +618,7 @@ mod tests { limit: Some(1), }); - let result = handle_shared_plugin_event( + let result = dispatch( &query_manager, &payload, SharedPluginEventContext { plugin_name: "@yaak/test", workspace_id: Some("wk_test") }, @@ -575,6 +634,105 @@ mod tests { } } + /// A store that answers from memory, standing in for whatever holds the + /// bytes — the point being that the dispatch below never learns which. + struct FakeBodyStore { + body: Vec, + reads: RefCell>, + } + + impl ResponseBodyStore for FakeBodyStore { + fn info(&self, _response_id: &str) -> crate::error::Result { + Ok(ResponseBodyInfo { + content_length: self.body.len() as u64, + content_type: Some("text/plain; charset=utf-8".to_string()), + complete: true, + }) + } + + fn read_chunk( + &self, + _response_id: &str, + offset: u64, + length: u64, + ) -> crate::error::Result> { + self.reads.borrow_mut().push((offset, length)); + let start = (offset as usize).min(self.body.len()); + let end = (start + length as usize).min(self.body.len()); + Ok(self.body[start..end].to_vec()) + } + } + + #[test] + fn response_body_is_read_by_id_through_the_store() { + let (query_manager, _temp_dir) = seed_query_manager(); + let store = FakeBodyStore { body: b"hello".to_vec(), reads: RefCell::new(Vec::new()) }; + + let info_payload = InternalEventPayload::GetHttpResponseBodyInfoRequest( + GetHttpResponseBodyInfoRequest { response_id: "rs_test".to_string() }, + ); + let info = handle_shared_plugin_event( + &query_manager, + &store, + &info_payload, + SharedPluginEventContext { plugin_name: "@yaak/test", workspace_id: None }, + ); + match info { + GroupedPluginEvent::Handled(Some( + InternalEventPayload::GetHttpResponseBodyInfoResponse(resp), + )) => { + assert_eq!(resp.content_length, 5); + assert_eq!(resp.content_type.as_deref(), Some("text/plain; charset=utf-8")); + } + other => panic!("unexpected body info result: {other:?}"), + } + + let chunk_payload = InternalEventPayload::ReadHttpResponseBodyChunkRequest( + ReadHttpResponseBodyChunkRequest { + response_id: "rs_test".to_string(), + offset: 1, + length: 3, + }, + ); + let chunk = handle_shared_plugin_event( + &query_manager, + &store, + &chunk_payload, + SharedPluginEventContext { plugin_name: "@yaak/test", workspace_id: None }, + ); + match chunk { + GroupedPluginEvent::Handled(Some( + InternalEventPayload::ReadHttpResponseBodyChunkResponse(resp), + )) => { + assert_eq!(resp.length, 3); + assert_eq!(BASE64_STANDARD.decode(resp.data).unwrap(), b"ell"); + } + other => panic!("unexpected body chunk result: {other:?}"), + } + + assert_eq!(*store.reads.borrow(), vec![(1, 3)]); + } + + #[test] + fn an_unreadable_response_body_becomes_an_error_reply() { + let (query_manager, _temp_dir) = seed_query_manager(); + let payload = InternalEventPayload::GetHttpResponseBodyInfoRequest( + GetHttpResponseBodyInfoRequest { response_id: "rs_never_persisted".to_string() }, + ); + let result = dispatch( + &query_manager, + &payload, + SharedPluginEventContext { plugin_name: "@yaak/test", workspace_id: None }, + ); + + match result { + GroupedPluginEvent::Handled(Some(InternalEventPayload::ErrorResponse(resp))) => { + assert!(resp.error.contains("rs_never_persisted"), "unhelpful error: {}", resp.error) + } + other => panic!("unexpected missing-response result: {other:?}"), + } + } + #[test] fn upsert_and_delete_model_are_shared_handled() { let (query_manager, _temp_dir) = seed_query_manager(); @@ -590,7 +748,7 @@ mod tests { }), }); - let upsert_result = handle_shared_plugin_event( + let upsert_result = dispatch( &query_manager, &upsert_payload, SharedPluginEventContext { plugin_name: "@yaak/test", workspace_id: Some("wk_test") }, @@ -609,7 +767,7 @@ mod tests { model: "http_request".to_string(), id: "rq_test".to_string(), }); - let delete_result = handle_shared_plugin_event( + let delete_result = dispatch( &query_manager, &delete_payload, SharedPluginEventContext { plugin_name: "@yaak/test", workspace_id: Some("wk_test") }, @@ -631,7 +789,7 @@ mod tests { let payload = InternalEventPayload::WindowInfoRequest(WindowInfoRequest { label: "main".to_string(), }); - let result = handle_shared_plugin_event( + let result = dispatch( &query_manager, &payload, SharedPluginEventContext { plugin_name: "@yaak/test", workspace_id: None }, diff --git a/crates/yaak/src/response_body.rs b/crates/yaak/src/response_body.rs new file mode 100644 index 00000000..38c9daab --- /dev/null +++ b/crates/yaak/src/response_body.rs @@ -0,0 +1,232 @@ +//! Reading response bodies back out, by response id. +//! +//! Plugins only ever name a response. Where its bytes actually live — files the +//! engine wrote under `/responses/` today, blob rows later — is +//! behind [`ResponseBodyStore`], so moving the bytes is a change to this file +//! and nothing a plugin can see. +//! +//! Only saved responses are reachable by id. A send that saved nothing hands +//! its body back with the reply instead, which is the only copy of it there is. + +use crate::error::Result; +use std::fs::File; +use std::io::{Read, Seek, SeekFrom}; +use yaak_models::models::HttpResponseState; +use yaak_models::query_manager::QueryManager; + +/// The most bytes one read will hand back, however much was asked for. +/// +/// A chunk is buffered whole and, on the desktop transport, base64'd into a +/// single WebSocket frame, so an unbounded request is a way to make the host +/// allocate on a plugin's say-so. +pub const MAX_CHUNK_BYTES: u64 = 8 * 1024 * 1024; + +/// What a stored body is, without reading any of it. +#[derive(Debug, Clone, Default)] +pub struct ResponseBodyInfo { + /// Bytes actually stored, which is not necessarily what `Content-Length` + /// claimed. Zero when the response has no body. + pub content_length: u64, + /// The response's `Content-Type` header, verbatim. + pub content_type: Option, + /// Whether the response has finished arriving, so `content_length` is + /// final. A body still being written grows past it. + pub complete: bool, +} + +/// Somewhere response bodies can be read from, a window at a time. +/// +/// Reads are repeatable — the bytes are durable, so nothing is consumed by +/// looking at it. +pub trait ResponseBodyStore { + fn info(&self, response_id: &str) -> Result; + + /// Bytes `[offset, offset + length)`, clamped to what is there. A short + /// read means the body ended. + fn read_chunk(&self, response_id: &str, offset: u64, length: u64) -> Result>; +} + +/// The desktop and CLI store: the database says where the file is, and the +/// filesystem holds it. +pub struct FileResponseBodyStore<'a> { + query_manager: &'a QueryManager, +} + +impl<'a> FileResponseBodyStore<'a> { + pub fn new(query_manager: &'a QueryManager) -> Self { + Self { query_manager } + } + + /// The file backing a response, or `None` when it stored no body. + /// + /// Only responses the store knows about are reachable here. A send with no + /// request behind it never reaches the store at all, and its bytes come + /// back from the send instead — see `SendHttpRequestResponse::body`. + fn body_path(&self, response_id: &str) -> Result> { + Ok(self.query_manager.connect().get_http_response(response_id)?.body_path) + } +} + +impl ResponseBodyStore for FileResponseBodyStore<'_> { + fn info(&self, response_id: &str) -> Result { + let response = self.query_manager.connect().get_http_response(response_id)?; + + let content_type = response + .headers + .iter() + .find(|h| h.name.eq_ignore_ascii_case("content-type")) + .map(|h| h.value.clone()); + + let content_length = match response.body_path { + Some(path) => std::fs::metadata(path)?.len(), + None => 0, + }; + + Ok(ResponseBodyInfo { + content_length, + content_type, + // Closed is the one terminal state: success, error, and cancel all end there. + complete: matches!(response.state, HttpResponseState::Closed), + }) + } + + fn read_chunk(&self, response_id: &str, offset: u64, length: u64) -> Result> { + let Some(path) = self.body_path(response_id)? else { + return Ok(Vec::new()); + }; + + let length = length.min(MAX_CHUNK_BYTES); + if length == 0 { + return Ok(Vec::new()); + } + + let mut file = File::open(path)?; + file.seek(SeekFrom::Start(offset))?; + + let mut buf = Vec::new(); + file.take(length).read_to_end(&mut buf)?; + Ok(buf) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + use tempfile::TempDir; + use yaak_models::models::{HttpRequest, HttpResponse, HttpResponseHeader, Workspace}; + use yaak_models::util::UpdateSource; + + fn seed(body: Option<&[u8]>) -> (QueryManager, TempDir, String) { + let temp_dir = TempDir::new().unwrap(); + let (query_manager, blob_manager, _rx) = yaak_models::init_standalone( + &temp_dir.path().join("db.sqlite"), + &temp_dir.path().join("blobs.sqlite"), + ) + .unwrap(); + + query_manager + .connect() + .upsert_workspace( + &Workspace { id: "wk_test".to_string(), ..Default::default() }, + &UpdateSource::Sync, + ) + .unwrap(); + + query_manager + .connect() + .upsert_http_request( + &HttpRequest { + id: "rq_test".to_string(), + workspace_id: "wk_test".to_string(), + ..Default::default() + }, + &UpdateSource::Sync, + ) + .unwrap(); + + let body_path = body.map(|bytes| { + let path = temp_dir.path().join("body"); + let mut f = std::fs::File::create(&path).unwrap(); + f.write_all(bytes).unwrap(); + path.to_string_lossy().to_string() + }); + + let response = query_manager + .connect() + .upsert_http_response( + &HttpResponse { + workspace_id: "wk_test".to_string(), + request_id: "rq_test".to_string(), + body_path, + headers: vec![HttpResponseHeader { + name: "Content-Type".to_string(), + value: "application/json; charset=utf-8".to_string(), + }], + ..Default::default() + }, + &UpdateSource::Sync, + &blob_manager, + ) + .unwrap(); + + let id = response.id.clone(); + (query_manager, temp_dir, id) + } + + #[test] + fn info_reports_stored_size_and_content_type() { + let (qm, _tmp, id) = seed(Some(b"hello world")); + let info = FileResponseBodyStore::new(&qm).info(&id).unwrap(); + assert_eq!(info.content_length, 11); + assert_eq!(info.content_type.as_deref(), Some("application/json; charset=utf-8")); + } + + #[test] + fn chunks_cover_the_body_and_stop_short_at_the_end() { + let (qm, _tmp, id) = seed(Some(b"hello world")); + let store = FileResponseBodyStore::new(&qm); + assert_eq!(store.read_chunk(&id, 0, 5).unwrap(), b"hello"); + assert_eq!(store.read_chunk(&id, 6, 100).unwrap(), b"world"); + assert!(store.read_chunk(&id, 11, 100).unwrap().is_empty()); + // Reading the same window twice gives the same bytes; nothing is consumed. + assert_eq!(store.read_chunk(&id, 0, 5).unwrap(), b"hello"); + } + + #[test] + fn a_response_with_no_body_is_empty_not_an_error() { + let (qm, _tmp, id) = seed(None); + let store = FileResponseBodyStore::new(&qm); + assert_eq!(store.info(&id).unwrap().content_length, 0); + assert!(store.read_chunk(&id, 0, 100).unwrap().is_empty()); + } + + #[test] + fn complete_tracks_whether_the_response_has_closed() { + let (qm, _tmp, id) = seed(Some(b"partial")); + // Seeded responses default to Initialized: still arriving. + assert!(!FileResponseBodyStore::new(&qm).info(&id).unwrap().complete); + + let mut response = qm.connect().get_http_response(&id).unwrap(); + response.state = HttpResponseState::Closed; + qm.connect().update_http_response_if_id(&response, &UpdateSource::Sync).unwrap(); + + assert!(FileResponseBodyStore::new(&qm).info(&id).unwrap().complete); + } + + #[test] + fn an_unknown_response_fails() { + let (qm, _tmp, _id) = seed(Some(b"hi")); + assert!(FileResponseBodyStore::new(&qm).info("rs_nope").is_err()); + } + + #[test] + fn an_unsaved_response_is_not_reachable_by_id() { + // Its bytes rode back with the send; there is nothing here to find, and + // guessing at a file named for the id is exactly what this must not do. + let (qm, tmp, _id) = seed(Some(b"hi")); + std::fs::write(tmp.path().join("rs_ephemeral1"), b"access_token=abc").unwrap(); + + assert!(FileResponseBodyStore::new(&qm).info("rs_ephemeral1").is_err()); + } +} diff --git a/crates/yaak/src/send.rs b/crates/yaak/src/send.rs index 3b70c4df..bcd61386 100644 --- a/crates/yaak/src/send.rs +++ b/crates/yaak/src/send.rs @@ -354,6 +354,19 @@ pub enum ResponseBody { Returned(Vec), } +impl ResponseBody { + /// The bytes, when this is the only copy of them. + /// + /// Stored and streamed bodies belong to whoever holds them; only `Returned` + /// has to travel back to the caller. + pub fn returned_bytes(&self) -> Option<&[u8]> { + match self { + ResponseBody::Returned(bytes) => Some(bytes), + ResponseBody::Stored | ResponseBody::Streamed => None, + } + } +} + pub struct SendHttpRequestResult { pub rendered_request: HttpRequest, pub response: HttpResponse, diff --git a/package-lock.json b/package-lock.json index c56900b1..0c2cdd4a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16248,7 +16248,7 @@ }, "packages/plugin-runtime-types": { "name": "@yaakapp/api", - "version": "0.8.0", + "version": "0.9.0", "dependencies": { "@types/node": "^24.0.13" }, diff --git a/packages/plugin-runtime-types/package.json b/packages/plugin-runtime-types/package.json index d4dd862d..f282c3f0 100644 --- a/packages/plugin-runtime-types/package.json +++ b/packages/plugin-runtime-types/package.json @@ -1,6 +1,6 @@ { "name": "@yaakapp/api", - "version": "0.8.0", + "version": "0.9.0", "keywords": [ "api-client", "bruno-alternative", diff --git a/packages/plugin-runtime-types/src/bindings/gen_events.ts b/packages/plugin-runtime-types/src/bindings/gen_events.ts index 0dbd3212..2daff845 100644 --- a/packages/plugin-runtime-types/src/bindings/gen_events.ts +++ b/packages/plugin-runtime-types/src/bindings/gen_events.ts @@ -416,6 +416,32 @@ export type GetHttpRequestByIdRequest = { id: string, }; export type GetHttpRequestByIdResponse = { httpRequest: HttpRequest | null, }; +/** + * Ask what a response's body is, before deciding whether to pull it. + * + * Bodies are addressed by response id and never by path, so where the host + * keeps the bytes is its own business. + */ +export type GetHttpResponseBodyInfoRequest = { responseId: string, }; + +export type GetHttpResponseBodyInfoResponse = { +/** + * How many bytes are stored right now, which is not necessarily what the + * `Content-Length` header claimed. Zero when the response has no body. + */ +contentLength: number, +/** + * Whether the response has finished arriving. While it has not, the body + * keeps growing past `content_length`, and a reader that wants all of it + * asks again. + */ +complete: boolean, +/** + * The response's `Content-Type` header, verbatim, so the reader can pick a + * charset. + */ +contentType?: string | null, }; + export type GetKeyValueRequest = { key: string, }; export type GetKeyValueResponse = { value?: string, }; @@ -452,7 +478,7 @@ export type ImportResponse = { resources: ImportResources, }; export type InternalEvent = { id: string, pluginRefId: string, pluginName: string, replyId: string | null, context: PluginContext, payload: InternalEventPayload, }; -export type InternalEventPayload = { "type": "boot_request" } & BootRequest | { "type": "boot_response" } | { "type": "reload_response" } & ReloadResponse | { "type": "terminate_request" } | { "type": "terminate_response" } | { "type": "import_request" } & ImportRequest | { "type": "import_response" } & ImportResponse | { "type": "filter_request" } & FilterRequest | { "type": "filter_response" } & FilterResponse | { "type": "export_http_request_request" } & ExportHttpRequestRequest | { "type": "export_http_request_response" } & ExportHttpRequestResponse | { "type": "send_http_request_request" } & SendHttpRequestRequest | { "type": "send_http_request_response" } & SendHttpRequestResponse | { "type": "list_cookie_names_request" } & ListCookieNamesRequest | { "type": "list_cookie_names_response" } & ListCookieNamesResponse | { "type": "get_cookie_value_request" } & GetCookieValueRequest | { "type": "get_cookie_value_response" } & GetCookieValueResponse | { "type": "get_http_request_actions_request" } & EmptyPayload | { "type": "get_http_request_actions_response" } & GetHttpRequestActionsResponse | { "type": "call_http_request_action_request" } & CallHttpRequestActionRequest | { "type": "get_websocket_request_actions_request" } & EmptyPayload | { "type": "get_websocket_request_actions_response" } & GetWebsocketRequestActionsResponse | { "type": "call_websocket_request_action_request" } & CallWebsocketRequestActionRequest | { "type": "get_workspace_actions_request" } & EmptyPayload | { "type": "get_workspace_actions_response" } & GetWorkspaceActionsResponse | { "type": "call_workspace_action_request" } & CallWorkspaceActionRequest | { "type": "get_folder_actions_request" } & EmptyPayload | { "type": "get_folder_actions_response" } & GetFolderActionsResponse | { "type": "call_folder_action_request" } & CallFolderActionRequest | { "type": "get_grpc_request_actions_request" } & EmptyPayload | { "type": "get_grpc_request_actions_response" } & GetGrpcRequestActionsResponse | { "type": "call_grpc_request_action_request" } & CallGrpcRequestActionRequest | { "type": "get_template_function_summary_request" } & EmptyPayload | { "type": "get_template_function_summary_response" } & GetTemplateFunctionSummaryResponse | { "type": "get_template_function_config_request" } & GetTemplateFunctionConfigRequest | { "type": "get_template_function_config_response" } & GetTemplateFunctionConfigResponse | { "type": "call_template_function_request" } & CallTemplateFunctionRequest | { "type": "call_template_function_response" } & CallTemplateFunctionResponse | { "type": "get_http_authentication_summary_request" } & EmptyPayload | { "type": "get_http_authentication_summary_response" } & GetHttpAuthenticationSummaryResponse | { "type": "get_http_authentication_config_request" } & GetHttpAuthenticationConfigRequest | { "type": "get_http_authentication_config_response" } & GetHttpAuthenticationConfigResponse | { "type": "call_http_authentication_request" } & CallHttpAuthenticationRequest | { "type": "call_http_authentication_response" } & CallHttpAuthenticationResponse | { "type": "call_http_authentication_action_request" } & CallHttpAuthenticationActionRequest | { "type": "call_http_authentication_action_response" } & EmptyPayload | { "type": "copy_text_request" } & CopyTextRequest | { "type": "copy_text_response" } & EmptyPayload | { "type": "render_http_request_request" } & RenderHttpRequestRequest | { "type": "render_http_request_response" } & RenderHttpRequestResponse | { "type": "render_grpc_request_request" } & RenderGrpcRequestRequest | { "type": "render_grpc_request_response" } & RenderGrpcRequestResponse | { "type": "template_render_request" } & TemplateRenderRequest | { "type": "template_render_response" } & TemplateRenderResponse | { "type": "get_key_value_request" } & GetKeyValueRequest | { "type": "get_key_value_response" } & GetKeyValueResponse | { "type": "set_key_value_request" } & SetKeyValueRequest | { "type": "set_key_value_response" } & SetKeyValueResponse | { "type": "delete_key_value_request" } & DeleteKeyValueRequest | { "type": "delete_key_value_response" } & DeleteKeyValueResponse | { "type": "open_window_request" } & OpenWindowRequest | { "type": "window_navigate_event" } & WindowNavigateEvent | { "type": "window_close_event" } | { "type": "close_window_request" } & CloseWindowRequest | { "type": "open_external_url_request" } & OpenExternalUrlRequest | { "type": "open_external_url_response" } & EmptyPayload | { "type": "show_toast_request" } & ShowToastRequest | { "type": "show_toast_response" } & EmptyPayload | { "type": "prompt_text_request" } & PromptTextRequest | { "type": "prompt_text_response" } & PromptTextResponse | { "type": "prompt_form_request" } & PromptFormRequest | { "type": "prompt_form_response" } & PromptFormResponse | { "type": "window_info_request" } & WindowInfoRequest | { "type": "window_info_response" } & WindowInfoResponse | { "type": "list_open_workspaces_request" } & ListOpenWorkspacesRequest | { "type": "list_open_workspaces_response" } & ListOpenWorkspacesResponse | { "type": "get_http_request_by_id_request" } & GetHttpRequestByIdRequest | { "type": "get_http_request_by_id_response" } & GetHttpRequestByIdResponse | { "type": "find_http_responses_request" } & FindHttpResponsesRequest | { "type": "find_http_responses_response" } & FindHttpResponsesResponse | { "type": "list_http_requests_request" } & ListHttpRequestsRequest | { "type": "list_http_requests_response" } & ListHttpRequestsResponse | { "type": "list_folders_request" } & ListFoldersRequest | { "type": "list_folders_response" } & ListFoldersResponse | { "type": "upsert_model_request" } & UpsertModelRequest | { "type": "upsert_model_response" } & UpsertModelResponse | { "type": "delete_model_request" } & DeleteModelRequest | { "type": "delete_model_response" } & DeleteModelResponse | { "type": "get_themes_request" } & GetThemesRequest | { "type": "get_themes_response" } & GetThemesResponse | { "type": "empty_response" } & EmptyPayload | { "type": "error_response" } & ErrorResponse; +export type InternalEventPayload = { "type": "boot_request" } & BootRequest | { "type": "boot_response" } | { "type": "reload_response" } & ReloadResponse | { "type": "terminate_request" } | { "type": "terminate_response" } | { "type": "import_request" } & ImportRequest | { "type": "import_response" } & ImportResponse | { "type": "filter_request" } & FilterRequest | { "type": "filter_response" } & FilterResponse | { "type": "export_http_request_request" } & ExportHttpRequestRequest | { "type": "export_http_request_response" } & ExportHttpRequestResponse | { "type": "send_http_request_request" } & SendHttpRequestRequest | { "type": "send_http_request_response" } & SendHttpRequestResponse | { "type": "list_cookie_names_request" } & ListCookieNamesRequest | { "type": "list_cookie_names_response" } & ListCookieNamesResponse | { "type": "get_cookie_value_request" } & GetCookieValueRequest | { "type": "get_cookie_value_response" } & GetCookieValueResponse | { "type": "get_http_request_actions_request" } & EmptyPayload | { "type": "get_http_request_actions_response" } & GetHttpRequestActionsResponse | { "type": "call_http_request_action_request" } & CallHttpRequestActionRequest | { "type": "get_websocket_request_actions_request" } & EmptyPayload | { "type": "get_websocket_request_actions_response" } & GetWebsocketRequestActionsResponse | { "type": "call_websocket_request_action_request" } & CallWebsocketRequestActionRequest | { "type": "get_workspace_actions_request" } & EmptyPayload | { "type": "get_workspace_actions_response" } & GetWorkspaceActionsResponse | { "type": "call_workspace_action_request" } & CallWorkspaceActionRequest | { "type": "get_folder_actions_request" } & EmptyPayload | { "type": "get_folder_actions_response" } & GetFolderActionsResponse | { "type": "call_folder_action_request" } & CallFolderActionRequest | { "type": "get_grpc_request_actions_request" } & EmptyPayload | { "type": "get_grpc_request_actions_response" } & GetGrpcRequestActionsResponse | { "type": "call_grpc_request_action_request" } & CallGrpcRequestActionRequest | { "type": "get_template_function_summary_request" } & EmptyPayload | { "type": "get_template_function_summary_response" } & GetTemplateFunctionSummaryResponse | { "type": "get_template_function_config_request" } & GetTemplateFunctionConfigRequest | { "type": "get_template_function_config_response" } & GetTemplateFunctionConfigResponse | { "type": "call_template_function_request" } & CallTemplateFunctionRequest | { "type": "call_template_function_response" } & CallTemplateFunctionResponse | { "type": "get_http_authentication_summary_request" } & EmptyPayload | { "type": "get_http_authentication_summary_response" } & GetHttpAuthenticationSummaryResponse | { "type": "get_http_authentication_config_request" } & GetHttpAuthenticationConfigRequest | { "type": "get_http_authentication_config_response" } & GetHttpAuthenticationConfigResponse | { "type": "call_http_authentication_request" } & CallHttpAuthenticationRequest | { "type": "call_http_authentication_response" } & CallHttpAuthenticationResponse | { "type": "call_http_authentication_action_request" } & CallHttpAuthenticationActionRequest | { "type": "call_http_authentication_action_response" } & EmptyPayload | { "type": "copy_text_request" } & CopyTextRequest | { "type": "copy_text_response" } & EmptyPayload | { "type": "render_http_request_request" } & RenderHttpRequestRequest | { "type": "render_http_request_response" } & RenderHttpRequestResponse | { "type": "render_grpc_request_request" } & RenderGrpcRequestRequest | { "type": "render_grpc_request_response" } & RenderGrpcRequestResponse | { "type": "template_render_request" } & TemplateRenderRequest | { "type": "template_render_response" } & TemplateRenderResponse | { "type": "get_key_value_request" } & GetKeyValueRequest | { "type": "get_key_value_response" } & GetKeyValueResponse | { "type": "set_key_value_request" } & SetKeyValueRequest | { "type": "set_key_value_response" } & SetKeyValueResponse | { "type": "delete_key_value_request" } & DeleteKeyValueRequest | { "type": "delete_key_value_response" } & DeleteKeyValueResponse | { "type": "open_window_request" } & OpenWindowRequest | { "type": "window_navigate_event" } & WindowNavigateEvent | { "type": "window_close_event" } | { "type": "close_window_request" } & CloseWindowRequest | { "type": "open_external_url_request" } & OpenExternalUrlRequest | { "type": "open_external_url_response" } & EmptyPayload | { "type": "show_toast_request" } & ShowToastRequest | { "type": "show_toast_response" } & EmptyPayload | { "type": "prompt_text_request" } & PromptTextRequest | { "type": "prompt_text_response" } & PromptTextResponse | { "type": "prompt_form_request" } & PromptFormRequest | { "type": "prompt_form_response" } & PromptFormResponse | { "type": "window_info_request" } & WindowInfoRequest | { "type": "window_info_response" } & WindowInfoResponse | { "type": "list_open_workspaces_request" } & ListOpenWorkspacesRequest | { "type": "list_open_workspaces_response" } & ListOpenWorkspacesResponse | { "type": "get_http_request_by_id_request" } & GetHttpRequestByIdRequest | { "type": "get_http_request_by_id_response" } & GetHttpRequestByIdResponse | { "type": "find_http_responses_request" } & FindHttpResponsesRequest | { "type": "find_http_responses_response" } & FindHttpResponsesResponse | { "type": "get_http_response_body_info_request" } & GetHttpResponseBodyInfoRequest | { "type": "get_http_response_body_info_response" } & GetHttpResponseBodyInfoResponse | { "type": "read_http_response_body_chunk_request" } & ReadHttpResponseBodyChunkRequest | { "type": "read_http_response_body_chunk_response" } & ReadHttpResponseBodyChunkResponse | { "type": "list_http_requests_request" } & ListHttpRequestsRequest | { "type": "list_http_requests_response" } & ListHttpRequestsResponse | { "type": "list_folders_request" } & ListFoldersRequest | { "type": "list_folders_response" } & ListFoldersResponse | { "type": "upsert_model_request" } & UpsertModelRequest | { "type": "upsert_model_response" } & UpsertModelResponse | { "type": "delete_model_request" } & DeleteModelRequest | { "type": "delete_model_response" } & DeleteModelResponse | { "type": "get_themes_request" } & GetThemesRequest | { "type": "get_themes_response" } & GetThemesResponse | { "type": "empty_response" } & EmptyPayload | { "type": "error_response" } & ErrorResponse; export type JsonPrimitive = string | number | boolean | null; @@ -502,6 +528,27 @@ required?: boolean, }; export type PromptTextResponse = { value: string | null, }; +/** + * Pull one window of a response body. + * + * Reads are idempotent: the bytes live in durable storage, so the same window + * can be asked for as many times as the plugin likes. + */ +export type ReadHttpResponseBodyChunkRequest = { responseId: string, offset: number, length: number, }; + +export type ReadHttpResponseBodyChunkResponse = { +/** + * Base64, because the desktop transport is a WebSocket that only sends + * text frames today. A host that can carry binary sends the bytes as they + * are and fills this in from them. + */ +data: string, +/** + * Bytes decoded from `data`. Short of the requested length means the body + * ended here. + */ +length: number, }; + export type ReloadResponse = { silent: boolean, }; export type RenderGrpcRequestRequest = { grpcRequest: GrpcRequest, purpose: RenderPurpose, }; @@ -516,7 +563,16 @@ export type RenderPurpose = "send" | "preview"; export type SendHttpRequestRequest = { httpRequest: Partial, }; -export type SendHttpRequestResponse = { httpResponse: HttpResponse, }; +export type SendHttpRequestResponse = { httpResponse: HttpResponse, +/** + * The body, base64, when the send saved nothing. + * + * A request with no id behind it produces a response the model store never + * sees, so it cannot be read back by id later the way a saved one can. + * This is the only copy of it. `None` means the body was stored and should + * be read with `read_http_response_body_chunk_request`. + */ +body?: string | null, }; export type SetKeyValueRequest = { key: string, value: string, }; diff --git a/packages/plugin-runtime-types/src/bindings/gen_models.ts b/packages/plugin-runtime-types/src/bindings/gen_models.ts index d0ba2f1d..2f3542af 100644 --- a/packages/plugin-runtime-types/src/bindings/gen_models.ts +++ b/packages/plugin-runtime-types/src/bindings/gen_models.ts @@ -224,7 +224,6 @@ export type HttpResponse = { updatedAt: string; workspaceId: string; requestId: string; - bodyPath: string | null; contentLength: number | null; contentLengthCompressed: number | null; elapsed: number; diff --git a/packages/plugin-runtime-types/src/plugins/Context.ts b/packages/plugin-runtime-types/src/plugins/Context.ts index 0a1afa49..96854c2f 100644 --- a/packages/plugin-runtime-types/src/plugins/Context.ts +++ b/packages/plugin-runtime-types/src/plugins/Context.ts @@ -6,6 +6,7 @@ import type { GetCookieValueResponse, GetHttpRequestByIdRequest, GetHttpRequestByIdResponse, + GetHttpResponseBodyInfoRequest, JsonPrimitive, ListCookieNamesResponse, ListFoldersRequest, @@ -22,12 +23,11 @@ import type { RenderHttpRequestRequest, RenderHttpRequestResponse, SendHttpRequestRequest, - SendHttpRequestResponse, ShowToastRequest, TemplateRenderRequest, WorkspaceInfo, } from "../bindings/gen_events.ts"; -import type { Folder, HttpRequest } from "../bindings/gen_models.ts"; +import type { Folder, HttpRequest, HttpResponse } from "../bindings/gen_models.ts"; import type { JsonValue } from "../bindings/serde_json/JsonValue"; import type { MaybePromise } from "../helpers"; @@ -65,6 +65,84 @@ type DynamicPromptFormRequest = Omit & { export type WorkspaceHandle = Pick; +export interface ReadHttpResponseBodyOptions { + /** + * Refuse to buffer a body larger than this, in bytes. Defaults to 32 MiB. + * Pass `Infinity` to read whatever is there. + */ + maxBytes?: number; + + /** Bytes to pull from the host at a time. Defaults to 1 MiB. */ + chunkSize?: number; +} + +/** + * A response body, read back from wherever the host stored it. + * + * The accessors are named after `fetch`'s and behave the same way against a + * response that is still arriving: they wait for the rest, and one that never + * finishes is never finished reading — `chunks()` is the way to consume that. + * Unlike `fetch`, the body is not used up by reading it: the bytes are in + * durable storage, so every accessor can be called as many times as you like, + * in any order. + */ +export interface HttpResponseBody { + /** The response these bytes belong to. */ + readonly responseId: string; + + /** + * How many bytes were stored when this body was opened, which is not + * necessarily what the `Content-Length` header claimed. Zero when the + * response has no body. Final only if `complete`. + */ + readonly contentLength: number; + + /** The response's `Content-Type` header, verbatim, or null if it had none. */ + readonly contentType: string | null; + + /** + * Whether the response had finished arriving when this body was opened. + * When false, the accessors below will wait for the rest of it. + */ + readonly complete: boolean; + + /** + * The whole body decoded to a string, using the charset from `contentType` + * and falling back to UTF-8. Waits for a response still arriving. Throws + * once more than `maxBytes` has been read. + */ + text(options?: ReadHttpResponseBodyOptions): Promise; + + /** `text()`, parsed as JSON. */ + json(options?: ReadHttpResponseBodyOptions): Promise; + + /** The whole body as raw bytes. Waits and throws as `text()` does. */ + arrayBuffer(options?: ReadHttpResponseBodyOptions): Promise; + + /** + * The raw bytes, a chunk at a time, so a body of any size can be read + * without holding all of it at once. Not subject to `maxBytes`. + * + * Follows a response that is still arriving, yielding as it comes, and ends + * when the response does. Break out of the loop to stop early. + */ + chunks(options?: Pick): AsyncIterable; +} + +/** What a send came back with. */ +export interface SentHttpRequest { + httpResponse: HttpResponse; + + /** + * The response's body. + * + * Handed over here rather than looked up later, because a request with no id + * is not saved and this is the only copy of its body. Reading it is the same + * either way, so nothing has to know which kind of send it made. + */ + body: HttpResponseBody; +} + export interface Context { clipboard: { copyText(text: string): Promise; @@ -101,7 +179,13 @@ export interface Context { render(args: RenderGrpcRequestRequest): Promise; }; httpRequest: { - send(args: SendHttpRequestRequest): Promise; + /** + * Send a request and wait for the response and its body. + * + * The body comes back with the response because a request with no id is + * never saved, and there would be nothing to look up afterwards. + */ + send(args: SendHttpRequestRequest): Promise; getById(args: GetHttpRequestByIdRequest): Promise; render(args: RenderHttpRequestRequest): Promise; list(args?: ListHttpRequestsRequest): Promise; @@ -129,6 +213,15 @@ export interface Context { }; httpResponse: { find(args: FindHttpResponsesRequest): Promise; + /** + * Read a saved response's body by id. Where the host keeps the bytes — + * files on a desktop, rows in a database, somewhere else later — is not + * something a plugin sees or should depend on. + * + * Ids come from `find`. A response that was never saved has none to look + * up, so its body arrives with the send that made it instead. + */ + body(args: GetHttpResponseBodyInfoRequest): Promise; }; templates: { render(args: TemplateRenderRequest & { data: T }): Promise; diff --git a/packages/plugin-runtime-types/src/plugins/index.ts b/packages/plugin-runtime-types/src/plugins/index.ts index 86c2b67d..d5280ac1 100644 --- a/packages/plugin-runtime-types/src/plugins/index.ts +++ b/packages/plugin-runtime-types/src/plugins/index.ts @@ -13,7 +13,13 @@ import type { WorkspaceActionPlugin } from "./WorkspaceActionPlugin"; export type { Context }; export type { DynamicAuthenticationArg } from "./AuthenticationPlugin"; -export type { CallPromptFormDynamicArgs, DynamicPromptFormArg } from "./Context"; +export type { + CallPromptFormDynamicArgs, + DynamicPromptFormArg, + HttpResponseBody, + ReadHttpResponseBodyOptions, + SentHttpRequest, +} from "./Context"; export type { DynamicTemplateFunctionArg } from "./TemplateFunctionPlugin"; export type { TemplateFunctionPlugin }; export type { FolderActionPlugin } from "./FolderActionPlugin"; diff --git a/packages/plugin-runtime/src/PluginInstance.ts b/packages/plugin-runtime/src/PluginInstance.ts index 031a6d0f..b61761d3 100644 --- a/packages/plugin-runtime/src/PluginInstance.ts +++ b/packages/plugin-runtime/src/PluginInstance.ts @@ -21,11 +21,13 @@ import type { GetCookieValueRequest, GetCookieValueResponse, GetHttpRequestByIdResponse, + GetHttpResponseBodyInfoResponse, GetKeyValueResponse, GrpcRequestAction, HttpAuthenticationAction, HttpRequest, HttpRequestAction, + HttpResponse, ImportResources, InternalEvent, InternalEventPayload, @@ -37,6 +39,7 @@ import type { PluginContext, PromptFormResponse, PromptTextResponse, + ReadHttpResponseBodyChunkResponse, RenderGrpcRequestResponse, RenderHttpRequestResponse, SendHttpRequestResponse, @@ -49,6 +52,22 @@ import type { import { applyDynamicFormInput } from "./common"; import { EventChannel } from "./EventChannel"; import { migrateTemplateFunctionSelectOptions } from "./migrations"; +import { createResponseBody, decodeBase64Chunk } from "./responseBody"; + +/** + * A response as a plugin should see it. + * + * The host still puts `bodyPath` on the wire for its own callers, but it names + * a file on the host's disk — meaningless to a plugin, absent once bodies move + * off the filesystem, and impossible in a browser. Plugins address bodies by + * response id, so drop it here rather than let one grow a dependency on it. + */ +function forPlugin(httpResponse: HttpResponse): HttpResponse { + const { bodyPath: _bodyPath, ...rest } = httpResponse as HttpResponse & { + bodyPath?: string | null; + }; + return rest; +} export interface PluginWorkerData { bootRequest: BootRequest; @@ -552,6 +571,14 @@ export class PluginInstance { return this.#sendPayload(context, { type: "empty_response" }, replyId); } + /** + * Send a request to the host and wait for its reply. + * + * A host that cannot answer replies with an error, which becomes a thrown + * error here. The alternative is handing back a reply-shaped object with + * none of the fields the caller destructures, and letting it fail somewhere + * further along with no idea why. + */ #sendForReply>( context: PluginContext, payload: InternalEventPayload, @@ -560,11 +587,16 @@ export class PluginInstance { const eventToSend = this.#buildEventToSend(context, payload, null); // 2. Spawn listener in background - const promise = new Promise((resolve) => { + const promise = new Promise((resolve, reject) => { const cb = (event: InternalEvent) => { if (event.replyId === eventToSend.id) { this.#appToPluginEvents.unlisten(cb); // Unlisten, now that we're done const { type: _, ...payload } = event.payload; + if (event.payload.type === "error_response") { + const { error } = payload as { error?: string }; + reject(new Error(error || `Host failed to handle ${eventToSend.payload.type}`)); + return; + } resolve(payload as T); } }; @@ -598,6 +630,33 @@ export class PluginInstance { } #newCtx(context: PluginContext): Context { + /** Read a body the host has stored, a chunk at a time, following it if it is still arriving. */ + const storedBody = async (responseId: string) => { + const bodyInfo = () => + this.#sendForReply(context, { + type: "get_http_response_body_info_request", + responseId, + }); + const info = await bodyInfo(); + + return createResponseBody( + { + responseId, + contentLength: info.contentLength, + contentType: info.contentType ?? null, + complete: info.complete, + }, + async (offset, length) => { + const chunk = await this.#sendForReply( + context, + { type: "read_http_response_body_chunk_request", responseId, offset, length }, + ); + return decodeBase64Chunk(chunk.data); + }, + { refresh: bodyInfo }, + ); + }; + const _windowInfo = async () => { if (context.label == null) { throw new Error("Can't get window context without an active window"); @@ -749,8 +808,9 @@ export class PluginInstance { context, payload, ); - return httpResponses; + return httpResponses.map(forPlugin); }, + body: ({ responseId }) => storedBody(responseId), }, grpcRequest: { render: async (args) => { @@ -782,11 +842,34 @@ export class PluginInstance { type: "send_http_request_request", ...args, } as const; - const { httpResponse } = await this.#sendForReply( + const { httpResponse, body } = await this.#sendForReply( context, payload, ); - return httpResponse; + + // A send with no request behind it saves nothing, so the reply + // carries the only copy of its body. A saved one is read back from + // the host like any other. Callers get the same thing either way. + if (body == null) { + return { httpResponse: forPlugin(httpResponse), body: await storedBody(httpResponse.id) }; + } + + const bytes = decodeBase64Chunk(body); + return { + httpResponse: forPlugin(httpResponse), + body: createResponseBody( + { + responseId: httpResponse.id, + contentLength: bytes.byteLength, + contentType: + httpResponse.headers.find((h) => h.name.toLowerCase() === "content-type") + ?.value ?? null, + // The host waited for the whole send before replying. + complete: true, + }, + async (offset, length) => bytes.slice(offset, offset + length), + ), + }; }, render: async (args) => { const payload = { diff --git a/packages/plugin-runtime/src/responseBody.ts b/packages/plugin-runtime/src/responseBody.ts new file mode 100644 index 00000000..527cfe02 --- /dev/null +++ b/packages/plugin-runtime/src/responseBody.ts @@ -0,0 +1,194 @@ +import type { HttpResponseBody, ReadHttpResponseBodyOptions } from "@yaakapp/api"; + +/** Bytes pulled from the host per round trip, when the caller doesn't say. */ +const DEFAULT_CHUNK_SIZE = 1024 * 1024; + +/** + * The most a plugin buffers by default. + * + * Reading a body used to be unbounded, so any ceiling is an improvement; this + * one is set well above what an API returns and well below what makes the + * plugin runtime fall over. `chunks()` has no ceiling, and any caller that + * really wants the whole thing can raise `maxBytes`. + */ +const DEFAULT_MAX_BYTES = 32 * 1024 * 1024; + +/** How long to wait, having caught up with a body still arriving, before looking again. */ +const DEFAULT_POLL_INTERVAL_MS = 100; + +/** Fetch one window of body bytes from the host. */ +export type ReadResponseBodyChunk = (offset: number, length: number) => Promise; + +export interface ResponseBodyInfo { + responseId: string; + contentLength: number; + contentType: string | null; + /** Whether the response has finished arriving, so `contentLength` is final. */ + complete: boolean; +} + +/** What can change while a body is still arriving. */ +export type ResponseBodyProgress = Pick; + +export interface CreateResponseBodyOptions { + /** + * Ask the host where the body has got to. Needed only for a body that was + * not complete when opened; a reader that has caught up calls this to learn + * whether to wait for more or stop. + */ + refresh?: () => Promise; + pollIntervalMs?: number; +} + +export function createResponseBody( + info: ResponseBodyInfo, + readChunk: ReadResponseBodyChunk, + { refresh, pollIntervalMs = DEFAULT_POLL_INTERVAL_MS }: CreateResponseBodyOptions = {}, +): HttpResponseBody { + const { responseId, contentLength, contentType, complete } = info; + + /** + * Yield the body from the start until it has all arrived. + * + * A complete body is read up to its known length and no further. One still + * arriving is followed: on catching up, ask the host whether it has finished, + * and if not, wait and look again. So this ends when the response does — + * which for a stream that never closes means it doesn't, exactly as + * iterating `fetch`'s body would not. + */ + async function* chunks( + options?: Pick, + ): AsyncIterable { + const chunkSize = Math.max(1, Math.floor(options?.chunkSize ?? DEFAULT_CHUNK_SIZE)); + let known = contentLength; + let done = complete; + let offset = 0; + + while (true) { + if (done && offset >= known) return; + + const want = done ? Math.min(chunkSize, known - offset) : chunkSize; + const chunk = await readChunk(offset, want); + if (chunk.byteLength > 0) { + yield chunk; + offset += chunk.byteLength; + continue; + } + + // Caught up. A complete body that came up short simply ended sooner than + // the host said; one still arriving needs asking about. + if (done || refresh == null) return; + ({ contentLength: known, complete: done } = await refresh()); + if (offset < known) continue; + if (done) return; + await new Promise((resolve) => setTimeout(resolve, pollIntervalMs)); + } + } + + async function readAll(accessor: string, options?: ReadHttpResponseBodyOptions) { + const maxBytes = options?.maxBytes ?? DEFAULT_MAX_BYTES; + refuseIfTooBig(accessor, contentLength, maxBytes); + + const parts: Uint8Array[] = []; + let total = 0; + for await (const chunk of chunks(options)) { + total += chunk.byteLength; + // The size the host reported is a claim about a moment ago, so check the + // bytes actually arriving too. + refuseIfTooBig(accessor, total, maxBytes); + parts.push(chunk); + } + + const bytes = new Uint8Array(total); + let offset = 0; + for (const part of parts) { + bytes.set(part, offset); + offset += part.byteLength; + } + return bytes; + } + + return { + responseId, + contentLength, + contentType, + complete, + chunks, + async arrayBuffer(options) { + const bytes = await readAll("arrayBuffer", options); + return bytes.buffer as ArrayBuffer; + }, + async text(options) { + return decodeBody(await readAll("text", options), contentType); + }, + async json(options?: ReadHttpResponseBodyOptions) { + return JSON.parse(decodeBody(await readAll("json", options), contentType)) as T; + }, + }; +} + +function refuseIfTooBig(accessor: string, bytes: number, maxBytes: number) { + if (bytes <= maxBytes) return; + throw new Error( + `Response body is ${formatBytes(bytes)}, over the ${formatBytes(maxBytes)} limit for ` + + `${accessor}(). Read it with chunks() instead, or pass a larger maxBytes.`, + ); +} + +/** + * Decode using the charset the response declared. + * + * Assuming UTF-8 mangles every response that isn't, and the header is right + * there. An unknown label is the one case worth guessing on, since the + * alternative is refusing to read a body we can very likely still read. + */ +function decodeBody(bytes: Uint8Array, contentType: string | null): string { + const charset = parseCharset(contentType); + if (charset != null) { + try { + return new TextDecoder(charset).decode(bytes); + } catch { + // Not a label this runtime knows. + } + } + // TextDecoder drops a leading BOM on its own. + return new TextDecoder("utf-8").decode(bytes); +} + +function parseCharset(contentType: string | null): string | null { + const match = contentType?.match(/;\s*charset\s*=\s*"?([^";]+)"?/i); + return match?.[1]?.trim() || null; +} + +function formatBytes(bytes: number): string { + if (bytes === Infinity) return "unlimited"; + if (bytes < 1024) return `${bytes} B`; + const units = ["KB", "MB", "GB"]; + let value = bytes / 1024; + let unit = 0; + while (value >= 1024 && unit < units.length - 1) { + value /= 1024; + unit++; + } + return `${value.toFixed(1)} ${units[unit]}`; +} + +/** + * Decode a chunk that arrived as base64. + * + * The desktop transport is a WebSocket carrying JSON text frames, so bytes + * have to be spelled out. A host that can pass an ArrayBuffer along skips this. + */ +export function decodeBase64Chunk(data: string): Uint8Array { + if (typeof Buffer !== "undefined") { + const buf = Buffer.from(data, "base64"); + return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength); + } + + const binary = atob(data); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) { + bytes[i] = binary.charCodeAt(i); + } + return bytes; +} diff --git a/packages/plugin-runtime/tests/responseBody.test.ts b/packages/plugin-runtime/tests/responseBody.test.ts new file mode 100644 index 00000000..cee5f3b2 --- /dev/null +++ b/packages/plugin-runtime/tests/responseBody.test.ts @@ -0,0 +1,203 @@ +import { describe, expect, test } from "vite-plus/test"; +import { createResponseBody, decodeBase64Chunk } from "../src/responseBody"; + +/** A finished body, in a store that records every window it was asked for. */ +function fakeBody(bytes: Uint8Array, contentType: string | null) { + const reads: Array<[number, number]> = []; + const body = createResponseBody( + { responseId: "rs_test", contentLength: bytes.byteLength, contentType, complete: true }, + async (offset, length) => { + reads.push([offset, length]); + return bytes.slice(offset, offset + length); + }, + ); + return { body, reads }; +} + +/** + * A body still arriving: it grows by one script step every time the reader + * asks the host where it has got to, and completes on the last step. + */ +function streamingBody(steps: string[], contentType = "text/plain") { + let stored = new Uint8Array(); + let step = 0; + let refreshes = 0; + const advance = () => { + if (step < steps.length) { + const next = utf8(steps[step]!); + const grown = new Uint8Array(stored.byteLength + next.byteLength); + grown.set(stored); + grown.set(next, stored.byteLength); + stored = grown; + step++; + } + return { contentLength: stored.byteLength, complete: step >= steps.length }; + }; + const body = createResponseBody( + { responseId: "rs_live", contentLength: 0, contentType, complete: false }, + async (offset, length) => stored.slice(offset, offset + length), + { + refresh: async () => { + refreshes++; + return advance(); + }, + pollIntervalMs: 1, + }, + ); + return { body, refreshCount: () => refreshes }; +} + +function utf8(text: string) { + return new TextEncoder().encode(text); +} + +describe("response body", () => { + test("pulls a body in chunks and reassembles it", async () => { + const { body, reads } = fakeBody(utf8("abcdefghij"), "text/plain"); + + expect(await body.text({ chunkSize: 4 })).toEqual("abcdefghij"); + expect(reads).toEqual([ + [0, 4], + [4, 4], + [8, 2], + ]); + }); + + test("can be read more than once, unlike fetch", async () => { + const { body } = fakeBody(utf8('{"a":1}'), "application/json"); + + expect(await body.text()).toEqual('{"a":1}'); + expect(await body.json()).toEqual({ a: 1 }); + expect(new Uint8Array(await body.arrayBuffer())).toEqual(utf8('{"a":1}')); + }); + + test("decodes using the charset the response declared", async () => { + // "café naïve" as Latin-1, which is mojibake if read as UTF-8. + const latin1 = new Uint8Array([0x63, 0x61, 0x66, 0xe9, 0x20, 0x6e, 0x61, 0xef, 0x76, 0x65]); + + const declared = fakeBody(latin1, "text/plain; charset=iso-8859-1"); + expect(await declared.body.text()).toEqual("café naïve"); + + const undeclared = fakeBody(latin1, "text/plain"); + expect(await undeclared.body.text()).not.toEqual("café naïve"); + }); + + test("falls back to UTF-8 for a charset the runtime doesn't know", async () => { + const { body } = fakeBody(utf8("hello"), "text/plain; charset=not-a-real-charset"); + expect(await body.text()).toEqual("hello"); + }); + + test("drops a UTF-8 BOM", async () => { + const withBom = new Uint8Array([0xef, 0xbb, 0xbf, ...utf8('{"a":1}')]); + const { body } = fakeBody(withBom, "application/json"); + + expect(await body.text()).toEqual('{"a":1}'); + expect(await body.json()).toEqual({ a: 1 }); + }); + + test("refuses to buffer past maxBytes, and says what to do instead", async () => { + const { body } = fakeBody(utf8("x".repeat(100)), "text/plain"); + + await expect(body.text({ maxBytes: 50 })).rejects.toThrow(/chunks\(\)/); + await expect(body.json({ maxBytes: 50 })).rejects.toThrow(/over the/); + await expect(body.arrayBuffer({ maxBytes: 50 })).rejects.toThrow(/arrayBuffer\(\)/); + + // The ceiling is the caller's to raise. + expect(await body.text({ maxBytes: 100 })).toHaveLength(100); + }); + + test("streams past maxBytes through chunks()", async () => { + const { body } = fakeBody(utf8("x".repeat(100)), "text/plain"); + + let total = 0; + for await (const chunk of body.chunks({ chunkSize: 10 })) { + total += chunk.byteLength; + } + expect(total).toEqual(100); + }); + + test("stops early when the host runs out of bytes sooner than it claimed", async () => { + // contentLength says 100; the store only ever hands back 10. + const body = createResponseBody( + { responseId: "rs_test", contentLength: 100, contentType: "text/plain", complete: true }, + async (offset) => (offset === 0 ? utf8("0123456789") : new Uint8Array()), + ); + + expect(await body.text()).toEqual("0123456789"); + }); + + test("a response with no body reads as empty", async () => { + const { body, reads } = fakeBody(new Uint8Array(), "application/json"); + + expect(body.contentLength).toEqual(0); + expect(await body.text()).toEqual(""); + expect(reads).toEqual([]); + }); + + test("keeps binary bytes intact", async () => { + const png = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0xff]); + const { body } = fakeBody(png, "image/png"); + + expect(new Uint8Array(await body.arrayBuffer({ chunkSize: 3 }))).toEqual(png); + }); +}); + +describe("a response still arriving", () => { + test("chunks() follows it until it finishes", async () => { + const { body, refreshCount } = streamingBody(["data: 1\n", "data: 2\n", "data: 3\n"]); + expect(body.complete).toBe(false); + + const seen: string[] = []; + for await (const chunk of body.chunks()) { + seen.push(new TextDecoder().decode(chunk)); + } + + expect(seen.join("")).toEqual("data: 1\ndata: 2\ndata: 3\n"); + // Asked once per catch-up, and stopped as soon as the host said it was done. + expect(refreshCount()).toEqual(3); + }); + + test("text() waits for the rest rather than returning a prefix", async () => { + const { body } = streamingBody(['{"token":', '"abc"}']); + expect(await body.json()).toEqual({ token: "abc" }); + }); + + test("keeps waiting through a stretch with nothing new", async () => { + // Two refreshes report no growth before the body finally moves. + const { body } = streamingBody(["", "", "late"]); + expect(await body.text()).toEqual("late"); + }); + + test("still refuses to buffer past maxBytes as it streams", async () => { + const { body } = streamingBody(["x".repeat(40), "x".repeat(40), "x".repeat(40)]); + await expect(body.text({ maxBytes: 100 })).rejects.toThrow(/chunks\(\)/); + }); + + test("a finished body never asks the host again", async () => { + let refreshes = 0; + const body = createResponseBody( + { responseId: "rs_done", contentLength: 5, contentType: null, complete: true }, + async (offset, length) => utf8("hello").slice(offset, offset + length), + { + refresh: async () => { + refreshes++; + return { contentLength: 5, complete: true }; + }, + }, + ); + expect(await body.text()).toEqual("hello"); + expect(refreshes).toEqual(0); + }); +}); + +describe("decodeBase64Chunk", () => { + test("round-trips arbitrary bytes", () => { + const bytes = new Uint8Array([0, 1, 127, 128, 254, 255]); + const base64 = Buffer.from(bytes).toString("base64"); + expect(decodeBase64Chunk(base64)).toEqual(bytes); + }); + + test("decodes an empty chunk", () => { + expect(decodeBase64Chunk("")).toEqual(new Uint8Array()); + }); +}); diff --git a/plugins-external/mcp-server/src/tools/httpRequest.ts b/plugins-external/mcp-server/src/tools/httpRequest.ts index a1aa342b..545c7504 100644 --- a/plugins-external/mcp-server/src/tools/httpRequest.ts +++ b/plugins-external/mcp-server/src/tools/httpRequest.ts @@ -80,7 +80,7 @@ export function registerHttpRequestTools(server: McpServer, ctx: McpServerContex throw new Error(`HTTP request with ID ${id} not found`); } - const response = await workspaceCtx.yaak.httpRequest.send({ httpRequest }); + const { httpResponse: response } = await workspaceCtx.yaak.httpRequest.send({ httpRequest }); return { content: [ diff --git a/plugins/auth-ntlm/src/index.ts b/plugins/auth-ntlm/src/index.ts index 9adfa091..f787de0d 100644 --- a/plugins/auth-ntlm/src/index.ts +++ b/plugins/auth-ntlm/src/index.ts @@ -67,7 +67,7 @@ export const plugin: PluginDefinition = { const type1 = ntlm.createType1Message(options); - const negotiateResponse = await ctx.httpRequest.send({ + const { httpResponse: negotiateResponse } = await ctx.httpRequest.send({ httpRequest: { method, url, diff --git a/plugins/auth-oauth2/src/fetchAccessToken.ts b/plugins/auth-oauth2/src/fetchAccessToken.ts index ccd28f26..270c2320 100644 --- a/plugins/auth-oauth2/src/fetchAccessToken.ts +++ b/plugins/auth-oauth2/src/fetchAccessToken.ts @@ -1,4 +1,3 @@ -import { readFileSync } from "node:fs"; import type { Context, HttpRequest, HttpUrlParameter } from "@yaakapp/api"; import type { AccessTokenRawResponse } from "./store"; @@ -57,7 +56,7 @@ export async function fetchAccessToken( } httpRequest.authenticationType = "none"; // Don't inherit workspace auth - const resp = await ctx.httpRequest.send({ httpRequest }); + const { httpResponse: resp, body: responseBody } = await ctx.httpRequest.send({ httpRequest }); console.log("[oauth2] Got access token response", resp.status); @@ -65,7 +64,10 @@ export async function fetchAccessToken( throw new Error(`Failed to fetch access token: ${resp.error}`); } - const body = resp.bodyPath ? readFileSync(resp.bodyPath, "utf8") : ""; + // A token request is sent ad-hoc, with no id, so nothing saves the response + // and this body is the only copy of it. An empty one parses to {} below, + // which is what reading a missing file used to give. + const body = await responseBody.text(); if (resp.status < 200 || resp.status >= 300) { throw new Error(`Failed to fetch access token with status=${resp.status} and body=${body}`); diff --git a/plugins/auth-oauth2/src/getOrRefreshAccessToken.ts b/plugins/auth-oauth2/src/getOrRefreshAccessToken.ts index eb2843f3..86aee3e9 100644 --- a/plugins/auth-oauth2/src/getOrRefreshAccessToken.ts +++ b/plugins/auth-oauth2/src/getOrRefreshAccessToken.ts @@ -1,4 +1,3 @@ -import { readFileSync } from "node:fs"; import type { Context, HttpRequest } from "@yaakapp/api"; import type { AccessToken, AccessTokenRawResponse, TokenStoreArgs } from "./store"; import { deleteToken, getToken, storeToken } from "./store"; @@ -71,7 +70,7 @@ export async function getOrRefreshAccessToken( } httpRequest.authenticationType = "none"; // Don't inherit workspace auth - const resp = await ctx.httpRequest.send({ httpRequest }); + const { httpResponse: resp, body: responseBody } = await ctx.httpRequest.send({ httpRequest }); if (resp.error) { throw new Error(`Failed to refresh access token: ${resp.error}`); @@ -85,7 +84,9 @@ export async function getOrRefreshAccessToken( return null; } - const body = resp.bodyPath ? readFileSync(resp.bodyPath, "utf8") : ""; + // Sent ad-hoc, so this body came back with the response rather than being + // saved anywhere to read later. + const body = await responseBody.text(); console.log("[oauth2] Got refresh token response", resp.status); diff --git a/plugins/template-function-response/src/index.ts b/plugins/template-function-response/src/index.ts index 9de85ffb..3edf954f 100644 --- a/plugins/template-function-response/src/index.ts +++ b/plugins/template-function-response/src/index.ts @@ -1,4 +1,3 @@ -import { readFileSync } from "node:fs"; import type { CallTemplateFunctionArgs, Context, @@ -196,17 +195,8 @@ export const plugin: PluginDefinition = { }); if (response == null) return null; - if (response.bodyPath == null) { - return null; - } - - const BOM = "\ufeff"; - let body: string; - try { - body = readFileSync(response.bodyPath, "utf-8").replace(BOM, ""); - } catch { - return null; - } + const body = await readResponseBody(ctx, response); + if (body == null) return null; try { const result: JSONPathResult = @@ -261,23 +251,32 @@ export const plugin: PluginDefinition = { }); if (response == null) return null; - if (response.bodyPath == null) { - return null; - } - - let body: string; - try { - body = readFileSync(response.bodyPath, "utf-8"); - } catch { - return null; - } - - return body; + return await readResponseBody(ctx, response); }, }, ], }; +/** + * The response's body as text, or null when there is nothing to read. + * + * The host is asked for it by response id, so this works wherever the bytes + * happen to live — including responses it never recorded, which still get an + * id. A body over the runtime's size limit throws rather than coming back + * empty, since a template silently rendering to nothing is worse than one that + * says why. + */ +async function readResponseBody(ctx: Context, response: HttpResponse): Promise { + // Belt and braces: everything reaching here came from find() or send() and so + // has an id. An empty one would just be an unreadable id. + if (!response.id) return null; + + const body = await ctx.httpResponse.body({ responseId: response.id }); + if (body.contentLength === 0) return null; + + return await body.text(); +} + async function getResponse( ctx: Context, { @@ -320,7 +319,7 @@ async function getResponse( // Explicitly render the request before send (instead of relying on send() to render) so that we can // preserve the render purpose. const renderedHttpRequest = await ctx.httpRequest.render({ httpRequest, purpose }); - response = await ctx.httpRequest.send({ httpRequest: renderedHttpRequest }); + response = (await ctx.httpRequest.send({ httpRequest: renderedHttpRequest })).httpResponse; } return response;