diff --git a/.gitattributes b/.gitattributes index 6438d80c..0b6a9cb7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,7 @@ src-tauri/vendored/**/* linguist-generated=true src-tauri/gen/schemas/**/* linguist-generated=true +**/bindings/* linguist-generated=true +src-tauri/yaak-templates/pkg/* linguist-generated=true + +# Ensure consistent line endings for test files that check exact content +src-tauri/yaak-http/tests/test.txt text eol=lf diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml new file mode 100644 index 00000000..d300267f --- /dev/null +++ b/.github/workflows/claude.yml @@ -0,0 +1,50 @@ +name: Claude Code + +on: + issue_comment: + types: [created] + pull_request_review_comment: + types: [created] + issues: + types: [opened, assigned] + pull_request_review: + types: [submitted] + +jobs: + claude: + if: | + (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) || + (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude'))) + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + issues: read + id-token: write + actions: read # Required for Claude to read CI results on PRs + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Run Claude Code + id: claude + uses: anthropics/claude-code-action@v1 + with: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + + # This is an optional setting that allows Claude to read CI results on PRs + additional_permissions: | + actions: read + + # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it. + # prompt: 'Update the pull request description to include a summary of changes.' + + # Optional: Add claude_args to customize behavior and configuration + # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md + # or https://code.claude.com/docs/en/cli-reference for available options + # claude_args: '--allowed-tools Bash(gh pr:*)' + diff --git a/.gitignore b/.gitignore index ee2c7428..e4995499 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,5 @@ out .tmp tmp +.zed +codebook.toml diff --git a/README.md b/README.md index 8bb37fb7..327cfd0d 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@

- User avatar: MVST-Solutions  User avatar: dharsanb  User avatar: railwayapp  User avatar: caseyamcl  User avatar:    + User avatar: MVST-Solutions  User avatar: dharsanb  User avatar: railwayapp  User avatar: caseyamcl  User avatar: bytebase  User avatar:   

User avatar: seanwash  User avatar: jerath  User avatar: itsa-sh  User avatar: dmmulroy  User avatar: timcole  User avatar: VLZH  User avatar: terasaka2k  User avatar: andriyor  User avatar: majudhu  User avatar: axelrindle  User avatar: jirizverina  User avatar: chip-well  User avatar: GRAYAH   diff --git a/package-lock.json b/package-lock.json index 2a3de264..86c09beb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1700,6 +1700,21 @@ "integrity": "sha512-l0h88YhZFyKdXIFNfSWpyjStDjGHwZ/U7iobcK1cQQD8sejsONdQtTVU+1wVN1PBw40PiiHB1vA5S7VTfQiP9g==", "license": "MIT" }, + "node_modules/@mjackson/headers": { + "version": "0.11.1", + "resolved": "https://registry.npmjs.org/@mjackson/headers/-/headers-0.11.1.tgz", + "integrity": "sha512-uXXhd4rtDdDwkqAuGef1nuafkCa1NlTmEc1Jzc0NL4YiA1yON1NFXuqJ3hOuKvNKQwkiDwdD+JJlKVyz4dunFA==", + "license": "MIT" + }, + "node_modules/@mjackson/multipart-parser": { + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/@mjackson/multipart-parser/-/multipart-parser-0.10.1.tgz", + "integrity": "sha512-cHMD6+ErH/DrEfC0N6Ru/+1eAdavxdV0C35PzSb5/SD7z3XoaDMc16xPJcb8CahWjSpqHY+Too9sAb6/UNuq7A==", + "license": "MIT", + "dependencies": { + "@mjackson/headers": "^0.11.1" + } + }, "node_modules/@mrmlnc/readdir-enhanced": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz", @@ -18680,6 +18695,7 @@ "@gilbarbara/deep-equal": "^0.3.1", "@lezer/highlight": "^1.1.3", "@lezer/lr": "^1.3.3", + "@mjackson/multipart-parser": "^0.10.1", "@prantlf/jsonlint": "^16.0.0", "@replit/codemirror-emacs": "^6.1.0", "@replit/codemirror-vim": "^6.3.0", diff --git a/packages/plugin-runtime-types/src/bindings/gen_models.ts b/packages/plugin-runtime-types/src/bindings/gen_models.ts index 6b2eb5c8..454903fe 100644 --- a/packages/plugin-runtime-types/src/bindings/gen_models.ts +++ b/packages/plugin-runtime-types/src/bindings/gen_models.ts @@ -12,7 +12,7 @@ export type HttpRequest = { model: "http_request", id: string, createdAt: string export type HttpRequestHeader = { enabled?: boolean, name: string, value: string, id?: string, }; -export type HttpResponse = { model: "http_response", id: string, createdAt: string, updatedAt: string, workspaceId: string, requestId: string, bodyPath: string | null, contentLength: number | null, elapsed: number, elapsedHeaders: number, error: string | null, headers: Array, remoteAddr: string | null, status: number, statusReason: string | null, state: HttpResponseState, url: string, version: string | null, }; +export type HttpResponse = { model: "http_response", id: string, createdAt: string, updatedAt: string, workspaceId: string, requestId: string, bodyPath: string | null, contentLength: number | null, contentLengthCompressed: number | null, elapsed: number, elapsedHeaders: number, error: string | null, headers: Array, remoteAddr: string | null, requestContentLength: number | null, requestHeaders: Array, status: number, statusReason: string | null, state: HttpResponseState, url: string, version: string | null, }; export type HttpResponseHeader = { name: string, value: string, }; diff --git a/plugins/importer-curl/src/index.ts b/plugins/importer-curl/src/index.ts index 49834ecc..5e4b6d98 100644 --- a/plugins/importer-curl/src/index.ts +++ b/plugins/importer-curl/src/index.ts @@ -194,11 +194,17 @@ function importCommand(parseEntries: ParseEntry[], workspaceId: string) { let value: string | boolean; const nextEntry = parseEntries[i + 1]; const hasValue = !BOOLEAN_FLAGS.includes(name); + // Check if nextEntry looks like a flag: + // - Single dash followed by a letter: -X, -H, -d + // - Double dash followed by a letter: --data-raw, --header + // This prevents mistaking data that starts with dashes (like multipart boundaries ------) as flags + const nextEntryIsFlag = typeof nextEntry === 'string' && + (nextEntry.match(/^-[a-zA-Z]/) || nextEntry.match(/^--[a-zA-Z]/)); if (isSingleDash && name.length > 1) { // Handle squished arguments like -XPOST value = name.slice(1); name = name.slice(0, 1); - } else if (typeof nextEntry === 'string' && hasValue && !nextEntry.startsWith('-')) { + } else if (typeof nextEntry === 'string' && hasValue && !nextEntryIsFlag) { // Next arg is not a flag, so assign it as the value value = nextEntry; i++; // Skip next one @@ -305,11 +311,32 @@ function importCommand(parseEntries: ParseEntry[], workspaceId: string) { } // Body (Text or Blob) - const dataParameters = pairsToDataParameters(flagsByName); const contentTypeHeader = headers.find((header) => header.name.toLowerCase() === 'content-type'); - const mimeType = contentTypeHeader ? contentTypeHeader.value.split(';')[0] : null; + const mimeType = contentTypeHeader ? contentTypeHeader.value.split(';')[0]?.trim() : null; - // Body (Multipart Form Data) + // Extract boundary from Content-Type header for multipart parsing + const boundaryMatch = contentTypeHeader?.value.match(/boundary=([^\s;]+)/i); + const boundary = boundaryMatch?.[1]; + + // Get raw data from --data-raw flags (before splitting by &) + const rawDataValues = [ + ...((flagsByName['data-raw'] as string[] | undefined) || []), + ...((flagsByName.d as string[] | undefined) || []), + ...((flagsByName.data as string[] | undefined) || []), + ...((flagsByName['data-binary'] as string[] | undefined) || []), + ...((flagsByName['data-ascii'] as string[] | undefined) || []), + ]; + + // Check if this is multipart form data in --data-raw (Chrome DevTools format) + let multipartFormDataFromRaw: { name: string; value?: string; file?: string; enabled: boolean }[] | null = null; + if (mimeType === 'multipart/form-data' && boundary && rawDataValues.length > 0) { + const rawBody = rawDataValues.join(''); + multipartFormDataFromRaw = parseMultipartFormData(rawBody, boundary); + } + + const dataParameters = pairsToDataParameters(flagsByName); + + // Body (Multipart Form Data from -F flags) const formDataParams = [ ...((flagsByName.form as string[] | undefined) || []), ...((flagsByName.F as string[] | undefined) || []), @@ -336,7 +363,13 @@ function importCommand(parseEntries: ParseEntry[], workspaceId: string) { let bodyType: string | null = null; const bodyAsGET = getPairValue(flagsByName, false, ['G', 'get']); - if (dataParameters.length > 0 && bodyAsGET) { + if (multipartFormDataFromRaw) { + // Handle multipart form data parsed from --data-raw (Chrome DevTools format) + bodyType = 'multipart/form-data'; + body = { + form: multipartFormDataFromRaw, + }; + } else if (dataParameters.length > 0 && bodyAsGET) { urlParameters.push(...dataParameters); } else if ( dataParameters.length > 0 && @@ -473,6 +506,71 @@ function splitOnce(str: string, sep: string): string[] { return [str]; } +/** + * Parses multipart form data from a raw body string + * Used when Chrome DevTools exports a cURL with --data-raw containing multipart data + */ +function parseMultipartFormData( + rawBody: string, + boundary: string, +): { name: string; value?: string; file?: string; enabled: boolean }[] | null { + const results: { name: string; value?: string; file?: string; enabled: boolean }[] = []; + + // The boundary in the body typically has -- prefix + const boundaryMarker = `--${boundary}`; + const parts = rawBody.split(boundaryMarker); + + for (const part of parts) { + // Skip empty parts and the closing boundary marker + if (!part || part.trim() === '--' || part.trim() === '--\r\n') { + continue; + } + + // Each part has headers and content separated by \r\n\r\n + const headerContentSplit = part.indexOf('\r\n\r\n'); + if (headerContentSplit === -1) { + continue; + } + + const headerSection = part.slice(0, headerContentSplit); + let content = part.slice(headerContentSplit + 4); // Skip \r\n\r\n + + // Remove trailing \r\n from content + if (content.endsWith('\r\n')) { + content = content.slice(0, -2); + } + + // Parse Content-Disposition header to get name and filename + const contentDispositionMatch = headerSection.match( + /Content-Disposition:\s*form-data;\s*name="([^"]+)"(?:;\s*filename="([^"]+)")?/i, + ); + + if (!contentDispositionMatch) { + continue; + } + + const name = contentDispositionMatch[1] ?? ''; + const filename = contentDispositionMatch[2]; + + const item: { name: string; value?: string; file?: string; enabled: boolean } = { + name, + enabled: true, + }; + + if (filename) { + // This is a file upload field + item.file = filename; + } else { + // This is a regular text field + item.value = content; + } + + results.push(item); + } + + return results.length > 0 ? results : null; +} + const idCount: Partial> = {}; function generateId(model: string): string { diff --git a/plugins/importer-curl/tests/index.test.ts b/plugins/importer-curl/tests/index.test.ts index 71e8cac1..6f75b8e4 100644 --- a/plugins/importer-curl/tests/index.test.ts +++ b/plugins/importer-curl/tests/index.test.ts @@ -441,6 +441,72 @@ describe('importer-curl', () => { }, }); }); + + test('Imports multipart form data from --data-raw (Chrome DevTools format)', () => { + // This is the format Chrome DevTools uses when copying a multipart form submission as cURL + const curlCommand = `curl 'http://localhost:8080/system' \ + -H 'Content-Type: multipart/form-data; boundary=----WebKitFormBoundaryHwsXKi4rKA6P5VBd' \ + --data-raw $'------WebKitFormBoundaryHwsXKi4rKA6P5VBd\r\nContent-Disposition: form-data; name="username"\r\n\r\njsgj\r\n------WebKitFormBoundaryHwsXKi4rKA6P5VBd\r\nContent-Disposition: form-data; name="password"\r\n\r\n654321\r\n------WebKitFormBoundaryHwsXKi4rKA6P5VBd\r\nContent-Disposition: form-data; name="captcha"; filename="test.xlsx"\r\nContent-Type: application/vnd.openxmlformats-officedocument.spreadsheetml.sheet\r\n\r\n\r\n------WebKitFormBoundaryHwsXKi4rKA6P5VBd--\r\n'`; + + expect(convertCurl(curlCommand)).toEqual({ + resources: { + workspaces: [baseWorkspace()], + httpRequests: [ + baseRequest({ + url: 'http://localhost:8080/system', + method: 'POST', + headers: [ + { + name: 'Content-Type', + value: 'multipart/form-data; boundary=----WebKitFormBoundaryHwsXKi4rKA6P5VBd', + enabled: true, + }, + ], + bodyType: 'multipart/form-data', + body: { + form: [ + { name: 'username', value: 'jsgj', enabled: true }, + { name: 'password', value: '654321', enabled: true }, + { name: 'captcha', file: 'test.xlsx', enabled: true }, + ], + }, + }), + ], + }, + }); + }); + + test('Imports multipart form data with text-only fields from --data-raw', () => { + const curlCommand = `curl 'http://example.com/api' \ + -H 'Content-Type: multipart/form-data; boundary=----FormBoundary123' \ + --data-raw $'------FormBoundary123\r\nContent-Disposition: form-data; name="field1"\r\n\r\nvalue1\r\n------FormBoundary123\r\nContent-Disposition: form-data; name="field2"\r\n\r\nvalue2\r\n------FormBoundary123--\r\n'`; + + expect(convertCurl(curlCommand)).toEqual({ + resources: { + workspaces: [baseWorkspace()], + httpRequests: [ + baseRequest({ + url: 'http://example.com/api', + method: 'POST', + headers: [ + { + name: 'Content-Type', + value: 'multipart/form-data; boundary=----FormBoundary123', + enabled: true, + }, + ], + bodyType: 'multipart/form-data', + body: { + form: [ + { name: 'field1', value: 'value1', enabled: true }, + { name: 'field2', value: 'value2', enabled: true }, + ], + }, + }), + ], + }, + }); + }); }); const idCount: Partial> = {}; diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 30ab8329..a91f4c0a 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -192,12 +192,14 @@ version = "0.4.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b37fc50485c4f3f736a4fb14199f6d5f5ba008d7f28fe710306c92780f004c07" dependencies = [ - "brotli", + "brotli 8.0.1", "flate2", "futures-core", "memchr", "pin-project-lite", "tokio", + "zstd", + "zstd-safe", ] [[package]] @@ -536,6 +538,17 @@ dependencies = [ "syn 2.0.101", ] +[[package]] +name = "brotli" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor 4.0.3", +] + [[package]] name = "brotli" version = "8.0.1" @@ -544,7 +557,17 @@ checksum = "9991eea70ea4f293524138648e41ee89b0b2b12ddef3b255effa43c8056e0e0d" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", - "brotli-decompressor", + "brotli-decompressor 5.0.0", +] + +[[package]] +name = "brotli-decompressor" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a334ef7c9e23abf0ce748e8cd309037da93e606ad52eb372e4ce327a0dcfbdfd" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", ] [[package]] @@ -5762,7 +5785,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa9844cefcf99554a16e0a278156ae73b0d8680bbc0e2ad1e4287aadd8489cf" dependencies = [ "base64 0.22.1", - "brotli", + "brotli 8.0.1", "ico", "json-patch", "plist", @@ -6094,7 +6117,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76a423c51176eb3616ee9b516a9fa67fed5f0e78baaba680e44eb5dd2cc37490" dependencies = [ "anyhow", - "brotli", + "brotli 8.0.1", "cargo_metadata", "ctor", "dunce", @@ -7903,6 +7926,7 @@ dependencies = [ "thiserror 2.0.17", "tokio", "tokio-stream", + "tokio-util", "ts-rs", "uuid", "yaak-common", @@ -7929,6 +7953,7 @@ dependencies = [ "regex", "reqwest", "serde", + "serde_json", "tauri", "thiserror 2.0.17", ] @@ -8010,19 +8035,30 @@ dependencies = [ name = "yaak-http" version = "0.1.0" dependencies = [ + "async-compression", + "async-trait", + "brotli 7.0.0", + "bytes", + "flate2", + "futures-util", "hyper-util", "log", + "mime_guess", "regex", "reqwest", "reqwest_cookie_store", "serde", + "serde_json", "tauri", "thiserror 2.0.17", "tokio", + "tokio-util", "tower-service", "urlencoding", + "yaak-common", "yaak-models", "yaak-tls", + "zstd", ] [[package]] diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index af5ce6c2..19be0850 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -74,6 +74,7 @@ tauri-plugin-window-state = "2.4.1" thiserror = { workspace = true } tokio = { workspace = true, features = ["sync"] } tokio-stream = "0.1.17" +tokio-util = { version = "0.7", features = ["codec"] } ts-rs = { workspace = true } uuid = "1.12.1" yaak-common = { workspace = true } diff --git a/src-tauri/src/error.rs b/src-tauri/src/error.rs index 4797fb0d..e0541b51 100644 --- a/src-tauri/src/error.rs +++ b/src-tauri/src/error.rs @@ -59,7 +59,7 @@ pub enum Error { #[error("Request error: {0}")] RequestError(#[from] reqwest::Error), - #[error("Generic error: {0}")] + #[error("{0}")] GenericError(String), } diff --git a/src-tauri/src/http_request.rs b/src-tauri/src/http_request.rs index dfde8946..64230cbf 100644 --- a/src-tauri/src/http_request.rs +++ b/src-tauri/src/http_request.rs @@ -1,33 +1,30 @@ use crate::error::Error::GenericError; use crate::error::Result; use crate::render::render_http_request; -use crate::response_err; -use http::header::{ACCEPT, USER_AGENT}; -use http::{HeaderMap, HeaderName, HeaderValue}; -use log::{debug, error, warn}; -use mime_guess::Mime; -use reqwest::{Method, Response}; -use reqwest::{Url, multipart}; +use log::{debug, warn}; use reqwest_cookie_store::{CookieStore, CookieStoreMutex}; -use serde_json::Value; -use std::collections::BTreeMap; -use std::path::PathBuf; -use std::str::FromStr; +use std::pin::Pin; use std::sync::Arc; -use std::time::Duration; -use tauri::{Manager, Runtime, WebviewWindow}; -use tokio::fs; +use std::time::{Duration, Instant}; +use tauri::{AppHandle, Manager, Runtime, WebviewWindow}; use tokio::fs::{File, create_dir_all}; -use tokio::io::AsyncWriteExt; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; use tokio::sync::watch::Receiver; -use tokio::sync::{Mutex, oneshot}; +use tokio_util::bytes::Bytes; use yaak_http::client::{ HttpConnectionOptions, HttpConnectionProxySetting, HttpConnectionProxySettingAuth, }; use yaak_http::manager::HttpConnectionManager; +use yaak_http::sender::ReqwestSender; +use yaak_http::tee_reader::TeeReader; +use yaak_http::transaction::HttpTransaction; +use yaak_http::types::{ + SendableBody, SendableHttpRequest, SendableHttpRequestOptions, append_query_params, +}; +use yaak_models::blob_manager::{BlobManagerExt, BodyChunk}; use yaak_models::models::{ - Cookie, CookieJar, Environment, HttpRequest, HttpResponse, HttpResponseHeader, - HttpResponseState, ProxySetting, ProxySettingAuth, + Cookie, CookieJar, Environment, HttpRequest, HttpResponse, HttpResponseEvent, + HttpResponseHeader, HttpResponseState, ProxySetting, ProxySettingAuth, }; use yaak_models::query_manager::QueryManagerExt; use yaak_models::util::UpdateSource; @@ -36,9 +33,58 @@ use yaak_plugins::events::{ }; use yaak_plugins::manager::PluginManager; use yaak_plugins::template_callback::PluginTemplateCallback; -use yaak_templates::{RenderErrorBehavior, RenderOptions}; +use yaak_templates::RenderOptions; use yaak_tls::find_client_certificate; +/// Chunk size for storing request bodies (1MB) +const REQUEST_BODY_CHUNK_SIZE: usize = 1024 * 1024; + +/// Context for managing response state during HTTP transactions. +/// Handles both persisted responses (stored in DB) and ephemeral responses (in-memory only). +struct ResponseContext { + app_handle: AppHandle, + response: HttpResponse, + update_source: UpdateSource, +} + +impl ResponseContext { + fn new(app_handle: AppHandle, response: HttpResponse, update_source: UpdateSource) -> Self { + Self { app_handle, response, update_source } + } + + /// Whether this response is persisted (has a non-empty ID) + fn is_persisted(&self) -> bool { + !self.response.id.is_empty() + } + + /// Update the response state. For persisted responses, fetches from DB, applies the + /// closure, and updates the DB. For ephemeral responses, just applies the closure + /// to the in-memory response. + fn update(&mut self, func: F) -> Result<()> + where + F: FnOnce(&mut HttpResponse), + { + if self.is_persisted() { + let r = self.app_handle.with_tx(|tx| { + let mut r = tx.get_http_response(&self.response.id)?; + func(&mut r); + tx.update_http_response_if_id(&r, &self.update_source)?; + Ok(r) + })?; + self.response = r; + Ok(()) + } else { + func(&mut self.response); + Ok(()) + } + } + + /// Get the current response state + fn response(&self) -> &HttpResponse { + &self.response + } +} + pub async fn send_http_request( window: &WebviewWindow, unrendered_request: &HttpRequest, @@ -65,62 +111,81 @@ pub async fn send_http_request_with_context( og_response: &HttpResponse, environment: Option, cookie_jar: Option, - cancelled_rx: &mut Receiver, + cancelled_rx: &Receiver, plugin_context: &PluginContext, +) -> Result { + let app_handle = window.app_handle().clone(); + let update_source = UpdateSource::from_window(window); + let mut response_ctx = + ResponseContext::new(app_handle.clone(), og_response.clone(), update_source); + + // Execute the inner send logic and handle errors consistently + let start = Instant::now(); + let result = send_http_request_inner( + window, + unrendered_request, + environment, + cookie_jar, + cancelled_rx, + plugin_context, + &mut response_ctx, + ) + .await; + + match result { + Ok(response) => Ok(response), + Err(e) => { + let error = e.to_string(); + let elapsed = start.elapsed().as_millis() as i32; + warn!("Failed to send request: {error:?}"); + let _ = response_ctx.update(|r| { + r.state = HttpResponseState::Closed; + r.elapsed = elapsed; + if r.elapsed_headers == 0 { + r.elapsed_headers = elapsed; + } + r.error = Some(error); + }); + Ok(response_ctx.response().clone()) + } + } +} + +async fn send_http_request_inner( + window: &WebviewWindow, + unrendered_request: &HttpRequest, + environment: Option, + cookie_jar: Option, + cancelled_rx: &Receiver, + plugin_context: &PluginContext, + response_ctx: &mut ResponseContext, ) -> Result { let app_handle = window.app_handle().clone(); let plugin_manager = app_handle.state::(); let connection_manager = app_handle.state::(); let settings = window.db().get_settings(); - let workspace = window.db().get_workspace(&unrendered_request.workspace_id)?; + let workspace_id = &unrendered_request.workspace_id; + let folder_id = unrendered_request.folder_id.as_deref(); let environment_id = environment.map(|e| e.id); - let environment_chain = window.db().resolve_environments( - &unrendered_request.workspace_id, - unrendered_request.folder_id.as_deref(), - environment_id.as_deref(), - )?; - - let response_id = og_response.id.clone(); - let response = Arc::new(Mutex::new(og_response.clone())); - - let update_source = UpdateSource::from_window(window); - - let (resolved_request, auth_context_id) = match resolve_http_request(window, unrendered_request) - { - Ok(r) => r, - Err(e) => { - return Ok(response_err( - &app_handle, - &*response.lock().await, - e.to_string(), - &update_source, - )); - } - }; - + let workspace = window.db().get_workspace(workspace_id)?; + let (resolved, auth_context_id) = resolve_http_request(window, unrendered_request)?; let cb = PluginTemplateCallback::new(window.app_handle(), &plugin_context, RenderPurpose::Send); + let env_chain = + window.db().resolve_environments(&workspace.id, folder_id, environment_id.as_deref())?; + let request = render_http_request(&resolved, env_chain, &cb, &RenderOptions::throw()).await?; - let opt = RenderOptions { error_behavior: RenderErrorBehavior::Throw }; - - let request = match render_http_request(&resolved_request, environment_chain, &cb, &opt).await { - Ok(r) => r, - Err(e) => { - return Ok(response_err( - &app_handle, - &*response.lock().await, - e.to_string(), - &update_source, - )); - } + // Build the sendable request using the new SendableHttpRequest type + let options = SendableHttpRequestOptions { + follow_redirects: workspace.setting_follow_redirects, + timeout: if workspace.setting_request_timeout > 0 { + Some(Duration::from_millis(workspace.setting_request_timeout.unsigned_abs() as u64)) + } else { + None + }, }; + let mut sendable_request = SendableHttpRequest::from_http_request(&request, options).await?; - let mut url_string = request.url.clone(); - - url_string = ensure_proto(&url_string); - if !url_string.starts_with("http://") && !url_string.starts_with("https://") { - url_string = format!("http://{}", url_string); - } - debug!("Sending request to {} {url_string}", request.method); + debug!("Sending request to {} {}", sendable_request.method, sendable_request.url); let proxy_setting = match settings.proxy { None => HttpConnectionProxySetting::System, @@ -144,7 +209,8 @@ pub async fn send_http_request_with_context( } }; - let client_certificate = find_client_certificate(&url_string, &settings.client_certificates); + let client_certificate = + find_client_certificate(&sendable_request.url, &settings.client_certificates); // Add cookie store if specified let maybe_cookie_manager = match cookie_jar.clone() { @@ -175,523 +241,73 @@ pub async fn send_http_request_with_context( let client = connection_manager .get_client(&HttpConnectionOptions { id: plugin_context.id.clone(), - follow_redirects: workspace.setting_follow_redirects, validate_certificates: workspace.setting_validate_certificates, proxy: proxy_setting, cookie_provider: maybe_cookie_manager.as_ref().map(|(p, _)| Arc::clone(&p)), client_certificate, - timeout: if workspace.setting_request_timeout > 0 { - Some(Duration::from_millis(workspace.setting_request_timeout.unsigned_abs() as u64)) - } else { - None - }, }) .await?; - // Render query parameters - let mut query_params = Vec::new(); - for p in request.url_parameters.clone() { - if !p.enabled || p.name.is_empty() { - continue; - } - query_params.push((p.name, p.value)); - } + // Apply authentication to the request + apply_authentication( + &window, + &mut sendable_request, + &request, + auth_context_id, + &plugin_manager, + plugin_context, + ) + .await?; - let url = match Url::from_str(&url_string) { - Ok(u) => u, - Err(e) => { - return Ok(response_err( - &app_handle, - &*response.lock().await, - format!("Failed to parse URL \"{}\": {}", url_string, e.to_string()), - &update_source, - )); + let result = + execute_transaction(client, sendable_request, response_ctx, cancelled_rx.clone()).await; + + // Wait for blob writing to complete and check for errors + let final_result = match result { + Ok((response, maybe_blob_write_handle)) => { + // Check if blob writing failed + if let Some(handle) = maybe_blob_write_handle { + if let Ok(Err(e)) = handle.await { + // Update response with the storage error + let _ = response_ctx.update(|r| { + let error_msg = + format!("Request succeeded but failed to store request body: {}", e); + r.error = Some(match &r.error { + Some(existing) => format!("{}; {}", existing, error_msg), + None => error_msg, + }); + }); + } + } + Ok(response) } + Err(e) => Err(e), }; - let m = Method::from_str(&request.method.to_uppercase()) - .map_err(|e| GenericError(e.to_string()))?; - let mut request_builder = client.request(m, url).query(&query_params); - - let mut headers = HeaderMap::new(); - headers.insert(USER_AGENT, HeaderValue::from_static("yaak")); - headers.insert(ACCEPT, HeaderValue::from_static("*/*")); - - // TODO: Set cookie header ourselves once we also handle redirects. We need to do this - // because reqwest doesn't give us a way to inspect the headers it sent (we have to do - // everything manually to know that). - // if let Some(cookie_store) = maybe_cookie_store.clone() { - // let values1 = cookie_store.get_request_values(&url); - // let raw_value = cookie_store.get_request_values(&url) - // .map(|(name, value)| format!("{}={}", name, value)) - // .collect::>() - // .join("; "); - // headers.insert( - // COOKIE, - // HeaderValue::from_str(&raw_value).expect("Failed to create cookie header"), - // ); - // } - - for h in request.headers.clone() { - if h.name.is_empty() && h.value.is_empty() { - continue; - } - - if !h.enabled { - continue; - } - - let header_name = match HeaderName::from_str(&h.name) { - Ok(n) => n, - Err(e) => { - error!("Failed to create header name: {}", e); - continue; - } - }; - let header_value = match HeaderValue::from_str(&h.value) { - Ok(n) => n, - Err(e) => { - error!("Failed to create header value: {}", e); - continue; - } - }; - - headers.insert(header_name, header_value); - } - - let request_body = request.body.clone(); - if let Some(body_type) = &request.body_type.clone() { - if body_type == "graphql" { - let query = get_str_h(&request_body, "query"); - let variables = get_str_h(&request_body, "variables"); - if request.method.to_lowercase() == "get" { - request_builder = request_builder.query(&[("query", query)]); - if !variables.trim().is_empty() { - request_builder = request_builder.query(&[("variables", variables)]); - } - } else { - let body = if variables.trim().is_empty() { - format!(r#"{{"query":{}}}"#, serde_json::to_string(query).unwrap_or_default()) - } else { - format!( - r#"{{"query":{},"variables":{variables}}}"#, - serde_json::to_string(query).unwrap_or_default() - ) - }; - request_builder = request_builder.body(body.to_owned()); - } - } else if body_type == "application/x-www-form-urlencoded" - && request_body.contains_key("form") - { - let mut form_params = Vec::new(); - let form = request_body.get("form"); - if let Some(f) = form { - match f.as_array() { - None => {} - Some(a) => { - for p in a { - let enabled = get_bool(p, "enabled", true); - let name = get_str(p, "name"); - if !enabled || name.is_empty() { - continue; - } - let value = get_str(p, "value"); - form_params.push((name, value)); - } - } - } - } - request_builder = request_builder.form(&form_params); - } else if body_type == "binary" && request_body.contains_key("filePath") { - let file_path = request_body - .get("filePath") - .ok_or(GenericError("filePath not set".to_string()))? - .as_str() - .unwrap_or_default(); - - match fs::read(file_path).await.map_err(|e| e.to_string()) { - Ok(f) => { - request_builder = request_builder.body(f); - } - Err(e) => { - return Ok(response_err( - &app_handle, - &*response.lock().await, - e, - &update_source, - )); - } - } - } else if body_type == "multipart/form-data" && request_body.contains_key("form") { - let mut multipart_form = multipart::Form::new(); - if let Some(form_definition) = request_body.get("form") { - match form_definition.as_array() { - None => {} - Some(fd) => { - for p in fd { - let enabled = get_bool(p, "enabled", true); - let name = get_str(p, "name").to_string(); - - if !enabled || name.is_empty() { - continue; - } - - let file_path = get_str(p, "file").to_owned(); - let value = get_str(p, "value").to_owned(); - - let mut part = if file_path.is_empty() { - multipart::Part::text(value.clone()) - } else { - match fs::read(file_path.clone()).await { - Ok(f) => multipart::Part::bytes(f), - Err(e) => { - return Ok(response_err( - &app_handle, - &*response.lock().await, - e.to_string(), - &update_source, - )); - } - } - }; - - let content_type = get_str(p, "contentType"); - - // Set or guess mimetype - if !content_type.is_empty() { - part = match part.mime_str(content_type) { - Ok(p) => p, - Err(e) => { - return Ok(response_err( - &app_handle, - &*response.lock().await, - format!("Invalid mime for multi-part entry {e:?}"), - &update_source, - )); - } - }; - } else if !file_path.is_empty() { - let default_mime = - Mime::from_str("application/octet-stream").unwrap(); - let mime = - mime_guess::from_path(file_path.clone()).first_or(default_mime); - part = match part.mime_str(mime.essence_str()) { - Ok(p) => p, - Err(e) => { - return Ok(response_err( - &app_handle, - &*response.lock().await, - format!("Invalid mime for multi-part entry {e:?}"), - &update_source, - )); - } - }; - } - - // Set a file path if it is not empty - if !file_path.is_empty() { - let user_filename = get_str(p, "filename").to_owned(); - let filename = if user_filename.is_empty() { - PathBuf::from(file_path) - .file_name() - .unwrap_or_default() - .to_string_lossy() - .to_string() - } else { - user_filename - }; - part = part.file_name(filename); - } - - multipart_form = multipart_form.part(name, part); - } - } - } - } - headers.remove("Content-Type"); // reqwest will add this automatically - request_builder = request_builder.multipart(multipart_form); - } else if request_body.contains_key("text") { - let body = get_str_h(&request_body, "text"); - request_builder = request_builder.body(body.to_owned()); - } else { - warn!("Unsupported body type: {}", body_type); - } - } else { - // No body set - let method = request.method.to_ascii_lowercase(); - let is_body_method = method == "post" || method == "put" || method == "patch"; - // Add Content-Length for methods that commonly accept a body because some servers - // will error if they don't receive it. - if is_body_method && !headers.contains_key("content-length") { - headers.insert("Content-Length", HeaderValue::from_static("0")); - } - } - - // Add headers last, because previous steps may modify them - request_builder = request_builder.headers(headers); - - let mut sendable_req = match request_builder.build() { - Ok(r) => r, - Err(e) => { - warn!("Failed to build request builder {e:?}"); - return Ok(response_err( - &app_handle, - &*response.lock().await, - e.to_string(), - &update_source, - )); - } - }; - - match request.authentication_type { - None => { - // No authentication found. Not even inherited - } - Some(authentication_type) if authentication_type == "none" => { - // Explicitly no authentication - } - Some(authentication_type) => { - let req = CallHttpAuthenticationRequest { - context_id: format!("{:x}", md5::compute(auth_context_id)), - values: serde_json::from_value(serde_json::to_value(&request.authentication)?)?, - url: sendable_req.url().to_string(), - method: sendable_req.method().to_string(), - headers: sendable_req - .headers() - .iter() - .map(|(name, value)| HttpHeader { - name: name.to_string(), - value: value.to_str().unwrap_or_default().to_string(), + // Persist cookies back to the database after the request completes + if let Some((cookie_store, mut cj)) = maybe_cookie_manager { + match cookie_store.lock() { + Ok(store) => { + let cookies: Vec = store + .iter_any() + .filter_map(|c| { + // Convert cookie_store::Cookie -> yaak_models::Cookie via serde + let json_cookie = serde_json::to_value(c).ok()?; + serde_json::from_value(json_cookie).ok() }) - .collect(), - }; - let auth_result = plugin_manager - .call_http_authentication(&window, &authentication_type, req, plugin_context) - .await; - let plugin_result = match auth_result { - Ok(r) => r, - Err(e) => { - return Ok(response_err( - &app_handle, - &*response.lock().await, - e.to_string(), - &update_source, - )); + .collect(); + cj.cookies = cookies; + if let Err(e) = window.db().upsert_cookie_jar(&cj, &UpdateSource::Background) { + warn!("Failed to persist cookies to database: {}", e); } - }; - - let headers = sendable_req.headers_mut(); - for header in plugin_result.set_headers.unwrap_or_default() { - match (HeaderName::from_str(&header.name), HeaderValue::from_str(&header.value)) { - (Ok(name), Ok(value)) => { - headers.insert(name, value); - } - _ => continue, - }; } - - if let Some(params) = plugin_result.set_query_parameters { - let mut query_pairs = sendable_req.url_mut().query_pairs_mut(); - for p in params { - query_pairs.append_pair(&p.name, &p.value); - } + Err(e) => { + warn!("Failed to lock cookie store: {}", e); } } } - let (resp_tx, resp_rx) = oneshot::channel::>(); - let (done_tx, done_rx) = oneshot::channel::(); - - let start = std::time::Instant::now(); - - tokio::spawn(async move { - let _ = resp_tx.send(client.execute(sendable_req).await); - }); - - let raw_response = tokio::select! { - Ok(r) = resp_rx => r, - _ = cancelled_rx.changed() => { - let mut r = response.lock().await; - r.elapsed_headers = start.elapsed().as_millis() as i32; - r.elapsed = start.elapsed().as_millis() as i32; - return Ok(response_err(&app_handle, &r, "Request was cancelled".to_string(), &update_source)); - } - }; - - { - let app_handle = app_handle.clone(); - let window = window.clone(); - let cancelled_rx = cancelled_rx.clone(); - let response_id = response_id.clone(); - let response = response.clone(); - let update_source = update_source.clone(); - tokio::spawn(async move { - match raw_response { - Ok(mut v) => { - let content_length = v.content_length(); - let response_headers = v.headers().clone(); - let dir = app_handle.path().app_data_dir().unwrap(); - let base_dir = dir.join("responses"); - create_dir_all(base_dir.clone()).await.expect("Failed to create responses dir"); - let body_path = if response_id.is_empty() { - base_dir.join(uuid::Uuid::new_v4().to_string()) - } else { - base_dir.join(response_id.clone()) - }; - - { - let mut r = response.lock().await; - r.body_path = Some(body_path.to_str().unwrap().to_string()); - r.elapsed_headers = start.elapsed().as_millis() as i32; - r.elapsed = start.elapsed().as_millis() as i32; - r.status = v.status().as_u16() as i32; - r.status_reason = v.status().canonical_reason().map(|s| s.to_string()); - r.headers = response_headers - .iter() - .map(|(k, v)| HttpResponseHeader { - name: k.as_str().to_string(), - value: v.to_str().unwrap_or_default().to_string(), - }) - .collect(); - r.url = v.url().to_string(); - r.remote_addr = v.remote_addr().map(|a| a.to_string()); - r.version = match v.version() { - reqwest::Version::HTTP_09 => Some("HTTP/0.9".to_string()), - reqwest::Version::HTTP_10 => Some("HTTP/1.0".to_string()), - reqwest::Version::HTTP_11 => Some("HTTP/1.1".to_string()), - reqwest::Version::HTTP_2 => Some("HTTP/2".to_string()), - reqwest::Version::HTTP_3 => Some("HTTP/3".to_string()), - _ => None, - }; - - r.state = HttpResponseState::Connected; - app_handle - .db() - .update_http_response_if_id(&r, &update_source) - .expect("Failed to update response after connected"); - } - - // Write body to FS - let mut f = File::options() - .create(true) - .truncate(true) - .write(true) - .open(&body_path) - .await - .expect("Failed to open file"); - - let mut written_bytes: usize = 0; - loop { - let chunk = v.chunk().await; - if *cancelled_rx.borrow() { - // Request was canceled - return; - } - match chunk { - Ok(Some(bytes)) => { - let mut r = response.lock().await; - r.elapsed = start.elapsed().as_millis() as i32; - f.write_all(&bytes).await.expect("Failed to write to file"); - f.flush().await.expect("Failed to flush file"); - written_bytes += bytes.len(); - r.content_length = Some(written_bytes as i32); - app_handle - .db() - .update_http_response_if_id(&r, &update_source) - .expect("Failed to update response"); - } - Ok(None) => { - break; - } - Err(e) => { - response_err( - &app_handle, - &*response.lock().await, - e.to_string(), - &update_source, - ); - break; - } - } - } - - // Set the final content length - { - let mut r = response.lock().await; - r.content_length = match content_length { - Some(l) => Some(l as i32), - None => Some(written_bytes as i32), - }; - r.state = HttpResponseState::Closed; - app_handle - .db() - .update_http_response_if_id(&r, &UpdateSource::from_window(&window)) - .expect("Failed to update response"); - }; - - // Add cookie store if specified - if let Some((cookie_store, mut cookie_jar)) = maybe_cookie_manager { - // let cookies = response_headers.get_all(SET_COOKIE).iter().map(|h| { - // println!("RESPONSE COOKIE: {}", h.to_str().unwrap()); - // cookie_store::RawCookie::from_str(h.to_str().unwrap()) - // .expect("Failed to parse cookie") - // }); - // store.store_response_cookies(cookies, &url); - - let json_cookies: Vec = cookie_store - .lock() - .unwrap() - .iter_any() - .map(|c| { - let json_cookie = - serde_json::to_value(&c).expect("Failed to serialize cookie"); - serde_json::from_value(json_cookie) - .expect("Failed to deserialize cookie") - }) - .collect::>(); - cookie_jar.cookies = json_cookies; - if let Err(e) = app_handle - .db() - .upsert_cookie_jar(&cookie_jar, &UpdateSource::from_window(&window)) - { - error!("Failed to update cookie jar: {}", e); - }; - } - } - Err(e) => { - warn!("Failed to execute request {e}"); - response_err( - &app_handle, - &*response.lock().await, - format!("{e} → {e:?}"), - &update_source, - ); - } - }; - - let r = response.lock().await.clone(); - done_tx.send(r).unwrap(); - }); - }; - - let app_handle = app_handle.clone(); - Ok(tokio::select! { - Ok(r) = done_rx => r, - _ = cancelled_rx.changed() => { - match app_handle.with_db(|c| c.get_http_response(&response_id)) { - Ok(mut r) => { - r.state = HttpResponseState::Closed; - r.elapsed = start.elapsed().as_millis() as i32; - r.elapsed_headers = start.elapsed().as_millis() as i32; - app_handle.db().update_http_response_if_id(&r, &UpdateSource::from_window(window)) - .expect("Failed to update response") - }, - _ => { - response_err(&app_handle, &*response.lock().await, "Ephemeral request was cancelled".to_string(), &update_source) - }.clone(), - } - } - }) + final_result } pub fn resolve_http_request( @@ -711,46 +327,365 @@ pub fn resolve_http_request( Ok((new_request, authentication_context_id)) } -fn ensure_proto(url_str: &str) -> String { - if url_str.starts_with("http://") || url_str.starts_with("https://") { - return url_str.to_string(); - } +async fn execute_transaction( + client: reqwest::Client, + mut sendable_request: SendableHttpRequest, + response_ctx: &mut ResponseContext, + mut cancelled_rx: Receiver, +) -> Result<(HttpResponse, Option>>)> { + let app_handle = &response_ctx.app_handle.clone(); + let response_id = response_ctx.response().id.clone(); + let workspace_id = response_ctx.response().workspace_id.clone(); + let is_persisted = response_ctx.is_persisted(); - // Url::from_str will fail without a proto, so add one - let parseable_url = format!("http://{}", url_str); - if let Ok(u) = Url::from_str(parseable_url.as_str()) { - match u.host() { - Some(host) => { - let h = host.to_string(); - // These TLDs force HTTPS - if h.ends_with(".app") || h.ends_with(".dev") || h.ends_with(".page") { - return format!("https://{url_str}"); + let sender = ReqwestSender::with_client(client); + let transaction = HttpTransaction::new(sender); + let start = Instant::now(); + + // Capture request headers before sending + let request_headers: Vec = sendable_request + .headers + .iter() + .map(|(name, value)| HttpResponseHeader { name: name.clone(), value: value.clone() }) + .collect(); + + // Update response with headers info + response_ctx.update(|r| { + r.url = sendable_request.url.clone(); + r.request_headers = request_headers; + })?; + + // Create bounded channel for receiving events and spawn a task to store them in DB + // Buffer size of 100 events provides backpressure if DB writes are slow + let (event_tx, mut event_rx) = + tokio::sync::mpsc::channel::(100); + + // Write events to DB in a task (only for persisted responses) + if is_persisted { + let response_id = response_id.clone(); + let app_handle = app_handle.clone(); + let update_source = response_ctx.update_source.clone(); + let workspace_id = workspace_id.clone(); + tokio::spawn(async move { + while let Some(event) = event_rx.recv().await { + let db_event = HttpResponseEvent::new(&response_id, &workspace_id, event.into()); + let _ = app_handle.db().upsert_http_response_event(&db_event, &update_source); + } + }); + } else { + // For ephemeral responses, just drain the events + tokio::spawn(async move { while event_rx.recv().await.is_some() {} }); + }; + + // Capture request body as it's sent (only for persisted responses) + let body_id = format!("{}.request", response_id); + let maybe_blob_write_handle = match sendable_request.body { + Some(SendableBody::Bytes(bytes)) => { + if is_persisted { + write_bytes_to_db_sync(response_ctx, &body_id, bytes.clone())?; + } + sendable_request.body = Some(SendableBody::Bytes(bytes)); + None + } + Some(SendableBody::Stream(stream)) => { + // Wrap stream with TeeReader to capture data as it's read + // Use unbounded channel to ensure all data is captured without blocking the HTTP request + let (body_chunk_tx, body_chunk_rx) = tokio::sync::mpsc::unbounded_channel::>(); + let tee_reader = TeeReader::new(stream, body_chunk_tx); + let pinned: Pin> = Box::pin(tee_reader); + + let handle = if is_persisted { + // Spawn task to write request body chunks to blob DB + let app_handle = app_handle.clone(); + let response_id = response_id.clone(); + let workspace_id = workspace_id.clone(); + let body_id = body_id.clone(); + let update_source = response_ctx.update_source.clone(); + Some(tauri::async_runtime::spawn(async move { + write_stream_chunks_to_db( + app_handle, + &body_id, + &workspace_id, + &response_id, + &update_source, + body_chunk_rx, + ) + .await + })) + } else { + // For ephemeral responses, just drain the body chunks + tauri::async_runtime::spawn(async move { + let mut rx = body_chunk_rx; + while rx.recv().await.is_some() {} + }); + None + }; + + sendable_request.body = Some(SendableBody::Stream(pinned)); + handle + } + None => { + sendable_request.body = None; + None + } + }; + + // Execute the transaction with cancellation support + // This returns the response with headers, but body is not yet consumed + // Events (headers, settings, chunks) are sent through the channel + let mut http_response = transaction + .execute_with_cancellation(sendable_request, cancelled_rx.clone(), event_tx) + .await?; + + // Prepare the response path before consuming the body + let body_path = if response_id.is_empty() { + // Ephemeral responses: use OS temp directory for automatic cleanup + let temp_dir = std::env::temp_dir().join("yaak-ephemeral-responses"); + create_dir_all(&temp_dir).await?; + temp_dir.join(uuid::Uuid::new_v4().to_string()) + } else { + // Persisted responses: use app data directory + let dir = app_handle.path().app_data_dir()?; + let base_dir = dir.join("responses"); + create_dir_all(&base_dir).await?; + base_dir.join(&response_id) + }; + + // Extract metadata before consuming the body (headers are available immediately) + // Url might change, so update again + response_ctx.update(|r| { + r.body_path = Some(body_path.to_string_lossy().to_string()); + r.elapsed_headers = start.elapsed().as_millis() as i32; + r.status = http_response.status as i32; + r.status_reason = http_response.status_reason.clone(); + r.url = http_response.url.clone(); + r.remote_addr = http_response.remote_addr.clone(); + r.version = http_response.version.clone(); + r.headers = http_response + .headers + .iter() + .map(|(name, value)| HttpResponseHeader { name: name.clone(), value: value.clone() }) + .collect(); + r.content_length = http_response.content_length.map(|l| l as i64); + r.state = HttpResponseState::Connected; + r.request_headers = http_response + .request_headers + .iter() + .map(|(n, v)| HttpResponseHeader { name: n.clone(), value: v.clone() }) + .collect(); + })?; + + // Get the body stream for manual consumption + let mut body_stream = http_response.into_body_stream()?; + + // Open file for writing + let mut file = File::options() + .create(true) + .truncate(true) + .write(true) + .open(&body_path) + .await + .map_err(|e| GenericError(format!("Failed to open file: {}", e)))?; + + // Stream body to file, with throttled DB updates to avoid excessive writes + let mut written_bytes: usize = 0; + let mut last_update_time = start; + let mut buf = [0u8; 8192]; + + // Throttle settings: update DB at most every 100ms + const UPDATE_INTERVAL_MS: u128 = 100; + + loop { + // Check for cancellation. If we already have headers/body, just close cleanly without error + if *cancelled_rx.borrow() { + break; + } + + // Use select! to race between reading and cancellation, so cancellation is immediate + let read_result = tokio::select! { + biased; + _ = cancelled_rx.changed() => { + break; + } + result = body_stream.read(&mut buf) => result, + }; + + match read_result { + Ok(0) => break, // EOF + Ok(n) => { + file.write_all(&buf[..n]) + .await + .map_err(|e| GenericError(format!("Failed to write to file: {}", e)))?; + file.flush() + .await + .map_err(|e| GenericError(format!("Failed to flush file: {}", e)))?; + written_bytes += n; + + // Throttle DB updates: only update if enough time has passed + let now = Instant::now(); + let elapsed_since_update = now.duration_since(last_update_time).as_millis(); + + if elapsed_since_update >= UPDATE_INTERVAL_MS { + response_ctx.update(|r| { + r.elapsed = start.elapsed().as_millis() as i32; + r.content_length = Some(written_bytes as i64); + })?; + last_update_time = now; } } - None => {} + Err(e) => { + return Err(GenericError(format!("Failed to read response body: {}", e))); + } } } - format!("http://{url_str}") + // Final update with closed state and accurate byte count + response_ctx.update(|r| { + r.elapsed = start.elapsed().as_millis() as i32; + r.content_length = Some(written_bytes as i64); + r.state = HttpResponseState::Closed; + })?; + + Ok((response_ctx.response().clone(), maybe_blob_write_handle)) } -fn get_bool(v: &Value, key: &str, fallback: bool) -> bool { - match v.get(key) { - None => fallback, - Some(v) => v.as_bool().unwrap_or(fallback), +fn write_bytes_to_db_sync( + response_ctx: &mut ResponseContext, + body_id: &str, + data: Bytes, +) -> Result<()> { + if data.is_empty() { + return Ok(()); } + + // Write in chunks if data is large + let mut offset = 0; + let mut chunk_index = 0; + while offset < data.len() { + let end = std::cmp::min(offset + REQUEST_BODY_CHUNK_SIZE, data.len()); + let chunk_data = data.slice(offset..end).to_vec(); + let chunk = BodyChunk::new(body_id, chunk_index, chunk_data); + response_ctx.app_handle.blobs().insert_chunk(&chunk)?; + offset = end; + chunk_index += 1; + } + + // Update the response with the total request body size + response_ctx.update(|r| { + r.request_content_length = Some(data.len() as i64); + })?; + + Ok(()) } -fn get_str<'a>(v: &'a Value, key: &str) -> &'a str { - match v.get(key) { - None => "", - Some(v) => v.as_str().unwrap_or_default(), +async fn write_stream_chunks_to_db( + app_handle: AppHandle, + body_id: &str, + workspace_id: &str, + response_id: &str, + update_source: &UpdateSource, + mut rx: tokio::sync::mpsc::UnboundedReceiver>, +) -> Result<()> { + let mut buffer = Vec::with_capacity(REQUEST_BODY_CHUNK_SIZE); + let mut chunk_index = 0; + let mut total_bytes: usize = 0; + + while let Some(data) = rx.recv().await { + total_bytes += data.len(); + buffer.extend_from_slice(&data); + + // Flush when buffer reaches chunk size + while buffer.len() >= REQUEST_BODY_CHUNK_SIZE { + debug!("Writing chunk {chunk_index} to DB"); + let chunk_data: Vec = buffer.drain(..REQUEST_BODY_CHUNK_SIZE).collect(); + let chunk = BodyChunk::new(body_id, chunk_index, chunk_data); + app_handle.blobs().insert_chunk(&chunk)?; + app_handle.db().upsert_http_response_event( + &HttpResponseEvent::new( + response_id, + workspace_id, + yaak_http::sender::HttpResponseEvent::ChunkSent { + bytes: REQUEST_BODY_CHUNK_SIZE, + } + .into(), + ), + update_source, + )?; + chunk_index += 1; + } } + + // Flush remaining data + if !buffer.is_empty() { + let chunk = BodyChunk::new(body_id, chunk_index, buffer); + debug!("Flushing remaining data {chunk_index} {}", chunk.data.len()); + app_handle.blobs().insert_chunk(&chunk)?; + app_handle.db().upsert_http_response_event( + &HttpResponseEvent::new( + response_id, + workspace_id, + yaak_http::sender::HttpResponseEvent::ChunkSent { bytes: chunk.data.len() }.into(), + ), + update_source, + )?; + } + + // Update the response with the total request body size + app_handle.with_tx(|tx| { + debug!("Updating final body length {total_bytes}"); + if let Ok(mut response) = tx.get_http_response(&response_id) { + response.request_content_length = Some(total_bytes as i64); + tx.update_http_response_if_id(&response, update_source)?; + } + Ok(()) + })?; + + Ok(()) } -fn get_str_h<'a>(v: &'a BTreeMap, key: &str) -> &'a str { - match v.get(key) { - None => "", - Some(v) => v.as_str().unwrap_or_default(), +async fn apply_authentication( + window: &WebviewWindow, + sendable_request: &mut SendableHttpRequest, + request: &HttpRequest, + auth_context_id: String, + plugin_manager: &PluginManager, + plugin_context: &PluginContext, +) -> Result<()> { + match &request.authentication_type { + None => { + // No authentication found. Not even inherited + } + Some(authentication_type) if authentication_type == "none" => { + // Explicitly no authentication + } + Some(authentication_type) => { + let req = CallHttpAuthenticationRequest { + context_id: format!("{:x}", md5::compute(auth_context_id)), + values: serde_json::from_value(serde_json::to_value(&request.authentication)?)?, + url: sendable_request.url.clone(), + method: sendable_request.method.clone(), + headers: sendable_request + .headers + .iter() + .map(|(name, value)| HttpHeader { + name: name.to_string(), + value: value.to_string(), + }) + .collect(), + }; + let plugin_result = plugin_manager + .call_http_authentication(&window, &authentication_type, req, plugin_context) + .await?; + + for header in plugin_result.set_headers.unwrap_or_default() { + sendable_request.insert_header((header.name, header.value)); + } + + if let Some(params) = plugin_result.set_query_parameters { + let params = params.into_iter().map(|p| (p.name, p.value)).collect::>(); + sendable_request.url = append_query_params(&sendable_request.url, params); + } + } } + Ok(()) } diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 680758b3..89be63ae 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -32,10 +32,11 @@ use yaak_common::window::WorkspaceWindowTrait; use yaak_grpc::manager::GrpcHandle; use yaak_grpc::{Code, ServiceDefinition, serialize_message}; use yaak_mac_window::AppHandleMacWindowExt; +use yaak_models::blob_manager::BlobManagerExt; use yaak_models::models::{ AnyModel, CookieJar, Environment, GrpcConnection, GrpcConnectionState, GrpcEvent, - GrpcEventType, GrpcRequest, HttpRequest, HttpResponse, HttpResponseState, Plugin, Workspace, - WorkspaceMeta, + GrpcEventType, GrpcRequest, HttpRequest, HttpResponse, HttpResponseEvent, HttpResponseState, + Plugin, Workspace, WorkspaceMeta, }; use yaak_models::query_manager::QueryManagerExt; use yaak_models::util::{BatchUpsertResult, UpdateSource, get_workspace_export_resources}; @@ -785,7 +786,7 @@ async fn cmd_http_response_body( ) -> YaakResult { let body_path = match response.body_path { None => { - return Err(GenericError("Response body path not set".to_string())); + return Ok(FilterResponse { content: String::new(), error: None }); } Some(p) => p, }; @@ -810,6 +811,23 @@ async fn cmd_http_response_body( } } +#[tauri::command] +async fn cmd_http_request_body( + app_handle: AppHandle, + response_id: &str, +) -> YaakResult>> { + let body_id = format!("{}.request", response_id); + let chunks = app_handle.blobs().get_chunks(&body_id)?; + + if chunks.is_empty() { + return Ok(None); + } + + // Concatenate all chunks + let body: Vec = chunks.into_iter().flat_map(|c| c.data).collect(); + Ok(Some(body)) +} + #[tauri::command] async fn cmd_get_sse_events(file_path: &str) -> YaakResult> { let body = fs::read(file_path)?; @@ -831,6 +849,15 @@ async fn cmd_get_sse_events(file_path: &str) -> YaakResult> Ok(events) } +#[tauri::command] +async fn cmd_get_http_response_events( + app_handle: AppHandle, + response_id: &str, +) -> YaakResult> { + let events: Vec = app_handle.db().list_http_response_events(response_id)?; + Ok(events) +} + #[tauri::command] async fn cmd_import_data( window: WebviewWindow, @@ -1139,6 +1166,7 @@ async fn cmd_send_http_request( // that has not yet been saved in the DB. request: HttpRequest, ) -> YaakResult { + let blobs = app_handle.blob_manager(); let response = app_handle.db().upsert_http_response( &HttpResponse { request_id: request.id.clone(), @@ -1146,6 +1174,7 @@ async fn cmd_send_http_request( ..Default::default() }, &UpdateSource::from_window(&window), + &blobs, )?; let (cancel_tx, mut cancel_rx) = tokio::sync::watch::channel(false); @@ -1191,6 +1220,7 @@ async fn cmd_send_http_request( ..resp }, &UpdateSource::from_window(&window), + &blobs, )? } }; @@ -1198,23 +1228,6 @@ async fn cmd_send_http_request( Ok(r) } -fn response_err( - app_handle: &AppHandle, - response: &HttpResponse, - error: String, - update_source: &UpdateSource, -) -> HttpResponse { - warn!("Failed to send request: {error:?}"); - let mut response = response.clone(); - response.state = HttpResponseState::Closed; - response.error = Some(error.clone()); - response = app_handle - .db() - .update_http_response_if_id(&response, update_source) - .expect("Failed to update response"); - response -} - #[tauri::command] async fn cmd_install_plugin( directory: &str, @@ -1493,11 +1506,13 @@ pub fn run() { cmd_delete_send_history, cmd_dismiss_notification, cmd_export_data, + cmd_http_request_body, cmd_http_response_body, cmd_format_json, cmd_get_http_authentication_summaries, cmd_get_http_authentication_config, cmd_get_sse_events, + cmd_get_http_response_events, cmd_get_workspace_meta, cmd_grpc_go, cmd_grpc_reflect, diff --git a/src-tauri/src/plugin_events.rs b/src-tauri/src/plugin_events.rs index 9087418c..5d20eff7 100644 --- a/src-tauri/src/plugin_events.rs +++ b/src-tauri/src/plugin_events.rs @@ -12,6 +12,7 @@ use log::error; use tauri::{AppHandle, Emitter, Manager, Runtime}; use tauri_plugin_clipboard_manager::ClipboardExt; use yaak_common::window::WorkspaceWindowTrait; +use yaak_models::blob_manager::BlobManagerExt; use yaak_models::models::{HttpResponse, Plugin}; use yaak_models::queries::any_request::AnyRequest; use yaak_models::query_manager::QueryManagerExt; @@ -219,6 +220,7 @@ pub(crate) async fn handle_plugin_event( let http_response = if http_request.id.is_empty() { HttpResponse::default() } else { + let blobs = window.blob_manager(); window.db().upsert_http_response( &HttpResponse { request_id: http_request.id.clone(), @@ -226,6 +228,7 @@ pub(crate) async fn handle_plugin_event( ..Default::default() }, &UpdateSource::Plugin, + &blobs, )? }; diff --git a/src-tauri/src/render.rs b/src-tauri/src/render.rs index f915bda9..f5ad8fad 100644 --- a/src-tauri/src/render.rs +++ b/src-tauri/src/render.rs @@ -157,7 +157,7 @@ pub async fn render_http_request( let url = parse_and_render(r.url.clone().as_str(), vars, cb, &opt).await?; // This doesn't fit perfectly with the concept of "rendering" but it kind of does - let (url, url_parameters) = apply_path_placeholders(&url, url_parameters); + let (url, url_parameters) = apply_path_placeholders(&url, &url_parameters); Ok(HttpRequest { url, url_parameters, headers, body, authentication, ..r.to_owned() }) } diff --git a/src-tauri/yaak-common/Cargo.toml b/src-tauri/yaak-common/Cargo.toml index be033751..3e872f12 100644 --- a/src-tauri/yaak-common/Cargo.toml +++ b/src-tauri/yaak-common/Cargo.toml @@ -10,3 +10,4 @@ reqwest = { workspace = true, features = ["system-proxy", "gzip"] } thiserror = { workspace = true } regex = "1.11.0" serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } diff --git a/src-tauri/yaak-common/src/lib.rs b/src-tauri/yaak-common/src/lib.rs index d315fede..cac9bcc6 100644 --- a/src-tauri/yaak-common/src/lib.rs +++ b/src-tauri/yaak-common/src/lib.rs @@ -1,4 +1,5 @@ pub mod api_client; pub mod error; pub mod platform; +pub mod serde; pub mod window; diff --git a/src-tauri/yaak-common/src/serde.rs b/src-tauri/yaak-common/src/serde.rs new file mode 100644 index 00000000..683cc25d --- /dev/null +++ b/src-tauri/yaak-common/src/serde.rs @@ -0,0 +1,23 @@ +use serde_json::Value; +use std::collections::BTreeMap; + +pub fn get_bool(v: &Value, key: &str, fallback: bool) -> bool { + match v.get(key) { + None => fallback, + Some(v) => v.as_bool().unwrap_or(fallback), + } +} + +pub fn get_str<'a>(v: &'a Value, key: &str) -> &'a str { + match v.get(key) { + None => "", + Some(v) => v.as_str().unwrap_or_default(), + } +} + +pub fn get_str_map<'a>(v: &'a BTreeMap, key: &str) -> &'a str { + match v.get(key) { + None => "", + Some(v) => v.as_str().unwrap_or_default(), + } +} diff --git a/src-tauri/yaak-http/Cargo.toml b/src-tauri/yaak-http/Cargo.toml index c51ff89f..ec7367a5 100644 --- a/src-tauri/yaak-http/Cargo.toml +++ b/src-tauri/yaak-http/Cargo.toml @@ -5,16 +5,27 @@ edition = "2024" publish = false [dependencies] +async-compression = { version = "0.4", features = ["tokio", "gzip", "deflate", "brotli", "zstd"] } +async-trait = "0.1" +brotli = "7" +bytes = "1.5.0" +flate2 = "1" +futures-util = "0.3" +zstd = "0.13" hyper-util = { version = "0.1.17", default-features = false, features = ["client-legacy"] } log = { workspace = true } +mime_guess = "2.0.5" regex = "1.11.1" -reqwest = { workspace = true, features = ["multipart", "cookies", "gzip", "brotli", "deflate", "json", "rustls-tls-manual-roots-no-provider", "socks", "http2"] } +reqwest = { workspace = true, features = ["cookies", "rustls-tls-manual-roots-no-provider", "socks", "http2", "stream"] } reqwest_cookie_store = { workspace = true } serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } tauri = { workspace = true } thiserror = { workspace = true } -tokio = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt", "fs", "io-util"] } +tokio-util = { version = "0.7", features = ["codec", "io", "io-util"] } tower-service = "0.3.3" urlencoding = "2.1.3" +yaak-common = { workspace = true } yaak-models = { workspace = true } yaak-tls = { workspace = true } diff --git a/src-tauri/yaak-http/src/chained_reader.rs b/src-tauri/yaak-http/src/chained_reader.rs new file mode 100644 index 00000000..7b5f10a3 --- /dev/null +++ b/src-tauri/yaak-http/src/chained_reader.rs @@ -0,0 +1,78 @@ +use std::io; +use std::pin::Pin; +use std::task::{Context, Poll}; +use tokio::io::{AsyncRead, ReadBuf}; + +/// A stream that chains multiple AsyncRead sources together +pub(crate) struct ChainedReader { + readers: Vec, + current_index: usize, + current_reader: Option>, +} + +#[derive(Clone)] +pub(crate) enum ReaderType { + Bytes(Vec), + FilePath(String), +} + +impl ChainedReader { + pub(crate) fn new(readers: Vec) -> Self { + Self { readers, current_index: 0, current_reader: None } + } +} + +impl AsyncRead for ChainedReader { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + loop { + // Try to read from current reader if we have one + if let Some(ref mut reader) = self.current_reader { + let before_len = buf.filled().len(); + return match Pin::new(reader).poll_read(cx, buf) { + Poll::Ready(Ok(())) => { + if buf.filled().len() == before_len && buf.remaining() > 0 { + // Current reader is exhausted, move to next + self.current_reader = None; + continue; + } + Poll::Ready(Ok(())) + } + Poll::Ready(Err(e)) => Poll::Ready(Err(e)), + Poll::Pending => Poll::Pending, + }; + } + + // We need to get the next reader + if self.current_index >= self.readers.len() { + // No more readers + return Poll::Ready(Ok(())); + } + + // Get the next reader + let reader_type = self.readers[self.current_index].clone(); + self.current_index += 1; + + match reader_type { + ReaderType::Bytes(bytes) => { + self.current_reader = Some(Box::new(io::Cursor::new(bytes))); + } + ReaderType::FilePath(path) => { + // We need to handle file opening synchronously in poll_read + // This is a limitation - we'll use blocking file open + match std::fs::File::open(&path) { + Ok(file) => { + // Convert std File to tokio File + let tokio_file = tokio::fs::File::from_std(file); + self.current_reader = Some(Box::new(tokio_file)); + } + Err(e) => return Poll::Ready(Err(e)), + } + } + } + } + } +} diff --git a/src-tauri/yaak-http/src/client.rs b/src-tauri/yaak-http/src/client.rs index f8d06acc..f43ebaaf 100644 --- a/src-tauri/yaak-http/src/client.rs +++ b/src-tauri/yaak-http/src/client.rs @@ -1,11 +1,9 @@ use crate::dns::LocalhostResolver; use crate::error::Result; use log::{debug, info, warn}; -use reqwest::redirect::Policy; -use reqwest::{Client, Proxy}; +use reqwest::{Client, Proxy, redirect}; use reqwest_cookie_store::CookieStoreMutex; use std::sync::Arc; -use std::time::Duration; use yaak_tls::{ClientCertificateConfig, get_tls_config}; #[derive(Clone)] @@ -29,11 +27,9 @@ pub enum HttpConnectionProxySetting { #[derive(Clone)] pub struct HttpConnectionOptions { pub id: String, - pub follow_redirects: bool, pub validate_certificates: bool, pub proxy: HttpConnectionProxySetting, pub cookie_provider: Option>, - pub timeout: Option, pub client_certificate: Option, } @@ -41,9 +37,11 @@ impl HttpConnectionOptions { pub(crate) fn build_client(&self) -> Result { let mut client = Client::builder() .connection_verbose(true) - .gzip(true) - .brotli(true) - .deflate(true) + .redirect(redirect::Policy::none()) + // Decompression is handled by HttpTransaction, not reqwest + .no_gzip() + .no_brotli() + .no_deflate() .referer(false) .tls_info(true); @@ -55,12 +53,6 @@ impl HttpConnectionOptions { // Configure DNS resolver client = client.dns_resolver(LocalhostResolver::new()); - // Configure redirects - client = client.redirect(match self.follow_redirects { - true => Policy::limited(10), // TODO: Handle redirects natively - false => Policy::none(), - }); - // Configure cookie provider if let Some(p) = &self.cookie_provider { client = client.cookie_provider(Arc::clone(&p)); @@ -79,11 +71,6 @@ impl HttpConnectionOptions { } } - // Configure timeout - if let Some(d) = self.timeout { - client = client.timeout(d); - } - info!( "Building new HTTP client validate_certificates={} client_cert={}", self.validate_certificates, diff --git a/src-tauri/yaak-http/src/decompress.rs b/src-tauri/yaak-http/src/decompress.rs new file mode 100644 index 00000000..e3764ea6 --- /dev/null +++ b/src-tauri/yaak-http/src/decompress.rs @@ -0,0 +1,188 @@ +use crate::error::{Error, Result}; +use async_compression::tokio::bufread::{ + BrotliDecoder, DeflateDecoder as AsyncDeflateDecoder, GzipDecoder, + ZstdDecoder as AsyncZstdDecoder, +}; +use flate2::read::{DeflateDecoder, GzDecoder}; +use std::io::Read; +use tokio::io::{AsyncBufRead, AsyncRead}; + +/// Supported compression encodings +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ContentEncoding { + Gzip, + Deflate, + Brotli, + Zstd, + Identity, +} + +impl ContentEncoding { + /// Parse a Content-Encoding header value into an encoding type. + /// Returns Identity for unknown or missing encodings. + pub fn from_header(value: Option<&str>) -> Self { + match value.map(|s| s.trim().to_lowercase()).as_deref() { + Some("gzip") | Some("x-gzip") => ContentEncoding::Gzip, + Some("deflate") => ContentEncoding::Deflate, + Some("br") => ContentEncoding::Brotli, + Some("zstd") => ContentEncoding::Zstd, + _ => ContentEncoding::Identity, + } + } +} + +/// Result of decompression, containing both the decompressed data and size info +#[derive(Debug)] +pub struct DecompressResult { + pub data: Vec, + pub compressed_size: u64, + pub decompressed_size: u64, +} + +/// Decompress data based on the Content-Encoding. +/// Returns the original data unchanged if encoding is Identity or unknown. +pub fn decompress(data: Vec, encoding: ContentEncoding) -> Result { + let compressed_size = data.len() as u64; + + let decompressed = match encoding { + ContentEncoding::Identity => data, + ContentEncoding::Gzip => decompress_gzip(&data)?, + ContentEncoding::Deflate => decompress_deflate(&data)?, + ContentEncoding::Brotli => decompress_brotli(&data)?, + ContentEncoding::Zstd => decompress_zstd(&data)?, + }; + + let decompressed_size = decompressed.len() as u64; + + Ok(DecompressResult { data: decompressed, compressed_size, decompressed_size }) +} + +fn decompress_gzip(data: &[u8]) -> Result> { + let mut decoder = GzDecoder::new(data); + let mut decompressed = Vec::new(); + decoder + .read_to_end(&mut decompressed) + .map_err(|e| Error::DecompressionError(format!("gzip decompression failed: {}", e)))?; + Ok(decompressed) +} + +fn decompress_deflate(data: &[u8]) -> Result> { + let mut decoder = DeflateDecoder::new(data); + let mut decompressed = Vec::new(); + decoder + .read_to_end(&mut decompressed) + .map_err(|e| Error::DecompressionError(format!("deflate decompression failed: {}", e)))?; + Ok(decompressed) +} + +fn decompress_brotli(data: &[u8]) -> Result> { + let mut decompressed = Vec::new(); + brotli::BrotliDecompress(&mut std::io::Cursor::new(data), &mut decompressed) + .map_err(|e| Error::DecompressionError(format!("brotli decompression failed: {}", e)))?; + Ok(decompressed) +} + +fn decompress_zstd(data: &[u8]) -> Result> { + zstd::stream::decode_all(std::io::Cursor::new(data)) + .map_err(|e| Error::DecompressionError(format!("zstd decompression failed: {}", e))) +} + +/// Create a streaming decompressor that wraps an async reader. +/// Returns an AsyncRead that decompresses data on-the-fly. +pub fn streaming_decoder( + reader: R, + encoding: ContentEncoding, +) -> Box { + match encoding { + ContentEncoding::Identity => Box::new(reader), + ContentEncoding::Gzip => Box::new(GzipDecoder::new(reader)), + ContentEncoding::Deflate => Box::new(AsyncDeflateDecoder::new(reader)), + ContentEncoding::Brotli => Box::new(BrotliDecoder::new(reader)), + ContentEncoding::Zstd => Box::new(AsyncZstdDecoder::new(reader)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use flate2::Compression; + use flate2::write::GzEncoder; + use std::io::Write; + + #[test] + fn test_content_encoding_from_header() { + assert_eq!(ContentEncoding::from_header(Some("gzip")), ContentEncoding::Gzip); + assert_eq!(ContentEncoding::from_header(Some("x-gzip")), ContentEncoding::Gzip); + assert_eq!(ContentEncoding::from_header(Some("GZIP")), ContentEncoding::Gzip); + assert_eq!(ContentEncoding::from_header(Some("deflate")), ContentEncoding::Deflate); + assert_eq!(ContentEncoding::from_header(Some("br")), ContentEncoding::Brotli); + assert_eq!(ContentEncoding::from_header(Some("zstd")), ContentEncoding::Zstd); + assert_eq!(ContentEncoding::from_header(Some("identity")), ContentEncoding::Identity); + assert_eq!(ContentEncoding::from_header(Some("unknown")), ContentEncoding::Identity); + assert_eq!(ContentEncoding::from_header(None), ContentEncoding::Identity); + } + + #[test] + fn test_decompress_identity() { + let data = b"hello world".to_vec(); + let result = decompress(data.clone(), ContentEncoding::Identity).unwrap(); + assert_eq!(result.data, data); + assert_eq!(result.compressed_size, 11); + assert_eq!(result.decompressed_size, 11); + } + + #[test] + fn test_decompress_gzip() { + // Compress some data with gzip + let original = b"hello world, this is a test of gzip compression"; + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(original).unwrap(); + let compressed = encoder.finish().unwrap(); + + let result = decompress(compressed.clone(), ContentEncoding::Gzip).unwrap(); + assert_eq!(result.data, original); + assert_eq!(result.compressed_size, compressed.len() as u64); + assert_eq!(result.decompressed_size, original.len() as u64); + } + + #[test] + fn test_decompress_deflate() { + // Compress some data with deflate + let original = b"hello world, this is a test of deflate compression"; + let mut encoder = flate2::write::DeflateEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(original).unwrap(); + let compressed = encoder.finish().unwrap(); + + let result = decompress(compressed.clone(), ContentEncoding::Deflate).unwrap(); + assert_eq!(result.data, original); + assert_eq!(result.compressed_size, compressed.len() as u64); + assert_eq!(result.decompressed_size, original.len() as u64); + } + + #[test] + fn test_decompress_brotli() { + // Compress some data with brotli + let original = b"hello world, this is a test of brotli compression"; + let mut compressed = Vec::new(); + let mut writer = brotli::CompressorWriter::new(&mut compressed, 4096, 4, 22); + writer.write_all(original).unwrap(); + drop(writer); + + let result = decompress(compressed.clone(), ContentEncoding::Brotli).unwrap(); + assert_eq!(result.data, original); + assert_eq!(result.compressed_size, compressed.len() as u64); + assert_eq!(result.decompressed_size, original.len() as u64); + } + + #[test] + fn test_decompress_zstd() { + // Compress some data with zstd + let original = b"hello world, this is a test of zstd compression"; + let compressed = zstd::stream::encode_all(std::io::Cursor::new(original), 3).unwrap(); + + let result = decompress(compressed.clone(), ContentEncoding::Zstd).unwrap(); + assert_eq!(result.data, original); + assert_eq!(result.compressed_size, compressed.len() as u64); + assert_eq!(result.decompressed_size, original.len() as u64); + } +} diff --git a/src-tauri/yaak-http/src/error.rs b/src-tauri/yaak-http/src/error.rs index a4ef6ac0..bfb063a0 100644 --- a/src-tauri/yaak-http/src/error.rs +++ b/src-tauri/yaak-http/src/error.rs @@ -8,6 +8,21 @@ pub enum Error { #[error(transparent)] TlsError(#[from] yaak_tls::error::Error), + + #[error("Request failed with {0:?}")] + RequestError(String), + + #[error("Request canceled")] + RequestCanceledError, + + #[error("Timeout of {0:?} reached")] + RequestTimeout(std::time::Duration), + + #[error("Decompression error: {0}")] + DecompressionError(String), + + #[error("Failed to read response body: {0}")] + BodyReadError(String), } impl Serialize for Error { diff --git a/src-tauri/yaak-http/src/lib.rs b/src-tauri/yaak-http/src/lib.rs index cdc9fff1..8c25fe7d 100644 --- a/src-tauri/yaak-http/src/lib.rs +++ b/src-tauri/yaak-http/src/lib.rs @@ -2,11 +2,18 @@ use crate::manager::HttpConnectionManager; use tauri::plugin::{Builder, TauriPlugin}; use tauri::{Manager, Runtime}; +mod chained_reader; pub mod client; +pub mod decompress; pub mod dns; pub mod error; pub mod manager; pub mod path_placeholders; +mod proto; +pub mod sender; +pub mod tee_reader; +pub mod transaction; +pub mod types; pub fn init() -> TauriPlugin { Builder::new("yaak-http") diff --git a/src-tauri/yaak-http/src/path_placeholders.rs b/src-tauri/yaak-http/src/path_placeholders.rs index f80b4a5f..9a700e82 100644 --- a/src-tauri/yaak-http/src/path_placeholders.rs +++ b/src-tauri/yaak-http/src/path_placeholders.rs @@ -2,7 +2,7 @@ use yaak_models::models::HttpUrlParameter; pub fn apply_path_placeholders( url: &str, - parameters: Vec, + parameters: &Vec, ) -> (String, Vec) { let mut new_parameters = Vec::new(); @@ -18,7 +18,7 @@ pub fn apply_path_placeholders( // Remove as param if it modified the URL if old_url_string == *url { - new_parameters.push(p); + new_parameters.push(p.to_owned()); } } @@ -156,7 +156,7 @@ mod placeholder_tests { ..Default::default() }; - let (url, url_parameters) = apply_path_placeholders(&req.url, req.url_parameters); + let (url, url_parameters) = apply_path_placeholders(&req.url, &req.url_parameters); // Pattern match back to access it assert_eq!(url, "example.com/aaa/bar"); diff --git a/src-tauri/yaak-http/src/proto.rs b/src-tauri/yaak-http/src/proto.rs new file mode 100644 index 00000000..fd36a0ed --- /dev/null +++ b/src-tauri/yaak-http/src/proto.rs @@ -0,0 +1,29 @@ +use reqwest::Url; +use std::str::FromStr; + +pub(crate) fn ensure_proto(url_str: &str) -> String { + if url_str.is_empty() { + return "".to_string(); + } + + if url_str.starts_with("http://") || url_str.starts_with("https://") { + return url_str.to_string(); + } + + // Url::from_str will fail without a proto, so add one + let parseable_url = format!("http://{}", url_str); + if let Ok(u) = Url::from_str(parseable_url.as_str()) { + match u.host() { + Some(host) => { + let h = host.to_string(); + // These TLDs force HTTPS + if h.ends_with(".app") || h.ends_with(".dev") || h.ends_with(".page") { + return format!("https://{url_str}"); + } + } + None => {} + } + } + + format!("http://{url_str}") +} diff --git a/src-tauri/yaak-http/src/sender.rs b/src-tauri/yaak-http/src/sender.rs new file mode 100644 index 00000000..1d56cff4 --- /dev/null +++ b/src-tauri/yaak-http/src/sender.rs @@ -0,0 +1,483 @@ +use crate::decompress::{ContentEncoding, streaming_decoder}; +use crate::error::{Error, Result}; +use crate::types::{SendableBody, SendableHttpRequest}; +use async_trait::async_trait; +use futures_util::StreamExt; +use reqwest::{Client, Method, Version}; +use std::collections::HashMap; +use std::fmt::Display; +use std::pin::Pin; +use std::task::{Context, Poll}; +use std::time::Duration; +use tokio::io::{AsyncRead, AsyncReadExt, BufReader, ReadBuf}; +use tokio::sync::mpsc; +use tokio_util::io::StreamReader; + +#[derive(Debug, Clone)] +pub enum RedirectBehavior { + /// 307/308: Method and body are preserved + Preserve, + /// 303 or 301/302 with POST: Method changed to GET, body dropped + DropBody, +} + +#[derive(Debug, Clone)] +pub enum HttpResponseEvent { + Setting(String, String), + Info(String), + Redirect { + url: String, + status: u16, + behavior: RedirectBehavior, + }, + SendUrl { + method: String, + path: String, + }, + ReceiveUrl { + version: Version, + status: String, + }, + HeaderUp(String, String), + HeaderDown(String, String), + ChunkSent { + bytes: usize, + }, + ChunkReceived { + bytes: usize, + }, +} + +impl Display for HttpResponseEvent { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + HttpResponseEvent::Setting(name, value) => write!(f, "* Setting {}={}", name, value), + HttpResponseEvent::Info(s) => write!(f, "* {}", s), + HttpResponseEvent::Redirect { url, status, behavior } => { + let behavior_str = match behavior { + RedirectBehavior::Preserve => "preserve", + RedirectBehavior::DropBody => "drop body", + }; + write!(f, "* Redirect {} -> {} ({})", status, url, behavior_str) + } + HttpResponseEvent::SendUrl { method, path } => write!(f, "> {} {}", method, path), + HttpResponseEvent::ReceiveUrl { version, status } => { + write!(f, "< {} {}", version_to_str(version), status) + } + HttpResponseEvent::HeaderUp(name, value) => write!(f, "> {}: {}", name, value), + HttpResponseEvent::HeaderDown(name, value) => write!(f, "< {}: {}", name, value), + HttpResponseEvent::ChunkSent { bytes } => write!(f, "> [{} bytes sent]", bytes), + HttpResponseEvent::ChunkReceived { bytes } => write!(f, "< [{} bytes received]", bytes), + } + } +} + +impl From for yaak_models::models::HttpResponseEventData { + fn from(event: HttpResponseEvent) -> Self { + use yaak_models::models::HttpResponseEventData as D; + match event { + HttpResponseEvent::Setting(name, value) => D::Setting { name, value }, + HttpResponseEvent::Info(message) => D::Info { message }, + HttpResponseEvent::Redirect { url, status, behavior } => D::Redirect { + url, + status, + behavior: match behavior { + RedirectBehavior::Preserve => "preserve".to_string(), + RedirectBehavior::DropBody => "drop_body".to_string(), + }, + }, + HttpResponseEvent::SendUrl { method, path } => D::SendUrl { method, path }, + HttpResponseEvent::ReceiveUrl { version, status } => { + D::ReceiveUrl { version: format!("{:?}", version), status } + } + HttpResponseEvent::HeaderUp(name, value) => D::HeaderUp { name, value }, + HttpResponseEvent::HeaderDown(name, value) => D::HeaderDown { name, value }, + HttpResponseEvent::ChunkSent { bytes } => D::ChunkSent { bytes }, + HttpResponseEvent::ChunkReceived { bytes } => D::ChunkReceived { bytes }, + } + } +} + +/// Statistics about the body after consumption +#[derive(Debug, Default, Clone)] +pub struct BodyStats { + /// Size of the body as received over the wire (before decompression) + pub size_compressed: u64, + /// Size of the body after decompression + pub size_decompressed: u64, +} + +/// An AsyncRead wrapper that sends chunk events as data is read +pub struct TrackingRead { + inner: R, + event_tx: mpsc::Sender, + ended: bool, +} + +impl TrackingRead { + pub fn new(inner: R, event_tx: mpsc::Sender) -> Self { + Self { inner, event_tx, ended: false } + } +} + +impl AsyncRead for TrackingRead { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + let before = buf.filled().len(); + let result = Pin::new(&mut self.inner).poll_read(cx, buf); + if let Poll::Ready(Ok(())) = &result { + let bytes_read = buf.filled().len() - before; + if bytes_read > 0 { + // Ignore send errors - receiver may have been dropped or channel is full + let _ = + self.event_tx.try_send(HttpResponseEvent::ChunkReceived { bytes: bytes_read }); + } else if !self.ended { + self.ended = true; + } + } + result + } +} + +/// Type alias for the body stream +type BodyStream = Pin>; + +/// HTTP response with deferred body consumption. +/// Headers are available immediately after send(), body can be consumed in different ways. +/// Note: Debug is manually implemented since BodyStream doesn't implement Debug. +pub struct HttpResponse { + /// HTTP status code + pub status: u16, + /// HTTP status reason phrase (e.g., "OK", "Not Found") + pub status_reason: Option, + /// Response headers + pub headers: HashMap, + /// Request headers + pub request_headers: HashMap, + /// Content-Length from headers (may differ from actual body size) + pub content_length: Option, + /// Final URL (after redirects) + pub url: String, + /// Remote address of the server + pub remote_addr: Option, + /// HTTP version (e.g., "HTTP/1.1", "HTTP/2") + pub version: Option, + + /// The body stream (consumed when calling bytes(), text(), write_to_file(), or drain()) + body_stream: Option, + /// Content-Encoding for decompression + encoding: ContentEncoding, +} + +impl std::fmt::Debug for HttpResponse { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("HttpResponse") + .field("status", &self.status) + .field("status_reason", &self.status_reason) + .field("headers", &self.headers) + .field("content_length", &self.content_length) + .field("url", &self.url) + .field("remote_addr", &self.remote_addr) + .field("version", &self.version) + .field("body_stream", &"") + .field("encoding", &self.encoding) + .finish() + } +} + +impl HttpResponse { + /// Create a new HttpResponse with an unconsumed body stream + #[allow(clippy::too_many_arguments)] + pub fn new( + status: u16, + status_reason: Option, + headers: HashMap, + request_headers: HashMap, + content_length: Option, + url: String, + remote_addr: Option, + version: Option, + body_stream: BodyStream, + encoding: ContentEncoding, + ) -> Self { + Self { + status, + status_reason, + headers, + request_headers, + content_length, + url, + remote_addr, + version, + body_stream: Some(body_stream), + encoding, + } + } + + /// Consume the body and return it as bytes (loads entire body into memory). + /// Also decompresses the body if Content-Encoding is set. + pub async fn bytes(mut self) -> Result<(Vec, BodyStats)> { + let stream = self.body_stream.take().ok_or_else(|| { + Error::RequestError("Response body has already been consumed".to_string()) + })?; + + let buf_reader = BufReader::new(stream); + let mut decoder = streaming_decoder(buf_reader, self.encoding); + + let mut decompressed = Vec::new(); + let mut bytes_read = 0u64; + + // Read through the decoder in chunks to track compressed size + let mut buf = [0u8; 8192]; + loop { + match decoder.read(&mut buf).await { + Ok(0) => break, + Ok(n) => { + decompressed.extend_from_slice(&buf[..n]); + bytes_read += n as u64; + } + Err(e) => { + return Err(Error::BodyReadError(e.to_string())); + } + } + } + + let stats = BodyStats { + // For now, we can't easily track compressed size when streaming through decoder + // Use content_length as an approximation, or decompressed size if identity encoding + size_compressed: self.content_length.unwrap_or(bytes_read), + size_decompressed: decompressed.len() as u64, + }; + + Ok((decompressed, stats)) + } + + /// Consume the body and return it as a UTF-8 string. + pub async fn text(self) -> Result<(String, BodyStats)> { + let (bytes, stats) = self.bytes().await?; + let text = String::from_utf8(bytes) + .map_err(|e| Error::RequestError(format!("Response is not valid UTF-8: {}", e)))?; + Ok((text, stats)) + } + + /// Take the body stream for manual consumption. + /// Returns an AsyncRead that decompresses on-the-fly if Content-Encoding is set. + /// The caller is responsible for reading and processing the stream. + pub fn into_body_stream(&mut self) -> Result> { + let stream = self.body_stream.take().ok_or_else(|| { + Error::RequestError("Response body has already been consumed".to_string()) + })?; + + let buf_reader = BufReader::new(stream); + let decoder = streaming_decoder(buf_reader, self.encoding); + + Ok(decoder) + } + + /// Discard the body without reading it (useful for redirects). + pub async fn drain(mut self) -> Result<()> { + let stream = self.body_stream.take().ok_or_else(|| { + Error::RequestError("Response body has already been consumed".to_string()) + })?; + + // Just read and discard all bytes + let mut reader = stream; + let mut buf = [0u8; 8192]; + loop { + match reader.read(&mut buf).await { + Ok(0) => break, + Ok(_) => continue, + Err(e) => { + return Err(Error::RequestError(format!( + "Failed to drain response body: {}", + e + ))); + } + } + } + + Ok(()) + } +} + +/// Trait for sending HTTP requests +#[async_trait] +pub trait HttpSender: Send + Sync { + /// Send an HTTP request and return the response with headers. + /// The body is not consumed until you call bytes(), text(), write_to_file(), or drain(). + /// Events are sent through the provided channel. + async fn send( + &self, + request: SendableHttpRequest, + event_tx: mpsc::Sender, + ) -> Result; +} + +/// Reqwest-based implementation of HttpSender +pub struct ReqwestSender { + client: Client, +} + +impl ReqwestSender { + /// Create a new ReqwestSender with a default client + pub fn new() -> Result { + let client = Client::builder().build().map_err(Error::Client)?; + Ok(Self { client }) + } + + /// Create a new ReqwestSender with a custom client + pub fn with_client(client: Client) -> Self { + Self { client } + } +} + +#[async_trait] +impl HttpSender for ReqwestSender { + async fn send( + &self, + request: SendableHttpRequest, + event_tx: mpsc::Sender, + ) -> Result { + // Helper to send events (ignores errors if receiver is dropped or channel is full) + let send_event = |event: HttpResponseEvent| { + let _ = event_tx.try_send(event); + }; + + // Parse the HTTP method + let method = Method::from_bytes(request.method.as_bytes()) + .map_err(|e| Error::RequestError(format!("Invalid HTTP method: {}", e)))?; + + // Build the request + let mut req_builder = self.client.request(method, &request.url); + + // Add headers + for header in request.headers { + req_builder = req_builder.header(&header.0, &header.1); + } + + // Configure timeout + if let Some(d) = request.options.timeout + && !d.is_zero() + { + req_builder = req_builder.timeout(d); + } + + // Add body + match request.body { + None => {} + Some(SendableBody::Bytes(bytes)) => { + req_builder = req_builder.body(bytes); + } + Some(SendableBody::Stream(stream)) => { + // Convert AsyncRead stream to reqwest Body + let stream = tokio_util::io::ReaderStream::new(stream); + let body = reqwest::Body::wrap_stream(stream); + req_builder = req_builder.body(body); + } + } + + // Send the request + let sendable_req = req_builder.build()?; + send_event(HttpResponseEvent::Setting( + "timeout".to_string(), + if request.options.timeout.unwrap_or_default().is_zero() { + "Infinity".to_string() + } else { + format!("{:?}", request.options.timeout) + }, + )); + + send_event(HttpResponseEvent::SendUrl { + path: sendable_req.url().path().to_string(), + method: sendable_req.method().to_string(), + }); + + let mut request_headers = HashMap::new(); + for (name, value) in sendable_req.headers() { + let v = value.to_str().unwrap_or_default().to_string(); + request_headers.insert(name.to_string(), v.clone()); + send_event(HttpResponseEvent::HeaderUp(name.to_string(), v)); + } + send_event(HttpResponseEvent::Info("Sending request to server".to_string())); + + // Map some errors to our own, so they look nicer + let response = self.client.execute(sendable_req).await.map_err(|e| { + if reqwest::Error::is_timeout(&e) { + Error::RequestTimeout( + request.options.timeout.unwrap_or(Duration::from_secs(0)).clone(), + ) + } else { + Error::Client(e) + } + })?; + + let status = response.status().as_u16(); + let status_reason = response.status().canonical_reason().map(|s| s.to_string()); + let url = response.url().to_string(); + let remote_addr = response.remote_addr().map(|a| a.to_string()); + let version = Some(version_to_str(&response.version())); + let content_length = response.content_length(); + + send_event(HttpResponseEvent::ReceiveUrl { + version: response.version(), + status: response.status().to_string(), + }); + + // Extract headers + let mut headers = HashMap::new(); + for (key, value) in response.headers() { + if let Ok(v) = value.to_str() { + send_event(HttpResponseEvent::HeaderDown(key.to_string(), v.to_string())); + headers.insert(key.to_string(), v.to_string()); + } + } + + // Determine content encoding for decompression + // HTTP headers are case-insensitive, so we need to search for any casing + let encoding = ContentEncoding::from_header( + headers + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("content-encoding")) + .map(|(_, v)| v.as_str()), + ); + + // Get the byte stream instead of loading into memory + let byte_stream = response.bytes_stream(); + + // Convert the stream to an AsyncRead + let stream_reader = StreamReader::new( + byte_stream.map(|result| result.map_err(|e| std::io::Error::other(e))), + ); + + // Wrap the stream with tracking to emit chunk received events via the same channel + let tracking_reader = TrackingRead::new(stream_reader, event_tx); + let body_stream: BodyStream = Box::pin(tracking_reader); + + Ok(HttpResponse::new( + status, + status_reason, + headers, + request_headers, + content_length, + url, + remote_addr, + version, + body_stream, + encoding, + )) + } +} + +fn version_to_str(version: &Version) -> String { + match *version { + Version::HTTP_09 => "HTTP/0.9".to_string(), + Version::HTTP_10 => "HTTP/1.0".to_string(), + Version::HTTP_11 => "HTTP/1.1".to_string(), + Version::HTTP_2 => "HTTP/2".to_string(), + Version::HTTP_3 => "HTTP/3".to_string(), + _ => "unknown".to_string(), + } +} diff --git a/src-tauri/yaak-http/src/tee_reader.rs b/src-tauri/yaak-http/src/tee_reader.rs new file mode 100644 index 00000000..2ee70088 --- /dev/null +++ b/src-tauri/yaak-http/src/tee_reader.rs @@ -0,0 +1,159 @@ +use std::io; +use std::pin::Pin; +use std::task::{Context, Poll}; +use tokio::io::{AsyncRead, ReadBuf}; +use tokio::sync::mpsc; + +/// A reader that forwards all read data to a channel while also returning it to the caller. +/// This allows capturing request body data as it's being sent. +/// Uses an unbounded channel to ensure all data is captured without blocking the request. +pub struct TeeReader { + inner: R, + tx: mpsc::UnboundedSender>, +} + +impl TeeReader { + pub fn new(inner: R, tx: mpsc::UnboundedSender>) -> Self { + Self { inner, tx } + } +} + +impl AsyncRead for TeeReader { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + let before_len = buf.filled().len(); + + match Pin::new(&mut self.inner).poll_read(cx, buf) { + Poll::Ready(Ok(())) => { + let after_len = buf.filled().len(); + if after_len > before_len { + // Data was read, send a copy to the channel + let data = buf.filled()[before_len..after_len].to_vec(); + // Send to unbounded channel - this never blocks + // Ignore error if receiver is closed + let _ = self.tx.send(data); + } + Poll::Ready(Ok(())) + } + Poll::Ready(Err(e)) => Poll::Ready(Err(e)), + Poll::Pending => Poll::Pending, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + use tokio::io::AsyncReadExt; + + #[tokio::test] + async fn test_tee_reader_captures_all_data() { + let data = b"Hello, World!"; + let cursor = Cursor::new(data.to_vec()); + let (tx, mut rx) = mpsc::unbounded_channel(); + + let mut tee = TeeReader::new(cursor, tx); + let mut output = Vec::new(); + tee.read_to_end(&mut output).await.unwrap(); + + // Verify the reader returns the correct data + assert_eq!(output, data); + + // Verify the channel received the data + let mut captured = Vec::new(); + while let Ok(chunk) = rx.try_recv() { + captured.extend(chunk); + } + assert_eq!(captured, data); + } + + #[tokio::test] + async fn test_tee_reader_with_chunked_reads() { + let data = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + let cursor = Cursor::new(data.to_vec()); + let (tx, mut rx) = mpsc::unbounded_channel(); + + let mut tee = TeeReader::new(cursor, tx); + + // Read in small chunks + let mut buf = [0u8; 5]; + let mut output = Vec::new(); + loop { + let n = tee.read(&mut buf).await.unwrap(); + if n == 0 { + break; + } + output.extend_from_slice(&buf[..n]); + } + + // Verify the reader returns the correct data + assert_eq!(output, data); + + // Verify the channel received all chunks + let mut captured = Vec::new(); + while let Ok(chunk) = rx.try_recv() { + captured.extend(chunk); + } + assert_eq!(captured, data); + } + + #[tokio::test] + async fn test_tee_reader_empty_data() { + let data: Vec = vec![]; + let cursor = Cursor::new(data.clone()); + let (tx, mut rx) = mpsc::unbounded_channel(); + + let mut tee = TeeReader::new(cursor, tx); + let mut output = Vec::new(); + tee.read_to_end(&mut output).await.unwrap(); + + // Verify empty output + assert!(output.is_empty()); + + // Verify no data was sent to channel + assert!(rx.try_recv().is_err()); + } + + #[tokio::test] + async fn test_tee_reader_works_when_receiver_dropped() { + let data = b"Hello, World!"; + let cursor = Cursor::new(data.to_vec()); + let (tx, rx) = mpsc::unbounded_channel(); + + // Drop the receiver before reading + drop(rx); + + let mut tee = TeeReader::new(cursor, tx); + let mut output = Vec::new(); + + // Should still work even though receiver is dropped + tee.read_to_end(&mut output).await.unwrap(); + assert_eq!(output, data); + } + + #[tokio::test] + async fn test_tee_reader_large_data() { + // Test with 1MB of data + let data: Vec = (0..1024 * 1024).map(|i| (i % 256) as u8).collect(); + let cursor = Cursor::new(data.clone()); + let (tx, mut rx) = mpsc::unbounded_channel(); + + let mut tee = TeeReader::new(cursor, tx); + let mut output = Vec::new(); + tee.read_to_end(&mut output).await.unwrap(); + + // Verify the reader returns the correct data + assert_eq!(output, data); + + // Verify the channel received all data + let mut captured = Vec::new(); + while let Ok(chunk) = rx.try_recv() { + captured.extend(chunk); + } + assert_eq!(captured, data); + } +} diff --git a/src-tauri/yaak-http/src/transaction.rs b/src-tauri/yaak-http/src/transaction.rs new file mode 100644 index 00000000..43a59de0 --- /dev/null +++ b/src-tauri/yaak-http/src/transaction.rs @@ -0,0 +1,391 @@ +use crate::error::Result; +use crate::sender::{HttpResponse, HttpResponseEvent, HttpSender, RedirectBehavior}; +use crate::types::SendableHttpRequest; +use tokio::sync::mpsc; +use tokio::sync::watch::Receiver; + +/// HTTP Transaction that manages the lifecycle of a request, including redirect handling +pub struct HttpTransaction { + sender: S, + max_redirects: usize, +} + +impl HttpTransaction { + /// Create a new transaction with default settings + pub fn new(sender: S) -> Self { + Self { sender, max_redirects: 10 } + } + + /// Create a new transaction with custom max redirects + pub fn with_max_redirects(sender: S, max_redirects: usize) -> Self { + Self { sender, max_redirects } + } + + /// Execute the request with cancellation support. + /// Returns an HttpResponse with unconsumed body - caller decides how to consume it. + /// Events are sent through the provided channel. + pub async fn execute_with_cancellation( + &self, + request: SendableHttpRequest, + mut cancelled_rx: Receiver, + event_tx: mpsc::Sender, + ) -> Result { + let mut redirect_count = 0; + let mut current_url = request.url; + let mut current_method = request.method; + let mut current_headers = request.headers; + let mut current_body = request.body; + + // Helper to send events (ignores errors if receiver is dropped or channel is full) + let send_event = |event: HttpResponseEvent| { + let _ = event_tx.try_send(event); + }; + + loop { + // Check for cancellation before each request + if *cancelled_rx.borrow() { + return Err(crate::error::Error::RequestCanceledError); + } + + // Build request for this iteration + let req = SendableHttpRequest { + url: current_url.clone(), + method: current_method.clone(), + headers: current_headers.clone(), + body: current_body, + options: request.options.clone(), + }; + + // Send the request + send_event(HttpResponseEvent::Setting( + "redirects".to_string(), + request.options.follow_redirects.to_string(), + )); + + // Execute with cancellation support + let response = tokio::select! { + result = self.sender.send(req, event_tx.clone()) => result?, + _ = cancelled_rx.changed() => { + return Err(crate::error::Error::RequestCanceledError); + } + }; + + if !Self::is_redirect(response.status) { + // Not a redirect - return the response for caller to consume body + return Ok(response); + } + + if !request.options.follow_redirects { + // Redirects disabled - return the redirect response as-is + return Ok(response); + } + + // Check if we've exceeded max redirects + if redirect_count >= self.max_redirects { + // Drain the response before returning error + let _ = response.drain().await; + return Err(crate::error::Error::RequestError(format!( + "Maximum redirect limit ({}) exceeded", + self.max_redirects + ))); + } + + // Extract Location header before draining (headers are available immediately) + // HTTP headers are case-insensitive, so we need to search for any casing + let location = response + .headers + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("location")) + .map(|(_, v)| v.clone()) + .ok_or_else(|| { + crate::error::Error::RequestError( + "Redirect response missing Location header".to_string(), + ) + })?; + + // Also get status before draining + let status = response.status; + + send_event(HttpResponseEvent::Info("Ignoring the response body".to_string())); + + // Drain the redirect response body before following + response.drain().await?; + + // Update the request URL + current_url = if location.starts_with("http://") || location.starts_with("https://") { + // Absolute URL + location + } else if location.starts_with('/') { + // Absolute path - need to extract base URL from current request + let base_url = Self::extract_base_url(¤t_url)?; + format!("{}{}", base_url, location) + } else { + // Relative path - need to resolve relative to current path + let base_path = Self::extract_base_path(¤t_url)?; + format!("{}/{}", base_path, location) + }; + + // Determine redirect behavior based on status code and method + let behavior = if status == 303 { + // 303 See Other always changes to GET + RedirectBehavior::DropBody + } else if (status == 301 || status == 302) && current_method == "POST" { + // For 301/302, change POST to GET (common browser behavior) + RedirectBehavior::DropBody + } else { + // For 307 and 308, the method and body are preserved + // Also for 301/302 with non-POST methods + RedirectBehavior::Preserve + }; + + send_event(HttpResponseEvent::Redirect { + url: current_url.clone(), + status, + behavior: behavior.clone(), + }); + + // Handle method changes for certain redirect codes + if matches!(behavior, RedirectBehavior::DropBody) { + if current_method != "GET" { + current_method = "GET".to_string(); + } + // Remove content-related headers + current_headers.retain(|h| { + let name_lower = h.0.to_lowercase(); + !name_lower.starts_with("content-") && name_lower != "transfer-encoding" + }); + } + + // Reset body for next iteration (since it was moved in the send call) + // For redirects that change method to GET or for all redirects since body was consumed + current_body = None; + + redirect_count += 1; + } + } + + /// Check if a status code indicates a redirect + fn is_redirect(status: u16) -> bool { + matches!(status, 301 | 302 | 303 | 307 | 308) + } + + /// Extract the base URL (scheme + host) from a full URL + fn extract_base_url(url: &str) -> Result { + // Find the position after "://" + let scheme_end = url.find("://").ok_or_else(|| { + crate::error::Error::RequestError(format!("Invalid URL format: {}", url)) + })?; + + // Find the first '/' after the scheme + let path_start = url[scheme_end + 3..].find('/'); + + if let Some(idx) = path_start { + Ok(url[..scheme_end + 3 + idx].to_string()) + } else { + // No path, return entire URL + Ok(url.to_string()) + } + } + + /// Extract the base path (everything except the last segment) from a URL + fn extract_base_path(url: &str) -> Result { + if let Some(last_slash) = url.rfind('/') { + // Don't include the trailing slash if it's part of the host + if url[..last_slash].ends_with("://") || url[..last_slash].ends_with(':') { + Ok(url.to_string()) + } else { + Ok(url[..last_slash].to_string()) + } + } else { + Ok(url.to_string()) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::decompress::ContentEncoding; + use crate::sender::{HttpResponseEvent, HttpSender}; + use async_trait::async_trait; + use std::collections::HashMap; + use std::pin::Pin; + use std::sync::Arc; + use tokio::io::AsyncRead; + use tokio::sync::Mutex; + + /// Mock sender for testing + struct MockSender { + responses: Arc>>, + } + + struct MockResponse { + status: u16, + headers: HashMap, + body: Vec, + } + + impl MockSender { + fn new(responses: Vec) -> Self { + Self { responses: Arc::new(Mutex::new(responses)) } + } + } + + #[async_trait] + impl HttpSender for MockSender { + async fn send( + &self, + _request: SendableHttpRequest, + _event_tx: mpsc::Sender, + ) -> Result { + let mut responses = self.responses.lock().await; + if responses.is_empty() { + Err(crate::error::Error::RequestError("No more mock responses".to_string())) + } else { + let mock = responses.remove(0); + // Create a simple in-memory stream from the body + let body_stream: Pin> = + Box::pin(std::io::Cursor::new(mock.body)); + Ok(HttpResponse::new( + mock.status, + None, // status_reason + mock.headers, + HashMap::new(), + None, // content_length + "https://example.com".to_string(), // url + None, // remote_addr + Some("HTTP/1.1".to_string()), // version + body_stream, + ContentEncoding::Identity, + )) + } + } + } + + #[tokio::test] + async fn test_transaction_no_redirect() { + let response = MockResponse { status: 200, headers: HashMap::new(), body: b"OK".to_vec() }; + let sender = MockSender::new(vec![response]); + let transaction = HttpTransaction::new(sender); + + let request = SendableHttpRequest { + url: "https://example.com".to_string(), + method: "GET".to_string(), + headers: vec![], + ..Default::default() + }; + + let (_tx, rx) = tokio::sync::watch::channel(false); + let (event_tx, _event_rx) = mpsc::channel(100); + let result = transaction.execute_with_cancellation(request, rx, event_tx).await.unwrap(); + assert_eq!(result.status, 200); + + // Consume the body to verify it + let (body, _) = result.bytes().await.unwrap(); + assert_eq!(body, b"OK"); + } + + #[tokio::test] + async fn test_transaction_single_redirect() { + let mut redirect_headers = HashMap::new(); + redirect_headers.insert("Location".to_string(), "https://example.com/new".to_string()); + + let responses = vec![ + MockResponse { status: 302, headers: redirect_headers, body: vec![] }, + MockResponse { status: 200, headers: HashMap::new(), body: b"Final".to_vec() }, + ]; + + let sender = MockSender::new(responses); + let transaction = HttpTransaction::new(sender); + + let request = SendableHttpRequest { + url: "https://example.com/old".to_string(), + method: "GET".to_string(), + options: crate::types::SendableHttpRequestOptions { + follow_redirects: true, + ..Default::default() + }, + ..Default::default() + }; + + let (_tx, rx) = tokio::sync::watch::channel(false); + let (event_tx, _event_rx) = mpsc::channel(100); + let result = transaction.execute_with_cancellation(request, rx, event_tx).await.unwrap(); + assert_eq!(result.status, 200); + + let (body, _) = result.bytes().await.unwrap(); + assert_eq!(body, b"Final"); + } + + #[tokio::test] + async fn test_transaction_max_redirects_exceeded() { + let mut redirect_headers = HashMap::new(); + redirect_headers.insert("Location".to_string(), "https://example.com/loop".to_string()); + + // Create more redirects than allowed + let responses: Vec = (0..12) + .map(|_| MockResponse { status: 302, headers: redirect_headers.clone(), body: vec![] }) + .collect(); + + let sender = MockSender::new(responses); + let transaction = HttpTransaction::with_max_redirects(sender, 10); + + let request = SendableHttpRequest { + url: "https://example.com/start".to_string(), + method: "GET".to_string(), + options: crate::types::SendableHttpRequestOptions { + follow_redirects: true, + ..Default::default() + }, + ..Default::default() + }; + + let (_tx, rx) = tokio::sync::watch::channel(false); + let (event_tx, _event_rx) = mpsc::channel(100); + let result = transaction.execute_with_cancellation(request, rx, event_tx).await; + if let Err(crate::error::Error::RequestError(msg)) = result { + assert!(msg.contains("Maximum redirect limit")); + } else { + panic!("Expected RequestError with max redirect message. Got {result:?}"); + } + } + + #[test] + fn test_is_redirect() { + assert!(HttpTransaction::::is_redirect(301)); + assert!(HttpTransaction::::is_redirect(302)); + assert!(HttpTransaction::::is_redirect(303)); + assert!(HttpTransaction::::is_redirect(307)); + assert!(HttpTransaction::::is_redirect(308)); + assert!(!HttpTransaction::::is_redirect(200)); + assert!(!HttpTransaction::::is_redirect(404)); + assert!(!HttpTransaction::::is_redirect(500)); + } + + #[test] + fn test_extract_base_url() { + let result = + HttpTransaction::::extract_base_url("https://example.com/path/to/resource"); + assert_eq!(result.unwrap(), "https://example.com"); + + let result = HttpTransaction::::extract_base_url("http://localhost:8080/api"); + assert_eq!(result.unwrap(), "http://localhost:8080"); + + let result = HttpTransaction::::extract_base_url("invalid-url"); + assert!(result.is_err()); + } + + #[test] + fn test_extract_base_path() { + let result = HttpTransaction::::extract_base_path( + "https://example.com/path/to/resource", + ); + assert_eq!(result.unwrap(), "https://example.com/path/to"); + + let result = HttpTransaction::::extract_base_path("https://example.com/single"); + assert_eq!(result.unwrap(), "https://example.com"); + + let result = HttpTransaction::::extract_base_path("https://example.com/"); + assert_eq!(result.unwrap(), "https://example.com"); + } +} diff --git a/src-tauri/yaak-http/src/types.rs b/src-tauri/yaak-http/src/types.rs new file mode 100644 index 00000000..135ce8bb --- /dev/null +++ b/src-tauri/yaak-http/src/types.rs @@ -0,0 +1,981 @@ +use crate::chained_reader::{ChainedReader, ReaderType}; +use crate::error::Error::RequestError; +use crate::error::Result; +use crate::path_placeholders::apply_path_placeholders; +use crate::proto::ensure_proto; +use bytes::Bytes; +use log::warn; +use std::collections::BTreeMap; +use std::pin::Pin; +use std::time::Duration; +use tokio::io::AsyncRead; +use yaak_common::serde::{get_bool, get_str, get_str_map}; +use yaak_models::models::HttpRequest; + +pub(crate) const MULTIPART_BOUNDARY: &str = "------YaakFormBoundary"; + +pub enum SendableBody { + Bytes(Bytes), + Stream(Pin>), +} + +enum SendableBodyWithMeta { + Bytes(Bytes), + Stream { + data: Pin>, + content_length: Option, + }, +} + +impl From for SendableBody { + fn from(value: SendableBodyWithMeta) -> Self { + match value { + SendableBodyWithMeta::Bytes(b) => SendableBody::Bytes(b), + SendableBodyWithMeta::Stream { data, .. } => SendableBody::Stream(data), + } + } +} + +#[derive(Default)] +pub struct SendableHttpRequest { + pub url: String, + pub method: String, + pub headers: Vec<(String, String)>, + pub body: Option, + pub options: SendableHttpRequestOptions, +} + +#[derive(Default, Clone)] +pub struct SendableHttpRequestOptions { + pub timeout: Option, + pub follow_redirects: bool, +} + +impl SendableHttpRequest { + pub async fn from_http_request( + r: &HttpRequest, + options: SendableHttpRequestOptions, + ) -> Result { + let initial_headers = build_headers(r); + let (body, headers) = build_body(&r.method, &r.body_type, &r.body, initial_headers).await?; + + Ok(Self { + url: build_url(r), + method: r.method.to_uppercase(), + headers, + body: body.into(), + options, + }) + } + + pub fn insert_header(&mut self, header: (String, String)) { + if let Some(existing) = + self.headers.iter_mut().find(|h| h.0.to_lowercase() == header.0.to_lowercase()) + { + existing.1 = header.1; + } else { + self.headers.push(header); + } + } +} + +pub fn append_query_params(url: &str, params: Vec<(String, String)>) -> String { + let url_string = url.to_string(); + if params.is_empty() { + return url.to_string(); + } + + // Build query string + let query_string = params + .iter() + .map(|(name, value)| { + format!("{}={}", urlencoding::encode(name), urlencoding::encode(value)) + }) + .collect::>() + .join("&"); + + // Split URL into parts: base URL, query, and fragment + let (base_and_query, fragment) = if let Some(hash_pos) = url_string.find('#') { + let (before_hash, after_hash) = url_string.split_at(hash_pos); + (before_hash.to_string(), Some(after_hash.to_string())) + } else { + (url_string, None) + }; + + // Now handle query parameters on the base URL (without fragment) + let mut result = if base_and_query.contains('?') { + // Check if there's already a query string after the '?' + let parts: Vec<&str> = base_and_query.splitn(2, '?').collect(); + if parts.len() == 2 && !parts[1].trim().is_empty() { + // Append with & if there are existing parameters + format!("{}&{}", base_and_query, query_string) + } else { + // Just append the new parameters directly (URL ends with '?') + format!("{}{}", base_and_query, query_string) + } + } else { + // No existing query parameters, add with '?' + format!("{}?{}", base_and_query, query_string) + }; + + // Re-append the fragment if it exists + if let Some(fragment) = fragment { + result.push_str(&fragment); + } + + result +} + +fn build_url(r: &HttpRequest) -> String { + let (url_string, params) = apply_path_placeholders(&ensure_proto(&r.url), &r.url_parameters); + append_query_params( + &url_string, + params + .iter() + .filter(|p| p.enabled && !p.name.is_empty()) + .map(|p| (p.name.clone(), p.value.clone())) + .collect(), + ) +} + +fn build_headers(r: &HttpRequest) -> Vec<(String, String)> { + r.headers + .iter() + .filter_map(|h| { + if h.enabled && !h.name.is_empty() { + Some((h.name.clone(), h.value.clone())) + } else { + None + } + }) + .collect() +} + +async fn build_body( + method: &str, + body_type: &Option, + body: &BTreeMap, + headers: Vec<(String, String)>, +) -> Result<(Option, Vec<(String, String)>)> { + let body_type = match &body_type { + None => return Ok((None, headers)), + Some(t) => t, + }; + + let (body, content_type) = match body_type.as_str() { + "binary" => (build_binary_body(&body).await?, None), + "graphql" => (build_graphql_body(&method, &body), Some("application/json".to_string())), + "application/x-www-form-urlencoded" => { + (build_form_body(&body), Some("application/x-www-form-urlencoded".to_string())) + } + "multipart/form-data" => build_multipart_body(&body, &headers).await?, + _ if body.contains_key("text") => (build_text_body(&body), None), + t => { + warn!("Unsupported body type: {}", t); + (None, None) + } + }; + + // Add or update the Content-Type header + let mut headers = headers; + if let Some(ct) = content_type { + if let Some(existing) = headers.iter_mut().find(|h| h.0.to_lowercase() == "content-type") { + existing.1 = ct; + } else { + headers.push(("Content-Type".to_string(), ct)); + } + } + + // Check if Transfer-Encoding: chunked is already set + let has_chunked_encoding = headers.iter().any(|h| { + h.0.to_lowercase() == "transfer-encoding" && h.1.to_lowercase().contains("chunked") + }); + + // Add a Content-Length header only if chunked encoding is not being used + if !has_chunked_encoding { + let content_length = match body { + Some(SendableBodyWithMeta::Bytes(ref bytes)) => Some(bytes.len()), + Some(SendableBodyWithMeta::Stream { content_length, .. }) => content_length, + None => None, + }; + + if let Some(cl) = content_length { + headers.push(("Content-Length".to_string(), cl.to_string())); + } + } + + Ok((body.map(|b| b.into()), headers)) +} + +fn build_form_body(body: &BTreeMap) -> Option { + let form_params = match body.get("form").map(|f| f.as_array()) { + Some(Some(f)) => f, + _ => return None, + }; + + let mut body = String::new(); + for p in form_params { + let enabled = get_bool(p, "enabled", true); + let name = get_str(p, "name"); + if !enabled || name.is_empty() { + continue; + } + let value = get_str(p, "value"); + if !body.is_empty() { + body.push('&'); + } + body.push_str(&urlencoding::encode(&name)); + body.push('='); + body.push_str(&urlencoding::encode(&value)); + } + + if body.is_empty() { None } else { Some(SendableBodyWithMeta::Bytes(Bytes::from(body))) } +} + +async fn build_binary_body( + body: &BTreeMap, +) -> Result> { + let file_path = match body.get("filePath").map(|f| f.as_str()) { + Some(Some(f)) => f, + _ => return Ok(None), + }; + + // Open a file for streaming + let content_length = tokio::fs::metadata(file_path) + .await + .map_err(|e| RequestError(format!("Failed to get file metadata: {}", e)))? + .len(); + + let file = tokio::fs::File::open(file_path) + .await + .map_err(|e| RequestError(format!("Failed to open file: {}", e)))?; + + Ok(Some(SendableBodyWithMeta::Stream { + data: Box::pin(file), + content_length: Some(content_length as usize), + })) +} + +fn build_text_body(body: &BTreeMap) -> Option { + let text = get_str_map(body, "text"); + if text.is_empty() { + None + } else { + Some(SendableBodyWithMeta::Bytes(Bytes::from(text.to_string()))) + } +} + +fn build_graphql_body( + method: &str, + body: &BTreeMap, +) -> Option { + let query = get_str_map(body, "query"); + let variables = get_str_map(body, "variables"); + + if method.to_lowercase() == "get" { + // GraphQL GET requests use query parameters, not a body + return None; + } + + let body = if variables.trim().is_empty() { + format!(r#"{{"query":{}}}"#, serde_json::to_string(&query).unwrap_or_default()) + } else { + format!( + r#"{{"query":{},"variables":{}}}"#, + serde_json::to_string(&query).unwrap_or_default(), + variables + ) + }; + + Some(SendableBodyWithMeta::Bytes(Bytes::from(body))) +} + +async fn build_multipart_body( + body: &BTreeMap, + headers: &Vec<(String, String)>, +) -> Result<(Option, Option)> { + let boundary = extract_boundary_from_headers(headers); + + let form_params = match body.get("form").map(|f| f.as_array()) { + Some(Some(f)) => f, + _ => return Ok((None, None)), + }; + + // Build a list of readers for streaming and calculate total content length + let mut readers: Vec = Vec::new(); + let mut has_content = false; + let mut total_size: usize = 0; + + for p in form_params { + let enabled = get_bool(p, "enabled", true); + let name = get_str(p, "name"); + if !enabled || name.is_empty() { + continue; + } + + has_content = true; + + // Add boundary delimiter + let boundary_bytes = format!("--{}\r\n", boundary).into_bytes(); + total_size += boundary_bytes.len(); + readers.push(ReaderType::Bytes(boundary_bytes)); + + let file_path = get_str(p, "file"); + let value = get_str(p, "value"); + let content_type = get_str(p, "contentType"); + + if file_path.is_empty() { + // Text field + let header = if !content_type.is_empty() { + format!( + "Content-Disposition: form-data; name=\"{}\"\r\nContent-Type: {}\r\n\r\n{}", + name, content_type, value + ) + } else { + format!("Content-Disposition: form-data; name=\"{}\"\r\n\r\n{}", name, value) + }; + let header_bytes = header.into_bytes(); + total_size += header_bytes.len(); + readers.push(ReaderType::Bytes(header_bytes)); + } else { + // File field - validate that file exists first + if !tokio::fs::try_exists(file_path).await.unwrap_or(false) { + return Err(RequestError(format!("File not found: {}", file_path))); + } + + // Get file size for content length calculation + let file_metadata = tokio::fs::metadata(file_path) + .await + .map_err(|e| RequestError(format!("Failed to get file metadata: {}", e)))?; + let file_size = file_metadata.len() as usize; + + let filename = get_str(p, "filename"); + let filename = if filename.is_empty() { + std::path::Path::new(file_path) + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("file") + } else { + filename + }; + + // Add content type + let mime_type = if !content_type.is_empty() { + content_type.to_string() + } else { + // Guess mime type from file extension + mime_guess::from_path(file_path).first_or_octet_stream().to_string() + }; + + let header = format!( + "Content-Disposition: form-data; name=\"{}\"; filename=\"{}\"\r\nContent-Type: {}\r\n\r\n", + name, filename, mime_type + ); + let header_bytes = header.into_bytes(); + total_size += header_bytes.len(); + total_size += file_size; + readers.push(ReaderType::Bytes(header_bytes)); + + // Add a file path for streaming + readers.push(ReaderType::FilePath(file_path.to_string())); + } + + let line_ending = b"\r\n".to_vec(); + total_size += line_ending.len(); + readers.push(ReaderType::Bytes(line_ending)); + } + + if has_content { + // Add the final boundary + let final_boundary = format!("--{}--\r\n", boundary).into_bytes(); + total_size += final_boundary.len(); + readers.push(ReaderType::Bytes(final_boundary)); + + let content_type = format!("multipart/form-data; boundary={}", boundary); + let stream = ChainedReader::new(readers); + Ok(( + Some(SendableBodyWithMeta::Stream { + data: Box::pin(stream), + content_length: Some(total_size), + }), + Some(content_type), + )) + } else { + Ok((None, None)) + } +} + +fn extract_boundary_from_headers(headers: &Vec<(String, String)>) -> String { + headers + .iter() + .find(|h| h.0.to_lowercase() == "content-type") + .and_then(|h| { + // Extract boundary from the Content-Type header (e.g., "multipart/form-data; boundary=xyz") + h.1.split(';') + .find(|part| part.trim().starts_with("boundary=")) + .and_then(|boundary_part| boundary_part.split('=').nth(1)) + .map(|b| b.trim().to_string()) + }) + .unwrap_or_else(|| MULTIPART_BOUNDARY.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use bytes::Bytes; + use serde_json::json; + use std::collections::BTreeMap; + use yaak_models::models::{HttpRequest, HttpUrlParameter}; + + #[test] + fn test_build_url_no_params() { + let r = HttpRequest { + url: "https://example.com/api".to_string(), + url_parameters: vec![], + ..Default::default() + }; + + let result = build_url(&r); + assert_eq!(result, "https://example.com/api"); + } + + #[test] + fn test_build_url_with_params() { + let r = HttpRequest { + url: "https://example.com/api".to_string(), + url_parameters: vec![ + HttpUrlParameter { + enabled: true, + name: "foo".to_string(), + value: "bar".to_string(), + id: None, + }, + HttpUrlParameter { + enabled: true, + name: "baz".to_string(), + value: "qux".to_string(), + id: None, + }, + ], + ..Default::default() + }; + + let result = build_url(&r); + assert_eq!(result, "https://example.com/api?foo=bar&baz=qux"); + } + + #[test] + fn test_build_url_with_disabled_params() { + let r = HttpRequest { + url: "https://example.com/api".to_string(), + url_parameters: vec![ + HttpUrlParameter { + enabled: false, + name: "disabled".to_string(), + value: "value".to_string(), + id: None, + }, + HttpUrlParameter { + enabled: true, + name: "enabled".to_string(), + value: "value".to_string(), + id: None, + }, + ], + ..Default::default() + }; + + let result = build_url(&r); + assert_eq!(result, "https://example.com/api?enabled=value"); + } + + #[test] + fn test_build_url_with_existing_query() { + let r = HttpRequest { + url: "https://example.com/api?existing=param".to_string(), + url_parameters: vec![HttpUrlParameter { + enabled: true, + name: "new".to_string(), + value: "value".to_string(), + id: None, + }], + ..Default::default() + }; + + let result = build_url(&r); + assert_eq!(result, "https://example.com/api?existing=param&new=value"); + } + + #[test] + fn test_build_url_with_empty_existing_query() { + let r = HttpRequest { + url: "https://example.com/api?".to_string(), + url_parameters: vec![HttpUrlParameter { + enabled: true, + name: "new".to_string(), + value: "value".to_string(), + id: None, + }], + ..Default::default() + }; + + let result = build_url(&r); + assert_eq!(result, "https://example.com/api?new=value"); + } + + #[test] + fn test_build_url_with_special_chars() { + let r = HttpRequest { + url: "https://example.com/api".to_string(), + url_parameters: vec![HttpUrlParameter { + enabled: true, + name: "special chars!@#".to_string(), + value: "value with spaces & symbols".to_string(), + id: None, + }], + ..Default::default() + }; + + let result = build_url(&r); + assert_eq!( + result, + "https://example.com/api?special%20chars%21%40%23=value%20with%20spaces%20%26%20symbols" + ); + } + + #[test] + fn test_build_url_adds_protocol() { + let r = HttpRequest { + url: "example.com/api".to_string(), + url_parameters: vec![HttpUrlParameter { + enabled: true, + name: "foo".to_string(), + value: "bar".to_string(), + id: None, + }], + ..Default::default() + }; + + let result = build_url(&r); + // ensure_proto defaults to http:// for regular domains + assert_eq!(result, "http://example.com/api?foo=bar"); + } + + #[test] + fn test_build_url_adds_https_for_dev_domain() { + let r = HttpRequest { + url: "example.dev/api".to_string(), + url_parameters: vec![HttpUrlParameter { + enabled: true, + name: "foo".to_string(), + value: "bar".to_string(), + id: None, + }], + ..Default::default() + }; + + let result = build_url(&r); + // .dev domains force https + assert_eq!(result, "https://example.dev/api?foo=bar"); + } + + #[test] + fn test_build_url_with_fragment() { + let r = HttpRequest { + url: "https://example.com/api#section".to_string(), + url_parameters: vec![HttpUrlParameter { + enabled: true, + name: "foo".to_string(), + value: "bar".to_string(), + id: None, + }], + ..Default::default() + }; + + let result = build_url(&r); + assert_eq!(result, "https://example.com/api?foo=bar#section"); + } + + #[test] + fn test_build_url_with_existing_query_and_fragment() { + let r = HttpRequest { + url: "https://yaak.app?foo=bar#some-hash".to_string(), + url_parameters: vec![HttpUrlParameter { + enabled: true, + name: "baz".to_string(), + value: "qux".to_string(), + id: None, + }], + ..Default::default() + }; + + let result = build_url(&r); + assert_eq!(result, "https://yaak.app?foo=bar&baz=qux#some-hash"); + } + + #[test] + fn test_build_url_with_empty_query_and_fragment() { + let r = HttpRequest { + url: "https://example.com/api?#section".to_string(), + url_parameters: vec![HttpUrlParameter { + enabled: true, + name: "foo".to_string(), + value: "bar".to_string(), + id: None, + }], + ..Default::default() + }; + + let result = build_url(&r); + assert_eq!(result, "https://example.com/api?foo=bar#section"); + } + + #[test] + fn test_build_url_with_fragment_containing_special_chars() { + let r = HttpRequest { + url: "https://example.com#section/with/slashes?and=fake&query".to_string(), + url_parameters: vec![HttpUrlParameter { + enabled: true, + name: "real".to_string(), + value: "param".to_string(), + id: None, + }], + ..Default::default() + }; + + let result = build_url(&r); + assert_eq!(result, "https://example.com?real=param#section/with/slashes?and=fake&query"); + } + + #[test] + fn test_build_url_preserves_empty_fragment() { + let r = HttpRequest { + url: "https://example.com/api#".to_string(), + url_parameters: vec![HttpUrlParameter { + enabled: true, + name: "foo".to_string(), + value: "bar".to_string(), + id: None, + }], + ..Default::default() + }; + + let result = build_url(&r); + assert_eq!(result, "https://example.com/api?foo=bar#"); + } + + #[test] + fn test_build_url_with_multiple_fragments() { + // Testing edge case where the URL has multiple # characters (though technically invalid) + let r = HttpRequest { + url: "https://example.com#section#subsection".to_string(), + url_parameters: vec![HttpUrlParameter { + enabled: true, + name: "foo".to_string(), + value: "bar".to_string(), + id: None, + }], + ..Default::default() + }; + + let result = build_url(&r); + // Should treat everything after first # as fragment + assert_eq!(result, "https://example.com?foo=bar#section#subsection"); + } + + #[tokio::test] + async fn test_text_body() { + let mut body = BTreeMap::new(); + body.insert("text".to_string(), json!("Hello, World!")); + + let result = build_text_body(&body); + match result { + Some(SendableBodyWithMeta::Bytes(bytes)) => { + assert_eq!(bytes, Bytes::from("Hello, World!")) + } + _ => panic!("Expected Some(SendableBody::Bytes)"), + } + } + + #[tokio::test] + async fn test_text_body_empty() { + let mut body = BTreeMap::new(); + body.insert("text".to_string(), json!("")); + + let result = build_text_body(&body); + assert!(result.is_none()); + } + + #[tokio::test] + async fn test_text_body_missing() { + let body = BTreeMap::new(); + + let result = build_text_body(&body); + assert!(result.is_none()); + } + + #[tokio::test] + async fn test_form_urlencoded_body() -> Result<()> { + let mut body = BTreeMap::new(); + body.insert( + "form".to_string(), + json!([ + { "enabled": true, "name": "basic", "value": "aaa"}, + { "enabled": true, "name": "fUnkey Stuff!$*#(", "value": "*)%&#$)@ *$#)@&"}, + { "enabled": false, "name": "disabled", "value": "won't show"}, + ]), + ); + + let result = build_form_body(&body); + match result { + Some(SendableBodyWithMeta::Bytes(bytes)) => { + let expected = "basic=aaa&fUnkey%20Stuff%21%24%2A%23%28=%2A%29%25%26%23%24%29%40%20%2A%24%23%29%40%26"; + assert_eq!(bytes, Bytes::from(expected)); + } + _ => panic!("Expected Some(SendableBody::Bytes)"), + } + Ok(()) + } + + #[tokio::test] + async fn test_form_urlencoded_body_missing_form() { + let body = BTreeMap::new(); + let result = build_form_body(&body); + assert!(result.is_none()); + } + + #[tokio::test] + async fn test_binary_body() -> Result<()> { + let mut body = BTreeMap::new(); + body.insert("filePath".to_string(), json!("./tests/test.txt")); + + let result = build_binary_body(&body).await?; + assert!(matches!(result, Some(SendableBodyWithMeta::Stream { .. }))); + Ok(()) + } + + #[tokio::test] + async fn test_binary_body_file_not_found() { + let mut body = BTreeMap::new(); + body.insert("filePath".to_string(), json!("./nonexistent/file.txt")); + + let result = build_binary_body(&body).await; + assert!(result.is_err()); + if let Err(e) = result { + assert!(matches!(e, RequestError(_))); + } + } + + #[tokio::test] + async fn test_graphql_body_with_variables() { + let mut body = BTreeMap::new(); + body.insert("query".to_string(), json!("{ user(id: $id) { name } }")); + body.insert("variables".to_string(), json!(r#"{"id": "123"}"#)); + + let result = build_graphql_body("POST", &body); + match result { + Some(SendableBodyWithMeta::Bytes(bytes)) => { + let expected = + r#"{"query":"{ user(id: $id) { name } }","variables":{"id": "123"}}"#; + assert_eq!(bytes, Bytes::from(expected)); + } + _ => panic!("Expected Some(SendableBody::Bytes)"), + } + } + + #[tokio::test] + async fn test_graphql_body_without_variables() { + let mut body = BTreeMap::new(); + body.insert("query".to_string(), json!("{ users { name } }")); + body.insert("variables".to_string(), json!("")); + + let result = build_graphql_body("POST", &body); + match result { + Some(SendableBodyWithMeta::Bytes(bytes)) => { + let expected = r#"{"query":"{ users { name } }"}"#; + assert_eq!(bytes, Bytes::from(expected)); + } + _ => panic!("Expected Some(SendableBody::Bytes)"), + } + } + + #[tokio::test] + async fn test_graphql_body_get_method() { + let mut body = BTreeMap::new(); + body.insert("query".to_string(), json!("{ users { name } }")); + + let result = build_graphql_body("GET", &body); + assert!(result.is_none()); + } + + #[tokio::test] + async fn test_multipart_body_text_fields() -> Result<()> { + let mut body = BTreeMap::new(); + body.insert( + "form".to_string(), + json!([ + { "enabled": true, "name": "field1", "value": "value1", "file": "" }, + { "enabled": true, "name": "field2", "value": "value2", "file": "" }, + { "enabled": false, "name": "disabled", "value": "won't show", "file": "" }, + ]), + ); + + let (result, content_type) = build_multipart_body(&body, &vec![]).await?; + assert!(content_type.is_some()); + + match result { + Some(SendableBodyWithMeta::Stream { data: mut stream, content_length }) => { + // Read the entire stream to verify content + let mut buf = Vec::new(); + use tokio::io::AsyncReadExt; + stream.read_to_end(&mut buf).await.expect("Failed to read stream"); + let body_str = String::from_utf8_lossy(&buf); + assert_eq!( + body_str, + "--------YaakFormBoundary\r\nContent-Disposition: form-data; name=\"field1\"\r\n\r\nvalue1\r\n--------YaakFormBoundary\r\nContent-Disposition: form-data; name=\"field2\"\r\n\r\nvalue2\r\n--------YaakFormBoundary--\r\n", + ); + assert_eq!(content_length, Some(body_str.len())); + } + _ => panic!("Expected Some(SendableBody::Stream)"), + } + + assert_eq!( + content_type.unwrap(), + format!("multipart/form-data; boundary={}", MULTIPART_BOUNDARY) + ); + + Ok(()) + } + + #[tokio::test] + async fn test_multipart_body_with_file() -> Result<()> { + let mut body = BTreeMap::new(); + body.insert( + "form".to_string(), + json!([ + { "enabled": true, "name": "file_field", "file": "./tests/test.txt", "filename": "custom.txt", "contentType": "text/plain" }, + ]), + ); + + let (result, content_type) = build_multipart_body(&body, &vec![]).await?; + assert!(content_type.is_some()); + + match result { + Some(SendableBodyWithMeta::Stream { data: mut stream, content_length }) => { + // Read the entire stream to verify content + let mut buf = Vec::new(); + use tokio::io::AsyncReadExt; + stream.read_to_end(&mut buf).await.expect("Failed to read stream"); + let body_str = String::from_utf8_lossy(&buf); + assert_eq!( + body_str, + "--------YaakFormBoundary\r\nContent-Disposition: form-data; name=\"file_field\"; filename=\"custom.txt\"\r\nContent-Type: text/plain\r\n\r\nThis is a test file!\n\r\n--------YaakFormBoundary--\r\n" + ); + assert_eq!(content_length, Some(body_str.len())); + } + _ => panic!("Expected Some(SendableBody::Stream)"), + } + + assert_eq!( + content_type.unwrap(), + format!("multipart/form-data; boundary={}", MULTIPART_BOUNDARY) + ); + + Ok(()) + } + + #[tokio::test] + async fn test_multipart_body_empty() -> Result<()> { + let body = BTreeMap::new(); + let (result, content_type) = build_multipart_body(&body, &vec![]).await?; + assert!(result.is_none()); + assert_eq!(content_type, None); + Ok(()) + } + + #[test] + fn test_extract_boundary_from_headers_with_custom_boundary() { + let headers = vec![( + "Content-Type".to_string(), + "multipart/form-data; boundary=customBoundary123".to_string(), + )]; + let boundary = extract_boundary_from_headers(&headers); + assert_eq!(boundary, "customBoundary123"); + } + + #[test] + fn test_extract_boundary_from_headers_default() { + let headers = vec![("Accept".to_string(), "*/*".to_string())]; + let boundary = extract_boundary_from_headers(&headers); + assert_eq!(boundary, MULTIPART_BOUNDARY); + } + + #[test] + fn test_extract_boundary_from_headers_no_boundary_in_content_type() { + let headers = vec![("Content-Type".to_string(), "multipart/form-data".to_string())]; + let boundary = extract_boundary_from_headers(&headers); + assert_eq!(boundary, MULTIPART_BOUNDARY); + } + + #[test] + fn test_extract_boundary_case_insensitive() { + let headers = vec![( + "Content-Type".to_string(), + "multipart/form-data; boundary=myBoundary".to_string(), + )]; + let boundary = extract_boundary_from_headers(&headers); + assert_eq!(boundary, "myBoundary"); + } + + #[tokio::test] + async fn test_no_content_length_with_chunked_encoding() -> Result<()> { + let mut body = BTreeMap::new(); + body.insert("text".to_string(), json!("Hello, World!")); + + // Headers with Transfer-Encoding: chunked + let headers = vec![("Transfer-Encoding".to_string(), "chunked".to_string())]; + + let (_, result_headers) = + build_body("POST", &Some("text/plain".to_string()), &body, headers).await?; + + // Verify that Content-Length is NOT present when Transfer-Encoding: chunked is set + let has_content_length = + result_headers.iter().any(|h| h.0.to_lowercase() == "content-length"); + assert!(!has_content_length, "Content-Length should not be present with chunked encoding"); + + // Verify that the Transfer-Encoding header is still present + let has_chunked = result_headers.iter().any(|h| { + h.0.to_lowercase() == "transfer-encoding" && h.1.to_lowercase().contains("chunked") + }); + assert!(has_chunked, "Transfer-Encoding: chunked should be preserved"); + + Ok(()) + } + + #[tokio::test] + async fn test_content_length_without_chunked_encoding() -> Result<()> { + let mut body = BTreeMap::new(); + body.insert("text".to_string(), json!("Hello, World!")); + + // Headers without Transfer-Encoding: chunked + let headers = vec![]; + + let (_, result_headers) = + build_body("POST", &Some("text/plain".to_string()), &body, headers).await?; + + // Verify that Content-Length IS present when Transfer-Encoding: chunked is NOT set + let content_length_header = + result_headers.iter().find(|h| h.0.to_lowercase() == "content-length"); + assert!( + content_length_header.is_some(), + "Content-Length should be present without chunked encoding" + ); + assert_eq!( + content_length_header.unwrap().1, + "13", + "Content-Length should match the body size" + ); + + Ok(()) + } +} diff --git a/src-tauri/yaak-http/tests/test.txt b/src-tauri/yaak-http/tests/test.txt new file mode 100644 index 00000000..c66d471e --- /dev/null +++ b/src-tauri/yaak-http/tests/test.txt @@ -0,0 +1 @@ +This is a test file! diff --git a/src-tauri/yaak-models/bindings/gen_models.ts b/src-tauri/yaak-models/bindings/gen_models.ts index ce16e13a..844c0deb 100644 --- a/src-tauri/yaak-models/bindings/gen_models.ts +++ b/src-tauri/yaak-models/bindings/gen_models.ts @@ -1,6 +1,6 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export type AnyModel = CookieJar | Environment | Folder | GraphQlIntrospection | GrpcConnection | GrpcEvent | GrpcRequest | HttpRequest | HttpResponse | KeyValue | Plugin | Settings | SyncState | WebsocketConnection | WebsocketEvent | WebsocketRequest | Workspace | WorkspaceMeta; +export type AnyModel = CookieJar | Environment | Folder | GraphQlIntrospection | GrpcConnection | GrpcEvent | GrpcRequest | HttpRequest | HttpResponse | HttpResponseEvent | KeyValue | Plugin | Settings | SyncState | WebsocketConnection | WebsocketEvent | WebsocketRequest | Workspace | WorkspaceMeta; export type ClientCertificate = { host: string, port: number | null, crtFile: string | null, keyFile: string | null, pfxFile: string | null, passphrase: string | null, enabled?: boolean, }; @@ -38,7 +38,16 @@ export type HttpRequest = { model: "http_request", id: string, createdAt: string export type HttpRequestHeader = { enabled?: boolean, name: string, value: string, id?: string, }; -export type HttpResponse = { model: "http_response", id: string, createdAt: string, updatedAt: string, workspaceId: string, requestId: string, bodyPath: string | null, contentLength: number | null, elapsed: number, elapsedHeaders: number, error: string | null, headers: Array, remoteAddr: string | null, status: number, statusReason: string | null, state: HttpResponseState, url: string, version: string | null, }; +export type HttpResponse = { model: "http_response", id: string, createdAt: string, updatedAt: string, workspaceId: string, requestId: string, bodyPath: string | null, contentLength: number | null, contentLengthCompressed: number | null, elapsed: number, elapsedHeaders: number, error: string | null, headers: Array, remoteAddr: string | null, requestContentLength: number | null, requestHeaders: Array, status: number, statusReason: string | null, state: HttpResponseState, url: string, version: string | null, }; + +export type HttpResponseEvent = { model: "http_response_event", id: string, createdAt: string, updatedAt: string, workspaceId: string, responseId: string, event: HttpResponseEventData, }; + +/** + * Serializable representation of HTTP response events for DB storage. + * This mirrors `yaak_http::sender::HttpResponseEvent` but with serde support. + * The `From` impl is in yaak-http to avoid circular dependencies. + */ +export type HttpResponseEventData = { "type": "setting", name: string, value: string, } | { "type": "info", message: string, } | { "type": "redirect", url: string, status: number, behavior: string, } | { "type": "send_url", method: string, path: string, } | { "type": "receive_url", version: string, status: string, } | { "type": "header_up", name: string, value: string, } | { "type": "header_down", name: string, value: string, } | { "type": "chunk_sent", bytes: number, } | { "type": "chunk_received", bytes: number, }; export type HttpResponseHeader = { name: string, value: string, }; diff --git a/src-tauri/yaak-models/blob_migrations/00000000000000_init.sql b/src-tauri/yaak-models/blob_migrations/00000000000000_init.sql new file mode 100644 index 00000000..7589cb4e --- /dev/null +++ b/src-tauri/yaak-models/blob_migrations/00000000000000_init.sql @@ -0,0 +1,12 @@ +CREATE TABLE body_chunks +( + id TEXT PRIMARY KEY, + body_id TEXT NOT NULL, + chunk_index INTEGER NOT NULL, + data BLOB NOT NULL, + created_at DATETIME DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')) NOT NULL, + + UNIQUE (body_id, chunk_index) +); + +CREATE INDEX idx_body_chunks_body_id ON body_chunks (body_id, chunk_index); diff --git a/src-tauri/yaak-models/guest-js/atoms.ts b/src-tauri/yaak-models/guest-js/atoms.ts index fc12373d..da3ad730 100644 --- a/src-tauri/yaak-models/guest-js/atoms.ts +++ b/src-tauri/yaak-models/guest-js/atoms.ts @@ -15,6 +15,7 @@ export const grpcEventsAtom = createOrderedModelAtom('grpc_event', 'createdAt', export const grpcRequestsAtom = createModelAtom('grpc_request'); export const httpRequestsAtom = createModelAtom('http_request'); export const httpResponsesAtom = createOrderedModelAtom('http_response', 'createdAt', 'desc'); +export const httpResponseEventsAtom = createOrderedModelAtom('http_response_event', 'createdAt', 'asc'); export const keyValuesAtom = createModelAtom('key_value'); export const pluginsAtom = createModelAtom('plugin'); export const settingsAtom = createSingularModelAtom('settings'); diff --git a/src-tauri/yaak-models/guest-js/util.ts b/src-tauri/yaak-models/guest-js/util.ts index 412fa98b..81981114 100644 --- a/src-tauri/yaak-models/guest-js/util.ts +++ b/src-tauri/yaak-models/guest-js/util.ts @@ -11,6 +11,7 @@ export function newStoreData(): ModelStoreData { grpc_request: {}, http_request: {}, http_response: {}, + http_response_event: {}, key_value: {}, plugin: {}, settings: {}, diff --git a/src-tauri/yaak-models/migrations/20251219074602_default-workspace-headers.sql b/src-tauri/yaak-models/migrations/20251219074602_default-workspace-headers.sql new file mode 100644 index 00000000..8793b2a5 --- /dev/null +++ b/src-tauri/yaak-models/migrations/20251219074602_default-workspace-headers.sql @@ -0,0 +1,15 @@ +-- Add default User-Agent header to workspaces that don't already have one (case-insensitive check) +UPDATE workspaces +SET headers = json_insert(headers, '$[#]', json('{"enabled":true,"name":"User-Agent","value":"yaak"}')) +WHERE NOT EXISTS ( + SELECT 1 FROM json_each(workspaces.headers) + WHERE LOWER(json_extract(value, '$.name')) = 'user-agent' +); + +-- Add default Accept header to workspaces that don't already have one (case-insensitive check) +UPDATE workspaces +SET headers = json_insert(headers, '$[#]', json('{"enabled":true,"name":"Accept","value":"*/*"}')) +WHERE NOT EXISTS ( + SELECT 1 FROM json_each(workspaces.headers) + WHERE LOWER(json_extract(value, '$.name')) = 'accept' +); diff --git a/src-tauri/yaak-models/migrations/20251220000000_response-request-headers.sql b/src-tauri/yaak-models/migrations/20251220000000_response-request-headers.sql new file mode 100644 index 00000000..36e73aeb --- /dev/null +++ b/src-tauri/yaak-models/migrations/20251220000000_response-request-headers.sql @@ -0,0 +1,3 @@ +-- Add request_headers and content_length_compressed columns to http_responses table +ALTER TABLE http_responses ADD COLUMN request_headers TEXT NOT NULL DEFAULT '[]'; +ALTER TABLE http_responses ADD COLUMN content_length_compressed INTEGER; diff --git a/src-tauri/yaak-models/migrations/20251221000000_http-response-events.sql b/src-tauri/yaak-models/migrations/20251221000000_http-response-events.sql new file mode 100644 index 00000000..ad9f0fa1 --- /dev/null +++ b/src-tauri/yaak-models/migrations/20251221000000_http-response-events.sql @@ -0,0 +1,15 @@ +CREATE TABLE http_response_events +( + id TEXT NOT NULL + PRIMARY KEY, + model TEXT DEFAULT 'http_response_event' NOT NULL, + workspace_id TEXT NOT NULL + REFERENCES workspaces + ON DELETE CASCADE, + response_id TEXT NOT NULL + REFERENCES http_responses + ON DELETE CASCADE, + created_at DATETIME DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')) NOT NULL, + updated_at DATETIME DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')) NOT NULL, + event TEXT NOT NULL +); diff --git a/src-tauri/yaak-models/migrations/20251221100000_request-content-length.sql b/src-tauri/yaak-models/migrations/20251221100000_request-content-length.sql new file mode 100644 index 00000000..16625cf5 --- /dev/null +++ b/src-tauri/yaak-models/migrations/20251221100000_request-content-length.sql @@ -0,0 +1,2 @@ +ALTER TABLE http_responses + ADD COLUMN request_content_length INTEGER; diff --git a/src-tauri/yaak-models/src/blob_manager.rs b/src-tauri/yaak-models/src/blob_manager.rs new file mode 100644 index 00000000..cb817db9 --- /dev/null +++ b/src-tauri/yaak-models/src/blob_manager.rs @@ -0,0 +1,372 @@ +use crate::error::Result; +use crate::util::generate_prefixed_id; +use include_dir::{Dir, include_dir}; +use log::{debug, info}; +use r2d2::Pool; +use r2d2_sqlite::SqliteConnectionManager; +use rusqlite::{OptionalExtension, params}; +use std::sync::{Arc, Mutex}; +use tauri::{Manager, Runtime, State}; + +static BLOB_MIGRATIONS_DIR: Dir = include_dir!("$CARGO_MANIFEST_DIR/blob_migrations"); + +/// A chunk of body data stored in the blob database. +#[derive(Debug, Clone)] +pub struct BodyChunk { + pub id: String, + pub body_id: String, + pub chunk_index: i32, + pub data: Vec, +} + +impl BodyChunk { + pub fn new(body_id: impl Into, chunk_index: i32, data: Vec) -> Self { + Self { id: generate_prefixed_id("bc"), body_id: body_id.into(), chunk_index, data } + } +} + +/// Extension trait for accessing the blob manager from app handle. +pub trait BlobManagerExt<'a, R> { + fn blob_manager(&'a self) -> State<'a, BlobManager>; + fn blobs(&'a self) -> BlobContext; +} + +impl<'a, R: Runtime, M: Manager> BlobManagerExt<'a, R> for M { + fn blob_manager(&'a self) -> State<'a, BlobManager> { + self.state::() + } + + fn blobs(&'a self) -> BlobContext { + let manager = self.state::(); + manager.inner().connect() + } +} + +/// Manages the blob database connection pool. +#[derive(Debug, Clone)] +pub struct BlobManager { + pool: Arc>>, +} + +impl BlobManager { + pub fn new(pool: Pool) -> Self { + Self { pool: Arc::new(Mutex::new(pool)) } + } + + pub fn connect(&self) -> BlobContext { + let conn = self + .pool + .lock() + .expect("Failed to gain lock on blob DB") + .get() + .expect("Failed to get blob DB connection from pool"); + BlobContext { conn } + } +} + +/// Context for blob database operations. +pub struct BlobContext { + conn: r2d2::PooledConnection, +} + +impl BlobContext { + /// Insert a single chunk. + pub fn insert_chunk(&self, chunk: &BodyChunk) -> Result<()> { + self.conn.execute( + "INSERT INTO body_chunks (id, body_id, chunk_index, data) VALUES (?1, ?2, ?3, ?4)", + params![chunk.id, chunk.body_id, chunk.chunk_index, chunk.data], + )?; + Ok(()) + } + + /// Get all chunks for a body, ordered by chunk_index. + pub fn get_chunks(&self, body_id: &str) -> Result> { + let mut stmt = self.conn.prepare( + "SELECT id, body_id, chunk_index, data FROM body_chunks + WHERE body_id = ?1 ORDER BY chunk_index ASC", + )?; + + let chunks = stmt + .query_map(params![body_id], |row| { + Ok(BodyChunk { + id: row.get(0)?, + body_id: row.get(1)?, + chunk_index: row.get(2)?, + data: row.get(3)?, + }) + })? + .collect::, _>>()?; + + Ok(chunks) + } + + /// Delete all chunks for a body. + pub fn delete_chunks(&self, body_id: &str) -> Result<()> { + self.conn.execute("DELETE FROM body_chunks WHERE body_id = ?1", params![body_id])?; + Ok(()) + } + + /// Delete all chunks matching a body_id prefix (e.g., "rs_abc123.%" to delete all bodies for a response). + pub fn delete_chunks_like(&self, body_id_prefix: &str) -> Result<()> { + self.conn + .execute("DELETE FROM body_chunks WHERE body_id LIKE ?1", params![body_id_prefix])?; + Ok(()) + } +} + +/// Get total size of a body without loading data. +impl BlobContext { + pub fn get_body_size(&self, body_id: &str) -> Result { + let size: i64 = self + .conn + .query_row( + "SELECT COALESCE(SUM(LENGTH(data)), 0) FROM body_chunks WHERE body_id = ?1", + params![body_id], + |row| row.get(0), + ) + .unwrap_or(0); + Ok(size as usize) + } + + /// Check if a body exists. + pub fn body_exists(&self, body_id: &str) -> Result { + let count: i64 = self + .conn + .query_row( + "SELECT COUNT(*) FROM body_chunks WHERE body_id = ?1", + params![body_id], + |row| row.get(0), + ) + .unwrap_or(0); + Ok(count > 0) + } +} + +/// Run migrations for the blob database. +pub fn migrate_blob_db(pool: &Pool) -> Result<()> { + info!("Running blob database migrations"); + + // Create migrations tracking table + pool.get()?.execute( + "CREATE TABLE IF NOT EXISTS _blob_migrations ( + version TEXT PRIMARY KEY, + description TEXT NOT NULL, + applied_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL + )", + [], + )?; + + // Read and sort all .sql files + let mut entries: Vec<_> = BLOB_MIGRATIONS_DIR + .entries() + .iter() + .filter(|e| e.path().extension().map(|ext| ext == "sql").unwrap_or(false)) + .collect(); + + entries.sort_by_key(|e| e.path()); + + let mut ran_migrations = 0; + for entry in &entries { + let filename = entry.path().file_name().unwrap().to_str().unwrap(); + let version = filename.split('_').next().unwrap(); + + // Check if already applied + let already_applied: Option = pool + .get()? + .query_row("SELECT 1 FROM _blob_migrations WHERE version = ?", [version], |r| r.get(0)) + .optional()?; + + if already_applied.is_some() { + debug!("Skipping already applied blob migration: {}", filename); + continue; + } + + let sql = + entry.as_file().unwrap().contents_utf8().expect("Failed to read blob migration file"); + + info!("Applying blob migration: {}", filename); + let conn = pool.get()?; + conn.execute_batch(sql)?; + + // Record migration + conn.execute( + "INSERT INTO _blob_migrations (version, description) VALUES (?, ?)", + params![version, filename], + )?; + + ran_migrations += 1; + } + + if ran_migrations == 0 { + info!("No blob migrations to run"); + } else { + info!("Ran {} blob migration(s)", ran_migrations); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn create_test_pool() -> Pool { + let manager = SqliteConnectionManager::memory(); + let pool = Pool::builder().max_size(1).build(manager).unwrap(); + migrate_blob_db(&pool).unwrap(); + pool + } + + #[test] + fn test_insert_and_get_chunks() { + let pool = create_test_pool(); + let manager = BlobManager::new(pool); + let ctx = manager.connect(); + + let body_id = "rs_test123.request"; + let chunk1 = BodyChunk::new(body_id, 0, b"Hello, ".to_vec()); + let chunk2 = BodyChunk::new(body_id, 1, b"World!".to_vec()); + + ctx.insert_chunk(&chunk1).unwrap(); + ctx.insert_chunk(&chunk2).unwrap(); + + let chunks = ctx.get_chunks(body_id).unwrap(); + assert_eq!(chunks.len(), 2); + assert_eq!(chunks[0].chunk_index, 0); + assert_eq!(chunks[0].data, b"Hello, "); + assert_eq!(chunks[1].chunk_index, 1); + assert_eq!(chunks[1].data, b"World!"); + } + + #[test] + fn test_get_chunks_ordered_by_index() { + let pool = create_test_pool(); + let manager = BlobManager::new(pool); + let ctx = manager.connect(); + + let body_id = "rs_test123.request"; + + // Insert out of order + ctx.insert_chunk(&BodyChunk::new(body_id, 2, b"C".to_vec())).unwrap(); + ctx.insert_chunk(&BodyChunk::new(body_id, 0, b"A".to_vec())).unwrap(); + ctx.insert_chunk(&BodyChunk::new(body_id, 1, b"B".to_vec())).unwrap(); + + let chunks = ctx.get_chunks(body_id).unwrap(); + assert_eq!(chunks.len(), 3); + assert_eq!(chunks[0].data, b"A"); + assert_eq!(chunks[1].data, b"B"); + assert_eq!(chunks[2].data, b"C"); + } + + #[test] + fn test_delete_chunks() { + let pool = create_test_pool(); + let manager = BlobManager::new(pool); + let ctx = manager.connect(); + + let body_id = "rs_test123.request"; + ctx.insert_chunk(&BodyChunk::new(body_id, 0, b"data".to_vec())).unwrap(); + + assert!(ctx.body_exists(body_id).unwrap()); + + ctx.delete_chunks(body_id).unwrap(); + + assert!(!ctx.body_exists(body_id).unwrap()); + assert_eq!(ctx.get_chunks(body_id).unwrap().len(), 0); + } + + #[test] + fn test_delete_chunks_like() { + let pool = create_test_pool(); + let manager = BlobManager::new(pool); + let ctx = manager.connect(); + + // Insert chunks for same response but different body types + ctx.insert_chunk(&BodyChunk::new("rs_abc.request", 0, b"req".to_vec())).unwrap(); + ctx.insert_chunk(&BodyChunk::new("rs_abc.response", 0, b"resp".to_vec())).unwrap(); + ctx.insert_chunk(&BodyChunk::new("rs_other.request", 0, b"other".to_vec())).unwrap(); + + // Delete all bodies for rs_abc + ctx.delete_chunks_like("rs_abc.%").unwrap(); + + // rs_abc bodies should be gone + assert!(!ctx.body_exists("rs_abc.request").unwrap()); + assert!(!ctx.body_exists("rs_abc.response").unwrap()); + + // rs_other should still exist + assert!(ctx.body_exists("rs_other.request").unwrap()); + } + + #[test] + fn test_get_body_size() { + let pool = create_test_pool(); + let manager = BlobManager::new(pool); + let ctx = manager.connect(); + + let body_id = "rs_test123.request"; + ctx.insert_chunk(&BodyChunk::new(body_id, 0, b"Hello".to_vec())).unwrap(); + ctx.insert_chunk(&BodyChunk::new(body_id, 1, b"World".to_vec())).unwrap(); + + let size = ctx.get_body_size(body_id).unwrap(); + assert_eq!(size, 10); // "Hello" + "World" = 10 bytes + } + + #[test] + fn test_get_body_size_empty() { + let pool = create_test_pool(); + let manager = BlobManager::new(pool); + let ctx = manager.connect(); + + let size = ctx.get_body_size("nonexistent").unwrap(); + assert_eq!(size, 0); + } + + #[test] + fn test_body_exists() { + let pool = create_test_pool(); + let manager = BlobManager::new(pool); + let ctx = manager.connect(); + + assert!(!ctx.body_exists("rs_test.request").unwrap()); + + ctx.insert_chunk(&BodyChunk::new("rs_test.request", 0, b"data".to_vec())).unwrap(); + + assert!(ctx.body_exists("rs_test.request").unwrap()); + } + + #[test] + fn test_multiple_bodies_isolated() { + let pool = create_test_pool(); + let manager = BlobManager::new(pool); + let ctx = manager.connect(); + + ctx.insert_chunk(&BodyChunk::new("body1", 0, b"data1".to_vec())).unwrap(); + ctx.insert_chunk(&BodyChunk::new("body2", 0, b"data2".to_vec())).unwrap(); + + let chunks1 = ctx.get_chunks("body1").unwrap(); + let chunks2 = ctx.get_chunks("body2").unwrap(); + + assert_eq!(chunks1.len(), 1); + assert_eq!(chunks1[0].data, b"data1"); + assert_eq!(chunks2.len(), 1); + assert_eq!(chunks2[0].data, b"data2"); + } + + #[test] + fn test_large_chunk() { + let pool = create_test_pool(); + let manager = BlobManager::new(pool); + let ctx = manager.connect(); + + // 1MB chunk + let large_data: Vec = (0..1024 * 1024).map(|i| (i % 256) as u8).collect(); + let body_id = "rs_large.request"; + + ctx.insert_chunk(&BodyChunk::new(body_id, 0, large_data.clone())).unwrap(); + + let chunks = ctx.get_chunks(body_id).unwrap(); + assert_eq!(chunks.len(), 1); + assert_eq!(chunks[0].data, large_data); + assert_eq!(ctx.get_body_size(body_id).unwrap(), 1024 * 1024); + } +} diff --git a/src-tauri/yaak-models/src/commands.rs b/src-tauri/yaak-models/src/commands.rs index b16fb2e6..f7bab2e1 100644 --- a/src-tauri/yaak-models/src/commands.rs +++ b/src-tauri/yaak-models/src/commands.rs @@ -1,3 +1,4 @@ +use crate::blob_manager::BlobManagerExt; use crate::error::Error::GenericError; use crate::error::Result; use crate::models::{AnyModel, GraphQlIntrospection, GrpcEvent, Settings, WebsocketEvent}; @@ -8,6 +9,7 @@ use tauri::{AppHandle, Runtime, WebviewWindow}; #[tauri::command] pub(crate) fn upsert(window: WebviewWindow, model: AnyModel) -> Result { let db = window.db(); + let blobs = window.blob_manager(); let source = &UpdateSource::from_window(&window); let id = match model { AnyModel::CookieJar(m) => db.upsert_cookie_jar(&m, source)?.id, @@ -15,7 +17,7 @@ pub(crate) fn upsert(window: WebviewWindow, model: AnyModel) -> R AnyModel::Folder(m) => db.upsert_folder(&m, source)?.id, AnyModel::GrpcRequest(m) => db.upsert_grpc_request(&m, source)?.id, AnyModel::HttpRequest(m) => db.upsert_http_request(&m, source)?.id, - AnyModel::HttpResponse(m) => db.upsert_http_response(&m, source)?.id, + AnyModel::HttpResponse(m) => db.upsert_http_response(&m, source, &blobs)?.id, AnyModel::KeyValue(m) => db.upsert_key_value(&m, source)?.id, AnyModel::Plugin(m) => db.upsert_plugin(&m, source)?.id, AnyModel::Settings(m) => db.upsert_settings(&m, source)?.id, @@ -30,6 +32,7 @@ pub(crate) fn upsert(window: WebviewWindow, model: AnyModel) -> R #[tauri::command] pub(crate) fn delete(window: WebviewWindow, model: AnyModel) -> Result { + let blobs = window.blob_manager(); // Use transaction for deletions because it might recurse window.with_tx(|tx| { let source = &UpdateSource::from_window(&window); @@ -40,7 +43,7 @@ pub(crate) fn delete(window: WebviewWindow, model: AnyModel) -> R AnyModel::GrpcConnection(m) => tx.delete_grpc_connection(&m, source)?.id, AnyModel::GrpcRequest(m) => tx.delete_grpc_request(&m, source)?.id, AnyModel::HttpRequest(m) => tx.delete_http_request(&m, source)?.id, - AnyModel::HttpResponse(m) => tx.delete_http_response(&m, source)?.id, + AnyModel::HttpResponse(m) => tx.delete_http_response(&m, source, &blobs)?.id, AnyModel::Plugin(m) => tx.delete_plugin(&m, source)?.id, AnyModel::WebsocketConnection(m) => tx.delete_websocket_connection(&m, source)?.id, AnyModel::WebsocketRequest(m) => tx.delete_websocket_request(&m, source)?.id, diff --git a/src-tauri/yaak-models/src/db_context.rs b/src-tauri/yaak-models/src/db_context.rs index f20795ea..31479ccf 100644 --- a/src-tauri/yaak-models/src/db_context.rs +++ b/src-tauri/yaak-models/src/db_context.rs @@ -67,7 +67,7 @@ impl<'a> DbContext<'a> { .expect("Failed to run find on DB") } - pub fn find_all<'s, M>(&self) -> Result> + pub(crate) fn find_all<'s, M>(&self) -> Result> where M: Into + Clone + UpsertModelInfo, { @@ -82,7 +82,7 @@ impl<'a> DbContext<'a> { Ok(items.map(|v| v.unwrap()).collect()) } - pub fn find_many<'s, M>( + pub(crate) fn find_many<'s, M>( &self, col: impl IntoColumnRef, value: impl Into, @@ -115,7 +115,7 @@ impl<'a> DbContext<'a> { Ok(items.map(|v| v.unwrap()).collect()) } - pub fn upsert(&self, model: &M, source: &UpdateSource) -> Result + pub(crate) fn upsert(&self, model: &M, source: &UpdateSource) -> Result where M: Into + From + UpsertModelInfo + Clone, { diff --git a/src-tauri/yaak-models/src/error.rs b/src-tauri/yaak-models/src/error.rs index 466e75b6..f92785a1 100644 --- a/src-tauri/yaak-models/src/error.rs +++ b/src-tauri/yaak-models/src/error.rs @@ -18,7 +18,7 @@ pub enum Error { #[error("Model serialization error: {0}")] ModelSerializationError(String), - #[error("Model error: {0}")] + #[error("HTTP error: {0}")] GenericError(String), #[error("DB Migration Failed: {0}")] diff --git a/src-tauri/yaak-models/src/lib.rs b/src-tauri/yaak-models/src/lib.rs index b41fb4d5..7af9be14 100644 --- a/src-tauri/yaak-models/src/lib.rs +++ b/src-tauri/yaak-models/src/lib.rs @@ -1,3 +1,4 @@ +use crate::blob_manager::{BlobManager, migrate_blob_db}; use crate::commands::*; use crate::migrate::migrate_db; use crate::query_manager::QueryManager; @@ -14,6 +15,7 @@ use tauri_plugin_dialog::{DialogExt, MessageDialogKind}; mod commands; +pub mod blob_manager; mod connection_or_tx; pub mod db_context; pub mod error; @@ -50,7 +52,9 @@ pub fn init() -> TauriPlugin { create_dir_all(app_path.clone()).expect("Problem creating App directory!"); let db_file_path = app_path.join("db.sqlite"); + let blob_db_file_path = app_path.join("blobs.sqlite"); + // Main database pool let manager = SqliteConnectionManager::file(db_file_path); let pool = Pool::builder() .max_size(100) // Up from 10 (just in case) @@ -68,7 +72,26 @@ pub fn init() -> TauriPlugin { return Err(Box::from(e.to_string())); }; + // Blob database pool + let blob_manager = SqliteConnectionManager::file(blob_db_file_path); + let blob_pool = Pool::builder() + .max_size(50) + .connection_timeout(Duration::from_secs(10)) + .build(blob_manager) + .unwrap(); + + if let Err(e) = migrate_blob_db(&blob_pool) { + error!("Failed to run blob database migration {e:?}"); + app_handle + .dialog() + .message(e.to_string()) + .kind(MessageDialogKind::Error) + .blocking_show(); + return Err(Box::from(e.to_string())); + }; + app_handle.manage(SqliteConnection::new(pool.clone())); + app_handle.manage(BlobManager::new(blob_pool)); { let (tx, rx) = mpsc::channel(); diff --git a/src-tauri/yaak-models/src/models.rs b/src-tauri/yaak-models/src/models.rs index 9b773b83..6ec3ef67 100644 --- a/src-tauri/yaak-models/src/models.rs +++ b/src-tauri/yaak-models/src/models.rs @@ -1322,12 +1322,15 @@ pub struct HttpResponse { pub request_id: String, pub body_path: Option, - pub content_length: Option, + pub content_length: Option, + pub content_length_compressed: Option, pub elapsed: i32, pub elapsed_headers: i32, pub error: Option, pub headers: Vec, pub remote_addr: Option, + pub request_content_length: Option, + pub request_headers: Vec, pub status: i32, pub status_reason: Option, pub state: HttpResponseState, @@ -1368,16 +1371,19 @@ impl UpsertModelInfo for HttpResponse { (WorkspaceId, self.workspace_id.into()), (BodyPath, self.body_path.into()), (ContentLength, self.content_length.into()), + (ContentLengthCompressed, self.content_length_compressed.into()), (Elapsed, self.elapsed.into()), (ElapsedHeaders, self.elapsed_headers.into()), (Error, self.error.into()), (Headers, serde_json::to_string(&self.headers)?.into()), (RemoteAddr, self.remote_addr.into()), + (RequestHeaders, serde_json::to_string(&self.request_headers)?.into()), (State, serde_json::to_value(self.state)?.as_str().into()), (Status, self.status.into()), (StatusReason, self.status_reason.into()), (Url, self.url.into()), (Version, self.version.into()), + (RequestContentLength, self.request_content_length.into()), ]) } @@ -1386,11 +1392,14 @@ impl UpsertModelInfo for HttpResponse { HttpResponseIden::UpdatedAt, HttpResponseIden::BodyPath, HttpResponseIden::ContentLength, + HttpResponseIden::ContentLengthCompressed, HttpResponseIden::Elapsed, HttpResponseIden::ElapsedHeaders, HttpResponseIden::Error, HttpResponseIden::Headers, HttpResponseIden::RemoteAddr, + HttpResponseIden::RequestContentLength, + HttpResponseIden::RequestHeaders, HttpResponseIden::State, HttpResponseIden::Status, HttpResponseIden::StatusReason, @@ -1415,6 +1424,7 @@ impl UpsertModelInfo for HttpResponse { error: r.get("error")?, url: r.get("url")?, content_length: r.get("content_length")?, + content_length_compressed: r.get("content_length_compressed").unwrap_or_default(), version: r.get("version")?, elapsed: r.get("elapsed")?, elapsed_headers: r.get("elapsed_headers")?, @@ -1424,10 +1434,152 @@ impl UpsertModelInfo for HttpResponse { state: serde_json::from_str(format!(r#""{state}""#).as_str()).unwrap(), body_path: r.get("body_path")?, headers: serde_json::from_str(headers.as_str()).unwrap_or_default(), + request_content_length: r.get("request_content_length").unwrap_or_default(), + request_headers: serde_json::from_str( + r.get::<_, String>("request_headers").unwrap_or_default().as_str(), + ) + .unwrap_or_default(), }) } } +/// Serializable representation of HTTP response events for DB storage. +/// This mirrors `yaak_http::sender::HttpResponseEvent` but with serde support. +/// The `From` impl is in yaak-http to avoid circular dependencies. +#[derive(Debug, Clone, Serialize, Deserialize, TS)] +#[serde(tag = "type", rename_all = "snake_case")] +#[ts(export, export_to = "gen_models.ts")] +pub enum HttpResponseEventData { + Setting { + name: String, + value: String, + }, + Info { + message: String, + }, + Redirect { + url: String, + status: u16, + behavior: String, + }, + SendUrl { + method: String, + path: String, + }, + ReceiveUrl { + version: String, + status: String, + }, + HeaderUp { + name: String, + value: String, + }, + HeaderDown { + name: String, + value: String, + }, + ChunkSent { + bytes: usize, + }, + ChunkReceived { + bytes: usize, + }, +} + +impl Default for HttpResponseEventData { + fn default() -> Self { + Self::Info { message: String::new() } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default, TS)] +#[serde(default, rename_all = "camelCase")] +#[ts(export, export_to = "gen_models.ts")] +#[enum_def(table_name = "http_response_events")] +pub struct HttpResponseEvent { + #[ts(type = "\"http_response_event\"")] + pub model: String, + pub id: String, + pub created_at: NaiveDateTime, + pub updated_at: NaiveDateTime, + pub workspace_id: String, + pub response_id: String, + pub event: HttpResponseEventData, +} + +impl UpsertModelInfo for HttpResponseEvent { + fn table_name() -> impl IntoTableRef + IntoIden { + HttpResponseEventIden::Table + } + + fn id_column() -> impl IntoIden + Eq + Clone { + HttpResponseEventIden::Id + } + + fn generate_id() -> String { + generate_prefixed_id("re") + } + + fn order_by() -> (impl IntoColumnRef, Order) { + (HttpResponseEventIden::CreatedAt, Order::Asc) + } + + fn get_id(&self) -> String { + self.id.clone() + } + + fn insert_values( + self, + source: &UpdateSource, + ) -> Result)>> { + use HttpResponseEventIden::*; + Ok(vec![ + (CreatedAt, upsert_date(source, self.created_at)), + (UpdatedAt, upsert_date(source, self.updated_at)), + (WorkspaceId, self.workspace_id.into()), + (ResponseId, self.response_id.into()), + (Event, serde_json::to_string(&self.event)?.into()), + ]) + } + + fn update_columns() -> Vec { + vec![ + HttpResponseEventIden::UpdatedAt, + HttpResponseEventIden::Event, + ] + } + + fn from_row(r: &Row) -> rusqlite::Result + where + Self: Sized, + { + let event: String = r.get("event")?; + Ok(Self { + id: r.get("id")?, + model: r.get("model")?, + workspace_id: r.get("workspace_id")?, + response_id: r.get("response_id")?, + created_at: r.get("created_at")?, + updated_at: r.get("updated_at")?, + event: serde_json::from_str(&event).unwrap_or_default(), + }) + } +} + +impl HttpResponseEvent { + pub fn new(response_id: &str, workspace_id: &str, event: HttpResponseEventData) -> Self { + Self { + model: "http_response_event".to_string(), + id: Self::generate_id(), + created_at: Utc::now().naive_utc(), + updated_at: Utc::now().naive_utc(), + workspace_id: workspace_id.to_string(), + response_id: response_id.to_string(), + event, + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize, Default, TS)] #[serde(default, rename_all = "camelCase")] #[ts(export, export_to = "gen_models.ts")] @@ -2178,6 +2330,7 @@ define_any_model! { GrpcRequest, HttpRequest, HttpResponse, + HttpResponseEvent, KeyValue, Plugin, Settings, diff --git a/src-tauri/yaak-models/src/queries/http_response_events.rs b/src-tauri/yaak-models/src/queries/http_response_events.rs new file mode 100644 index 00000000..145ea444 --- /dev/null +++ b/src-tauri/yaak-models/src/queries/http_response_events.rs @@ -0,0 +1,18 @@ +use crate::db_context::DbContext; +use crate::error::Result; +use crate::models::{HttpResponseEvent, HttpResponseEventIden}; +use crate::util::UpdateSource; + +impl<'a> DbContext<'a> { + pub fn list_http_response_events(&self, response_id: &str) -> Result> { + self.find_many(HttpResponseEventIden::ResponseId, response_id, None) + } + + pub fn upsert_http_response_event( + &self, + http_response_event: &HttpResponseEvent, + source: &UpdateSource, + ) -> Result { + self.upsert(http_response_event, source) + } +} diff --git a/src-tauri/yaak-models/src/queries/http_responses.rs b/src-tauri/yaak-models/src/queries/http_responses.rs index 4647446d..a555b276 100644 --- a/src-tauri/yaak-models/src/queries/http_responses.rs +++ b/src-tauri/yaak-models/src/queries/http_responses.rs @@ -1,3 +1,4 @@ +use crate::blob_manager::BlobManager; use crate::db_context::DbContext; use crate::error::Result; use crate::models::{HttpResponse, HttpResponseIden, HttpResponseState}; @@ -58,6 +59,7 @@ impl<'a> DbContext<'a> { &self, http_response: &HttpResponse, source: &UpdateSource, + blob_manager: &BlobManager, ) -> Result { // Delete the body file if it exists if let Some(p) = http_response.body_path.clone() { @@ -66,6 +68,13 @@ impl<'a> DbContext<'a> { }; } + // Delete request body blobs (pattern: {response_id}.request) + let blob_ctx = blob_manager.connect(); + let body_id = format!("{}.request", http_response.id); + if let Err(e) = blob_ctx.delete_chunks(&body_id) { + error!("Failed to delete request body blobs: {}", e); + } + Ok(self.delete(http_response, source)?) } @@ -73,12 +82,13 @@ impl<'a> DbContext<'a> { &self, http_response: &HttpResponse, source: &UpdateSource, + blob_manager: &BlobManager, ) -> Result { let responses = self.list_http_responses_for_request(&http_response.request_id, None)?; for m in responses.iter().skip(MAX_HISTORY_ITEMS - 1) { debug!("Deleting old HTTP response {}", http_response.id); - self.delete_http_response(&m, source)?; + self.delete_http_response(&m, source, blob_manager)?; } self.upsert(http_response, source) diff --git a/src-tauri/yaak-models/src/queries/mod.rs b/src-tauri/yaak-models/src/queries/mod.rs index 983ac621..fd1553d9 100644 --- a/src-tauri/yaak-models/src/queries/mod.rs +++ b/src-tauri/yaak-models/src/queries/mod.rs @@ -8,6 +8,7 @@ mod grpc_connections; mod grpc_events; mod grpc_requests; mod http_requests; +mod http_response_events; mod http_responses; mod key_values; mod plugin_key_values; diff --git a/src-tauri/yaak-plugins/bindings/gen_models.ts b/src-tauri/yaak-plugins/bindings/gen_models.ts index 6b2eb5c8..454903fe 100644 --- a/src-tauri/yaak-plugins/bindings/gen_models.ts +++ b/src-tauri/yaak-plugins/bindings/gen_models.ts @@ -12,7 +12,7 @@ export type HttpRequest = { model: "http_request", id: string, createdAt: string export type HttpRequestHeader = { enabled?: boolean, name: string, value: string, id?: string, }; -export type HttpResponse = { model: "http_response", id: string, createdAt: string, updatedAt: string, workspaceId: string, requestId: string, bodyPath: string | null, contentLength: number | null, elapsed: number, elapsedHeaders: number, error: string | null, headers: Array, remoteAddr: string | null, status: number, statusReason: string | null, state: HttpResponseState, url: string, version: string | null, }; +export type HttpResponse = { model: "http_response", id: string, createdAt: string, updatedAt: string, workspaceId: string, requestId: string, bodyPath: string | null, contentLength: number | null, contentLengthCompressed: number | null, elapsed: number, elapsedHeaders: number, error: string | null, headers: Array, remoteAddr: string | null, requestContentLength: number | null, requestHeaders: Array, status: number, statusReason: string | null, state: HttpResponseState, url: string, version: string | null, }; export type HttpResponseHeader = { name: string, value: string, }; diff --git a/src-tauri/yaak-sync/src/models.rs b/src-tauri/yaak-sync/src/models.rs index 41d1fff9..9c3c75fe 100644 --- a/src-tauri/yaak-sync/src/models.rs +++ b/src-tauri/yaak-sync/src/models.rs @@ -208,6 +208,7 @@ impl TryFrom for SyncModel { AnyModel::GrpcConnection(m) => return Err(UnknownModel(m.model)), AnyModel::GrpcEvent(m) => return Err(UnknownModel(m.model)), AnyModel::HttpResponse(m) => return Err(UnknownModel(m.model)), + AnyModel::HttpResponseEvent(m) => return Err(UnknownModel(m.model)), AnyModel::KeyValue(m) => return Err(UnknownModel(m.model)), AnyModel::Plugin(m) => return Err(UnknownModel(m.model)), AnyModel::Settings(m) => return Err(UnknownModel(m.model)), diff --git a/src-tauri/yaak-templates/pkg/yaak_templates.d.ts b/src-tauri/yaak-templates/pkg/yaak_templates.d.ts index aed6c395..5d24deef 100644 --- a/src-tauri/yaak-templates/pkg/yaak_templates.d.ts +++ b/src-tauri/yaak-templates/pkg/yaak_templates.d.ts @@ -1,5 +1,5 @@ /* tslint:disable */ /* eslint-disable */ export function unescape_template(template: string): any; -export function parse_template(template: string): any; export function escape_template(template: string): any; +export function parse_template(template: string): any; diff --git a/src-tauri/yaak-templates/pkg/yaak_templates.js b/src-tauri/yaak-templates/pkg/yaak_templates.js index 7ca3c562..8d2a7738 100644 --- a/src-tauri/yaak-templates/pkg/yaak_templates.js +++ b/src-tauri/yaak-templates/pkg/yaak_templates.js @@ -1,4 +1,5 @@ import * as wasm from "./yaak_templates_bg.wasm"; export * from "./yaak_templates_bg.js"; import { __wbg_set_wasm } from "./yaak_templates_bg.js"; -__wbg_set_wasm(wasm); \ No newline at end of file +__wbg_set_wasm(wasm); +wasm.__wbindgen_start(); diff --git a/src-tauri/yaak-templates/pkg/yaak_templates_bg.js b/src-tauri/yaak-templates/pkg/yaak_templates_bg.js index 98b3c8f5..4d11efa6 100644 --- a/src-tauri/yaak-templates/pkg/yaak_templates_bg.js +++ b/src-tauri/yaak-templates/pkg/yaak_templates_bg.js @@ -4,35 +4,6 @@ export function __wbg_set_wasm(val) { } -const heap = new Array(128).fill(undefined); - -heap.push(undefined, null, true, false); - -let heap_next = heap.length; - -function addHeapObject(obj) { - if (heap_next === heap.length) heap.push(heap.length + 1); - const idx = heap_next; - heap_next = heap[idx]; - - heap[idx] = obj; - return idx; -} - -function getObject(idx) { return heap[idx]; } - -function dropObject(idx) { - if (idx < 132) return; - heap[idx] = heap_next; - heap_next = idx; -} - -function takeObject(idx) { - const ret = getObject(idx); - dropObject(idx); - return ret; -} - function debugString(val) { // primitive types const type = typeof val; @@ -184,48 +155,24 @@ function getStringFromWasm0(ptr, len) { ptr = ptr >>> 0; return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len)); } + +function takeFromExternrefTable0(idx) { + const value = wasm.__wbindgen_export_2.get(idx); + wasm.__externref_table_dealloc(idx); + return value; +} /** * @param {string} template * @returns {any} */ export function unescape_template(template) { - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - const ptr0 = passStringToWasm0(template, wasm.__wbindgen_export_0, wasm.__wbindgen_export_1); - const len0 = WASM_VECTOR_LEN; - wasm.unescape_template(retptr, ptr0, len0); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true); - if (r2) { - throw takeObject(r1); - } - return takeObject(r0); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - } -} - -/** - * @param {string} template - * @returns {any} - */ -export function parse_template(template) { - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - const ptr0 = passStringToWasm0(template, wasm.__wbindgen_export_0, wasm.__wbindgen_export_1); - const len0 = WASM_VECTOR_LEN; - wasm.parse_template(retptr, ptr0, len0); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true); - if (r2) { - throw takeObject(r1); - } - return takeObject(r0); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); + const ptr0 = passStringToWasm0(template, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.unescape_template(ptr0, len0); + if (ret[2]) { + throw takeFromExternrefTable0(ret[1]); } + return takeFromExternrefTable0(ret[0]); } /** @@ -233,61 +180,69 @@ export function parse_template(template) { * @returns {any} */ export function escape_template(template) { - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - const ptr0 = passStringToWasm0(template, wasm.__wbindgen_export_0, wasm.__wbindgen_export_1); - const len0 = WASM_VECTOR_LEN; - wasm.escape_template(retptr, ptr0, len0); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true); - if (r2) { - throw takeObject(r1); - } - return takeObject(r0); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); + const ptr0 = passStringToWasm0(template, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.escape_template(ptr0, len0); + if (ret[2]) { + throw takeFromExternrefTable0(ret[1]); } + return takeFromExternrefTable0(ret[0]); +} + +/** + * @param {string} template + * @returns {any} + */ +export function parse_template(template) { + const ptr0 = passStringToWasm0(template, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_template(ptr0, len0); + if (ret[2]) { + throw takeFromExternrefTable0(ret[1]); + } + return takeFromExternrefTable0(ret[0]); } export function __wbg_new_405e22f390576ce2() { const ret = new Object(); - return addHeapObject(ret); + return ret; }; export function __wbg_new_78feb108b6472713() { const ret = new Array(); - return addHeapObject(ret); + return ret; }; export function __wbg_set_37837023f3d740e8(arg0, arg1, arg2) { - getObject(arg0)[arg1 >>> 0] = takeObject(arg2); + arg0[arg1 >>> 0] = arg2; }; export function __wbg_set_3f1d0b984ed272ed(arg0, arg1, arg2) { - getObject(arg0)[takeObject(arg1)] = takeObject(arg2); + arg0[arg1] = arg2; }; export function __wbindgen_debug_string(arg0, arg1) { - const ret = debugString(getObject(arg1)); - const ptr1 = passStringToWasm0(ret, wasm.__wbindgen_export_0, wasm.__wbindgen_export_1); + const ret = debugString(arg1); + const ptr1 = passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); const len1 = WASM_VECTOR_LEN; getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true); getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true); }; -export function __wbindgen_object_clone_ref(arg0) { - const ret = getObject(arg0); - return addHeapObject(ret); -}; - -export function __wbindgen_object_drop_ref(arg0) { - takeObject(arg0); +export function __wbindgen_init_externref_table() { + const table = wasm.__wbindgen_export_2; + const offset = table.grow(4); + table.set(0, undefined); + table.set(offset + 0, undefined); + table.set(offset + 1, null); + table.set(offset + 2, true); + table.set(offset + 3, false); + ; }; export function __wbindgen_string_new(arg0, arg1) { const ret = getStringFromWasm0(arg0, arg1); - return addHeapObject(ret); + return ret; }; export function __wbindgen_throw(arg0, arg1) { diff --git a/src-tauri/yaak-templates/pkg/yaak_templates_bg.wasm b/src-tauri/yaak-templates/pkg/yaak_templates_bg.wasm index dceb1699..dfa7764d 100644 Binary files a/src-tauri/yaak-templates/pkg/yaak_templates_bg.wasm and b/src-tauri/yaak-templates/pkg/yaak_templates_bg.wasm differ diff --git a/src-tauri/yaak-templates/pkg/yaak_templates_bg.wasm.d.ts b/src-tauri/yaak-templates/pkg/yaak_templates_bg.wasm.d.ts index 55be67ff..d8bbabb6 100644 --- a/src-tauri/yaak-templates/pkg/yaak_templates_bg.wasm.d.ts +++ b/src-tauri/yaak-templates/pkg/yaak_templates_bg.wasm.d.ts @@ -1,9 +1,11 @@ /* tslint:disable */ /* eslint-disable */ export const memory: WebAssembly.Memory; -export const escape_template: (a: number, b: number, c: number) => void; -export const parse_template: (a: number, b: number, c: number) => void; -export const unescape_template: (a: number, b: number, c: number) => void; -export const __wbindgen_export_0: (a: number, b: number) => number; -export const __wbindgen_export_1: (a: number, b: number, c: number, d: number) => number; -export const __wbindgen_add_to_stack_pointer: (a: number) => number; +export const escape_template: (a: number, b: number) => [number, number, number]; +export const parse_template: (a: number, b: number) => [number, number, number]; +export const unescape_template: (a: number, b: number) => [number, number, number]; +export const __wbindgen_malloc: (a: number, b: number) => number; +export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number; +export const __wbindgen_export_2: WebAssembly.Table; +export const __externref_table_dealloc: (a: number) => void; +export const __wbindgen_start: () => void; diff --git a/src-tauri/yaak-templates/src/renderer.rs b/src-tauri/yaak-templates/src/renderer.rs index 73ccee55..495e1120 100644 --- a/src-tauri/yaak-templates/src/renderer.rs +++ b/src-tauri/yaak-templates/src/renderer.rs @@ -77,6 +77,12 @@ pub struct RenderOptions { pub error_behavior: RenderErrorBehavior, } +impl RenderOptions { + pub fn throw() -> Self { + Self { error_behavior: RenderErrorBehavior::Throw } + } +} + impl RenderErrorBehavior { pub fn handle(&self, r: Result) -> Result { match (self, r) { diff --git a/src-tauri/yaak-ws/src/commands.rs b/src-tauri/yaak-ws/src/commands.rs index a61ad5ea..be1073af 100644 --- a/src-tauri/yaak-ws/src/commands.rs +++ b/src-tauri/yaak-ws/src/commands.rs @@ -216,7 +216,7 @@ pub(crate) async fn connect( &UpdateSource::from_window(&window), )?; - let (mut url, url_parameters) = apply_path_placeholders(&request.url, request.url_parameters); + let (mut url, url_parameters) = apply_path_placeholders(&request.url, &request.url_parameters); if !url.starts_with("ws://") && !url.starts_with("wss://") { url.insert_str(0, "ws://"); } diff --git a/src-web/components/ConfirmLargeResponseRequest.tsx b/src-web/components/ConfirmLargeResponseRequest.tsx new file mode 100644 index 00000000..80e9fb30 --- /dev/null +++ b/src-web/components/ConfirmLargeResponseRequest.tsx @@ -0,0 +1,58 @@ +import type { HttpResponse } from '@yaakapp-internal/models'; +import { type ReactNode, useMemo } from 'react'; +import { getRequestBodyText as getHttpResponseRequestBodyText } from '../hooks/useHttpRequestBody'; +import { useToggle } from '../hooks/useToggle'; +import { isProbablyTextContentType } from '../lib/contentType'; +import { getContentTypeFromHeaders } from '../lib/model_util'; +import { CopyButton } from './CopyButton'; +import { Banner } from './core/Banner'; +import { Button } from './core/Button'; +import { InlineCode } from './core/InlineCode'; +import { SizeTag } from './core/SizeTag'; +import { HStack } from './core/Stacks'; + +interface Props { + children: ReactNode; + response: HttpResponse; +} + +const LARGE_BYTES = 2 * 1000 * 1000; + +export function ConfirmLargeResponseRequest({ children, response }: Props) { + const [showLargeResponse, toggleShowLargeResponse] = useToggle(); + const isProbablyText = useMemo(() => { + const contentType = getContentTypeFromHeaders(response.headers); + return isProbablyTextContentType(contentType); + }, [response.headers]); + + const contentLength = response.requestContentLength ?? 0; + const isLarge = contentLength > LARGE_BYTES; + if (!showLargeResponse && isLarge) { + return ( + +

+ Showing content over{' '} + + + {' '} + may impact performance +

+ + + {isProbablyText && ( + getHttpResponseRequestBodyText(response).then((d) => d?.bodyText ?? '')} + /> + )} + + + ); + } + + return <>{children}; +} diff --git a/src-web/components/ExportDataDialog.tsx b/src-web/components/ExportDataDialog.tsx index 0c7c1ec7..89942fed 100644 --- a/src-web/components/ExportDataDialog.tsx +++ b/src-web/components/ExportDataDialog.tsx @@ -128,7 +128,7 @@ function ExportDataDialogContent({ ))} - + • • - + ) : ( diff --git a/src-web/components/HeadersEditor.tsx b/src-web/components/HeadersEditor.tsx index 7e6d76a6..47f036e1 100644 --- a/src-web/components/HeadersEditor.tsx +++ b/src-web/components/HeadersEditor.tsx @@ -34,11 +34,11 @@ export function HeadersEditor({ const validInheritedHeaders = inheritedHeaders?.filter((pair) => pair.enabled && (pair.name || pair.value)) ?? []; return ( -
+
{validInheritedHeaders.length > 0 ? ( Inherited diff --git a/src-web/components/HttpResponsePane.tsx b/src-web/components/HttpResponsePane.tsx index e29bda0e..02c575c8 100644 --- a/src-web/components/HttpResponsePane.tsx +++ b/src-web/components/HttpResponsePane.tsx @@ -4,11 +4,14 @@ import type { ComponentType, CSSProperties } from 'react'; import { lazy, Suspense, useCallback, useMemo } from 'react'; import { useLocalStorage } from 'react-use'; import { useCancelHttpResponse } from '../hooks/useCancelHttpResponse'; +import { useHttpResponseEvents } from '../hooks/useHttpResponseEvents'; import { usePinnedHttpResponse } from '../hooks/usePinnedHttpResponse'; +import { useResponseBodyBytes, useResponseBodyText } from '../hooks/useResponseBodyText'; import { useResponseViewMode } from '../hooks/useResponseViewMode'; import { getMimeTypeFromContentType } from '../lib/contentType'; import { getContentTypeFromHeaders } from '../lib/model_util'; import { ConfirmLargeResponse } from './ConfirmLargeResponse'; +import { ConfirmLargeResponseRequest } from './ConfirmLargeResponseRequest'; import { Banner } from './core/Banner'; import { Button } from './core/Button'; import { CountBadge } from './core/CountBadge'; @@ -22,7 +25,9 @@ import type { TabItem } from './core/Tabs/Tabs'; import { TabContent, Tabs } from './core/Tabs/Tabs'; import { EmptyStateText } from './EmptyStateText'; import { ErrorBoundary } from './ErrorBoundary'; +import { HttpResponseTimeline } from './HttpResponseTimeline'; import { RecentHttpResponsesDropdown } from './RecentHttpResponsesDropdown'; +import { RequestBodyViewer } from './RequestBodyViewer'; import { ResponseHeaders } from './ResponseHeaders'; import { ResponseInfo } from './ResponseInfo'; import { AudioViewer } from './responseViewers/AudioViewer'; @@ -30,6 +35,7 @@ import { CsvViewer } from './responseViewers/CsvViewer'; import { EventStreamViewer } from './responseViewers/EventStreamViewer'; import { HTMLOrTextViewer } from './responseViewers/HTMLOrTextViewer'; import { ImageViewer } from './responseViewers/ImageViewer'; +import { MultipartViewer } from './responseViewers/MultipartViewer'; import { SvgViewer } from './responseViewers/SvgViewer'; import { VideoViewer } from './responseViewers/VideoViewer'; @@ -44,8 +50,10 @@ interface Props { } const TAB_BODY = 'body'; +const TAB_REQUEST = 'request'; const TAB_HEADERS = 'headers'; const TAB_INFO = 'info'; +const TAB_TIMELINE = 'timeline'; export function HttpResponsePane({ style, className, activeRequestId }: Props) { const { activeResponse, setPinnedResponseId, responses } = usePinnedHttpResponse(activeRequestId); @@ -57,6 +65,8 @@ export function HttpResponsePane({ style, className, activeRequestId }: Props) { const contentType = getContentTypeFromHeaders(activeResponse?.headers ?? null); const mimeType = contentType == null ? null : getMimeTypeFromContentType(contentType).essence; + const responseEvents = useHttpResponseEvents(activeResponse); + const tabs = useMemo( () => [ { @@ -71,21 +81,41 @@ export function HttpResponsePane({ style, className, activeRequestId }: Props) { ], }, }, + { + value: TAB_REQUEST, + label: 'Request', + rightSlot: + (activeResponse?.requestContentLength ?? 0) > 0 ? : null, + }, { value: TAB_HEADERS, label: 'Headers', rightSlot: ( h.name && h.value).length ?? 0} + count2={activeResponse?.headers.length ?? 0} + count={activeResponse?.requestHeaders.length ?? 0} /> ), }, + { + value: TAB_TIMELINE, + label: 'Timeline', + rightSlot: , + }, { value: TAB_INFO, label: 'Info', }, ], - [activeResponse?.headers, mimeType, setViewMode, viewMode], + [ + activeResponse?.headers, + activeResponse?.requestContentLength, + activeResponse?.requestHeaders.length, + mimeType, + responseEvents.data?.length, + setViewMode, + viewMode, + ], ); const activeTab = activeTabs?.[activeRequestId]; const setActiveTab = useCallback( @@ -133,7 +163,10 @@ export function HttpResponsePane({ style, className, activeRequestId }: Props) { • • - +
- {activeResponse?.error ? ( - - {activeResponse.error} - - ) : ( +
+ {activeResponse?.error && ( + + {activeResponse.error} + + )} + {/* Show tabs if we have any data (headers, body, etc.) even if there's an error */} @@ -177,22 +212,24 @@ export function HttpResponsePane({ style, className, activeRequestId }: Props) { ) : activeResponse.state === 'closed' && - activeResponse.contentLength === 0 ? ( - Empty + (activeResponse.contentLength ?? 0) === 0 ? ( + Empty ) : mimeType?.match(/^text\/event-stream/i) && viewMode === 'pretty' ? ( ) : mimeType?.match(/^image\/svg/) ? ( - + ) : mimeType?.match(/^image/i) ? ( ) : mimeType?.match(/^audio/i) ? ( ) : mimeType?.match(/^video/i) ? ( + ) : mimeType?.match(/^multipart/i) && viewMode === 'pretty' ? ( + ) : mimeType?.match(/pdf/i) ? ( ) : mimeType?.match(/csv|tab-separated/i) ? ( - + ) : ( + + + + + + + + - )} +
)}
@@ -240,3 +285,28 @@ function EnsureCompleteResponse({ return ; } + +function HttpSvgViewer({ response }: { response: HttpResponse }) { + const body = useResponseBodyText({ response, filter: null }); + + if (!body.data) return null; + + return ; +} + +function HttpCsvViewer({ response, className }: { response: HttpResponse; className?: string }) { + const body = useResponseBodyText({ response, filter: null }); + + return ; +} + +function HttpMultipartViewer({ response }: { response: HttpResponse }) { + const body = useResponseBodyBytes({ response }); + + if (body.data == null) return null; + + const contentTypeHeader = getContentTypeFromHeaders(response.headers); + const boundary = contentTypeHeader?.split('boundary=')[1] ?? 'unknown'; + + return ; +} diff --git a/src-web/components/HttpResponseTimeline.tsx b/src-web/components/HttpResponseTimeline.tsx new file mode 100644 index 00000000..fe5fbde3 --- /dev/null +++ b/src-web/components/HttpResponseTimeline.tsx @@ -0,0 +1,323 @@ +import type { + HttpResponse, + HttpResponseEvent, + HttpResponseEventData, +} from '@yaakapp-internal/models'; +import classNames from 'classnames'; +import { format } from 'date-fns'; +import { type ReactNode, useMemo, useState } from 'react'; +import { useHttpResponseEvents } from '../hooks/useHttpResponseEvents'; +import { AutoScroller } from './core/AutoScroller'; +import { Banner } from './core/Banner'; +import { HttpMethodTagRaw } from './core/HttpMethodTag'; +import { HttpStatusTagRaw } from './core/HttpStatusTag'; +import { Icon, type IconProps } from './core/Icon'; +import { KeyValueRow, KeyValueRows } from './core/KeyValueRow'; +import { Separator } from './core/Separator'; +import { SplitLayout } from './core/SplitLayout'; + +interface Props { + response: HttpResponse; +} + +export function HttpResponseTimeline({ response }: Props) { + return ; +} + +function Inner({ response }: Props) { + const [activeEventIndex, setActiveEventIndex] = useState(null); + const { data: events, error, isLoading } = useHttpResponseEvents(response); + + const activeEvent = useMemo( + () => (activeEventIndex == null ? null : events?.[activeEventIndex]), + [activeEventIndex, events], + ); + + if (isLoading) { + return
Loading events...
; + } + + if (error) { + return ( + + {String(error)} + + ); + } + + if (!events || events.length === 0) { + return
No events recorded
; + } + + return ( + ( + ( + { + if (i === activeEventIndex) setActiveEventIndex(null); + else setActiveEventIndex(i); + }} + /> + )} + /> + )} + secondSlot={ + activeEvent + ? () => ( +
+
+ +
+
+ +
+
+ ) + : null + } + /> + ); +} + +function EventRow({ + onClick, + isActive, + event, +}: { + onClick: () => void; + isActive: boolean; + event: HttpResponseEvent; +}) { + const display = getEventDisplay(event.event); + const { icon, color, summary } = display; + + return ( +
+ +
+ ); +} + +function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; +} + +function EventDetails({ event }: { event: HttpResponseEvent }) { + const { label } = getEventDisplay(event.event); + const timestamp = format(new Date(`${event.createdAt}Z`), 'HH:mm:ss.SSS'); + const e = event.event; + + // Headers - show name and value with Editor for JSON + if (e.type === 'header_up' || e.type === 'header_down') { + return ( +
+ + + {e.name} + {e.value} + +
+ ); + } + + // Request URL - show method and path separately + if (e.type === 'send_url') { + return ( +
+ + + + + + {e.path} + +
+ ); + } + + // Response status - show version and status separately + if (e.type === 'receive_url') { + return ( +
+ + + {e.version} + + + + +
+ ); + } + + // Redirect - show status, URL, and behavior + if (e.type === 'redirect') { + return ( +
+ + + + + + {e.url} + + {e.behavior === 'drop_body' ? 'Drop body, change to GET' : 'Preserve method and body'} + + +
+ ); + } + + // Settings - show as key/value + if (e.type === 'setting') { + return ( +
+ + + {e.name} + {e.value} + +
+ ); + } + + // Chunks - show formatted bytes + if (e.type === 'chunk_sent' || e.type === 'chunk_received') { + const direction = e.type === 'chunk_sent' ? 'Sent' : 'Received'; + return ( +
+ +
{formatBytes(e.bytes)}
+
+ ); + } + + // Default - use summary + const { summary } = getEventDisplay(event.event); + return ( +
+ +
{summary}
+
+ ); +} + +function DetailHeader({ title, timestamp }: { title: string; timestamp: string }) { + return ( +
+

{title}

+ {timestamp} +
+ ); +} + +type EventDisplay = { + icon: IconProps['icon']; + color: IconProps['color']; + label: string; + summary: ReactNode; +}; + +function getEventDisplay(event: HttpResponseEventData): EventDisplay { + switch (event.type) { + case 'setting': + return { + icon: 'settings', + color: 'secondary', + label: 'Setting', + summary: `${event.name} = ${event.value}`, + }; + case 'info': + return { + icon: 'info', + color: 'secondary', + label: 'Info', + summary: event.message, + }; + case 'redirect': + return { + icon: 'arrow_big_right_dash', + color: 'warning', + label: 'Redirect', + summary: `Redirecting ${event.status} ${event.url}${event.behavior === 'drop_body' ? ' (drop body)' : ''}`, + }; + case 'send_url': + return { + icon: 'arrow_big_up_dash', + color: 'primary', + label: 'Request', + summary: `${event.method} ${event.path}`, + }; + case 'receive_url': + return { + icon: 'arrow_big_down_dash', + color: 'info', + label: 'Response', + summary: `${event.version} ${event.status}`, + }; + case 'header_up': + return { + icon: 'arrow_big_up_dash', + color: 'primary', + label: 'Header', + summary: `${event.name}: ${event.value}`, + }; + case 'header_down': + return { + icon: 'arrow_big_down_dash', + color: 'info', + label: 'Header', + summary: `${event.name}: ${event.value}`, + }; + + case 'chunk_sent': + return { + icon: 'info', + color: 'secondary', + label: 'Chunk', + summary: `${formatBytes(event.bytes)} chunk sent`, + }; + case 'chunk_received': + return { + icon: 'info', + color: 'secondary', + label: 'Chunk', + summary: `${formatBytes(event.bytes)} chunk received`, + }; + default: + return { + icon: 'info', + color: 'secondary', + label: 'Unknown', + summary: 'Unknown event', + }; + } +} diff --git a/src-web/components/RequestBodyViewer.tsx b/src-web/components/RequestBodyViewer.tsx new file mode 100644 index 00000000..7e0ccd51 --- /dev/null +++ b/src-web/components/RequestBodyViewer.tsx @@ -0,0 +1,102 @@ +import type { HttpResponse } from '@yaakapp-internal/models'; +import { lazy, Suspense } from 'react'; +import { useHttpRequestBody } from '../hooks/useHttpRequestBody'; +import { getMimeTypeFromContentType, languageFromContentType } from '../lib/contentType'; +import { EmptyStateText } from './EmptyStateText'; +import { LoadingIcon } from './core/LoadingIcon'; +import { AudioViewer } from './responseViewers/AudioViewer'; +import { CsvViewer } from './responseViewers/CsvViewer'; +import { ImageViewer } from './responseViewers/ImageViewer'; +import { MultipartViewer } from './responseViewers/MultipartViewer'; +import { SvgViewer } from './responseViewers/SvgViewer'; +import { TextViewer } from './responseViewers/TextViewer'; +import { VideoViewer } from './responseViewers/VideoViewer'; +import { WebPageViewer } from './responseViewers/WebPageViewer'; + +const PdfViewer = lazy(() => + import('./responseViewers/PdfViewer').then((m) => ({ default: m.PdfViewer })), +); + +interface Props { + response: HttpResponse; +} + +export function RequestBodyViewer({ response }: Props) { + return ; +} + +function RequestBodyViewerInner({ response }: Props) { + const { data, isLoading, error } = useHttpRequestBody(response); + + if (isLoading) { + return ( + + + + ); + } + + if (error) { + return Error loading request body: {error.message}; + } + + if (data?.bodyText == null || data.bodyText.length === 0) { + return No request body; + } + + const { bodyText, body } = data; + + // Try to detect language from content-type header that was sent + const contentTypeHeader = response.requestHeaders.find( + (h) => h.name.toLowerCase() === 'content-type', + ); + const contentType = contentTypeHeader?.value ?? null; + const mimeType = contentType ? getMimeTypeFromContentType(contentType).essence : null; + const language = languageFromContentType(contentType, bodyText); + + // Route to appropriate viewer based on content type + if (mimeType?.match(/^multipart/i)) { + const boundary = contentType?.split('boundary=')[1] ?? 'unknown'; + // Create a copy because parseMultipart may detach the buffer + const bodyCopy = new Uint8Array(body); + return ( + + ); + } + + if (mimeType?.match(/^image\/svg/i)) { + return ; + } + + if (mimeType?.match(/^image/i)) { + return ; + } + + if (mimeType?.match(/^audio/i)) { + return ; + } + + if (mimeType?.match(/^video/i)) { + return ; + } + + if (mimeType?.match(/csv|tab-separated/i)) { + return ; + } + + if (mimeType?.match(/^text\/html/i)) { + return ; + } + + if (mimeType?.match(/pdf/i)) { + return ( + }> + + + ); + } + + return ( + + ); +} diff --git a/src-web/components/ResponseHeaders.tsx b/src-web/components/ResponseHeaders.tsx index 04bffc07..e109c806 100644 --- a/src-web/components/ResponseHeaders.tsx +++ b/src-web/components/ResponseHeaders.tsx @@ -1,5 +1,7 @@ import type { HttpResponse } from '@yaakapp-internal/models'; import { useMemo } from 'react'; +import { CountBadge } from './core/CountBadge'; +import { DetailsBanner } from './core/DetailsBanner'; import { KeyValueRow, KeyValueRows } from './core/KeyValueRow'; interface Props { @@ -7,20 +9,69 @@ interface Props { } export function ResponseHeaders({ response }: Props) { - const sortedHeaders = useMemo( - () => [...response.headers].sort((a, b) => a.name.localeCompare(b.name)), + const responseHeaders = useMemo( + () => + [...response.headers].sort((a, b) => + a.name.toLocaleLowerCase().localeCompare(b.name.toLocaleLowerCase()), + ), [response.headers], ); + const requestHeaders = useMemo( + () => + [...response.requestHeaders].sort((a, b) => + a.name.toLocaleLowerCase().localeCompare(b.name.toLocaleLowerCase()), + ), + [response.requestHeaders], + ); return ( -
- - {sortedHeaders.map((h, i) => ( - // biome-ignore lint/suspicious/noArrayIndexKey: none - - {h.value} - - ))} - +
+ + Request + + } + > + {requestHeaders.length === 0 ? ( + + ) : ( + + {requestHeaders.map((h, i) => ( + // biome-ignore lint/suspicious/noArrayIndexKey: none + + {h.value} + + ))} + + )} + + + Response + + } + > + {responseHeaders.length === 0 ? ( + + ) : ( + + {responseHeaders.map((h, i) => ( + // biome-ignore lint/suspicious/noArrayIndexKey: none + + {h.value} + + ))} + + )} +
); } + +function NoHeaders() { + return No Headers; +} diff --git a/src-web/components/ResponseInfo.tsx b/src-web/components/ResponseInfo.tsx index ea17ae74..dcb68816 100644 --- a/src-web/components/ResponseInfo.tsx +++ b/src-web/components/ResponseInfo.tsx @@ -12,10 +12,10 @@ export function ResponseInfo({ response }: Props) {
- {response.version} + {response.version ?? --} - {response.remoteAddr} + {response.remoteAddr ?? --}
{stack}
diff --git a/src-web/components/Settings/SettingsCertificates.tsx b/src-web/components/Settings/SettingsCertificates.tsx index d7f069ea..e6081e72 100644 --- a/src-web/components/Settings/SettingsCertificates.tsx +++ b/src-web/components/Settings/SettingsCertificates.tsx @@ -53,7 +53,7 @@ function CertificateEditor({ certificate, index, onUpdate, onRemove }: Certifica return ( diff --git a/src-web/components/core/Button.tsx b/src-web/components/core/Button.tsx index c7fa3097..3d08249a 100644 --- a/src-web/components/core/Button.tsx +++ b/src-web/components/core/Button.tsx @@ -61,7 +61,6 @@ export const Button = forwardRef(function Button 'x-theme-button', `x-theme-button--${variant}`, `x-theme-button--${variant}--${color}`, - 'text-text', 'border', // They all have borders to ensure the same width 'max-w-full min-w-0', // Help with truncation 'hocus:opacity-100', // Force opacity for certain hover effects @@ -81,7 +80,7 @@ export const Button = forwardRef(function Button variant === 'solid' && color === 'custom' && 'focus-visible:outline-2 outline-border-focus', variant === 'solid' && color !== 'custom' && - 'enabled:hocus:text-text enabled:hocus:bg-surface-highlight outline-border-subtle', + 'text-text enabled:hocus:text-text enabled:hocus:bg-surface-highlight outline-border-subtle', variant === 'solid' && color !== 'custom' && color !== 'default' && 'bg-surface', // Borders diff --git a/src-web/components/core/CountBadge.tsx b/src-web/components/core/CountBadge.tsx index a719c275..d7b8d53e 100644 --- a/src-web/components/core/CountBadge.tsx +++ b/src-web/components/core/CountBadge.tsx @@ -3,12 +3,15 @@ import classNames from 'classnames'; interface Props { count: number | true; + count2?: number | true; className?: string; color?: Color; + showZero?: boolean; } -export function CountBadge({ count, className, color }: Props) { - if (count === 0) return null; +export function CountBadge({ count, count2, className, color, showZero }: Props) { + if (count === 0 && !showZero) return null; + return (
+ / + {count2 === true ? ( +
+ ) : ( + count2 + )} + + )}
); } diff --git a/src-web/components/core/DetailsBanner.tsx b/src-web/components/core/DetailsBanner.tsx index 32368c32..df0f3c4c 100644 --- a/src-web/components/core/DetailsBanner.tsx +++ b/src-web/components/core/DetailsBanner.tsx @@ -1,19 +1,48 @@ import classNames from 'classnames'; +import { atom, useAtom } from 'jotai'; import type { HTMLAttributes, ReactNode } from 'react'; +import { useMemo } from 'react'; +import { atomWithKVStorage } from '../../lib/atoms/atomWithKVStorage'; import type { BannerProps } from './Banner'; import { Banner } from './Banner'; interface Props extends HTMLAttributes { summary: ReactNode; color?: BannerProps['color']; - open?: boolean; + defaultOpen?: boolean; + storageKey?: string; } -export function DetailsBanner({ className, color, summary, children, ...extraProps }: Props) { +export function DetailsBanner({ + className, + color, + summary, + children, + defaultOpen, + storageKey, + ...extraProps +}: Props) { + // biome-ignore lint/correctness/useExhaustiveDependencies: We only want to recompute the atom when storageKey changes + const openAtom = useMemo( + () => + storageKey + ? atomWithKVStorage(['details_banner', storageKey], defaultOpen ?? false) + : atom(defaultOpen ?? false), + [storageKey], + ); + + const [isOpen, setIsOpen] = useAtom(openAtom); + + const handleToggle = (e: React.SyntheticEvent) => { + if (storageKey) { + setIsOpen(e.currentTarget.open); + } + }; + return ( -
- +
+
{ + describe('${[var]} format (valid template tags)', () => { + test('parses simple variable as Tag', () => { + expect(hasTag('${[var]}')).toBe(true); + expect(hasError('${[var]}')).toBe(false); + }); + + test('parses variable with whitespace as Tag', () => { + expect(hasTag('${[ var ]}')).toBe(true); + expect(hasError('${[ var ]}')).toBe(false); + }); + + test('parses embedded variable as Tag', () => { + expect(hasTag('hello ${[name]} world')).toBe(true); + expect(hasError('hello ${[name]} world')).toBe(false); + }); + + test('parses function call as Tag', () => { + expect(hasTag('${[fn()]}')).toBe(true); + expect(hasError('${[fn()]}')).toBe(false); + }); + }); + + describe('${var} format (should be plain text, not tags)', () => { + test('parses ${var} as plain Text without errors', () => { + expect(hasTag('${var}')).toBe(false); + expect(hasError('${var}')).toBe(false); + }); + + test('parses embedded ${var} as plain Text', () => { + expect(hasTag('hello ${name} world')).toBe(false); + expect(hasError('hello ${name} world')).toBe(false); + }); + + test('parses JSON with ${var} as plain Text', () => { + const json = '{"key": "${value}"}'; + expect(hasTag(json)).toBe(false); + expect(hasError(json)).toBe(false); + }); + + test('parses multiple ${var} as plain Text', () => { + expect(hasTag('${a} and ${b}')).toBe(false); + expect(hasError('${a} and ${b}')).toBe(false); + }); + }); + + describe('mixed content', () => { + test('distinguishes ${var} from ${[var]} in same string', () => { + const input = '${plain} and ${[tag]}'; + expect(hasTag(input)).toBe(true); + expect(hasError(input)).toBe(false); + }); + + test('parses JSON with ${[var]} as having Tag', () => { + const json = '{"key": "${[value]}"}'; + expect(hasTag(json)).toBe(true); + expect(hasError(json)).toBe(false); + }); + }); + + describe('edge cases', () => { + test('handles $ at end of string', () => { + expect(hasError('hello$')).toBe(false); + expect(hasTag('hello$')).toBe(false); + }); + + test('handles ${ at end of string without crash', () => { + // Incomplete syntax may produce errors, but should not crash + expect(() => parser.parse('hello${')).not.toThrow(); + }); + + test('handles ${[ without closing without crash', () => { + // Unclosed tag may produce partial match, but should not crash + expect(() => parser.parse('${[unclosed')).not.toThrow(); + }); + + test('handles empty ${[]}', () => { + // Empty tags may or may not be valid depending on grammar + // Just ensure no crash + expect(() => parser.parse('${[]}')).not.toThrow(); + }); + }); +}); diff --git a/src-web/components/core/Editor/twig/twig.ts b/src-web/components/core/Editor/twig/twig.ts index 20c2ff32..41f91be7 100644 --- a/src-web/components/core/Editor/twig/twig.ts +++ b/src-web/components/core/Editor/twig/twig.ts @@ -1,20 +1,18 @@ // This file was generated by lezer-generator. You probably shouldn't edit it. -import { LocalTokenGroup, LRParser } from '@lezer/lr'; -import { highlight } from './highlight'; +import {LRParser, LocalTokenGroup} from "@lezer/lr" +import {highlight} from "./highlight" export const parser = LRParser.deserialize({ version: 14, - states: - "!^QQOPOOOOOO'#C_'#C_OYOQO'#C^OOOO'#Cc'#CcQQOPOOOOOO'#Cd'#CdO_OQO,58xOOOO-E6a-E6aOOOO-E6b-E6bOOOO1G.d1G.d", - stateData: 'g~OUROYPO~OSTO~OSTOTXO~O', - goto: 'nXPPY^PPPbhTROSTQOSQSORVSQUQRWU', - nodeNames: '⚠ Template Tag TagOpen TagContent TagClose Text', + states: "!^QQOPOOOOOO'#C_'#C_OYOQO'#C^OOOO'#Cc'#CcQQOPOOOOOO'#Cd'#CdO_OQO,58xOOOO-E6a-E6aOOOO-E6b-E6bOOOO1G.d1G.d", + stateData: "g~OUROYPO~OSTO~OSTOTXO~O", + goto: "nXPPY^PPPbhTROSTQOSQSORVSQUQRWU", + nodeNames: "⚠ Template Tag TagOpen TagContent TagClose Text", maxTerm: 10, propSources: [highlight], skippedNodes: [0], repeatNodeCount: 2, - tokenData: - "#]~RTOtbtu!hu;'Sb;'S;=`!]<%lOb~gTU~Otbtuvu;'Sb;'S;=`!]<%lOb~yUO#ob#p;'Sb;'S;=`!]<%l~b~Ob~~!c~!`P;=`<%lb~!hOU~~!kVO#ob#o#p#Q#p;'Sb;'S;=`!]<%l~b~Ob~~!c~#TP!}#O#W~#]OY~", - tokenizers: [1, new LocalTokenGroup('b~RP#P#QU~XP#q#r[~aOT~~', 17, 4)], - topRules: { Template: [0, 1] }, - tokenPrec: 0, -}); + tokenData: "#{~RTOtbtu!zu;'Sb;'S;=`!o<%lOb~gTU~Otbtuvu;'Sb;'S;=`!o<%lOb~yVO#ob#o#p!`#p;'Sb;'S;=`!o<%l~b~Ob~~!u~!cSO!}b#O;'Sb;'S;=`!o<%lOb~!rP;=`<%lb~!zOU~~!}VO#ob#o#p#d#p;'Sb;'S;=`!o<%l~b~Ob~~!u~#gTO!}b!}#O#v#O;'Sb;'S;=`!o<%lOb~#{OY~", + tokenizers: [1, new LocalTokenGroup("b~RP#P#QU~XP#q#r[~aOT~~", 17, 4)], + topRules: {"Template":[0,1]}, + tokenPrec: 0 +}) diff --git a/src-web/components/core/FormattedError.tsx b/src-web/components/core/FormattedError.tsx index 0d95cade..b7889467 100644 --- a/src-web/components/core/FormattedError.tsx +++ b/src-web/components/core/FormattedError.tsx @@ -3,12 +3,14 @@ import type { ReactNode } from 'react'; interface Props { children: ReactNode; + className?: string; } -export function FormattedError({ children }: Props) { +export function FormattedError({ children, className }: Props) { return ( 
;
+}
 
+export function HttpStatusTagRaw({
+  status,
+  state,
+  className,
+  showReason,
+  statusReason,
+  short,
+}: Omit & {
+  status: number | string;
+  state?: HttpResponseState;
+  statusReason?: string | null;
+}) {
   let colorClass: string;
   let label = `${status}`;
+  const statusN = typeof status === 'number' ? status : parseInt(status, 10);
 
   if (state === 'initialized') {
     label = short ? 'CONN' : 'CONNECTING';
     colorClass = 'text-text-subtle';
-  } else if (status < 100) {
+  } else if (statusN < 100) {
     label = short ? 'ERR' : 'ERROR';
     colorClass = 'text-danger';
-  } else if (status < 200) {
+  } else if (statusN < 200) {
     colorClass = 'text-info';
-  } else if (status < 300) {
+  } else if (statusN < 300) {
     colorClass = 'text-success';
-  } else if (status < 400) {
+  } else if (statusN < 400) {
     colorClass = 'text-primary';
-  } else if (status < 500) {
+  } else if (statusN < 500) {
     colorClass = 'text-warning';
   } else {
     colorClass = 'text-danger';
@@ -34,7 +50,7 @@ export function HttpStatusTag({ response, className, showReason, short }: Props)
 
   return (
     
-      {label} {showReason && 'statusReason' in response ? response.statusReason : null}
+      {label} {showReason && statusReason}
     
   );
 }
diff --git a/src-web/components/core/Icon.tsx b/src-web/components/core/Icon.tsx
index 224a2922..80011b98 100644
--- a/src-web/components/core/Icon.tsx
+++ b/src-web/components/core/Icon.tsx
@@ -12,7 +12,9 @@ import {
   ArrowDownIcon,
   ArrowDownToDotIcon,
   ArrowDownToLineIcon,
+  ArrowLeftIcon,
   ArrowRightCircleIcon,
+  ArrowRightIcon,
   ArrowUpDownIcon,
   ArrowUpFromDotIcon,
   ArrowUpFromLineIcon,
@@ -54,6 +56,7 @@ import {
   EyeIcon,
   EyeOffIcon,
   FileCodeIcon,
+  FileIcon,
   FileTextIcon,
   FilterIcon,
   FlameIcon,
@@ -108,7 +111,7 @@ import {
   Rows2Icon,
   SaveIcon,
   SearchIcon,
-  SendHorizonalIcon,
+  SendHorizontalIcon,
   SettingsIcon,
   ShieldAlertIcon,
   ShieldCheckIcon,
@@ -142,6 +145,8 @@ const icons = {
   arrow_down: ArrowDownIcon,
   arrow_down_to_dot: ArrowDownToDotIcon,
   arrow_down_to_line: ArrowDownToLineIcon,
+  arrow_left: ArrowLeftIcon,
+  arrow_right: ArrowRightIcon,
   arrow_right_circle: ArrowRightCircleIcon,
   arrow_up: ArrowUpIcon,
   arrow_up_down: ArrowUpDownIcon,
@@ -183,7 +188,9 @@ const icons = {
   external_link: ExternalLinkIcon,
   eye: EyeIcon,
   eye_closed: EyeOffIcon,
+  file: FileIcon,
   file_code: FileCodeIcon,
+  file_text: FileTextIcon,
   filter: FilterIcon,
   flame: FlameIcon,
   flask: FlaskConicalIcon,
@@ -238,7 +245,7 @@ const icons = {
   rows_2: Rows2Icon,
   save: SaveIcon,
   search: SearchIcon,
-  send_horizontal: SendHorizonalIcon,
+  send_horizontal: SendHorizontalIcon,
   settings: SettingsIcon,
   shield: ShieldIcon,
   shield_check: ShieldCheckIcon,
diff --git a/src-web/components/core/KeyValueRow.tsx b/src-web/components/core/KeyValueRow.tsx
index a6ba7fc8..ec08d6ba 100644
--- a/src-web/components/core/KeyValueRow.tsx
+++ b/src-web/components/core/KeyValueRow.tsx
@@ -10,7 +10,7 @@ interface Props {
 export function KeyValueRows({ children }: Props) {
   children = Array.isArray(children) ? children : [children];
   return (
-    
+    

         {children.map((child, i) => (
           // biome-ignore lint/suspicious/noArrayIndexKey: none
diff --git a/src-web/components/core/PairEditor.tsx b/src-web/components/core/PairEditor.tsx
index 67beb4aa..3824c513 100644
--- a/src-web/components/core/PairEditor.tsx
+++ b/src-web/components/core/PairEditor.tsx
@@ -678,7 +678,7 @@ export function PairEditorRow({
             size="xs"
             icon={isLast || disabled ? 'empty' : 'chevron_down'}
             title="Select form data type"
-            className="text-text-subtle"
+            className="text-text-subtlest"
           />
         
       )}
@@ -798,7 +798,13 @@ function FileActionsDropdown({
       items={fileItems}
       itemsAfter={itemsAfter}
     >
-      
+      
     
   );
 }
diff --git a/src-web/components/core/SizeTag.tsx b/src-web/components/core/SizeTag.tsx
index 845a8d42..05bf8286 100644
--- a/src-web/components/core/SizeTag.tsx
+++ b/src-web/components/core/SizeTag.tsx
@@ -2,11 +2,18 @@ import { formatSize } from '@yaakapp-internal/lib/formatSize';
 
 interface Props {
   contentLength: number;
+  contentLengthCompressed?: number | null;
 }
 
-export function SizeTag({ contentLength }: Props) {
+export function SizeTag({ contentLength, contentLengthCompressed }: Props) {
   return (
-    
+    
       {formatSize(contentLength)}
     
   );
diff --git a/src-web/components/responseViewers/AudioViewer.tsx b/src-web/components/responseViewers/AudioViewer.tsx
index 8eb616b8..a38c7d7a 100644
--- a/src-web/components/responseViewers/AudioViewer.tsx
+++ b/src-web/components/responseViewers/AudioViewer.tsx
@@ -1,11 +1,26 @@
 import { convertFileSrc } from '@tauri-apps/api/core';
+import { useEffect, useState } from 'react';
 
 interface Props {
-  bodyPath: string;
+  bodyPath?: string;
+  data?: Uint8Array;
 }
 
-export function AudioViewer({ bodyPath }: Props) {
-  const src = convertFileSrc(bodyPath);
+export function AudioViewer({ bodyPath, data }: Props) {
+  const [src, setSrc] = useState();
+
+  useEffect(() => {
+    if (bodyPath) {
+      setSrc(convertFileSrc(bodyPath));
+    } else if (data) {
+      const blob = new Blob([data], { type: 'audio/mpeg' });
+      const url = URL.createObjectURL(blob);
+      setSrc(url);
+      return () => URL.revokeObjectURL(url);
+    } else {
+      setSrc(undefined);
+    }
+  }, [bodyPath, data]);
 
   // biome-ignore lint/a11y/useMediaCaption: none
   return 

       

-        
+      

+        
+          
+            {parsed.meta.fields?.map((field) => (
+              {field}
+            ))}
+          
+        
+        
           {parsed.data.map((row, i) => (
             // biome-ignore lint/suspicious/noArrayIndexKey: none
-             0 && 'border-b')}>
-              {row.map((col, j) => (
-                // biome-ignore lint/suspicious/noArrayIndexKey: none
-                
+            
+              {parsed.meta.fields?.map((key) => (
+                {row[key] ?? ''}
               ))}
-            
+            
           ))}
-        
-      

-                  {col}
-                

+        
+
); } diff --git a/src-web/components/responseViewers/HTMLOrTextViewer.tsx b/src-web/components/responseViewers/HTMLOrTextViewer.tsx index 90938b81..f1bb5a92 100644 --- a/src-web/components/responseViewers/HTMLOrTextViewer.tsx +++ b/src-web/components/responseViewers/HTMLOrTextViewer.tsx @@ -1,7 +1,9 @@ import type { HttpResponse } from '@yaakapp-internal/models'; +import { useMemo, useState } from 'react'; import { useResponseBodyText } from '../../hooks/useResponseBodyText'; import { languageFromContentType } from '../../lib/contentType'; import { getContentTypeFromHeaders } from '../../lib/model_util'; +import type { EditorProps } from '../core/Editor/Editor'; import { EmptyStateText } from '../EmptyStateText'; import { TextViewer } from './TextViewer'; import { WebPageViewer } from './WebPageViewer'; @@ -22,19 +24,54 @@ export function HTMLOrTextViewer({ response, pretty, textViewerClassName }: Prop } if (language === 'html' && pretty) { - return ; + return ; } if (rawTextBody.data == null) { return Empty response; } return ( - + ); +} + +interface HttpTextViewerProps { + response: HttpResponse; + text: string; + language: EditorProps['language']; + pretty: boolean; + className?: string; +} + +function HttpTextViewer({ response, text, language, pretty, className }: HttpTextViewerProps) { + const [currentFilter, setCurrentFilter] = useState(null); + const filteredBody = useResponseBodyText({ response, filter: currentFilter }); + + const filterCallback = useMemo( + () => (filter: string) => { + setCurrentFilter(filter); + return { + data: filteredBody.data, + isPending: filteredBody.isPending, + error: !!filteredBody.error, + }; + }, + [filteredBody], + ); + + return ( + ); } diff --git a/src-web/components/responseViewers/ImageViewer.tsx b/src-web/components/responseViewers/ImageViewer.tsx index 6967f27e..a584c7c4 100644 --- a/src-web/components/responseViewers/ImageViewer.tsx +++ b/src-web/components/responseViewers/ImageViewer.tsx @@ -1,10 +1,39 @@ import { convertFileSrc } from '@tauri-apps/api/core'; +import classNames from 'classnames'; +import { useEffect, useState } from 'react'; -interface Props { - bodyPath: string; -} +type Props = { className?: string } & ( + | { + bodyPath: string; + } + | { + data: ArrayBuffer; + } +); -export function ImageViewer({ bodyPath }: Props) { - const src = convertFileSrc(bodyPath); - return Response preview; +export function ImageViewer({ className, ...props }: Props) { + const [src, setSrc] = useState(); + const bodyPath = 'bodyPath' in props ? props.bodyPath : null; + const data = 'data' in props ? props.data : null; + + useEffect(() => { + if (bodyPath != null) { + setSrc(convertFileSrc(bodyPath)); + } else if (data != null) { + const blob = new Blob([data], { type: 'image/png' }); + const url = URL.createObjectURL(blob); + setSrc(url); + return () => URL.revokeObjectURL(url); + } else { + setSrc(undefined); + } + }, [bodyPath, data]); + + return ( + Response preview + ); } diff --git a/src-web/components/responseViewers/JsonViewer.tsx b/src-web/components/responseViewers/JsonViewer.tsx index 28c5ceaf..603d8b60 100644 --- a/src-web/components/responseViewers/JsonViewer.tsx +++ b/src-web/components/responseViewers/JsonViewer.tsx @@ -1,21 +1,15 @@ -import type { HttpResponse } from '@yaakapp-internal/models'; import classNames from 'classnames'; -import { useResponseBodyText } from '../../hooks/useResponseBodyText'; import { JsonAttributeTree } from '../core/JsonAttributeTree'; interface Props { - response: HttpResponse; + text: string; className?: string; } -export function JsonViewer({ response, className }: Props) { - const rawBody = useResponseBodyText({ response, filter: null }); - - if (rawBody.isLoading || rawBody.data == null) return null; - +export function JsonViewer({ text, className }: Props) { let parsed = {}; try { - parsed = JSON.parse(rawBody.data); + parsed = JSON.parse(text); } catch { // Nothing yet } diff --git a/src-web/components/responseViewers/MultipartViewer.tsx b/src-web/components/responseViewers/MultipartViewer.tsx new file mode 100644 index 00000000..c21b2a71 --- /dev/null +++ b/src-web/components/responseViewers/MultipartViewer.tsx @@ -0,0 +1,138 @@ +import { type MultipartPart, parseMultipart } from '@mjackson/multipart-parser'; +import { lazy, Suspense, useMemo, useState } from 'react'; +import { languageFromContentType } from '../../lib/contentType'; +import { Banner } from '../core/Banner'; +import { Icon } from '../core/Icon'; +import { LoadingIcon } from '../core/LoadingIcon'; +import { TabContent, Tabs } from '../core/Tabs/Tabs'; +import { AudioViewer } from './AudioViewer'; +import { CsvViewer } from './CsvViewer'; +import { ImageViewer } from './ImageViewer'; +import { SvgViewer } from './SvgViewer'; +import { TextViewer } from './TextViewer'; +import { VideoViewer } from './VideoViewer'; +import { WebPageViewer } from './WebPageViewer'; + +const PdfViewer = lazy(() => import('./PdfViewer').then((m) => ({ default: m.PdfViewer }))); + +interface Props { + data: Uint8Array; + boundary: string; + idPrefix?: string; +} + +export function MultipartViewer({ data, boundary, idPrefix = 'multipart' }: Props) { + const [tab, setTab] = useState(); + + const parseResult = useMemo(() => { + try { + const maxFileSize = 1024 * 1024 * 10; // 10MB + const parsed = parseMultipart(data, { boundary, maxFileSize }); + const parts = Array.from(parsed); + return { parts, error: null }; + } catch (err) { + return { parts: [], error: err instanceof Error ? err.message : String(err) }; + } + }, [data, boundary]); + + const { parts, error } = parseResult; + + if (error) { + return ( + + Failed to parse multipart data: {error} + + ); + } + + if (parts.length === 0) { + return ( + + No multipart parts found + + ); + } + + return ( + ({ + label: part.name ?? '', + value: part.name ?? '', + rightSlot: + part.filename && part.headers.contentType.mediaType?.startsWith('image/') ? ( +
+ +
+ ) : part.filename ? ( + + ) : null, + }))} + > + {parts.map((part, i) => ( + + + + ))} + + ); +} + +function Part({ part }: { part: MultipartPart }) { + const mimeType = part.headers.contentType.mediaType ?? null; + const contentTypeHeader = part.headers.get('content-type'); + + const { uint8Array, content, detectedLanguage } = useMemo(() => { + const uint8Array = new Uint8Array(part.arrayBuffer); + const content = new TextDecoder().decode(part.arrayBuffer); + const detectedLanguage = languageFromContentType(contentTypeHeader, content); + return { uint8Array, content, detectedLanguage }; + }, [part, contentTypeHeader]); + + if (mimeType?.match(/^image\/svg/i)) { + return ; + } + + if (mimeType?.match(/^image/i)) { + return ; + } + + if (mimeType?.match(/^audio/i)) { + return ; + } + + if (mimeType?.match(/^video/i)) { + return ; + } + + if (mimeType?.match(/csv|tab-separated/i)) { + return ; + } + + if (mimeType?.match(/^text\/html/i) || detectedLanguage === 'html') { + return ; + } + + if (mimeType?.match(/pdf/i)) { + return ( + }> + + + ); + } + + return ; +} diff --git a/src-web/components/responseViewers/PdfViewer.tsx b/src-web/components/responseViewers/PdfViewer.tsx index 1af697b4..d2fd239f 100644 --- a/src-web/components/responseViewers/PdfViewer.tsx +++ b/src-web/components/responseViewers/PdfViewer.tsx @@ -3,7 +3,7 @@ import 'react-pdf/dist/Page/AnnotationLayer.css'; import { convertFileSrc } from '@tauri-apps/api/core'; import './PdfViewer.css'; import type { PDFDocumentProxy } from 'pdfjs-dist'; -import { useRef, useState } from 'react'; +import { useEffect, useRef, useState } from 'react'; import { Document, Page } from 'react-pdf'; import { useContainerSize } from '../../hooks/useContainerQuery'; @@ -15,7 +15,8 @@ import('react-pdf').then(({ pdfjs }) => { }); interface Props { - bodyPath: string; + bodyPath?: string; + data?: Uint8Array; } const options = { @@ -23,17 +24,29 @@ const options = { standardFontDataUrl: '/standard_fonts/', }; -export function PdfViewer({ bodyPath }: Props) { +export function PdfViewer({ bodyPath, data }: Props) { const containerRef = useRef(null); const [numPages, setNumPages] = useState(); + const [src, setSrc] = useState(); const { width: containerWidth } = useContainerSize(containerRef); + useEffect(() => { + if (bodyPath) { + setSrc(convertFileSrc(bodyPath)); + } else if (data) { + // Create a copy to avoid "Buffer is already detached" errors + // This happens when the ArrayBuffer is transferred/detached elsewhere + const dataCopy = new Uint8Array(data); + setSrc({ data: dataCopy }); + } else { + setSrc(undefined); + } + }, [bodyPath, data]); + const onDocumentLoadSuccess = ({ numPages: nextNumPages }: PDFDocumentProxy): void => { setNumPages(nextNumPages); }; - - const src = convertFileSrc(bodyPath); return (
(null); useEffect(() => { - if (!rawTextBody.data) { + if (!text) { return setSrc(null); } - const blob = new Blob([rawTextBody.data], { type: 'image/svg+xml;charset=utf-8' }); + const blob = new Blob([text], { type: 'image/svg+xml;charset=utf-8' }); const url = URL.createObjectURL(blob); setSrc(url); return () => URL.revokeObjectURL(url); - }, [rawTextBody.data]); + }, [text]); if (src == null) { return null; } - return Response preview; + return ( + Response preview + ); } diff --git a/src-web/components/responseViewers/TextViewer.tsx b/src-web/components/responseViewers/TextViewer.tsx index c0095a0e..c03bc7e6 100644 --- a/src-web/components/responseViewers/TextViewer.tsx +++ b/src-web/components/responseViewers/TextViewer.tsx @@ -1,11 +1,9 @@ -import type { HttpResponse } from '@yaakapp-internal/models'; import classNames from 'classnames'; import type { ReactNode } from 'react'; import { useCallback, useMemo } from 'react'; import { createGlobalState } from 'react-use'; import { useDebouncedValue } from '../../hooks/useDebouncedValue'; import { useFormatText } from '../../hooks/useFormatText'; -import { useResponseBodyText } from '../../hooks/useResponseBodyText'; import type { EditorProps } from '../core/Editor/Editor'; import { hyperlink } from '../core/Editor/hyperlink/extension'; import { Editor } from '../core/Editor/LazyEditor'; @@ -15,29 +13,37 @@ import { Input } from '../core/Input'; const extraExtensions = [hyperlink]; interface Props { - pretty: boolean; - className?: string; text: string; language: EditorProps['language']; - response: HttpResponse; - requestId: string; + stateKey: string | null; + pretty?: boolean; + className?: string; + onFilter?: (filter: string) => { + data: string | null | undefined; + isPending: boolean; + error: boolean; + }; } const useFilterText = createGlobalState>({}); -export function TextViewer({ language, text, response, requestId, pretty, className }: Props) { +export function TextViewer({ language, text, stateKey, pretty, className, onFilter }: Props) { const [filterTextMap, setFilterTextMap] = useFilterText(); - const filterText = filterTextMap[requestId] ?? null; + const filterText = stateKey ? (filterTextMap[stateKey] ?? null) : null; const debouncedFilterText = useDebouncedValue(filterText); const setFilterText = useCallback( (v: string | null) => { - setFilterTextMap((m) => ({ ...m, [requestId]: v })); + if (!stateKey) return; + setFilterTextMap((m) => ({ ...m, [stateKey]: v })); }, - [setFilterTextMap, requestId], + [setFilterTextMap, stateKey], ); const isSearching = filterText != null; - const filteredResponse = useResponseBodyText({ response, filter: debouncedFilterText ?? null }); + const filteredResponse = + onFilter && debouncedFilterText + ? onFilter(debouncedFilterText) + : { data: null, isPending: false, error: false }; const toggleSearch = useCallback(() => { if (isSearching) { @@ -47,7 +53,7 @@ export function TextViewer({ language, text, response, requestId, pretty, classN } }, [isSearching, setFilterText]); - const canFilter = language === 'json' || language === 'xml' || language === 'html'; + const canFilter = onFilter && (language === 'json' || language === 'xml' || language === 'html'); const actions = useMemo(() => { const nodes: ReactNode[] = []; @@ -58,7 +64,7 @@ export function TextViewer({ language, text, response, requestId, pretty, classN nodes.push(
e.key === 'Escape' && toggleSearch()} onChange={setFilterText} - stateKey={`filter.${response.id}`} + stateKey={stateKey ? `filter.${stateKey}` : null} />
, ); @@ -96,13 +102,12 @@ export function TextViewer({ language, text, response, requestId, pretty, classN filteredResponse.isPending, isSearching, language, - requestId, - response, + stateKey, setFilterText, toggleSearch, ]); - const formattedBody = useFormatText({ text, language, pretty }); + const formattedBody = useFormatText({ text, language, pretty: pretty ?? false }); if (formattedBody == null) { return null; } @@ -132,8 +137,7 @@ export function TextViewer({ language, text, response, requestId, pretty, classN language={language} actions={actions} extraExtensions={extraExtensions} - // State key for storing fold state - stateKey={`response.body.${response.id}`} + stateKey={stateKey} /> ); } diff --git a/src-web/components/responseViewers/VideoViewer.tsx b/src-web/components/responseViewers/VideoViewer.tsx index 49a14b7a..0bad1975 100644 --- a/src-web/components/responseViewers/VideoViewer.tsx +++ b/src-web/components/responseViewers/VideoViewer.tsx @@ -1,11 +1,26 @@ import { convertFileSrc } from '@tauri-apps/api/core'; +import { useEffect, useState } from 'react'; interface Props { - bodyPath: string; + bodyPath?: string; + data?: Uint8Array; } -export function VideoViewer({ bodyPath }: Props) { - const src = convertFileSrc(bodyPath); +export function VideoViewer({ bodyPath, data }: Props) { + const [src, setSrc] = useState(); + + useEffect(() => { + if (bodyPath) { + setSrc(convertFileSrc(bodyPath)); + } else if (data) { + const blob = new Blob([data], { type: 'video/mp4' }); + const url = URL.createObjectURL(blob); + setSrc(url); + return () => URL.revokeObjectURL(url); + } else { + setSrc(undefined); + } + }, [bodyPath, data]); // biome-ignore lint/a11y/useMediaCaption: none return