Store and show request body in UI (#327)

src-tauri/yaak-models/src/blob_manager.rs

@@ -0,0 +1,372 @@
+use crate::error::Result;
+use crate::util::generate_prefixed_id;
+use include_dir::{Dir, include_dir};
+use log::{debug, info};
+use r2d2::Pool;
+use r2d2_sqlite::SqliteConnectionManager;
+use rusqlite::{OptionalExtension, params};
+use std::sync::{Arc, Mutex};
+use tauri::{Manager, Runtime, State};
+
+static BLOB_MIGRATIONS_DIR: Dir = include_dir!("$CARGO_MANIFEST_DIR/blob_migrations");
+
+/// A chunk of body data stored in the blob database.
+#[derive(Debug, Clone)]
+pub struct BodyChunk {
+    pub id: String,
+    pub body_id: String,
+    pub chunk_index: i32,
+    pub data: Vec<u8>,
+}
+
+impl BodyChunk {
+    pub fn new(body_id: impl Into<String>, chunk_index: i32, data: Vec<u8>) -> Self {
+        Self { id: generate_prefixed_id("bc"), body_id: body_id.into(), chunk_index, data }
+    }
+}
+
+/// Extension trait for accessing the blob manager from app handle.
+pub trait BlobManagerExt<'a, R> {
+    fn blob_manager(&'a self) -> State<'a, BlobManager>;
+    fn blobs(&'a self) -> BlobContext;
+}
+
+impl<'a, R: Runtime, M: Manager<R>> BlobManagerExt<'a, R> for M {
+    fn blob_manager(&'a self) -> State<'a, BlobManager> {
+        self.state::<BlobManager>()
+    }
+
+    fn blobs(&'a self) -> BlobContext {
+        let manager = self.state::<BlobManager>();
+        manager.inner().connect()
+    }
+}
+
+/// Manages the blob database connection pool.
+#[derive(Debug, Clone)]
+pub struct BlobManager {
+    pool: Arc<Mutex<Pool<SqliteConnectionManager>>>,
+}
+
+impl BlobManager {
+    pub fn new(pool: Pool<SqliteConnectionManager>) -> Self {
+        Self { pool: Arc::new(Mutex::new(pool)) }
+    }
+
+    pub fn connect(&self) -> BlobContext {
+        let conn = self
+            .pool
+            .lock()
+            .expect("Failed to gain lock on blob DB")
+            .get()
+            .expect("Failed to get blob DB connection from pool");
+        BlobContext { conn }
+    }
+}
+
+/// Context for blob database operations.
+pub struct BlobContext {
+    conn: r2d2::PooledConnection<SqliteConnectionManager>,
+}
+
+impl BlobContext {
+    /// Insert a single chunk.
+    pub fn insert_chunk(&self, chunk: &BodyChunk) -> Result<()> {
+        self.conn.execute(
+            "INSERT INTO body_chunks (id, body_id, chunk_index, data) VALUES (?1, ?2, ?3, ?4)",
+            params![chunk.id, chunk.body_id, chunk.chunk_index, chunk.data],
+        )?;
+        Ok(())
+    }
+
+    /// Get all chunks for a body, ordered by chunk_index.
+    pub fn get_chunks(&self, body_id: &str) -> Result<Vec<BodyChunk>> {
+        let mut stmt = self.conn.prepare(
+            "SELECT id, body_id, chunk_index, data FROM body_chunks
+             WHERE body_id = ?1 ORDER BY chunk_index ASC",
+        )?;
+
+        let chunks = stmt
+            .query_map(params![body_id], |row| {
+                Ok(BodyChunk {
+                    id: row.get(0)?,
+                    body_id: row.get(1)?,
+                    chunk_index: row.get(2)?,
+                    data: row.get(3)?,
+                })
+            })?
+            .collect::<std::result::Result<Vec<_>, _>>()?;
+
+        Ok(chunks)
+    }
+
+    /// Delete all chunks for a body.
+    pub fn delete_chunks(&self, body_id: &str) -> Result<()> {
+        self.conn.execute("DELETE FROM body_chunks WHERE body_id = ?1", params![body_id])?;
+        Ok(())
+    }
+
+    /// Delete all chunks matching a body_id prefix (e.g., "rs_abc123.%" to delete all bodies for a response).
+    pub fn delete_chunks_like(&self, body_id_prefix: &str) -> Result<()> {
+        self.conn
+            .execute("DELETE FROM body_chunks WHERE body_id LIKE ?1", params![body_id_prefix])?;
+        Ok(())
+    }
+}
+
+/// Get total size of a body without loading data.
+impl BlobContext {
+    pub fn get_body_size(&self, body_id: &str) -> Result<usize> {
+        let size: i64 = self
+            .conn
+            .query_row(
+                "SELECT COALESCE(SUM(LENGTH(data)), 0) FROM body_chunks WHERE body_id = ?1",
+                params![body_id],
+                |row| row.get(0),
+            )
+            .unwrap_or(0);
+        Ok(size as usize)
+    }
+
+    /// Check if a body exists.
+    pub fn body_exists(&self, body_id: &str) -> Result<bool> {
+        let count: i64 = self
+            .conn
+            .query_row(
+                "SELECT COUNT(*) FROM body_chunks WHERE body_id = ?1",
+                params![body_id],
+                |row| row.get(0),
+            )
+            .unwrap_or(0);
+        Ok(count > 0)
+    }
+}
+
+/// Run migrations for the blob database.
+pub fn migrate_blob_db(pool: &Pool<SqliteConnectionManager>) -> Result<()> {
+    info!("Running blob database migrations");
+
+    // Create migrations tracking table
+    pool.get()?.execute(
+        "CREATE TABLE IF NOT EXISTS _blob_migrations (
+            version     TEXT PRIMARY KEY,
+            description TEXT NOT NULL,
+            applied_at  DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL
+        )",
+        [],
+    )?;
+
+    // Read and sort all .sql files
+    let mut entries: Vec<_> = BLOB_MIGRATIONS_DIR
+        .entries()
+        .iter()
+        .filter(|e| e.path().extension().map(|ext| ext == "sql").unwrap_or(false))
+        .collect();
+
+    entries.sort_by_key(|e| e.path());
+
+    let mut ran_migrations = 0;
+    for entry in &entries {
+        let filename = entry.path().file_name().unwrap().to_str().unwrap();
+        let version = filename.split('_').next().unwrap();
+
+        // Check if already applied
+        let already_applied: Option<i64> = pool
+            .get()?
+            .query_row("SELECT 1 FROM _blob_migrations WHERE version = ?", [version], |r| r.get(0))
+            .optional()?;
+
+        if already_applied.is_some() {
+            debug!("Skipping already applied blob migration: {}", filename);
+            continue;
+        }
+
+        let sql =
+            entry.as_file().unwrap().contents_utf8().expect("Failed to read blob migration file");
+
+        info!("Applying blob migration: {}", filename);
+        let conn = pool.get()?;
+        conn.execute_batch(sql)?;
+
+        // Record migration
+        conn.execute(
+            "INSERT INTO _blob_migrations (version, description) VALUES (?, ?)",
+            params![version, filename],
+        )?;
+
+        ran_migrations += 1;
+    }
+
+    if ran_migrations == 0 {
+        info!("No blob migrations to run");
+    } else {
+        info!("Ran {} blob migration(s)", ran_migrations);
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn create_test_pool() -> Pool<SqliteConnectionManager> {
+        let manager = SqliteConnectionManager::memory();
+        let pool = Pool::builder().max_size(1).build(manager).unwrap();
+        migrate_blob_db(&pool).unwrap();
+        pool
+    }
+
+    #[test]
+    fn test_insert_and_get_chunks() {
+        let pool = create_test_pool();
+        let manager = BlobManager::new(pool);
+        let ctx = manager.connect();
+
+        let body_id = "rs_test123.request";
+        let chunk1 = BodyChunk::new(body_id, 0, b"Hello, ".to_vec());
+        let chunk2 = BodyChunk::new(body_id, 1, b"World!".to_vec());
+
+        ctx.insert_chunk(&chunk1).unwrap();
+        ctx.insert_chunk(&chunk2).unwrap();
+
+        let chunks = ctx.get_chunks(body_id).unwrap();
+        assert_eq!(chunks.len(), 2);
+        assert_eq!(chunks[0].chunk_index, 0);
+        assert_eq!(chunks[0].data, b"Hello, ");
+        assert_eq!(chunks[1].chunk_index, 1);
+        assert_eq!(chunks[1].data, b"World!");
+    }
+
+    #[test]
+    fn test_get_chunks_ordered_by_index() {
+        let pool = create_test_pool();
+        let manager = BlobManager::new(pool);
+        let ctx = manager.connect();
+
+        let body_id = "rs_test123.request";
+
+        // Insert out of order
+        ctx.insert_chunk(&BodyChunk::new(body_id, 2, b"C".to_vec())).unwrap();
+        ctx.insert_chunk(&BodyChunk::new(body_id, 0, b"A".to_vec())).unwrap();
+        ctx.insert_chunk(&BodyChunk::new(body_id, 1, b"B".to_vec())).unwrap();
+
+        let chunks = ctx.get_chunks(body_id).unwrap();
+        assert_eq!(chunks.len(), 3);
+        assert_eq!(chunks[0].data, b"A");
+        assert_eq!(chunks[1].data, b"B");
+        assert_eq!(chunks[2].data, b"C");
+    }
+
+    #[test]
+    fn test_delete_chunks() {
+        let pool = create_test_pool();
+        let manager = BlobManager::new(pool);
+        let ctx = manager.connect();
+
+        let body_id = "rs_test123.request";
+        ctx.insert_chunk(&BodyChunk::new(body_id, 0, b"data".to_vec())).unwrap();
+
+        assert!(ctx.body_exists(body_id).unwrap());
+
+        ctx.delete_chunks(body_id).unwrap();
+
+        assert!(!ctx.body_exists(body_id).unwrap());
+        assert_eq!(ctx.get_chunks(body_id).unwrap().len(), 0);
+    }
+
+    #[test]
+    fn test_delete_chunks_like() {
+        let pool = create_test_pool();
+        let manager = BlobManager::new(pool);
+        let ctx = manager.connect();
+
+        // Insert chunks for same response but different body types
+        ctx.insert_chunk(&BodyChunk::new("rs_abc.request", 0, b"req".to_vec())).unwrap();
+        ctx.insert_chunk(&BodyChunk::new("rs_abc.response", 0, b"resp".to_vec())).unwrap();
+        ctx.insert_chunk(&BodyChunk::new("rs_other.request", 0, b"other".to_vec())).unwrap();
+
+        // Delete all bodies for rs_abc
+        ctx.delete_chunks_like("rs_abc.%").unwrap();
+
+        // rs_abc bodies should be gone
+        assert!(!ctx.body_exists("rs_abc.request").unwrap());
+        assert!(!ctx.body_exists("rs_abc.response").unwrap());
+
+        // rs_other should still exist
+        assert!(ctx.body_exists("rs_other.request").unwrap());
+    }
+
+    #[test]
+    fn test_get_body_size() {
+        let pool = create_test_pool();
+        let manager = BlobManager::new(pool);
+        let ctx = manager.connect();
+
+        let body_id = "rs_test123.request";
+        ctx.insert_chunk(&BodyChunk::new(body_id, 0, b"Hello".to_vec())).unwrap();
+        ctx.insert_chunk(&BodyChunk::new(body_id, 1, b"World".to_vec())).unwrap();
+
+        let size = ctx.get_body_size(body_id).unwrap();
+        assert_eq!(size, 10); // "Hello" + "World" = 10 bytes
+    }
+
+    #[test]
+    fn test_get_body_size_empty() {
+        let pool = create_test_pool();
+        let manager = BlobManager::new(pool);
+        let ctx = manager.connect();
+
+        let size = ctx.get_body_size("nonexistent").unwrap();
+        assert_eq!(size, 0);
+    }
+
+    #[test]
+    fn test_body_exists() {
+        let pool = create_test_pool();
+        let manager = BlobManager::new(pool);
+        let ctx = manager.connect();
+
+        assert!(!ctx.body_exists("rs_test.request").unwrap());
+
+        ctx.insert_chunk(&BodyChunk::new("rs_test.request", 0, b"data".to_vec())).unwrap();
+
+        assert!(ctx.body_exists("rs_test.request").unwrap());
+    }
+
+    #[test]
+    fn test_multiple_bodies_isolated() {
+        let pool = create_test_pool();
+        let manager = BlobManager::new(pool);
+        let ctx = manager.connect();
+
+        ctx.insert_chunk(&BodyChunk::new("body1", 0, b"data1".to_vec())).unwrap();
+        ctx.insert_chunk(&BodyChunk::new("body2", 0, b"data2".to_vec())).unwrap();
+
+        let chunks1 = ctx.get_chunks("body1").unwrap();
+        let chunks2 = ctx.get_chunks("body2").unwrap();
+
+        assert_eq!(chunks1.len(), 1);
+        assert_eq!(chunks1[0].data, b"data1");
+        assert_eq!(chunks2.len(), 1);
+        assert_eq!(chunks2[0].data, b"data2");
+    }
+
+    #[test]
+    fn test_large_chunk() {
+        let pool = create_test_pool();
+        let manager = BlobManager::new(pool);
+        let ctx = manager.connect();
+
+        // 1MB chunk
+        let large_data: Vec<u8> = (0..1024 * 1024).map(|i| (i % 256) as u8).collect();
+        let body_id = "rs_large.request";
+
+        ctx.insert_chunk(&BodyChunk::new(body_id, 0, large_data.clone())).unwrap();
+
+        let chunks = ctx.get_chunks(body_id).unwrap();
+        assert_eq!(chunks.len(), 1);
+        assert_eq!(chunks[0].data, large_data);
+        assert_eq!(ctx.get_body_size(body_id).unwrap(), 1024 * 1024);
+    }
+}

src-tauri/yaak-models/src/commands.rs

@@ -1,3 +1,4 @@
+use crate::blob_manager::BlobManagerExt;
 use crate::error::Error::GenericError;
 use crate::error::Result;
 use crate::models::{AnyModel, GraphQlIntrospection, GrpcEvent, Settings, WebsocketEvent};
@@ -8,6 +9,7 @@ use tauri::{AppHandle, Runtime, WebviewWindow};
 #[tauri::command]
 pub(crate) fn upsert<R: Runtime>(window: WebviewWindow<R>, model: AnyModel) -> Result<String> {
     let db = window.db();
+    let blobs = window.blob_manager();
     let source = &UpdateSource::from_window(&window);
     let id = match model {
         AnyModel::CookieJar(m) => db.upsert_cookie_jar(&m, source)?.id,
@@ -15,7 +17,7 @@ pub(crate) fn upsert<R: Runtime>(window: WebviewWindow<R>, model: AnyModel) -> R
         AnyModel::Folder(m) => db.upsert_folder(&m, source)?.id,
         AnyModel::GrpcRequest(m) => db.upsert_grpc_request(&m, source)?.id,
         AnyModel::HttpRequest(m) => db.upsert_http_request(&m, source)?.id,
-        AnyModel::HttpResponse(m) => db.upsert_http_response(&m, source)?.id,
+        AnyModel::HttpResponse(m) => db.upsert_http_response(&m, source, &blobs)?.id,
         AnyModel::KeyValue(m) => db.upsert_key_value(&m, source)?.id,
         AnyModel::Plugin(m) => db.upsert_plugin(&m, source)?.id,
         AnyModel::Settings(m) => db.upsert_settings(&m, source)?.id,
@@ -30,6 +32,7 @@ pub(crate) fn upsert<R: Runtime>(window: WebviewWindow<R>, model: AnyModel) -> R
 
 #[tauri::command]
 pub(crate) fn delete<R: Runtime>(window: WebviewWindow<R>, model: AnyModel) -> Result<String> {
+    let blobs = window.blob_manager();
     // Use transaction for deletions because it might recurse
     window.with_tx(|tx| {
         let source = &UpdateSource::from_window(&window);
@@ -40,7 +43,7 @@ pub(crate) fn delete<R: Runtime>(window: WebviewWindow<R>, model: AnyModel) -> R
             AnyModel::GrpcConnection(m) => tx.delete_grpc_connection(&m, source)?.id,
             AnyModel::GrpcRequest(m) => tx.delete_grpc_request(&m, source)?.id,
             AnyModel::HttpRequest(m) => tx.delete_http_request(&m, source)?.id,
-            AnyModel::HttpResponse(m) => tx.delete_http_response(&m, source)?.id,
+            AnyModel::HttpResponse(m) => tx.delete_http_response(&m, source, &blobs)?.id,
             AnyModel::Plugin(m) => tx.delete_plugin(&m, source)?.id,
             AnyModel::WebsocketConnection(m) => tx.delete_websocket_connection(&m, source)?.id,
             AnyModel::WebsocketRequest(m) => tx.delete_websocket_request(&m, source)?.id,

src-tauri/yaak-models/src/db_context.rs

@@ -67,7 +67,7 @@ impl<'a> DbContext<'a> {
             .expect("Failed to run find on DB")
     }
 
-    pub fn find_all<'s, M>(&self) -> Result<Vec<M>>
+    pub(crate) fn find_all<'s, M>(&self) -> Result<Vec<M>>
     where
         M: Into<AnyModel> + Clone + UpsertModelInfo,
     {
@@ -82,7 +82,7 @@ impl<'a> DbContext<'a> {
         Ok(items.map(|v| v.unwrap()).collect())
     }
 
-    pub fn find_many<'s, M>(
+    pub(crate) fn find_many<'s, M>(
         &self,
         col: impl IntoColumnRef,
         value: impl Into<SimpleExpr>,
@@ -115,7 +115,7 @@ impl<'a> DbContext<'a> {
         Ok(items.map(|v| v.unwrap()).collect())
     }
 
-    pub fn upsert<M>(&self, model: &M, source: &UpdateSource) -> Result<M>
+    pub(crate) fn upsert<M>(&self, model: &M, source: &UpdateSource) -> Result<M>
     where
         M: Into<AnyModel> + From<AnyModel> + UpsertModelInfo + Clone,
     {

src-tauri/yaak-models/src/lib.rs

@@ -1,3 +1,4 @@
+use crate::blob_manager::{BlobManager, migrate_blob_db};
 use crate::commands::*;
 use crate::migrate::migrate_db;
 use crate::query_manager::QueryManager;
@@ -14,6 +15,7 @@ use tauri_plugin_dialog::{DialogExt, MessageDialogKind};
 
 mod commands;
 
+pub mod blob_manager;
 mod connection_or_tx;
 pub mod db_context;
 pub mod error;
@@ -50,7 +52,9 @@ pub fn init<R: Runtime>() -> TauriPlugin<R> {
             create_dir_all(app_path.clone()).expect("Problem creating App directory!");
 
             let db_file_path = app_path.join("db.sqlite");
+            let blob_db_file_path = app_path.join("blobs.sqlite");
 
+            // Main database pool
             let manager = SqliteConnectionManager::file(db_file_path);
             let pool = Pool::builder()
                 .max_size(100) // Up from 10 (just in case)
@@ -68,7 +72,26 @@ pub fn init<R: Runtime>() -> TauriPlugin<R> {
                 return Err(Box::from(e.to_string()));
             };
 
+            // Blob database pool
+            let blob_manager = SqliteConnectionManager::file(blob_db_file_path);
+            let blob_pool = Pool::builder()
+                .max_size(50)
+                .connection_timeout(Duration::from_secs(10))
+                .build(blob_manager)
+                .unwrap();
+
+            if let Err(e) = migrate_blob_db(&blob_pool) {
+                error!("Failed to run blob database migration {e:?}");
+                app_handle
+                    .dialog()
+                    .message(e.to_string())
+                    .kind(MessageDialogKind::Error)
+                    .blocking_show();
+                return Err(Box::from(e.to_string()));
+            };
+
             app_handle.manage(SqliteConnection::new(pool.clone()));
+            app_handle.manage(BlobManager::new(blob_pool));
 
             {
                 let (tx, rx) = mpsc::channel();

src-tauri/yaak-models/src/models.rs

@@ -1322,13 +1322,14 @@ pub struct HttpResponse {
     pub request_id: String,
 
     pub body_path: Option<String>,
-    pub content_length: Option<i32>,
-    pub content_length_compressed: Option<i32>,
+    pub content_length: Option<i64>,
+    pub content_length_compressed: Option<i64>,
     pub elapsed: i32,
     pub elapsed_headers: i32,
     pub error: Option<String>,
     pub headers: Vec<HttpResponseHeader>,
     pub remote_addr: Option<String>,
+    pub request_content_length: Option<i64>,
     pub request_headers: Vec<HttpResponseHeader>,
     pub status: i32,
     pub status_reason: Option<String>,
@@ -1382,6 +1383,7 @@ impl UpsertModelInfo for HttpResponse {
             (StatusReason, self.status_reason.into()),
             (Url, self.url.into()),
             (Version, self.version.into()),
+            (RequestContentLength, self.request_content_length.into()),
         ])
     }
 
@@ -1396,6 +1398,7 @@ impl UpsertModelInfo for HttpResponse {
             HttpResponseIden::Error,
             HttpResponseIden::Headers,
             HttpResponseIden::RemoteAddr,
+            HttpResponseIden::RequestContentLength,
             HttpResponseIden::RequestHeaders,
             HttpResponseIden::State,
             HttpResponseIden::Status,
@@ -1431,6 +1434,7 @@ impl UpsertModelInfo for HttpResponse {
             state: serde_json::from_str(format!(r#""{state}""#).as_str()).unwrap(),
             body_path: r.get("body_path")?,
             headers: serde_json::from_str(headers.as_str()).unwrap_or_default(),
+            request_content_length: r.get("request_content_length").unwrap_or_default(),
             request_headers: serde_json::from_str(
                 r.get::<_, String>("request_headers").unwrap_or_default().as_str(),
             )

src-tauri/yaak-models/src/queries/http_response_events.rs

@@ -0,0 +1,18 @@
+use crate::db_context::DbContext;
+use crate::error::Result;
+use crate::models::{HttpResponseEvent, HttpResponseEventIden};
+use crate::util::UpdateSource;
+
+impl<'a> DbContext<'a> {
+    pub fn list_http_response_events(&self, response_id: &str) -> Result<Vec<HttpResponseEvent>> {
+        self.find_many(HttpResponseEventIden::ResponseId, response_id, None)
+    }
+
+    pub fn upsert_http_response_event(
+        &self,
+        http_response_event: &HttpResponseEvent,
+        source: &UpdateSource,
+    ) -> Result<HttpResponseEvent> {
+        self.upsert(http_response_event, source)
+    }
+}

src-tauri/yaak-models/src/queries/http_responses.rs

@@ -1,3 +1,4 @@
+use crate::blob_manager::BlobManager;
 use crate::db_context::DbContext;
 use crate::error::Result;
 use crate::models::{HttpResponse, HttpResponseIden, HttpResponseState};
@@ -58,6 +59,7 @@ impl<'a> DbContext<'a> {
         &self,
         http_response: &HttpResponse,
         source: &UpdateSource,
+        blob_manager: &BlobManager,
     ) -> Result<HttpResponse> {
         // Delete the body file if it exists
         if let Some(p) = http_response.body_path.clone() {
@@ -66,6 +68,13 @@ impl<'a> DbContext<'a> {
             };
         }
 
+        // Delete request body blobs (pattern: {response_id}.request)
+        let blob_ctx = blob_manager.connect();
+        let body_id = format!("{}.request", http_response.id);
+        if let Err(e) = blob_ctx.delete_chunks(&body_id) {
+            error!("Failed to delete request body blobs: {}", e);
+        }
+
         Ok(self.delete(http_response, source)?)
     }
 
@@ -73,12 +82,13 @@ impl<'a> DbContext<'a> {
         &self,
         http_response: &HttpResponse,
         source: &UpdateSource,
+        blob_manager: &BlobManager,
     ) -> Result<HttpResponse> {
         let responses = self.list_http_responses_for_request(&http_response.request_id, None)?;
 
         for m in responses.iter().skip(MAX_HISTORY_ITEMS - 1) {
             debug!("Deleting old HTTP response {}", http_response.id);
-            self.delete_http_response(&m, source)?;
+            self.delete_http_response(&m, source, blob_manager)?;
         }
 
         self.upsert(http_response, source)

src-tauri/yaak-models/src/queries/mod.rs

@@ -8,6 +8,7 @@ mod grpc_connections;
 mod grpc_events;
 mod grpc_requests;
 mod http_requests;
+mod http_response_events;
 mod http_responses;
 mod key_values;
 mod plugin_key_values;