Add the browser send proxy and web sender (#572)

crates/yaak-web/Cargo.toml

@@ -27,6 +27,8 @@ serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
 yaak-lifecycle = { workspace = true }
 yaak-models = { workspace = true }
+# No default features: the template exports belong to @yaakapp-internal/templates, not this module
+yaak-templates = { path = "../yaak-templates", default-features = false }
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 console_error_panic_hook = "0.1"

crates/yaak-web/index.ts

@@ -3,4 +3,4 @@
 // This is loaded by the SharedWorker in packages/platform/src/web/worker.ts and
 // nowhere else: it owns a SQLite database, and there must be exactly one of it
 // per origin.
-export { blob_delete, blob_get, blob_put, boot, rpc } from "./pkg";
+export { blob_delete, blob_get, blob_put, boot, prepare_http_send, rpc } from "./pkg";

crates/yaak-web/pkg/yaak_web.d.ts

@@ -25,6 +25,17 @@ export function blob_put(id: string, bytes: Uint8Array): void;
  */
 export function boot(): Promise<void>;
 
+/**
+ * Resolve and render a request for sending, exactly as the desktop does before it puts the
+ * request on the network: the environment chain, inherited headers and auth, request
+ * settings, the cookie jar. Nothing here touches a socket. What comes back is what the tab
+ * posts to the send proxy.
+ *
+ * Refuses, with a message the user can act on, when the request needs something this host
+ * doesn't have: an authentication plugin, or a template function.
+ */
+export function prepare_http_send(payload: any): Promise<any>;
+
 /**
  * Run one command as `label` (the calling tab's identity, which stands in for
  * the desktop's window label on every write it makes).

crates/yaak-web/pkg/yaak_web_bg.js

@@ -62,6 +62,22 @@ export function boot() {
     return ret;
 }
 
+/**
+ * Resolve and render a request for sending, exactly as the desktop does before it puts the
+ * request on the network: the environment chain, inherited headers and auth, request
+ * settings, the cookie jar. Nothing here touches a socket. What comes back is what the tab
+ * posts to the send proxy.
+ *
+ * Refuses, with a message the user can act on, when the request needs something this host
+ * doesn't have: an authentication plugin, or a template function.
+ * @param {any} payload
+ * @returns {Promise<any>}
+ */
+export function prepare_http_send(payload) {
+    const ret = wasm.prepare_http_send(payload);
+    return ret;
+}
+
 /**
  * Run one command as `label` (the calling tab's identity, which stands in for
  * the desktop's window label on every write it makes).
@@ -496,7 +512,7 @@ export function __wbg_new_typed_c072c4ce9a2a0cdf(arg0, arg1) {
             const a = state0.a;
             state0.a = 0;
             try {
-                return wasm_bindgen__convert__closures_____invoke__h2c72ca09e851b7f3(a, state0.b, arg0, arg1);
+                return wasm_bindgen__convert__closures_____invoke__h2cf3f4cce3b29948(a, state0.b, arg0, arg1);
             } finally {
                 state0.a = a;
             }
@@ -681,23 +697,23 @@ export function __wbg_warn_b6f36cac66fc96a4(arg0, arg1) {
     console.warn(arg0, arg1);
 }
 export function __wbindgen_cast_0000000000000001(arg0, arg1) {
-    // Cast intrinsic for `Closure(Closure { owned: true, function: Function { arguments: [Externref], shim_idx: 1103, ret: Result(Unit), inner_ret: Some(Result(Unit)) }, mutable: true }) -> Externref`.
-    const ret = makeMutClosure(arg0, arg1, wasm_bindgen__convert__closures_____invoke__hf84d53817e0238b4);
+    // Cast intrinsic for `Closure(Closure { owned: true, function: Function { arguments: [Externref], shim_idx: 1117, ret: Result(Unit), inner_ret: Some(Result(Unit)) }, mutable: true }) -> Externref`.
+    const ret = makeMutClosure(arg0, arg1, wasm_bindgen__convert__closures_____invoke__ha1c2fa93df0107f3);
     return ret;
 }
 export function __wbindgen_cast_0000000000000002(arg0, arg1) {
-    // Cast intrinsic for `Closure(Closure { owned: true, function: Function { arguments: [NamedExternref("Event")], shim_idx: 200, ret: Unit, inner_ret: Some(Unit) }, mutable: true }) -> Externref`.
-    const ret = makeMutClosure(arg0, arg1, wasm_bindgen__convert__closures_____invoke__h7e06bb925b918fbb);
+    // Cast intrinsic for `Closure(Closure { owned: true, function: Function { arguments: [NamedExternref("Event")], shim_idx: 212, ret: Unit, inner_ret: Some(Unit) }, mutable: true }) -> Externref`.
+    const ret = makeMutClosure(arg0, arg1, wasm_bindgen__convert__closures_____invoke__ha7903b6e296dd8f4);
     return ret;
 }
 export function __wbindgen_cast_0000000000000003(arg0, arg1) {
-    // Cast intrinsic for `Closure(Closure { owned: true, function: Function { arguments: [NamedExternref("IDBVersionChangeEvent")], shim_idx: 177, ret: Result(Unit), inner_ret: Some(Result(Unit)) }, mutable: true }) -> Externref`.
-    const ret = makeMutClosure(arg0, arg1, wasm_bindgen__convert__closures_____invoke__h400c17219073e521);
+    // Cast intrinsic for `Closure(Closure { owned: true, function: Function { arguments: [NamedExternref("IDBVersionChangeEvent")], shim_idx: 83, ret: Result(Unit), inner_ret: Some(Result(Unit)) }, mutable: true }) -> Externref`.
+    const ret = makeMutClosure(arg0, arg1, wasm_bindgen__convert__closures_____invoke__h38d884a456ef1afe);
     return ret;
 }
 export function __wbindgen_cast_0000000000000004(arg0, arg1) {
-    // Cast intrinsic for `Closure(Closure { owned: true, function: Function { arguments: [], shim_idx: 198, ret: Unit, inner_ret: Some(Unit) }, mutable: true }) -> Externref`.
-    const ret = makeMutClosure(arg0, arg1, wasm_bindgen__convert__closures_____invoke__h87640adb2bbfa2fc);
+    // Cast intrinsic for `Closure(Closure { owned: true, function: Function { arguments: [], shim_idx: 210, ret: Unit, inner_ret: Some(Unit) }, mutable: true }) -> Externref`.
+    const ret = makeMutClosure(arg0, arg1, wasm_bindgen__convert__closures_____invoke__ha1b480b83daa641f);
     return ret;
 }
 export function __wbindgen_cast_0000000000000005(arg0) {
@@ -734,30 +750,30 @@ export function __wbindgen_init_externref_table() {
     table.set(offset + 2, true);
     table.set(offset + 3, false);
 }
-function wasm_bindgen__convert__closures_____invoke__h87640adb2bbfa2fc(arg0, arg1) {
-    wasm.wasm_bindgen__convert__closures_____invoke__h87640adb2bbfa2fc(arg0, arg1);
+function wasm_bindgen__convert__closures_____invoke__ha1b480b83daa641f(arg0, arg1) {
+    wasm.wasm_bindgen__convert__closures_____invoke__ha1b480b83daa641f(arg0, arg1);
 }
 
-function wasm_bindgen__convert__closures_____invoke__h7e06bb925b918fbb(arg0, arg1, arg2) {
-    wasm.wasm_bindgen__convert__closures_____invoke__h7e06bb925b918fbb(arg0, arg1, arg2);
+function wasm_bindgen__convert__closures_____invoke__ha7903b6e296dd8f4(arg0, arg1, arg2) {
+    wasm.wasm_bindgen__convert__closures_____invoke__ha7903b6e296dd8f4(arg0, arg1, arg2);
 }
 
-function wasm_bindgen__convert__closures_____invoke__hf84d53817e0238b4(arg0, arg1, arg2) {
-    const ret = wasm.wasm_bindgen__convert__closures_____invoke__hf84d53817e0238b4(arg0, arg1, arg2);
+function wasm_bindgen__convert__closures_____invoke__ha1c2fa93df0107f3(arg0, arg1, arg2) {
+    const ret = wasm.wasm_bindgen__convert__closures_____invoke__ha1c2fa93df0107f3(arg0, arg1, arg2);
     if (ret[1]) {
         throw takeFromExternrefTable0(ret[0]);
     }
 }
 
-function wasm_bindgen__convert__closures_____invoke__h400c17219073e521(arg0, arg1, arg2) {
-    const ret = wasm.wasm_bindgen__convert__closures_____invoke__h400c17219073e521(arg0, arg1, arg2);
+function wasm_bindgen__convert__closures_____invoke__h38d884a456ef1afe(arg0, arg1, arg2) {
+    const ret = wasm.wasm_bindgen__convert__closures_____invoke__h38d884a456ef1afe(arg0, arg1, arg2);
     if (ret[1]) {
         throw takeFromExternrefTable0(ret[0]);
     }
 }
 
-function wasm_bindgen__convert__closures_____invoke__h2c72ca09e851b7f3(arg0, arg1, arg2, arg3) {
-    wasm.wasm_bindgen__convert__closures_____invoke__h2c72ca09e851b7f3(arg0, arg1, arg2, arg3);
+function wasm_bindgen__convert__closures_____invoke__h2cf3f4cce3b29948(arg0, arg1, arg2, arg3) {
+    wasm.wasm_bindgen__convert__closures_____invoke__h2cf3f4cce3b29948(arg0, arg1, arg2, arg3);
 }
 
 

crates/yaak-web/pkg/yaak_web_bg.wasm.d.ts

@@ -5,6 +5,7 @@ export const blob_delete: (a: number, b: number) => [number, number];
 export const blob_get: (a: number, b: number) => [number, number, number, number];
 export const blob_put: (a: number, b: number, c: number, d: number) => [number, number];
 export const boot: () => any;
+export const prepare_http_send: (a: any) => any;
 export const rpc: (a: number, b: number, c: any, d: number, e: number) => [number, number, number];
 export const rust_sqlite_wasm_abort: () => void;
 export const rust_sqlite_wasm_assert_fail: (a: number, b: number, c: number, d: number) => void;
@@ -16,11 +17,11 @@ export const rust_sqlite_wasm_malloc: (a: number) => number;
 export const rust_sqlite_wasm_realloc: (a: number, b: number) => number;
 export const sqlite3_os_end: () => number;
 export const sqlite3_os_init: () => number;
-export const wasm_bindgen__convert__closures_____invoke__hf84d53817e0238b4: (a: number, b: number, c: any) => [number, number];
-export const wasm_bindgen__convert__closures_____invoke__h400c17219073e521: (a: number, b: number, c: any) => [number, number];
-export const wasm_bindgen__convert__closures_____invoke__h2c72ca09e851b7f3: (a: number, b: number, c: any, d: any) => void;
-export const wasm_bindgen__convert__closures_____invoke__h7e06bb925b918fbb: (a: number, b: number, c: any) => void;
-export const wasm_bindgen__convert__closures_____invoke__h87640adb2bbfa2fc: (a: number, b: number) => void;
+export const wasm_bindgen__convert__closures_____invoke__ha1c2fa93df0107f3: (a: number, b: number, c: any) => [number, number];
+export const wasm_bindgen__convert__closures_____invoke__h38d884a456ef1afe: (a: number, b: number, c: any) => [number, number];
+export const wasm_bindgen__convert__closures_____invoke__h2cf3f4cce3b29948: (a: number, b: number, c: any, d: any) => void;
+export const wasm_bindgen__convert__closures_____invoke__ha7903b6e296dd8f4: (a: number, b: number, c: any) => void;
+export const wasm_bindgen__convert__closures_____invoke__ha1b480b83daa641f: (a: number, b: number) => void;
 export const __wbindgen_malloc: (a: number, b: number) => number;
 export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
 export const __wbindgen_exn_store: (a: number) => void;

crates/yaak-web/src/lib.rs

@@ -12,8 +12,10 @@
 //! JavaScript side owns that; this crate assumes it is the only writer.
 //!
 //! The command surface is deliberately narrow: what the frontend needs to keep
-//! its model store coherent, and blob storage. Sending, plugins, git, sync and
-//! everything else with a socket or a filesystem behind it lives elsewhere.
+//! its model store coherent, blob storage, and the "prepare" half of a send
+//! (resolve, inherit, render — see [`prepare_http_send`]). Putting bytes on the
+//! network, plugins, git, sync and everything else with a socket or a
+//! filesystem behind it lives elsewhere.
 
 // Nothing in here means anything off wasm32, and building it there would drag
 // SQLite's wasm C shim into a native compile. So on any other target the crate
@@ -24,12 +26,19 @@ use std::cell::RefCell;
 use std::sync::mpsc;
 
 use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
 use wasm_bindgen::prelude::*;
 use yaak_models::blob_manager::{BlobManager, BodyChunk};
-use yaak_models::models::AnyModel;
+use yaak_models::cookies::apply_cookie_changes;
+use yaak_models::models::{
+    AnyModel, Cookie, CookieJar, HttpRequest, HttpResponseEvent, HttpResponseEventData,
+    HttpSendSettings,
+};
 use yaak_models::models_ops;
 use yaak_models::query_manager::QueryManager;
+use yaak_models::render::render_http_request;
 use yaak_models::util::{ModelPayload, UpdateSource};
+use yaak_templates::{RenderOptions, TemplateCallback};
 
 /// Names inside the VFS, not paths on any disk. Two files because the desktop
 /// keeps two: models in one, blobs in the other.
@@ -209,6 +218,28 @@ struct UpsertIntrospectionReq {
     content: Option<String>,
 }
 
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct ResponseIdReq {
+    response_id: String,
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct PersistSendCookiesReq {
+    cookie_jar_id: String,
+    before: Vec<Cookie>,
+    after: Vec<Cookie>,
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct InsertResponseEventsReq {
+    response_id: String,
+    workspace_id: String,
+    events: Vec<HttpResponseEventData>,
+}
+
 fn dispatch(
     host: &Host,
     cmd: &str,
@@ -313,6 +344,48 @@ fn dispatch(
         // Nothing here can open a socket, so no connection ever produced any.
         "models_grpc_events" | "models_websocket_events" => to_json(Vec::<()>::new()),
 
+        "web_get_http_request" => {
+            let req: RequestIdReq = from_js(payload)?;
+            to_json(host.queries.connect().get_http_request(&req.request_id).map_err(js_error)?)
+        }
+
+        "cmd_get_http_response_events" => {
+            let req: ResponseIdReq = from_js(payload)?;
+            to_json(
+                host.queries
+                    .connect()
+                    .list_http_response_events(&req.response_id)
+                    .map_err(js_error)?,
+            )
+        }
+
+        // The cookies a send set or cleared, applied to the jar as it is *now* rather than
+        // written over it, so an edit made while the send was in flight survives.
+        "web_persist_send_cookies" => {
+            let req: PersistSendCookiesReq = from_js(payload)?;
+            if req.before == req.after {
+                return to_json(());
+            }
+            let db = host.queries.connect();
+            let jar = db.get_cookie_jar(&req.cookie_jar_id).map_err(js_error)?;
+            let cookies = apply_cookie_changes(jar.cookies.clone(), &req.before, &req.after);
+            db.upsert_cookie_jar(&CookieJar { cookies, ..jar }, source).map_err(js_error)?;
+            to_json(())
+        }
+
+        // The tab's half of the send timeline: the events the proxy streamed back, recorded
+        // under the response they belong to. Same rows the desktop's send task writes, and the
+        // writes fan out to every tab as `model_writes` like any other.
+        "web_insert_http_response_events" => {
+            let req: InsertResponseEventsReq = from_js(payload)?;
+            let db = host.queries.connect();
+            for event in req.events {
+                let model = HttpResponseEvent::new(&req.response_id, &req.workspace_id, event);
+                db.upsert_http_response_event(&model, source).map_err(js_error)?;
+            }
+            to_json(())
+        }
+
         "cmd_get_workspace_meta" => {
             let req: WorkspaceIdReq = from_js(payload)?;
             let db = host.queries.connect();
@@ -346,6 +419,131 @@ fn dispatch(
     }
 }
 
+/* -------------------------------------------------------------------------- */
+/* Preparing a send                                                            */
+/* -------------------------------------------------------------------------- */
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct PrepareHttpSendReq {
+    request_id: String,
+    environment_id: Option<String>,
+    cookie_jar_id: Option<String>,
+}
+
+/// Everything a send needs that lives in the database, resolved and rendered: the desktop's
+/// `HttpSendInputs`, in the shape a tab hands to the proxy and keeps for itself.
+#[derive(Serialize)]
+#[serde(rename_all = "camelCase")]
+struct PreparedHttpSend {
+    /// The request with inherited headers and authentication applied and every template
+    /// rendered. What the proxy sends, and what the response records as its request.
+    request: HttpRequest,
+    settings: HttpSendSettings,
+    /// The `* Setting name=value` timeline lines the desktop writes at the top of a send,
+    /// sources and all. The tab records them before the proxy's own events.
+    setting_events: Vec<HttpResponseEventData>,
+    /// The jar the send starts with, so the tab can write it back with the proxy's changes.
+    cookie_jar: Option<CookieJar>,
+}
+
+/// A template callback for a host with no plugins. Variables render; a function is a clear
+/// refusal naming the function, so the user knows what the request needs rather than seeing
+/// an empty string sent in its place.
+struct NoPluginsCallback;
+
+impl TemplateCallback for NoPluginsCallback {
+    fn run(
+        &self,
+        fn_name: &str,
+        _args: HashMap<String, serde_json::Value>,
+    ) -> impl std::future::Future<Output = yaak_templates::error::Result<String>> + Send {
+        let message = format!(
+            "This request uses the template function \"{fn_name}\", which needs plugins. \
+             Plugins aren't available in the browser yet"
+        );
+        async move { Err(yaak_templates::error::Error::RenderError(message)) }
+    }
+
+    fn transform_arg(
+        &self,
+        _fn_name: &str,
+        _arg_name: &str,
+        arg_value: &str,
+    ) -> yaak_templates::error::Result<String> {
+        Ok(arg_value.to_string())
+    }
+}
+
+/// Resolve and render a request for sending, exactly as the desktop does before it puts the
+/// request on the network: the environment chain, inherited headers and auth, request
+/// settings, the cookie jar. Nothing here touches a socket. What comes back is what the tab
+/// posts to the send proxy.
+///
+/// Refuses, with a message the user can act on, when the request needs something this host
+/// doesn't have: an authentication plugin, or a template function.
+#[wasm_bindgen]
+pub async fn prepare_http_send(payload: JsValue) -> Result<JsValue> {
+    let req: PrepareHttpSendReq = from_js(payload)?;
+
+    // Everything from the database first, then release the host borrow before rendering.
+    let (request, environment_chain, settings, cookie_jar) = with_host(|host| {
+        let db = host.queries.connect();
+        let request = db.get_http_request(&req.request_id).map_err(js_error)?;
+        let environment_chain = db
+            .resolve_environments(
+                &request.workspace_id,
+                request.folder_id.as_deref(),
+                req.environment_id.as_deref(),
+            )
+            .map_err(js_error)?;
+        let (authentication_type, authentication, _auth_context_id) =
+            db.resolve_auth_for_http_request(&request).map_err(js_error)?;
+        let headers = db.resolve_headers_for_http_request(&request).map_err(js_error)?;
+        let settings = db.resolve_settings_for_http_request(&request).map_err(js_error)?;
+        let cookie_jar = match req.cookie_jar_id.as_deref() {
+            Some(id) => Some(db.get_cookie_jar(id).map_err(js_error)?),
+            None => None,
+        };
+        let request = HttpRequest { authentication_type, authentication, headers, ..request };
+        Ok((request, environment_chain, settings, cookie_jar))
+    })?;
+
+    let rendered = render_http_request(
+        &request,
+        environment_chain,
+        &NoPluginsCallback,
+        &RenderOptions::throw(),
+    )
+    .await
+    .map_err(js_error)?;
+
+    // Authentication is applied by a plugin on the desktop. There is no plugin here, and a
+    // request sent without the auth it asked for is worse than one refused with the reason.
+    let auth_disabled =
+        rendered.authentication.get("disabled").and_then(|v| v.as_bool()) == Some(true);
+    if let Some(auth_type) = rendered.authentication_type.as_deref()
+        && auth_type != "none"
+        && !auth_disabled
+    {
+        return Err(js_error(format!(
+            "This request uses {auth_type} authentication, which needs plugins. \
+             Plugins aren't available in the browser yet"
+        )));
+    }
+
+    let prepared = PreparedHttpSend {
+        request: rendered,
+        settings: HttpSendSettings::from(&settings),
+        setting_events: settings.timeline_events(),
+        cookie_jar,
+    };
+    // JSON-compatible, as `rpc` does: the tab posts this to the proxy with `JSON.stringify`,
+    // and the default serializer's `Map` for the request body would stringify to `{}`.
+    use serde::Serialize as _;
+    prepared.serialize(&serde_wasm_bindgen::Serializer::json_compatible()).map_err(js_error)
+}
+
 /* -------------------------------------------------------------------------- */
 /* Blobs                                                                       */
 /* -------------------------------------------------------------------------- */