Add the browser send proxy and web sender (#572)

crates/yaak-models/src/cookies.rs

@@ -0,0 +1,107 @@
+//! Carrying a send's cookie changes back into a jar.
+//!
+//! A send starts from a snapshot of the jar and hands back the jar as the
+//! transaction left it. Writing that whole result over the jar would also
+//! write over anything the user changed *while* the send was in flight — a
+//! cookie edited or deleted in the jar view, or set by another send. So the
+//! send's contribution is taken as a difference (what it added, changed, or
+//! removed relative to its snapshot) and applied to whatever the jar holds now.
+
+use crate::models::{Cookie, CookieDomain};
+
+/// The identity of a cookie in a jar: two cookies with the same name, domain
+/// and path are the same cookie, whatever their value or attributes.
+type CookieKey = (String, CookieDomain, String);
+
+fn key(c: &Cookie) -> CookieKey {
+    (c.name.clone(), c.domain.clone(), c.path.clone())
+}
+
+/// Apply the changes between `before` (the snapshot a send started from) and
+/// `after` (the jar as the send left it) to `current` (the jar as it is now).
+///
+/// Cookies the send removed are removed; cookies it added or changed replace
+/// their counterpart in `current`, or are appended. Cookies the send did not
+/// touch are left exactly as `current` has them.
+pub fn apply_cookie_changes(
+    current: Vec<Cookie>,
+    before: &[Cookie],
+    after: &[Cookie],
+) -> Vec<Cookie> {
+    let removed: Vec<CookieKey> =
+        before.iter().filter(|b| !after.iter().any(|a| key(a) == key(b))).map(key).collect();
+    let changed: Vec<&Cookie> = after.iter().filter(|a| !before.iter().any(|b| b == *a)).collect();
+
+    let mut result: Vec<Cookie> =
+        current.into_iter().filter(|c| !removed.contains(&key(c))).collect();
+    for cookie in changed {
+        match result.iter_mut().find(|c| key(c) == key(cookie)) {
+            Some(existing) => *existing = cookie.clone(),
+            None => result.push(cookie.clone()),
+        }
+    }
+    result
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::models::CookieExpires;
+
+    fn cookie(name: &str, value: &str) -> Cookie {
+        Cookie {
+            name: name.to_string(),
+            value: value.to_string(),
+            domain: CookieDomain::HostOnly("example.com".to_string()),
+            expires: CookieExpires::SessionEnd,
+            path: "/".to_string(),
+            secure: false,
+            http_only: false,
+            same_site: None,
+        }
+    }
+
+    #[test]
+    fn a_send_that_changed_nothing_leaves_the_jar_alone() {
+        let before = vec![cookie("a", "1")];
+        let current = vec![cookie("a", "edited"), cookie("b", "2")];
+        assert_eq!(apply_cookie_changes(current.clone(), &before, &before), current);
+    }
+
+    #[test]
+    fn additions_and_changes_land_without_touching_concurrent_edits() {
+        let before = vec![cookie("a", "1"), cookie("b", "2")];
+        let after = vec![cookie("a", "1"), cookie("b", "3"), cookie("c", "4")];
+        // Meanwhile the user edited `a` and added `d`.
+        let current = vec![cookie("a", "edited"), cookie("b", "2"), cookie("d", "5")];
+        assert_eq!(
+            apply_cookie_changes(current, &before, &after),
+            vec![
+                cookie("a", "edited"),
+                cookie("b", "3"),
+                cookie("d", "5"),
+                cookie("c", "4")
+            ]
+        );
+    }
+
+    #[test]
+    fn a_cookie_the_send_removed_is_removed() {
+        let before = vec![cookie("a", "1"), cookie("b", "2")];
+        let after = vec![cookie("b", "2")];
+        let current = vec![cookie("a", "1"), cookie("b", "2"), cookie("c", "3")];
+        assert_eq!(
+            apply_cookie_changes(current, &before, &after),
+            vec![cookie("b", "2"), cookie("c", "3")]
+        );
+    }
+
+    #[test]
+    fn a_cookie_the_user_deleted_mid_send_stays_deleted_unless_the_send_set_it() {
+        let before = vec![cookie("a", "1")];
+        let after = vec![cookie("a", "1")]; // untouched by the send
+        assert_eq!(apply_cookie_changes(vec![], &before, &after), vec![]);
+        let after = vec![cookie("a", "fresh")]; // the send set it again
+        assert_eq!(apply_cookie_changes(vec![], &before, &after), vec![cookie("a", "fresh")]);
+    }
+}

crates/yaak-models/src/lib.rs

@@ -10,11 +10,13 @@ use yaak_database::SqlitePool;
 
 pub mod blob_manager;
 pub mod client_db;
+pub mod cookies;
 mod connection_or_tx;
 pub mod error;
 pub mod migrate;
 pub mod models;
 pub mod models_ops;
+pub mod path_placeholders;
 pub mod queries;
 pub mod query_manager;
 pub mod render;

crates/yaak-models/src/models.rs

@@ -158,6 +158,70 @@ impl Default for ResolvedHttpRequestSettings {
     }
 }
 
+impl ResolvedHttpRequestSettings {
+    /// The `* Setting name=value` lines a send writes at the top of its timeline, sources and
+    /// all. Built here, once, so every host that runs a send — the desktop, the CLI, the browser
+    /// tab handing off to a proxy — records the same lines the same way.
+    pub fn timeline_events(&self) -> Vec<HttpResponseEventData> {
+        fn event<T>(
+            name: &str,
+            value: String,
+            setting: &ResolvedSetting<T>,
+        ) -> HttpResponseEventData {
+            HttpResponseEventData::Setting {
+                name: name.to_string(),
+                value,
+                source_model: Some(setting.source_model.clone()),
+                source_id: setting.source_id.clone(),
+                source_name: setting.source_name.clone(),
+            }
+        }
+        let timeout = if self.request_timeout.value > 0 {
+            format!("{:?}", std::time::Duration::from_millis(self.request_timeout.value as u64))
+        } else {
+            "Infinity".to_string()
+        };
+        vec![
+            event(
+                "validate_certificates",
+                self.validate_certificates.value.to_string(),
+                &self.validate_certificates,
+            ),
+            event("redirects", self.follow_redirects.value.to_string(), &self.follow_redirects),
+            event("timeout", timeout, &self.request_timeout),
+            event("send_cookies", self.send_cookies.value.to_string(), &self.send_cookies),
+            event("store_cookies", self.store_cookies.value.to_string(), &self.store_cookies),
+        ]
+    }
+}
+
+/// The resolved send settings, values only: what an executor has to obey, with the sources
+/// (which model each came from) left behind in [`ResolvedHttpRequestSettings`]. This is what
+/// crosses from a tab to the send proxy, and what the proxy reads.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export, export_to = "gen_models.ts")]
+pub struct HttpSendSettings {
+    pub validate_certificates: bool,
+    pub follow_redirects: bool,
+    /// Milliseconds. Zero or negative means no timeout.
+    pub timeout_ms: i32,
+    pub send_cookies: bool,
+    pub store_cookies: bool,
+}
+
+impl From<&ResolvedHttpRequestSettings> for HttpSendSettings {
+    fn from(s: &ResolvedHttpRequestSettings) -> Self {
+        Self {
+            validate_certificates: s.validate_certificates.value,
+            follow_redirects: s.follow_redirects.value,
+            timeout_ms: s.request_timeout.value,
+            send_cookies: s.send_cookies.value,
+            store_cookies: s.store_cookies.value,
+        }
+    }
+}
+
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema, TS)]
 #[serde(default, rename_all = "camelCase")]
 #[ts(export, export_to = "gen_models.ts")]

crates/yaak-models/src/path_placeholders.rs

@@ -0,0 +1,220 @@
+use crate::models::HttpUrlParameter;
+
+pub fn apply_path_placeholders(
+    url: &str,
+    parameters: &Vec<HttpUrlParameter>,
+) -> (String, Vec<HttpUrlParameter>) {
+    let mut new_parameters = Vec::new();
+
+    let mut url = url.to_string();
+    for p in parameters {
+        if !p.enabled || p.name.is_empty() {
+            continue;
+        }
+
+        // Replace path parameters with values from URL parameters
+        let old_url_string = url.clone();
+        url = replace_path_placeholder(&p, url.as_str());
+
+        // Remove as param if it modified the URL
+        if old_url_string == *url {
+            new_parameters.push(p.to_owned());
+        }
+    }
+
+    (url, new_parameters)
+}
+
+fn replace_path_placeholder(p: &HttpUrlParameter, url: &str) -> String {
+    if !p.enabled {
+        return url.to_string();
+    }
+
+    if !p.name.starts_with(":") {
+        return url.to_string();
+    }
+
+    // A placeholder is `/` followed by the parameter's name (which starts with `:`), and it
+    // ends at `/`, `?`, `#`, a literal `:`, or the end of the URL. The `:` boundary is what
+    // lets `/:id:increment-importance` substitute the `:id` placeholder while leaving
+    // `:increment-importance` as literal text. `/:foooo` is not a match for `:foo`.
+    //
+    // A plain scan rather than a regex: the name is matched literally, so a name containing
+    // `.` or `+` means exactly that, and nothing else in the model layer needs a regex engine.
+    let name = p.name.as_str();
+    let value = urlencoding::encode(p.value.as_str());
+    let mut result = String::with_capacity(url.len());
+    let mut rest = url;
+    while let Some(slash) = rest.find('/') {
+        let after_slash = &rest[slash + 1..];
+        let is_placeholder = after_slash.starts_with(name)
+            && after_slash[name.len()..]
+                .chars()
+                .next()
+                .is_none_or(|c| matches!(c, '/' | '?' | '#' | ':'));
+        if is_placeholder {
+            result.push_str(&rest[..=slash]);
+            result.push_str(&value);
+            rest = &after_slash[name.len()..];
+        } else {
+            result.push_str(&rest[..=slash]);
+            rest = after_slash;
+        }
+    }
+    result.push_str(rest);
+    result
+}
+
+#[cfg(test)]
+mod placeholder_tests {
+    use crate::models::{HttpRequest, HttpUrlParameter};
+    use crate::path_placeholders::{apply_path_placeholders, replace_path_placeholder};
+
+    #[test]
+    fn placeholder_middle() {
+        let p =
+            HttpUrlParameter { name: ":foo".into(), value: "xxx".into(), enabled: true, id: None };
+        assert_eq!(
+            replace_path_placeholder(&p, "https://example.com/:foo/bar"),
+            "https://example.com/xxx/bar",
+        );
+    }
+
+    #[test]
+    fn placeholder_end() {
+        let p =
+            HttpUrlParameter { name: ":foo".into(), value: "xxx".into(), enabled: true, id: None };
+        assert_eq!(
+            replace_path_placeholder(&p, "https://example.com/:foo"),
+            "https://example.com/xxx",
+        );
+    }
+
+    #[test]
+    fn placeholder_query() {
+        let p =
+            HttpUrlParameter { name: ":foo".into(), value: "xxx".into(), enabled: true, id: None };
+        assert_eq!(
+            replace_path_placeholder(&p, "https://example.com/:foo?:foo"),
+            "https://example.com/xxx?:foo",
+        );
+    }
+
+    #[test]
+    fn placeholder_followed_by_literal_colon() {
+        // AIP-136-style custom method: `:id` is the placeholder, `:increment-importance`
+        // is literal text in the same path segment.
+        let p =
+            HttpUrlParameter { name: ":id".into(), value: "42".into(), enabled: true, id: None };
+        assert_eq!(
+            replace_path_placeholder(&p, "https://example.com/tasks/:id:increment-importance"),
+            "https://example.com/tasks/42:increment-importance",
+        );
+    }
+
+    #[test]
+    fn placeholder_name_is_matched_literally() {
+        // `.` in a name is a dot, not "any character".
+        let p = HttpUrlParameter {
+            name: ":id.v2".into(),
+            value: "xxx".into(),
+            enabled: true,
+            id: None,
+        };
+        assert_eq!(
+            replace_path_placeholder(&p, "https://example.com/:id.v2/:idXv2"),
+            "https://example.com/xxx/:idXv2",
+        );
+    }
+
+    #[test]
+    fn placeholder_repeated() {
+        let p = HttpUrlParameter { name: ":id".into(), value: "7".into(), enabled: true, id: None };
+        assert_eq!(
+            replace_path_placeholder(&p, "https://example.com/:id/:id"),
+            "https://example.com/7/7",
+        );
+    }
+
+    #[test]
+    fn placeholder_missing() {
+        let p = HttpUrlParameter {
+            enabled: true,
+            name: "".to_string(),
+            value: "".to_string(),
+            id: None,
+        };
+        assert_eq!(
+            replace_path_placeholder(&p, "https://example.com/:missing"),
+            "https://example.com/:missing",
+        );
+    }
+
+    #[test]
+    fn placeholder_disabled() {
+        let p = HttpUrlParameter {
+            enabled: false,
+            name: ":foo".to_string(),
+            value: "xxx".to_string(),
+            id: None,
+        };
+        assert_eq!(
+            replace_path_placeholder(&p, "https://example.com/:foo"),
+            "https://example.com/:foo",
+        );
+    }
+
+    #[test]
+    fn placeholder_prefix() {
+        let p =
+            HttpUrlParameter { name: ":foo".into(), value: "xxx".into(), enabled: true, id: None };
+        assert_eq!(
+            replace_path_placeholder(&p, "https://example.com/:foooo"),
+            "https://example.com/:foooo",
+        );
+    }
+
+    #[test]
+    fn placeholder_encode() {
+        let p = HttpUrlParameter {
+            name: ":foo".into(),
+            value: "Hello World".into(),
+            enabled: true,
+            id: None,
+        };
+        assert_eq!(
+            replace_path_placeholder(&p, "https://example.com/:foo"),
+            "https://example.com/Hello%20World",
+        );
+    }
+
+    #[test]
+    fn apply_placeholder() {
+        let req = HttpRequest {
+            url: "example.com/:a/bar".to_string(),
+            url_parameters: vec![
+                HttpUrlParameter {
+                    name: "b".to_string(),
+                    value: "bbb".to_string(),
+                    enabled: true,
+                    id: None,
+                },
+                HttpUrlParameter {
+                    name: ":a".to_string(),
+                    value: "aaa".to_string(),
+                    enabled: true,
+                    id: None,
+                },
+            ],
+            ..Default::default()
+        };
+
+        let (url, url_parameters) = apply_path_placeholders(&req.url, &req.url_parameters);
+
+        // Pattern match back to access it
+        assert_eq!(url, "example.com/aaa/bar");
+        assert_eq!(url_parameters.len(), 1);
+        assert_eq!(url_parameters[0].name, "b");
+        assert_eq!(url_parameters[0].value, "bbb");
+    }
+}

crates/yaak-models/src/render.rs

@@ -1,5 +1,159 @@
-use crate::models::{Environment, EnvironmentVariable};
-use std::collections::HashMap;
+//! Rendering requests against an environment chain.
+//!
+//! Lives here rather than beside the send engine so that the browser's wasm
+//! host, which has the model layer but no sockets, renders exactly what the
+//! desktop renders.
+
+use crate::models::{
+    Environment, EnvironmentVariable, GrpcRequest, HttpRequest, HttpRequestHeader, HttpUrlParameter,
+};
+use crate::path_placeholders::apply_path_placeholders;
+use log::info;
+use serde_json::Value;
+use std::collections::{BTreeMap, HashMap};
+use yaak_templates::{RenderOptions, TemplateCallback, parse_and_render, render_json_value_raw};
+
+/// Render every template in an HTTP request against an environment chain.
+pub async fn render_http_request<T: TemplateCallback>(
+    request: &HttpRequest,
+    environment_chain: Vec<Environment>,
+    callback: &T,
+    options: &RenderOptions,
+) -> yaak_templates::error::Result<HttpRequest> {
+    let vars = &make_vars_hashmap(environment_chain);
+
+    let mut url_parameters = Vec::new();
+    for parameter in request.url_parameters.clone() {
+        if !parameter.enabled {
+            continue;
+        }
+
+        url_parameters.push(HttpUrlParameter {
+            enabled: parameter.enabled,
+            name: parse_and_render(parameter.name.as_str(), vars, callback, options).await?,
+            value: parse_and_render(parameter.value.as_str(), vars, callback, options).await?,
+            id: parameter.id,
+        })
+    }
+
+    let mut headers = Vec::new();
+    for header in request.headers.clone() {
+        if !header.enabled {
+            continue;
+        }
+
+        headers.push(HttpRequestHeader {
+            enabled: header.enabled,
+            name: parse_and_render(header.name.as_str(), vars, callback, options).await?,
+            value: parse_and_render(header.value.as_str(), vars, callback, options).await?,
+            id: header.id,
+        })
+    }
+
+    let mut body = BTreeMap::new();
+    for (key, value) in request.body.clone() {
+        let value = if key == "form" { strip_disabled_form_entries(value) } else { value };
+        body.insert(key, render_json_value_raw(value, vars, callback, options).await?);
+    }
+
+    let authentication = {
+        let mut disabled = false;
+        let mut auth = BTreeMap::new();
+
+        match request.authentication.get("disabled") {
+            Some(Value::Bool(true)) => {
+                disabled = true;
+            }
+            Some(Value::String(template)) => {
+                disabled = parse_and_render(template.as_str(), vars, callback, options)
+                    .await
+                    .unwrap_or_default()
+                    .is_empty();
+                info!(
+                    "Rendering authentication.disabled as a template: {disabled} from \"{template}\""
+                );
+            }
+            _ => {}
+        }
+
+        if disabled {
+            auth.insert("disabled".to_string(), Value::Bool(true));
+        } else {
+            for (key, value) in request.authentication.clone() {
+                if key == "disabled" {
+                    auth.insert(key, Value::Bool(false));
+                } else {
+                    auth.insert(key, render_json_value_raw(value, vars, callback, options).await?);
+                }
+            }
+        }
+
+        auth
+    };
+
+    let url = parse_and_render(request.url.clone().as_str(), vars, callback, options).await?;
+    let (url, url_parameters) = apply_path_placeholders(&url, &url_parameters);
+
+    Ok(HttpRequest { url, url_parameters, headers, body, authentication, ..request.to_owned() })
+}
+
+pub async fn render_grpc_request<T: TemplateCallback>(
+    r: &GrpcRequest,
+    environment_chain: Vec<Environment>,
+    cb: &T,
+    opt: &RenderOptions,
+) -> yaak_templates::error::Result<GrpcRequest> {
+    let vars = &make_vars_hashmap(environment_chain);
+
+    let mut metadata = Vec::new();
+    for p in r.metadata.clone() {
+        if !p.enabled {
+            continue;
+        }
+        metadata.push(HttpRequestHeader {
+            enabled: p.enabled,
+            name: parse_and_render(p.name.as_str(), vars, cb, opt).await?,
+            value: parse_and_render(p.value.as_str(), vars, cb, opt).await?,
+            id: p.id,
+        })
+    }
+
+    let authentication = {
+        let mut disabled = false;
+        let mut auth = BTreeMap::new();
+        match r.authentication.get("disabled") {
+            Some(Value::Bool(true)) => {
+                disabled = true;
+            }
+            Some(Value::String(tmpl)) => {
+                disabled = parse_and_render(tmpl.as_str(), vars, cb, opt)
+                    .await
+                    .unwrap_or_default()
+                    .is_empty();
+                info!(
+                    "Rendering authentication.disabled as a template: {disabled} from \"{tmpl}\""
+                );
+            }
+            _ => {}
+        }
+        if disabled {
+            auth.insert("disabled".to_string(), Value::Bool(true));
+        } else {
+            for (k, v) in r.authentication.clone() {
+                if k == "disabled" {
+                    auth.insert(k, Value::Bool(false));
+                } else {
+                    auth.insert(k, render_json_value_raw(v, vars, cb, opt).await?);
+                }
+            }
+        }
+        auth
+    };
+
+    let url = parse_and_render(r.url.as_str(), vars, cb, opt).await?;
+
+    Ok(GrpcRequest { url, metadata, authentication, ..r.to_owned() })
+}
 
 pub fn make_vars_hashmap(environment_chain: Vec<Environment>) -> HashMap<String, String> {
     let mut variables = HashMap::new();
@@ -27,3 +181,70 @@ fn add_variable_to_map(
 
     map
 }
+
+fn strip_disabled_form_entries(v: Value) -> Value {
+    match v {
+        Value::Array(items) => Value::Array(
+            items
+                .into_iter()
+                .filter(|item| item.get("enabled").and_then(|e| e.as_bool()).unwrap_or(true))
+                .collect(),
+        ),
+        v => v,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use serde_json::json;
+
+    #[test]
+    fn test_strip_disabled_form_entries() {
+        let input = json!([
+            {"enabled": true, "name": "foo", "value": "bar"},
+            {"enabled": false, "name": "disabled", "value": "gone"},
+            {"enabled": true, "name": "baz", "value": "qux"},
+        ]);
+        let result = strip_disabled_form_entries(input);
+        assert_eq!(
+            result,
+            json!([
+                {"enabled": true, "name": "foo", "value": "bar"},
+                {"enabled": true, "name": "baz", "value": "qux"},
+            ])
+        );
+    }
+
+    #[test]
+    fn test_strip_disabled_form_entries_all_disabled() {
+        let input = json!([
+            {"enabled": false, "name": "a", "value": "b"},
+            {"enabled": false, "name": "c", "value": "d"},
+        ]);
+        let result = strip_disabled_form_entries(input);
+        assert_eq!(result, json!([]));
+    }
+
+    #[test]
+    fn test_strip_disabled_form_entries_missing_enabled_defaults_to_kept() {
+        let input = json!([
+            {"name": "no_enabled_field", "value": "kept"},
+            {"enabled": false, "name": "disabled", "value": "gone"},
+        ]);
+        let result = strip_disabled_form_entries(input);
+        assert_eq!(
+            result,
+            json!([
+                {"name": "no_enabled_field", "value": "kept"},
+            ])
+        );
+    }
+
+    #[test]
+    fn test_strip_disabled_form_entries_non_array_passthrough() {
+        let input = json!("just a string");
+        let result = strip_disabled_form_entries(input.clone());
+        assert_eq!(result, input);
+    }
+}