mirror of
https://github.com/mountain-loop/yaak.git
synced 2026-08-24 20:34:05 +02:00
Add the browser send proxy and web sender
crates-server/yaak-send-proxy: a stateless executor over yaak-http's HttpTransaction. It takes a rendered request, streams timeline events, the response head, body chunks and the resulting cookies back as NDJSON, and keeps nothing. Private/loopback/link-local/metadata ranges are refused after DNS on every hop (an AddressFilter on the resolver plus a per-hop URL check), with size caps, a timeout ceiling, a rate limit, host allow/deny lists and an optional token. The web host now sends through it: the wasm worker resolves and renders the request (render_http_request moved into yaak-models so it builds for wasm; re-exported from its old paths), the tab posts it, and stores what comes back where the desktop stores it. Requests needing auth plugins or template functions are refused with the reason until plugins run in the browser.
This commit is contained in:
@@ -49,6 +49,7 @@ function toSyncUnsubscribe(pending: Promise<Unsubscribe>): Unsubscribe {
|
||||
}
|
||||
|
||||
const ALL_CAPABILITIES: PlatformCapabilities = {
|
||||
httpSending: true,
|
||||
grpc: true,
|
||||
websocket: true,
|
||||
git: true,
|
||||
|
||||
@@ -237,6 +237,8 @@ export interface Platform {
|
||||
* from the cargo features they were built with.
|
||||
*/
|
||||
export interface PlatformCapabilities {
|
||||
/** Send HTTP requests and see the whole response: every header, the redirect chain, timing. */
|
||||
httpSending: boolean;
|
||||
/** Send gRPC requests. Needs HTTP/2 trailers, so it needs a real backend. */
|
||||
grpc: boolean;
|
||||
/** Send WebSocket requests with custom headers and auth. */
|
||||
|
||||
@@ -24,8 +24,10 @@ installs the Tauri host exactly as before.
|
||||
|
||||
```
|
||||
tab (index.ts, commands.ts) ──MessagePort──▶ worker.ts ──▶ @yaakapp-internal/web (wasm)
|
||||
◀── model_writes ── crates/yaak-web → yaak-models → SQLite
|
||||
└─ pages in IndexedDB
|
||||
│ ◀── model_writes ── crates/yaak-web → yaak-models → SQLite
|
||||
│ └─ pages in IndexedDB
|
||||
└── send.ts ──POST rendered request──▶ yaak-send-proxy (crates-server) ──▶ the internet
|
||||
◀── NDJSON: events, response, body, cookies ──
|
||||
```
|
||||
|
||||
| File | What it is |
|
||||
@@ -33,13 +35,19 @@ tab (index.ts, commands.ts) ──MessagePort──▶ worker.ts ──▶ @yaak
|
||||
| `index.ts` | The `Platform` implementation. |
|
||||
| `commands.ts` | The command table: model commands forward to the worker; the rest is fixed answers and refusals-with-a-reason. |
|
||||
| `connection.ts` | A tab's end of the wire: request/response over a `MessagePort`, event delivery, and the tab's identity (`label`). |
|
||||
| `send.ts` | Sending: the worker renders (`prepare_http_send`), the proxy executes, this file stores what comes back where the desktop stores it. |
|
||||
| `proxy.ts` | The proxy's location and wire shapes, mirrored by hand from `crates-server/yaak-send-proxy/src/wire.rs`. |
|
||||
| `worker.ts` | The process that owns the database. Loads the wasm, opens the DB once, answers each port, fans `model_writes` out to every port. |
|
||||
| `protocol.ts` | The message shapes both sides import. |
|
||||
| `errors.ts` | `UnsupportedCommandError`, the structured refusal. |
|
||||
| `storage.ts` | `navigator.storage.persist()`. |
|
||||
|
||||
The Rust side is `crates/yaak-web` (`@yaakapp-internal/web`): `boot()`,
|
||||
`rpc(cmd, payload, label)` returning `{ result, events }`, and blob get/put.
|
||||
`rpc(cmd, payload, label)` returning `{ result, events }`, blob get/put, and
|
||||
`prepare_http_send(payload)` — the database half of a send (environment chain,
|
||||
inherited headers and auth, request settings, cookie jar, rendering), which is
|
||||
`yaak_models::render::render_http_request`, the same function the desktop
|
||||
renders with.
|
||||
Its `pkg/` is committed; rebuilding needs a clang with a WebAssembly backend
|
||||
(`brew install llvm`), and `build-wasm.cjs` skips with a notice when there
|
||||
isn't one, so a desktop `npm run bootstrap` never depends on it.
|
||||
@@ -70,13 +78,14 @@ Behaviours worth knowing before changing anything:
|
||||
## Commands
|
||||
|
||||
109 commands are declared in `@yaakapp-internal/rpc-schema`. This host answers
|
||||
31, declines 44 by name with a reason, and refuses the remaining 34 generically.
|
||||
32, declines 43 by name with a reason, and refuses the remaining 34 generically.
|
||||
|
||||
### Implemented (31)
|
||||
### Implemented (32)
|
||||
|
||||
| Group | Commands |
|
||||
| --- | --- |
|
||||
| Models | `models_workspace_models`, `models_upsert`, `models_delete`, `models_duplicate`, `models_get_settings`, `models_get_graphql_introspection`, `models_upsert_graphql_introspection`, `models_grpc_events`, `models_websocket_events` |
|
||||
| Sending | `cmd_send_http_request` (through the send proxy; see below) |
|
||||
| App | `cmd_metadata`, `cmd_get_workspace_meta`, `cmd_default_headers`, `cmd_get_themes`, `cmd_check_for_updates`, `cmd_dismiss_notification`, `cmd_plugin_init_errors` |
|
||||
| Bodies | `cmd_http_response_body`, `cmd_http_response_body_path`, `cmd_http_request_body`, `cmd_get_http_response_events`, `cmd_get_sse_events` |
|
||||
| Plugin surfaces (empty results) | `cmd_http_request_actions`, `cmd_websocket_request_actions`, `cmd_grpc_request_actions`, `cmd_workspace_actions`, `cmd_folder_actions`, `cmd_template_function_summaries`, `cmd_get_http_authentication_summaries`, `cmd_get_http_authentication_config` |
|
||||
@@ -97,7 +106,7 @@ Some of these answer honestly rather than fully, and the difference matters:
|
||||
- `cmd_metadata` reports empty strings for the data, log, plugin and project
|
||||
directories. There is no filesystem behind this host.
|
||||
|
||||
### Declined by name (44)
|
||||
### Declined by name (43)
|
||||
|
||||
Each returns an `UnsupportedCommandError` carrying `cmd`, a user-facing
|
||||
`message`, and the `capability` a caller should have checked. The UI turns it
|
||||
@@ -105,7 +114,7 @@ into a toast.
|
||||
|
||||
| Reason | Commands |
|
||||
| --- | --- |
|
||||
| Sending isn't available yet (slice 2) | `cmd_send_http_request`, `cmd_send_ephemeral_request`, `cmd_delete_send_history`, `cmd_delete_all_http_responses`, `cmd_import_url` |
|
||||
| Sending, the parts not wired yet | `cmd_send_ephemeral_request`, `cmd_delete_send_history`, `cmd_delete_all_http_responses`, `cmd_import_url` |
|
||||
| No plugin runtime | `cmd_reload_plugins`, `cmd_plugin_info`, `cmd_plugins_search`, `cmd_plugins_install`, `cmd_plugins_install_from_directory`, `cmd_plugins_uninstall`, `cmd_plugins_updates`, `cmd_plugins_update_all`, `cmd_template_function_config`, `cmd_template_tokens_to_string`, `cmd_call_http_request_action`, `cmd_call_websocket_request_action`, `cmd_call_grpc_request_action`, `cmd_call_workspace_action`, `cmd_call_folder_action`, `cmd_call_http_authentication_action`, `cmd_curl_to_request`, `cmd_format_graphql` |
|
||||
| No filesystem | `cmd_import_data`, `cmd_export_data`, `cmd_save_response`, `cmd_save_base64_to_binary` |
|
||||
| Needs a real socket | `cmd_grpc_reflect`, `cmd_grpc_go`, `cmd_delete_all_grpc_connections`, `cmd_ws_connect`, `cmd_ws_send`, `cmd_ws_close`, `cmd_ws_delete_connections` |
|
||||
@@ -129,7 +138,7 @@ Reported honestly, so callers gate on the question rather than on the host:
|
||||
|
||||
| True | False |
|
||||
| --- | --- |
|
||||
| `cookieJar` (the jar stores and edits here; only filling it needs the sender) | `grpc`, `websocket`, `git`, `sync`, `tlsOptions`, `localFiles`, `timeline`, `multiWindow`, `plugins`, `encryption`, `updater`, `clipboardRead`, `systemFonts`, `license` |
|
||||
| `httpSending`, `timeline`, `cookieJar` | `grpc`, `websocket`, `git`, `sync`, `tlsOptions`, `localFiles`, `multiWindow`, `plugins`, `encryption`, `updater`, `clipboardRead`, `systemFonts`, `license` |
|
||||
|
||||
`multiWindow: false` means the host cannot open a *second window* on demand —
|
||||
what `cmd_new_child_window` does for Settings and workspace switching. It is not
|
||||
@@ -168,26 +177,35 @@ other's writes for an echo of their own and drop them.
|
||||
crates for their types), so the crate declares the handful of request shapes
|
||||
it needs locally, and `commands.ts` stays typed against `RpcSchema`.
|
||||
|
||||
## What slice 2 (the send proxy) will need from this layer
|
||||
## Sending
|
||||
|
||||
Sending becomes a stateless hosted service; this layer stays the only place data
|
||||
lives. Concretely:
|
||||
A page cannot see a response the way a desktop app can — CORS exposes a handful
|
||||
of headers, redirects are followed silently, there is no timeline — so the
|
||||
network half of a send runs on a small stateless proxy,
|
||||
`crates-server/yaak-send-proxy`. This layer stays the only place data lives:
|
||||
|
||||
1. **A rendered request to send.** The client assembles `HttpSendInputs` and
|
||||
posts it. Nothing about the workspace is uploaded except what this request
|
||||
needs.
|
||||
2. **Cookies out, cookies in.** The active `cookie_jar` model's `cookies` array
|
||||
goes up with the request; the proxy returns the jar as the exchange left it,
|
||||
and the client upserts it back through `models_upsert` like any other write.
|
||||
The proxy keeps nothing.
|
||||
3. **A response body sink.** `blob_put(responseId, bytes)` in the worker
|
||||
writes through the desktop's `blob_manager`, chunked the way it chunks.
|
||||
Streaming will want an append path rather than one whole-body write.
|
||||
4. **A request body sink** under `${responseId}.request`, which
|
||||
`cmd_http_request_body` already reads.
|
||||
5. **Response and timeline models.** `cmd_send_http_request` currently declines;
|
||||
it will instead upsert an `http_response` as the exchange progresses, plus
|
||||
`http_response_event` rows once `timeline` becomes true. Both flow through
|
||||
the same `write()` helper, so other tabs see a send land live.
|
||||
6. **Blob cleanup is the desktop's.** `delete_http_response` and
|
||||
`delete_workspace` in `yaak-models` already remove blob chunks.
|
||||
1. `send.ts` creates the `http_response` row (state `initialized`), as the
|
||||
desktop does, so anything that goes wrong lands in the response pane.
|
||||
2. The worker resolves and renders the request (`prepare_http_send`): the
|
||||
environment chain, inherited headers and auth, request settings, the cookie
|
||||
jar. This is the desktop's `HttpSendInputs`, in Rust, on the same model layer,
|
||||
with `yaak_models::render::render_http_request`. Variables (`${[ name ]}`)
|
||||
render here with no plugins involved.
|
||||
3. The rendered request, the settings and the jar's cookies are POSTed to the
|
||||
proxy. It streams back timeline events, the response head, body chunks and a
|
||||
terminal frame carrying the jar as the send left it.
|
||||
4. Each frame is written where the desktop writes it: the response row as it
|
||||
progresses, `http_response_event` rows for the timeline (which is why
|
||||
`timeline` is true), the body under the response id via `blob_put`, and the
|
||||
cookie jar through `models_upsert`. Every write fans out to every tab.
|
||||
|
||||
**What sends today:** any saved request whose templates are variables and whose
|
||||
authentication is none, or an inline header. Sending a request that needs a
|
||||
template *function* (`${[ timestamp() ]}`) or an authentication plugin (bearer,
|
||||
basic, OAuth, …) is refused before anything leaves the tab, with a message naming
|
||||
what it needs; those light up when plugins run in the browser. Requests with a
|
||||
file body or multipart file fields are refused by the proxy (it has no access to
|
||||
your files, and must not read its own).
|
||||
|
||||
The proxy URL is `VITE_YAAK_SEND_PROXY_URL` at build time, defaulting to
|
||||
`http://127.0.0.1:9227` (see `proxy.ts`). Run one with `cargo run -p yaak-send-proxy`.
|
||||
|
||||
@@ -21,6 +21,7 @@ import type { RpcSchema } from "@yaakapp-internal/rpc-schema";
|
||||
import type { CapabilityName, RpcPayload } from "../types";
|
||||
import type { WorkerConnection } from "./connection";
|
||||
import { unsupported } from "./errors";
|
||||
import { sendHttpRequest } from "./send";
|
||||
|
||||
export type AppCmd = keyof RpcSchema;
|
||||
|
||||
@@ -66,6 +67,18 @@ const HANDLERS: Partial<Record<AppCmd, Handler>> = {
|
||||
models_websocket_events: (payload, db) => db.rpc("models_websocket_events", payload),
|
||||
cmd_get_workspace_meta: (payload, db) => db.rpc("cmd_get_workspace_meta", payload),
|
||||
|
||||
/* ------------------------------- sending ------------------------------- */
|
||||
|
||||
// The tab renders and stores; a stateless proxy puts the bytes on the wire.
|
||||
// See send.ts for the whole shape of it.
|
||||
cmd_send_http_request: (payload, db) =>
|
||||
sendHttpRequest(
|
||||
db,
|
||||
text(payload, "requestId"),
|
||||
str(payload, "environmentId"),
|
||||
str(payload, "cookieJarId"),
|
||||
),
|
||||
|
||||
/* -------------------------------- app ---------------------------------- */
|
||||
|
||||
async cmd_metadata() {
|
||||
@@ -203,9 +216,8 @@ const HANDLERS: Partial<Record<AppCmd, Handler>> = {
|
||||
return bytes == null ? null : Array.from(bytes);
|
||||
},
|
||||
|
||||
async cmd_get_http_response_events() {
|
||||
return [];
|
||||
},
|
||||
// The rows the sender wrote for that response, same table as the desktop.
|
||||
cmd_get_http_response_events: (payload, db) => db.rpc("cmd_get_http_response_events", payload),
|
||||
|
||||
async cmd_get_sse_events() {
|
||||
return [];
|
||||
@@ -237,16 +249,10 @@ const HTTP_AUTHENTICATION_SUMMARIES = [
|
||||
* while the first is a slice away.
|
||||
*/
|
||||
const DECLINED: Partial<Record<AppCmd, [reason: string, capability: CapabilityName | null]>> = {
|
||||
// Sending — the next slice. Everything else about a request works today;
|
||||
// only the part that puts bytes on the network is missing.
|
||||
cmd_send_http_request: [
|
||||
"Sending isn't available in the browser yet — everything else about this request is saved",
|
||||
null,
|
||||
],
|
||||
cmd_send_ephemeral_request: [
|
||||
"Sending isn't available in the browser yet — everything else about this request is saved",
|
||||
null,
|
||||
],
|
||||
// Saved requests send through the proxy (see send.ts). Ephemeral sends — the
|
||||
// ones nothing stores, used for GraphQL introspection — take the same road but
|
||||
// return the body inline; not wired yet.
|
||||
cmd_send_ephemeral_request: ["Sending unsaved requests isn't available in the browser yet", null],
|
||||
cmd_curl_to_request: ["Importing from cURL needs a plugin, which this host doesn't run", null],
|
||||
|
||||
// Protocols that need a real socket.
|
||||
|
||||
@@ -163,6 +163,11 @@ export class WorkerConnection {
|
||||
return this.request<T>((id) => ({ type: "rpc", id, cmd, payload, label: this.label }));
|
||||
}
|
||||
|
||||
/** See `prepare_http_send` in crates/yaak-web: the database half of a send. */
|
||||
prepareHttpSend<T>(payload: unknown): Promise<T> {
|
||||
return this.request<T>((id) => ({ type: "prepare_http_send", id, payload }));
|
||||
}
|
||||
|
||||
async blobGet(blobId: string): Promise<Uint8Array<ArrayBuffer> | null> {
|
||||
const buf = await this.request<ArrayBuffer | null>((id) => ({ type: "blob_get", id, blobId }));
|
||||
return buf == null ? null : new Uint8Array(buf);
|
||||
|
||||
@@ -7,11 +7,12 @@
|
||||
* the origin, so two tabs stay coherent for the same reason two desktop windows
|
||||
* do: one process holds the data and pushes every write to all of them.
|
||||
*
|
||||
* What a page genuinely cannot do is not faked. There is no file dialog, no
|
||||
* second window, no clipboard read without a prompt, and — in this slice — no
|
||||
* sending. Those report false through `capabilities` and refuse with a reason
|
||||
* if called anyway, so a missing feature shows up as a disabled control or a
|
||||
* toast that explains itself, never as a silent no-op.
|
||||
* Sending goes through a small stateless proxy, because a page cannot see a
|
||||
* response the way a desktop app can (see send.ts). What a page genuinely
|
||||
* cannot do is not faked: there is no file dialog, no second window, no
|
||||
* clipboard read without a prompt. Those report false through `capabilities`
|
||||
* and refuse with a reason if called anyway, so a missing feature shows up as a
|
||||
* disabled control or a toast that explains itself, never as a silent no-op.
|
||||
*/
|
||||
|
||||
import type {
|
||||
@@ -32,17 +33,21 @@ import { requestPersistence } from "./storage";
|
||||
/** What this host can do, reported honestly. */
|
||||
function capabilitiesFor(): PlatformCapabilities {
|
||||
return {
|
||||
// Through the send proxy: the tab renders, the proxy executes, the tab
|
||||
// stores. Requests needing plugin auth or template functions are refused
|
||||
// with the reason until plugins run here.
|
||||
httpSending: true,
|
||||
grpc: false,
|
||||
websocket: false,
|
||||
git: false,
|
||||
sync: false,
|
||||
// Certificates and proxies are decided by whoever puts the bytes on the
|
||||
// wire. Nothing in the browser does yet.
|
||||
// wire, and the send proxy uses its own.
|
||||
tlsOptions: false,
|
||||
// The jar can be edited and stored here; only filling it needs the sender.
|
||||
cookieJar: true,
|
||||
localFiles: false,
|
||||
timeline: false,
|
||||
// The proxy streams the engine's events back and the sender stores them.
|
||||
timeline: true,
|
||||
// Whether the host can put a *second window* on this data on demand — what
|
||||
// `cmd_new_child_window` does for Settings and workspace switching. A tab
|
||||
// can't, so those open in place instead. This is not a claim that nothing
|
||||
|
||||
@@ -10,6 +10,12 @@
|
||||
/** Tab → worker */
|
||||
export type ToWorker =
|
||||
| { type: "rpc"; id: number; cmd: string; payload: unknown; label: string }
|
||||
/**
|
||||
* The prepare half of a send: resolve, inherit and render a request against
|
||||
* the database. Its own message rather than an `rpc` command because it is
|
||||
* async in the engine (rendering is), where every `rpc` command is not.
|
||||
*/
|
||||
| { type: "prepare_http_send"; id: number; payload: unknown }
|
||||
| { type: "blob_get"; id: number; blobId: string }
|
||||
| { type: "blob_put"; id: number; blobId: string; bytes: ArrayBuffer }
|
||||
| { type: "blob_delete"; id: number; blobId: string }
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
/**
|
||||
* The wire to the send proxy: where it is, what goes up, and what comes back.
|
||||
*
|
||||
* These shapes mirror `crates-server/yaak-send-proxy/src/wire.rs` by hand. The
|
||||
* proxy is a separate binary with its own release cadence, so the contract is
|
||||
* written down on both sides rather than generated across them; a change to one
|
||||
* is a change to the other, and the frame `type` tags are the versioning.
|
||||
*/
|
||||
|
||||
/* ------------------------------- location -------------------------------- */
|
||||
|
||||
/**
|
||||
* Where the tab sends. Build-time configuration for now: `VITE_YAAK_SEND_PROXY_URL`
|
||||
* (Vite exposes `VITE_*` to the bundle), defaulting to a proxy on this machine at
|
||||
* its default port. A per-user setting can replace this later without touching
|
||||
* the callers, which only ever ask for the URL.
|
||||
*/
|
||||
export function proxyBaseUrl(): string {
|
||||
const env = (import.meta as unknown as { env?: Record<string, string | undefined> }).env;
|
||||
const configured = env?.VITE_YAAK_SEND_PROXY_URL?.trim();
|
||||
return (configured || "http://127.0.0.1:9227").replace(/\/+$/, "");
|
||||
}
|
||||
|
||||
export function proxySendUrl(): string {
|
||||
return `${proxyBaseUrl()}/v1/http/send`;
|
||||
}
|
||||
|
||||
/* --------------------------------- up ------------------------------------ */
|
||||
|
||||
/** The body of `POST /v1/http/send`. */
|
||||
export interface ProxyRequestBody {
|
||||
/** The rendered request, in the model shape (see `wire.rs` `SendRequest.request`). */
|
||||
request: Record<string, unknown>;
|
||||
settings: {
|
||||
validateCertificates: boolean;
|
||||
followRedirects: boolean;
|
||||
timeoutMs: number;
|
||||
sendCookies: boolean;
|
||||
storeCookies: boolean;
|
||||
};
|
||||
/** The jar's cookies to start from, or `null` for no jar at all. */
|
||||
cookies: unknown[] | null;
|
||||
}
|
||||
|
||||
/* -------------------------------- down ----------------------------------- */
|
||||
|
||||
interface WireHeader {
|
||||
name: string;
|
||||
value: string;
|
||||
}
|
||||
|
||||
export interface ProxySendResponse {
|
||||
type: "response";
|
||||
status: number;
|
||||
statusReason: string | null;
|
||||
url: string;
|
||||
remoteAddr: string | null;
|
||||
version: string | null;
|
||||
headers: WireHeader[];
|
||||
requestHeaders: WireHeader[];
|
||||
contentLength: number | null;
|
||||
elapsedHeaders: number;
|
||||
elapsedDns: number;
|
||||
}
|
||||
|
||||
export type ProxyFrame =
|
||||
/** A timeline event in the `http_response_event.event` shape. */
|
||||
| { type: "event"; event: unknown }
|
||||
| ProxySendResponse
|
||||
/** A body chunk, decompressed, base64. */
|
||||
| { type: "body"; data: string }
|
||||
| {
|
||||
type: "done";
|
||||
elapsed: number;
|
||||
contentLength: number;
|
||||
contentLengthCompressed: number;
|
||||
cookies: unknown[] | null;
|
||||
}
|
||||
| { type: "error"; message: string; cookies: unknown[] | null };
|
||||
|
||||
/**
|
||||
* Yield frames from an NDJSON stream as they arrive. A partial trailing line is
|
||||
* held until its newline comes; anything left when the stream ends is dropped,
|
||||
* because a frame without its newline is a frame the proxy didn't finish writing.
|
||||
*/
|
||||
export async function* readFrames(stream: ReadableStream<Uint8Array>): AsyncGenerator<ProxyFrame> {
|
||||
const reader = stream.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = "";
|
||||
try {
|
||||
while (true) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) break;
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
let newline = buffer.indexOf("\n");
|
||||
while (newline !== -1) {
|
||||
const line = buffer.slice(0, newline);
|
||||
buffer = buffer.slice(newline + 1);
|
||||
if (line.trim() !== "") yield JSON.parse(line) as ProxyFrame;
|
||||
newline = buffer.indexOf("\n");
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,386 @@
|
||||
/**
|
||||
* Sending an HTTP request from a tab.
|
||||
*
|
||||
* A tab can't see a response the way the desktop can — CORS hides most headers,
|
||||
* redirects are followed silently, there is no timeline — so the network half of
|
||||
* a send happens on a small stateless proxy (`crates-server/yaak-send-proxy`).
|
||||
* Everything else happens here, against this tab's own database, in the same
|
||||
* order the desktop does it:
|
||||
*
|
||||
* 1. create the `http_response` row (state: initialized);
|
||||
* 2. resolve and render the request in the worker (`prepare_http_send`: the
|
||||
* environment chain, inherited headers and auth, request settings, cookie
|
||||
* jar — the desktop's `HttpSendInputs`, in Rust, on the same model layer);
|
||||
* 3. POST the rendered request to the proxy and consume its stream: timeline
|
||||
* events, the response head, body chunks, and a terminal frame;
|
||||
* 4. write what comes back where the desktop writes it — the response row as
|
||||
* it progresses, `http_response_event` rows for the timeline, the body
|
||||
* blob under the response id, the cookie jar with the proxy's changes.
|
||||
*
|
||||
* The proxy keeps nothing. Every byte it sees comes from this tab and every
|
||||
* byte it returns is stored by this tab.
|
||||
*/
|
||||
|
||||
import type { WorkerConnection } from "./connection";
|
||||
import type { ProxyFrame, ProxyRequestBody, ProxySendResponse } from "./proxy";
|
||||
import { proxySendUrl, readFrames } from "./proxy";
|
||||
|
||||
/* -------------------------------- shapes --------------------------------- */
|
||||
|
||||
// The model types this file writes, spelled out rather than imported from
|
||||
// `@yaakapp-internal/models`: the platform package sits underneath the model
|
||||
// package in the dependency graph and must not import it.
|
||||
|
||||
interface HttpResponseHeader {
|
||||
name: string;
|
||||
value: string;
|
||||
}
|
||||
|
||||
/** The subset of the `http_response` model this sender writes. */
|
||||
interface ResponsePatch {
|
||||
model: "http_response";
|
||||
id: string;
|
||||
requestId: string;
|
||||
workspaceId: string;
|
||||
state: "initialized" | "connected" | "closed";
|
||||
url: string;
|
||||
status: number;
|
||||
statusReason: string | null;
|
||||
version: string | null;
|
||||
remoteAddr: string | null;
|
||||
headers: HttpResponseHeader[];
|
||||
requestHeaders: HttpResponseHeader[];
|
||||
contentLength: number | null;
|
||||
contentLengthCompressed: number | null;
|
||||
elapsed: number;
|
||||
elapsedHeaders: number;
|
||||
elapsedDns: number;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
interface CookieJarModel {
|
||||
model: "cookie_jar";
|
||||
id: string;
|
||||
cookies: unknown[];
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
/** What `prepare_http_send` (crates/yaak-web) hands back. */
|
||||
interface PreparedHttpSend {
|
||||
request: { url: string; [key: string]: unknown };
|
||||
settings: ProxyRequestBody["settings"];
|
||||
settingEvents: unknown[];
|
||||
cookieJar: CookieJarModel | null;
|
||||
}
|
||||
|
||||
/** The desktop writes progress at most this often while a body streams in. */
|
||||
const PROGRESS_INTERVAL_MS = 100;
|
||||
|
||||
/* --------------------------------- send ---------------------------------- */
|
||||
|
||||
export async function sendHttpRequest(
|
||||
db: WorkerConnection,
|
||||
requestId: string,
|
||||
environmentId: string | null,
|
||||
cookieJarId: string | null,
|
||||
): Promise<unknown> {
|
||||
// The response row exists before anything can go wrong, as on the desktop, so
|
||||
// a failure to render or to reach the proxy lands in the response pane as
|
||||
// that response's error rather than as a toast that names no request.
|
||||
const workspaceId = await workspaceIdOfRequest(db, requestId);
|
||||
const response = new ResponseWriter(db, {
|
||||
model: "http_response",
|
||||
id: "",
|
||||
requestId,
|
||||
workspaceId,
|
||||
state: "initialized",
|
||||
url: "",
|
||||
status: 0,
|
||||
statusReason: null,
|
||||
version: null,
|
||||
remoteAddr: null,
|
||||
headers: [],
|
||||
requestHeaders: [],
|
||||
contentLength: null,
|
||||
contentLengthCompressed: null,
|
||||
elapsed: 0,
|
||||
elapsedHeaders: 0,
|
||||
elapsedDns: 0,
|
||||
error: null,
|
||||
});
|
||||
await response.create();
|
||||
|
||||
const cancel = new AbortController();
|
||||
const unlistenCancel = db.listen(`cancel_http_response_${response.id}`, () => cancel.abort());
|
||||
|
||||
try {
|
||||
await runSend(db, response, requestId, environmentId, cookieJarId, cancel.signal);
|
||||
} catch (err) {
|
||||
const message = cancel.signal.aborted ? "Request canceled" : errorMessage(err);
|
||||
await response.finish({ error: message });
|
||||
} finally {
|
||||
unlistenCancel();
|
||||
}
|
||||
return response.current();
|
||||
}
|
||||
|
||||
async function runSend(
|
||||
db: WorkerConnection,
|
||||
response: ResponseWriter,
|
||||
requestId: string,
|
||||
environmentId: string | null,
|
||||
cookieJarId: string | null,
|
||||
signal: AbortSignal,
|
||||
): Promise<void> {
|
||||
const prepared = await db.prepareHttpSend<PreparedHttpSend>({
|
||||
requestId,
|
||||
environmentId,
|
||||
cookieJarId,
|
||||
});
|
||||
await response.patch({ url: prepared.request.url });
|
||||
|
||||
const timeline = new TimelineWriter(db, response.id, response.workspaceId);
|
||||
timeline.push(prepared.settingEvents);
|
||||
|
||||
const body: ProxyRequestBody = {
|
||||
request: prepared.request,
|
||||
settings: prepared.settings,
|
||||
cookies: prepared.cookieJar?.cookies ?? null,
|
||||
};
|
||||
const startedAt = performance.now();
|
||||
const res = await fetch(proxySendUrl(), {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify(body),
|
||||
signal,
|
||||
}).catch((err: unknown) => {
|
||||
if (signal.aborted) throw err;
|
||||
throw new Error(`Couldn't reach the send proxy at ${proxySendUrl()}: ${errorMessage(err)}`);
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
// A refusal, not a failed send: bad destination, rate limit, a body the
|
||||
// proxy can't build. It comes as JSON with the reason.
|
||||
const text = await res.text();
|
||||
let reason = text;
|
||||
try {
|
||||
reason = (JSON.parse(text) as { error?: string }).error ?? text;
|
||||
} catch {
|
||||
/* not JSON; the text is the reason */
|
||||
}
|
||||
throw new Error(reason || `The send proxy answered ${res.status}`);
|
||||
}
|
||||
if (res.body == null) throw new Error("The send proxy sent no body");
|
||||
|
||||
const chunks: Uint8Array[] = [];
|
||||
let received = 0;
|
||||
let lastProgress = startedAt;
|
||||
let terminal: ProxyFrame | null = null;
|
||||
|
||||
for await (const frame of readFrames(res.body)) {
|
||||
switch (frame.type) {
|
||||
case "event":
|
||||
timeline.push([frame.event]);
|
||||
break;
|
||||
case "response":
|
||||
await response.patch(headOf(frame));
|
||||
break;
|
||||
case "body": {
|
||||
const bytes = base64ToBytes(frame.data);
|
||||
chunks.push(bytes);
|
||||
received += bytes.byteLength;
|
||||
const now = performance.now();
|
||||
if (now - lastProgress >= PROGRESS_INTERVAL_MS) {
|
||||
lastProgress = now;
|
||||
await response.patch({
|
||||
contentLength: received,
|
||||
elapsed: Math.round(now - startedAt),
|
||||
});
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "done":
|
||||
case "error":
|
||||
terminal = frame;
|
||||
break;
|
||||
}
|
||||
if (terminal != null) break;
|
||||
}
|
||||
|
||||
// Everything the proxy said about the timeline is in the database before the
|
||||
// response is marked closed, so a reader that wakes on "closed" sees all of it.
|
||||
await timeline.flush();
|
||||
|
||||
if (terminal == null) {
|
||||
throw new Error("The send proxy closed the stream without finishing");
|
||||
}
|
||||
|
||||
// Cookies come back on both outcomes: a hop before the failing one may have
|
||||
// set some, and the desktop keeps those too.
|
||||
if (prepared.cookieJar != null && terminal.cookies != null) {
|
||||
await persistCookies(db, prepared.cookieJar, terminal.cookies);
|
||||
}
|
||||
|
||||
if (terminal.type === "error") {
|
||||
throw new Error(terminal.message);
|
||||
}
|
||||
|
||||
// The body is written under the response id, which is how every reader —
|
||||
// `cmd_http_response_body`, the image viewer, the download button — asks for
|
||||
// it. One write, once the whole body is here: the worker's blob store has no
|
||||
// append, and a body larger than memory is over the proxy's cap anyway.
|
||||
await db.blobPut(response.id, concat(chunks, received));
|
||||
await response.finish({
|
||||
contentLength: terminal.contentLength,
|
||||
contentLengthCompressed: terminal.contentLengthCompressed,
|
||||
elapsed: terminal.elapsed,
|
||||
});
|
||||
}
|
||||
|
||||
function headOf(frame: ProxySendResponse): Partial<ResponsePatch> {
|
||||
return {
|
||||
state: "connected",
|
||||
status: frame.status,
|
||||
statusReason: frame.statusReason,
|
||||
url: frame.url,
|
||||
remoteAddr: frame.remoteAddr,
|
||||
version: frame.version,
|
||||
headers: frame.headers,
|
||||
requestHeaders: frame.requestHeaders,
|
||||
contentLength: frame.contentLength,
|
||||
elapsedHeaders: frame.elapsedHeaders,
|
||||
elapsedDns: frame.elapsedDns,
|
||||
};
|
||||
}
|
||||
|
||||
/* ------------------------------- helpers --------------------------------- */
|
||||
|
||||
/**
|
||||
* The response row, written the way the desktop writes it: created empty,
|
||||
* patched as the send progresses, closed at the end. Each write goes through
|
||||
* `models_upsert`, so every tab on this database sees the response land.
|
||||
*/
|
||||
class ResponseWriter {
|
||||
private state: ResponsePatch;
|
||||
|
||||
constructor(
|
||||
private readonly db: WorkerConnection,
|
||||
initial: ResponsePatch,
|
||||
) {
|
||||
this.state = initial;
|
||||
}
|
||||
|
||||
get id(): string {
|
||||
return this.state.id;
|
||||
}
|
||||
|
||||
get workspaceId(): string {
|
||||
return this.state.workspaceId;
|
||||
}
|
||||
|
||||
current(): ResponsePatch {
|
||||
return this.state;
|
||||
}
|
||||
|
||||
async create(): Promise<void> {
|
||||
const { id: _, ...withoutId } = this.state;
|
||||
const id = await this.db.rpc<string>("models_upsert", { model: withoutId });
|
||||
this.state = { ...this.state, id };
|
||||
}
|
||||
|
||||
async patch(patch: Partial<ResponsePatch>): Promise<void> {
|
||||
// Structured clone carries `undefined` across to the worker as a present
|
||||
// key, and the model layer reads that as "wrong type" and refuses the whole
|
||||
// model. Nothing here should produce one, but a missing wire field must
|
||||
// not take the response row down with it.
|
||||
const defined = Object.fromEntries(Object.entries(patch).filter(([, v]) => v !== undefined));
|
||||
this.state = { ...this.state, ...defined };
|
||||
await this.db.rpc("models_upsert", { model: this.state });
|
||||
}
|
||||
|
||||
async finish(patch: Partial<ResponsePatch>): Promise<void> {
|
||||
await this.patch({ ...patch, state: "closed" });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Timeline events, written in the order they arrived. Writes are chained rather
|
||||
* than awaited inline so a burst of `header_down` events doesn't serialise the
|
||||
* body read behind a database round trip each, and `flush()` is the point at
|
||||
* which the whole timeline is known to be in the database.
|
||||
*/
|
||||
class TimelineWriter {
|
||||
private queue: unknown[] = [];
|
||||
private inFlight: Promise<void> = Promise.resolve();
|
||||
|
||||
constructor(
|
||||
private readonly db: WorkerConnection,
|
||||
private readonly responseId: string,
|
||||
private readonly workspaceId: string,
|
||||
) {}
|
||||
|
||||
push(events: unknown[]): void {
|
||||
if (events.length === 0) return;
|
||||
this.queue.push(...events);
|
||||
this.inFlight = this.inFlight.then(() => this.drain());
|
||||
}
|
||||
|
||||
private async drain(): Promise<void> {
|
||||
if (this.queue.length === 0) return;
|
||||
const events = this.queue;
|
||||
this.queue = [];
|
||||
await this.db.rpc("web_insert_http_response_events", {
|
||||
responseId: this.responseId,
|
||||
workspaceId: this.workspaceId,
|
||||
events,
|
||||
});
|
||||
}
|
||||
|
||||
flush(): Promise<void> {
|
||||
return this.inFlight;
|
||||
}
|
||||
}
|
||||
|
||||
async function persistCookies(
|
||||
db: WorkerConnection,
|
||||
jar: CookieJarModel,
|
||||
cookies: unknown[],
|
||||
): Promise<void> {
|
||||
// The desktop compares before writing so a jar edited mid-send isn't clobbered
|
||||
// by an unchanged copy. Structural equality is enough here: cookies are plain
|
||||
// data and the proxy hands back the whole jar.
|
||||
if (JSON.stringify(cookies) === JSON.stringify(jar.cookies)) return;
|
||||
await db.rpc("models_upsert", { model: { ...jar, cookies } });
|
||||
}
|
||||
|
||||
/**
|
||||
* The request's workspace, needed to create the response row before the worker
|
||||
* has resolved the request (which is where a render refusal would land).
|
||||
*/
|
||||
async function workspaceIdOfRequest(db: WorkerConnection, requestId: string): Promise<string> {
|
||||
const req = await db.rpc<{ workspaceId: string }>("web_get_http_request", { requestId });
|
||||
return req.workspaceId;
|
||||
}
|
||||
|
||||
function errorMessage(err: unknown): string {
|
||||
if (err instanceof Error) return err.message;
|
||||
return String(err);
|
||||
}
|
||||
|
||||
function base64ToBytes(data: string): Uint8Array {
|
||||
const bin = atob(data);
|
||||
const out = new Uint8Array(bin.length);
|
||||
for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i);
|
||||
return out;
|
||||
}
|
||||
|
||||
function concat(chunks: Uint8Array[], total: number): Uint8Array {
|
||||
if (chunks.length === 1) return chunks[0]!;
|
||||
const out = new Uint8Array(total);
|
||||
let offset = 0;
|
||||
for (const c of chunks) {
|
||||
out.set(c, offset);
|
||||
offset += c.byteLength;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -122,7 +122,7 @@ async function handle(port: MessagePort, message: ToWorker): Promise<void> {
|
||||
send(port, { type: "error", id: message.id, message: bootError ?? "Database failed to open" });
|
||||
return;
|
||||
}
|
||||
const { rpc, blob_get, blob_put, blob_delete } = engine!;
|
||||
const { rpc, blob_get, blob_put, blob_delete, prepare_http_send } = engine!;
|
||||
|
||||
try {
|
||||
switch (message.type) {
|
||||
@@ -142,6 +142,11 @@ async function handle(port: MessagePort, message: ToWorker): Promise<void> {
|
||||
}
|
||||
return;
|
||||
}
|
||||
case "prepare_http_send": {
|
||||
const prepared = await prepare_http_send(message.payload);
|
||||
send(port, { type: "result", id: message.id, result: prepared });
|
||||
return;
|
||||
}
|
||||
case "blob_get": {
|
||||
const bytes = blob_get(message.blobId);
|
||||
if (bytes == null) {
|
||||
|
||||
Reference in New Issue
Block a user