diff --git a/Cargo.lock b/Cargo.lock index 927a8f5b8..d18d8bebb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4156,6 +4156,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95b4103cffefa72eb8428cb6b47d6627161e51c2739fc5e3b734584157bc642a" dependencies = [ "cc", + "openssl-sys", "pkg-config", "vcpkg", ] @@ -11612,9 +11613,11 @@ dependencies = [ "base32", "base64 0.23.1", "chacha20poly1305", + "dbus-secret-service", "keyring", "log 0.4.34", "serde", + "tempfile", "thiserror 2.0.21", "yaak-models", ] @@ -11628,6 +11631,7 @@ dependencies = [ "include_dir", "log 0.4.34", "nanoid", + "openssl-sys", "r2d2", "r2d2_sqlite", "rusqlite", @@ -11635,6 +11639,7 @@ dependencies = [ "sea-query-rusqlite", "serde", "serde_json", + "tempfile", "thiserror 2.0.21", "ts-rs", "uuid", @@ -11951,6 +11956,7 @@ dependencies = [ "ts-rs", "url", "yaak-common", + "yaak-crypto", "yaak-database", "yaak-filter", "yaak-grpc", diff --git a/apps/yaak-proxy/components/ClearFlowsButton.tsx b/apps/yaak-proxy/components/ClearFlowsButton.tsx index 522ac1142..08cc5cd2a 100644 --- a/apps/yaak-proxy/components/ClearFlowsButton.tsx +++ b/apps/yaak-proxy/components/ClearFlowsButton.tsx @@ -1,14 +1,16 @@ import { Dropdown, type DropdownItem, IconButton } from "@yaakapp-internal/ui"; import { useAtomValue, useSetAtom } from "jotai"; import { useState } from "react"; -import { clearFlowsAtom, flowTotalAtom } from "../lib/store"; +import { clearFlowsAtom, flowTotalAtom, lockedAtom } from "../lib/store"; /** * Clears every flow, once confirmed in its menu, which its hotkey opens too. The list empties at - * once; the flows are deleted in the background. + * once; the flows are deleted in the background. Disabled with none to clear, or while captured + * traffic is locked. */ export function ClearFlowsButton() { const total = useAtomValue(flowTotalAtom); + const locked = useAtomValue(lockedAtom); const clear = useSetAtom(clearFlowsAtom); const [error, setError] = useState(null); @@ -45,6 +47,20 @@ export function ClearFlowsButton() { }, ]; + // Nothing to clear, or nothing that can be read while traffic is locked: no menu, nor hotkey. + if (locked != null || total === 0) { + return ( + + ); + } + return ( @@ -113,9 +116,17 @@ export function FirstRun() { yaak mcp gives it the proxy's flows and actions. +
+ Captured traffic is encrypted on disk with a key kept in your keychain, so macOS may ask + to let Yaak Proxy use it. +
); diff --git a/apps/yaak-proxy/components/LockedState.tsx b/apps/yaak-proxy/components/LockedState.tsx new file mode 100644 index 000000000..c0941f2dc --- /dev/null +++ b/apps/yaak-proxy/components/LockedState.tsx @@ -0,0 +1,53 @@ +import type { ProxyLock } from "@yaakapp-internal/proxy-lib"; +import { Button, Icon, LoadingIcon } from "@yaakapp-internal/ui"; +import { useAtomValue } from "jotai"; +import { useRpcMutation } from "../hooks/useRpcMutation"; +import { encryptingPercent } from "../lib/encrypting"; +import { lockedLine } from "../lib/locked"; +import { proxyStateAtom } from "../lib/store"; +import { getOsType } from "../lib/tauri"; + +/** + * Over the list and the detail pane while captured traffic can't be read: its key is + * unavailable, or traffic from before encryption is being encrypted. Retry reads the key again + * and opens it. + */ +export function LockedState({ lock }: { lock: ProxyLock }) { + const proxyState = useAtomValue(proxyStateAtom); + const retry = useRpcMutation("take_over"); + const encrypting = encryptingPercent(proxyState); + + return ( +
+
+ {encrypting != null ? ( + + ) : ( + + )} +
+ {encrypting != null ? `Encrypting stored traffic, ${encrypting}` : "Traffic is locked"} +
+
+ {encrypting != null + ? "Traffic captured before this version is being encrypted on disk. It shows here when it's done." + : lockedLine(lock, getOsType())} +
+ {encrypting == null && ( + + )} + {encrypting == null && retry.error != null && ( +
{retry.error.message}
+ )} +
+
+ ); +} diff --git a/apps/yaak-proxy/components/ProxyLayout.tsx b/apps/yaak-proxy/components/ProxyLayout.tsx index fb90f28c5..5c936cb9b 100644 --- a/apps/yaak-proxy/components/ProxyLayout.tsx +++ b/apps/yaak-proxy/components/ProxyLayout.tsx @@ -12,12 +12,13 @@ import { useState } from "react"; import { useLocalStorage } from "react-use"; import { flowLayoutAtom, settingsRequestedAtom } from "../lib/atoms"; import { setupIncomplete } from "../lib/firstRun"; -import { noFlowsAtom, systemProxyEverOnAtom } from "../lib/store"; +import { lockedAtom, noFlowsAtom, systemProxyEverOnAtom } from "../lib/store"; import { getOsType } from "../lib/tauri"; import { ClearFlowsButton } from "./ClearFlowsButton"; import { FirstRun } from "./FirstRun"; import { FlowDetail } from "./FlowDetail"; import { FlowList } from "./FlowList"; +import { LockedState } from "./LockedState"; import { ProxyStatus, useProxySetup } from "./ProxyStatus"; import { HelperDialog } from "./HelperDialog"; import { SaveDialog } from "./SaveDialog"; @@ -157,10 +158,12 @@ export function ProxyLayout() { /** * The flow list beside or above the detail pane, or, while setup is incomplete and there are no * flows, the first-run card across the whole area. Once setup is done an empty list says so - * itself, after a Clear too. + * itself, after a Clear too. While captured traffic is locked, the locked state takes the whole + * area, or the first-run card does, saying why, while setup is incomplete too. */ function FlowsOrFirstRun({ flowLayout }: { flowLayout: SplitLayoutLayout }) { - const noFlows = useAtomValue(noFlowsAtom); + const locked = useAtomValue(lockedAtom); + const noFlows = useAtomValue(noFlowsAtom) || locked != null; const { caStatus, systemProxy } = useProxySetup(); const systemProxyEverOn = useAtomValue(systemProxyEverOnAtom); const incomplete = setupIncomplete({ @@ -169,6 +172,7 @@ function FlowsOrFirstRun({ flowLayout }: { flowLayout: SplitLayoutLayout }) { systemProxyEverOn, }); if (noFlows && incomplete) return ; + if (locked != null) return ; return ( } + title={`Encrypting stored traffic, ${encrypting}`} + description="Traffic captured before this version is being encrypted on disk. Recording starts when it's done." + /> + ) : other != null ? ( } title="Running from the CLI" @@ -252,8 +262,20 @@ export function ProxyStatus() { return ( <> - - {other != null ? ( + + {encrypting != null ? ( + <> + + Encrypting + {encrypting} + + ) : other != null ? ( <> CLI diff --git a/apps/yaak-proxy/components/SearchBar.tsx b/apps/yaak-proxy/components/SearchBar.tsx index d376558b9..91954ac83 100644 --- a/apps/yaak-proxy/components/SearchBar.tsx +++ b/apps/yaak-proxy/components/SearchBar.tsx @@ -3,6 +3,7 @@ import { useAtom, useAtomValue, useSetAtom, useStore } from "jotai"; import { memo, type RefObject, useEffect, useRef } from "react"; import { allTrafficTotalAtom, + lockedAtom, searchTermsAtom, searchTextAtom, searchTextVersionAtom, @@ -24,6 +25,8 @@ export const SearchBar = memo(function SearchBar() { const setTerms = useSetAtom(searchTermsAtom); const setSearchText = useSetAtom(setSearchTextAtom); const version = useAtomValue(searchTextVersionAtom); + // Nothing to search while captured traffic is locked. + const locked = useAtomValue(lockedAtom) != null; // Read, not subscribed: `PlaceholderCount` keeps it current without re-rendering the field. const total = useStore().get(allTrafficTotalAtom); // PlainInput hands back its as the ref @@ -59,7 +62,8 @@ export const SearchBar = memo(function SearchBar() { hideLabel size="sm" className="font-mono text-xs" - placeholder={placeholderFor(total)} + placeholder={locked ? "Traffic is locked" : placeholderFor(total)} + disabled={locked} defaultValue={text} forceUpdateKey={String(version)} onChange={setText} diff --git a/apps/yaak-proxy/harness/LockedHarness.tsx b/apps/yaak-proxy/harness/LockedHarness.tsx new file mode 100644 index 000000000..c3b2a80be --- /dev/null +++ b/apps/yaak-proxy/harness/LockedHarness.tsx @@ -0,0 +1,90 @@ +import { HotkeyProvider } from "@yaakapp-internal/ui"; +import type { GetProxyStateResponse } from "@yaakapp-internal/proxy-lib"; +import { useStore } from "jotai"; +import { type ReactNode, useEffect, useState } from "react"; +import { ProxyLayout } from "../components/ProxyLayout"; +import { useHotkeyContextValue } from "../lib/hotkeys"; +import { syncFlows, syncProxyState, syncSidebarSelection } from "../lib/store"; + +const LOCKED = + "Traffic is locked: Yaak Proxy stores traffic encrypted and couldn't read its key from the " + + "keychain: Encryption error: User canceled the operation. Allow it access, then retry."; + +const STATE: GetProxyStateResponse = { + state: "stopped", + port: null, + configuredPort: 9090, + recording: true, + systemProxyAtLaunch: true, + httpVersion: "auto", + startError: LOCKED.slice("Traffic is locked: ".length), + encrypting: null, + locked: { message: LOCKED, keychain: true }, + otherOwner: null, +}; + +/** What the backend answers while the keychain refuses: the state, and every read refused */ +function answer(cmd: string): unknown { + switch (cmd) { + case "get_proxy_state": + return STATE; + case "ca_status": + return { trusted: true, installed: true, certPath: "/tmp/yaak-proxy-ca.pem" }; + case "system_proxy_status": + return { + enabled: false, + port: 9090, + services: [], + backend: "helper", + helper: "enabled", + supported: true, + }; + case "list_actions": + return { actions: [] }; + default: + throw new Error(LOCKED); + } +} + +/** Stands in for Tauri's IPC, answering as a locked proxy does */ +export function installLockedBackend() { + const internals = (window as unknown as { __TAURI_INTERNALS__: Record }) + .__TAURI_INTERNALS__; + internals.invoke = async (cmd: string, args?: { cmd?: string }) => { + if (cmd === "rpc") return answer(args?.cmd ?? ""); + if (cmd.startsWith("plugin:event|")) return 0; + throw new Error(`Not in the harness: ${cmd}`); + }; + (window as unknown as { __TAURI_OS_PLUGIN_INTERNALS__: Record }) + .__TAURI_OS_PLUGIN_INTERNALS__ = { + os_type: "macos", + platform: "macos", + family: "unix", + version: "15.0", + eol: "\n", + }; + // Setup was done before, as for anyone whose traffic is locked after an upgrade. + localStorage.setItem("system_proxy_ever_on", "true"); +} + +/** The whole window, as the app renders it, while captured traffic is locked */ +export function LockedHarness() { + const store = useStore(); + const [synced, setSynced] = useState(false); + useEffect(() => { + syncProxyState(store); + syncFlows(store); + syncSidebarSelection(store); + setSynced(true); + }, [store]); + if (!synced) return null; + return ( + + + + ); +} + +function Hotkeys({ children }: { children: ReactNode }) { + return {children}; +} diff --git a/apps/yaak-proxy/harness/main.tsx b/apps/yaak-proxy/harness/main.tsx index 8caad3fe1..85a58af3a 100644 --- a/apps/yaak-proxy/harness/main.tsx +++ b/apps/yaak-proxy/harness/main.tsx @@ -4,18 +4,26 @@ import { createStore, Provider } from "jotai"; import { StrictMode } from "react"; import { createRoot } from "react-dom/client"; import { FlowPanesHarness } from "./FlowPanesHarness"; +import { installLockedBackend, LockedHarness } from "./LockedHarness"; import { MessagesHarness } from "./MessagesHarness"; import "../main.css"; const params = new URLSearchParams(location.search); const dark = params.get("theme") !== "light"; applyThemeToDocument(dark ? defaultDarkTheme : defaultLightTheme); +if (params.has("locked")) installLockedBackend(); createRoot(document.getElementById("root") as HTMLElement).render( - {params.has("stream") ? : } + {params.has("locked") ? ( + + ) : params.has("stream") ? ( + + ) : ( + + )} , diff --git a/apps/yaak-proxy/lib/encrypting.test.ts b/apps/yaak-proxy/lib/encrypting.test.ts new file mode 100644 index 000000000..076836670 --- /dev/null +++ b/apps/yaak-proxy/lib/encrypting.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, test } from "vite-plus/test"; +import { encryptingPercent } from "./encrypting"; + +describe("encryptingPercent", () => { + const state = (encrypting: number | null) => + ({ encrypting }) as Parameters[0]; + + test("is null when nothing is being encrypted", () => { + expect(encryptingPercent(null)).toBeNull(); + expect(encryptingPercent(state(null))).toBeNull(); + }); + + test("rounds down and stays within 0 and 100", () => { + expect(encryptingPercent(state(0))).toBe("0%"); + expect(encryptingPercent(state(0.426))).toBe("42%"); + expect(encryptingPercent(state(0.999))).toBe("99%"); + expect(encryptingPercent(state(1))).toBe("100%"); + expect(encryptingPercent(state(1.2))).toBe("100%"); + }); +}); diff --git a/apps/yaak-proxy/lib/encrypting.ts b/apps/yaak-proxy/lib/encrypting.ts new file mode 100644 index 000000000..12e53ad0a --- /dev/null +++ b/apps/yaak-proxy/lib/encrypting.ts @@ -0,0 +1,11 @@ +import type { GetProxyStateResponse } from "@yaakapp-internal/proxy-lib"; + +/** + * How far encrypting traffic stored before encryption has got, as "42%", while it runs; null + * otherwise. Rounded down, so it reads 100% only once it is done. + */ +export function encryptingPercent(state: GetProxyStateResponse | null | undefined): string | null { + const fraction = state?.encrypting; + if (fraction == null) return null; + return `${Math.floor(Math.min(Math.max(fraction, 0), 1) * 100)}%`; +} diff --git a/apps/yaak-proxy/lib/flowActions.ts b/apps/yaak-proxy/lib/flowActions.ts index deb1b35e4..985d306fd 100644 --- a/apps/yaak-proxy/lib/flowActions.ts +++ b/apps/yaak-proxy/lib/flowActions.ts @@ -7,7 +7,7 @@ import { useCallback } from "react"; import { base64ToBytes, bodyFileName } from "./body"; import { curlBodyText, toCurl } from "./curl"; import { rpc } from "./rpc"; -import { activeQueryAtom } from "./store"; +import { activeQueryAtom, lockedAtom } from "./store"; import { useShowToast } from "./toast"; /** The flow the save dialog is open for */ @@ -68,10 +68,13 @@ export async function exportHar( export function useExportShownHar(): () => DropdownItem { const showToast = useShowToast(); const query = useAtomValue(activeQueryAtom); + // Nothing can be read to export while captured traffic is locked. + const locked = useAtomValue(lockedAtom) != null; return useCallback( () => ({ label: query == null ? "Export All Flows as HAR" : "Export Matching Flows as HAR", icon: "download", + disabled: locked, onSelect: async () => { try { const written = await exportHar(query == null ? {} : { filter: query }, "flows.har"); @@ -86,7 +89,7 @@ export function useExportShownHar(): () => DropdownItem { } }, }), - [query, showToast], + [query, locked, showToast], ); } diff --git a/apps/yaak-proxy/lib/locked.test.ts b/apps/yaak-proxy/lib/locked.test.ts new file mode 100644 index 000000000..acce6513e --- /dev/null +++ b/apps/yaak-proxy/lib/locked.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, test } from "vite-plus/test"; +import { lockedLine } from "./locked"; + +describe("lockedLine", () => { + test("points at the platform's key store when it refused", () => { + const lock = { keychain: true, message: "Traffic is locked: anything" }; + expect(lockedLine(lock, "macos")).toBe( + "Allow Yaak Proxy to use its key in Keychain Access, then retry.", + ); + expect(lockedLine(lock, "windows")).toContain("Credential Manager"); + expect(lockedLine(lock, "linux")).toContain("your keyring"); + }); + + test("gives any other reason without the title's words", () => { + const lock = { + keychain: false, + message: "Traffic is locked: the proxy is already running in the yaak CLI (pid 4).", + }; + expect(lockedLine(lock, "macos")).toBe("The proxy is already running in the yaak CLI (pid 4)."); + }); +}); diff --git a/apps/yaak-proxy/lib/locked.ts b/apps/yaak-proxy/lib/locked.ts new file mode 100644 index 000000000..374881fc4 --- /dev/null +++ b/apps/yaak-proxy/lib/locked.ts @@ -0,0 +1,30 @@ +import type { ProxyLock } from "@yaakapp-internal/proxy-lib"; + +/** What every refused read starts with, before its reason */ +const LOCKED_PREFIX = "Traffic is locked: "; + +/** Where each platform keeps the key, by the name its users know it by */ +function keyStoreName(os: string): string { + switch (os) { + case "macos": + return "Keychain Access"; + case "windows": + return "Credential Manager"; + default: + return "your keyring"; + } +} + +/** + * The locked state's one line: how to let Yaak Proxy read its key when the key store is what + * refused, or else the reason itself, without the "Traffic is locked" its title already says. + */ +export function lockedLine(lock: ProxyLock, os: string): string { + if (lock.keychain) { + return `Allow Yaak Proxy to use its key in ${keyStoreName(os)}, then retry.`; + } + const reason = lock.message.startsWith(LOCKED_PREFIX) + ? lock.message.slice(LOCKED_PREFIX.length) + : lock.message; + return reason.charAt(0).toUpperCase() + reason.slice(1); +} diff --git a/apps/yaak-proxy/lib/store.ts b/apps/yaak-proxy/lib/store.ts index 0cddb9697..df3b09735 100644 --- a/apps/yaak-proxy/lib/store.ts +++ b/apps/yaak-proxy/lib/store.ts @@ -185,6 +185,9 @@ export const followAtom = atom( /** The engine's state, kept current from its events, and polled while another process has it */ export const proxyStateAtom = atom(null); +/** Why captured traffic can't be read, while it can't: nothing that reads it is offered then */ +export const lockedAtom = atom((get) => get(proxyStateAtom)?.locked ?? null); + /** When recording was seen to pause, in this session; null while recording or not known */ export const pausedAtAtom = atom(null); diff --git a/crates-proxy/yaak-proxy-lib/Cargo.toml b/crates-proxy/yaak-proxy-lib/Cargo.toml index e4a62cf6e..bf52d7d8c 100644 --- a/crates-proxy/yaak-proxy-lib/Cargo.toml +++ b/crates-proxy/yaak-proxy-lib/Cargo.toml @@ -26,12 +26,14 @@ tokio = { workspace = true, features = ["io-util", "macros", "net", "process", " ts-rs = { workspace = true, features = ["chrono-impl"] } url = "2" yaak-common = { workspace = true } -yaak-database = { workspace = true } +yaak-crypto = { workspace = true } +# Its databases are encrypted whole with SQLCipher. +yaak-database = { workspace = true, features = ["sqlcipher"] } yaak-filter = { workspace = true } yaak-grpc = { workspace = true } yaak-http = { workspace = true } yaak-lifecycle = { workspace = true } -yaak-models = { workspace = true } +yaak-models = { workspace = true, features = ["sqlcipher"] } yaak-proxy = { workspace = true } yaak-proxy-helper = { workspace = true, features = ["ts"] } yaak-rpc = { workspace = true } @@ -82,4 +84,5 @@ tonic-reflection = "0.14.6" reqwest = { workspace = true } tempfile = "3" tokio = { workspace = true, features = ["macros", "net", "rt-multi-thread", "time"] } +yaak-crypto = { workspace = true, features = ["test-support"] } yaak-proxy-helper = { workspace = true, features = ["test-support"] } diff --git a/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts b/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts index ab4d28fdc..bd9125d90 100644 --- a/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts +++ b/crates-proxy/yaak-proxy-lib/bindings/gen_rpc.ts @@ -250,6 +250,15 @@ httpVersion: HttpVersion, * Why the engine last failed to start, until it starts. */ startError: string | null, +/** + * How far encrypting traffic stored before encryption has got, from 0 to 1, while it + * runs. The engine starts once it is done. + */ +encrypting: number | null, +/** + * Captured traffic cannot be read: its key is unavailable, or it is being encrypted. + */ +locked: ProxyLock | null, /** * Another process that owns the engine, such as the CLI daemon. While one does, this app * cannot start its own, and new flows reach it only through the database. @@ -336,6 +345,20 @@ export type Owner = "app" | "cli"; export type PreviewSaveRequest = { flowId: string, }; +/** + * Why captured traffic cannot be read, for the window's locked state. + */ +export type ProxyLock = { +/** + * What every refused read says. + */ +message: string, +/** + * The keychain would not give up the key, or has not been asked yet, so allowing Yaak + * Proxy in Keychain Access is the way out. + */ +keychain: boolean, }; + export type ProxySetting = { enabled: boolean, server: string, port: number, }; export type ProxyState = "running" | "stopped"; diff --git a/crates-proxy/yaak-proxy-lib/src/bodies.rs b/crates-proxy/yaak-proxy-lib/src/bodies.rs index c840b3c1c..12b69fb9c 100644 --- a/crates-proxy/yaak-proxy-lib/src/bodies.rs +++ b/crates-proxy/yaak-proxy-lib/src/bodies.rs @@ -322,7 +322,8 @@ mod tests { fn proxy_db() -> (tempfile::TempDir, ProxyQueryManager) { let dir = tempfile::tempdir().unwrap(); - let db = ProxyQueryManager::new(&dir.path().join("proxy.db")).unwrap(); + let db = + ProxyQueryManager::new(&dir.path().join("proxy.db"), &crate::db::test_key()).unwrap(); (dir, db) } diff --git a/crates-proxy/yaak-proxy-lib/src/client.rs b/crates-proxy/yaak-proxy-lib/src/client.rs index 4b6267866..f1601fb0f 100644 --- a/crates-proxy/yaak-proxy-lib/src/client.rs +++ b/crates-proxy/yaak-proxy-lib/src/client.rs @@ -9,6 +9,7 @@ use crate::control::{ SystemProxyRequest, TRUNCATED_HEADER, }; use crate::db::ProxyQueryManager; +use crate::db::Stores; use crate::flows::{self, FlowDetail, FlowFilter}; use crate::har::{self, Har}; use crate::macos::SystemProxyStatus; @@ -18,6 +19,7 @@ use crate::settings::ProxySettings; use serde::Serialize; use serde::de::DeserializeOwned; use std::path::Path; +use yaak_crypto::MasterKeySource; use yaak_models::blob_manager::BlobManager; /// Where captured traffic is read from. @@ -29,11 +31,13 @@ pub enum ProxyConnection { } impl ProxyConnection { - /// The live owner in `data_dir`, or its databases when there is none. - pub fn connect(data_dir: &Path) -> Result { + /// The live owner in `data_dir`, or its databases when there is none, read with the data + /// key the master key in `keys` unwraps. The app's keychain item, so on macOS a CLI asking + /// for it is prompted to be allowed. + pub fn connect(data_dir: &Path, keys: &MasterKeySource) -> Result { match control::live_owner(data_dir) { Some(info) => Ok(Self::Live(ControlClient::new(info))), - None => Ok(Self::Offline(OfflineStore::open(data_dir)?)), + None => Ok(Self::Offline(OfflineStore::open(data_dir, keys)?)), } } @@ -200,18 +204,17 @@ pub struct OfflineStore { } impl OfflineStore { - pub fn open(data_dir: &Path) -> Result { - let db_path = data_dir.join("proxy.db"); + /// Opens the databases in `data_dir`, decrypting with the data key the master key in `keys` + /// unwraps. Ones from before encryption, not yet converted, are read as they are. + pub fn open(data_dir: &Path, keys: &MasterKeySource) -> Result { + let db_path = data_dir.join(crate::db::PROXY_DB); if !db_path.exists() { return Err(format!( "No captured traffic yet: {} does not exist. Start Yaak Proxy to capture some.", db_path.display() )); } - let db = ProxyQueryManager::open_read_only(&db_path) - .map_err(|e| format!("Failed to open {}: {e}", db_path.display()))?; - let blobs = crate::db::open_blobs_read_only(&data_dir.join("blobs.sqlite")) - .map_err(|e| format!("Failed to open the proxy's blob store: {e}"))?; + let Stores { db, blobs, .. } = Stores::open_read_only(data_dir, keys)?; let settings = ProxySettings::load(&data_dir.join("settings.json"))?; Ok(Self { db, blobs, capture_body_limit: settings.capture_body_limit }) } diff --git a/crates-proxy/yaak-proxy-lib/src/control.rs b/crates-proxy/yaak-proxy-lib/src/control.rs index 99ad7c23f..1a2e67948 100644 --- a/crates-proxy/yaak-proxy-lib/src/control.rs +++ b/crates-proxy/yaak-proxy-lib/src/control.rs @@ -33,7 +33,6 @@ use std::sync::mpsc as std_mpsc; use std::time::Duration; use tokio::net::TcpListener; use ts_rs::TS; -use yaak_proxy::CaStore; pub const CONTROL_FILE: &str = "control.json"; const LOCK_FILE: &str = "control.lock"; @@ -212,10 +211,14 @@ pub struct ControlStatus { pub configured_port: u16, /// Flows are stored. Paused, traffic still passes through, unrecorded. pub recording: bool, + /// Flows stored; zero while they are locked. pub flow_count: usize, pub ca: CaStatus, /// `None` when it could not be read. pub system_proxy: Option, + /// Why captured traffic cannot be read, while it cannot. + #[serde(default)] + pub locked: Option, } #[derive(Debug, Clone, Default, Serialize, Deserialize)] @@ -458,10 +461,21 @@ impl ApiError { impl From for ApiError { fn from(message: String) -> Self { + // Locked, said through a call that returns its reason as text. + if message.starts_with(crate::LOCKED) { + return Self::new(StatusCode::SERVICE_UNAVAILABLE, message); + } Self::new(StatusCode::INTERNAL_SERVER_ERROR, message) } } +/// Captured traffic is locked for now: unavailable, not missing or broken. +impl From for ApiError { + fn from(locked: crate::Locked) -> Self { + Self::new(StatusCode::SERVICE_UNAVAILABLE, locked.to_string()) + } +} + async fn handle(state: &ServerState, req: Request) -> ControlResponse { if !authorized(&req, &state.token) { return error_response(ApiError::new( @@ -515,7 +529,7 @@ async fn route(state: &ServerState, req: Request) -> Result { let filter = FlowFilter::from_query(&query).map_err(bad_request)?; - json(&flows::list(&ctx.db, &filter)?) + json(&flows::list(&ctx.db()?, &filter)?) } (&Method::POST, ["flows", "clear"]) => { let ctx = ctx.clone(); @@ -524,7 +538,7 @@ async fn route(state: &ServerState, req: Request) -> Result { found(ctx, id)?; let limit = body_limit(&query)?; - json(&flows::detail(&ctx.db, &ctx.blobs, id, limit).await?) + json(&flows::detail(&ctx.db()?, &ctx.blobs()?, id, limit).await?) } (&Method::GET, ["flows", id, "request-body"]) => raw_body(ctx, id, FlowBody::Request).await, (&Method::GET, ["flows", id, "response-body"]) => { @@ -533,20 +547,20 @@ async fn route(state: &ServerState, req: Request) -> Result { found(ctx, id)?; let (after_seq, limit) = message_page(&query)?; - let messages = messages::list(&ctx.db, id, after_seq, limit)?; - let total = messages::count(&ctx.db, id)?; + let messages = messages::list(&ctx.db()?, id, after_seq, limit)?; + let total = messages::count(&ctx.db()?, id)?; json(&ListMessagesResponse { messages, total }) } (&Method::GET, ["flows", id, "messages", seq]) => { - let flow = flows::find(&ctx.db, id) + let flow = flows::find(&ctx.db()?, id) .ok_or_else(|| ApiError::new(StatusCode::NOT_FOUND, format!("No flow {id}")))?; let seq: i64 = seq.parse().map_err(|_| bad_request(format!("Invalid seq {seq:?}")))?; - let message = messages::find(&ctx.db, id, seq)?.ok_or_else(|| { + let message = messages::find(&ctx.db()?, id, seq)?.ok_or_else(|| { ApiError::new(StatusCode::NOT_FOUND, format!("No message {seq} in flow {id}")) })?; - let payload = messages::payload(&ctx.db, &ctx.blobs, &message)?; + let payload = messages::payload(&ctx.db()?, &ctx.blobs()?, &message)?; let (decoded, decode_error) = if message.kind == MessageKind::GrpcMessage { - match grpc::decode(&ctx.db, &ctx.blobs, &ctx.grpc, &flow, seq, false).await { + match grpc::decode(&ctx.db()?, &ctx.blobs()?, &ctx.grpc, &flow, seq, false).await { Ok(decoded) => (Some(decoded), None), Err(e) => (None, Some(e)), } @@ -556,20 +570,22 @@ async fn route(state: &ServerState, req: Request) -> Result json(&grpc::list(&ctx.db)?), + (&Method::GET, ["grpc-schemas"]) => json(&grpc::list(&ctx.db()?)?), (&Method::POST, ["grpc-schemas"]) => { let body: AddGrpcSchemaRequest = read_json(req).await?; - let added = grpc::add(&ctx.db, &ctx.grpc, &body.host_pattern, &body.proto_paths); + let db = ctx.db()?; + let added = grpc::add(&db, &ctx.grpc, &body.host_pattern, &body.proto_paths); json(&added.await.map_err(bad_request)?) } (&Method::POST, ["grpc-schemas", "reflect"]) => { let body: ReflectGrpcHostRequest = read_json(req).await?; let (host, plaintext) = grpc::reflection_host(&body.host); - let reflected = grpc::reflect(&ctx.db, &ctx.grpc, &host, plaintext).await; + let db = ctx.db()?; + let reflected = grpc::reflect(&db, &ctx.grpc, &host, plaintext).await; json(&reflected.map_err(bad_request)?) } (&Method::DELETE, ["grpc-schemas", id]) => { - json(&RemovedResponse { removed: grpc::remove(&ctx.db, &ctx.grpc, id)? }) + json(&RemovedResponse { removed: grpc::remove(&ctx.db()?, &ctx.grpc, id)? }) } (&Method::POST, ["flows", id, "save"]) => { found(ctx, id)?; @@ -579,17 +595,23 @@ async fn route(state: &ServerState, req: Request) -> Result { let filter = FlowFilter::from_query(&query).map_err(bad_request)?; - let flows = flows::list(&ctx.db, &filter)?; - json(&har::export(&ctx.db, &ctx.blobs, &flows, ctx.capture_body_limit()?).await?) + let flows = flows::list(&ctx.db()?, &filter)?; + json(&har::export(&ctx.db()?, &ctx.blobs()?, &flows, ctx.capture_body_limit()?).await?) } (&Method::GET, ["ca"]) => { - let ca = CaStore::load_or_create(&ctx.ca_dir()) - .map_err(|e| format!("Failed to load proxy CA: {e}"))?; + // Off the runtime's few workers: it may wait for the keychain or a conversion. + let ca = { + let ctx = ctx.clone(); + blocking(move || ctx.load_ca()).await? + }; Ok(response(StatusCode::OK, "application/x-pem-file", ca.pem().to_string())) } (&Method::POST, ["ca", "trust"]) => { - let ca = CaStore::load_or_create(&ctx.ca_dir()) - .map_err(|e| format!("Failed to load proxy CA: {e}"))?; + // Off the runtime's few workers: it may wait for the keychain or a conversion. + let ca = { + let ctx = ctx.clone(); + blocking(move || ctx.load_ca()).await? + }; ca::trust(ca.cert_path()).await.map_err(bad_request)?; ctx.forget_passthrough(); json(&ca::status(ca.cert_path()).await?) @@ -625,24 +647,30 @@ async fn status(ctx: &ProxyCtx) -> Result { proxy_port, configured_port: ctx.settings()?.port, recording: ctx.recording(), - flow_count: flows::count(&ctx.db)?, + // Answered while locked too, saying so. + flow_count: match ctx.db() { + Ok(db) => flows::count(&db)?, + Err(_) => 0, + }, ca, system_proxy, + locked: ctx.locked().map(|lock| lock.message), }) } fn found(ctx: &ProxyCtx, id: &str) -> Result<(), ApiError> { - match flows::find(&ctx.db, id) { + match flows::find(&ctx.db()?, id) { Some(_) => Ok(()), None => Err(ApiError::new(StatusCode::NOT_FOUND, format!("No flow {id}"))), } } async fn raw_body(ctx: &ProxyCtx, id: &str, which: FlowBody) -> Result { - let flow = flows::find(&ctx.db, id) + let flow = flows::find(&ctx.db()?, id) .ok_or_else(|| ApiError::new(StatusCode::NOT_FOUND, format!("No flow {id}")))?; let body = - bodies::read_body(&ctx.db, &ctx.blobs, &flow, which, ctx.capture_body_limit()?).await?; + bodies::read_body(&ctx.db()?, &ctx.blobs()?, &flow, which, ctx.capture_body_limit()?) + .await?; let content_type = find_header(flow.headers(which), "content-type").unwrap_or("application/octet-stream"); let mut resp = response(StatusCode::OK, content_type, body.data); diff --git a/crates-proxy/yaak-proxy-lib/src/db.rs b/crates-proxy/yaak-proxy-lib/src/db.rs index e9e42f108..13b556277 100644 --- a/crates-proxy/yaak-proxy-lib/src/db.rs +++ b/crates-proxy/yaak-proxy-lib/src/db.rs @@ -1,15 +1,30 @@ +//! The one place the proxy's databases are opened. `proxy.db` and `blobs.sqlite` are encrypted +//! whole with SQLCipher, keyed from the data key [`Stores::open`] is given: every connection is +//! keyed as it opens, and a plain database left by a build from before is converted before +//! anything writes to it. Only reading, an unconverted one is read as it is. + use include_dir::{Dir, include_dir}; use r2d2::Pool; use r2d2_sqlite::SqliteConnectionManager; use rusqlite::functions::FunctionFlags; use rusqlite::{OpenFlags, Transaction, TransactionBehavior}; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::time::Duration; +use yaak_crypto::MasterKeySource; +use yaak_crypto::data_key::DataKey; +use yaak_database::encryption::{self, DatabaseKey, FileState}; use yaak_database::{ConnectionOrTx, DbContext, Error, Result, SqlitePool, run_migrations}; use yaak_models::blob_manager::BlobManager; static MIGRATIONS: Dir<'static> = include_dir!("$CARGO_MANIFEST_DIR/migrations"); +/// Flows, their headers and messages. +pub const PROXY_DB: &str = "proxy.db"; +/// Request and response bodies, and message payloads. +pub const BLOBS_DB: &str = "blobs.sqlite"; +/// The key both are encrypted with, wrapped by the app's master key in the keychain. +pub const DATA_KEY: &str = "data.key"; + /// Reads come from a pool; writes go through one connection, as in yaak-models' /// `QueryManager`. WAL lets the CLI read `proxy.db` while the app writes it. #[derive(Clone)] @@ -19,19 +34,23 @@ pub struct ProxyQueryManager { } impl ProxyQueryManager { - pub fn new(db_path: &Path) -> Result { + /// Opens `proxy.db` encrypted with `key`, made so when it is new and converted when a build + /// from before left it plain, and brings its schema up to date. + pub fn new(db_path: &Path, key: &DatabaseKey) -> Result { if let Some(parent) = db_path.parent() { std::fs::create_dir_all(parent)?; } - let writer = file_pool(db_path, 1)?; + encryption::encrypt_in_place(db_path, key, &|_| {})?; + let writer = file_pool(db_path, 1, key)?; run_migrations(&writer, &MIGRATIONS)?; - Ok(Self { readers: file_pool(db_path, 5)?, writer }) + Ok(Self { readers: file_pool(db_path, 5, key)?, writer }) } /// `proxy.db` for reading while another process may be writing it, or none is. Writes - /// fail. No migrations run, so an older database reads as it is. - pub fn open_read_only(db_path: &Path) -> Result { - let readers = read_only_pool(db_path)?; + /// fail. No migrations run, so an older database reads as it is. An encrypted one needs + /// `key`; a plain one, not yet converted, is read without it. + pub fn open_read_only(db_path: &Path, key: Option<&DatabaseKey>) -> Result { + let readers = read_only_pool(db_path, key)?; Ok(Self { writer: readers.clone(), readers }) } @@ -71,12 +90,150 @@ impl ProxyQueryManager { } } +/// `blobs.sqlite` encrypted with `key`, converted first when it is plain, migrated. +pub fn open_blobs(path: &Path, key: &DatabaseKey) -> Result { + encryption::encrypt_in_place(path, key, &|_| {})?; + yaak_models::init_encrypted_blob_store(path, key).map_err(|e| Error::Database(e.to_string())) +} + /// `blobs.sqlite` for reading, alongside [`ProxyQueryManager::open_read_only`]. -pub fn open_blobs_read_only(path: &Path) -> Result { - let readers = read_only_pool(path)?; +pub fn open_blobs_read_only(path: &Path, key: Option<&DatabaseKey>) -> Result { + let readers = read_only_pool(path, key)?; Ok(BlobManager::new(readers.clone(), readers)) } +/// The SQLCipher key for the databases, from the data key. +pub fn database_key(data_key: &DataKey) -> DatabaseKey { + DatabaseKey::from_bytes(data_key.raw_bytes()) +} + +/// Where the data key is kept in `data_dir`. +pub fn data_key_path(data_dir: &Path) -> PathBuf { + data_dir.join(DATA_KEY) +} + +/// The data key for the databases in `data_dir`, unwrapped with the master key in `keys`, and +/// made the first time. Not made again once a database is encrypted, should its file go: a new +/// one could not open it, and would replace the CA. +pub fn load_or_create_data_key( + keys: &MasterKeySource, + data_dir: &Path, +) -> std::result::Result { + let path = data_key_path(data_dir); + if any_encrypted(data_dir) { + DataKey::load_file(keys, &path) + .map_err(|e| key_unavailable(&e.to_string()))? + .ok_or_else(|| missing_data_key(data_dir)) + } else { + DataKey::load_or_create_file(keys, &path).map_err(|e| key_unavailable(&e.to_string())) + } +} + +fn any_encrypted(data_dir: &Path) -> bool { + [PROXY_DB, BLOBS_DB] + .iter() + .any(|name| encryption::file_state(&data_dir.join(name)).ok() == Some(FileState::Encrypted)) +} + +fn missing_data_key(data_dir: &Path) -> String { + format!( + "The proxy's captured traffic is encrypted, but its key is missing from {}", + data_dir.display() + ) +} + +/// Both databases, opened together. +#[derive(Clone)] +pub struct Stores { + pub db: ProxyQueryManager, + pub blobs: BlobManager, +} + +impl Stores { + /// The databases in `data_dir`, encrypted with `key`. Ones a build from before left plain + /// are converted first, without progress; [`encrypt_existing`] shows it. + pub fn open(data_dir: &Path, key: &DatabaseKey) -> std::result::Result { + let db = ProxyQueryManager::new(&data_dir.join(PROXY_DB), key) + .map_err(|e| format!("Failed to open proxy database: {e}"))?; + let blobs = open_blobs(&data_dir.join(BLOBS_DB), key) + .map_err(|e| format!("Failed to open proxy blob store: {e}"))?; + Ok(Self { db, blobs }) + } + + /// The databases in `data_dir` for reading, with the data key the master key in `keys` + /// unwraps. Asks the keychain only when one is encrypted, and never makes a key. + pub fn open_read_only( + data_dir: &Path, + keys: &MasterKeySource, + ) -> std::result::Result { + let db_path = data_dir.join(PROXY_DB); + let blobs_path = data_dir.join(BLOBS_DB); + let key = if any_encrypted(data_dir) { + let data_key = DataKey::load_file(keys, &data_key_path(data_dir)) + .map_err(|e| key_unavailable(&e.to_string()))? + .ok_or_else(|| missing_data_key(data_dir))?; + Some(database_key(&data_key)) + } else { + None + }; + let db = ProxyQueryManager::open_read_only(&db_path, key.as_ref()) + .map_err(|e| format!("Failed to open {}: {e}", db_path.display()))?; + let blobs = open_blobs_read_only(&blobs_path, key.as_ref()) + .map_err(|e| format!("Failed to open the proxy's blob store: {e}"))?; + Ok(Self { db, blobs }) + } +} + +/// Either database in `data_dir` is plain, left by a build from before encryption. +pub fn needs_encrypting(data_dir: &Path) -> bool { + [PROXY_DB, BLOBS_DB] + .iter() + .any(|name| encryption::file_state(&data_dir.join(name)).ok() == Some(FileState::Plain)) +} + +/// Converts the plain databases in `data_dir` to ones encrypted with `key`, telling `progress` +/// how far it has got, from 0 to 1. Returns whether there were any. Nothing may have them +/// open meanwhile. +pub fn encrypt_existing( + data_dir: &Path, + key: &DatabaseKey, + progress: &(dyn Fn(f64) + Sync), +) -> std::result::Result { + let paths: Vec = [PROXY_DB, BLOBS_DB] + .iter() + .map(|name| data_dir.join(name)) + .filter(|p| encryption::file_state(p).ok() == Some(FileState::Plain)) + .collect(); + let total = paths.iter().map(|p| encryption::plain_size(p)).sum::().max(1) as f64; + let mut before = 0u64; + let mut converted = false; + for path in &paths { + let size = encryption::plain_size(path); + let done = before; + let file_progress = move |written: u64| { + progress(((done + written.min(size)) as f64 / total).min(1.0)); + }; + converted |= encryption::encrypt_in_place(path, key, &file_progress) + .map_err(|e| format!("Failed to encrypt {}: {e}", path.display()))?; + before += size; + } + progress(1.0); + Ok(converted) +} + +const KEY_UNAVAILABLE: &str = + "Yaak Proxy stores traffic encrypted and couldn't read its key from the keychain"; + +/// Why the databases cannot be opened, or the CA made, without the data key. +pub fn key_unavailable(reason: &str) -> String { + format!("{KEY_UNAVAILABLE}: {reason}. Allow it access, then retry.") +} + +/// `reason` is [`key_unavailable`]'s: the keychain would not give up the key. +pub fn is_key_unavailable(reason: &str) -> bool { + reason.starts_with(KEY_UNAVAILABLE) +} + /// The functions filters use: `sidebar_path(url)`, the app sidebar's place for a URL, and /// `under_site(url, site)`, for filtering by its nodes; `folder_name(dir)`, for naming a program /// by its folder; and `is_system(bundle_id, path)`, for macOS's own daemons. @@ -99,10 +256,24 @@ fn add_functions(conn: &mut rusqlite::Connection) -> rusqlite::Result<()> { }) } -fn read_only_pool(path: &Path) -> Result { +fn read_only_pool(path: &Path, key: Option<&DatabaseKey>) -> Result { + encryption::init_crypto(); + let key = match (encryption::file_state(path)?, key) { + (FileState::Plain, _) => None, + (_, Some(key)) => Some(key.clone()), + (_, None) => { + return Err(Error::Database(format!( + "{} is encrypted, with no key to read it", + path.display() + ))); + } + }; let manager = SqliteConnectionManager::file(path) .with_flags(OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX) - .with_init(|conn| { + .with_init(move |conn| { + if let Some(key) = &key { + encryption::apply_key(conn, key)?; + } conn.busy_timeout(Duration::from_millis(5000))?; add_functions(conn) }); @@ -113,8 +284,11 @@ fn read_only_pool(path: &Path) -> Result { .map_err(|e| Error::Database(e.to_string())) } -fn file_pool(path: &Path, max_size: u32) -> Result { - let manager = SqliteConnectionManager::file(path).with_init(|conn| { +fn file_pool(path: &Path, max_size: u32, key: &DatabaseKey) -> Result { + encryption::init_crypto(); + let key = key.clone(); + let manager = SqliteConnectionManager::file(path).with_init(move |conn| { + encryption::apply_key(conn, &key)?; conn.pragma_update(None, "journal_mode", "WAL")?; conn.pragma_update(None, "synchronous", "NORMAL")?; conn.busy_timeout(Duration::from_millis(5000))?; @@ -127,6 +301,21 @@ fn file_pool(path: &Path, max_size: u32) -> Result { .map_err(|e| Error::Database(e.to_string())) } +/// The key unit tests encrypt their databases with. +#[cfg(test)] +pub(crate) fn test_key() -> DatabaseKey { + DatabaseKey::from_bytes([7; 32]) +} + +/// A connection to a database encrypted with [`test_key`], for altering it as another build +/// would have. +#[cfg(test)] +pub(crate) fn test_conn(path: &Path) -> rusqlite::Connection { + let conn = rusqlite::Connection::open(path).unwrap(); + encryption::apply_key(&conn, &test_key()).unwrap(); + conn +} + #[cfg(test)] mod tests { use super::*; @@ -142,10 +331,10 @@ mod tests { use crate::models::Flow; let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("proxy.db"); - drop(ProxyQueryManager::new(&path).unwrap()); + drop(ProxyQueryManager::new(&path, &crate::db::test_key()).unwrap()); // As a build from before the saved request and client columns left it. - let conn = Connection::open(&path).unwrap(); + let conn = test_conn(&path); conn.execute_batch( "ALTER TABLE flows DROP COLUMN saved_request_id; ALTER TABLE flows DROP COLUMN client_pid; @@ -164,7 +353,7 @@ mod tests { .unwrap(); drop(conn); - let db = ProxyQueryManager::open_read_only(&path).unwrap(); + let db = ProxyQueryManager::open_read_only(&path, Some(&test_key())).unwrap(); let flows = db.with_conn(|c| c.find_all::()).unwrap(); assert_eq!(flows.len(), 1); assert_eq!((flows[0].id.as_str(), flows[0].saved_request_id.as_deref()), ("fl_old", None)); @@ -178,7 +367,7 @@ mod tests { // Filters by app work without the folder column, and count without the clear mark. drop(db); - let conn = Connection::open(&path).unwrap(); + let conn = test_conn(&path); conn.execute_batch( "ALTER TABLE flows ADD COLUMN client_name TEXT; ALTER TABLE flows ADD COLUMN client_bundle_id TEXT; @@ -189,7 +378,7 @@ mod tests { ) .unwrap(); drop(conn); - let db = ProxyQueryManager::open_read_only(&path).unwrap(); + let db = ProxyQueryManager::open_read_only(&path, Some(&test_key())).unwrap(); let filter = crate::flows::FlowFilter { app: Some(vec!["curl".into()]), ..Default::default() }; let (page, total, matching) = crate::flows::page(&db, &filter).unwrap(); @@ -219,10 +408,10 @@ mod tests { .unwrap(); drop(conn); - let db = ProxyQueryManager::new(&path).unwrap(); + let db = ProxyQueryManager::new(&path, &crate::db::test_key()).unwrap(); drop(db); - let conn = Connection::open(&path).unwrap(); + let conn = test_conn(&path); let tables: Vec = conn .prepare("SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name") .unwrap() @@ -312,9 +501,9 @@ mod tests { .unwrap(); drop(conn); - drop(ProxyQueryManager::new(&path).unwrap()); + drop(ProxyQueryManager::new(&path, &crate::db::test_key()).unwrap()); - let conn = Connection::open(&path).unwrap(); + let conn = test_conn(&path); assert!(columns(&conn, "flows").contains(&"client_cwd".to_string())); let applied: Vec = conn .prepare("SELECT version FROM _sqlx_migrations ORDER BY version") @@ -329,8 +518,9 @@ mod tests { drop(conn); // The stream and its messages come through, and clearing then deleting takes both. - let db = ProxyQueryManager::new(&path).unwrap(); - let blobs = yaak_models::init_blob_store(dir.path().join("blobs.sqlite")).unwrap(); + let db = ProxyQueryManager::new(&path, &crate::db::test_key()).unwrap(); + let blobs = crate::db::open_blobs(&dir.path().join("blobs.sqlite"), &crate::db::test_key()) + .unwrap(); let listed = crate::flows::list(&db, &Default::default()).unwrap(); assert_eq!(listed.len(), 1); assert_eq!((listed[0].id.as_str(), listed[0].message_count), ("fl_ws", 2)); diff --git a/crates-proxy/yaak-proxy-lib/src/flows.rs b/crates-proxy/yaak-proxy-lib/src/flows.rs index ec974618b..565dad4aa 100644 --- a/crates-proxy/yaak-proxy-lib/src/flows.rs +++ b/crates-proxy/yaak-proxy-lib/src/flows.rs @@ -753,7 +753,8 @@ mod tests { fn db_of(flows: Vec) -> (tempfile::TempDir, ProxyQueryManager) { use yaak_database::UpdateSource; let dir = tempfile::tempdir().unwrap(); - let db = ProxyQueryManager::new(&dir.path().join("proxy.db")).unwrap(); + let db = + ProxyQueryManager::new(&dir.path().join("proxy.db"), &crate::db::test_key()).unwrap(); for (i, flow) in flows.into_iter().enumerate() { let started_at = chrono::DateTime::from_timestamp(i as i64, 0).unwrap().naive_utc(); let flow = Flow { id: format!("fl_{i:03}"), started_at, ..flow }; @@ -964,7 +965,8 @@ mod tests { fn after_follows_flows_as_they_are_recorded() { use yaak_database::UpdateSource; let dir = tempfile::tempdir().unwrap(); - let db = ProxyQueryManager::new(&dir.path().join("proxy.db")).unwrap(); + let db = + ProxyQueryManager::new(&dir.path().join("proxy.db"), &crate::db::test_key()).unwrap(); let at = |s| chrono::DateTime::from_timestamp(s, 0).unwrap().naive_utc(); // A slow flow that started first but ended, and so was written, after a quick one. for (id, started) in [("fl_quick", 20), ("fl_slow", 10), ("fl_next", 30)] { @@ -986,7 +988,8 @@ mod tests { fn before_pages_back_through_older_flows() { use yaak_database::UpdateSource; let dir = tempfile::tempdir().unwrap(); - let db = ProxyQueryManager::new(&dir.path().join("proxy.db")).unwrap(); + let db = + ProxyQueryManager::new(&dir.path().join("proxy.db"), &crate::db::test_key()).unwrap(); let at = |s| chrono::DateTime::from_timestamp(s, 0).unwrap().naive_utc(); // Two started together, which their ids order. for (id, started) in [ @@ -1033,8 +1036,10 @@ mod tests { use yaak_database::UpdateSource; use yaak_models::blob_manager::BodyChunkWriter; let dir = tempfile::tempdir().unwrap(); - let db = ProxyQueryManager::new(&dir.path().join("proxy.db")).unwrap(); - let blobs = yaak_models::init_blob_store(dir.path().join("blobs.sqlite")).unwrap(); + let db = + ProxyQueryManager::new(&dir.path().join("proxy.db"), &crate::db::test_key()).unwrap(); + let blobs = crate::db::open_blobs(&dir.path().join("blobs.sqlite"), &crate::db::test_key()) + .unwrap(); let write_body = |id: &str, which| { let mut writer = BodyChunkWriter::new(blobs.clone(), Flow::body_id(id, which)); writer.write(b"body").unwrap(); @@ -1079,7 +1084,11 @@ mod tests { assert_eq!(page(&db, &narrowed).unwrap().1, 1); // Bodies that fail to delete keep their flows, to delete again. - let read_only = crate::db::open_blobs_read_only(&dir.path().join("blobs.sqlite")).unwrap(); + let read_only = crate::db::open_blobs_read_only( + &dir.path().join("blobs.sqlite"), + Some(&crate::db::test_key()), + ) + .unwrap(); assert!(delete_cleared(&db, &read_only, 2, "cln_a").is_err()); assert_eq!(rows(), 4); @@ -1117,8 +1126,10 @@ mod tests { fn one_cleaner_at_a_time_and_never_a_reused_rowid() { use yaak_database::UpdateSource; let dir = tempfile::tempdir().unwrap(); - let db = ProxyQueryManager::new(&dir.path().join("proxy.db")).unwrap(); - let blobs = yaak_models::init_blob_store(dir.path().join("blobs.sqlite")).unwrap(); + let db = + ProxyQueryManager::new(&dir.path().join("proxy.db"), &crate::db::test_key()).unwrap(); + let blobs = crate::db::open_blobs(&dir.path().join("blobs.sqlite"), &crate::db::test_key()) + .unwrap(); let write_flow = |id: &str| { let flow = Flow { id: id.into(), ..Default::default() }; db.with_tx(|tx| tx.upsert(&flow, &UpdateSource::Background)).unwrap().0 @@ -1169,7 +1180,8 @@ mod tests { fn numbers_flows_by_rowid_and_keeps_it_on_update() { use yaak_database::UpdateSource; let dir = tempfile::tempdir().unwrap(); - let db = ProxyQueryManager::new(&dir.path().join("proxy.db")).unwrap(); + let db = + ProxyQueryManager::new(&dir.path().join("proxy.db"), &crate::db::test_key()).unwrap(); let write = |flow: &Flow| db.with_tx(|tx| tx.upsert(flow, &UpdateSource::Background)); let (a, _) = write(&Flow { id: "fl_a".into(), ..Default::default() }).unwrap(); let (b, _) = write(&Flow { id: "fl_b".into(), ..Default::default() }).unwrap(); @@ -1186,7 +1198,8 @@ mod tests { fn filters_by_app_name_folder_site_and_system() { use yaak_database::UpdateSource; let dir = tempfile::tempdir().unwrap(); - let db = ProxyQueryManager::new(&dir.path().join("proxy.db")).unwrap(); + let db = + ProxyQueryManager::new(&dir.path().join("proxy.db"), &crate::db::test_key()).unwrap(); let flows = [ Flow { id: "fl_slack".into(), diff --git a/crates-proxy/yaak-proxy-lib/src/har.rs b/crates-proxy/yaak-proxy-lib/src/har.rs index 2716740ba..225fd278e 100644 --- a/crates-proxy/yaak-proxy-lib/src/har.rs +++ b/crates-proxy/yaak-proxy-lib/src/har.rs @@ -469,7 +469,8 @@ mod tests { }; let dir = tempfile::tempdir().unwrap(); - let db = ProxyQueryManager::new(&dir.path().join("proxy.db")).unwrap(); + let db = + ProxyQueryManager::new(&dir.path().join("proxy.db"), &crate::db::test_key()).unwrap(); let har = export(&db, &blobs, &[flow], 1024 * 1024).await.unwrap(); let entry = &har.log.entries[0]; let text_kept = entry.response.content.text.as_deref().unwrap(); diff --git a/crates-proxy/yaak-proxy-lib/src/lib.rs b/crates-proxy/yaak-proxy-lib/src/lib.rs index ff9ec51e8..f2fd75f3a 100644 --- a/crates-proxy/yaak-proxy-lib/src/lib.rs +++ b/crates-proxy/yaak-proxy-lib/src/lib.rs @@ -22,7 +22,7 @@ use crate::ca::CaStatus; use crate::client::ControlClient; use crate::control::{ControlFile, ControlInfo, ControlServer, Owner}; use crate::convert::DroppedHeader; -use crate::db::ProxyQueryManager; +use crate::db::{ProxyQueryManager, Stores}; use crate::grpc::{DecodedGrpcMessage, GrpcSchema, GrpcState}; use crate::helper::HelperState; use crate::macos::{SystemProxyBackend, SystemProxyStatus}; @@ -39,14 +39,16 @@ use std::collections::HashMap; use std::io::Write; use std::net::{SocketAddr, TcpStream}; use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex, OnceLock, PoisonError}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, OnceLock, PoisonError, RwLock}; use std::time::{Duration, SystemTime}; use ts_rs::TS; +pub use yaak_crypto::MasterKeySource; +use yaak_crypto::data_key::DataKey; use yaak_database::{ModelChangeEvent, UpdateSource, UpsertModelInfo}; pub use yaak_grpc::GrpcConfig; use yaak_models::blob_manager::{BlobManager, BodyChunkWriter}; -use yaak_proxy::{CaStore, ProxyConfig, ProxyEvent, ProxyHandle}; +use yaak_proxy::{CaStore, KeyCipher, ProxyConfig, ProxyEvent, ProxyHandle}; use yaak_rpc::{RpcError, RpcEventEmitter, define_rpc}; // -- Context -- @@ -78,21 +80,40 @@ pub struct ProxyCtx { /// This context's name as the one deleting cleared flows, which the database leases to one /// process at a time. cleaner_id: String, - pub db: ProxyQueryManager, - pub blobs: BlobManager, + /// Where the master key that unwraps the data key is kept: the keychain, normally. + keys: MasterKeySource, + /// The data key, once read. Held while it is read and the databases converted and opened, + /// so that happens once, and whoever needs them waits for it. + unlocked: Arc>>, + /// How far converting plain databases has got, from 0 to 1, while it runs. + encrypting: Arc>>, + /// Stops so far, for a start that waited for the databases to tell one came meanwhile. + stops: Arc, + /// `proxy.db` and `blobs.sqlite`, once unlocked. None while the data key is unavailable or + /// they are being converted: nothing is read or written then. + stores: Arc>>, + /// Why they could not be unlocked, the last time it was tried. + lock_reason: Arc>>, pub events: RpcEventEmitter, /// gRPC schemas as decoded, and the protoc that compiles attached protos. pub grpc: GrpcState, } impl ProxyCtx { - /// Opens `proxy.db` and `blobs.sqlite` in `data_dir`, the proxy app's data directory. - pub fn new(data_dir: &Path, events: RpcEventEmitter) -> Result { - let db = ProxyQueryManager::new(&data_dir.join("proxy.db")) - .map_err(|e| format!("Failed to open proxy database: {e}"))?; - let blobs = yaak_models::init_blob_store(data_dir.join("blobs.sqlite")) - .map_err(|e| format!("Failed to open proxy blob store: {e}"))?; - Ok(Self { + /// Opens `proxy.db` and `blobs.sqlite` in `data_dir`, the proxy app's data directory, + /// encrypted with a data key the master key in `keys` wraps, made the first time. Without + /// it nothing is captured, so this asks the keychain at launch; when it refuses, no database + /// is open and everything that reads or writes one is refused as [`Locked`], until starting + /// tries again. Databases a build from before left plain are converted in the background, + /// and the engine starts once they are. + pub fn new( + data_dir: &Path, + events: RpcEventEmitter, + keys: MasterKeySource, + ) -> Result { + // Before any database is opened, as it must be. + yaak_database::encryption::init_crypto(); + let ctx = Self { handle: Arc::new(Mutex::new(None)), data_dir: data_dir.to_path_buf(), owner: Owner::App, @@ -106,21 +127,160 @@ impl ProxyCtx { start_error: Arc::new(Mutex::new(None)), deleting_cleared: Arc::new(Mutex::new(false)), cleaner_id: yaak_database::generate_prefixed_id("cln"), - db, - blobs, + keys, + unlocked: Arc::new(Mutex::new(None)), + encrypting: Arc::new(Mutex::new(None)), + stops: Arc::new(AtomicU64::new(0)), + stores: Arc::new(RwLock::new(None)), + lock_reason: Arc::new(Mutex::new(None)), events, grpc: GrpcState::default(), + }; + if db::needs_encrypting(data_dir) { + let unlocking = ctx.clone(); + std::thread::spawn(move || { + if let Err(e) = unlocking.unlock() { + warn!("{e}"); + } + }); + } else if let Err(e) = ctx.unlock() { + // Kept as the reason the databases are locked; starting tries again. + warn!("{e}"); } - // A clear the last run did not finish deleting. - .with_cleared_deleting()) + Ok(ctx) } - fn with_cleared_deleting(self) -> Self { - let pending = self.db.with_conn(|c| flows::clear_mark(c.conn().resolve())) > 0; - if pending { + /// `proxy.db`, or why it is locked. + pub fn db(&self) -> Result { + Ok(self.stores()?.db) + } + + /// `blobs.sqlite`, or why it is locked. + pub fn blobs(&self) -> Result { + Ok(self.stores()?.blobs) + } + + fn stores(&self) -> Result { + let stores = self.stores.read().unwrap_or_else(PoisonError::into_inner); + stores.clone().ok_or_else(|| Locked(self.lock_reason())) + } + + /// Why the databases are locked, or None once they are open. + pub fn locked(&self) -> Option { + if self.stores.read().unwrap_or_else(PoisonError::into_inner).is_some() { + return None; + } + let reason = self.lock_reason.lock().unwrap_or_else(PoisonError::into_inner).clone(); + Some(ProxyLock { + keychain: reason.as_deref().is_none_or(db::is_key_unavailable), + message: Locked(self.lock_reason()).to_string(), + }) + } + + fn lock_reason(&self) -> String { + if self.encrypting().is_some() { + return "traffic captured before this version is being encrypted.".to_string(); + } + let reason = self.lock_reason.lock().unwrap_or_else(PoisonError::into_inner).clone(); + reason.unwrap_or_else(|| "its key has not been read yet.".to_string()) + } + + /// How far converting databases from before encryption has got, from 0 to 1, while it runs. + pub fn encrypting(&self) -> Option { + *self.encrypting.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// The data key, read the first time it is asked for, with the databases converted when + /// they are plain, then opened. The CA's key is encrypted on the spot. Callers meanwhile + /// wait for it, so capture starts after any conversion. Why it failed is kept, for what the + /// databases' absence refuses to say. + fn unlock(&self) -> Result { + let unlocked = self.try_unlock(); + let reason = unlocked.as_ref().err().cloned(); + let mut kept = self.lock_reason.lock().unwrap_or_else(PoisonError::into_inner); + if *kept != reason { + *kept = reason; + drop(kept); + self.emit_state(); + } + unlocked + } + + fn try_unlock(&self) -> Result { + let mut unlocked = self.unlocked.lock().unwrap_or_else(PoisonError::into_inner); + if let Some(key) = unlocked.as_ref() { + return Ok(key.clone()); + } + let key = db::load_or_create_data_key(&self.keys, &self.data_dir)?; + let converting = db::needs_encrypting(&self.data_dir); + // A build from before encryption that owns the engine, a CLI daemon, has the plain + // databases and CA open, and converting them would pull them away from under it. They + // are converted once it lets go, which starting here asks it to. + if converting + && let Some(owner) = control::live_owner(&self.data_dir) + && owner.pid != std::process::id() + { + return Err(format!( + "{}, so the traffic it stored can't be encrypted yet. Take over or quit it, then \ + retry.", + control::already_owned(&owner) + )); + } + if self.ca_dir().join("ca.key").exists() + && let Err(e) = self.ca_with(&key) + { + warn!("{e}"); + } + + let db_key = db::database_key(&key); + if converting { + let converted = self.encrypt_existing(&db_key); + self.set_encrypting(None); + converted?; + } + let stores = Stores::open(&self.data_dir, &db_key)?; + *self.stores.write().unwrap_or_else(PoisonError::into_inner) = Some(stores); + *unlocked = Some(key.clone()); + drop(unlocked); + + // Nothing could be read while locked, so the app reads again. + self.events.emit("flows_cleared", &FlowsClearedPayload {}); + // A clear the last run did not finish deleting. + if self.db().is_ok_and(|db| db.with_conn(|c| flows::clear_mark(c.conn().resolve())) > 0) { self.delete_cleared(); } - self + Ok(key) + } + + /// Converts the plain databases, telling the app how far it has got a few times a second. + fn encrypt_existing(&self, key: &yaak_database::encryption::DatabaseKey) -> Result<(), String> { + let started = std::time::Instant::now(); + self.set_encrypting(Some(0.0)); + db::encrypt_existing(&self.data_dir, key, &|fraction| self.set_encrypting(Some(fraction)))?; + log::info!("Encrypted the proxy's databases in {:?}", started.elapsed()); + Ok(()) + } + + fn set_encrypting(&self, fraction: Option) { + *self.encrypting.lock().unwrap_or_else(PoisonError::into_inner) = fraction; + self.emit_state(); + } + + /// Tells the app to read the proxy's state again. + fn emit_state(&self) { + let state = if self.port().is_some() { ProxyState::Running } else { ProxyState::Stopped }; + self.events.emit("proxy_state_changed", &ProxyStatePayload { state }); + } + + /// The proxy's CA, made the first time, its private key encrypted with the data key. + pub fn load_ca(&self) -> Result { + let key = self.unlock()?; + self.ca_with(&key) + } + + fn ca_with(&self, key: &DataKey) -> Result { + CaStore::load_or_create_encrypted(&self.ca_dir(), &CaKeyCipher(key.clone())) + .map_err(|e| format!("Failed to load proxy CA: {e}")) } /// The protoc that compiles `.proto` files attached for gRPC decoding, as the app bundles @@ -194,15 +354,19 @@ impl ProxyCtx { { return Err(control::already_owned(&owner)); } + // Refused without the key, so nothing is ever stored unencrypted. Waits for databases + // from before encryption to be converted, outside the handle's lock so the state reads + // meanwhile, and so a stop may come meanwhile too. + let stops = self.stops.load(Ordering::SeqCst); + let ca = self.load_ca()?; let mut handle = self.handle.lock().map_err(|_| "lock poisoned".to_string())?; - if handle.is_some() { + // Stopped while waiting: the stop was answered, so this start is not. + if handle.is_some() || self.stops.load(Ordering::SeqCst) != stops { return Ok(()); } let settings = self.settings()?; self.end_stale_streams(); - let ca = CaStore::load_or_create(&self.ca_dir()) - .map_err(|e| format!("Failed to load proxy CA: {e}"))?; let mut config = ProxyConfig::new(port.unwrap_or(settings.port), Arc::new(ca)); config.capture_body_limit = settings.capture_body_limit; config.http2 = settings.http_version == HttpVersion::Auto; @@ -223,11 +387,12 @@ impl ProxyCtx { if let Some(event_rx) = proxy_handle.take_event_rx() { let sink = FlowSink { - db: self.db.clone(), - blobs: self.blobs.clone(), + db: self.db()?, + blobs: self.blobs()?, events: self.events.clone(), attribution, recording: self.recording.clone(), + finished: Default::default(), }; std::thread::spawn(move || run_event_loop(event_rx, sink)); } @@ -260,7 +425,11 @@ impl ProxyCtx { /// Marks streams an engine left open, as one that crashed does, as ended. Only the engine's /// owner, about to start it, may: until then another process's streams may be open. fn end_stale_streams(&self) { - let ended = self.db.with_tx(|c| { + // Only after unlocking, so open. + let Ok(db) = self.db() else { + return; + }; + let ended = db.with_tx(|c| { c.conn().resolve().execute( "UPDATE flows SET error = ?1 WHERE stream_kind IS NOT NULL \ AND elapsed_ms IS NULL AND error IS NULL AND NOT cancelled", @@ -304,6 +473,8 @@ impl ProxyCtx { /// Stops the engine, returning the port it was on. fn take_handle(&self) -> Result, String> { let mut handle = self.handle.lock().map_err(|_| "lock poisoned".to_string())?; + // Under the handle's lock, so a start waiting for the databases sees it. + self.stops.fetch_add(1, Ordering::SeqCst); // Dropping the handle stops the listener and joins its thread. let port = handle.take().map(|h| h.port); drop(handle); @@ -605,14 +776,14 @@ impl ProxyCtx { flow_id: &str, req: &SaveFlowRequest, ) -> Result { - let flow = flows::get(&self.db, flow_id)?; + let flow = flows::get(&self.db()?, flow_id)?; let limit = self.capture_body_limit()?; let body = - bodies::read_body(&self.db, &self.blobs, &flow, FlowBody::Request, limit).await?; + bodies::read_body(&self.db()?, &self.blobs()?, &flow, FlowBody::Request, limit).await?; let saved = ClientDb::open(&self.client_data_dir)?.save(&flow, &body, req)?; let flow = Flow { saved_request_id: Some(saved.request.id.clone()), ..flow }; - self.db + self.db()? .with_tx(|tx| { // Cleared while saving: recording it would bring the flow back without bodies. if flows::find_in(tx.conn().resolve(), flow_id).is_none() { @@ -633,7 +804,7 @@ impl ProxyCtx { /// Clears every flow at once and tells the app so its list empties, then deletes them and /// their bodies in the background. Returns how many there were. pub fn clear_flows(&self) -> Result { - let cleared = flows::clear(&self.db)?; + let cleared = flows::clear(&self.db()?)?; // After the commit, so the app reloads what is left. Flows written before the clear sent // their events before it began. self.events.emit("flows_cleared", &FlowsClearedPayload {}); @@ -644,15 +815,14 @@ impl ProxyCtx { /// Deletes cleared flows on a background thread, a batch per transaction, unless one is /// already doing so. Yields the writer between batches, so new flows are written meanwhile. fn delete_cleared(&self) { + // Only ever after a clear or unlocking, so open. + let Ok(Stores { db, blobs }) = self.stores() else { + return; + }; if !claim(&self.deleting_cleared) { return; } - let (db, blobs, running, cleaner) = ( - self.db.clone(), - self.blobs.clone(), - self.deleting_cleared.clone(), - self.cleaner_id.clone(), - ); + let (running, cleaner) = (self.deleting_cleared.clone(), self.cleaner_id.clone()); let spawned = std::thread::Builder::new().name("proxy-clear".into()).spawn(move || { lower_thread_priority(); let mut retry = DELETE_PAUSE; @@ -838,11 +1008,54 @@ pub struct GetProxyStateResponse { pub http_version: HttpVersion, /// Why the engine last failed to start, until it starts. pub start_error: Option, + /// How far encrypting traffic stored before encryption has got, from 0 to 1, while it + /// runs. The engine starts once it is done. + pub encrypting: Option, + /// Captured traffic cannot be read: its key is unavailable, or it is being encrypted. + pub locked: Option, /// Another process that owns the engine, such as the CLI daemon. While one does, this app /// cannot start its own, and new flows reach it only through the database. pub other_owner: Option, } +/// The databases are locked: their key is unavailable, or they are being encrypted. What reads +/// or writes captured traffic is refused with this rather than reading or writing nothing. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Locked(pub String); + +/// What [`Locked`] says first. +pub const LOCKED: &str = "Traffic is locked"; + +impl std::fmt::Display for Locked { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{LOCKED}: {}", self.0) + } +} + +impl From for String { + fn from(locked: Locked) -> Self { + locked.to_string() + } +} + +impl From for RpcError { + fn from(locked: Locked) -> Self { + rpc_error(locked) + } +} + +/// Why captured traffic cannot be read, for the window's locked state. +#[derive(Serialize, TS)] +#[ts(export, export_to = "gen_rpc.ts")] +#[serde(rename_all = "camelCase")] +pub struct ProxyLock { + /// What every refused read says. + pub message: String, + /// The keychain would not give up the key, or has not been asked yet, so allowing Yaak + /// Proxy in Keychain Access is the way out. + pub keychain: bool, +} + #[derive(Serialize, TS)] #[ts(export, export_to = "gen_rpc.ts")] #[serde(rename_all = "camelCase")] @@ -1175,8 +1388,8 @@ fn list_messages( ) -> Result { find_flow(ctx, &req.flow_id)?; let messages = - messages::list(&ctx.db, &req.flow_id, req.after_seq, req.limit).map_err(rpc_error)?; - let total = messages::count(&ctx.db, &req.flow_id).map_err(rpc_error)?; + messages::list(&ctx.db()?, &req.flow_id, req.after_seq, req.limit).map_err(rpc_error)?; + let total = messages::count(&ctx.db()?, &req.flow_id).map_err(rpc_error)?; Ok(ListMessagesResponse { messages, total }) } @@ -1184,8 +1397,8 @@ fn get_message_payload( ctx: &ProxyCtx, req: GetMessagePayloadRequest, ) -> Result { - let message = messages::get(&ctx.db, &req.flow_id, req.seq).map_err(rpc_error)?; - let data = messages::payload(&ctx.db, &ctx.blobs, &message).map_err(rpc_error)?; + let message = messages::get(&ctx.db()?, &req.flow_id, req.seq).map_err(rpc_error)?; + let data = messages::payload(&ctx.db()?, &ctx.blobs()?, &message).map_err(rpc_error)?; Ok(GetMessagePayloadResponse { data: BASE64.encode(data), truncated: message.truncated }) } @@ -1194,7 +1407,7 @@ async fn decode_grpc_message( req: DecodeGrpcMessageRequest, ) -> Result { let flow = find_flow(&ctx, &req.flow_id)?; - grpc::decode(&ctx.db, &ctx.blobs, &ctx.grpc, &flow, req.seq, req.schemaless) + grpc::decode(&ctx.db()?, &ctx.blobs()?, &ctx.grpc, &flow, req.seq, req.schemaless) .await .map_err(rpc_error) } @@ -1203,15 +1416,16 @@ fn list_grpc_schemas( ctx: &ProxyCtx, _req: ListGrpcSchemasRequest, ) -> Result, RpcError> { - grpc::list(&ctx.db).map_err(rpc_error) + grpc::list(&ctx.db()?).map_err(rpc_error) } async fn add_grpc_schema(ctx: ProxyCtx, req: AddGrpcSchemaRequest) -> Result { - grpc::add(&ctx.db, &ctx.grpc, &req.host_pattern, &req.proto_paths).await.map_err(rpc_error) + let db = ctx.db()?; + grpc::add(&db, &ctx.grpc, &req.host_pattern, &req.proto_paths).await.map_err(rpc_error) } fn remove_grpc_schema(ctx: &ProxyCtx, req: RemoveGrpcSchemaRequest) -> Result { - grpc::remove(&ctx.db, &ctx.grpc, &req.id).map_err(rpc_error) + grpc::remove(&ctx.db()?, &ctx.grpc, &req.id).map_err(rpc_error) } async fn reflect_grpc_host( @@ -1219,7 +1433,8 @@ async fn reflect_grpc_host( req: ReflectGrpcHostRequest, ) -> Result { let (host, plaintext) = grpc::reflection_host(&req.host); - grpc::reflect(&ctx.db, &ctx.grpc, &host, plaintext).await.map_err(rpc_error) + let db = ctx.db()?; + grpc::reflect(&db, &ctx.grpc, &host, plaintext).await.map_err(rpc_error) } fn execute_action(ctx: &ProxyCtx, invocation: ActionInvocation) -> Result { @@ -1257,6 +1472,8 @@ async fn get_proxy_state( system_proxy_at_launch: settings.system_proxy_at_launch, http_version: settings.http_version, start_error: ctx.start_error(), + encrypting: ctx.encrypting(), + locked: ctx.locked(), other_owner, }) }) @@ -1307,7 +1524,7 @@ async fn list_workspaces( async fn preview_save(ctx: ProxyCtx, req: PreviewSaveRequest) -> Result { let flow = find_flow(&ctx, &req.flow_id)?; let limit = settings(&ctx)?.capture_body_limit; - let body = bodies::read_body(&ctx.db, &ctx.blobs, &flow, FlowBody::Request, limit) + let body = bodies::read_body(&ctx.db()?, &ctx.blobs()?, &flow, FlowBody::Request, limit) .await .map_err(rpc_error)?; let conversion = convert::flow_to_request(&flow, &body); @@ -1352,15 +1569,15 @@ async fn save_flow(ctx: ProxyCtx, req: SaveFlowToYaakRequest) -> Result Result { let flows = match req.ids { - Some(ids) => ids.iter().map(|id| flows::get(&ctx.db, id)).collect::, _>>(), + Some(ids) => ids.iter().map(|id| flows::get(&ctx.db()?, id)).collect::, _>>(), None => { let filter = flows::FlowFilter { limit: None, ..req.filter.unwrap_or_default() }; - flows::list(&ctx.db, &filter) + flows::list(&ctx.db()?, &filter) } } .map_err(rpc_error)?; let limit = settings(&ctx)?.capture_body_limit; - let har = har::export(&ctx.db, &ctx.blobs, &flows, limit).await.map_err(rpc_error)?; + let har = har::export(&ctx.db()?, &ctx.blobs()?, &flows, limit).await.map_err(rpc_error)?; let path = PathBuf::from(req.path); blocking(move || { let file = std::fs::File::create(&path) @@ -1384,7 +1601,7 @@ async fn get_flow_body( ) -> Result { let flow = find_flow(&ctx, &req.flow_id)?; let limit = settings(&ctx)?.capture_body_limit; - let body = bodies::read_body_or_raw(&ctx.db, &ctx.blobs, &flow, req.which, limit) + let body = bodies::read_body_or_raw(&ctx.db()?, &ctx.blobs()?, &flow, req.which, limit) .await .map_err(rpc_error)?; Ok(GetFlowBodyResponse { @@ -1404,7 +1621,7 @@ async fn save_flow_body( ) -> Result { let flow = find_flow(&ctx, &req.flow_id)?; let limit = settings(&ctx)?.capture_body_limit; - let body = bodies::read_body_or_raw(&ctx.db, &ctx.blobs, &flow, req.which, limit) + let body = bodies::read_body_or_raw(&ctx.db()?, &ctx.blobs()?, &flow, req.which, limit) .await .map_err(rpc_error)?; // Nothing to save; an empty file would say the body was empty. @@ -1433,8 +1650,9 @@ async fn viewer_body( which: FlowBody, limit: usize, ) -> Result { - let body = - bodies::read_body(&ctx.db, &ctx.blobs, flow, which, limit).await.map_err(rpc_error)?; + let body = bodies::read_body(&ctx.db()?, &ctx.blobs()?, flow, which, limit) + .await + .map_err(rpc_error)?; match body.decode_error { Some(e) => Err(rpc_error(e)), None => Ok(body), @@ -1465,9 +1683,12 @@ async fn ca_status(ctx: ProxyCtx, _req: CaStatusRequest) -> Result Result { - // Creates the CA when the engine has never run, so there is something to trust. - let ca = CaStore::load_or_create(&ctx.ca_dir()) - .map_err(|e| rpc_error(format!("Failed to load proxy CA: {e}")))?; + // Creates the CA when the engine has never run, so there is something to trust. Off the + // async threads: it may wait for the keychain or a conversion. + let ca = { + let ctx = ctx.clone(); + blocking(move || ctx.load_ca()).await? + }; ca::trust(ca.cert_path()).await.map_err(rpc_error)?; ctx.forget_passthrough(); ca::status(ca.cert_path()).await.map_err(rpc_error) @@ -1514,7 +1735,7 @@ fn settings(ctx: &ProxyCtx) -> Result { /// A flow by id, with its number; not one that is cleared, though its row may wait to be /// deleted. fn find_flow(ctx: &ProxyCtx, flow_id: &str) -> Result { - flows::get(&ctx.db, flow_id).map_err(rpc_error) + flows::get(&ctx.db()?, flow_id).map_err(rpc_error) } fn rpc_error(e: impl ToString) -> RpcError { @@ -1529,7 +1750,7 @@ fn list_actions( } fn list_flows(ctx: &ProxyCtx, req: flows::FlowFilter) -> Result { - let (flows, total, matching) = flows::page(&ctx.db, &req).map_err(rpc_error)?; + let (flows, total, matching) = flows::page(&ctx.db()?, &req).map_err(rpc_error)?; Ok(ListFlowsResponse { flows, total, matching }) } @@ -1546,6 +1767,19 @@ pub const ANOTHER_APP_OWNS: &str = /// How long taking over waits for the owner to let go of the engine. const TAKE_OVER_WAIT: Duration = Duration::from_secs(10); +/// The CA's private key, encrypted with the data key. +struct CaKeyCipher(DataKey); + +impl KeyCipher for CaKeyCipher { + fn encrypt(&self, data: &[u8]) -> Result, String> { + self.0.encrypt(data).map_err(|e| e.to_string()) + } + + fn decrypt(&self, data: &[u8]) -> Result, String> { + self.0.decrypt(data).map_err(|e| e.to_string()) + } +} + /// The longest a new client connection waits for its app to be found before it is served. const ACCEPT_ATTRIBUTION_WAIT: Duration = Duration::from_millis(20); @@ -1626,8 +1860,35 @@ struct FlowSink { events: RpcEventEmitter, attribution: Attribution, recording: Arc>, + /// Flows that have ended, waiting to be written together. + finished: std::cell::RefCell, } +/// Flows that have ended and are yet to be written, with when the first of them ended. Each +/// commit encrypts every page it writes, so flows ending together share one: a batch is +/// written once no more events are waiting, or it reaches [`FLOW_BATCH`], or its first flow +/// has waited [`FLOW_BATCH_WAIT`]. What an engine that crashes loses with it is the batch +/// open then, as it loses the events still queued and the flows still in flight. +#[derive(Default)] +struct FinishedFlows { + flows: Vec, + since: Option, +} + +struct FinishedFlow { + flow: Flow, + persisted: bool, + /// The last chunk of each of its bodies, written with the batch. + chunks: Vec, +} + +/// Flows written in one transaction at most. +const FLOW_BATCH: usize = 50; + +/// The longest a flow that has ended waits for others to be written with, while events keep +/// coming. Alone, it is written at once. +const FLOW_BATCH_WAIT: Duration = Duration::from_millis(20); + fn run_event_loop(rx: std::sync::mpsc::Receiver, sink: FlowSink) { use std::sync::mpsc::{RecvTimeoutError, TryRecvError}; let mut in_flight: HashMap = HashMap::new(); @@ -1637,11 +1898,14 @@ fn run_event_loop(rx: std::sync::mpsc::Receiver, sink: FlowSink) { loop { // Messages are written once no more are waiting, or a batch is full; flows changed by // them once their interval is up. - let event = if !pending.is_empty() { + // Flows that have ended likewise, so under load many share a transaction, and one + // that comes alone is written at once. + let event = if !pending.is_empty() || sink.has_finished() { match rx.try_recv() { Ok(event) => event, Err(TryRecvError::Empty) => { sink.write_messages(&mut in_flight, &mut pending); + sink.write_finished(); sink.write_due_flows(&mut in_flight); continue; } @@ -1673,6 +1937,11 @@ fn run_event_loop(rx: std::sync::mpsc::Receiver, sink: FlowSink) { sink.write_messages(&mut in_flight, &mut pending); sink.handle(&mut in_flight, event); } + // Traffic that never lets up never empties the queue, so a batch is written once it + // is full or its first flow has waited long enough. + if sink.finished_due() { + sink.write_finished(); + } // Traffic that never lets up never empties the queue, so flows due to be written are // looked for here too, every so often, their messages written first. if last_due_check.elapsed() >= DUE_CHECK_INTERVAL { @@ -1684,13 +1953,14 @@ fn run_event_loop(rx: std::sync::mpsc::Receiver, sink: FlowSink) { sink.write_messages(&mut in_flight, &mut pending); // The engine stopped: a stream still open ends with it. Other flows in flight are dropped, - // as they always have been. + // as they always have been. Every flow that ended is written. for (_, mut f) in in_flight.drain() { if f.persisted { f.flow.error = Some(STOPPED_MID_STREAM.to_string()); sink.finish_flow(f); } } + sink.write_finished(); } /// When the next changed streaming flow is due to be written. @@ -2029,19 +2299,24 @@ impl FlowSink { } } + /// Puts a flow that has ended, with the rest of its bodies, in the batch to be written. fn finish_flow(&self, mut f: InFlight) { // Ended while its upload still went on: what came of it is kept, cut. let upload_cut = bodies::from_messages(&f.flow, FlowBody::Request) && f.request_body.is_some() && f.flow.request_size > 0; f.flow.request_truncated |= upload_cut; - finish_body(f.request_body); - finish_body(f.response_body); - let id = f.flow.id.clone(); if f.cleared { - self.forget_cleared(&id); + // Nothing more of it is kept, so its bodies are not finished. + drop((f.request_body, f.response_body)); + self.forget_cleared(&f.flow.id); return; } + let chunks = [f.request_body, f.response_body] + .into_iter() + .flatten() + .filter_map(BodyChunkWriter::finish_deferred) + .collect(); let mut flow = f.flow; flow.content_type = find_header(&flow.response_headers, "content-type").map(String::from); @@ -2052,11 +2327,74 @@ impl FlowSink { flow.client_path = Some(app.path); flow.client_cwd = app.cwd; } - let written = self.db.with_tx(|ctx| self.write_flow(ctx, flow, f.persisted)); - match written { - Ok(true) => {} - Ok(false) => self.forget_cleared(&id), - Err(e) => warn!("Failed to write proxy flow: {e}"), + let mut finished = self.finished.borrow_mut(); + finished.since.get_or_insert_with(std::time::Instant::now); + finished.flows.push(FinishedFlow { flow, persisted: f.persisted, chunks }); + } + + fn has_finished(&self) -> bool { + !self.finished.borrow().flows.is_empty() + } + + /// The batch is full, or its first flow has waited long enough. + fn finished_due(&self) -> bool { + let finished = self.finished.borrow(); + finished.flows.len() >= FLOW_BATCH + || finished.since.is_some_and(|t| t.elapsed() >= FLOW_BATCH_WAIT) + } + + /// Writes the flows that have ended: their bodies' last chunks in one transaction, then + /// the flows in another, telling the app of each. Bodies first, so a flow the app hears of + /// has its bodies. A flow that fails to write is rolled back alone, and a batch of bodies + /// that fails is written again a body at a time, so one bad row loses no others. + fn write_finished(&self) { + let flows = { + let mut finished = self.finished.borrow_mut(); + finished.since = None; + std::mem::take(&mut finished.flows) + }; + if flows.is_empty() { + return; + } + + let chunks: Vec<_> = flows.iter().flat_map(|f| f.chunks.iter()).collect(); + if !chunks.is_empty() { + let stored = self.blobs.with_tx(|b| chunks.iter().try_for_each(|c| b.insert_chunk(c))); + if let Err(e) = stored { + warn!("Failed to store proxy body chunks together, so each alone: {e}"); + for chunk in &chunks { + if let Err(e) = self.blobs.with_tx(|b| b.insert_chunk(chunk)) { + warn!("Failed to store proxy body chunk: {e}"); + } + } + } + } + + let mut cleared = Vec::new(); + let written = self.db.with_tx(|ctx| { + let conn = ctx.conn().resolve(); + for FinishedFlow { flow, persisted, .. } in flows { + let id = flow.id.clone(); + conn.execute_batch("SAVEPOINT finished_flow")?; + match self.write_flow(ctx, flow, persisted) { + Ok(true) => conn.execute_batch("RELEASE finished_flow")?, + Ok(false) => { + conn.execute_batch("RELEASE finished_flow")?; + cleared.push(id); + } + Err(e) => { + warn!("Failed to write proxy flow: {e}"); + conn.execute_batch("ROLLBACK TO finished_flow; RELEASE finished_flow")?; + } + } + } + Ok::<_, yaak_database::Error>(()) + }); + if let Err(e) = written { + warn!("Failed to write proxy flows: {e}"); + } + for id in cleared { + self.forget_cleared(&id); } } @@ -2190,7 +2528,7 @@ mod tests { use yaak_proxy_helper::networksetup::{ProxySetting, ServiceProxy, Snapshot}; let dir = tempfile::tempdir().unwrap(); let (events, _rx) = RpcEventEmitter::new(); - let ctx = ProxyCtx::new(dir.path(), events).unwrap(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); assert_eq!(ctx.backend_for(HelperState::Enabled), SystemProxyBackend::Helper); // A restore parked an unplugged adapter's settings, at port 0. @@ -2230,11 +2568,12 @@ mod tests { ) -> (std::sync::mpsc::Sender, std::thread::JoinHandle<()>) { let (tx, rx) = std::sync::mpsc::channel(); let sink = FlowSink { - db: ctx.db.clone(), - blobs: ctx.blobs.clone(), + db: ctx.db().unwrap(), + blobs: ctx.blobs().unwrap(), events: ctx.events.clone(), attribution: Attribution::new(0), recording: ctx.recording.clone(), + finished: Default::default(), }; (tx, std::thread::spawn(move || run_event_loop(rx, sink))) } @@ -2285,7 +2624,7 @@ mod tests { fn wait_for_flow(ctx: &ProxyCtx, check: impl Fn(&Flow) -> bool) -> Flow { let deadline = std::time::Instant::now() + Duration::from_secs(5); loop { - let flows = ctx.db.with_conn(|c| c.find_all::().unwrap()); + let flows = ctx.db().unwrap().with_conn(|c| c.find_all::().unwrap()); if let Some(flow) = flows.into_iter().find(&check) { return flow; } @@ -2301,7 +2640,7 @@ mod tests { use yaak_proxy::{MessageDirection as D, MessageKind as K}; let dir = tempfile::tempdir().unwrap(); let (events, _event_rx) = RpcEventEmitter::new(); - let ctx = ProxyCtx::new(dir.path(), events).unwrap(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); let (tx, event_loop) = event_loop(&ctx); let send = |event| tx.send(event).unwrap(); send(ProxyEvent::RequestStart { @@ -2369,15 +2708,16 @@ mod tests { drop(tx); event_loop.join().unwrap(); - let flow = flows::list(&ctx.db, &Default::default()).unwrap().remove(0); + let flow = flows::list(&ctx.db().unwrap(), &Default::default()).unwrap().remove(0); assert_eq!((flow.message_count, flow.grpc_status), (3, Some(0))); assert_eq!((flow.request_size, flow.response_size), (10, 16), "sizes on the wire"); assert_eq!((flow.request_truncated, flow.response_truncated), (false, true)); - let blobs = ctx.blobs.connect(); - assert!(!blobs.body_exists(&Flow::body_id(&flow.id, FlowBody::Request)).unwrap()); - assert!(!blobs.body_exists(&Flow::body_id(&flow.id, FlowBody::Response)).unwrap()); + let (db, blobs) = (ctx.db().unwrap(), ctx.blobs().unwrap()); + let stored = blobs.connect(); + assert!(!stored.body_exists(&Flow::body_id(&flow.id, FlowBody::Request)).unwrap()); + assert!(!stored.body_exists(&Flow::body_id(&flow.id, FlowBody::Response)).unwrap()); - let read = |which| bodies::read_body(&ctx.db, &ctx.blobs, &flow, which, 1 << 20); + let read = |which| bodies::read_body(&db, &blobs, &flow, which, 1 << 20); let request_body = read(FlowBody::Request).await.unwrap(); assert_eq!((request_body.data.as_slice(), request_body.truncated), (&request[..], false)); // HTTP/2 trailers are no part of the body. @@ -2385,7 +2725,8 @@ mod tests { // Framed with the length it was sent with, so it reads back as it was on the wire, cut. assert_eq!(response_body.data, b"\x00\x00\x00\x00\x0b\x0a\x09Hel"); assert!(response_body.truncated); - let har = har::export(&ctx.db, &ctx.blobs, &[flow], 1 << 20).await.unwrap(); + let har = + har::export(&ctx.db().unwrap(), &ctx.blobs().unwrap(), &[flow], 1 << 20).await.unwrap(); assert_eq!(har.log.entries[0].response.content.size, 10); } @@ -2398,7 +2739,7 @@ mod tests { use yaak_proxy::{MessageDirection as D, MessageKind as K}; let dir = tempfile::tempdir().unwrap(); let (events, _event_rx) = RpcEventEmitter::new(); - let ctx = ProxyCtx::new(dir.path(), events).unwrap(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); let (tx, event_loop) = event_loop(&ctx); let raw = "data: hi\n\n".to_string(); let mut gzip = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); @@ -2462,12 +2803,12 @@ mod tests { drop(tx); event_loop.join().unwrap(); - let flows = flows::list(&ctx.db, &Default::default()).unwrap(); + let flows = flows::list(&ctx.db().unwrap(), &Default::default()).unwrap(); let flow = |host: &str| flows.iter().find(|f| f.url.contains(host)).unwrap().clone(); let (decoded, undecoded) = (flow("a.example"), flow("b.example")); assert_eq!(decoded.message_count, 1); - let read = - |flow| bodies::read_body_or_raw(&ctx.db, &ctx.blobs, flow, FlowBody::Response, 1024); + let (db, blobs) = (ctx.db().unwrap(), ctx.blobs().unwrap()); + let read = |flow| bodies::read_body_or_raw(&db, &blobs, flow, FlowBody::Response, 1024); let body = read(&decoded).await.unwrap(); assert_eq!((body.data.as_slice(), body.decode_error), (raw.as_bytes(), None)); let body = read(&undecoded).await.unwrap(); @@ -2480,7 +2821,7 @@ mod tests { use yaak_proxy::{MessageDirection as D, MessageKind as K}; let dir = tempfile::tempdir().unwrap(); let (events, _event_rx) = RpcEventEmitter::new(); - let ctx = ProxyCtx::new(dir.path(), events).unwrap(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); let (tx, event_loop) = event_loop(&ctx); let send = |event| tx.send(event).unwrap(); let header = |name: &str, value: &str| (name.to_string(), value.to_string()); @@ -2539,15 +2880,21 @@ mod tests { drop(tx); event_loop.join().unwrap(); - let flow = flows::list(&ctx.db, &Default::default()).unwrap().remove(0); + let flow = flows::list(&ctx.db().unwrap(), &Default::default()).unwrap().remove(0); assert_eq!((flow.request_truncated, flow.response_truncated), (true, true)); for (which, kept) in [ (FlowBody::Request, &b"junk"[..]), (FlowBody::Response, b"\x1f\x8bnope"), ] { // The bytes are kept, and read as a cut body is: what decoded of them, cut short. - assert_eq!(bodies::read_raw_body(&ctx.blobs, &flow.id, which).unwrap(), kept); - let body = bodies::read_body(&ctx.db, &ctx.blobs, &flow, which, 1024).await.unwrap(); + assert_eq!( + bodies::read_raw_body(&ctx.blobs().unwrap(), &flow.id, which).unwrap(), + kept + ); + let body = + bodies::read_body(&ctx.db().unwrap(), &ctx.blobs().unwrap(), &flow, which, 1024) + .await + .unwrap(); assert!(body.truncated, "{which:?}"); } } @@ -2559,7 +2906,7 @@ mod tests { use yaak_proxy::{MessageDirection as D, MessageKind as K}; let dir = tempfile::tempdir().unwrap(); let (events, _event_rx) = RpcEventEmitter::new(); - let ctx = ProxyCtx::new(dir.path(), events).unwrap(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); let (tx, event_loop) = event_loop(&ctx); let send = |event| tx.send(event).unwrap(); send(ProxyEvent::RequestStart { @@ -2609,11 +2956,17 @@ mod tests { drop(tx); event_loop.join().unwrap(); - let flow = flows::list(&ctx.db, &Default::default()).unwrap().remove(0); + let flow = flows::list(&ctx.db().unwrap(), &Default::default()).unwrap().remove(0); assert!(flow.request_truncated); - let body = bodies::read_body(&ctx.db, &ctx.blobs, &flow, FlowBody::Request, 1 << 20) - .await - .unwrap(); + let body = bodies::read_body( + &ctx.db().unwrap(), + &ctx.blobs().unwrap(), + &flow, + FlowBody::Request, + 1 << 20, + ) + .await + .unwrap(); assert_eq!((body.data.as_slice(), body.truncated), (&uploaded[..], true)); } @@ -2622,7 +2975,7 @@ mod tests { use yaak_proxy::{MessageDirection as D, MessageKind as K}; let dir = tempfile::tempdir().unwrap(); let (events, event_rx) = RpcEventEmitter::new(); - let ctx = ProxyCtx::new(dir.path(), events).unwrap(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); let (tx, event_loop) = event_loop(&ctx); open_websocket(&tx, 1, "https://example.com/socket"); let big: Vec = (0..10_000u32).map(|i| i as u8).collect(); @@ -2638,9 +2991,9 @@ mod tests { drop(tx); event_loop.join().unwrap(); - let flow = flows::list(&ctx.db, &Default::default()).unwrap().remove(0); + let flow = flows::list(&ctx.db().unwrap(), &Default::default()).unwrap().remove(0); assert_eq!((flow.elapsed_ms, flow.error.as_deref()), (Some(50), None)); - let stored = messages::list(&ctx.db, &flow.id, None, None).unwrap(); + let stored = messages::list(&ctx.db().unwrap(), &flow.id, None, None).unwrap(); let kinds: Vec<_> = stored.iter().map(|m| (m.seq, m.kind, m.direction)).collect(); assert_eq!( kinds, @@ -2657,13 +3010,25 @@ mod tests { (Some(1000), Some("bye")) ); // The large payload is in the blob store, the small ones in their rows. - assert_eq!(messages::payload(&ctx.db, &ctx.blobs, &stored[1]).unwrap(), big); - assert!(ctx.blobs.connect().body_exists(&stored[1].blob_id()).unwrap()); - assert!(!ctx.blobs.connect().body_exists(&stored[0].blob_id()).unwrap()); - assert_eq!(messages::payload(&ctx.db, &ctx.blobs, &stored[0]).unwrap(), b"hi"); + assert_eq!( + messages::payload(&ctx.db().unwrap(), &ctx.blobs().unwrap(), &stored[1]).unwrap(), + big + ); + assert!(ctx.blobs().unwrap().connect().body_exists(&stored[1].blob_id()).unwrap()); + assert!(!ctx.blobs().unwrap().connect().body_exists(&stored[0].blob_id()).unwrap()); + assert_eq!( + messages::payload(&ctx.db().unwrap(), &ctx.blobs().unwrap(), &stored[0]).unwrap(), + b"hi" + ); - let har = - har::export(&ctx.db, &ctx.blobs, std::slice::from_ref(&flow), 1 << 20).await.unwrap(); + let har = har::export( + &ctx.db().unwrap(), + &ctx.blobs().unwrap(), + std::slice::from_ref(&flow), + 1 << 20, + ) + .await + .unwrap(); let har = serde_json::to_value(&har).unwrap(); let sent = &har["log"]["entries"][0]["_webSocketMessages"]; assert_eq!(sent.as_array().unwrap().len(), 2, "data messages only, as devtools exports"); @@ -2675,9 +3040,15 @@ mod tests { ); assert_eq!(sent[1]["data"], BASE64.encode(&big)); // No body after the 101, and it says why. - let body = bodies::read_body(&ctx.db, &ctx.blobs, &flow, FlowBody::Response, 1 << 20) - .await - .unwrap(); + let body = bodies::read_body( + &ctx.db().unwrap(), + &ctx.blobs().unwrap(), + &flow, + FlowBody::Response, + 1 << 20, + ) + .await + .unwrap(); assert_eq!((body.data.len(), body.note.as_deref()), (0, Some(bodies::WEBSOCKET_BODY_NOTE))); let names: Vec<_> = event_rx.try_iter().map(|e| e.event).collect(); @@ -2686,10 +3057,10 @@ mod tests { // Gone from listings at once, and with its messages and their payloads once the // background delete has run. ctx.clear_flows().unwrap(); - assert_eq!(flows::count(&ctx.db).unwrap(), 0); + assert_eq!(flows::count(&ctx.db().unwrap()).unwrap(), 0); wait_for_cleared(&ctx); - assert!(!ctx.blobs.connect().body_exists(&stored[1].blob_id()).unwrap()); - let left: i64 = ctx.db.with_conn(|c| { + assert!(!ctx.blobs().unwrap().connect().body_exists(&stored[1].blob_id()).unwrap()); + let left: i64 = ctx.db().unwrap().with_conn(|c| { c.conn() .resolve() .query_row("SELECT count(*) FROM flow_messages", [], |r| r.get(0)) @@ -2701,7 +3072,7 @@ mod tests { /// Waits for the background delete a clear starts to finish. fn wait_for_cleared(ctx: &ProxyCtx) { let deadline = std::time::Instant::now() + Duration::from_secs(10); - while ctx.db.with_conn(|c| flows::clear_mark(c.conn().resolve())) > 0 { + while ctx.db().unwrap().with_conn(|c| flows::clear_mark(c.conn().resolve())) > 0 { assert!(std::time::Instant::now() < deadline, "cleared flows were not deleted"); std::thread::sleep(Duration::from_millis(10)); } @@ -2712,7 +3083,7 @@ mod tests { use yaak_proxy::{MessageDirection as D, MessageKind as K}; let dir = tempfile::tempdir().unwrap(); let (events, _event_rx) = RpcEventEmitter::new(); - let ctx = ProxyCtx::new(dir.path(), events).unwrap(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); let (tx, event_loop) = event_loop(&ctx); open_websocket(&tx, 1, "https://example.com/socket"); tx.send(ws_message(1, D::Client, K::WebSocketText, b"before")).unwrap(); @@ -2724,18 +3095,18 @@ mod tests { let data = bytes::Bytes::from(vec![b'x'; 2 << 20]); tx.send(ProxyEvent::ResponseBodyChunk { id: 1, len: data.len(), data }).unwrap(); std::thread::sleep(Duration::from_millis(500)); - let blobs = ctx.blobs.connect().list_body_ids().unwrap(); + let blobs = ctx.blobs().unwrap().connect().list_body_ids().unwrap(); assert!(blobs.iter().all(|b| !b.starts_with(&flow.id)), "while open: {blobs:?}"); tx.send(ProxyEvent::StreamEnd { id: 1, elapsed_ms: 9, error: None }).unwrap(); drop(tx); event_loop.join().unwrap(); - assert_eq!(flows::count(&ctx.db).unwrap(), 0, "not brought back by its end"); + assert_eq!(flows::count(&ctx.db().unwrap()).unwrap(), 0, "not brought back by its end"); // The messages written before the clear go with the background delete. wait_for_cleared(&ctx); - assert_eq!(flows::count(&ctx.db).unwrap(), 0, "not brought back by its end"); - assert_eq!(messages::count(&ctx.db, &flow.id).unwrap(), 0); - let blobs = ctx.blobs.connect().list_body_ids().unwrap(); + assert_eq!(flows::count(&ctx.db().unwrap()).unwrap(), 0, "not brought back by its end"); + assert_eq!(messages::count(&ctx.db().unwrap(), &flow.id).unwrap(), 0); + let blobs = ctx.blobs().unwrap().connect().list_body_ids().unwrap(); assert!(blobs.iter().all(|b| !b.starts_with(&flow.id)), "{blobs:?}"); } @@ -2744,7 +3115,7 @@ mod tests { use yaak_proxy::{MessageDirection as D, MessageKind as K}; let dir = tempfile::tempdir().unwrap(); let (events, _event_rx) = RpcEventEmitter::new(); - let ctx = ProxyCtx::new(dir.path(), events).unwrap(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); let (tx, event_loop) = event_loop(&ctx); open_websocket(&tx, 1, "https://example.com/socket"); wait_for_flow(&ctx, |f| f.stream_kind.is_some()); @@ -2767,7 +3138,7 @@ mod tests { fn an_open_call_shows_its_response_before_any_message() { let dir = tempfile::tempdir().unwrap(); let (events, _event_rx) = RpcEventEmitter::new(); - let ctx = ProxyCtx::new(dir.path(), events).unwrap(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); let (tx, event_loop) = event_loop(&ctx); tx.send(ProxyEvent::RequestStart { id: 1, @@ -2805,17 +3176,46 @@ mod tests { event_loop.join().unwrap(); } + /// A stop that comes while a start waits for the databases, as it does through a + /// conversion, wins: the start does not go on to capture. + #[test] + fn a_stop_while_starting_waits_is_kept() { + let dir = tempfile::tempdir().unwrap(); + let settings = + ProxySettings { port: 0, system_proxy_at_launch: false, ..Default::default() }; + settings.save(&dir.path().join("settings.json")).unwrap(); + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + + // Held as a conversion holds it, so the start waits. + let unlocked = ctx.unlocked.lock().unwrap(); + let starting = std::thread::spawn({ + let ctx = ctx.clone(); + move || ctx.start() + }); + std::thread::sleep(Duration::from_millis(100)); + ctx.stop().unwrap(); + drop(unlocked); + starting.join().unwrap().unwrap(); + assert_eq!(ctx.port(), None); + + // A start after the stop starts. + ctx.start().unwrap(); + assert!(ctx.port().is_some()); + ctx.stop().unwrap(); + } + #[test] fn streams_still_open_end_when_the_engine_stops() { let dir = tempfile::tempdir().unwrap(); let (events, _event_rx) = RpcEventEmitter::new(); - let ctx = ProxyCtx::new(dir.path(), events).unwrap(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); let (tx, event_loop) = event_loop(&ctx); open_websocket(&tx, 1, "https://example.com/socket"); wait_for_flow(&ctx, |f| f.stream_kind.is_some()); drop(tx); event_loop.join().unwrap(); - let flow = flows::list(&ctx.db, &Default::default()).unwrap().remove(0); + let flow = flows::list(&ctx.db().unwrap(), &Default::default()).unwrap().remove(0); assert_eq!(flow.error.as_deref(), Some(STOPPED_MID_STREAM)); } @@ -2831,9 +3231,9 @@ mod tests { }], ..Default::default() }; - ctx.db.with_tx(|tx| tx.upsert(&flow, &UpdateSource::Background)).unwrap(); + ctx.db().unwrap().with_tx(|tx| tx.upsert(&flow, &UpdateSource::Background)).unwrap(); let mut writer = - BodyChunkWriter::new(ctx.blobs.clone(), Flow::body_id(id, FlowBody::Response)); + BodyChunkWriter::new(ctx.blobs().unwrap(), Flow::body_id(id, FlowBody::Response)); writer.write(body).unwrap(); writer.finish().unwrap(); } @@ -2843,7 +3243,7 @@ mod tests { async fn filter_reads_the_body_in_its_charset() { let dir = tempfile::tempdir().unwrap(); let (events, _rx) = RpcEventEmitter::new(); - let ctx = ProxyCtx::new(dir.path(), events).unwrap(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); let xml = "ZürichKraków"; let utf16: Vec = xml.encode_utf16().flat_map(u16::to_le_bytes).collect(); @@ -2874,7 +3274,7 @@ mod tests { async fn the_app_still_hears_why_a_body_failed_to_decode() { let dir = tempfile::tempdir().unwrap(); let (events, _rx) = RpcEventEmitter::new(); - let ctx = ProxyCtx::new(dir.path(), events).unwrap(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); let flow = Flow { id: "fl_bad".into(), status: Some(200), @@ -2884,9 +3284,9 @@ mod tests { }], ..Default::default() }; - ctx.db.with_tx(|tx| tx.upsert(&flow, &UpdateSource::Background)).unwrap(); + ctx.db().unwrap().with_tx(|tx| tx.upsert(&flow, &UpdateSource::Background)).unwrap(); let mut writer = - BodyChunkWriter::new(ctx.blobs.clone(), Flow::body_id("fl_bad", FlowBody::Response)); + BodyChunkWriter::new(ctx.blobs().unwrap(), Flow::body_id("fl_bad", FlowBody::Response)); writer.write(b"not a gzip stream").unwrap(); writer.finish().unwrap(); @@ -2917,4 +3317,219 @@ mod tests { }; assert!(filter_body(ctx, req).await.is_err()); } + + /// Sends one whole flow for `url`, with a response body, as the engine does. + fn send_flow(tx: &std::sync::mpsc::Sender, id: u64, url: &str) { + tx.send(ProxyEvent::RequestStart { + id, + started_at: SystemTime::now(), + method: "GET".into(), + url: url.into(), + http_version: "HTTP/1.1".into(), + connection: 1, + client_addr: "127.0.0.1:1".parse().unwrap(), + }) + .unwrap(); + tx.send(ProxyEvent::ResponseStart { + id, + status: 200, + http_version: "HTTP/1.1".into(), + remote_addr: None, + elapsed_ms: 1, + }) + .unwrap(); + let data = bytes::Bytes::from(format!("body of {url}")); + tx.send(ProxyEvent::ResponseBodyChunk { id, len: data.len(), data }).unwrap(); + tx.send(ProxyEvent::ResponseBodyComplete { + id, + size: 0, + truncated: false, + elapsed_ms: 2, + error: None, + cancelled: false, + }) + .unwrap(); + } + + /// Flows that end together are written together, and every one the engine sent is written + /// when it stops, mid-batch or not, each with its body. + #[test] + fn a_stop_mid_batch_writes_every_flow() { + let dir = tempfile::tempdir().unwrap(); + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + let (tx, event_loop) = event_loop(&ctx); + // More than a batch, sent faster than they are written, then the engine stops. + for id in 1..=(FLOW_BATCH as u64 * 2 + 7) { + send_flow(&tx, id, &format!("https://example.com/{id}")); + } + drop(tx); + event_loop.join().unwrap(); + + let stored = flows::list(&ctx.db().unwrap(), &Default::default()).unwrap(); + assert_eq!(stored.len(), FLOW_BATCH * 2 + 7); + for flow in &stored { + let body = + bodies::read_raw_body(&ctx.blobs().unwrap(), &flow.id, FlowBody::Response).unwrap(); + assert_eq!(body, format!("body of {}", flow.url).into_bytes()); + } + } + + /// A flow that ends alone is written at once, not after the batch's wait. + #[test] + fn a_lone_flow_shows_at_once() { + let dir = tempfile::tempdir().unwrap(); + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + let (tx, event_loop) = event_loop(&ctx); + let sent = std::time::Instant::now(); + send_flow(&tx, 1, "https://example.com/alone"); + wait_for_flow(&ctx, |f| f.url == "https://example.com/alone"); + let shown = sent.elapsed(); + assert!(shown < Duration::from_millis(50), "shown after {shown:?}"); + drop(tx); + event_loop.join().unwrap(); + } + + /// A flow that fails to write is rolled back alone: the others in its batch are kept. + #[test] + fn one_bad_flow_loses_no_others_in_its_batch() { + let dir = tempfile::tempdir().unwrap(); + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + ctx.db() + .unwrap() + .with_tx(|tx| { + tx.conn().resolve().execute_batch( + "CREATE TEMP TRIGGER refuse BEFORE INSERT ON flows + WHEN NEW.url = 'https://example.com/bad' + BEGIN SELECT RAISE(ABORT, 'refused'); END;", + )?; + Ok::<_, yaak_database::Error>(()) + }) + .unwrap(); + let sink = FlowSink { + db: ctx.db().unwrap(), + blobs: ctx.blobs().unwrap(), + events: ctx.events.clone(), + attribution: Attribution::new(0), + recording: ctx.recording.clone(), + finished: Default::default(), + }; + // One batch of three, the middle one refused. + for url in ["before", "bad", "after"] { + let flow = Flow { + id: Flow::generate_id(), + url: format!("https://example.com/{url}"), + ..Default::default() + }; + let chunks = vec![yaak_models::blob_manager::BodyChunk::new( + Flow::body_id(&flow.id, FlowBody::Response), + 0, + url.as_bytes().to_vec(), + )]; + sink.finished.borrow_mut().flows.push(FinishedFlow { flow, persisted: false, chunks }); + } + sink.write_finished(); + assert!(!sink.has_finished()); + + let mut urls: Vec = flows::list(&ctx.db().unwrap(), &Default::default()) + .unwrap() + .into_iter() + .map(|f| f.url) + .collect(); + urls.sort(); + assert_eq!(urls, ["https://example.com/after", "https://example.com/before"]); + } + + /// Capture throughput and flow-open latency at 3,000 flows, for weighing what encryption + /// costs. Run in release: `cargo test --release -p yaak-proxy-lib --lib times_capture -- + /// --ignored --nocapture`. + #[test] + #[ignore] + fn times_capture_and_flow_open() { + const FLOWS: u64 = 3_000; + let dir = tempfile::tempdir().unwrap(); + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + let (tx, event_loop) = event_loop(&ctx); + let body = bytes::Bytes::from(vec![b'x'; 2048]); + let started = std::time::Instant::now(); + for id in 1..=FLOWS { + tx.send(ProxyEvent::RequestStart { + id, + started_at: SystemTime::now(), + method: "GET".into(), + url: format!("https://api.example.com/v1/items/{id}?api_key=sk_test_123"), + http_version: "HTTP/1.1".into(), + // One kept-alive connection, so its app is looked up once and the time is + // the database's. + connection: 1, + client_addr: "127.0.0.1:1".parse().unwrap(), + }) + .unwrap(); + for h in 0..8 { + let (name, value) = (format!("x-header-{h}"), "a header value".into()); + tx.send(ProxyEvent::RequestHeader { id, name, value }).unwrap(); + } + tx.send(ProxyEvent::ResponseStart { + id, + status: 200, + http_version: "HTTP/1.1".into(), + remote_addr: None, + elapsed_ms: 1, + }) + .unwrap(); + let (name, value) = ("content-type".into(), "application/json".into()); + tx.send(ProxyEvent::ResponseHeader { id, name, value }).unwrap(); + let data = body.clone(); + tx.send(ProxyEvent::ResponseBodyChunk { id, len: data.len(), data }).unwrap(); + tx.send(ProxyEvent::ResponseBodyComplete { + id, + size: body.len() as u64, + truncated: false, + elapsed_ms: 2, + error: None, + cancelled: false, + }) + .unwrap(); + } + drop(tx); + event_loop.join().unwrap(); + let capture = started.elapsed(); + let db = ctx.db().unwrap(); + let blobs = ctx.blobs().unwrap(); + assert_eq!(flows::count(&db).unwrap(), FLOWS as usize); + + let ids: Vec = + flows::list(&db, &Default::default()).unwrap().into_iter().map(|f| f.id).collect(); + let started = std::time::Instant::now(); + for id in &ids { + let flow = flows::get(&db, id).unwrap(); + let body = bodies::read_raw_body(&blobs, &flow.id, FlowBody::Response).unwrap(); + assert_eq!(body.len(), 2048); + } + let open = started.elapsed() / ids.len() as u32; + let cipher: Vec = db.with_conn(|c| { + [ + "cipher_provider", + "cipher_provider_version", + "cipher_memory_security", + ] + .iter() + .map(|p| { + let value = c.conn().resolve().query_row(&format!("PRAGMA {p}"), [], |r| { + r.get::<_, rusqlite::types::Value>(0) + }); + format!("{p} {value:?}") + }) + .collect() + }); + eprintln!("{cipher:?}"); + eprintln!( + "{FLOWS} flows captured in {capture:?}: {:.0} flows/s; opening one, its row and \ + body: {open:?}", + FLOWS as f64 / capture.as_secs_f64() + ); + } } diff --git a/crates-proxy/yaak-proxy-lib/tests/at_rest.rs b/crates-proxy/yaak-proxy-lib/tests/at_rest.rs new file mode 100644 index 000000000..3c5dd487c --- /dev/null +++ b/crates-proxy/yaak-proxy-lib/tests/at_rest.rs @@ -0,0 +1,531 @@ +//! Captured traffic encrypted at rest: `proxy.db` and `blobs.sqlite` with SQLCipher, keyed from +//! a data key the master key in the keychain wraps, and `ca.key` with the data key. + +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use yaak_crypto::data_key::DataKey; +use yaak_database::UpdateSource; +use yaak_database::encryption::{FileState, apply_key, file_state}; +use yaak_models::blob_manager::BodyChunkWriter; +use yaak_proxy::CaStore; +use yaak_proxy_lib::bodies::read_raw_body; +use yaak_proxy_lib::client::ProxyConnection; +use yaak_proxy_lib::db::{BLOBS_DB, DATA_KEY, PROXY_DB, database_key}; +use yaak_proxy_lib::flows::{self, FlowFilter}; +use yaak_proxy_lib::models::{Flow, FlowBody, ProxyHeader}; +use yaak_proxy_lib::settings::ProxySettings; +use yaak_proxy_lib::{MasterKeySource, ProxyCtx}; +use yaak_rpc::RpcEventEmitter; + +const HEADER_SECRET: &str = "sk_live_HEADERSECRET123"; +const QUERY_SECRET: &str = "QUERYSECRET456"; +const BODY_SECRET: &str = "BODYSECRET789"; +const FLOWS: usize = 3_000; + +fn open_ctx(dir: &Path, keys: MasterKeySource) -> ProxyCtx { + let settings = ProxySettings { port: 0, system_proxy_at_launch: false, ..Default::default() }; + settings.save(&dir.join("settings.json")).unwrap(); + let (events, _rx) = RpcEventEmitter::new(); + ProxyCtx::new(dir, events, keys).unwrap() +} + +/// `n` flows, each with a bearer token in a header, an API key in its query and a token in a +/// 1 KB response body. +fn write_flows(ctx: &ProxyCtx, n: usize) { + let flows: Vec = (0..n) + .map(|i| Flow { + id: format!("fl_{i:05}"), + method: if i % 3 == 0 { "POST" } else { "GET" }.into(), + url: format!("https://api.example.com/v1/items/{i}?api_key={QUERY_SECRET}"), + status: Some(if i % 10 == 0 { 404 } else { 200 }), + content_type: Some("application/json".into()), + request_headers: vec![ProxyHeader { + name: "authorization".into(), + value: format!("Bearer {HEADER_SECRET}"), + }], + started_at: chrono::DateTime::from_timestamp(1_700_000_000 + i as i64, 0) + .unwrap() + .naive_utc(), + elapsed_ms: Some(5), + ..Default::default() + }) + .collect(); + ctx.db() + .unwrap() + .with_tx(|tx| { + flows.iter().try_for_each(|f| tx.upsert(f, &UpdateSource::Background).map(|_| ())) + }) + .unwrap(); + for flow in &flows { + let mut writer = + BodyChunkWriter::new(ctx.blobs().unwrap(), Flow::body_id(&flow.id, FlowBody::Response)); + let body = format!("{{\"token\":\"{BODY_SECRET}\",\"pad\":\"{}\"}}", "x".repeat(1000)); + writer.write(body.as_bytes()).unwrap(); + writer.finish().unwrap(); + } +} + +/// Every file under `dir`, recursively. +fn files(dir: &Path) -> Vec { + let mut found = vec![]; + for entry in std::fs::read_dir(dir).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + found.extend(files(&path)); + } else { + found.push(path); + } + } + found +} + +/// The files under `dir` that hold `needle` as it is. One gone by the time it is read, a WAL +/// its last connection closed meanwhile, holds nothing. +fn holding(dir: &Path, needle: &str) -> Vec { + files(dir) + .into_iter() + .filter(|p| { + std::fs::read(p) + .map(|b| b.windows(needle.len()).any(|w| w == needle.as_bytes())) + .unwrap_or(false) + }) + .collect() +} + +fn assert_no_plaintext(dir: &Path) { + for secret in [HEADER_SECRET, QUERY_SECRET, BODY_SECRET, "PRIVATE KEY"] { + assert_eq!(holding(dir, secret), Vec::::new(), "{secret} in plaintext"); + } +} + +/// `dir`'s databases as a build from before encryption left them: the same tables and rows, +/// plain, with a plaintext CA key. +fn make_plain(dir: &Path, keys: &MasterKeySource) { + let data_key = DataKey::load_file(keys, &dir.join(DATA_KEY)).unwrap().unwrap(); + let key = database_key(&data_key); + for name in [PROXY_DB, BLOBS_DB] { + let path = dir.join(name); + let plain = dir.join(format!("{name}.plain-export")); + let conn = rusqlite::Connection::open(&path).unwrap(); + apply_key(&conn, &key).unwrap(); + conn.execute("ATTACH DATABASE ?1 AS plain KEY ''", [plain.to_string_lossy()]).unwrap(); + conn.query_row("SELECT sqlcipher_export('plain')", [], |_| Ok(())).unwrap(); + conn.execute_batch("DETACH DATABASE plain").unwrap(); + drop(conn); + for suffix in ["", "-wal", "-shm"] { + let _ = std::fs::remove_file(dir.join(format!("{name}{suffix}"))); + } + std::fs::rename(&plain, &path).unwrap(); + assert_eq!(file_state(&path).unwrap(), FileState::Plain); + } + std::fs::remove_file(dir.join(DATA_KEY)).unwrap(); + // The CA as it was written before: its key in plaintext PEM. + let ca_dir = dir.join("ca"); + std::fs::remove_dir_all(&ca_dir).ok(); + CaStore::load_or_create(&ca_dir).unwrap(); +} + +/// A data directory with `FLOWS` flows in plain databases, as before encryption. +fn plain_capture() -> tempfile::TempDir { + let dir = tempfile::tempdir().unwrap(); + let keys = MasterKeySource::memory(); + let ctx = open_ctx(dir.path(), keys.clone()); + write_flows(&ctx, FLOWS); + drop(ctx); + make_plain(dir.path(), &keys); + assert!(!holding(dir.path(), BODY_SECRET).is_empty()); + assert!(!holding(dir.path(), "PRIVATE KEY").is_empty()); + dir +} + +#[test] +fn stores_nothing_in_plaintext_and_reads_back_with_the_key() { + let dir = tempfile::tempdir().unwrap(); + let keys = MasterKeySource::memory(); + let ctx = open_ctx(dir.path(), keys.clone()); + ctx.load_ca().unwrap(); + write_flows(&ctx, 20); + // Through the WAL and checkpointed alike. + assert_no_plaintext(dir.path()); + for name in [PROXY_DB, BLOBS_DB] { + assert_eq!(file_state(&dir.path().join(name)).unwrap(), FileState::Encrypted); + } + drop(ctx); + assert_no_plaintext(dir.path()); + + // Opened again with the key, everything is there. + let ctx = open_ctx(dir.path(), keys.clone()); + assert_eq!(flows::count(&ctx.db().unwrap()).unwrap(), 20); + let body = read_raw_body(&ctx.blobs().unwrap(), "fl_00007", FlowBody::Response).unwrap(); + assert!(String::from_utf8(body).unwrap().contains(BODY_SECRET)); + drop(ctx); + + // Read offline, as the CLI does, with the same keychain. + let offline = ProxyConnection::Offline( + yaak_proxy_lib::client::OfflineStore::open(dir.path(), &keys).unwrap(), + ); + let rt = tokio::runtime::Runtime::new().unwrap(); + let listed = rt.block_on(offline.flows(&FlowFilter::default())).unwrap(); + assert_eq!(listed.len(), 20); + + // Without the master key that wraps it, nothing opens. + let err = yaak_proxy_lib::client::OfflineStore::open(dir.path(), &MasterKeySource::memory()) + .err() + .unwrap(); + assert!(err.contains("The encryption key is missing from the keychain"), "{err}"); + let err = yaak_proxy_lib::client::OfflineStore::open( + dir.path(), + &MasterKeySource::refusing("User canceled the operation"), + ) + .err() + .unwrap(); + assert!(err.contains("couldn't read its key from the keychain"), "{err}"); + // Nor without any key at all. + let raw = rusqlite::Connection::open(dir.path().join(PROXY_DB)).unwrap(); + let err = raw.query_row("SELECT count(*) FROM flows", [], |r| r.get::<_, i64>(0)).unwrap_err(); + assert!(err.to_string().contains("file is not a database"), "{err}"); +} + +/// A capture from before encryption, 3,000 flows with bodies, is converted on first launch: +/// the engine starts once it is done, the app hears how far it has got, nothing is lost and no +/// secret is left readable in any file. +#[test] +fn converts_a_plain_capture_on_first_launch() { + let dir = plain_capture(); + let pem_before = std::fs::read_to_string(dir.path().join("ca/ca.pem")).unwrap(); + let keys = MasterKeySource::memory(); + // Flows deleted since, as a clear leaves gaps in their numbers. + let plain = rusqlite::Connection::open(dir.path().join(PROXY_DB)).unwrap(); + let deleted = plain.execute("DELETE FROM flows WHERE id < 'fl_00010'", []).unwrap(); + let number: i64 = + plain.query_row("SELECT rowid FROM flows WHERE id = 'fl_01234'", [], |r| r.get(0)).unwrap(); + drop(plain); + + let ctx = open_ctx(dir.path(), keys.clone()); + let seen = Arc::new(Mutex::new(Vec::::new())); + let watching = Arc::new(AtomicBool::new(true)); + let watcher = std::thread::spawn({ + let (ctx, seen, watching) = (ctx.clone(), seen.clone(), watching.clone()); + move || { + while watching.load(Ordering::Relaxed) { + if let Some(fraction) = ctx.encrypting() { + seen.lock().unwrap().push(fraction); + } + std::thread::sleep(Duration::from_micros(200)); + } + } + }); + // Waits for the conversion, then starts. + ctx.start().unwrap(); + watching.store(false, Ordering::Relaxed); + watcher.join().unwrap(); + assert!(ctx.port().is_some()); + assert_eq!(ctx.encrypting(), None); + let seen = seen.lock().unwrap(); + assert!(!seen.is_empty(), "no progress shown"); + assert!(seen.iter().all(|f| (0.0..=1.0).contains(f)), "{seen:?}"); + + for name in [PROXY_DB, BLOBS_DB] { + assert_eq!(file_state(&dir.path().join(name)).unwrap(), FileState::Encrypted); + } + assert!(dir.path().join(DATA_KEY).exists()); + let leftovers: Vec<_> = files(dir.path()) + .into_iter() + .filter(|p| { + let name = p.file_name().unwrap().to_string_lossy(); + name.ends_with("-plain") || name.ends_with("-encrypting") + }) + .collect(); + assert!(leftovers.is_empty(), "{leftovers:?}"); + + // Every flow and body came across, each flow under its own number. + assert_eq!(flows::count(&ctx.db().unwrap()).unwrap(), FLOWS - deleted); + let body = read_raw_body(&ctx.blobs().unwrap(), "fl_02999", FlowBody::Response).unwrap(); + assert!(String::from_utf8(body).unwrap().contains(BODY_SECRET)); + let flow = flows::get(&ctx.db().unwrap(), "fl_01234").unwrap(); + assert_eq!(flow.request_headers[0].value, format!("Bearer {HEADER_SECRET}")); + assert_eq!(flow.number, number); + // The same CA, so it stays trusted, its key now encrypted. + let ca = ctx.load_ca().unwrap(); + assert_eq!(ca.pem(), pem_before); + + ctx.stop().unwrap(); + drop(ctx); + assert_no_plaintext(dir.path()); +} + +/// Search, the `q` filter, paging and the sidebar's filters read the encrypted database as +/// they read a plain one, and a full-text table works inside it. +#[test] +fn search_reaches_inside_the_encrypted_database() { + let dir = plain_capture(); + let ctx = open_ctx(dir.path(), MasterKeySource::memory()); + ctx.load_ca().unwrap(); + let db = ctx.db().unwrap(); + + // Query strings are searchable, being inside the encrypted file. + let q = |q: &str| FlowFilter { q: Some(q.into()), limit: Some(50), ..Default::default() }; + let (page, total, matching) = flows::page(&db, &q(QUERY_SECRET)).unwrap(); + assert_eq!((page.len(), total, matching), (50, FLOWS, Some(FLOWS))); + let (page, _, matching) = flows::page(&db, &q("items/2999")).unwrap(); + assert_eq!((page[0].id.as_str(), matching), ("fl_02999", Some(1))); + let posts_not_found = FlowFilter { + method: Some("post".into()), + status: Some("4xx".into()), + limit: Some(1), + ..Default::default() + }; + let (_, _, matching) = flows::page(&db, &posts_not_found).unwrap(); + assert_eq!(matching, Some((0..FLOWS).filter(|i| i % 3 == 0 && i % 10 == 0).count())); + + // Paging back from the newest, as the list does. + let mut before = None; + let mut seen = 0; + loop { + let filter = FlowFilter { limit: Some(500), before: before.clone(), ..Default::default() }; + let (page, _, _) = flows::page(&db, &filter).unwrap(); + if page.is_empty() { + break; + } + seen += page.len(); + before = page.last().map(|f| f.id.clone()); + } + assert_eq!(seen, FLOWS); + + // The sidebar's site and path filters, which run the database's own functions. + let under = |u: &str| FlowFilter { under: Some(vec![u.into()]), ..Default::default() }; + assert_eq!(flows::list(&db, &under("api.example.com")).unwrap().len(), FLOWS); + assert_eq!(flows::list(&db, &under("*.example.com")).unwrap().len(), FLOWS); + assert_eq!(flows::list(&db, &under("api.example.com/v1/items/42")).unwrap().len(), 1); + + // A full-text table, as body search will want, made and searched in the encrypted file. + let hits: i64 = db + .with_tx(|tx| { + let conn = tx.conn().resolve(); + conn.execute_batch(&format!( + "CREATE VIRTUAL TABLE fts_probe USING fts5(body); + INSERT INTO fts_probe VALUES ('{{\"token\":\"{BODY_SECRET}\"}}'), ('nothing');" + ))?; + let hits = conn.query_row( + &format!( + "SELECT count(*) FROM fts_probe WHERE fts_probe MATCH '\"{BODY_SECRET}\"'" + ), + [], + |r| r.get(0), + )?; + conn.execute_batch("DROP TABLE fts_probe")?; + Ok::<_, yaak_database::Error>(hits) + }) + .unwrap(); + assert_eq!(hits, 1); + drop((db, ctx)); + assert_no_plaintext(dir.path()); +} + +/// Two first launches at once, as the app and the CLI daemon might: one data key, one +/// conversion, one CA, and both read everything. +#[test] +fn concurrent_first_unlocks_agree() { + let dir = plain_capture(); + let keys = MasterKeySource::memory(); + let barrier = std::sync::Barrier::new(2); + let pems: Vec = std::thread::scope(|s| { + let handles: Vec<_> = (0..2) + .map(|_| { + s.spawn(|| { + barrier.wait(); + let ctx = open_ctx(dir.path(), keys.clone()); + let pem = ctx.load_ca().unwrap().pem().to_string(); + assert_eq!(flows::count(&ctx.db().unwrap()).unwrap(), FLOWS); + pem + }) + }) + .collect(); + handles.into_iter().map(|h| h.join().unwrap()).collect() + }); + assert_eq!(pems[0], pems[1]); + let ctx = open_ctx(dir.path(), keys); + assert_eq!(flows::count(&ctx.db().unwrap()).unwrap(), FLOWS); + drop(ctx); + assert_no_plaintext(dir.path()); +} + +/// A keychain that refuses leaves capture refused, saying why, and touches nothing: no key or +/// database is made, and traffic from before stays as it was until the key can be read. +#[test] +fn a_refused_key_refuses_capture() { + let refusing = MasterKeySource::refusing("User canceled the operation"); + let empty = tempfile::tempdir().unwrap(); + let ctx = open_ctx(empty.path(), refusing.clone()); + let err = ctx.start().unwrap_err(); + assert_eq!( + err, + "Yaak Proxy stores traffic encrypted and couldn't read its key from the keychain: \ + Encryption error: User canceled the operation. Allow it access, then retry." + ); + assert_eq!(ctx.start_error().as_deref(), Some(err.as_str())); + assert_eq!(ctx.port(), None); + assert!(ctx.load_ca().is_err()); + // Nothing is open: what reads or writes captured traffic is refused, saying why. + let locked = ctx.db().err().unwrap(); + assert_eq!(locked.to_string(), format!("Traffic is locked: {err}")); + assert!(ctx.blobs().is_err()); + let lock = ctx.locked().unwrap(); + assert!(lock.keychain); + assert_eq!(lock.message, locked.to_string()); + let made: Vec<_> = files(empty.path()) + .into_iter() + .filter(|p| p.extension().is_none_or(|e| e != "lock") && !p.ends_with("settings.json")) + .collect(); + assert!(made.is_empty(), "{made:?}"); + + let dir = plain_capture(); + let ctx = open_ctx(dir.path(), refusing); + let refused = ctx.start().unwrap_err(); + assert_eq!(ctx.encrypting(), None); + for name in [PROXY_DB, BLOBS_DB] { + assert_eq!(file_state(&dir.path().join(name)).unwrap(), FileState::Plain); + } + // Clear is refused too, so the captured flows are not left to come back. + assert_eq!(ctx.clear_flows().unwrap_err(), format!("Traffic is locked: {refused}")); + // Retrying with access allowed converts it then, every flow still there. + drop(ctx); + let ctx = open_ctx(dir.path(), MasterKeySource::memory()); + ctx.start().unwrap(); + assert_eq!(flows::count(&ctx.db().unwrap()).unwrap(), FLOWS); + // And clears them now. + assert_eq!(ctx.clear_flows().unwrap(), FLOWS); + assert_eq!(flows::count(&ctx.db().unwrap()).unwrap(), 0); +} + +/// A build from before encryption that owns the engine, a CLI daemon, keeps its plain databases +/// until it lets go: converting them under it would pull them away from its connections. +#[cfg(unix)] +#[test] +fn waits_for_an_older_owner_before_converting() { + use yaak_proxy_lib::control::{ControlInfo, Owner, control_path}; + let dir = plain_capture(); + let ca_key = std::fs::read(dir.path().join("ca/ca.key")).unwrap(); + // Another live process owns it: this one's parent, answering on a port of its own. + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let owner = ControlInfo { + pid: std::os::unix::process::parent_id(), + control_port: listener.local_addr().unwrap().port(), + proxy_port: None, + token: "tok".into(), + owner: Owner::Cli, + started_at: chrono::Utc::now(), + }; + let control = control_path(dir.path()); + std::fs::write(&control, serde_json::to_string(&owner).unwrap()).unwrap(); + + let ctx = open_ctx(dir.path(), MasterKeySource::memory()); + assert!(ctx.load_ca().err().unwrap().contains("can't be encrypted yet")); + assert!(ctx.start().unwrap_err().starts_with("The proxy is already running in the yaak CLI")); + for name in [PROXY_DB, BLOBS_DB] { + assert_eq!(file_state(&dir.path().join(name)).unwrap(), FileState::Plain); + } + assert_eq!(std::fs::read(dir.path().join("ca/ca.key")).unwrap(), ca_key, "CA left as it was"); + + // Once it has let go, starting converts them. + drop(listener); + std::fs::remove_file(&control).unwrap(); + ctx.start().unwrap(); + assert_eq!(flows::count(&ctx.db().unwrap()).unwrap(), FLOWS); + for name in [PROXY_DB, BLOBS_DB] { + assert_eq!(file_state(&dir.path().join(name)).unwrap(), FileState::Encrypted); + } +} + +/// The data key's file gone from beside encrypted databases is refused too, rather than a new +/// key being made that could open nothing and would replace the trusted CA. +#[test] +fn a_lost_data_key_is_not_replaced() { + let dir = tempfile::tempdir().unwrap(); + let keys = MasterKeySource::memory(); + let ctx = open_ctx(dir.path(), keys.clone()); + let pem = ctx.load_ca().unwrap().pem().to_string(); + write_flows(&ctx, 3); + drop(ctx); + + std::fs::remove_file(dir.path().join(DATA_KEY)).unwrap(); + let ctx = open_ctx(dir.path(), keys); + let err = ctx.start().unwrap_err(); + assert!(err.starts_with("The proxy's captured traffic is encrypted, but its key is missing")); + assert!(!dir.path().join(DATA_KEY).exists()); + assert_eq!(std::fs::read_to_string(dir.path().join("ca/ca.pem")).unwrap(), pem); + assert!(ctx.db().is_err(), "locked"); + assert!(!ctx.locked().unwrap().keychain, "not the keychain's doing"); +} + +/// While locked, every RPC that reads or writes captured traffic is refused with why, and so is +/// the control API, with 503; the proxy's state says so, and starting, once the key can be +/// read, opens the databases. +#[tokio::test(flavor = "multi_thread")] +async fn locked_rpcs_and_control_api_refuse() { + let dir = plain_capture(); + let ctx = open_ctx(dir.path(), MasterKeySource::refusing("User canceled the operation")); + let router = yaak_proxy_lib::build_router(); + let rpc = |cmd: &'static str, payload: serde_json::Value| { + let (router, ctx) = (&router, ctx.clone()); + async move { router.dispatch(cmd, payload, &ctx).await.map_err(|e| e.message) } + }; + let flow_id = "fl_00001"; + for (cmd, payload) in [ + ("list_flows", serde_json::json!({})), + ("clear_flows", serde_json::json!({})), + ("list_messages", serde_json::json!({ "flowId": flow_id })), + ("get_flow_body", serde_json::json!({ "flowId": flow_id, "which": "response" })), + ("preview_save", serde_json::json!({ "flowId": flow_id })), + ("list_grpc_schemas", serde_json::json!({})), + ("remove_grpc_schema", serde_json::json!({ "id": "gs_1" })), + ("execute_action", serde_json::json!({ "scope": "global", "action": "flows_clear" })), + ] { + let err = rpc(cmd, payload).await.unwrap_err(); + assert!(err.starts_with("Traffic is locked: "), "{cmd}: {err}"); + } + let state = rpc("get_proxy_state", serde_json::json!({})).await.unwrap(); + assert_eq!(state["locked"]["keychain"], true); + assert!(state["locked"]["message"].as_str().unwrap().starts_with("Traffic is locked: ")); + + ctx.serve_control_api().unwrap(); + let info = ctx.control_info().unwrap(); + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + for path in ["/flows", &format!("/flows/{flow_id}"), "/har"] { + let resp = http + .get(format!("{}{path}", info.base_url())) + .bearer_auth(&info.token) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 503, "{path}"); + assert!(resp.text().await.unwrap().contains("Traffic is locked: "), "{path}"); + } + let cleared = http + .post(format!("{}/flows/clear", info.base_url())) + .bearer_auth(&info.token) + .send() + .await + .unwrap(); + assert_eq!(cleared.status(), 503); + // The status still answers, saying why. + let status: serde_json::Value = http + .get(format!("{}/status", info.base_url())) + .bearer_auth(&info.token) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(status["flowCount"], 0); + assert!(status["locked"].as_str().unwrap().starts_with("Traffic is locked: ")); + ctx.stop_control_api(); + // Nothing was cleared or converted meanwhile. + for name in [PROXY_DB, BLOBS_DB] { + assert_eq!(file_state(&dir.path().join(name)).unwrap(), FileState::Plain); + } +} diff --git a/crates-proxy/yaak-proxy-lib/tests/control.rs b/crates-proxy/yaak-proxy-lib/tests/control.rs index e5a402ad3..ba22e7fca 100644 --- a/crates-proxy/yaak-proxy-lib/tests/control.rs +++ b/crates-proxy/yaak-proxy-lib/tests/control.rs @@ -20,7 +20,7 @@ use yaak_proxy_lib::flows::{BodyEncoding, FlowFilter}; use yaak_proxy_lib::models::{Flow, FlowBody}; use yaak_proxy_lib::save::SaveFlowRequest; use yaak_proxy_lib::settings::ProxySettings; -use yaak_proxy_lib::{ProxyCtx, ProxyState}; +use yaak_proxy_lib::{MasterKeySource, ProxyCtx, ProxyState}; use yaak_rpc::RpcEventEmitter; async fn route(_req: Request) -> Result>, Infallible> { @@ -44,12 +44,18 @@ async fn spawn_upstream() -> SocketAddr { addr } +/// The keychain every context and reader here shares, as the app and the CLI share one. +fn keys() -> MasterKeySource { + static KEYS: std::sync::OnceLock = std::sync::OnceLock::new(); + KEYS.get_or_init(MasterKeySource::memory).clone() +} + /// A context over `dir` whose engine picks a free port, saving into Yaak's data in `client`. fn ctx(dir: &Path, client: &Path) -> ProxyCtx { let settings = ProxySettings { port: 0, system_proxy_at_launch: false, ..Default::default() }; settings.save(&dir.join("settings.json")).unwrap(); let (events, _rx) = RpcEventEmitter::new(); - ProxyCtx::new(dir, events).unwrap().with_client_data_dir(client) + ProxyCtx::new(dir, events, keys()).unwrap().with_client_data_dir(client) } /// Sends one POST through the engine and waits for its flow. @@ -72,7 +78,7 @@ async fn capture_flow(ctx: &ProxyCtx) -> Flow { let deadline = Instant::now() + Duration::from_secs(5); loop { - let flows = ctx.db.with_conn(|db| db.find_all::().unwrap()); + let flows = ctx.db().unwrap().with_conn(|db| db.find_all::().unwrap()); if let Some(flow) = flows.into_iter().find(|f| f.url == url) { return flow; } @@ -117,7 +123,8 @@ async fn round_trip() { let wrong = http.get(&url).bearer_auth("not-the-token").send().await.unwrap(); assert_eq!(wrong.status(), 401); - let ProxyConnection::Live(client) = ProxyConnection::connect(dir.path()).unwrap() else { + let ProxyConnection::Live(client) = ProxyConnection::connect(dir.path(), &keys()).unwrap() + else { panic!("expected the live owner"); }; let status = client.status().await.unwrap(); @@ -268,7 +275,7 @@ async fn offline_reads_proxy_db() { flow }; - let connection = ProxyConnection::connect(dir.path()).unwrap(); + let connection = ProxyConnection::connect(dir.path(), &keys()).unwrap(); assert!(matches!(connection, ProxyConnection::Offline(_))); let flows = connection.flows(&FlowFilter::default()).await.unwrap(); assert_eq!(flows.iter().map(|f| &f.id).collect::>(), vec![&flow.id]); @@ -280,7 +287,7 @@ async fn offline_reads_proxy_db() { assert_eq!(connection.har(&posts).await.unwrap().log.entries.len(), 1); let empty = tempfile::tempdir().unwrap(); - let err = ProxyConnection::connect(empty.path()).err().unwrap(); + let err = ProxyConnection::connect(empty.path(), &keys()).err().unwrap(); assert!(err.starts_with("No captured traffic yet"), "{err}"); } @@ -365,7 +372,7 @@ async fn save_writes_the_request_into_yaak() { assert!(source.contains("sync"), "{source}"); // The flow remembers what it was saved as. - let flow = yaak_proxy_lib::flows::get(&ctx.db, &flow.id).unwrap(); + let flow = yaak_proxy_lib::flows::get(&ctx.db().unwrap(), &flow.id).unwrap(); assert_eq!(flow.saved_request_id, Some(saved.request.id)); // Without a name, through the control API as the CLI saves, it stays unnamed, as Yaak @@ -564,7 +571,8 @@ async fn the_save_dialog_reads_workspaces_previews_and_saves() { (workspace.id.as_str(), Some(folder.id.as_str())) ); let stored = ctx - .db + .db() + .unwrap() .with_conn(|db| db.find_one::(yaak_proxy_lib::models::FlowIden::Id, &flow.id)) .unwrap(); assert_eq!(stored.saved_request_id.as_deref(), Some(request_id)); @@ -636,7 +644,7 @@ async fn flows_name_the_app_that_sent_them() { let deadline = Instant::now() + Duration::from_secs(5); let flow = loop { - let flows = ctx.db.with_conn(|db| db.find_all::().unwrap()); + let flows = ctx.db().unwrap().with_conn(|db| db.find_all::().unwrap()); if let Some(flow) = flows.into_iter().find(|f| f.url == url) { break flow; } @@ -650,7 +658,7 @@ async fn flows_name_the_app_that_sent_them() { // The search and the sidebar's Apps group find it by its path. let filter = FlowFilter { app: Some(vec!["/usr/bin/curl".into()]), ..Default::default() }; - assert_eq!(yaak_proxy_lib::flows::list(&ctx.db, &filter).unwrap().len(), 1); + assert_eq!(yaak_proxy_lib::flows::list(&ctx.db().unwrap(), &filter).unwrap().len(), 1); } /// Sends one GET through the engine, without waiting for a flow. @@ -682,9 +690,12 @@ async fn pausing_stores_no_flows_and_resuming_stores_them_again() { assert_eq!((state["state"].as_str(), &state["recording"]), (Some("running"), &json!(false))); let paused = send_through(&ctx, "/while-paused").await; tokio::time::sleep(Duration::from_millis(300)).await; - let flows = ctx.db.with_conn(|db| db.find_all::().unwrap()); + let flows = ctx.db().unwrap().with_conn(|db| db.find_all::().unwrap()); assert!(flows.iter().all(|f| f.url != paused), "a flow was stored while paused"); - assert!(ctx.blobs.connect().list_body_ids().unwrap().is_empty(), "a body was stored"); + assert!( + ctx.blobs().unwrap().connect().list_body_ids().unwrap().is_empty(), + "a body was stored" + ); // Resumed through the control API, as the CLI would. let status = client.set_recording(true).await.unwrap(); diff --git a/crates-proxy/yaak-proxy-lib/tests/flow.rs b/crates-proxy/yaak-proxy-lib/tests/flow.rs index 387d9c4c0..303c7904c 100644 --- a/crates-proxy/yaak-proxy-lib/tests/flow.rs +++ b/crates-proxy/yaak-proxy-lib/tests/flow.rs @@ -16,10 +16,10 @@ use hyper::{Request, Response}; use hyper_util::rt::TokioIo; use serde_json::{Value, json}; use tokio::net::TcpListener; -use yaak_proxy_lib::ProxyCtx; use yaak_proxy_lib::bodies::{DecodedBody, read_body, read_raw_body}; use yaak_proxy_lib::models::{Flow, FlowBody}; use yaak_proxy_lib::settings::ProxySettings; +use yaak_proxy_lib::{MasterKeySource, ProxyCtx}; use yaak_rpc::RpcEventEmitter; const CAPTURE_LIMIT: usize = 1024; @@ -95,7 +95,7 @@ async fn spawn_upstream() -> SocketAddr { async fn wait_for_flow(ctx: &ProxyCtx, url: &str) -> Flow { let deadline = Instant::now() + Duration::from_secs(5); loop { - let flows = ctx.db.with_conn(|db| db.find_all::().unwrap()); + let flows = ctx.db().unwrap().with_conn(|db| db.find_all::().unwrap()); if let Some(flow) = flows.into_iter().find(|f| f.url == url) { return flow; } @@ -116,7 +116,7 @@ async fn flow_is_stored_with_its_bodies() { settings.save(&dir.path().join("settings.json")).unwrap(); let (events, _event_rx) = RpcEventEmitter::new(); - let ctx = ProxyCtx::new(dir.path(), events).unwrap(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); ctx.start().unwrap(); let port = ctx.port().unwrap(); assert!(dir.path().join("ca/ca.pem").exists()); @@ -149,10 +149,23 @@ async fn flow_is_stored_with_its_bodies() { assert!(flow.request_headers.iter().any(|h| h.name == "content-length")); assert_eq!(flow.error, None); - assert_eq!(read_raw_body(&ctx.blobs, &flow.id, FlowBody::Request).unwrap(), b"hello proxy"); - assert_eq!(read_raw_body(&ctx.blobs, &flow.id, FlowBody::Response).unwrap(), compressed); - let decoded = - read_body(&ctx.db, &ctx.blobs, &flow, FlowBody::Response, CAPTURE_LIMIT).await.unwrap(); + assert_eq!( + read_raw_body(&ctx.blobs().unwrap(), &flow.id, FlowBody::Request).unwrap(), + b"hello proxy" + ); + assert_eq!( + read_raw_body(&ctx.blobs().unwrap(), &flow.id, FlowBody::Response).unwrap(), + compressed + ); + let decoded = read_body( + &ctx.db().unwrap(), + &ctx.blobs().unwrap(), + &flow, + FlowBody::Response, + CAPTURE_LIMIT, + ) + .await + .unwrap(); let expected = DecodedBody { data: b"hello proxy".to_vec(), stored_size: compressed.len(), @@ -170,9 +183,15 @@ async fn flow_is_stored_with_its_bodies() { assert_eq!(flow.request_size, 0); assert_eq!(flow.response_size, BIG_BODY as i64); assert!(flow.response_truncated); - let stored = read_raw_body(&ctx.blobs, &flow.id, FlowBody::Response).unwrap(); + let stored = read_raw_body(&ctx.blobs().unwrap(), &flow.id, FlowBody::Response).unwrap(); assert_eq!(stored, vec![b'x'; CAPTURE_LIMIT]); - assert!(!ctx.blobs.connect().body_exists(&Flow::body_id(&flow.id, FlowBody::Request)).unwrap()); + assert!( + !ctx.blobs() + .unwrap() + .connect() + .body_exists(&Flow::body_id(&flow.id, FlowBody::Request)) + .unwrap() + ); ctx.stop().unwrap(); assert_eq!(ctx.port(), None); @@ -190,7 +209,7 @@ async fn early_response_keeps_request_totals() { settings.save(&dir.path().join("settings.json")).unwrap(); let (events, _event_rx) = RpcEventEmitter::new(); - let ctx = ProxyCtx::new(dir.path(), events).unwrap(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); ctx.start().unwrap(); let port = ctx.port().unwrap(); @@ -205,7 +224,8 @@ async fn early_response_keeps_request_totals() { let flow = wait_for_flow(&ctx, &url).await; assert_eq!(flow.status, Some(200)); - let stored = read_raw_body(&ctx.blobs, &flow.id, FlowBody::Request).unwrap().len() as i64; + let stored = + read_raw_body(&ctx.blobs().unwrap(), &flow.id, FlowBody::Request).unwrap().len() as i64; assert!(stored > 0); assert!(flow.request_size >= stored, "{} < {stored}", flow.request_size); assert_eq!(flow.request_truncated, flow.request_size > stored); @@ -223,7 +243,7 @@ async fn cancelled_flow_is_not_an_error() { settings.save(&dir.path().join("settings.json")).unwrap(); let (events, _event_rx) = RpcEventEmitter::new(); - let ctx = ProxyCtx::new(dir.path(), events).unwrap(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); ctx.start().unwrap(); let port = ctx.port().unwrap(); @@ -251,7 +271,7 @@ async fn http_version_setting_applies_to_new_connections() { settings.save(&dir.path().join("settings.json")).unwrap(); let (events, _event_rx) = RpcEventEmitter::new(); - let ctx = ProxyCtx::new(dir.path(), events).unwrap(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); ctx.start().unwrap(); let port = ctx.port().unwrap(); let ca = std::fs::read(dir.path().join("ca/ca.pem")).unwrap(); @@ -296,7 +316,7 @@ async fn body_rpcs() { settings.save(&dir.path().join("settings.json")).unwrap(); let (events, _event_rx) = RpcEventEmitter::new(); - let ctx = ProxyCtx::new(dir.path(), events).unwrap(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); ctx.start().unwrap(); let port = ctx.port().unwrap(); @@ -311,6 +331,8 @@ async fn body_rpcs() { "systemProxyAtLaunch": false, "httpVersion": "auto", "startError": null, + "encrypting": null, + "locked": null, "otherOwner": null, }) ); @@ -393,3 +415,72 @@ async fn body_rpcs() { let formatted = rpc(&ctx, "format_body", json!({"text": r#"{"a":[1,2]}"#})).await.unwrap(); assert_eq!(formatted, json!("{\n \"a\": [\n 1,\n 2\n ]\n}")); } + +/// Capture throughput through the engine, end to end: 3,000 requests, 16 at a time over +/// kept-alive connections, each with a 4 KB response, timed until every flow is stored, and how +/// long opening one then takes. Run in release: `cargo test --release -p yaak-proxy-lib --test +/// flow times_capture -- --ignored --nocapture`. +#[tokio::test(flavor = "multi_thread")] +#[ignore] +async fn times_capture_through_the_engine() { + const FLOWS: usize = 3_000; + const AT_ONCE: usize = 16; + let upstream = spawn_upstream().await; + let dir = tempfile::tempdir().unwrap(); + let settings = ProxySettings { port: 0, system_proxy_at_launch: false, ..Default::default() }; + settings.save(&dir.path().join("settings.json")).unwrap(); + let (events, _rx) = RpcEventEmitter::new(); + let ctx = ProxyCtx::new(dir.path(), events, MasterKeySource::memory()).unwrap(); + ctx.start().unwrap(); + let proxy = format!("http://127.0.0.1:{}", ctx.port().unwrap()); + let client = reqwest::Client::builder() + .proxy(reqwest::Proxy::http(&proxy).unwrap()) + .pool_max_idle_per_host(AT_ONCE) + .build() + .unwrap(); + + let started = Instant::now(); + let next = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let workers: Vec<_> = (0..AT_ONCE) + .map(|_| { + let (client, next) = (client.clone(), next.clone()); + tokio::spawn(async move { + loop { + let i = next.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + if i >= FLOWS { + return; + } + let url = format!("http://{upstream}/v1/items/{i}?api_key=sk_test_123"); + let resp = client.get(&url).header("authorization", "Bearer abc").send(); + assert_eq!(resp.await.unwrap().bytes().await.unwrap().len(), BIG_BODY); + } + }) + }) + .collect(); + for worker in workers { + worker.await.unwrap(); + } + let db = ctx.db().unwrap(); + while yaak_proxy_lib::flows::count(&db).unwrap() < FLOWS { + tokio::time::sleep(Duration::from_millis(1)).await; + } + let capture = started.elapsed(); + + let ids: Vec = yaak_proxy_lib::flows::list(&db, &Default::default()) + .unwrap() + .into_iter() + .map(|f| f.id) + .collect(); + let blobs = ctx.blobs().unwrap(); + let started = Instant::now(); + for id in &ids { + let flow = yaak_proxy_lib::flows::get(&db, id).unwrap(); + read_raw_body(&blobs, &flow.id, FlowBody::Response).unwrap(); + } + let open = started.elapsed() / ids.len() as u32; + ctx.stop().unwrap(); + eprintln!( + "{FLOWS} flows through the engine in {capture:?}: {:.0} flows/s; opening one: {open:?}", + FLOWS as f64 / capture.as_secs_f64() + ); +} diff --git a/crates-proxy/yaak-proxy-lib/tests/streams.rs b/crates-proxy/yaak-proxy-lib/tests/streams.rs index b1695d5b5..b34184fa0 100644 --- a/crates-proxy/yaak-proxy-lib/tests/streams.rs +++ b/crates-proxy/yaak-proxy-lib/tests/streams.rs @@ -22,7 +22,7 @@ use tokio::net::TcpListener; use yaak_proxy_lib::bodies::read_raw_body; use yaak_proxy_lib::models::{Flow, FlowBody}; use yaak_proxy_lib::settings::ProxySettings; -use yaak_proxy_lib::{GrpcConfig, ProxyCtx}; +use yaak_proxy_lib::{GrpcConfig, MasterKeySource, ProxyCtx}; use yaak_rpc::RpcEventEmitter; type BoxError = Box; @@ -210,7 +210,7 @@ fn start(dir: &Path) -> (ProxyCtx, std::sync::mpsc::Receiver }; settings.save(&dir.join("settings.json")).unwrap(); let (events, event_rx) = RpcEventEmitter::new(); - let ctx = ProxyCtx::new(dir, events).unwrap().with_protoc(protoc()); + let ctx = ProxyCtx::new(dir, events, MasterKeySource::memory()).unwrap().with_protoc(protoc()); ctx.start().unwrap(); (ctx, event_rx) } @@ -228,7 +228,7 @@ async fn rpc(ctx: &ProxyCtx, cmd: &str, payload: Value) -> Result async fn wait_for(ctx: &ProxyCtx, url: &str, check: impl Fn(&Flow) -> bool) -> Flow { let deadline = Instant::now() + Duration::from_secs(10); loop { - let flows = ctx.db.with_conn(|db| db.find_all::().unwrap()); + let flows = ctx.db().unwrap().with_conn(|db| db.find_all::().unwrap()); if let Some(flow) = flows.into_iter().find(|f| f.url == url && check(f)) { return flow; } @@ -280,7 +280,9 @@ async fn server_sent_events_are_stored_while_the_stream_is_open() { // The events are the body's record: nothing of it went to the blob store, and it reads // back as the client read it. - assert!(read_raw_body(&ctx.blobs, &ended.id, FlowBody::Response).unwrap().is_empty()); + assert!( + read_raw_body(&ctx.blobs().unwrap(), &ended.id, FlowBody::Response).unwrap().is_empty() + ); let body = rpc(&ctx, "get_flow_body", json!({ "flowId": ended.id, "which": "response" })) .await .unwrap(); @@ -342,7 +344,10 @@ async fn grpc_web_calls_are_stored_and_decoded() { (FlowBody::Request, "request"), (FlowBody::Response, "response"), ] { - assert!(read_raw_body(&ctx.blobs, &flow.id, which).unwrap().is_empty(), "{side}"); + assert!( + read_raw_body(&ctx.blobs().unwrap(), &flow.id, which).unwrap().is_empty(), + "{side}" + ); } let read = |which: &str| rpc(&ctx, "get_flow_body", json!({ "flowId": flow.id, "which": which })); @@ -381,7 +386,7 @@ async fn grpc_web_calls_are_stored_and_decoded() { .unwrap(); let down = wait_for(&ctx, &gateway, |f| f.elapsed_ms.is_some()).await; assert_eq!( - read_raw_body(&ctx.blobs, &down.id, FlowBody::Response).unwrap(), + read_raw_body(&ctx.blobs().unwrap(), &down.id, FlowBody::Response).unwrap(), b"upstream down" ); let page = rpc(&ctx, "get_flow_body", json!({ "flowId": down.id, "which": "response" })) @@ -515,11 +520,13 @@ async fn grpc_web_calls_are_stored_and_decoded() { assert!(rpc(&ctx, "list_messages", json!({ "flowId": flow.id })).await.is_err()); // The rows go with the background delete the clear starts. let deadline = Instant::now() + Duration::from_secs(10); - while ctx.db.with_conn(|c| yaak_proxy_lib::flows::clear_mark(c.conn().resolve())) > 0 { + while ctx.db().unwrap().with_conn(|c| yaak_proxy_lib::flows::clear_mark(c.conn().resolve())) + > 0 + { assert!(Instant::now() < deadline, "cleared flows were not deleted"); tokio::time::sleep(Duration::from_millis(10)).await; } - let left: i64 = ctx.db.with_conn(|c| { + let left: i64 = ctx.db().unwrap().with_conn(|c| { c.conn() .resolve() .query_row("SELECT COUNT(*) FROM flow_messages", [], |r| r.get(0)) diff --git a/crates-tauri/yaak-app-proxy/src/lib.rs b/crates-tauri/yaak-app-proxy/src/lib.rs index 5856a3c8e..b5e7d52df 100644 --- a/crates-tauri/yaak-app-proxy/src/lib.rs +++ b/crates-tauri/yaak-app-proxy/src/lib.rs @@ -2,7 +2,7 @@ use log::{error, info, warn}; use tauri::Runtime; use tauri::path::BaseDirectory; use tauri::{Emitter, Manager, RunEvent, State, WebviewWindow}; -use yaak_proxy_lib::ProxyCtx; +use yaak_proxy_lib::{MasterKeySource, ProxyCtx}; use yaak_rpc::{RpcEventEmitter, RpcRouter}; use yaak_window::window::CreateWindowConfig; @@ -88,7 +88,14 @@ pub fn run() { std::fs::create_dir_all(&data_dir).expect("failed to create app data dir"); let (emitter, event_rx) = RpcEventEmitter::new(); - let mut ctx = ProxyCtx::new(&data_dir, emitter)?; + // Captured traffic is encrypted with a key wrapped by a master key in the keychain, + // under this app's identifier, so a development build keeps its own. Linux without + // a Secret Service keeps it in a file only the user can read, with a warning. + let keys = MasterKeySource::keychain_or_file( + &app.config().identifier, + data_dir.join("encryption.key"), + ); + let mut ctx = ProxyCtx::new(&data_dir, emitter, keys)?; // Compiles protos attached for gRPC decoding when the bundle has protoc, as Yaak's // does; otherwise one on PATH is used. if let Some(config) = bundled_protoc(app.handle()) { diff --git a/crates/common/yaak-database/Cargo.toml b/crates/common/yaak-database/Cargo.toml index 5fb06e287..89be8eeb7 100644 --- a/crates/common/yaak-database/Cargo.toml +++ b/crates/common/yaak-database/Cargo.toml @@ -17,7 +17,16 @@ serde_json = { workspace = true } thiserror = { workspace = true } ts-rs = { workspace = true } +[features] +# Whole-database encryption: links SQLCipher in place of SQLite, with OpenSSL built in. +sqlcipher = ["rusqlite/bundled-sqlcipher-vendored-openssl", "dep:openssl-sys"] + +[dev-dependencies] +tempfile = "3" + [target.'cfg(not(target_arch = "wasm32"))'.dependencies] +# The OpenSSL SQLCipher links, initialized without its exit-time cleanup. +openssl-sys = { version = "0.9", optional = true } r2d2 = "0.8.10" r2d2_sqlite = { version = "0.32" } diff --git a/crates/common/yaak-database/src/encryption.rs b/crates/common/yaak-database/src/encryption.rs new file mode 100644 index 000000000..109279bb0 --- /dev/null +++ b/crates/common/yaak-database/src/encryption.rs @@ -0,0 +1,490 @@ +//! Whole-database encryption with SQLCipher: the key a connection is opened with, and the +//! one-time conversion of a plain database into an encrypted one. +//! +//! An encrypted file is SQLite inside, so every query, index and full-text table works as on a +//! plain one; pages are encrypted as they are written and decrypted as they are read. Built only +//! with the `sqlcipher` feature, which links SQLCipher in place of SQLite: a build without it +//! cannot open an encrypted file, and one with it opens plain files without a key as before. + +use crate::error::{Error, Result}; +use rusqlite::Connection; +use std::fmt; +use std::fs; +use std::io::{self, Read, Write}; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +/// A database's 256-bit key, given to SQLCipher raw: no passphrase, so no key derivation runs +/// as each connection opens. The file's salt stays in its first 16 bytes, as SQLCipher keeps it. +#[derive(Clone)] +pub struct DatabaseKey([u8; 32]); + +impl DatabaseKey { + pub fn from_bytes(bytes: [u8; 32]) -> Self { + Self(bytes) + } + + /// The key as `PRAGMA key` and `ATTACH … KEY` take it raw: `"x'…'"`. + fn sql_literal(&self) -> String { + let hex: String = self.0.iter().map(|b| format!("{b:02X}")).collect(); + format!("\"x'{hex}'\"") + } +} + +/// Never prints the key. +impl fmt::Debug for DatabaseKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("DatabaseKey(..)") + } +} + +/// What is at a database's path. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FileState { + /// Nothing, or an empty file: one opened with a key starts out encrypted. + Missing, + /// A plain SQLite database, which [`encrypt_in_place`] converts. + Plain, + /// Anything else, which only the right key opens. + Encrypted, +} + +const PLAIN_HEADER: &[u8; 16] = b"SQLite format 3\0"; + +/// Whether `path` holds a plain database, an encrypted one, or none. Reads the first 16 bytes: +/// a plain file starts with SQLite's header, an encrypted one with its random salt. +pub fn file_state(path: &Path) -> io::Result { + let mut file = match fs::File::open(path) { + Ok(file) => file, + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(FileState::Missing), + Err(e) => return Err(e), + }; + let mut header = [0u8; 16]; + let mut read = 0; + while read < header.len() { + match file.read(&mut header[read..])? { + 0 => break, + n => read += n, + } + } + Ok(match read { + 0 => FileState::Missing, + _ if &header == PLAIN_HEADER => FileState::Plain, + _ => FileState::Encrypted, + }) +} + +/// Keys `conn`, which must have just been opened and not yet read: every pool's connection +/// init calls this first. Reads the schema at once, so a wrong key, or a plain file, fails here +/// ("file is not a database") rather than at the first query. +pub fn apply_key(conn: &Connection, key: &DatabaseKey) -> rusqlite::Result<()> { + init_crypto(); + conn.execute_batch(&format!("PRAGMA key = {};", key.sql_literal()))?; + conn.query_row("SELECT count(*) FROM sqlite_master", [], |_| Ok(())) +} + +/// Initializes OpenSSL without the cleanup it otherwise registers to run at exit, which frees +/// its state while threads still reading or writing an encrypted database use it, so the process +/// crashes as it quits. Must run before the process opens its first SQLite connection: SQLCipher +/// uses OpenSSL as SQLite initializes, and a cleanup registered then cannot be taken back. Every +/// opener here calls it; one that opens a connection itself first calls it too. +pub fn init_crypto() { + openssl_sys::init(); +} + +/// The bytes a plain database at `path` takes, with its WAL: what [`encrypt_in_place`] has to +/// write, for showing its progress. +pub fn plain_size(path: &Path) -> u64 { + [path.to_path_buf(), sidecar(path, "-wal")] + .iter() + .filter_map(|p| fs::metadata(p).ok()) + .map(|m| m.len()) + .sum() +} + +/// Converts a plain database at `path` into one encrypted with `key`, WAL and all, and returns +/// whether there was one to convert. Exports into a new file beside it, checks the new file +/// opens with `key` and holds every table, swaps it in, then overwrites the plain file, its WAL +/// and shared memory with zeros before deleting them. `progress` hears the bytes written so +/// far, a few times a second. +/// +/// Nothing may have the database open meanwhile. A lock beside it serializes callers across +/// processes; the second finds it converted. An interrupted conversion is finished, or undone +/// and redone, by the next call. +pub fn encrypt_in_place( + path: &Path, + key: &DatabaseKey, + progress: &(dyn Fn(u64) + Sync), +) -> Result { + init_crypto(); + let exported = sidecar(path, "-encrypting"); + let replaced = sidecar(path, "-plain"); + // Nothing to convert and nothing left over, the usual case, takes no lock. + if file_state(path)? != FileState::Plain && !exported.exists() && !replaced.exists() { + return Ok(false); + } + + let lock = fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(sidecar(path, "-encrypt.lock"))?; + lock.lock()?; + recover(path, &exported, &replaced)?; + if file_state(path)? != FileState::Plain { + return Ok(false); + } + + let conn = Connection::open(path)?; + conn.busy_timeout(Duration::from_secs(5))?; + keep_wal_on_close(&conn)?; + // Brings in what the WAL holds, so the export is complete however the last run ended. + conn.query_row("PRAGMA wal_checkpoint(FULL)", [], |_| Ok(()))?; + let original = fingerprint(&conn)?; + let user_version: i64 = conn.query_row("PRAGMA user_version", [], |r| r.get(0))?; + + let exporting = || -> rusqlite::Result<()> { + conn.execute( + &format!("ATTACH DATABASE ?1 AS encrypted KEY {}", key.sql_literal()), + [exported.to_string_lossy()], + )?; + conn.query_row("SELECT sqlcipher_export('encrypted')", [], |_| Ok(()))?; + conn.execute_batch(&format!( + "PRAGMA encrypted.user_version = {user_version}; DETACH DATABASE encrypted;" + )) + }; + let done = std::sync::atomic::AtomicBool::new(false); + let exported_result = std::thread::scope(|s| { + s.spawn(|| { + while !done.load(std::sync::atomic::Ordering::Relaxed) { + progress(fs::metadata(&exported).map(|m| m.len()).unwrap_or(0)); + std::thread::sleep(Duration::from_millis(200)); + } + }); + let result = exporting(); + done.store(true, std::sync::atomic::Ordering::Relaxed); + result + }); + if let Err(e) = exported_result { + drop(conn); + let _ = fs::remove_file(&exported); + return Err(e.into()); + } + // Closing checkpoints what is left; the WAL stays, to be overwritten below. + drop(conn); + // Writable, as Windows flushes only a handle that can write. + fs::OpenOptions::new().write(true).open(&exported)?.sync_all()?; + progress(fs::metadata(&exported)?.len()); + + // The copy must open with the key and hold everything, every row under its own rowid, + // before the original goes. + let check = Connection::open(&exported)?; + apply_key(&check, key)?; + let copied = fingerprint(&check)?; + drop(check); + if copied != original { + let _ = fs::remove_file(&exported); + return Err(Error::Database(format!( + "The encrypted copy of {} differs from it, so it is left as it was", + path.display() + ))); + } + + for suffix in ["-wal", "-shm", "-journal"] { + wipe(&sidecar(path, suffix))?; + } + fs::rename(path, &replaced)?; + fs::rename(&exported, path)?; + sync_dir(path); + wipe(&replaced)?; + // Another caller waiting on the lock holds its own handle, so this only tidies. + let _ = fs::remove_file(sidecar(path, "-encrypt.lock")); + Ok(true) +} + +/// Finishes or undoes a conversion that stopped part way: the copy swapped in when the plain +/// file had already moved aside, the plain file put back when there was no copy to swap in, +/// and a half-written copy deleted. +fn recover(path: &Path, exported: &Path, replaced: &Path) -> Result<()> { + if replaced.exists() { + if !path.exists() { + if file_state(exported)? == FileState::Encrypted { + fs::rename(exported, path)?; + } else { + fs::rename(replaced, path)?; + } + } + if replaced.exists() { + wipe(replaced)?; + } + } + if exported.exists() { + fs::remove_file(exported)?; + } + Ok(()) +} + +/// What a copy must match: every table and index by name, and each table's row count and the +/// total of its rowids, so rows renumbered in the copy show. `sqlcipher_export` keeps rowids by +/// copying each table whole, which SQLite does for an empty table of the same schema. +fn fingerprint(conn: &Connection) -> rusqlite::Result> { + let names: Vec<(String, String)> = conn + .prepare( + "SELECT type, name FROM sqlite_master WHERE type IN ('table', 'index') ORDER BY 1, 2", + )? + .query_map([], |r| Ok((r.get(0)?, r.get(1)?)))? + .collect::>()?; + names + .into_iter() + .map(|(kind, name)| { + if kind != "table" { + return Ok((format!("{kind} {name}"), 0, 0.0)); + } + let table = format!("\"{}\"", name.replace('"', "\"\"")); + let count = |sql: String| conn.query_row(&sql, [], |r| Ok((r.get(0)?, r.get(1)?))); + // A table without rowids, or a virtual one that has none, by its rows alone. + let (rows, rowids) = count(format!("SELECT count(*), total(rowid) FROM {table}")) + .or_else(|_| count(format!("SELECT count(*), 0.0 FROM {table}")))?; + Ok((format!("{kind} {name}"), rows, rowids)) + }) + .collect() +} + +/// Keeps the WAL file when `conn` closes, rather than SQLite deleting it unread, so it can be +/// overwritten before it goes. +fn keep_wal_on_close(conn: &Connection) -> Result<()> { + let mut on: std::ffi::c_int = 1; + // SAFETY: `conn` is open for the call, "main" is a NUL-terminated schema name, and + // SQLITE_FCNTL_PERSIST_WAL takes a pointer to an int it reads and may write. + let rc = unsafe { + rusqlite::ffi::sqlite3_file_control( + conn.handle(), + c"main".as_ptr(), + rusqlite::ffi::SQLITE_FCNTL_PERSIST_WAL, + (&mut on as *mut std::ffi::c_int).cast(), + ) + }; + if rc != rusqlite::ffi::SQLITE_OK { + return Err(Error::Database(format!("Failed to keep the WAL on close: {rc}"))); + } + Ok(()) +} + +/// Overwrites the file at `path` with zeros, flushed, then deletes it. Nothing there is fine. +fn wipe(path: &Path) -> Result<()> { + let mut file = match fs::OpenOptions::new().write(true).open(path) { + Ok(file) => file, + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(e) => return Err(e.into()), + }; + let len = file.metadata()?.len(); + let zeros = vec![0u8; 1 << 20]; + let mut left = len; + while left > 0 { + let n = left.min(zeros.len() as u64) as usize; + file.write_all(&zeros[..n])?; + left -= n as u64; + } + file.sync_all()?; + drop(file); + fs::remove_file(path)?; + Ok(()) +} + +/// Flushes the directory holding `path`, so its renames survive a crash. Only Unix can. +fn sync_dir(path: &Path) { + #[cfg(unix)] + if let Some(dir) = path.parent() + && let Ok(dir) = fs::File::open(dir) + { + let _ = dir.sync_all(); + } + #[cfg(not(unix))] + let _ = path; +} + +/// `path` with `suffix` added, as SQLite names a database's WAL `-wal`. +fn sidecar(path: &Path, suffix: &str) -> PathBuf { + let mut name = path.as_os_str().to_os_string(); + name.push(suffix); + PathBuf::from(name) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn key(n: u8) -> DatabaseKey { + DatabaseKey::from_bytes([n; 32]) + } + + fn contains(path: &Path, needle: &[u8]) -> bool { + fs::read(path).map(|b| b.windows(needle.len()).any(|w| w == needle)).unwrap_or(false) + } + + #[test] + fn opens_only_with_its_key() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("db.sqlite"); + let conn = Connection::open(&path).unwrap(); + apply_key(&conn, &key(1)).unwrap(); + conn.execute_batch("CREATE TABLE t (v TEXT); INSERT INTO t VALUES ('Bearer secret');") + .unwrap(); + drop(conn); + assert_eq!(file_state(&path).unwrap(), FileState::Encrypted); + assert!(!contains(&path, b"Bearer secret")); + + let wrong = Connection::open(&path).unwrap(); + let err = apply_key(&wrong, &key(2)).unwrap_err(); + assert!(err.to_string().contains("file is not a database"), "{err}"); + let none = Connection::open(&path).unwrap(); + assert!(none.query_row("SELECT count(*) FROM t", [], |r| r.get::<_, i64>(0)).is_err()); + + let right = Connection::open(&path).unwrap(); + apply_key(&right, &key(1)).unwrap(); + let v: String = right.query_row("SELECT v FROM t", [], |r| r.get(0)).unwrap(); + assert_eq!(v, "Bearer secret"); + assert_eq!(format!("{:?}", key(1)), "DatabaseKey(..)"); + } + + #[test] + fn a_plain_file_does_not_open_with_a_key() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("db.sqlite"); + assert_eq!(file_state(&path).unwrap(), FileState::Missing); + Connection::open(&path).unwrap().execute_batch("CREATE TABLE t (v TEXT)").unwrap(); + assert_eq!(file_state(&path).unwrap(), FileState::Plain); + assert!(apply_key(&Connection::open(&path).unwrap(), &key(1)).is_err()); + } + + #[test] + fn converts_a_plain_database_with_its_wal() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("db.sqlite"); + let live = dir.path().join("live.sqlite"); + let conn = Connection::open(&live).unwrap(); + conn.pragma_update(None, "journal_mode", "WAL").unwrap(); + conn.execute_batch( + "PRAGMA user_version = 7; + CREATE TABLE t (v TEXT); CREATE INDEX t_v ON t (v); + INSERT INTO t VALUES ('in the main file');", + ) + .unwrap(); + conn.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(())).unwrap(); + // Left in the WAL, as a crash would leave it. + keep_wal_on_close(&conn).unwrap(); + conn.execute_batch("PRAGMA wal_autocheckpoint = 0; INSERT INTO t VALUES ('only in wal');") + .unwrap(); + // The files as a crash leaves them, nothing checkpointed or closed: copied while the + // connection is open, which then closes, so no handle outlives the test. + fs::copy(&live, &path).unwrap(); + fs::copy(sidecar(&live, "-wal"), sidecar(&path, "-wal")).unwrap(); + drop(conn); + assert!(contains(&sidecar(&path, "-wal"), b"only in wal")); + assert!(!contains(&path, b"only in wal")); + assert!(plain_size(&path) > 0); + + let seen = std::sync::Mutex::new(Vec::new()); + assert!(encrypt_in_place(&path, &key(1), &|n| seen.lock().unwrap().push(n)).unwrap()); + assert!(seen.lock().unwrap().last().is_some_and(|n| *n > 0)); + assert_eq!(file_state(&path).unwrap(), FileState::Encrypted); + for suffix in ["-wal", "-shm", "-plain", "-encrypting"] { + assert!(!sidecar(&path, suffix).exists(), "{suffix} left behind"); + } + assert!(!contains(&path, b"only in wal") && !contains(&path, b"in the main file")); + + let conn = Connection::open(&path).unwrap(); + apply_key(&conn, &key(1)).unwrap(); + let rows: Vec = conn + .prepare("SELECT v FROM t ORDER BY v") + .unwrap() + .query_map([], |r| r.get(0)) + .unwrap() + .map(|r| r.unwrap()) + .collect(); + assert_eq!(rows, ["in the main file", "only in wal"]); + let version: i64 = conn.query_row("PRAGMA user_version", [], |r| r.get(0)).unwrap(); + assert_eq!(version, 7); + drop(conn); + + // Already encrypted, the second call does nothing. + assert!(!encrypt_in_place(&path, &key(1), &|_| {}).unwrap()); + } + + /// Rows keep their rowids, gaps from deleted rows and all, as flows' numbers are theirs. + #[test] + fn keeps_rowids() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("db.sqlite"); + let conn = Connection::open(&path).unwrap(); + conn.execute_batch( + "CREATE TABLE flows (id TEXT NOT NULL PRIMARY KEY, started INTEGER); + CREATE INDEX flows_started ON flows (started, id); + CREATE TABLE marks (id INTEGER PRIMARY KEY, max_rowid INTEGER);", + ) + .unwrap(); + for i in 0..20 { + conn.execute("INSERT INTO flows VALUES (?1, ?1)", [i]).unwrap(); + } + conn.execute_batch( + "DELETE FROM flows WHERE started < 5 OR started IN (9, 12); + INSERT INTO marks VALUES (3, 12);", + ) + .unwrap(); + let rowids = |conn: &Connection| -> Vec<(i64, String)> { + conn.prepare("SELECT rowid, id FROM flows ORDER BY rowid") + .unwrap() + .query_map([], |r| Ok((r.get(0)?, r.get(1)?))) + .unwrap() + .map(|r| r.unwrap()) + .collect() + }; + let before = rowids(&conn); + drop(conn); + + assert!(encrypt_in_place(&path, &key(1), &|_| {}).unwrap()); + let conn = Connection::open(&path).unwrap(); + apply_key(&conn, &key(1)).unwrap(); + assert_eq!(rowids(&conn), before); + let mark: (i64, i64) = conn + .query_row("SELECT id, max_rowid FROM marks", [], |r| Ok((r.get(0)?, r.get(1)?))) + .unwrap(); + assert_eq!(mark, (3, 12)); + // New rows carry on from the last, not from the count. + conn.execute("INSERT INTO flows VALUES ('new', 99)", []).unwrap(); + let last: i64 = conn.query_row("SELECT max(rowid) FROM flows", [], |r| r.get(0)).unwrap(); + assert_eq!(last, 21); + } + + #[test] + fn finishes_a_swap_that_stopped_halfway() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("db.sqlite"); + let plain = sidecar(&path, "-plain"); + let exported = sidecar(&path, "-encrypting"); + Connection::open(&path).unwrap().execute_batch("CREATE TABLE t (v TEXT)").unwrap(); + + // Stopped mid-export: the half copy goes and the conversion runs again. + fs::write(&exported, b"half").unwrap(); + assert!(encrypt_in_place(&path, &key(1), &|_| {}).unwrap()); + assert!(!exported.exists()); + + // Stopped between the renames: the plain file aside, the copy not yet in place. + fs::rename(&path, &exported).unwrap(); + Connection::open(&path).unwrap().execute_batch("CREATE TABLE t (v TEXT)").unwrap(); + fs::rename(&path, &plain).unwrap(); + assert!(!encrypt_in_place(&path, &key(1), &|_| {}).unwrap()); + assert_eq!(file_state(&path).unwrap(), FileState::Encrypted); + assert!(!plain.exists() && !exported.exists()); + + // Stopped with the plain file aside and no copy: it is put back and converted. + fs::remove_file(&path).unwrap(); + Connection::open(&plain).unwrap().execute_batch("CREATE TABLE t (v TEXT)").unwrap(); + assert!(encrypt_in_place(&path, &key(1), &|_| {}).unwrap()); + assert!(!plain.exists()); + let conn = Connection::open(&path).unwrap(); + apply_key(&conn, &key(1)).unwrap(); + assert_eq!(fingerprint(&conn).unwrap(), [("table t".to_string(), 0, 0.0)]); + } +} diff --git a/crates/common/yaak-database/src/lib.rs b/crates/common/yaak-database/src/lib.rs index da45ac08c..8d23f2a2b 100644 --- a/crates/common/yaak-database/src/lib.rs +++ b/crates/common/yaak-database/src/lib.rs @@ -1,5 +1,7 @@ pub mod connection_or_tx; pub mod db_context; +#[cfg(all(feature = "sqlcipher", not(target_arch = "wasm32")))] +pub mod encryption; pub mod error; pub mod migrate; pub mod pool; diff --git a/crates/yaak-crypto/Cargo.toml b/crates/yaak-crypto/Cargo.toml index 51a670038..ee2e6c750 100644 --- a/crates/yaak-crypto/Cargo.toml +++ b/crates/yaak-crypto/Cargo.toml @@ -13,3 +13,14 @@ log = { workspace = true } serde = { workspace = true, features = ["derive"] } thiserror = { workspace = true } yaak-models = { workspace = true } + +# The Secret Service backend keyring uses, to tell a missing service from a refusal. +[target.'cfg(target_os = "linux")'.dependencies] +dbus-secret-service = "4.1.0" + +[features] +# `MasterKeySource::memory` and `MasterKeySource::refusing`, for other crates' tests. +test-support = [] + +[dev-dependencies] +tempfile = "3" diff --git a/crates/yaak-crypto/src/data_key.rs b/crates/yaak-crypto/src/data_key.rs new file mode 100644 index 000000000..f4de9b39b --- /dev/null +++ b/crates/yaak-crypto/src/data_key.rs @@ -0,0 +1,218 @@ +//! A key for an app's own data at rest, outside any workspace. The data key encrypts the data; +//! it is kept beside it wrapped (encrypted) by the app's master key, so only the master key +//! lives in the OS credential store. + +use crate::encryption::{decrypt_data, encrypt_data}; +use crate::error::Error::{GenericError, IncorrectMasterKey, MissingMasterKey}; +use crate::error::Result; +use crate::master_key::{write_secret_file, MasterKey, MasterKeySource}; +use base64::prelude::BASE64_STANDARD; +use base64::Engine; +use chacha20poly1305::aead::{Generate, Key}; +use chacha20poly1305::XChaCha20Poly1305; +use std::fmt; +use std::fs; +use std::io; +use std::path::Path; + +#[derive(Clone)] +pub struct DataKey { + key: Key, +} + +impl DataKey { + /// Bytes [`DataKey::encrypt`] adds to what it encrypts: a tag and version, the nonce and the + /// authentication tag. + pub const OVERHEAD: usize = 48; + + /// A new data key, and it wrapped by the master key in `source`, for storing. Makes the + /// master key when `source` has none. + pub fn create(source: &MasterKeySource) -> Result<(Self, String)> { + let master = MasterKey::get_or_create_in(source.store())?; + let key = Key::::generate(); + let wrapped = BASE64_STANDARD.encode(master.encrypt(key.as_slice())?); + Ok((Self { key }, wrapped)) + } + + /// The data key in `wrapped`, as [`DataKey::create`] gave it, unwrapped with the master key + /// in `source`. Never makes a master key: one made now could not unwrap it. + pub fn from_wrapped(source: &MasterKeySource, wrapped: &str) -> Result { + let master = MasterKey::get_in(source.store())?.ok_or(MissingMasterKey)?; + let wrapped = BASE64_STANDARD + .decode(wrapped.trim()) + .map_err(|e| GenericError(format!("Failed to decode the wrapped data key: {e}")))?; + let raw = master.decrypt(&wrapped).map_err(|_| IncorrectMasterKey)?; + let key = Key::::try_from(raw.as_slice()) + .map_err(|_| GenericError("The data key is the wrong length".to_string()))?; + Ok(Self { key }) + } + + /// The data key kept wrapped in the file at `path`, made and written there the first time. + /// Safe to call from several threads and processes at once: a lock beside the file + /// serializes them, so every caller gets the one key that ends up written. + pub fn load_or_create_file(source: &MasterKeySource, path: &Path) -> Result { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + let lock = fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(path.with_extension("lock"))?; + lock.lock()?; + + match fs::read_to_string(path) { + Ok(wrapped) => Self::from_wrapped(source, &wrapped), + Err(e) if e.kind() == io::ErrorKind::NotFound => { + let (key, wrapped) = Self::create(source)?; + // Written beside and renamed over, so a crash never leaves half a key. + let tmp = path.with_extension("tmp"); + write_secret_file(&tmp, &wrapped)?; + fs::rename(&tmp, path)?; + Ok(key) + } + Err(e) => Err(e.into()), + } + } + + /// The data key kept wrapped in the file at `path`, or None when there is no file. Never + /// makes one, for a reader that must not create keys. + pub fn load_file(source: &MasterKeySource, path: &Path) -> Result> { + match fs::read_to_string(path) { + Ok(wrapped) => Self::from_wrapped(source, &wrapped).map(Some), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(e.into()), + } + } + + /// The key's 32 bytes, for a cipher outside this crate, such as SQLCipher's raw key. + pub fn raw_bytes(&self) -> [u8; 32] { + let mut bytes = [0u8; 32]; + bytes.copy_from_slice(self.key.as_slice()); + bytes + } + + pub fn encrypt(&self, data: &[u8]) -> Result> { + encrypt_data(data, &self.key) + } + + pub fn decrypt(&self, data: &[u8]) -> Result> { + decrypt_data(data, &self.key) + } +} + +/// Never prints the key. +impl fmt::Debug for DataKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("DataKey(..)") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::error::Error; + + #[test] + fn round_trips_through_its_wrapped_form() -> Result<()> { + let source = MasterKeySource::memory(); + let (key, wrapped) = DataKey::create(&source)?; + let sealed = key.encrypt(b"authorization: Bearer abc")?; + assert_eq!(sealed.len(), b"authorization: Bearer abc".len() + DataKey::OVERHEAD); + assert!(!sealed.windows(6).any(|w| w == b"Bearer")); + + let unwrapped = DataKey::from_wrapped(&source, &wrapped)?; + assert_eq!(unwrapped.decrypt(&sealed)?, b"authorization: Bearer abc"); + assert_eq!(key.decrypt(&unwrapped.encrypt(b"")?)?, b""); + assert_eq!(unwrapped.raw_bytes(), key.raw_bytes()); + Ok(()) + } + + #[test] + fn unwrapping_needs_the_same_master_key() -> Result<()> { + let (_, wrapped) = DataKey::create(&MasterKeySource::memory())?; + + // One that has never had a master key does not get one made. + let empty = MasterKeySource::memory(); + assert!(matches!(DataKey::from_wrapped(&empty, &wrapped), Err(Error::MissingMasterKey))); + assert!(empty.store().get()?.is_none()); + + let other = MasterKeySource::memory(); + DataKey::create(&other)?; + assert!(matches!(DataKey::from_wrapped(&other, &wrapped), Err(Error::IncorrectMasterKey))); + + let refusing = MasterKeySource::refusing("denied"); + assert!(DataKey::create(&refusing).is_err()); + assert!(DataKey::from_wrapped(&refusing, &wrapped).is_err()); + Ok(()) + } + + #[test] + fn a_wrong_key_does_not_decrypt() -> Result<()> { + let source = MasterKeySource::memory(); + let (a, _) = DataKey::create(&source)?; + let (b, _) = DataKey::create(&source)?; + let sealed = a.encrypt(b"secret")?; + assert!(matches!(b.decrypt(&sealed), Err(Error::DecryptionError))); + assert_eq!(format!("{a:?}"), "DataKey(..)"); + Ok(()) + } + + #[test] + fn the_key_file_is_made_once_and_read_back() -> Result<()> { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("data.key"); + let source = MasterKeySource::memory(); + let made = DataKey::load_or_create_file(&source, &path)?; + let stored = fs::read_to_string(&path)?; + assert!(!stored.is_empty()); + assert!(!dir.path().join("data.tmp").exists()); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!(fs::metadata(&path)?.permissions().mode() & 0o777, 0o600); + } + + let read = DataKey::load_or_create_file(&source, &path)?; + assert_eq!(read.raw_bytes(), made.raw_bytes()); + let loaded = DataKey::load_file(&source, &path)?.unwrap(); + assert_eq!(loaded.raw_bytes(), made.raw_bytes()); + assert!(DataKey::load_file(&source, &dir.path().join("none.key"))?.is_none()); + assert!(!dir.path().join("none.key").exists()); + assert_eq!(fs::read_to_string(&path)?, stored, "not rewritten"); + + // Another master key cannot read it, and does not replace it. + let other = MasterKeySource::memory(); + assert!(matches!( + DataKey::load_or_create_file(&other, &path), + Err(Error::MissingMasterKey) + )); + assert!(DataKey::load_or_create_file(&MasterKeySource::refusing("denied"), &path).is_err()); + assert_eq!(fs::read_to_string(&path)?, stored); + Ok(()) + } + + /// Everyone asking at once, the first time, gets the same key: one master key, one data key. + #[test] + fn concurrent_first_loads_agree() -> Result<()> { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("data.key"); + let source = MasterKeySource::memory(); + let barrier = std::sync::Barrier::new(8); + let keys: Vec<[u8; 32]> = std::thread::scope(|s| { + let handles: Vec<_> = (0..8) + .map(|_| { + s.spawn(|| { + barrier.wait(); + DataKey::load_or_create_file(&source, &path).unwrap().raw_bytes() + }) + }) + .collect(); + handles.into_iter().map(|h| h.join().unwrap()).collect() + }); + assert!(keys.windows(2).all(|w| w[0] == w[1])); + assert_eq!(DataKey::load_or_create_file(&source, &path)?.raw_bytes(), keys[0]); + Ok(()) + } +} diff --git a/crates/yaak-crypto/src/error.rs b/crates/yaak-crypto/src/error.rs index ca70e97fa..0745ca641 100644 --- a/crates/yaak-crypto/src/error.rs +++ b/crates/yaak-crypto/src/error.rs @@ -19,6 +19,12 @@ pub enum Error { #[error("Failed to decrypt workspace key: {0}")] WorkspaceKeyDecryptionError(String), + #[error("The encryption key is missing from the keychain")] + MissingMasterKey, + + #[error("The keychain's encryption key does not match the one this data was encrypted with")] + IncorrectMasterKey, + #[error("Crypto IO error: {0}")] IoError(#[from] io::Error), diff --git a/crates/yaak-crypto/src/lib.rs b/crates/yaak-crypto/src/lib.rs index 6fe9224ce..fdf9ab3cc 100644 --- a/crates/yaak-crypto/src/lib.rs +++ b/crates/yaak-crypto/src/lib.rs @@ -1,7 +1,10 @@ extern crate core; +pub mod data_key; pub mod encryption; pub mod error; pub mod manager; mod master_key; mod workspace_key; + +pub use master_key::MasterKeySource; diff --git a/crates/yaak-crypto/src/master_key.rs b/crates/yaak-crypto/src/master_key.rs index 887601d57..5e3f0d073 100644 --- a/crates/yaak-crypto/src/master_key.rs +++ b/crates/yaak-crypto/src/master_key.rs @@ -5,7 +5,9 @@ use base32::Alphabet; use chacha20poly1305::aead::{Generate, Key}; use chacha20poly1305::XChaCha20Poly1305; use keyring::{Entry, Error}; -use log::info; +use log::{info, warn}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; const HUMAN_PREFIX: &str = "YKM_"; @@ -16,31 +18,38 @@ pub(crate) struct MasterKey { impl MasterKey { pub(crate) fn get_or_create(app_id: &str, user: &str) -> Result { - let id = format!("{app_id}.EncryptionKey"); - let entry = Entry::new(&id, user)?; - - let key = match entry.get_password() { - Ok(encoded) => { - let without_prefix = encoded.strip_prefix(HUMAN_PREFIX).unwrap_or(&encoded); - let key_bytes = base32::decode(Alphabet::Crockford {}, &without_prefix) - .ok_or(GenericError("Failed to decode master key".to_string()))?; - Key::::try_from(key_bytes.as_slice()) - .map_err(|_| GenericError("Master key is the wrong length".to_string()))? - } - Err(Error::NoEntry) => { - info!("Creating new master key"); - let key = Key::::generate(); - let encoded = base32::encode(Alphabet::Crockford {}, key.as_slice()); - let with_prefix = format!("{HUMAN_PREFIX}{encoded}"); - entry.set_password(&with_prefix)?; - key - } - Err(e) => return Err(GenericError(e.to_string())), - }; + Self::get_or_create_in(&KeychainStore::new(app_id, user)) + } + /// The master key in `store`, or a new one put there when it has none. Callers in this + /// process take turns, so two asking at once for the first time do not each make one, the + /// second replacing the first and leaving what it wrapped unreadable. + pub(crate) fn get_or_create_in(store: &dyn SecretStore) -> Result { + static CREATING: std::sync::Mutex<()> = std::sync::Mutex::new(()); + let _turn = CREATING.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(key) = Self::get_in(store)? { + return Ok(key); + } + info!("Creating new master key"); + let key = Key::::generate(); + let encoded = base32::encode(Alphabet::Crockford {}, key.as_slice()); + store.set(&format!("{HUMAN_PREFIX}{encoded}"))?; Ok(Self { key }) } + /// The master key in `store`, or None when it has none. + pub(crate) fn get_in(store: &dyn SecretStore) -> Result> { + let Some(encoded) = store.get()? else { + return Ok(None); + }; + let without_prefix = encoded.strip_prefix(HUMAN_PREFIX).unwrap_or(&encoded); + let key_bytes = base32::decode(Alphabet::Crockford {}, without_prefix) + .ok_or(GenericError("Failed to decode master key".to_string()))?; + let key = Key::::try_from(key_bytes.as_slice()) + .map_err(|_| GenericError("Master key is the wrong length".to_string()))?; + Ok(Some(Self { key })) + } + pub(crate) fn encrypt(&self, data: &[u8]) -> Result> { encrypt_data(data, &self.key) } @@ -56,10 +65,223 @@ impl MasterKey { } } +/// Where a master key is kept, as the text [`MasterKey`] encodes it to. +pub(crate) trait SecretStore: Send + Sync { + /// The secret, or None when none has been set. + fn get(&self) -> Result>; + fn set(&self, secret: &str) -> Result<()>; +} + +/// Where an app's master key is kept: the OS credential store, normally. Cheap to clone. +#[derive(Clone)] +pub struct MasterKeySource { + store: Arc, +} + +impl MasterKeySource { + /// The OS credential store, under `{app_id}.EncryptionKey`, where Yaak keeps its own: the + /// macOS Keychain, the Windows Credential Manager or the Secret Service on Linux. + pub fn keychain(app_id: &str) -> Self { + Self { store: Arc::new(KeychainStore::new(app_id, KEYCHAIN_USER)) } + } + + /// [`Self::keychain`], but on Linux without a Secret Service running, the key is kept in + /// `path`, readable by the user alone, with a warning, rather than not at all. + pub fn keychain_or_file(app_id: &str, path: impl Into) -> Self { + let store = KeychainStore::new(app_id, KEYCHAIN_USER).with_fallback_file(path.into()); + Self { store: Arc::new(store) } + } + + /// A key held in memory for as long as this source and its clones live. + #[cfg(any(test, feature = "test-support"))] + pub fn memory() -> Self { + Self { store: Arc::new(test_support::MemoryStore::default()) } + } + + /// A store that refuses with `reason`, as a keychain does when access is denied. + #[cfg(any(test, feature = "test-support"))] + pub fn refusing(reason: impl Into) -> Self { + Self { store: Arc::new(test_support::RefusingStore(reason.into())) } + } + + pub(crate) fn store(&self) -> &dyn SecretStore { + self.store.as_ref() + } +} + +/// The account master keys are kept under, as [`crate::manager::EncryptionManager`] keeps +/// Yaak's. +const KEYCHAIN_USER: &str = "encryption-key"; + +/// The OS credential store, with a file to fall back to on Linux without a Secret Service. +struct KeychainStore { + service: String, + user: String, + fallback_file: Option, +} + +impl KeychainStore { + fn new(app_id: &str, user: &str) -> Self { + Self { service: format!("{app_id}.EncryptionKey"), user: user.into(), fallback_file: None } + } + + fn with_fallback_file(self, path: PathBuf) -> Self { + Self { fallback_file: Some(path), ..self } + } + + /// The fallback file, when `e` says there is no credential store to use and there is one. + /// Denied access is not that: it stays an error. + fn fallback_for(&self, e: &Error) -> Option<&Path> { + let path = self.fallback_file.as_deref()?; + is_missing_secret_service(e).then_some(path) + } + + /// A key kept in the fallback file while there was no credential store. Read before + /// creating one, so a Secret Service that turns up later does not replace it. + fn fallback_secret(&self) -> Result> { + match &self.fallback_file { + Some(path) => read_secret_file(path), + None => Ok(None), + } + } +} + +impl SecretStore for KeychainStore { + fn get(&self) -> Result> { + let entry = Entry::new(&self.service, &self.user)?; + match entry.get_password() { + Ok(secret) => Ok(Some(secret)), + Err(Error::NoEntry) => self.fallback_secret(), + Err(e) => match self.fallback_for(&e) { + Some(path) => read_secret_file(path), + None => Err(GenericError(e.to_string())), + }, + } + } + + fn set(&self, secret: &str) -> Result<()> { + let entry = Entry::new(&self.service, &self.user)?; + match entry.set_password(secret) { + Ok(()) => Ok(()), + Err(e) => match self.fallback_for(&e) { + Some(path) => { + warn!( + "No Secret Service is running ({e}), so the encryption key for {} is kept \ + in {}, readable only by you, instead of the keyring", + self.service, + path.display() + ); + write_secret_file(path, secret) + } + None => Err(e.into()), + }, + } + } +} + +/// D-Bus errors that mean there is no session bus, or no Secret Service on it. +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +const MISSING_SERVICE_ERRORS: [&str; 7] = [ + "org.freedesktop.DBus.Error.ServiceUnknown", + "org.freedesktop.DBus.Error.NameHasNoOwner", + "org.freedesktop.DBus.Error.NoServer", + "org.freedesktop.DBus.Error.FileNotFound", + "org.freedesktop.DBus.Error.BadAddress", + // Autolaunching a session bus without X11 + "org.freedesktop.DBus.Error.NotSupported", + "org.freedesktop.DBus.Error.Spawn.ExecFailed", +]; + +/// `e` says there is no Secret Service to keep a key in: no session bus, or nothing on it +/// providing `org.freedesktop.secrets`. Every other failure, access denied, a locked +/// collection or a dismissed prompt among them, is not, so it stays an error rather than +/// putting the key in a file. +#[cfg(target_os = "linux")] +fn is_missing_secret_service(e: &Error) -> bool { + let Error::PlatformFailure(inner) = e else { + return false; + }; + match inner.downcast_ref::() { + Some(dbus_secret_service::Error::Unavailable) => true, + Some(dbus_secret_service::Error::Dbus(e)) => { + e.name().is_some_and(|name| MISSING_SERVICE_ERRORS.contains(&name)) + } + _ => false, + } +} + +/// Only Linux keeps keys in a service that may not be running. +#[cfg(not(target_os = "linux"))] +fn is_missing_secret_service(_e: &Error) -> bool { + false +} + +fn read_secret_file(path: &Path) -> Result> { + match std::fs::read_to_string(path) { + Ok(secret) => Ok(Some(secret.trim().to_string())), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(e.into()), + } +} + +/// Writes `secret` to `path`, readable and writable by the user alone where the platform has +/// such modes. +pub(crate) fn write_secret_file(path: &Path, secret: &str) -> Result<()> { + use std::io::Write; + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + let mut options = std::fs::OpenOptions::new(); + options.write(true).create(true).truncate(true); + #[cfg(unix)] + std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600); + let mut file = options.open(path)?; + // A file that already existed keeps its mode through `open`. + #[cfg(unix)] + file.set_permissions(std::os::unix::fs::PermissionsExt::from_mode(0o600))?; + file.write_all(secret.as_bytes())?; + file.sync_all()?; + Ok(()) +} + +#[cfg(any(test, feature = "test-support"))] +mod test_support { + use super::SecretStore; + use crate::error::Error::GenericError; + use crate::error::Result; + use std::sync::Mutex; + + #[derive(Default)] + pub(super) struct MemoryStore(Mutex>); + + impl SecretStore for MemoryStore { + fn get(&self) -> Result> { + Ok(self.0.lock().unwrap().clone()) + } + + fn set(&self, secret: &str) -> Result<()> { + *self.0.lock().unwrap() = Some(secret.to_string()); + Ok(()) + } + } + + pub(super) struct RefusingStore(pub(super) String); + + impl SecretStore for RefusingStore { + fn get(&self) -> Result> { + Err(GenericError(self.0.clone())) + } + + fn set(&self, _secret: &str) -> Result<()> { + Err(GenericError(self.0.clone())) + } + } +} + #[cfg(test)] mod tests { + use super::*; use crate::error::Result; - use crate::master_key::MasterKey; #[test] fn test_master_key() -> Result<()> { @@ -75,4 +297,89 @@ mod tests { Ok(()) } + + #[test] + fn creates_a_master_key_once() -> Result<()> { + let source = MasterKeySource::memory(); + assert!(MasterKey::get_in(source.store())?.is_none()); + let created = MasterKey::get_or_create_in(source.store())?; + let stored = source.store().get()?.unwrap(); + assert!(stored.starts_with(HUMAN_PREFIX), "{stored}"); + + let again = MasterKey::get_or_create_in(source.store())?; + assert_eq!(again.key, created.key); + assert_eq!(source.store().get()?.unwrap(), stored); + Ok(()) + } + + /// Several asking at once for the first time all get the one key that is stored. + #[test] + fn concurrent_first_creates_agree() { + let source = MasterKeySource::memory(); + let barrier = std::sync::Barrier::new(8); + let keys: Vec<_> = std::thread::scope(|s| { + let handles: Vec<_> = (0..8) + .map(|_| { + s.spawn(|| { + barrier.wait(); + MasterKey::get_or_create_in(source.store()).unwrap().key + }) + }) + .collect(); + handles.into_iter().map(|h| h.join().unwrap()).collect() + }); + let stored = MasterKey::get_in(source.store()).unwrap().unwrap().key; + assert!(keys.iter().all(|k| *k == stored)); + } + + #[test] + fn a_refusing_store_gives_no_key() { + let source = MasterKeySource::refusing("User canceled the operation"); + let err = MasterKey::get_or_create_in(source.store()).unwrap_err(); + assert_eq!(err.to_string(), "Encryption error: User canceled the operation"); + } + + #[test] + fn the_fallback_file_is_private_and_read_back() -> Result<()> { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("keys").join("master.key"); + assert_eq!(read_secret_file(&path)?, None); + + write_secret_file(&path, "YKM_ABC")?; + assert_eq!(read_secret_file(&path)?.as_deref(), Some("YKM_ABC")); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(&path)?.permissions().mode(); + assert_eq!(mode & 0o777, 0o600); + } + + // Rewritten, it stays private and holds only the new key. + write_secret_file(&path, "YKM_D")?; + assert_eq!(read_secret_file(&path)?.as_deref(), Some("YKM_D")); + Ok(()) + } + + #[test] + fn falls_back_only_without_a_store() { + let store = KeychainStore::new("app.yaak.test", "u").with_fallback_file("k".into()); + // Failures that do not say the Secret Service is missing, such as a denied D-Bus call, + // which the backend also reports as a platform failure. + let other = Error::PlatformFailure("org.freedesktop.DBus.Error.AccessDenied".into()); + let denied = Error::NoStorageAccess("prompt dismissed".into()); + assert!(store.fallback_for(&other).is_none()); + assert!(store.fallback_for(&denied).is_none()); + assert!(store.fallback_for(&Error::NoEntry).is_none()); + } + + #[cfg(target_os = "linux")] + #[test] + fn falls_back_on_linux_without_a_secret_service() { + let store = KeychainStore::new("app.yaak.test", "u").with_fallback_file("k".into()); + let missing = Error::PlatformFailure(Box::new(dbus_secret_service::Error::Unavailable)); + assert!(store.fallback_for(&missing).is_some()); + assert!(KeychainStore::new("app.yaak.test", "u").fallback_for(&missing).is_none()); + let locked = Error::PlatformFailure(Box::new(dbus_secret_service::Error::Locked)); + assert!(store.fallback_for(&locked).is_none()); + } } diff --git a/crates/yaak-models/Cargo.toml b/crates/yaak-models/Cargo.toml index 3f2f6a5ce..f7014636c 100644 --- a/crates/yaak-models/Cargo.toml +++ b/crates/yaak-models/Cargo.toml @@ -24,6 +24,10 @@ ts-rs = { workspace = true, features = ["chrono-impl", "serde-json-impl"] } yaak-core = { workspace = true } yaak-templates = { path = "../yaak-templates", default-features = false } +[features] +# Opening databases encrypted with SQLCipher; see `yaak_database::encryption`. +sqlcipher = ["yaak-database/sqlcipher"] + [target.'cfg(not(target_arch = "wasm32"))'.dependencies] r2d2 = "0.8.10" r2d2_sqlite = { version = "0.32" } diff --git a/crates/yaak-models/src/blob_manager.rs b/crates/yaak-models/src/blob_manager.rs index 4f79df7b1..59e3ece52 100644 --- a/crates/yaak-models/src/blob_manager.rs +++ b/crates/yaak-models/src/blob_manager.rs @@ -223,6 +223,16 @@ impl BodyChunkWriter { Ok(self.total_bytes) } + /// [`Self::finish`] without inserting the partial last chunk: it is returned for the caller + /// to insert, in a transaction it shares with other writes. + pub fn finish_deferred(mut self) -> Option { + if self.buf.is_empty() { + return None; + } + let data = std::mem::take(&mut self.buf); + Some(BodyChunk::new(&self.body_id, self.chunk_index, data)) + } + fn insert(&mut self, data: Vec) -> Result<()> { let chunk = BodyChunk::new(&self.body_id, self.chunk_index, data); self.blob_manager.with_tx(|b| b.insert_chunk(&chunk))?; @@ -318,6 +328,21 @@ mod tests { .unwrap(); } + /// The last chunk is handed back unwritten, after the full chunks written as they filled. + #[test] + fn finish_deferred_returns_the_last_chunk_unwritten() { + let manager = create_test_manager(); + let mut writer = BodyChunkWriter::new(manager.clone(), "fl_1.response"); + writer.write(&vec![b'a'; BODY_CHUNK_SIZE + 3]).unwrap(); + let last = writer.finish_deferred().unwrap(); + assert_eq!((last.body_id.as_str(), last.chunk_index), ("fl_1.response", 1)); + assert_eq!(last.data, b"aaa"); + assert_eq!(manager.connect().get_chunks("fl_1.response").unwrap().len(), 1); + + let empty = BodyChunkWriter::new(manager.clone(), "fl_2.response"); + assert!(empty.finish_deferred().is_none()); + } + #[test] fn test_insert_and_get_chunks() { let manager = create_test_manager(); diff --git a/crates/yaak-models/src/lib.rs b/crates/yaak-models/src/lib.rs index fdcc8f1ee..899c9d359 100644 --- a/crates/yaak-models/src/lib.rs +++ b/crates/yaak-models/src/lib.rs @@ -58,6 +58,31 @@ mod open { std::fs::create_dir_all(parent)?; } let manager = SqliteConnectionManager::file(path).with_init(|c| init_file_connection(c)); + build(manager, max_size, min_idle) + } + + /// [`file_pool`] over a database encrypted with `key`, keying each connection first. + #[cfg(feature = "sqlcipher")] + pub fn encrypted_file_pool( + path: impl Into, + max_size: u32, + min_idle: u32, + key: &yaak_database::encryption::DatabaseKey, + ) -> Result { + let path: PathBuf = path.into(); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + yaak_database::encryption::init_crypto(); + let key = key.clone(); + let manager = SqliteConnectionManager::file(path).with_init(move |c| { + yaak_database::encryption::apply_key(c, &key)?; + init_file_connection(c) + }); + build(manager, max_size, min_idle) + } + + fn build(manager: SqliteConnectionManager, max_size: u32, min_idle: u32) -> Result { Pool::builder() .max_size(max_size) .min_idle(Some(min_idle)) @@ -160,6 +185,22 @@ pub fn init_blob_store(blob_path: impl AsRef) -> Result { Ok(BlobManager::new(blob_readers, blob_writer)) } +/// [`init_blob_store`] for a blobs database encrypted with `key`, as +/// [`yaak_database::encryption`] makes one. A plain one is converted first with +/// [`yaak_database::encryption::encrypt_in_place`]. +#[cfg(all(feature = "sqlcipher", not(target_arch = "wasm32")))] +pub fn init_encrypted_blob_store( + blob_path: impl AsRef, + key: &yaak_database::encryption::DatabaseKey, +) -> Result { + let blob_path = blob_path.as_ref(); + info!("Initializing encrypted blobs database {blob_path:?}"); + let readers = open::encrypted_file_pool(blob_path, 10, 1, key)?; + let writer = open::encrypted_file_pool(blob_path, 1, 1, key)?; + migrate_blob_db(&writer)?; + Ok(BlobManager::new(readers, writer)) +} + /// Initialize the database managers with in-memory SQLite databases. /// Useful for testing and CI environments. /// diff --git a/crates/yaak-proxy/src/cert.rs b/crates/yaak-proxy/src/cert.rs index a3674ef35..e07112d0a 100644 --- a/crates/yaak-proxy/src/cert.rs +++ b/crates/yaak-proxy/src/cert.rs @@ -28,10 +28,28 @@ pub struct CaStore { leaf_configs: Mutex>>, } +/// Encrypts the CA's private key at rest, for [`CaStore::load_or_create_encrypted`]. +pub trait KeyCipher: Send + Sync { + fn encrypt(&self, data: &[u8]) -> Result, String>; + fn decrypt(&self, data: &[u8]) -> Result, String>; +} + impl CaStore { /// Loads the CA from `dir`, or creates one there (key mode 0600) if either file is missing. /// Safe to call from several processes at once: `ca.lock` serializes them. pub fn load_or_create(dir: &Path) -> io::Result { + Self::open(dir, None) + } + + /// [`Self::load_or_create`], keeping `ca.key` encrypted with `cipher`. A plaintext `ca.key`, + /// as builds before encryption wrote, is encrypted in place. One `cipher` cannot decrypt, + /// encrypted with a key since lost, is replaced along with the certificate: unusable, it + /// could only leave the proxy unable to start, and the new one has to be trusted again. + pub fn load_or_create_encrypted(dir: &Path, cipher: &dyn KeyCipher) -> io::Result { + Self::open(dir, Some(cipher)) + } + + fn open(dir: &Path, cipher: Option<&dyn KeyCipher>) -> io::Result { let cert_path = dir.join("ca.pem"); let key_path = dir.join("ca.key"); @@ -43,22 +61,26 @@ impl CaStore { .open(dir.join("ca.lock"))?; lock.lock()?; - let (pem, key) = if cert_path.exists() && key_path.exists() { - let key = - KeyPair::from_pem(&fs::read_to_string(&key_path)?).map_err(io::Error::other)?; - (fs::read_to_string(&cert_path)?, key) + let stored = if cert_path.exists() && key_path.exists() { + load_key(&key_path, cipher)? } else { - log::info!("Creating proxy CA in {}", dir.display()); - let key = KeyPair::generate().map_err(io::Error::other)?; - let now = OffsetDateTime::now_utc(); - let mut params = ca_params(); - params.not_before = now - Duration::days(1); - params.not_after = now + Duration::days(CA_VALIDITY_DAYS); - let pem = params.self_signed(&key).map_err(io::Error::other)?.pem(); - remove_if_exists(&cert_path)?; - write_atomic(&key_path, key.serialize_pem().as_bytes(), true)?; - write_atomic(&cert_path, pem.as_bytes(), false)?; - (pem, key) + None + }; + let (pem, key) = match stored { + Some(key) => (fs::read_to_string(&cert_path)?, key), + None => { + log::info!("Creating proxy CA in {}", dir.display()); + let key = KeyPair::generate().map_err(io::Error::other)?; + let now = OffsetDateTime::now_utc(); + let mut params = ca_params(); + params.not_before = now - Duration::days(1); + params.not_after = now + Duration::days(CA_VALIDITY_DAYS); + let pem = params.self_signed(&key).map_err(io::Error::other)?.pem(); + remove_if_exists(&cert_path)?; + write_key(&key_path, &key, cipher)?; + write_atomic(&cert_path, pem.as_bytes(), false)?; + (pem, key) + } }; drop(lock); @@ -179,6 +201,48 @@ fn ca_params() -> CertificateParams { params } +/// The key in `ca.key`: PEM as written without a cipher, or encrypted with one. A plaintext key +/// is encrypted in place when there is a cipher. None when the cipher cannot decrypt it. +fn load_key(path: &Path, cipher: Option<&dyn KeyCipher>) -> io::Result> { + let stored = fs::read(path)?; + if stored.starts_with(b"-----BEGIN") { + let pem = String::from_utf8(stored).map_err(io::Error::other)?; + let key = KeyPair::from_pem(&pem).map_err(io::Error::other)?; + if cipher.is_some() { + write_key(path, &key, cipher)?; + log::info!("Encrypted the proxy CA's private key in {}", path.display()); + } + return Ok(Some(key)); + } + let Some(cipher) = cipher else { + return Err(io::Error::other(format!( + "{} is encrypted, with no key to read it", + path.display() + ))); + }; + let pem = match cipher.decrypt(&stored) { + Ok(pem) => String::from_utf8(pem).map_err(io::Error::other)?, + Err(e) => { + log::warn!( + "Failed to decrypt {} ({e}), so the proxy CA is replaced with a new one", + path.display() + ); + return Ok(None); + } + }; + KeyPair::from_pem(&pem).map(Some).map_err(io::Error::other) +} + +/// Writes `ca.key`, mode 0600, encrypted when there is a cipher. +fn write_key(path: &Path, key: &KeyPair, cipher: Option<&dyn KeyCipher>) -> io::Result<()> { + let pem = key.serialize_pem(); + let contents = match cipher { + Some(cipher) => cipher.encrypt(pem.as_bytes()).map_err(io::Error::other)?, + None => pem.into_bytes(), + }; + write_atomic(path, &contents, true) +} + fn remove_if_exists(path: &Path) -> io::Result<()> { match fs::remove_file(path) { Err(e) if e.kind() != io::ErrorKind::NotFound => Err(e), @@ -216,6 +280,76 @@ mod tests { use super::*; use std::sync::mpsc; + /// Flips every bit behind a marker that names the key, so a wrong key fails to decrypt. + struct TestCipher(u8); + + impl KeyCipher for TestCipher { + fn encrypt(&self, data: &[u8]) -> Result, String> { + Ok([b'k', self.0].into_iter().chain(data.iter().map(|b| !b)).collect()) + } + + fn decrypt(&self, data: &[u8]) -> Result, String> { + let rest = data.strip_prefix(&[b'k', self.0][..]).ok_or("wrong key")?; + Ok(rest.iter().map(|b| !b).collect()) + } + } + + fn private(path: &Path) { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!(fs::metadata(path).unwrap().permissions().mode() & 0o777, 0o600); + } + #[cfg(not(unix))] + let _ = path; + } + + #[test] + fn keeps_the_key_encrypted() { + let dir = tempfile::tempdir().unwrap(); + let created = CaStore::load_or_create_encrypted(dir.path(), &TestCipher(1)).unwrap(); + let stored = fs::read(created.key_path()).unwrap(); + assert!(!stored.windows(11).any(|w| w == b"PRIVATE KEY"), "ca.key is plaintext"); + private(created.key_path()); + // The certificate is public, and stays as it was. + assert!(fs::read_to_string(created.cert_path()).unwrap().starts_with("-----BEGIN")); + + let loaded = CaStore::load_or_create_encrypted(dir.path(), &TestCipher(1)).unwrap(); + assert_eq!(loaded.pem(), created.pem()); + assert_eq!(fs::read(loaded.key_path()).unwrap(), stored, "rewritten"); + loaded.server_config("example.test", true).unwrap(); + + let err = CaStore::load_or_create(dir.path()).err().unwrap(); + assert!(err.to_string().ends_with("ca.key is encrypted, with no key to read it"), "{err}"); + } + + #[test] + fn encrypts_a_plaintext_key_in_place() { + let dir = tempfile::tempdir().unwrap(); + let plain = CaStore::load_or_create(dir.path()).unwrap(); + let pem_key = fs::read_to_string(plain.key_path()).unwrap(); + + let encrypted = CaStore::load_or_create_encrypted(dir.path(), &TestCipher(1)).unwrap(); + assert_eq!(encrypted.pem(), plain.pem(), "the same CA, still trusted"); + let stored = fs::read(encrypted.key_path()).unwrap(); + assert_eq!(TestCipher(1).decrypt(&stored).unwrap(), pem_key.as_bytes()); + private(encrypted.key_path()); + assert!(!dir.path().join("ca.key.tmp").exists()); + } + + #[test] + fn replaces_a_ca_whose_key_cannot_be_decrypted() { + let dir = tempfile::tempdir().unwrap(); + let lost = CaStore::load_or_create_encrypted(dir.path(), &TestCipher(1)).unwrap(); + + let replaced = CaStore::load_or_create_encrypted(dir.path(), &TestCipher(2)).unwrap(); + assert_ne!(replaced.pem(), lost.pem()); + let stored = fs::read(replaced.key_path()).unwrap(); + assert!(TestCipher(2).decrypt(&stored).is_ok()); + let again = CaStore::load_or_create_encrypted(dir.path(), &TestCipher(2)).unwrap(); + assert_eq!(again.pem(), replaced.pem()); + } + /// A host whose leaf is still being signed holds up no lookup of another host's. #[test] fn signing_one_host_does_not_block_another() { diff --git a/crates/yaak-proxy/src/lib.rs b/crates/yaak-proxy/src/lib.rs index b5d8a8fe5..3c973569d 100644 --- a/crates/yaak-proxy/src/lib.rs +++ b/crates/yaak-proxy/src/lib.rs @@ -30,7 +30,7 @@ use rustls::ClientConfig; use rustls::pki_types::CertificateDer; use tokio::net::TcpListener; -pub use cert::CaStore; +pub use cert::{CaStore, KeyCipher}; use connection::handle_connection; pub use grpc::{detect as detect_grpc, service_and_method}; pub use messages::{MessageDirection, MessageKind, RAW_LIMIT, StreamKind, StreamMessage};