Skip Private Relay for the proxy engine's plain-HTTP upstream on macOS (#818)

Cargo.lock

@@ -11878,13 +11878,16 @@ name = "yaak-proxy"
 version = "0.1.0"
 dependencies = [
  "base64 0.23.1",
+ "block2 0.6.1",
  "bytes",
+ "dispatch2",
  "flate2",
  "futures-util",
  "http",
  "http-body-util",
  "hyper",
  "hyper-util",
+ "libc",
  "log 0.4.34",
  "pem",
  "rcgen",
@@ -11896,6 +11899,7 @@ dependencies = [
  "tokio",
  "tokio-rustls",
  "tokio-tungstenite",
+ "tower-service",
 ]
 
 [[package]]

crates/yaak-proxy/Cargo.toml

@@ -28,6 +28,12 @@ tokio-rustls = "0.26"
 pem = "3"
 time = "0.3"
 
+[target.'cfg(target_os = "macos")'.dependencies]
+block2 = "0.6"
+dispatch2 = "0.3"
+libc = "0.2"
+tower-service = "0.3"
+
 [dev-dependencies]
 futures-util = "0.3"
 hyper-util = { version = "0.1", features = ["server-auto"] }

crates/yaak-proxy/src/lib.rs

@@ -4,6 +4,8 @@ mod connection;
 mod decode;
 mod grpc;
 mod messages;
+#[cfg(target_os = "macos")]
+mod nw;
 mod request;
 mod sse;
 mod tunnel;
@@ -21,6 +23,7 @@ use std::time::SystemTime;
 use bytes::Bytes;
 use http::HeaderValue;
 use hyper_util::client::legacy::Client;
+#[cfg(not(target_os = "macos"))]
 use hyper_util::client::legacy::connect::HttpConnector;
 use hyper_util::rt::TokioExecutor;
 use rustls::ClientConfig;
@@ -242,7 +245,7 @@ pub(crate) struct Engine {
     events: std_mpsc::Sender<ProxyEvent>,
     next_id: AtomicU64,
     next_connection: AtomicU64,
-    http_client: Client<HttpConnector, BoxBody>,
+    http_client: HttpClient,
     upstream_tls: Arc<ClientConfig>,
     upstream_tls_h1: Arc<ClientConfig>,
     passthrough: Mutex<HashSet<String>>,
@@ -250,8 +253,8 @@ pub(crate) struct Engine {
     /// Offer HTTP/2 to clients in intercepted TLS.
     http2: AtomicBool,
     websocket_capture: bool,
-    /// `Via` value marking requests whose URL the engine rebuilt from `Host`, unique to this run,
+    /// `Via` value marking the plain-HTTP requests the engine forwards, unique to this run, so one
-    /// so one that comes back to the engine is refused instead of forwarded again.
+    /// that comes back to the engine is refused instead of forwarded again.
     via: HeaderValue,
 }
 
@@ -305,6 +308,21 @@ fn upstream_tls_config(
     Ok(Arc::new(config))
 }
 
+/// Sends plain HTTP upstream. On macOS it connects through Network.framework, so Private Relay
+/// can't route the engine's own port-80 connections back into the engine (see [`nw`]).
+#[cfg(target_os = "macos")]
+type HttpClient = Client<nw::Connector, BoxBody>;
+#[cfg(not(target_os = "macos"))]
+type HttpClient = Client<HttpConnector, BoxBody>;
+
+fn http_client() -> HttpClient {
+    let builder = Client::builder(TokioExecutor::new());
+    #[cfg(target_os = "macos")]
+    return builder.build(nw::Connector);
+    #[cfg(not(target_os = "macos"))]
+    builder.build_http()
+}
+
 fn via_marker() -> HeaderValue {
     let run = RandomState::new().build_hasher().finish();
     HeaderValue::try_from(format!("1.1 yaak-proxy-{run:016x}")).unwrap()
@@ -327,7 +345,7 @@ pub fn start_proxy(config: ProxyConfig) -> Result<ProxyHandle, String> {
         events: event_tx,
         next_id: AtomicU64::new(1),
         next_connection: AtomicU64::new(1),
-        http_client: Client::builder(TokioExecutor::new()).build_http(),
+        http_client: http_client(),
         upstream_tls: upstream_tls_config(&roots, &[b"h2", b"http/1.1"])?,
         upstream_tls_h1: upstream_tls_config(&roots, &[b"http/1.1"])?,
         passthrough: Mutex::new(HashSet::new()),

crates/yaak-proxy/src/request.rs

@@ -17,6 +17,7 @@ use hyper::http::request::Parts;
 use hyper::http::uri::{Authority, PathAndQuery, Scheme};
 use hyper::upgrade::OnUpgrade;
 use hyper::{Method, Request, Response, StatusCode, Uri};
+#[cfg(not(target_os = "macos"))]
 use hyper_util::client::legacy::connect::HttpInfo;
 use hyper_util::rt::{TokioExecutor, TokioIo};
 use log::{debug, info, warn};
@@ -296,35 +297,60 @@ pub(crate) async fn handle_request(
         return Ok(handle_connect(req, client, engine));
     }
 
+    // A request the engine sent that was routed back to it, as iCloud Private Relay does with
+    // port 80 while the system proxy is on: forwarding it again would loop forever. It is the
+    // engine's own copy of a flow already recorded, which gets this answer, so it records none.
+    if carries_via(&req, &engine) {
+        debug!("Refused {} {}: it looped back to Yaak Proxy", req.method(), req.uri());
+        return Ok(Response::builder()
+            .status(StatusCode::LOOP_DETECTED)
+            .header(CONTENT_TYPE, "text/plain; charset=utf-8")
+            .body(text_body("request looped back to Yaak Proxy"))
+            .unwrap());
+    }
+
     // Some clients send a bare path (`POST /api`) as they would to the server itself.
     let from_host = req.uri().authority().is_none();
-    let rejected = if from_host { url_from_host(&mut req, &engine).err() } else { None };
+    let rejected = if from_host { url_from_host(&mut req).err() } else { None };
     let flow = Flow::begin(engine.clone(), client, &req, req.uri().to_string());
     if let Some((status, error)) = rejected {
         return Ok(flow.reject(status, error));
     }
-    if from_host {
+    req.headers_mut().append(VIA, engine.via.clone());
-        req.headers_mut().append(VIA, engine.via.clone());
-    }
 
     let (parts, body) = req.into_parts();
     let outgoing = Request::from_parts(parts, flow.capture_request(body));
     Ok(match engine.http_client.request(outgoing).await {
         Ok(resp) => {
-            let remote_addr = resp.extensions().get::<HttpInfo>().map(|i| i.remote_addr());
+            let remote_addr = plain_remote_addr(&resp);
             flow.respond(resp, remote_addr)
         }
         Err(e) => flow.fail(error_chain(&e)),
     })
 }
 
+/// The address `Engine::http_client` connected to for `resp`.
+fn plain_remote_addr(resp: &Response<Incoming>) -> Option<SocketAddr> {
+    #[cfg(target_os = "macos")]
+    return resp.extensions().get::<crate::nw::RemoteAddr>().map(|a| a.0);
+    #[cfg(not(target_os = "macos"))]
+    resp.extensions().get::<HttpInfo>().map(|i| i.remote_addr())
+}
+
+/// Whether `req` carries this engine's `Via` mark, which it adds to every plain-HTTP request it
+/// forwards.
+fn carries_via(req: &Request<Incoming>, engine: &Engine) -> bool {
+    let via = engine.via.to_str().unwrap_or_default();
+    req.headers()
+        .get_all(VIA)
+        .iter()
+        .filter_map(|v| v.to_str().ok())
+        .any(|v| v.split(',').any(|hop| hop.trim() == via))
+}
+
 /// Gives a bare-path request the `http://` URL its `Host` names. Fails when there is no usable
-/// `Host`, or when the request carries this engine's `Via` mark: then it is one the engine sent
+/// `Host`.
-/// that was routed back to it, and forwarding it again would loop forever.
+fn url_from_host(req: &mut Request<Incoming>) -> Result<(), (StatusCode, &'static str)> {
-fn url_from_host(
-    req: &mut Request<Incoming>,
-    engine: &Engine,
-) -> Result<(), (StatusCode, &'static str)> {
     let Some(host) = req.headers().get(HOST) else {
         return Err((StatusCode::BAD_REQUEST, "no host in request"));
     };
@@ -339,17 +365,6 @@ fn url_from_host(
     }
     *req.uri_mut() =
         Uri::from_parts(parts).map_err(|_| (StatusCode::BAD_REQUEST, "invalid host in request"))?;
-
-    let via = engine.via.to_str().unwrap_or_default();
-    let looped = req
-        .headers()
-        .get_all(VIA)
-        .iter()
-        .filter_map(|v| v.to_str().ok())
-        .any(|v| v.split(',').any(|hop| hop.trim() == via));
-    if looped {
-        return Err((StatusCode::LOOP_DETECTED, "request looped back to Yaak Proxy"));
-    }
     Ok(())
 }
 

crates/yaak-proxy/src/tunnel.rs

@@ -8,13 +8,43 @@ use hyper_util::rt::TokioIo;
 use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, ReadBuf, copy_bidirectional};
 use tokio::net::TcpStream;
 
-/// Copies bytes both ways between `client` and `target`, after sending `prefix` (bytes already
+/// Copies bytes both ways between `client` and `target` (`host:port`), after sending `prefix`
-/// read from the client) upstream.
+/// (bytes already read from the client) upstream.
-pub(crate) async fn tunnel<S>(mut client: S, target: &str, prefix: &[u8]) -> io::Result<()>
+pub(crate) async fn tunnel<S>(client: S, target: &str, prefix: &[u8]) -> io::Result<()>
 where
     S: AsyncRead + AsyncWrite + Unpin,
 {
-    let mut upstream = TcpStream::connect(target).await?;
+    // Private Relay sends public port-80 connections into the system proxy, which may be this
+    // engine.
+    #[cfg(target_os = "macos")]
+    if let Some((host, "80")) = target.rsplit_once(':') {
+        return tunnel_skipping_proxy(client, host, 80, prefix).await;
+    }
+    relay(client, TcpStream::connect(target).await?, prefix).await
+}
+
+/// [`tunnel`] through Network.framework, which can skip the system proxy. When it can't, it falls
+/// back to the system proxy with a `CONNECT` that arrives here, and tunneling that out the same
+/// way would repeat it.
+#[cfg(target_os = "macos")]
+async fn tunnel_skipping_proxy<S>(client: S, host: &str, port: u16, prefix: &[u8]) -> io::Result<()>
+where
+    S: AsyncRead + AsyncWrite + Unpin,
+{
+    let host = host.trim_start_matches('[').trim_end_matches(']');
+    if crate::nw::is_own_fallback(host, port).await {
+        return Err(io::Error::other(format!(
+            "not tunneling Yaak Proxy's own connection to {host}:{port} back out"
+        )));
+    }
+    relay(client, crate::nw::connect(host, port).await?, prefix).await
+}
+
+async fn relay<S, U>(mut client: S, mut upstream: U, prefix: &[u8]) -> io::Result<()>
+where
+    S: AsyncRead + AsyncWrite + Unpin,
+    U: AsyncRead + AsyncWrite + Unpin,
+{
     upstream.write_all(prefix).await?;
     copy_bidirectional(&mut client, &mut upstream).await?;
     Ok(())
@@ -83,3 +113,50 @@ impl<S: AsyncWrite + Unpin> AsyncWrite for Rewind<S> {
         Pin::new(&mut self.inner).poll_shutdown(cx)
     }
 }
+
+#[cfg(all(test, target_os = "macos"))]
+mod tests {
+    use std::time::Duration;
+
+    use tokio::io::{AsyncReadExt, duplex};
+    use tokio::net::TcpListener;
+
+    use super::*;
+
+    /// A port-80 tunnel on macOS goes through Network.framework: the prefix goes first, bytes
+    /// cross both ways, and each side's close reaches the other.
+    #[tokio::test(flavor = "multi_thread")]
+    async fn tunnel_skipping_proxy_relays_both_ways_and_closes() {
+        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let port = listener.local_addr().unwrap().port();
+        tokio::spawn(async move {
+            let (mut socket, _) = listener.accept().await.unwrap();
+            let (mut read, mut write) = socket.split();
+            tokio::io::copy(&mut read, &mut write).await.unwrap();
+            write.shutdown().await.unwrap();
+        });
+
+        let (client, proxy_side) = duplex(1024);
+        let tunnel = tokio::spawn(async move {
+            tunnel_skipping_proxy(proxy_side, "localhost", port, b"first,").await
+        });
+
+        let payload: Vec<u8> = (0..300_000).map(|i| (i % 251) as u8).collect();
+        let (mut read, mut write) = tokio::io::split(client);
+        let (echoed, ()) = tokio::join!(
+            async {
+                let mut echoed = Vec::new();
+                read.read_to_end(&mut echoed).await.unwrap();
+                echoed
+            },
+            async {
+                write.write_all(&payload).await.unwrap();
+                write.shutdown().await.unwrap();
+            },
+        );
+        assert_eq!(&echoed[..6], b"first,");
+        assert!(echoed[6..] == payload[..], "echoed bytes differ from the payload");
+        let ended = tokio::time::timeout(Duration::from_secs(10), tunnel).await;
+        ended.expect("the tunnel stayed open").unwrap().unwrap();
+    }
+}

crates/yaak-proxy/tests/own_fallback.rs

@@ -0,0 +1,118 @@
+//! The engine's own upstream falling back to the system proxy, which is the engine. Its own test
+//! binary, since the proxy is set for the whole process.
+#![cfg(target_os = "macos")]
+
+use std::ffi::{CString, c_void};
+use std::sync::Arc;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::time::Duration;
+
+use rustls::pki_types::ServerName;
+use rustls::{ClientConfig, RootCertStore};
+use tempfile::TempDir;
+use tokio::io::{AsyncReadExt, AsyncWriteExt, copy_bidirectional};
+use tokio::net::{TcpListener, TcpStream};
+use tokio_rustls::TlsConnector;
+use yaak_proxy::{CaStore, ProxyConfig, start_proxy};
+
+type NwObject = *mut c_void;
+
+#[link(name = "Network", kind = "framework")]
+unsafe extern "C" {
+    static _nw_privacy_context_default_context: NwObject;
+    fn nw_endpoint_create_host(host: *const i8, port: *const i8) -> NwObject;
+    fn nw_proxy_config_create_http_connect(endpoint: NwObject, tls: NwObject) -> NwObject;
+    fn nw_proxy_config_set_failover_allowed(proxy_config: NwObject, allowed: bool);
+    fn nw_privacy_context_add_proxy(privacy_context: NwObject, proxy_config: NwObject);
+    fn nw_release(obj: NwObject);
+}
+
+/// Points every Network.framework connection in the process at an HTTP CONNECT proxy on
+/// `127.0.0.1:port`, as a system proxy would.
+fn set_process_proxy(port: u16) {
+    let host = CString::new("127.0.0.1").unwrap();
+    let port = CString::new(port.to_string()).unwrap();
+    // SAFETY: each object created here is released once, after the context retains it.
+    unsafe {
+        let endpoint = nw_endpoint_create_host(host.as_ptr(), port.as_ptr());
+        let config = nw_proxy_config_create_http_connect(endpoint, std::ptr::null_mut());
+        nw_proxy_config_set_failover_allowed(config, true);
+        nw_privacy_context_add_proxy(_nw_privacy_context_default_context, config);
+        nw_release(config);
+        nw_release(endpoint);
+    }
+}
+
+async fn read_head(stream: &mut (impl AsyncReadExt + Unpin)) -> String {
+    let mut head = Vec::new();
+    while !head.ends_with(b"\r\n\r\n") {
+        head.push(stream.read_u8().await.unwrap());
+    }
+    String::from_utf8(head).unwrap()
+}
+
+/// A host the engine passes through is tunneled as soon as its `CONNECT` arrives. When that
+/// `CONNECT` is the engine's own upstream falling back to the proxy, tunneling it out again would
+/// fall back again, without end. One `CONNECT`, then the client gets a 502.
+#[tokio::test(flavor = "multi_thread")]
+async fn own_fallback_through_a_passthrough_host_stops() {
+    const TARGET: &str = "yaak-proxy-own-fallback.invalid";
+    let dir = TempDir::new().unwrap();
+    let ca = Arc::new(CaStore::load_or_create(dir.path()).unwrap());
+    let proxy = start_proxy(ProxyConfig::new(0, ca)).unwrap();
+    let engine_port = proxy.port;
+
+    // Counts the CONNECTs on their way from Network.framework to the engine.
+    let gateway = TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let gateway_port = gateway.local_addr().unwrap().port();
+    let connects = Arc::new(AtomicUsize::new(0));
+    let seen = connects.clone();
+    tokio::spawn(async move {
+        loop {
+            let (mut inbound, _) = gateway.accept().await.unwrap();
+            let seen = seen.clone();
+            tokio::spawn(async move {
+                let head = read_head(&mut inbound).await;
+                if head.starts_with("CONNECT ") {
+                    seen.fetch_add(1, Ordering::SeqCst);
+                }
+                let mut engine = TcpStream::connect(("127.0.0.1", engine_port)).await.unwrap();
+                engine.write_all(head.as_bytes()).await.unwrap();
+                let _ = copy_bidirectional(&mut inbound, &mut engine).await;
+            });
+        }
+    });
+    set_process_proxy(gateway_port);
+
+    // A client that rejects the engine's certificate gets the target passed through.
+    let mut tcp = TcpStream::connect(("127.0.0.1", engine_port)).await.unwrap();
+    let connect = format!("CONNECT {TARGET}:80 HTTP/1.1\r\nHost: {TARGET}:80\r\n\r\n");
+    tcp.write_all(connect.as_bytes()).await.unwrap();
+    assert!(read_head(&mut tcp).await.starts_with("HTTP/1.1 200"));
+    let distrusting =
+        ClientConfig::builder_with_provider(Arc::new(rustls::crypto::ring::default_provider()))
+            .with_safe_default_protocol_versions()
+            .unwrap()
+            .with_root_certificates(RootCertStore::empty())
+            .with_no_client_auth();
+    let name = ServerName::try_from(TARGET).unwrap();
+    assert!(TlsConnector::from(Arc::new(distrusting)).connect(name, tcp).await.is_err());
+    let addr = format!("{TARGET}:80");
+    let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
+    while proxy.handshake_state(&addr) != (0, true) {
+        assert!(tokio::time::Instant::now() < deadline, "{addr} was never passed through");
+        tokio::time::sleep(Duration::from_millis(10)).await;
+    }
+
+    let mut client = TcpStream::connect(("127.0.0.1", engine_port)).await.unwrap();
+    let request = format!("GET http://{TARGET}/ HTTP/1.1\r\nHost: {TARGET}\r\n\r\n");
+    client.write_all(request.as_bytes()).await.unwrap();
+    let head = tokio::time::timeout(Duration::from_secs(10), read_head(&mut client))
+        .await
+        .expect("no response");
+    assert!(head.starts_with("HTTP/1.1 502"), "{head}");
+
+    // Give a runaway chain time to show.
+    tokio::time::sleep(Duration::from_secs(1)).await;
+    assert_eq!(connects.load(Ordering::SeqCst), 1);
+}

crates/yaak-proxy/tests/proxy.rs

@@ -1449,9 +1449,10 @@ async fn plain_http_bare_path_without_host_is_a_bad_request() {
     assert!(!events.iter().any(|e| matches!(e, ProxyEvent::ResponseStart { .. })));
 }
 
-/// A request whose connection upstream is routed back into the proxy, as happened to Telegram's
+/// A request whose connection upstream is routed back into the proxy, as Private Relay does with
-/// while the system proxy was on, arrives again with a bare path. Sending that on would loop
+/// port 80 while the system proxy is on, arrives again with the engine's `Via` mark. Sending
-/// forever.
+/// that on would loop forever. It is refused without a flow of its own, so the loop shows as the
+/// client's request answered with a 508.
 #[tokio::test(flavor = "multi_thread")]
 async fn plain_http_request_looped_back_to_the_proxy_stops() {
     let h = Harness::start(1024);
@@ -1462,8 +1463,33 @@ async fn plain_http_request_looped_back_to_the_proxy_stops() {
     assert_eq!(resp.text().await.unwrap(), "request looped back to Yaak Proxy");
 
     let events = h.flow_events();
-    let error = find!(events, ProxyEvent::Error { error, .. } => error.clone());
+    let starts: Vec<_> = events
-    assert_eq!(error, "request looped back to Yaak Proxy");
+        .iter()
+        .filter_map(|e| match e {
+            ProxyEvent::RequestStart { url, .. } => Some(url.clone()),
+            _ => None,
+        })
+        .collect();
+    assert_eq!(starts, [url]);
+    assert_eq!(find!(events, ProxyEvent::ResponseStart { status, .. } => *status), 508);
+    assert!(!events.iter().any(|e| matches!(e, ProxyEvent::Error { .. })));
+    let (body, _) = body(&events, false);
+    assert_eq!(body, b"request looped back to Yaak Proxy");
+}
+
+/// The same loop from a client that sends a bare path, as Telegram did.
+#[tokio::test(flavor = "multi_thread")]
+async fn bare_path_request_looped_back_to_the_proxy_stops() {
+    let h = Harness::start(1024);
+    let host = format!("127.0.0.1:{}", h.proxy.port);
+
+    let resp = h.send_as_written(telegram_request(Some(&host), "/api", &[0xefu8; 44])).await;
+    assert_eq!(resp.status(), 508);
+
+    let events = h.flow_events();
+    let flows: HashSet<u64> = events.iter().map(flow_id).collect();
+    assert_eq!(flows.len(), 1, "{events:?}");
+    assert_eq!(find!(events, ProxyEvent::ResponseStart { status, .. } => *status), 508);
 }
 
 // -- HTTP/2 clients --