Fix responses with more than 100 headers failing on HTTP/1 (#658)

Cargo.lock

@@ -685,6 +685,12 @@ version = "0.22.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
 
+[[package]]
+name = "base64"
+version = "0.23.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
+
 [[package]]
 name = "base64-simd"
 version = "0.8.0"
@@ -2942,7 +2948,7 @@ dependencies = [
  "libc",
  "libgit2-sys",
  "log 0.4.29",
- "openssl-probe",
+ "openssl-probe 0.1.6",
  "openssl-sys",
  "url",
 ]
@@ -4471,17 +4477,17 @@ dependencies = [
 
 [[package]]
 name = "native-tls"
-version = "0.2.14"
+version = "0.2.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e"
+checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2"
 dependencies = [
  "libc",
  "log 0.4.29",
  "openssl",
- "openssl-probe",
+ "openssl-probe 0.2.1",
  "openssl-sys",
  "schannel",
- "security-framework 2.11.1",
+ "security-framework 3.5.1",
  "security-framework-sys",
  "tempfile",
 ]
@@ -5055,6 +5061,12 @@ version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e"
 
+[[package]]
+name = "openssl-probe"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
+
 [[package]]
 name = "openssl-src"
 version = "300.6.1+3.6.3"
@@ -6717,12 +6729,11 @@ dependencies = [
 
 [[package]]
 name = "reqwest"
-version = "0.12.20"
+version = "0.13.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eabf4c97d9130e2bf606614eb937e86edac8292eaa6f422f995d7e8de1eb1813"
+checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029"
 dependencies = [
- "async-compression",
+ "base64 0.23.1",
- "base64 0.22.1",
  "bytes",
  "encoding_rs",
  "futures-core",
@@ -6744,6 +6755,7 @@ dependencies = [
  "pin-project-lite",
  "rustls",
  "rustls-pki-types",
+ "rustls-platform-verifier",
  "serde",
  "serde_json",
  "serde_urlencoded",
@@ -6758,46 +6770,7 @@ dependencies = [
  "url",
  "wasm-bindgen",
  "wasm-bindgen-futures",
- "wasm-streams 0.4.2",
+ "wasm-streams",
- "web-sys",
-]
-
-[[package]]
-name = "reqwest"
-version = "0.13.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0"
-dependencies = [
- "base64 0.22.1",
- "bytes",
- "futures-core",
- "futures-util",
- "http",
- "http-body",
- "http-body-util",
- "hyper",
- "hyper-rustls",
- "hyper-util",
- "js-sys",
- "log 0.4.29",
- "percent-encoding",
- "pin-project-lite",
- "rustls",
- "rustls-pki-types",
- "rustls-platform-verifier",
- "serde",
- "serde_json",
- "sync_wrapper",
- "tokio",
- "tokio-rustls",
- "tokio-util",
- "tower 0.5.2",
- "tower-http",
- "tower-service",
- "url",
- "wasm-bindgen",
- "wasm-bindgen-futures",
- "wasm-streams 0.5.0",
  "web-sys",
 ]
 
@@ -7431,7 +7404,7 @@ version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3"
 dependencies = [
- "openssl-probe",
+ "openssl-probe 0.1.6",
  "rustls-pki-types",
  "schannel",
  "security-framework 3.5.1",
@@ -7517,11 +7490,11 @@ dependencies = [
 
 [[package]]
 name = "schannel"
-version = "0.1.27"
+version = "0.1.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d"
+checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939"
 dependencies = [
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -8596,7 +8569,7 @@ dependencies = [
  "percent-encoding",
  "plist",
  "raw-window-handle",
- "reqwest 0.13.3",
+ "reqwest",
  "serde",
  "serde_json",
  "serde_repr",
@@ -8886,7 +8859,7 @@ dependencies = [
  "minisign-verify",
  "osakit",
  "percent-encoding",
- "reqwest 0.13.3",
+ "reqwest",
  "rustls",
  "semver",
  "serde",
@@ -10204,19 +10177,6 @@ dependencies = [
  "unicode-ident",
 ]
 
-[[package]]
-name = "wasm-streams"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65"
-dependencies = [
- "futures-util",
- "js-sys",
- "wasm-bindgen",
- "wasm-bindgen-futures",
- "web-sys",
-]
-
 [[package]]
 name = "wasm-streams"
 version = "0.5.0"
@@ -11240,7 +11200,7 @@ name = "yaak-api"
 version = "0.1.0"
 dependencies = [
  "log 0.4.29",
- "reqwest 0.12.20",
+ "reqwest",
  "sysproxy",
  "thiserror 2.0.17",
  "yaak-common",
@@ -11265,7 +11225,7 @@ dependencies = [
  "r2d2",
  "r2d2_sqlite",
  "rand 0.9.5",
- "reqwest 0.12.20",
+ "reqwest",
  "rlimit",
  "serde",
  "serde_json",
@@ -11351,7 +11311,7 @@ dependencies = [
  "oxc_resolver",
  "predicates",
  "rand 0.8.7",
- "reqwest 0.12.20",
+ "reqwest",
  "rolldown",
  "schemars 0.8.22",
  "serde",
@@ -11515,7 +11475,7 @@ dependencies = [
  "log 0.4.29",
  "mime_guess",
  "native-tls",
- "reqwest 0.12.20",
+ "reqwest",
  "serde",
  "serde_json",
  "tempfile",
@@ -11538,7 +11498,7 @@ version = "0.1.0"
 dependencies = [
  "chrono",
  "log 0.4.29",
- "reqwest 0.12.20",
+ "reqwest",
  "serde",
  "serde_json",
  "tauri",
@@ -11611,7 +11571,7 @@ dependencies = [
  "md5 0.7.0",
  "path-slash",
  "rand 0.9.5",
- "reqwest 0.12.20",
+ "reqwest",
  "serde",
  "serde_json",
  "sha2",

Cargo.toml

@@ -47,7 +47,14 @@ chrono = "0.4.42"
 hex = "0.4.3"
 keyring = "3.6.3"
 log = "0.4.29"
-reqwest = "0.12.20"
+reqwest = { version = "0.13.5", default-features = false, features = [
+  # reqwest 0.13 switched `default-tls` from native-tls to rustls; keep the
+  # rest of the old default set and stay on native-tls.
+  "charset",
+  "http2",
+  "system-proxy",
+  "native-tls",
+] }
 rustls = { version = "0.23.34", default-features = false }
 rustls-platform-verifier = "0.6.2"
 schemars = { version = "0.8.22", features = ["chrono"] }

crates-cli/yaak-cli/Cargo.toml

@@ -22,7 +22,7 @@ include_dir = "0.7"
 keyring = { workspace = true, features = ["apple-native", "windows-native", "sync-secret-service"] }
 log = { workspace = true }
 rand = "0.8"
-reqwest = { workspace = true }
+reqwest = { workspace = true, features = ["form"] }
 rolldown = "0.1.0"
 oxc_resolver = "=11.10.0"
 schemars = { workspace = true }

crates-tauri/yaak-app-client/Cargo.toml

@@ -48,7 +48,8 @@ reqwest = { workspace = true, features = [
   "brotli",
   "deflate",
   "json",
-  "rustls-tls-manual-roots-no-provider",
+  "query",
+  "rustls-no-provider",
   "socks",
   "http2",
 ] }

crates/yaak-http/Cargo.toml

@@ -20,9 +20,8 @@ log = { workspace = true }
 mime_guess = "2.0.5"
 native-tls = { version = "0.2", features = ["alpn"] }
 reqwest = { workspace = true, features = [
-  "rustls-tls-manual-roots-no-provider",
+  "rustls-no-provider",
   "native-tls",
-  "native-tls-alpn",
   "socks",
   "http2",
   "stream",
@@ -41,3 +40,4 @@ yaak-tls = { workspace = true }
 
 [dev-dependencies]
 tempfile = "3"
+tokio = { workspace = true, features = ["net"] }

crates/yaak-http/src/client.rs

@@ -10,8 +10,14 @@ use yaak_tls::{
 
 pub const HTTP2_MAX_RESPONSE_HEADER_LIST_SIZE: u32 = 1024 * 1024;
 
+/// hyper caps HTTP/1 responses at 100 headers, which real servers exceed.
+/// https://yaak.app/feedback/posts/when-response-headers-exceed-a-certain-count-hyper-throws-error
+pub const HTTP1_MAX_RESPONSE_HEADERS: usize = 1024;
+
 fn client_builder() -> ClientBuilder {
-    Client::builder().http2_max_header_list_size(HTTP2_MAX_RESPONSE_HEADER_LIST_SIZE)
+    Client::builder()
+        .http2_max_header_list_size(HTTP2_MAX_RESPONSE_HEADER_LIST_SIZE)
+        .http1_max_headers(HTTP1_MAX_RESPONSE_HEADERS)
 }
 
 #[derive(Clone)]
@@ -399,3 +405,100 @@ UFNAXPsoutUompC1Z57bKA6OKM2hRANCAATCYYKhzgHEaRaGsYVjJSoXvoroL8qe
         assert!(build_native_tls_identity(Some(empty)).unwrap().is_none());
     }
 }
+
+#[cfg(test)]
+mod header_limit_tests {
+    use super::*;
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};
+    use tokio::net::TcpListener;
+
+    /// Accepts one connection and replies with a response head of exactly
+    /// `total_headers` fields. The trailing `Content-Length` is one of them,
+    /// because hyper counts every field against its limit.
+    async fn serve_response_with_header_count(total_headers: usize) -> String {
+        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let addr = listener.local_addr().unwrap();
+
+        tokio::spawn(async move {
+            let (mut socket, _) = listener.accept().await.unwrap();
+
+            // Read the request head so the client is not writing into a closed socket
+            let mut buf = [0u8; 4096];
+            loop {
+                let n = socket.read(&mut buf).await.unwrap();
+                if n == 0 || buf[..n].windows(4).any(|w| w == b"\r\n\r\n") {
+                    break;
+                }
+            }
+
+            let mut response = String::from("HTTP/1.1 200 OK\r\n");
+            for i in 0..total_headers - 1 {
+                response.push_str(&format!("X-Test-{i}: value-{i}\r\n"));
+            }
+            response.push_str("Content-Length: 0\r\n\r\n");
+
+            // A response the client rejects mid-parse closes the socket under
+            // us, so a failed write here is an expected outcome, not a fault
+            let _ = socket.write_all(response.as_bytes()).await;
+            let _ = socket.flush().await;
+        });
+
+        format!("http://{addr}/")
+    }
+
+    fn options() -> HttpConnectionOptions {
+        HttpConnectionOptions {
+            id: "test".to_string(),
+            validate_certificates: true,
+            http_version: HttpVersion::Http1,
+            proxy: HttpConnectionProxySetting::Disabled,
+            client_certificate: None,
+            dns_overrides: Vec::new(),
+            address_filter: None,
+        }
+    }
+
+    /// hyper defaults to 100 header fields per HTTP/1 response and fails the
+    /// whole request past that, which is what users hit in the field.
+    #[tokio::test]
+    async fn responses_with_more_than_100_headers_are_accepted() {
+        let url = serve_response_with_header_count(150).await;
+        let (client, _resolver) = options().build_client().unwrap();
+
+        let response = client
+            .inner()
+            .get(&url)
+            .send()
+            .await
+            .expect("request with 150 response headers should succeed");
+
+        assert_eq!(response.status(), 200);
+        assert_eq!(response.headers().len(), 150);
+        for i in 0..149 {
+            assert_eq!(
+                response.headers().get(format!("x-test-{i}")).unwrap(),
+                format!("value-{i}").as_str(),
+            );
+        }
+    }
+
+    /// Pins both sides of the boundary. The sizes are written out rather than
+    /// derived from `HTTP1_MAX_RESPONSE_HEADERS`, so retuning the limit trips
+    /// this test instead of silently moving with it.
+    #[tokio::test]
+    async fn the_limit_is_exactly_1024_header_fields() {
+        let (client, _resolver) = options().build_client().unwrap();
+
+        let at_limit = serve_response_with_header_count(1024).await;
+        let response = client
+            .inner()
+            .get(&at_limit)
+            .send()
+            .await
+            .expect("a response with 1024 header fields should succeed");
+        assert_eq!(response.headers().len(), 1024);
+
+        let over_limit = serve_response_with_header_count(1025).await;
+        assert!(client.inner().get(&over_limit).send().await.is_err());
+    }
+}