feat: rewrite HTTP service with full Chrome 137 TLS/H2 impersonation via rquest

crates/bex-core/src/http_service.rs

@@ -1,53 +1,29 @@
-use reqwest::Client;
+//! HTTP Service — Full Chrome 137 Browser Impersonation
+//!
+//! This module uses `rquest` (BoringSSL-backed) to impersonate Chrome's exact
+//! TLS fingerprint (JA3/JA4), HTTP/2 SETTINGS frame, and header ordering.
+//! 
+//! WHY THIS MATTERS:
+//! Cloudflare uses 3 fingerprinting layers simultaneously:
+//!   1. JA3/JA4 — TLS ClientHello cipher suites, extensions, curves
+//!   2. Akamai H2 FP — HTTP/2 SETTINGS values (INITIAL_WINDOW_SIZE, etc.)
+//!   3. Header ordering — Chrome sends headers in a specific order
+//!
+//! `rustls` fails ALL THREE checks. `rquest` with BoringSSL passes all of them
+//! because it uses Chrome's actual TLS library (BoringSSL) with the exact same
+//! configuration Chrome uses.
+//!
+//! The result: Cloudflare sees an identical fingerprint to a real Chrome browser.
+//! No JS challenges, no blocks, no CAPTCHAs (except Turnstile which requires
+//! actual human interaction).
+
+use rquest::Client;
+use rquest::tls::Impersonate;
 use std::collections::HashMap;
 use std::sync::Arc;
 use std::time::Duration;
 use tokio::sync::RwLock;
 
-/// Real Chrome 137 headers used as defaults when plugins don't provide their own.
-/// This prevents bot detection by mimicking a real browser's header fingerprint.
-pub const DEFAULT_BROWSER_UA: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36";
-
-/// Standard browser headers that MUST be sent with every request to avoid detection.
-/// These are the headers Chrome sends automatically on every navigation/fetch.
-pub fn browser_default_headers() -> Vec<(&'static str, &'static str)> {
-    vec![
-        ("User-Agent", DEFAULT_BROWSER_UA),
-        ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8"),
-        ("Accept-Language", "en-US,en;q=0.9"),
-        ("Accept-Encoding", "gzip, deflate, br"),
-        ("Sec-CH-UA", "\"Google Chrome\";v=\"137\", \"Chromium\";v=\"137\", \"Not/A)Brand\";v=\"24\""),
-        ("Sec-CH-UA-Mobile", "?0"),
-        ("Sec-CH-UA-Platform", "\"Windows\""),
-        ("Sec-Fetch-Dest", "document"),
-        ("Sec-Fetch-Mode", "navigate"),
-        ("Sec-Fetch-Site", "none"),
-        ("Sec-Fetch-User", "?1"),
-        ("Upgrade-Insecure-Requests", "1"),
-        ("Connection", "keep-alive"),
-        ("DNT", "1"),
-    ]
-}
-
-/// Headers for XHR/API requests (Sec-Fetch differs from navigation)
-pub fn browser_xhr_headers(referer: &str) -> Vec<(String, String)> {
-    vec![
-        ("User-Agent".to_string(), DEFAULT_BROWSER_UA.to_string()),
-        ("Accept".to_string(), "application/json, text/javascript, */*; q=0.01".to_string()),
-        ("Accept-Language".to_string(), "en-US,en;q=0.9".to_string()),
-        ("Accept-Encoding".to_string(), "gzip, deflate, br".to_string()),
-        ("Sec-CH-UA".to_string(), "\"Google Chrome\";v=\"137\", \"Chromium\";v=\"137\", \"Not/A)Brand\";v=\"24\"".to_string()),
-        ("Sec-CH-UA-Mobile".to_string(), "?0".to_string()),
-        ("Sec-CH-UA-Platform".to_string(), "\"Windows\"".to_string()),
-        ("Sec-Fetch-Dest".to_string(), "empty".to_string()),
-        ("Sec-Fetch-Mode".to_string(), "cors".to_string()),
-        ("Sec-Fetch-Site".to_string(), "same-origin".to_string()),
-        ("X-Requested-With".to_string(), "XMLHttpRequest".to_string()),
-        ("Referer".to_string(), referer.to_string()),
-        ("Connection".to_string(), "keep-alive".to_string()),
-    ]
-}
-
 /// Cache entry for HTTP responses.
 #[derive(Clone)]
 struct CacheEntry {
@@ -71,43 +47,49 @@ pub struct HttpHostService {
 }
 
 impl HttpHostService {
+    /// Create a new HTTP service with full Chrome 137 impersonation.
+    ///
+    /// This sets up:
+    /// - TLS ClientHello identical to Chrome 137 (JA3/JA4 match)
+    /// - HTTP/2 SETTINGS frame identical to Chrome (Akamai FP match)
+    /// - Header ordering identical to Chrome
+    /// - GREASE values in TLS extensions (RFC 8701)
+    /// - Certificate compression (brotli) support
+    /// - Cookie store for CF session persistence
+    /// - gzip/br/deflate decompression
+    ///
+    /// No manual header configuration needed — rquest handles everything.
     pub fn new(
-        _user_agent: &str, // Ignored — we always use real browser UA
+        _user_agent: &str, // Ignored — Chrome 137 UA is set by impersonation
         timeout_ms: u32,
         pool_idle_timeout_ms: u64,
         pool_max_idle_per_host: usize,
     ) -> Self {
-        // Build client mimicking Chrome as closely as possible:
-        // - HTTP/2 enabled (Chrome always uses H2)
-        // - Proper header ordering via reqwest::header::HeaderMap
-        // - gzip/br/deflate decompression 
-        // - Real browser User-Agent (not "BexEngine/6.0")
-        let mut headers = reqwest::header::HeaderMap::new();
-        for (k, v) in browser_default_headers() {
-            if let (Ok(name), Ok(val)) = (
-                reqwest::header::HeaderName::from_bytes(k.as_bytes()),
-                reqwest::header::HeaderValue::from_str(v),
-            ) {
-                headers.insert(name, val);
-            }
-        }
-
+        // Build client with Chrome 137 impersonation.
+        // This single call configures:
+        //   - BoringSSL with Chrome's exact cipher suite order
+        //   - GREASE randomization in TLS extensions
+        //   - Chrome's HTTP/2 SETTINGS frame (INITIAL_WINDOW_SIZE=6291456, etc.)
+        //   - Chrome's header ordering (sec-ch-ua before accept, etc.)
+        //   - Chrome's User-Agent string
+        //   - Certificate compression (brotli)
+        //   - ALPS/ALPN negotiation
         let client = Client::builder()
+            .impersonate(Impersonate::Chrome137)
             .timeout(Duration::from_millis(timeout_ms as u64))
-            .redirect(reqwest::redirect::Policy::limited(10))
-            .default_headers(headers)
+            .redirect(rquest::redirect::Policy::limited(10))
             .pool_idle_timeout(Duration::from_millis(pool_idle_timeout_ms))
             .pool_max_idle_per_host(pool_max_idle_per_host)
-            .use_rustls_tls()
+            // Cookie store is CRITICAL for Cloudflare:
+            // CF sets cookies after passing a challenge; subsequent requests
+            // must include these cookies or get re-challenged.
+            .cookie_store(true)
+            // Content-encoding decompression (browsers always support these)
             .gzip(true)
             .brotli(true)
             .deflate(true)
-            // Enable HTTP/2 (Chrome always uses it)
-            .http2_prior_knowledge(false) // Allow HTTP/2 via ALPN negotiation
-            // Cookie store for session persistence (critical for CF challenges)
-            .cookie_store(true)
             .build()
-            .expect("failed to build HTTP client");
+            .expect("failed to build HTTP client with Chrome impersonation");
 
         Self {
             client,
@@ -131,7 +113,7 @@ impl HttpHostService {
         None
     }
 
-    /// Store response in cache
+    /// Store response in cache (only 2xx, < 2MB, respecting Cache-Control)
     async fn store_cache(
         &self,
         url: &str,
@@ -140,7 +122,7 @@ impl HttpHostService {
         headers: &HashMap<String, String>,
         final_url: &str,
     ) {
-        if status != 200 && status != 301 && status != 302 {
+        if status < 200 || status >= 300 {
             return;
         }
         if body.len() > 2 * 1024 * 1024 {
@@ -154,7 +136,9 @@ impl HttpHostService {
             if let Some(pos) = cc.find("max-age=") {
                 let rest = &cc[pos + 8..];
                 let end = rest.find(|c: char| !c.is_ascii_digit()).unwrap_or(rest.len());
-                rest[..end].parse::<u64>().ok()
+                rest[..end]
+                    .parse::<u64>()
+                    .ok()
                     .map(|secs| Duration::from_secs(secs.min(300)))
                     .unwrap_or(Duration::from_secs(60))
             } else {
@@ -165,12 +149,11 @@ impl HttpHostService {
         };
 
         let mut cache = self.cache.write().await;
-        // Evict stale entries before adding
         if cache.len() >= 500 {
             cache.retain(|_, v| v.is_fresh());
         }
         if cache.len() >= 500 {
-            return; // Still full after eviction
+            return;
         }
 
         cache.insert(
@@ -186,6 +169,21 @@ impl HttpHostService {
         );
     }
 
+    /// Send an HTTP request with full Chrome impersonation.
+    ///
+    /// IMPORTANT: Plugin-provided headers are applied AFTER the Chrome defaults.
+    /// This means plugins CAN override specific headers (e.g., custom Referer)
+    /// without breaking the fingerprint. The TLS fingerprint and H2 SETTINGS
+    /// are set at the connection level and cannot be overridden by headers.
+    ///
+    /// The only headers plugins should typically set are:
+    /// - Referer (for same-origin API calls)
+    /// - X-Requested-With (for XHR detection)
+    /// - Authorization / API keys
+    /// - Content-Type (for POST)
+    ///
+    /// Do NOT set User-Agent, Sec-CH-UA, Sec-Fetch-* — these are handled
+    /// by the impersonation layer and setting them manually may break ordering.
     pub async fn send_request(
         &self,
         method: &str,
@@ -210,20 +208,26 @@ impl HttpHostService {
             _ => self.client.get(url),
         };
 
-        // CRITICAL: Apply plugin headers AFTER default headers.
-        // This lets plugins override defaults (e.g., different Referer).
-        // reqwest merges: plugin headers take precedence over defaults.
+        // Apply plugin headers.
+        // IMPORTANT: Only set headers that the plugin explicitly provides.
+        // The Chrome impersonation already handles UA, Sec-CH-UA, Sec-Fetch-*, etc.
+        // Plugin headers that conflict with impersonation defaults will override them.
         for (k, v) in &headers {
             req = req.header(k.as_str(), v.as_str());
         }
 
-        // If no User-Agent provided by plugin, the default_headers has one.
-        // If no Referer provided, add the origin of the URL being requested.
-        let has_referer = headers.iter().any(|(k, _)| k.eq_ignore_ascii_case("referer"));
+        // Auto-generate Referer from URL origin if not provided by plugin.
+        // Chrome automatically sends Referer on same-origin requests.
+        let has_referer = headers
+            .iter()
+            .any(|(k, _)| k.eq_ignore_ascii_case("referer"));
         if !has_referer {
-            // Auto-generate Referer from URL origin (like a real browser)
             if let Ok(parsed) = url::Url::parse(url) {
-                let origin = format!("{}://{}/", parsed.scheme(), parsed.host_str().unwrap_or(""));
+                let origin = format!(
+                    "{}://{}/",
+                    parsed.scheme(),
+                    parsed.host_str().unwrap_or("")
+                );
                 req = req.header("Referer", &origin);
             }
         }