fix: restore portable reqwest HTTP service with browser-like defaults

crates/bex-core/src/http_service.rs

@@ -1,30 +1,52 @@
-//! HTTP Service — Full Chrome 137 Browser Impersonation
+//! HTTP Service — portable production backend.
 //!
-//! This module uses `rquest` (BoringSSL-backed) to impersonate Chrome's exact
-//! TLS fingerprint (JA3/JA4), HTTP/2 SETTINGS frame, and header ordering.
-//! 
-//! WHY THIS MATTERS:
-//! Cloudflare uses 3 fingerprinting layers simultaneously:
-//!   1. JA3/JA4 — TLS ClientHello cipher suites, extensions, curves
-//!   2. Akamai H2 FP — HTTP/2 SETTINGS values (INITIAL_WINDOW_SIZE, etc.)
-//!   3. Header ordering — Chrome sends headers in a specific order
+//! This backend uses `reqwest` + `rustls` for reliable cross-platform builds and
+//! native-library embedding in C++ apps. It sends browser-like HTTP headers,
+//! supports HTTP/2, cookies, gzip/brotli/deflate, caching, max response limits,
+//! and plugin-provided header overrides.
 //!
-//! `rustls` fails ALL THREE checks. `rquest` with BoringSSL passes all of them
-//! because it uses Chrome's actual TLS library (BoringSSL) with the exact same
-//! configuration Chrome uses.
+//! IMPORTANT LIMITATION:
+//! This default backend does NOT byte-for-byte impersonate Chrome's TLS JA3/JA4
+//! fingerprint. It is production-friendly and portable, but advanced Cloudflare,
+//! DataDome, PerimeterX, or Akamai Bot Manager deployments can still detect that
+//! the TLS stack is rustls, not Chrome/BoringSSL.
 //!
-//! The result: Cloudflare sees an identical fingerprint to a real Chrome browser.
-//! No JS challenges, no blocks, no CAPTCHAs (except Turnstile which requires
-//! actual human interaction).
+//! For strict anti-bot bypass, add a second optional backend based on a verified
+//! BoringSSL/curl-impersonate client and gate it behind a Cargo feature. Do not
+//! make that the default until it is compiled and tested on every target platform
+//! you intend to ship.
 
-use rquest::Client;
-use rquest::tls::Impersonate;
+use reqwest::Client;
 use std::collections::HashMap;
 use std::sync::Arc;
 use std::time::Duration;
 use tokio::sync::RwLock;
 
-/// Cache entry for HTTP responses.
+/// Current Chrome-like desktop UA. Kept centralized so plugins and host match.
+pub const DEFAULT_BROWSER_UA: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36";
+
+/// Default browser-ish navigation/fetch headers.
+///
+/// These help with simple header-based checks. They do not alter TLS JA3/JA4.
+pub fn browser_default_headers() -> Vec<(&'static str, &'static str)> {
+    vec![
+        ("User-Agent", DEFAULT_BROWSER_UA),
+        ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8"),
+        ("Accept-Language", "en-US,en;q=0.9"),
+        ("Accept-Encoding", "gzip, deflate, br"),
+        ("Sec-CH-UA", "\"Google Chrome\";v=\"137\", \"Chromium\";v=\"137\", \"Not/A)Brand\";v=\"24\""),
+        ("Sec-CH-UA-Mobile", "?0"),
+        ("Sec-CH-UA-Platform", "\"Windows\""),
+        ("Sec-Fetch-Dest", "document"),
+        ("Sec-Fetch-Mode", "navigate"),
+        ("Sec-Fetch-Site", "none"),
+        ("Sec-Fetch-User", "?1"),
+        ("Upgrade-Insecure-Requests", "1"),
+        ("Connection", "keep-alive"),
+        ("DNT", "1"),
+    ]
+}
+
 #[derive(Clone)]
 struct CacheEntry {
     body: Vec<u8>,
@@ -47,49 +69,36 @@ pub struct HttpHostService {
 }
 
 impl HttpHostService {
-    /// Create a new HTTP service with full Chrome 137 impersonation.
-    ///
-    /// This sets up:
-    /// - TLS ClientHello identical to Chrome 137 (JA3/JA4 match)
-    /// - HTTP/2 SETTINGS frame identical to Chrome (Akamai FP match)
-    /// - Header ordering identical to Chrome
-    /// - GREASE values in TLS extensions (RFC 8701)
-    /// - Certificate compression (brotli) support
-    /// - Cookie store for CF session persistence
-    /// - gzip/br/deflate decompression
-    ///
-    /// No manual header configuration needed — rquest handles everything.
     pub fn new(
-        _user_agent: &str, // Ignored — Chrome 137 UA is set by impersonation
+        _user_agent: &str,
         timeout_ms: u32,
         pool_idle_timeout_ms: u64,
         pool_max_idle_per_host: usize,
     ) -> Self {
-        // Build client with Chrome 137 impersonation.
-        // This single call configures:
-        //   - BoringSSL with Chrome's exact cipher suite order
-        //   - GREASE randomization in TLS extensions
-        //   - Chrome's HTTP/2 SETTINGS frame (INITIAL_WINDOW_SIZE=6291456, etc.)
-        //   - Chrome's header ordering (sec-ch-ua before accept, etc.)
-        //   - Chrome's User-Agent string
-        //   - Certificate compression (brotli)
-        //   - ALPS/ALPN negotiation
+        let mut headers = reqwest::header::HeaderMap::new();
+        for (k, v) in browser_default_headers() {
+            if let (Ok(name), Ok(val)) = (
+                reqwest::header::HeaderName::from_bytes(k.as_bytes()),
+                reqwest::header::HeaderValue::from_str(v),
+            ) {
+                headers.insert(name, val);
+            }
+        }
+
         let client = Client::builder()
-            .impersonate(Impersonate::Chrome137)
             .timeout(Duration::from_millis(timeout_ms as u64))
-            .redirect(rquest::redirect::Policy::limited(10))
+            .redirect(reqwest::redirect::Policy::limited(10))
+            .default_headers(headers)
             .pool_idle_timeout(Duration::from_millis(pool_idle_timeout_ms))
             .pool_max_idle_per_host(pool_max_idle_per_host)
-            // Cookie store is CRITICAL for Cloudflare:
-            // CF sets cookies after passing a challenge; subsequent requests
-            // must include these cookies or get re-challenged.
-            .cookie_store(true)
-            // Content-encoding decompression (browsers always support these)
+            .use_rustls_tls()
             .gzip(true)
             .brotli(true)
             .deflate(true)
+            .http2_prior_knowledge(false)
+            .cookie_store(true)
             .build()
-            .expect("failed to build HTTP client with Chrome impersonation");
+            .expect("failed to build HTTP client");
 
         Self {
             client,
@@ -97,23 +106,18 @@ impl HttpHostService {
         }
     }
 
-    /// Check cache for a fresh entry
     async fn check_cache(&self, url: &str) -> Option<(u16, Vec<u8>, HashMap<String, String>, String)> {
         let cache = self.cache.read().await;
-        if let Some(entry) = cache.get(url) {
-            if entry.is_fresh() {
-                return Some((
-                    entry.status,
-                    entry.body.clone(),
-                    entry.headers.clone(),
-                    entry.final_url.clone(),
-                ));
-            }
-        }
-        None
+        cache.get(url).filter(|entry| entry.is_fresh()).map(|entry| {
+            (
+                entry.status,
+                entry.body.clone(),
+                entry.headers.clone(),
+                entry.final_url.clone(),
+            )
+        })
     }
 
-    /// Store response in cache (only 2xx, < 2MB, respecting Cache-Control)
     async fn store_cache(
         &self,
         url: &str,
@@ -122,19 +126,17 @@ impl HttpHostService {
         headers: &HashMap<String, String>,
         final_url: &str,
     ) {
-        if status < 200 || status >= 300 {
-            return;
-        }
-        if body.len() > 2 * 1024 * 1024 {
+        if !(200..300).contains(&status) || body.len() > 2 * 1024 * 1024 {
             return;
         }
 
         let max_age = if let Some(cc) = headers.get("cache-control") {
-            if cc.contains("no-store") || cc.contains("no-cache") || cc.contains("private") {
+            let cc_l = cc.to_ascii_lowercase();
+            if cc_l.contains("no-store") || cc_l.contains("no-cache") || cc_l.contains("private") {
                 return;
             }
-            if let Some(pos) = cc.find("max-age=") {
-                let rest = &cc[pos + 8..];
+            if let Some(pos) = cc_l.find("max-age=") {
+                let rest = &cc_l[pos + 8..];
                 let end = rest.find(|c: char| !c.is_ascii_digit()).unwrap_or(rest.len());
                 rest[..end]
                     .parse::<u64>()
@@ -169,21 +171,6 @@ impl HttpHostService {
         );
     }
 
-    /// Send an HTTP request with full Chrome impersonation.
-    ///
-    /// IMPORTANT: Plugin-provided headers are applied AFTER the Chrome defaults.
-    /// This means plugins CAN override specific headers (e.g., custom Referer)
-    /// without breaking the fingerprint. The TLS fingerprint and H2 SETTINGS
-    /// are set at the connection level and cannot be overridden by headers.
-    ///
-    /// The only headers plugins should typically set are:
-    /// - Referer (for same-origin API calls)
-    /// - X-Requested-With (for XHR detection)
-    /// - Authorization / API keys
-    /// - Content-Type (for POST)
-    ///
-    /// Do NOT set User-Agent, Sec-CH-UA, Sec-Fetch-* — these are handled
-    /// by the impersonation layer and setting them manually may break ordering.
     pub async fn send_request(
         &self,
         method: &str,
@@ -192,7 +179,6 @@ impl HttpHostService {
         body: Option<Vec<u8>>,
         timeout_ms: Option<u32>,
     ) -> anyhow::Result<(u16, Vec<u8>, HashMap<String, String>, String)> {
-        // Cache check for GET
         if method == "GET" {
             if let Some(cached) = self.check_cache(url).await {
                 return Ok(cached);
@@ -208,34 +194,23 @@ impl HttpHostService {
             _ => self.client.get(url),
         };
 
-        // Apply plugin headers.
-        // IMPORTANT: Only set headers that the plugin explicitly provides.
-        // The Chrome impersonation already handles UA, Sec-CH-UA, Sec-Fetch-*, etc.
-        // Plugin headers that conflict with impersonation defaults will override them.
         for (k, v) in &headers {
             req = req.header(k.as_str(), v.as_str());
         }
 
-        // Auto-generate Referer from URL origin if not provided by plugin.
-        // Chrome automatically sends Referer on same-origin requests.
-        let has_referer = headers
-            .iter()
-            .any(|(k, _)| k.eq_ignore_ascii_case("referer"));
+        let has_referer = headers.iter().any(|(k, _)| k.eq_ignore_ascii_case("referer"));
         if !has_referer {
             if let Ok(parsed) = url::Url::parse(url) {
-                let origin = format!(
-                    "{}://{}/",
-                    parsed.scheme(),
-                    parsed.host_str().unwrap_or("")
-                );
-                req = req.header("Referer", &origin);
+                if let Some(host) = parsed.host_str() {
+                    let origin = format!("{}://{}/", parsed.scheme(), host);
+                    req = req.header("Referer", &origin);
+                }
             }
         }
 
         if let Some(b) = body {
             req = req.body(b);
         }
-
         if let Some(ms) = timeout_ms {
             req = req.timeout(Duration::from_millis(ms as u64));
         }
@@ -250,7 +225,6 @@ impl HttpHostService {
             .collect();
         let resp_body = resp.bytes().await?.to_vec();
 
-        // Cache GET responses
         if method == "GET" {
             self.store_cache(url, status, resp_body.clone(), &resp_headers, &final_url)
                 .await;
@@ -259,7 +233,6 @@ impl HttpHostService {
         Ok((status, resp_body, resp_headers, final_url))
     }
 
-    /// Clear the HTTP cache
     pub async fn clear_cache(&self) {
         self.cache.write().await.clear();
     }