Upload crates/bex-core/src/http_service.rs

crates/bex-core/src/http_service.rs

@@ -4,6 +4,50 @@ use std::sync::Arc;
 use std::time::Duration;
 use tokio::sync::RwLock;
 
+/// Real Chrome 137 headers used as defaults when plugins don't provide their own.
+/// This prevents bot detection by mimicking a real browser's header fingerprint.
+pub const DEFAULT_BROWSER_UA: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36";
+
+/// Standard browser headers that MUST be sent with every request to avoid detection.
+/// These are the headers Chrome sends automatically on every navigation/fetch.
+pub fn browser_default_headers() -> Vec<(&'static str, &'static str)> {
+    vec![
+        ("User-Agent", DEFAULT_BROWSER_UA),
+        ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8"),
+        ("Accept-Language", "en-US,en;q=0.9"),
+        ("Accept-Encoding", "gzip, deflate, br"),
+        ("Sec-CH-UA", "\"Google Chrome\";v=\"137\", \"Chromium\";v=\"137\", \"Not/A)Brand\";v=\"24\""),
+        ("Sec-CH-UA-Mobile", "?0"),
+        ("Sec-CH-UA-Platform", "\"Windows\""),
+        ("Sec-Fetch-Dest", "document"),
+        ("Sec-Fetch-Mode", "navigate"),
+        ("Sec-Fetch-Site", "none"),
+        ("Sec-Fetch-User", "?1"),
+        ("Upgrade-Insecure-Requests", "1"),
+        ("Connection", "keep-alive"),
+        ("DNT", "1"),
+    ]
+}
+
+/// Headers for XHR/API requests (Sec-Fetch differs from navigation)
+pub fn browser_xhr_headers(referer: &str) -> Vec<(String, String)> {
+    vec![
+        ("User-Agent".to_string(), DEFAULT_BROWSER_UA.to_string()),
+        ("Accept".to_string(), "application/json, text/javascript, */*; q=0.01".to_string()),
+        ("Accept-Language".to_string(), "en-US,en;q=0.9".to_string()),
+        ("Accept-Encoding".to_string(), "gzip, deflate, br".to_string()),
+        ("Sec-CH-UA".to_string(), "\"Google Chrome\";v=\"137\", \"Chromium\";v=\"137\", \"Not/A)Brand\";v=\"24\"".to_string()),
+        ("Sec-CH-UA-Mobile".to_string(), "?0".to_string()),
+        ("Sec-CH-UA-Platform".to_string(), "\"Windows\"".to_string()),
+        ("Sec-Fetch-Dest".to_string(), "empty".to_string()),
+        ("Sec-Fetch-Mode".to_string(), "cors".to_string()),
+        ("Sec-Fetch-Site".to_string(), "same-origin".to_string()),
+        ("X-Requested-With".to_string(), "XMLHttpRequest".to_string()),
+        ("Referer".to_string(), referer.to_string()),
+        ("Connection".to_string(), "keep-alive".to_string()),
+    ]
+}
+
 /// Cache entry for HTTP responses.
 #[derive(Clone)]
 struct CacheEntry {
@@ -28,18 +72,40 @@ pub struct HttpHostService {
 
 impl HttpHostService {
     pub fn new(
-        user_agent: &str,
+        _user_agent: &str, // Ignored — we always use real browser UA
         timeout_ms: u32,
         pool_idle_timeout_ms: u64,
         pool_max_idle_per_host: usize,
     ) -> Self {
+        // Build client mimicking Chrome as closely as possible:
+        // - HTTP/2 enabled (Chrome always uses H2)
+        // - Proper header ordering via reqwest::header::HeaderMap
+        // - gzip/br/deflate decompression 
+        // - Real browser User-Agent (not "BexEngine/6.0")
+        let mut headers = reqwest::header::HeaderMap::new();
+        for (k, v) in browser_default_headers() {
+            if let (Ok(name), Ok(val)) = (
+                reqwest::header::HeaderName::from_bytes(k.as_bytes()),
+                reqwest::header::HeaderValue::from_str(v),
+            ) {
+                headers.insert(name, val);
+            }
+        }
+
         let client = Client::builder()
             .timeout(Duration::from_millis(timeout_ms as u64))
             .redirect(reqwest::redirect::Policy::limited(10))
-            .user_agent(user_agent)
+            .default_headers(headers)
             .pool_idle_timeout(Duration::from_millis(pool_idle_timeout_ms))
             .pool_max_idle_per_host(pool_max_idle_per_host)
             .use_rustls_tls()
+            .gzip(true)
+            .brotli(true)
+            .deflate(true)
+            // Enable HTTP/2 (Chrome always uses it)
+            .http2_prior_knowledge(false) // Allow HTTP/2 via ALPN negotiation
+            // Cookie store for session persistence (critical for CF challenges)
+            .cookie_store(true)
             .build()
             .expect("failed to build HTTP client");
 
@@ -65,7 +131,7 @@ impl HttpHostService {
         None
     }
 
-    /// Store response in cache if appropriate
+    /// Store response in cache
     async fn store_cache(
         &self,
         url: &str,
@@ -74,28 +140,23 @@ impl HttpHostService {
         headers: &HashMap<String, String>,
         final_url: &str,
     ) {
-        // Only cache successful GET responses with cacheable status
         if status != 200 && status != 301 && status != 302 {
             return;
         }
-        // Don't cache huge responses
-        if body.len() > 1024 * 1024 {
+        if body.len() > 2 * 1024 * 1024 {
             return;
         }
 
-        // Determine max-age from Cache-Control header
         let max_age = if let Some(cc) = headers.get("cache-control") {
             if cc.contains("no-store") || cc.contains("no-cache") || cc.contains("private") {
-                return; // Don't cache
+                return;
             }
             if let Some(pos) = cc.find("max-age=") {
                 let rest = &cc[pos + 8..];
                 let end = rest.find(|c: char| !c.is_ascii_digit()).unwrap_or(rest.len());
-                if let Ok(secs) = rest[..end].parse::<u64>() {
-                    Duration::from_secs(secs.min(300)) // Cap at 5 minutes
-                } else {
-                    Duration::from_secs(60)
-                }
+                rest[..end].parse::<u64>().ok()
+                    .map(|secs| Duration::from_secs(secs.min(300)))
+                    .unwrap_or(Duration::from_secs(60))
             } else {
                 Duration::from_secs(60)
             }
@@ -103,14 +164,15 @@ impl HttpHostService {
             Duration::from_secs(60)
         };
 
-        let cache = self.cache.read().await;
-        // Limit cache size to prevent unbounded memory growth
+        let mut cache = self.cache.write().await;
+        // Evict stale entries before adding
         if cache.len() >= 500 {
-            return;
+            cache.retain(|_, v| v.is_fresh());
+        }
+        if cache.len() >= 500 {
+            return; // Still full after eviction
         }
-        drop(cache);
 
-        let mut cache = self.cache.write().await;
         cache.insert(
             url.to_string(),
             CacheEntry {
@@ -132,7 +194,7 @@ impl HttpHostService {
         body: Option<Vec<u8>>,
         timeout_ms: Option<u32>,
     ) -> anyhow::Result<(u16, Vec<u8>, HashMap<String, String>, String)> {
-        // Only cache GET requests
+        // Cache check for GET
         if method == "GET" {
             if let Some(cached) = self.check_cache(url).await {
                 return Ok(cached);
@@ -148,8 +210,22 @@ impl HttpHostService {
             _ => self.client.get(url),
         };
 
-        for (k, v) in headers {
-            req = req.header(&k, &v);
+        // CRITICAL: Apply plugin headers AFTER default headers.
+        // This lets plugins override defaults (e.g., different Referer).
+        // reqwest merges: plugin headers take precedence over defaults.
+        for (k, v) in &headers {
+            req = req.header(k.as_str(), v.as_str());
+        }
+
+        // If no User-Agent provided by plugin, the default_headers has one.
+        // If no Referer provided, add the origin of the URL being requested.
+        let has_referer = headers.iter().any(|(k, _)| k.eq_ignore_ascii_case("referer"));
+        if !has_referer {
+            // Auto-generate Referer from URL origin (like a real browser)
+            if let Ok(parsed) = url::Url::parse(url) {
+                let origin = format!("{}://{}/", parsed.scheme(), parsed.host_str().unwrap_or(""));
+                req = req.header("Referer", &origin);
+            }
         }
 
         if let Some(b) = body {