feat: add simple anti-bot challenge detection to HTTP service

crates/bex-core/src/http_service.rs

@@ -1,20 +1,14 @@
-//! HTTP Service — portable production backend.
+//! HTTP Service — portable production backend with simple challenge detection.
 //!
 //! This backend uses `reqwest` + `rustls` for reliable cross-platform builds and
 //! native-library embedding in C++ apps. It sends browser-like HTTP headers,
 //! supports HTTP/2, cookies, gzip/brotli/deflate, caching, max response limits,
 //! and plugin-provided header overrides.
 //!
-//! IMPORTANT LIMITATION:
-//! This default backend does NOT byte-for-byte impersonate Chrome's TLS JA3/JA4
-//! fingerprint. It is production-friendly and portable, but advanced Cloudflare,
-//! DataDome, PerimeterX, or Akamai Bot Manager deployments can still detect that
-//! the TLS stack is rustls, not Chrome/BoringSSL.
-//!
-//! For strict anti-bot bypass, add a second optional backend based on a verified
-//! BoringSSL/curl-impersonate client and gate it behind a Cargo feature. Do not
-//! make that the default until it is compiled and tested on every target platform
-//! you intend to ship.
+//! It also detects common anti-bot challenge pages and returns a structured
+//! `CHALLENGE_REQUIRED` error string. The host app can then decide whether to
+//! retry with cookies, a platform browser session, an external fetcher, or a
+//! user-visible WebView/browser flow.
 
 use reqwest::Client;
 use std::collections::HashMap;
@@ -22,12 +16,8 @@ use std::sync::Arc;
 use std::time::Duration;
 use tokio::sync::RwLock;
 
-/// Current Chrome-like desktop UA. Kept centralized so plugins and host match.
 pub const DEFAULT_BROWSER_UA: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36";
 
-/// Default browser-ish navigation/fetch headers.
-///
-/// These help with simple header-based checks. They do not alter TLS JA3/JA4.
 pub fn browser_default_headers() -> Vec<(&'static str, &'static str)> {
     vec![
         ("User-Agent", DEFAULT_BROWSER_UA),
@@ -68,6 +58,82 @@ pub struct HttpHostService {
     cache: Arc<RwLock<HashMap<String, CacheEntry>>>,
 }
 
+#[derive(Debug, Clone, serde::Serialize)]
+struct ChallengeInfo {
+    code: &'static str,
+    provider: String,
+    status: u16,
+    url: String,
+    final_url: String,
+    domain: String,
+    hint: &'static str,
+}
+
+fn header_value<'a>(headers: &'a HashMap<String, String>, name: &str) -> Option<&'a str> {
+    headers
+        .iter()
+        .find(|(k, _)| k.eq_ignore_ascii_case(name))
+        .map(|(_, v)| v.as_str())
+}
+
+fn detect_antibot_challenge(
+    status: u16,
+    url: &str,
+    final_url: &str,
+    headers: &HashMap<String, String>,
+    body: &[u8],
+) -> Option<ChallengeInfo> {
+    let status_suspicious = matches!(status, 403 | 429 | 503);
+    let body_sample = String::from_utf8_lossy(&body[..body.len().min(64 * 1024)]).to_ascii_lowercase();
+
+    let mut provider = None::<&str>;
+
+    if header_value(headers, "cf-ray").is_some()
+        || header_value(headers, "server").map(|v| v.to_ascii_lowercase().contains("cloudflare")).unwrap_or(false)
+        || body_sample.contains("cf-chl-")
+        || body_sample.contains("checking your browser")
+        || body_sample.contains("just a moment")
+        || body_sample.contains("cloudflare")
+        || final_url.contains("/cdn-cgi/challenge-platform/")
+    {
+        provider = Some("cloudflare");
+    } else if header_value(headers, "x-datadome").is_some()
+        || body_sample.contains("datadome")
+    {
+        provider = Some("datadome");
+    } else if body_sample.contains("px-captcha")
+        || body_sample.contains("perimeterx")
+        || header_value(headers, "x-px").is_some()
+    {
+        provider = Some("perimeterx");
+    } else if body_sample.contains("akamai") && (body_sample.contains("bot") || body_sample.contains("denied")) {
+        provider = Some("akamai");
+    } else if body_sample.contains("captcha") || body_sample.contains("turnstile") {
+        provider = Some("captcha");
+    }
+
+    let provider = provider?;
+    if !status_suspicious && provider == "captcha" {
+        return None;
+    }
+
+    let domain = url::Url::parse(final_url)
+        .or_else(|_| url::Url::parse(url))
+        .ok()
+        .and_then(|u| u.host_str().map(|s| s.to_string()))
+        .unwrap_or_default();
+
+    Some(ChallengeInfo {
+        code: "CHALLENGE_REQUIRED",
+        provider: provider.to_string(),
+        status,
+        url: url.to_string(),
+        final_url: final_url.to_string(),
+        domain,
+        hint: "Host app should retry with stored cookies, browser-backed fetch, or external fetcher.",
+    })
+}
+
 impl HttpHostService {
     pub fn new(
         _user_agent: &str,
@@ -225,6 +291,11 @@ impl HttpHostService {
             .collect();
         let resp_body = resp.bytes().await?.to_vec();
 
+        if let Some(challenge) = detect_antibot_challenge(status, url, &final_url, &resp_headers, &resp_body) {
+            let json = serde_json::to_string(&challenge).unwrap_or_else(|_| "{\"code\":\"CHALLENGE_REQUIRED\"}".to_string());
+            anyhow::bail!(json);
+        }
+
         if method == "GET" {
             self.store_cache(url, status, resp_body.clone(), &resp_headers, &final_url)
                 .await;