Download video via RapidAPI (own-CDN) -> faster-whisper transcript + local screenshots -> delete; retire proxies

app.py

@@ -47,8 +47,10 @@ _install_proxy_ca()
 import gradio as gr
 
 from pipeline import (
+    asr as asr_mod,
     captions as captions_mod,
     docx_builder,
+    downloader as downloader_mod,
     frames as frames_mod,
     search as search_mod,
     sentiment as sentiment_mod,
@@ -123,70 +125,46 @@ def _resolve_api_key(ui_key: str | None) -> str | None:
 
 
 def check_access():
-    """Health check: which secrets are set, the egress IP, and YouTube reachability."""
+    """Health check: are the required secrets set and is the video-download API live?"""
     import requests
 
-    proxy = _resolve_proxy()
-    proxies = {"http": proxy, "https": proxy} if proxy else None
+    rk = os.environ.get("RAPIDAPI_KEY", "").strip()
     lines = [
-        f"- **Transcript API** (`TRANSCRIPTAPI_KEY`, no proxy needed): "
-        f"{'set ✅' if os.environ.get('TRANSCRIPTAPI_KEY') else 'not set ⚪ (falls back to youtube-transcript-api via YT_PROXY)'}",
-        f"- **Proxy** (`YT_PROXY`): {'set ✅' if proxy else 'not set ⚪'}",
-        f"- **Media proxy** (`YT_MEDIA_PROXY`, screenshots): "
-        f"{'set ✅' if _resolve_media_proxy() else 'not set ⚪ (falls back to YT_PROXY)'}",
-        f"- **Cookies** (`YT_COOKIES`): {'set ✅' if os.environ.get('YT_COOKIES') else 'not set ⚪'}",
-        f"- **Data API key** (`YOUTUBE_API_KEY`): {'set ✅' if os.environ.get('YOUTUBE_API_KEY') else 'not set ⚪'}",
+        f"- **Video download** (`RAPIDAPI_KEY`): {'set ✅' if rk else 'not set ❌ — required'}",
+        f"- **Comment sentiment** (`YOUTUBE_API_KEY`): "
+        f"{'set ✅' if os.environ.get('YOUTUBE_API_KEY') else 'not set ⚪ (sentiment skipped)'}",
+        "- **Transcript**: faster-whisper, local (no key/proxy needed) ✅",
+        "- **Search**: Piped API (no key needed) ✅",
     ]
-    try:
-        ip = requests.get("https://api.ipify.org", proxies=proxies, timeout=20).text.strip()
-        lines.append(f"- **Egress IP** {'(via proxy)' if proxy else '(Space direct)'}: `{ip}`")
-    except Exception as exc:
-        lines.append(f"- **Egress IP**: ❌ {type(exc).__name__}")
-    def probe(url):
-        last = None
-        for _ in range(2):  # tolerate a single transient reset
-            try:
-                r = requests.get(url, proxies=proxies, timeout=20)
-                return True, f"HTTP {r.status_code}"
-            except Exception as exc:
-                last = exc
-                time.sleep(1.0)
-        return False, f"{type(last).__name__}: {str(last)[:90]}"
-
-    # Discriminating battery: neutral-small vs Google-family (same big cert as
-    # YouTube, different hostname) vs YouTube itself. The pass/fail pattern says
-    # whether it's a size/MTU blackhole, hostname-based egress filtering, or reset.
-    probes = [
-        ("example.com (neutral)", "https://example.com"),
-        ("google.com/204 (Google cert, non-YT name)", "https://www.google.com/generate_204"),
-        ("i.ytimg.com (YT image CDN)", "https://i.ytimg.com/generate_204"),
-        ("youtubei.googleapis.com (YT API)", "https://youtubei.googleapis.com/generate_204"),
-        ("youtube.com/robots.txt", "https://www.youtube.com/robots.txt"),
-        ("youtube.com/ (large body)", "https://www.youtube.com/"),
-    ]
-    results = {}
-    for label, url in probes:
-        ok, msg = probe(url)
-        results[label] = ok
-        lines.append(f"- {'✅' if ok else '❌'} {label}: {msg}")
-
-    yt_ok = results.get("youtube.com/robots.txt")
-    if yt_ok:
-        verdict = "### 🟢 YouTube is reachable — transcript + screenshots should work."
-    elif results.get("google.com/204 (Google cert, non-YT name)"):
-        verdict = ("### 🔴 YouTube is filtered by hostname.\n"
-                   "Google works but YouTube is reset → the network between the Space and the "
-                   "tunnel is dropping the plaintext `CONNECT www.youtube.com`. Fix: run the "
-                   "home proxy as an **HTTPS proxy** (TLS-wrapped) so the target host is hidden.")
-    elif results.get("example.com (neutral)") and not results.get("google.com/204 (Google cert, non-YT name)"):
-        verdict = ("### 🔴 Large TLS handshakes are being dropped (MTU blackhole).\n"
-                   "Small sites work; Google/YouTube (large cert chains) reset. Fix: clamp MSS on "
-                   "the tunnel path or switch tunnel provider (e.g. ngrok).")
+
+    api_ok = None
+    if rk:
+        host = "youtube-video-fast-downloader-24-7.p.rapidapi.com"
+        try:
+            r = requests.get(f"https://{host}/get-video-info/dQw4w9WgXcQ",
+                             headers={"X-RapidAPI-Key": rk, "X-RapidAPI-Host": host},
+                             timeout=25)
+            if r.status_code == 200:
+                api_ok = True
+                lines.append("- ✅ Download API reachable and key valid (HTTP 200)")
+            elif r.status_code in (401, 403):
+                api_ok = False
+                lines.append(f"- ❌ Download API rejected the key (HTTP {r.status_code}) — "
+                             "check RAPIDAPI_KEY and that you're subscribed")
+            else:
+                api_ok = False
+                lines.append(f"- ⚠️ Download API HTTP {r.status_code}: {r.text[:120]}")
+        except Exception as exc:
+            api_ok = False
+            lines.append(f"- ❌ Download API unreachable: {type(exc).__name__}")
+
+    if not rk:
+        verdict = "### 🔴 Set `RAPIDAPI_KEY` — it's required to download the source video."
+    elif api_ok:
+        verdict = ("### 🟢 Ready. Video downloads via the API; the transcript is produced "
+                   "locally by faster-whisper (first run downloads the model, ~1 min).")
     else:
-        verdict = ("### 🔴 YouTube is NOT reachable from the Space.\n"
-                   "The proxy connects (egress IP works) but TLS to YouTube is failing. "
-                   "Try again in a minute or refresh **`YT_PROXY`** via the panel in "
-                   "[`tools/`](tools/README.md).")
+        verdict = "### 🔴 The video-download API isn't working — see the details below."
     return verdict + "\n\n" + "\n".join(lines)
 
 
@@ -238,17 +216,14 @@ def run_pipeline(topic, hf_token, yt_api_key, llm_model, vlm_model,
 
     workdir = tempfile.mkdtemp(prefix="ytt_")
     frames_dir = os.path.join(workdir, "frames")
+    video_path = os.path.join(workdir, "source.mp4")
     try:
         api_key = _resolve_api_key(yt_api_key)
-        cookiefile = _cookiefile(workdir)
-        proxy = _resolve_proxy()
-        if proxy:
-            log.append("🔐 `YT_PROXY` is set — transcript + stream requests route through it.")
 
         # 1. Search ------------------------------------------------------------------
         progress(0.03, desc="Searching")
         yield status(f"🔍 Searching top videos for **{topic}** (engagement-ranked)…"), gr.update(), gr.update(), gr.update()
-        videos = search_mod.search_top5(topic, api_key=api_key, proxy=proxy)
+        videos = search_mod.search_top5(topic, api_key=api_key)
         eng_note = ""
         if videos and "engagement" in videos[0]:
             eng_note = " • ranked by likes+comments+subscribers−dislikes (normalized)"
@@ -269,12 +244,18 @@ def run_pipeline(topic, hf_token, yt_api_key, llm_model, vlm_model,
         yield (status(f"🏆 Picked **{best.get('title', best['video_id'])}** {picked_msg}."),
                ranking, gr.update(), gr.update())
 
-        # 3. Transcript (transcriptapi.com primary, youtube-transcript-api fallback) --
-        progress(0.3, desc="Transcript")
-        tsrc = ("transcriptapi.com (no proxy)" if os.environ.get("TRANSCRIPTAPI_KEY")
-                else "youtube-transcript-api via proxy")
-        yield status(f"📝 Fetching the timestamped transcript — {tsrc}…"), ranking, gr.update(), gr.update()
-        segs = transcribe_mod.get_segments(best["video_id"], proxy=proxy)
+        # 3. Download the source video once (RapidAPI, own-CDN — no proxy) -----------
+        progress(0.25, desc="Downloading video")
+        yield status("⬇️ Downloading the video (server-side prep can take up to ~5 min)…"), ranking, gr.update(), gr.update()
+        downloader_mod.download_video(best["video_id"], video_path, progress=progress)
+        size_mb = os.path.getsize(video_path) / 1024 / 1024
+        yield (status(f"Video downloaded ({size_mb:.0f} MB)."),
+               ranking, gr.update(), gr.update())
+
+        # 4. Transcript via faster-whisper on the local video ------------------------
+        progress(0.4, desc="Transcribing (faster-whisper)")
+        yield status("📝 Transcribing the audio with faster-whisper (can take a while on CPU)…"), ranking, gr.update(), gr.update()
+        segs = asr_mod.transcribe_file(video_path, progress=progress)
         transcript = transcribe_mod.transcript_text(segs)
         yield (status(f"Transcript ready ({len(segs)} segments)."),
                ranking, gr.update(value=transcript), gr.update())
@@ -295,27 +276,31 @@ def run_pipeline(topic, hf_token, yt_api_key, llm_model, vlm_model,
             yield (status(f"🔑 Primary keyword '{keywords['primary']}' appears {n}× in the post."),
                    ranking, gr.update(value=transcript), gr.update())
 
-        # 6. Screenshots (weighted timestamps -> yt-dlp clip download -> ffmpeg) ------
+        # 6. Screenshots from the local video (ffmpeg, no network) -------------------
         selected, caps = {}, {}
         times = frames_mod.compute_shot_times(
             tut["steps"], segs, w_llm=float(w_llm), w_whisper=float(w_whisper),
             lead=float(lead), max_shots=int(max_shots))
         if times:
-            progress(0.75, desc="Screenshots")
-            yield status(f"🎞️ Capturing {len(times)} screenshots at weighted timestamps…"), ranking, gr.update(value=transcript), gr.update()
+            progress(0.8, desc="Screenshots")
+            yield status(f"🎞️ Capturing {len(times)} screenshots from the video…"), ranking, gr.update(value=transcript), gr.update()
             try:
-                selected = frames_mod.capture_shots(
-                    times, best["video_id"], frames_dir, cookiefile, proxy,
-                    _resolve_media_proxy(), progress)
+                selected = frames_mod.capture_from_file(times, video_path, frames_dir, progress)
             except Exception as exc:
-                yield (status(f"⚠️ Couldn't fetch screenshots — text-only tutorial. "
-                              f"`{type(exc).__name__}: {str(exc)[:400]}`"),
+                yield (status(f"⚠️ Couldn't extract screenshots — text-only tutorial. "
+                              f"`{type(exc).__name__}: {str(exc)[:300]}`"),
                        ranking, gr.update(value=transcript), gr.update())
             if selected:
-                progress(0.88, desc="Captioning")
+                progress(0.9, desc="Captioning")
                 yield status(f"✍️ Captioning {len(selected)} screenshots with `{vlm_model}`…"), ranking, gr.update(value=transcript), gr.update()
                 caps = captions_mod.caption_frames(selected, tut["steps"], hf_token.strip(), vlm_model, progress)
 
+        # Done with the video — delete it (keep only the .docx for download).
+        try:
+            os.remove(video_path)
+        except OSError:
+            pass
+
         # 7. DOCX --------------------------------------------------------------------
         progress(0.96, desc="Building document")
         out_path = os.path.join(workdir, f"{_safe_name(tut['title'])}.docx")
@@ -331,6 +316,12 @@ def run_pipeline(topic, hf_token, yt_api_key, llm_model, vlm_model,
     except (transcribe_mod.TranscriptError, sentiment_mod.SentimentError,
             RuntimeError, ValueError) as exc:
         raise gr.Error(str(exc))
+    finally:
+        # Always delete the downloaded video (keep only the .docx for download).
+        try:
+            os.remove(video_path)
+        except OSError:
+            pass
 
 
 def build_ui():