Add PO-token support (manual PO token + visitor data) with README guide

- download + comment-fetch accept po_token/visitor_data and inject them into yt-dlp
extractor_args; when a PO token is present, web-family player clients are tried so the
token applies
- UI: PO token + visitor data fields in the "YouTube access" panel, with a pointer to the
README's PO-token guide; operator secrets YT_POT / YT_VISITOR_DATA as shared defaults
- README: step-by-step manual PO-token + visitor-data extraction guide, automated
(bgutil) note, and a link to the official yt-dlp PO Token Guide
- Block-error hint now suggests the PO-token route

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

README.md

@@ -57,6 +57,57 @@ on that topic.
   - A hosted Space **cannot** open or read a visitor's browser — auth must be supplied
     explicitly.
 
+## PO token guide (free, no proxy)
+
+YouTube increasingly requires a **Proof-of-Origin (PO) token** to serve video from
+datacenter IPs. A matching **PO token + visitor data** pair can get the Space past the
+"Sign in to confirm you're not a bot" wall **without any proxy** — and it's free. It's
+fiddly to extract by hand and the tokens expire (hours), so re-do this when downloads
+start failing.
+
+Paste the values into the app's **"YouTube access — cookies / proxy"** panel:
+**PO token** field and **Visitor data** field.
+
+### Get them manually (web client, ~2 min)
+
+1. Open a fresh **Incognito** window and go to <https://www.youtube.com> (a throwaway
+   logged-in account is fine; if you log in, also export that session's cookies).
+2. Open **DevTools → Network** (F12), then **play any video**.
+3. **Visitor data:** filter Network for `v1/player`, open the request to
+   `youtubei/v1/player`, and in the **Request payload** copy
+   `context.client.visitorData` (a string ending in `%3D%3D`). Paste it into the
+   **Visitor data** field.
+4. **PO token (gvs):** filter Network for `googlevideo.com`, open a `videoplayback`
+   request, and copy the value of its **`pot`** query parameter. In the **PO token**
+   field enter it prefixed with the client + context:
+
+   ```
+   web.gvs+<pot-value>
+   ```
+
+   You can supply more than one, comma-separated, e.g.
+   `web.gvs+<gvs-pot>, web.player+<player-pot>` (the player PO token comes from the
+   `v1/player` request's response, field `...poToken`).
+5. Run the Space. When a PO token is present it automatically uses the **web** player
+   client so the token applies.
+
+**Tips**
+- The PO token is bound to the **visitor data** (and cookies) it was made with — always
+  submit them together, from the same session.
+- Prefer using the **same Incognito session** for cookies + visitor data + PO token.
+- Tokens are short-lived; refresh when you next get blocked.
+
+### Automated / operator option
+
+yt-dlp can auto-generate PO tokens via the
+[`bgutil-ytdlp-pot-provider`](https://github.com/Brainicism/bgutil-ytdlp-pot-provider)
+plugin, but that needs a small provider server running alongside yt-dlp — not bundled in
+this hosted Space. A self-hosting operator can run it and set shared defaults via the
+`YT_POT` / `YT_VISITOR_DATA` Space secrets.
+
+See the authoritative, up-to-date method in the
+[yt-dlp PO Token Guide](https://github.com/yt-dlp/yt-dlp/wiki/PO-Token-Guide).
+
 ## Local run
 
 ```bash

app.py

@@ -93,6 +93,13 @@ def _resolve_proxy(raw: str | None = None) -> str | None:
     return proxy or None
 
 
+def _resolve_pot(po_token: str | None, visitor_data: str | None) -> tuple[str | None, str | None]:
+    """Per-user PO token + visitor data, falling back to YT_POT / YT_VISITOR_DATA secrets."""
+    pot = (po_token or "").strip() or os.environ.get("YT_POT", "").strip()
+    vis = (visitor_data or "").strip() or os.environ.get("YT_VISITOR_DATA", "").strip()
+    return (pot or None), (vis or None)
+
+
 def _ranking_rows(scored: list[dict]) -> list[list]:
     rows = []
     for rank, v in enumerate(scored, start=1):
@@ -129,7 +136,7 @@ def _collect_keywords(primary_kw, secondary_kw) -> dict:
 
 def run_pipeline(topic, hf_token, llm_model, vlm_model, w_llm, w_whisper, lead,
                  max_minutes, max_shots, primary_kw, secondary_kw,
-                 cookies_text, proxy_url,
+                 cookies_text, proxy_url, po_token_in, visitor_data_in,
                  progress=gr.Progress()):
     """Generator that yields (status_md, ranking_df, transcript, docx_file)."""
     log: list[str] = []
@@ -150,11 +157,14 @@ def run_pipeline(topic, hf_token, llm_model, vlm_model, w_llm, w_whisper, lead,
     try:
         cookiefile = _cookiefile(workdir, cookies_text)
         proxy = _resolve_proxy(proxy_url)
+        po_token, visitor_data = _resolve_pot(po_token_in, visitor_data_in)
         auth_bits = []
         if cookiefile:
             auth_bits.append("cookies")
         if proxy:
             auth_bits.append("proxy")
+        if po_token:
+            auth_bits.append("PO token")
         if auth_bits:
             yield status("🔐 Using " + " + ".join(auth_bits) + " for YouTube access."), gr.update(), gr.update(), gr.update()
 
@@ -166,7 +176,8 @@ def run_pipeline(topic, hf_token, llm_model, vlm_model, w_llm, w_whisper, lead,
 
         # 2. Sentiment ranking -------------------------------------------------------
         yield status("💬 Fetching comments and scoring sentiment…"), gr.update(), gr.update(), gr.update()
-        best, scored = sentiment_mod.rank_by_sentiment(videos, cookiefile, progress, proxy)
+        best, scored = sentiment_mod.rank_by_sentiment(
+            videos, cookiefile, progress, proxy, po_token, visitor_data)
         ranking = gr.update(value=_ranking_rows(scored))
         yield (status(f"🏆 Picked **{best.get('title', best['video_id'])}** "
                       f"({best['positive_share'] * 100:.0f}% positive)."),
@@ -176,7 +187,7 @@ def run_pipeline(topic, hf_token, llm_model, vlm_model, w_llm, w_whisper, lead,
         progress(0.25, desc="Downloading")
         yield status("⬇️ Downloading the chosen video…"), ranking, gr.update(), gr.update()
         video_path, duration = download_mod.download_video(
-            best["url"], workdir, cookiefile, int(max_minutes), proxy)
+            best["url"], workdir, cookiefile, int(max_minutes), proxy, po_token, visitor_data)
         wav = download_mod.extract_audio(video_path, workdir)
 
         # 4. Transcribe --------------------------------------------------------------
@@ -271,15 +282,25 @@ def build_ui():
                 "can get an account rate-limited or flagged.\n"
                 "- **Cookies:** export a `youtube.com` cookies.txt (Netscape format) from a "
                 "logged-in throwaway account and paste it (raw or base64) below.\n"
+                "- **PO token (free, no proxy):** a Proof-of-Origin token + visitor data "
+                "can pass the bot-check from a datacenter IP. **See this Space's README → "
+                "\"PO token\" guide** for how to get and paste them.\n"
                 "- **Proxy:** a **residential** proxy works; **free *datacenter* proxies "
                 "(e.g. Webshare's free tier) usually do NOT** get past YouTube's block and "
                 "have tight bandwidth caps.\n"
-                "- An operator can instead set Space secrets `YT_COOKIES` / `YT_PROXY` as "
-                "shared defaults."
+                "- An operator can instead set Space secrets `YT_COOKIES` / `YT_PROXY` / "
+                "`YT_POT` / `YT_VISITOR_DATA` as shared defaults."
             )
             cookies_text = gr.Textbox(
                 label="YouTube cookies (cookies.txt contents or base64)", lines=4,
                 placeholder="# Netscape HTTP Cookie File …  (or a base64 blob)")
+            with gr.Row():
+                po_token_in = gr.Textbox(
+                    label="PO token (see README) — CLIENT.CONTEXT+TOKEN", lines=2, scale=3,
+                    placeholder="web.gvs+AbC…,  web.player+XyZ…")
+                visitor_data_in = gr.Textbox(
+                    label="Visitor data (pairs with the PO token)", scale=2,
+                    placeholder="Cgt...%3D%3D")
             proxy_url = gr.Textbox(
                 label="Proxy URL (optional)", type="password",
                 placeholder="http://user:pass@host:port")
@@ -320,7 +341,7 @@ def build_ui():
             run_pipeline,
             inputs=[topic, hf_token, llm_model, vlm_model, w_llm, w_whisper, lead,
                     max_minutes, max_shots, primary_kw, secondary_kw,
-                    cookies_text, proxy_url],
+                    cookies_text, proxy_url, po_token_in, visitor_data_in],
             outputs=[status_md, ranking_df, transcript_box, docx_file],
         )
     return demo

pipeline/download.py

@@ -10,24 +10,44 @@ getting past datacenter-IP blocking without cookies.
 from __future__ import annotations
 
 import os
+import re
 import subprocess
 
 from yt_dlp import YoutubeDL
 
 # Player clients tried in order. "android"/"tv" sometimes succeed where "web" is blocked.
 _PLAYER_CLIENTS = ["default", "android", "tv", "web_safari"]
+# When a (web) PO token is supplied it only applies to web-family clients, so try those.
+_POT_CLIENTS = ["web_safari", "web", "default"]
 
 
 class DownloadError(RuntimeError):
     """Raised for user-actionable download failures (blocked IP, too long, etc.)."""
 
 
-def _base_opts(cookiefile: str | None, proxy: str | None, player_client: str) -> dict:
+def _parse_pot(po_token: str | None) -> list[str]:
+    """Split a PO-token field into yt-dlp entries (comma/semicolon/newline separated).
+
+    Each entry should be in yt-dlp's ``CLIENT.CONTEXT+TOKEN`` form, e.g. ``web.gvs+AbC``.
+    """
+    if not po_token:
+        return []
+    return [p.strip() for p in re.split(r"[,\n;]+", po_token) if p.strip()]
+
+
+def _base_opts(cookiefile: str | None, proxy: str | None, player_client: str,
+               po_token: str | None = None, visitor_data: str | None = None) -> dict:
+    yt_args: dict = {"player_client": [player_client]}
+    pots = _parse_pot(po_token)
+    if pots:
+        yt_args["po_token"] = pots
+    if visitor_data and visitor_data.strip():
+        yt_args["visitor_data"] = [visitor_data.strip()]
     opts = {
         "quiet": True,
         "no_warnings": True,
         "noplaylist": True,
-        "extractor_args": {"youtube": {"player_client": [player_client]}},
+        "extractor_args": {"youtube": yt_args},
     }
     if cookiefile:
         opts["cookiefile"] = cookiefile
@@ -37,15 +57,17 @@ def _base_opts(cookiefile: str | None, proxy: str | None, player_client: str) ->
 
 
 def _extract_with_fallback(url: str, base: dict, cookiefile: str | None,
-                           proxy: str | None, download: bool):
+                           proxy: str | None, download: bool,
+                           po_token: str | None = None, visitor_data: str | None = None):
     """Run extract_info, retrying across player clients on failure.
 
     Returns ``(info, ydl)`` from the first client that works; raises DownloadError with a
     helpful hint if all clients fail.
     """
+    clients = _POT_CLIENTS if _parse_pot(po_token) else _PLAYER_CLIENTS
     last_exc = None
-    for client in _PLAYER_CLIENTS:
-        opts = {**_base_opts(cookiefile, proxy, client), **base}
+    for client in clients:
+        opts = {**_base_opts(cookiefile, proxy, client, po_token, visitor_data), **base}
         try:
             ydl = YoutubeDL(opts)
             info = ydl.extract_info(url, download=download)
@@ -53,23 +75,27 @@ def _extract_with_fallback(url: str, base: dict, cookiefile: str | None,
         except Exception as exc:  # try the next client
             last_exc = exc
             continue
-    raise DownloadError(_blocked_hint(last_exc, bool(cookiefile), bool(proxy)))
+    raise DownloadError(
+        _blocked_hint(last_exc, bool(cookiefile), bool(proxy), bool(_parse_pot(po_token))))
 
 
-def probe_duration(url: str, cookiefile: str | None = None,
-                   proxy: str | None = None) -> float:
+def probe_duration(url: str, cookiefile: str | None = None, proxy: str | None = None,
+                   po_token: str | None = None, visitor_data: str | None = None) -> float:
     """Return the video duration in seconds without downloading."""
     info, _ = _extract_with_fallback(url, {"skip_download": True}, cookiefile, proxy,
-                                     download=False)
+                                     download=False, po_token=po_token,
+                                     visitor_data=visitor_data)
     return float(info.get("duration") or 0.0)
 
 
 def download_video(url: str, out_dir: str, cookiefile: str | None = None,
-                   max_minutes: int = 20, proxy: str | None = None) -> tuple[str, float]:
+                   max_minutes: int = 20, proxy: str | None = None,
+                   po_token: str | None = None,
+                   visitor_data: str | None = None) -> tuple[str, float]:
     """Download the video to ``out_dir``; return ``(mp4_path, duration_seconds)``."""
     os.makedirs(out_dir, exist_ok=True)
 
-    duration = probe_duration(url, cookiefile, proxy)
+    duration = probe_duration(url, cookiefile, proxy, po_token, visitor_data)
     if duration and duration > max_minutes * 60:
         raise DownloadError(
             f"Video is {duration / 60:.0f} min long, over the {max_minutes} min cap for "
@@ -81,7 +107,8 @@ def download_video(url: str, out_dir: str, cookiefile: str | None = None,
         "format": "bestvideo[height<=720][ext=mp4]+bestaudio[ext=m4a]/best[height<=720]/best",
         "merge_output_format": "mp4",
     }
-    info, ydl = _extract_with_fallback(url, base, cookiefile, proxy, download=True)
+    info, ydl = _extract_with_fallback(url, base, cookiefile, proxy, download=True,
+                                       po_token=po_token, visitor_data=visitor_data)
     path = ydl.prepare_filename(info)
     duration = float(info.get("duration") or duration or 0.0)
 
@@ -110,21 +137,26 @@ def extract_audio(video_path: str, out_dir: str) -> str:
     return wav_path
 
 
-def _blocked_hint(exc: Exception | None, had_cookies: bool, had_proxy: bool) -> str:
+def _blocked_hint(exc: Exception | None, had_cookies: bool, had_proxy: bool,
+                  had_pot: bool = False) -> str:
     msg = str(exc) if exc else "unknown error"
     lowered = msg.lower()
-    if any(k in lowered for k in ("sign in", "bot", "403", "429", "confirm you", "cookies")):
+    if any(k in lowered for k in ("sign in", "bot", "403", "429", "confirm you", "cookies",
+                                  "po_token", "po token")):
         used = []
         if had_cookies:
             used.append("cookies")
         if had_proxy:
             used.append("proxy")
-        ctx = (f" Your {' and '.join(used)} did not get past it"
-               if used else " No cookies or proxy were provided")
+        if had_pot:
+            used.append("PO token")
+        ctx = (f" Your {' + '.join(used)} did not get past it"
+               if used else " No cookies, proxy, or PO token were provided")
         return (
             "YouTube blocked this request (typical from datacenter IPs)." + ctx + ". "
-            "Provide throwaway-account cookies and/or a residential proxy and retry. "
-            "Note: free datacenter proxies usually do NOT bypass this block.\n"
+            "Try throwaway-account cookies, a PO token + visitor data (see the README "
+            "'PO token' guide), and/or a residential proxy. Note: free datacenter proxies "
+            "usually do NOT bypass this block.\n"
             f"Original error: {msg[:300]}"
         )
     return f"Download failed: {msg[:400]}"

pipeline/sentiment.py

@@ -8,6 +8,7 @@ single video that fails simply scores 0 and we never abort the whole ranking.
 from __future__ import annotations
 
 import os
+import re
 from functools import lru_cache
 
 from yt_dlp import YoutubeDL
@@ -50,13 +51,19 @@ def _polarity(scores) -> float:
     return val
 
 
-def _fetch_comments(url: str, cookiefile: str | None, proxy: str | None) -> list[str]:
+def _fetch_comments(url: str, cookiefile: str | None, proxy: str | None,
+                    po_token: str | None = None, visitor_data: str | None = None) -> list[str]:
+    yt_args: dict = {"max_comments": [str(MAX_COMMENTS)], "comment_sort": ["top"]}
+    if po_token:
+        yt_args["po_token"] = [p.strip() for p in re.split(r"[,\n;]+", po_token) if p.strip()]
+    if visitor_data and visitor_data.strip():
+        yt_args["visitor_data"] = [visitor_data.strip()]
     opts = {
         "skip_download": True,
         "getcomments": True,
         "quiet": True,
         "no_warnings": True,
-        "extractor_args": {"youtube": {"max_comments": [str(MAX_COMMENTS)], "comment_sort": ["top"]}},
+        "extractor_args": {"youtube": yt_args},
     }
     if cookiefile:
         opts["cookiefile"] = cookiefile
@@ -70,7 +77,9 @@ def _fetch_comments(url: str, cookiefile: str | None, proxy: str | None) -> list
 
 
 def rank_by_sentiment(videos: list[dict], cookiefile: str | None = None,
-                      progress=None, proxy: str | None = None) -> tuple[dict, list[dict]]:
+                      progress=None, proxy: str | None = None,
+                      po_token: str | None = None,
+                      visitor_data: str | None = None) -> tuple[dict, list[dict]]:
     """Score each video and return ``(best_video, scored)``.
 
     ``scored`` mirrors ``videos`` with added keys: ``positive_share`` (0..1),
@@ -87,7 +96,7 @@ def rank_by_sentiment(videos: list[dict], cookiefile: str | None = None,
             progress((i + 1) / len(videos), desc=f"Sentiment {i + 1}/{len(videos)}")
         item = dict(v)
         try:
-            comments = _fetch_comments(v["url"], cookiefile, proxy)
+            comments = _fetch_comments(v["url"], cookiefile, proxy, po_token, visitor_data)
             if comments:
                 results = clf(comments)  # list aligned with comments (each = list of labels)
                 pols = [_polarity(r) for r in results]