Replace keyword density with primary/secondary keyword placement + AEO structure

- UI: primary keyword + secondary keywords (comma-separated) instead of density inputs
- LLM prompt: use primary ~3x in body and place it in title/slug/meta/first-100-words of
intro and 1-2 H2 headings; each secondary keyword once
- AEO best practices (from GameMaster citation guide): answer-first "Quick answer" block,
descriptive title + meta description + URL slug, H2 step headings, FAQ section, and a
sources/last-updated trust footer
- docx_builder renders the new structure; tutorial JSON gains slug/meta_description/answer/faqs
- Add count_keyword() and surface the primary-keyword count in the run status

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

README.md

@@ -29,7 +29,10 @@ on that topic.
 5. **Candidate frames** are extracted densely from the video, then the **video is
    auto-deleted** — only small JPEGs and the transcript remain.
 6. **Tutorial text** — `deepseek-ai/DeepSeek-V3` (HF Inference Providers, billed to your
-   token) turns the transcript into structured steps.
+   token) turns the transcript into an **answer-engine-optimized** post: an answer-first
+   paragraph, H2 step headings, an FAQ, a meta description, a URL slug, and a
+   last-updated/source citation. Optional **primary/secondary keyword** placement (primary
+   ~3× + in title/slug/meta/intro/H2; each secondary once).
 7. **Screenshot selection** — a weighted indicator blends each step's LLM-suggested
    timestamp with Whisper's actual speech timing.
 8. **Captions** — a vision model (default `Qwen/Qwen2.5-VL-72B-Instruct`, billed to your

app.py

@@ -65,23 +65,24 @@ def _safe_name(text: str) -> str:
     return re.sub(r"[^A-Za-z0-9._-]+", "_", text).strip("_")[:60] or "tutorial"
 
 
-def _collect_keywords(kw1, d1, kw2, d2, kw3, d3) -> list[dict]:
-    """Build the [{keyword, density}] list from the three UI keyword/density pairs."""
-    keywords = []
-    for kw, dens in ((kw1, d1), (kw2, d2), (kw3, d3)):
-        kw = (kw or "").strip()
-        if not kw:
-            continue
-        try:
-            dval = float(dens or 0)
-        except (TypeError, ValueError):
-            dval = 0.0
-        keywords.append({"keyword": kw, "density": max(0.0, dval)})
-    return keywords
+def _collect_keywords(primary_kw, secondary_kw) -> dict:
+    """Build ``{"primary": str, "secondary": [str, ...]}`` from the two keyword inputs.
+
+    Secondary keywords are comma-separated. Duplicates and the primary are removed.
+    """
+    primary = (primary_kw or "").strip()
+    secondary = []
+    seen = {primary.lower()}
+    for part in (secondary_kw or "").split(","):
+        kw = part.strip()
+        if kw and kw.lower() not in seen:
+            seen.add(kw.lower())
+            secondary.append(kw)
+    return {"primary": primary, "secondary": secondary}
 
 
 def run_pipeline(topic, hf_token, llm_model, vlm_model, w_llm, w_whisper, lead,
-                 max_minutes, max_shots, kw1, d1, kw2, d2, kw3, d3,
+                 max_minutes, max_shots, primary_kw, secondary_kw,
                  progress=gr.Progress()):
     """Generator that yields (status_md, ranking_df, transcript, docx_file)."""
     log: list[str] = []
@@ -142,11 +143,16 @@ def run_pipeline(topic, hf_token, llm_model, vlm_model, w_llm, w_whisper, lead,
 
         # 6. Tutorial text -----------------------------------------------------------
         progress(0.72, desc="Writing tutorial")
-        keywords = _collect_keywords(kw1, d1, kw2, d2, kw3, d3)
-        kw_note = (" with keywords " + ", ".join(f"{k['keyword']} (~{k['density']:.1f}%)"
-                                                 for k in keywords)) if keywords else ""
+        keywords = _collect_keywords(primary_kw, secondary_kw)
+        kw_note = f" • primary: '{keywords['primary']}'" if keywords["primary"] else ""
+        if keywords["secondary"]:
+            kw_note += f" • secondary: {', '.join(keywords['secondary'])}"
         yield status(f"🤖 Generating tutorial with `{llm_model}`{kw_note}…"), ranking, gr.update(value=transcript), gr.update()
         tut = tutorial_mod.generate_tutorial(transcript, hf_token.strip(), llm_model, keywords)
+        if keywords["primary"]:
+            n = tutorial_mod.count_keyword(tut, keywords["primary"])
+            yield (status(f"🔑 Primary keyword '{keywords['primary']}' appears {n}× in the post."),
+                   ranking, gr.update(value=transcript), gr.update())
 
         # 7. Weighted screenshot selection ------------------------------------------
         selected = frames_mod.select_screenshots(
@@ -201,21 +207,18 @@ def build_ui():
                 vlm_model = gr.Dropdown(VLM_CHOICES, value=VLM_CHOICES[0],
                                         label="Vision model (captions)", allow_custom_value=True)
 
-        with gr.Accordion("SEO keywords (optional)", open=False):
+        with gr.Accordion("SEO / AEO keywords (optional)", open=False):
             gr.Markdown(
-                "Weave up to three keywords into the tutorial at a target **keyword "
-                "density** (% of total words). The writer aims for these but stays "
-                "natural — it may use a keyword somewhat less often than requested."
+                "The **primary keyword** is used naturally ~3× in the body and placed in "
+                "the title, URL slug, meta description, the first 100 words, and one or "
+                "two H2 headings. Each **secondary keyword** is used once. The post also "
+                "follows answer-engine best practices (direct answer up top, FAQ, "
+                "last-updated date, source citation)."
             )
-            with gr.Row():
-                kw1 = gr.Textbox(label="Keyword 1", scale=3)
-                d1 = gr.Number(label="Density % 1", value=0, minimum=0, maximum=100, scale=1)
-            with gr.Row():
-                kw2 = gr.Textbox(label="Keyword 2", scale=3)
-                d2 = gr.Number(label="Density % 2", value=0, minimum=0, maximum=100, scale=1)
-            with gr.Row():
-                kw3 = gr.Textbox(label="Keyword 3", scale=3)
-                d3 = gr.Number(label="Density % 3", value=0, minimum=0, maximum=100, scale=1)
+            primary_kw = gr.Textbox(label="Primary keyword",
+                                    placeholder="e.g. godot ai plugin")
+            secondary_kw = gr.Textbox(label="Secondary keywords (comma-separated)",
+                                      placeholder="e.g. gdscript assistant, ai game tools")
 
         with gr.Accordion("Advanced settings", open=False):
             with gr.Row():
@@ -239,7 +242,7 @@ def build_ui():
         run_btn.click(
             run_pipeline,
             inputs=[topic, hf_token, llm_model, vlm_model, w_llm, w_whisper, lead,
-                    max_minutes, max_shots, kw1, d1, kw2, d2, kw3, d3],
+                    max_minutes, max_shots, primary_kw, secondary_kw],
             outputs=[status_md, ranking_df, transcript_box, docx_file],
         )
     return demo

pipeline/docx_builder.py

@@ -1,12 +1,22 @@
-"""Stage 9: assemble the .docx tutorial (text + captioned screenshots)."""
+"""Stage 9: assemble the .docx tutorial (text + captioned screenshots).
+
+Layout follows answer-engine best practices: H1 title, an SEO metadata line
+(slug / meta description / last-updated), an answer-first "Quick answer" block, the
+intro, H2 step sections with captioned screenshots, an FAQ section, and a sources/
+trust footer citing the source video.
+"""
 from __future__ import annotations
 
+import datetime
 import os
 
 from docx import Document
 from docx.enum.text import WD_ALIGN_PARAGRAPH
 from docx.shared import Inches, Pt, RGBColor
 
+_GREY = RGBColor(0x66, 0x66, 0x66)
+_LIGHT = RGBColor(0x88, 0x88, 0x88)
+
 
 def _fmt_ts(seconds: float) -> str:
     m, s = divmod(int(seconds), 60)
@@ -14,27 +24,46 @@ def _fmt_ts(seconds: float) -> str:
     return f"{h:d}:{m:02d}:{s:02d}" if h else f"{m:d}:{s:02d}"
 
 
+def _grey_line(doc, text: str, size: int = 9, color: RGBColor = _GREY):
+    p = doc.add_paragraph()
+    run = p.add_run(text)
+    run.italic = True
+    run.font.size = Pt(size)
+    run.font.color.rgb = color
+    return p
+
+
 def build_docx(tutorial: dict, selected: dict[int, dict], captions: dict[int, str],
                out_path: str, source_url: str = "") -> str:
     """Write the tutorial to ``out_path`` and return it.
 
-    ``tutorial`` = {title, intro, steps}; ``selected`` = {step_idx: {time, path}};
-    ``captions`` = {step_idx: caption}.
+    ``tutorial`` = {title, slug, meta_description, answer, intro, steps, faqs};
+    ``selected`` = {step_idx: {time, path}}; ``captions`` = {step_idx: caption}.
     """
+    today = datetime.date.today().isoformat()
     doc = Document()
     doc.add_heading(tutorial["title"], level=0)
 
+    # SEO metadata line (slug + last-updated) and meta description.
+    meta_bits = []
+    if tutorial.get("slug"):
+        meta_bits.append(f"Slug: /{tutorial['slug']}/")
+    meta_bits.append(f"Last updated: {today}")
+    _grey_line(doc, "  •  ".join(meta_bits))
+    if tutorial.get("meta_description"):
+        _grey_line(doc, f"Meta description: {tutorial['meta_description']}")
+
+    # Answer-first block (quotable by AI assistants).
+    if tutorial.get("answer"):
+        doc.add_heading("Quick answer", level=2)
+        ans = doc.add_paragraph()
+        ans.add_run(tutorial["answer"]).bold = True
+
     if tutorial.get("intro"):
         doc.add_paragraph(tutorial["intro"])
 
-    if source_url:
-        src = doc.add_paragraph()
-        run = src.add_run(f"Source video: {source_url}")
-        run.italic = True
-        run.font.size = Pt(9)
-        run.font.color.rgb = RGBColor(0x66, 0x66, 0x66)
-
-    for i, step in enumerate(tutorial["steps"], start=0):
+    # Step sections.
+    for i, step in enumerate(tutorial["steps"]):
         doc.add_heading(f"{i + 1}. {step['heading']}", level=1)
         if step.get("body"):
             doc.add_paragraph(step["body"])
@@ -47,14 +76,26 @@ def build_docx(tutorial: dict, selected: dict[int, dict], captions: dict[int, st
 
             cap_par = doc.add_paragraph()
             cap_par.alignment = WD_ALIGN_PARAGRAPH.CENTER
-            caption = captions.get(i, "")
-            cap_run = cap_par.add_run(f"{caption}  ")
+            cap_run = cap_par.add_run(f"{captions.get(i, '')}  ")
             cap_run.italic = True
             cap_run.font.size = Pt(9)
             ts_run = cap_par.add_run(f"(@ {_fmt_ts(sel['time'])})")
             ts_run.italic = True
             ts_run.font.size = Pt(9)
-            ts_run.font.color.rgb = RGBColor(0x88, 0x88, 0x88)
+            ts_run.font.color.rgb = _LIGHT
+
+    # FAQ section (visible Q&A — AEO/FAQPage-friendly).
+    if tutorial.get("faqs"):
+        doc.add_heading("Frequently asked questions", level=1)
+        for f in tutorial["faqs"]:
+            doc.add_heading(f["q"], level=2)
+            doc.add_paragraph(f["a"])
+
+    # Sources / trust footer (E-E-A-T: citation + freshness).
+    doc.add_heading("Sources", level=1)
+    if source_url:
+        _grey_line(doc, f"Adapted from the source video: {source_url}", color=_GREY)
+    _grey_line(doc, f"Generated and last updated: {today}.", color=_GREY)
 
     os.makedirs(os.path.dirname(os.path.abspath(out_path)), exist_ok=True)
     doc.save(out_path)

pipeline/tutorial.py

@@ -1,8 +1,13 @@
-"""Stage 7: transform the timestamped transcript into a structured tutorial.
+"""Stage 7: transform the timestamped transcript into a structured, AEO-friendly tutorial.
 
 Uses an HF Inference Providers chat model (default DeepSeek-V3) billed to the user's
-token. The model is asked for strict JSON so the downstream weighted indicator and the
-.docx builder have stable fields to work with.
+token. The model returns strict JSON so the downstream weighted indicator and the .docx
+builder have stable fields to work with.
+
+The output follows answer-engine-optimization (AEO) / AI-citation best practices: an
+answer-first paragraph near the top, descriptive title + meta description + URL slug,
+clear H2 step headings, and an FAQ section. SEO keyword placement is explicit (see
+``_keyword_block``).
 """
 from __future__ import annotations
 
@@ -18,32 +23,44 @@ DEFAULT_LLM = "deepseek-ai/DeepSeek-V3"
 MAX_TRANSCRIPT_CHARS = 24000
 
 _SYSTEM = (
-    "You are a technical writer. You convert a timestamped video transcript into a clear, "
-    "step-by-step written tutorial. You ALWAYS respond with a single JSON object and no "
-    "prose outside it."
+    "You are a technical writer who optimizes for answer engines (ChatGPT, Claude, "
+    "Gemini, Perplexity, Google AI). You convert a timestamped video transcript into a "
+    "clear, citable, step-by-step written tutorial. You ALWAYS respond with a single JSON "
+    "object and no prose outside it."
 )
 
 _INSTRUCTIONS = """\
-Turn the transcript below into a tutorial. Return ONLY a JSON object with this schema:
+Turn the transcript below into a tutorial blog post. Return ONLY a JSON object with this schema:
 
 {
-  "title": "string - concise tutorial title",
-  "intro": "string - 2-4 sentence overview",
+  "title": "string - H1 / title tag; concise and descriptive",
+  "slug": "string - lowercase, hyphenated URL slug",
+  "meta_description": "string - ~150-160 chars, compelling, search-friendly",
+  "answer": "string - 1-3 sentence DIRECT answer/definition of the topic, placed first so "
+            "AI assistants can quote it on its own",
+  "intro": "string - 2-4 sentence overview that builds on the answer",
   "steps": [
     {
-      "heading": "string - short step title",
-      "body": "string - 1-3 paragraphs explaining this step in your own words",
+      "heading": "string - short, descriptive H2 step title",
+      "body": "string - 1-3 paragraphs in your own words; specific and verifiable",
       "quote": "string - a short, near-verbatim snippet (<=120 chars) copied from the "
                "transcript line this step is based on, used to locate the moment",
       "t_llm": number,        // best timestamp IN SECONDS for an illustrative screenshot
       "importance": number    // 0..1, how worth screenshotting this step is
     }
+  ],
+  "faqs": [
+    { "q": "string - a natural question a user might ask an AI assistant",
+      "a": "string - a concise, factual 1-3 sentence answer" }
   ]
 }
 
 Rules:
 - 4 to 10 steps. Keep quotes copied from the transcript so they can be matched back.
 - t_llm must be within the transcript's time range.
+- Provide 3-5 FAQs phrased as real questions ("Can ...?", "How do I ...?", "What is ...?").
+- Be specific and factual (citation-worthy); avoid marketing fluff. Note prerequisites or
+  limitations where relevant.
 - Output valid JSON only. No markdown, no comments in the actual output.
 
 {keyword_block}
@@ -55,27 +72,40 @@ Transcript (each line is "[mm:ss] text"):
 """
 
 
-def _keyword_block(keywords: list[dict] | None) -> str:
-    """Render the SEO-keyword guidance injected into the prompt."""
-    keywords = [k for k in (keywords or []) if k.get("keyword")]
-    if not keywords:
+def _keyword_block(keywords: dict | None) -> str:
+    """Render the SEO/keyword placement guidance injected into the prompt.
+
+    ``keywords`` = ``{"primary": str, "secondary": [str, ...]}``.
+    """
+    keywords = keywords or {}
+    primary = (keywords.get("primary") or "").strip()
+    secondary = [s for s in (keywords.get("secondary") or []) if s.strip()]
+    if not primary and not secondary:
         return "SEO keywords: none specified."
-    lines = [
-        "SEO keywords: naturally weave the following keywords into the title, intro and "
-        "step bodies, aiming for roughly the target keyword density (share of total "
-        "words). Treat each target as an upper guide: keep the writing natural and "
-        "readable, and prefer using a keyword somewhat LESS often than the target over "
-        "forcing awkward, repetitive phrasing (no keyword stuffing):",
-    ]
-    for k in keywords:
-        try:
-            dens = float(k.get("density", 0) or 0)
-        except (TypeError, ValueError):
-            dens = 0.0
-        lines.append(f'  - "{k["keyword"]}": target ~{dens:.1f}% of total words')
+
+    lines = ["SEO / keyword requirements (keep everything natural - never keyword-stuff):"]
+    if primary:
+        lines += [
+            f'- PRIMARY keyword: "{primary}".',
+            f'  * Use "{primary}" naturally about 3 times in the body text.',
+            f'  * Also place "{primary}" in: the title (H1), the URL slug, the meta '
+            f'description, within the FIRST 100 words of the intro, and in one or two '
+            f'H2 step headings where it fits naturally.',
+        ]
+    if secondary:
+        joined = ", ".join(f'"{s}"' for s in secondary)
+        lines.append(
+            f"- SECONDARY keywords ({joined}): use each one EXACTLY once, naturally, "
+            "somewhere in the body."
+        )
     return "\n".join(lines)
 
 
+def _slugify(text: str) -> str:
+    text = re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")
+    return text[:80] or "tutorial"
+
+
 def _extract_json(text: str) -> dict:
     """Parse the model output into a dict, tolerating code fences / stray prose."""
     text = text.strip()
@@ -111,19 +141,55 @@ def _normalize(data: dict) -> dict:
         })
     if not steps:
         raise RuntimeError("The LLM returned no usable steps.")
+
+    faqs = []
+    for f in data.get("faqs", []) or []:
+        q = str(f.get("q", "")).strip()
+        a = str(f.get("a", "")).strip()
+        if q and a:
+            faqs.append({"q": q, "a": a})
+
+    title = str(data.get("title", "Tutorial")).strip() or "Tutorial"
+    intro = str(data.get("intro", "")).strip()
+    answer = str(data.get("answer", "")).strip()
+    meta = str(data.get("meta_description", "")).strip() or (answer or intro)[:160]
+    slug = _slugify(str(data.get("slug", "")).strip() or title)
     return {
-        "title": str(data.get("title", "Tutorial")).strip() or "Tutorial",
-        "intro": str(data.get("intro", "")).strip(),
+        "title": title,
+        "slug": slug,
+        "meta_description": meta,
+        "answer": answer,
+        "intro": intro,
         "steps": steps,
+        "faqs": faqs,
     }
 
 
+def count_keyword(tutorial: dict, keyword: str) -> int:
+    """Count case-insensitive whole-keyword occurrences across all post text.
+
+    Used only for transparency in the UI status — placement isn't hard-enforced.
+    """
+    keyword = (keyword or "").strip()
+    if not keyword:
+        return 0
+    parts = [tutorial.get("title", ""), tutorial.get("slug", "").replace("-", " "),
+             tutorial.get("meta_description", ""), tutorial.get("answer", ""),
+             tutorial.get("intro", "")]
+    for s in tutorial.get("steps", []):
+        parts += [s.get("heading", ""), s.get("body", "")]
+    for f in tutorial.get("faqs", []):
+        parts += [f.get("q", ""), f.get("a", "")]
+    blob = "\n".join(parts).lower()
+    return len(re.findall(re.escape(keyword.lower()), blob))
+
+
 def generate_tutorial(transcript: str, hf_token: str, model: str = DEFAULT_LLM,
-                      keywords: list[dict] | None = None) -> dict:
-    """Call the chat model and return a normalized ``{title, intro, steps}`` dict.
+                      keywords: dict | None = None) -> dict:
+    """Call the chat model and return a normalized tutorial dict.
 
-    ``keywords`` is an optional list of ``{"keyword", "density"}`` SEO targets the model
-    is asked to weave in at roughly the given keyword density.
+    Returned keys: ``title, slug, meta_description, answer, intro, steps, faqs``.
+    ``keywords`` is ``{"primary": str, "secondary": [str, ...]}`` for SEO placement.
     """
     if not hf_token:
         raise ValueError("An HF token is required for the tutorial LLM (billed to your key).")