Upload 7 files

app.py

@@ -10,6 +10,7 @@ Run locally:
 
 import gradio as gr
 import numpy as np
+import os
 import uuid
 import json
 from datetime import datetime
@@ -24,6 +25,11 @@ ai_pipeline = HausaVoiceAIPipeline()
 # NLU backend chain: HF Inference API (if HF_TOKEN set) → local LLM → rules
 dm          = Orchestrator(crm=CRMClient(), nlu=NLU())
 
+# ── Streaming config ─────────────────────────────────────────────────────────
+# Partials cost an extra Whisper decode every ~900ms. On a CPU-only Space that
+# can saturate the box, so allow turning them off via env var.
+SHOW_PARTIALS = os.getenv("SHOW_PARTIALS", "1") == "1"
+
 # ── Demo phrases (for visitors who don't speak Hausa) ────────────────────────
 DEMO_PROMPTS = [
     ("Compound: balance + transfer",
@@ -285,10 +291,15 @@ ARCH_HTML = """
 
   <!-- Pipeline -->
   <div style="display:flex; flex-direction:column; gap:8px;">
+    <div style="background:rgba(220,38,38,0.1);border:1px solid rgba(220,38,38,0.3);
+                border-radius:8px; padding:10px 20px; text-align:center;">
+      <div style="font-size:11px; font-weight:700; color:#F87171;">VAD ENDPOINTING</div>
+      <div style="font-size:10px; color:#78716C;">Silero VAD · preroll · hangover · barge-in</div>
+    </div>
     <div style="background:rgba(245,158,11,0.1);border:1px solid rgba(245,158,11,0.3);
                 border-radius:8px; padding:10px 20px; text-align:center;">
-      <div style="font-size:11px; font-weight:700; color:#F59E0B;">WHISPER LARGE-V3</div>
-      <div style="font-size:10px; color:#78716C;">Hausa ASR  · openai/whisper-large-v3</div>
+      <div style="font-size:11px; font-weight:700; color:#F59E0B;">WHISPER (DUAL)</div>
+      <div style="font-size:10px; color:#78716C;">small → partials · large-v3 → final</div>
     </div>
     <div style="background:rgba(56,189,248,0.1);border:1px solid rgba(56,189,248,0.3);
                 border-radius:8px; padding:10px 20px; text-align:center;">
@@ -344,6 +355,9 @@ def _init_state():
     return {
         "conv":    dm.new_session(),
         "history": [],   # [{role, hausa, english, time}]
+        "sasr":    None, # StreamingASR — created on first streamed chunk
+        "partial": "",   # live (unconfirmed) transcript
+        "metrics": [],   # per-turn latency records
     }
 
 def _render_conversation(history: list) -> str:
@@ -404,30 +418,47 @@ def process_voice(audio, text_input, state):
     else:
         return None, _render_conversation(state["history"]), "⚠ No input provided", state
 
-    # 2. Hausa → English
+    audio_out, status = _handle_turn(hausa_text, state)
+    return audio_out, _render_conversation(state["history"]), status, state
+
+
+# ── Shared turn handler (used by both push-to-talk and streaming) ────────────
+
+def _handle_turn(hausa_text: str, state: dict, asr_ms: float = 0.0):
+    """
+    Runs MT → dialogue → MT → TTS for one finalized Hausa utterance.
+    Mutates state['history'] and state['conv']. Returns (audio_out, status).
+    """
+    import time as _time
+    t_mt0 = _time.perf_counter()
     english_text = ai_pipeline.hausa_to_english(hausa_text)
+    mt_in_ms = (_time.perf_counter() - t_mt0) * 1000
 
-    # 3. Dialogue
+    t_nlu0 = _time.perf_counter()
     english_response, conv_state, escalate = dm.respond(
         english_text, hausa_text, state["conv"]
     )
     state["conv"] = conv_state
+    nlu_ms = (_time.perf_counter() - t_nlu0) * 1000
 
-    # 4. English → Hausa
+    t_mt1 = _time.perf_counter()
     hausa_response = ai_pipeline.english_to_hausa(english_response)
+    mt_out_ms = (_time.perf_counter() - t_mt1) * 1000
 
-    # 5. TTS
+    t_tts = _time.perf_counter()
     sr, audio_out = ai_pipeline.hausa_text_to_audio(hausa_response)
+    tts_ms = (_time.perf_counter() - t_tts) * 1000
 
-    # 6. Update history
     now = datetime.now().strftime("%H:%M:%S")
-    state["history"].append({
-        "role": "user", "hausa": hausa_text,
-        "english": english_text, "time": now
-    })
-    state["history"].append({
-        "role": "agent", "hausa": hausa_response,
-        "english": english_response, "time": now
+    state["history"].append({"role": "user", "hausa": hausa_text,
+                             "english": english_text, "time": now})
+    state["history"].append({"role": "agent", "hausa": hausa_response,
+                             "english": english_response, "time": now})
+
+    total = asr_ms + mt_in_ms + nlu_ms + mt_out_ms + tts_ms
+    state["metrics"].append({
+        "asr": asr_ms, "mt_in": mt_in_ms, "nlu": nlu_ms,
+        "mt_out": mt_out_ms, "tts": tts_ms, "total": total,
     })
 
     open_tasks = [t for t in conv_state.tasks
@@ -437,15 +468,73 @@ def process_voice(audio, text_input, state):
               f"{sum(1 for t in conv_state.tasks if t.status == 'done')} done")
     if conv_state.active_task:
         status += f" · active: {conv_state.active_task.intent}"
+    status += (f"  |  ASR {asr_ms:.0f}ms · MT {mt_in_ms + mt_out_ms:.0f}ms · "
+               f"NLU {nlu_ms:.0f}ms · TTS {tts_ms:.0f}ms · "
+               f"total {total:.0f}ms")
     if escalate:
         status += "  ⚠ ESCALATED TO HUMAN"
 
-    return (sr, audio_out), _render_conversation(state["history"]), status, state
+    return (sr, audio_out), status
+
+
+# ── Streaming handler: VAD endpointing, no button press ──────────────────────
+
+def process_stream(stream_chunk, state):
+    """
+    Bound to gr.Audio(streaming=True).stream — fires every ~0.3s with a new
+    chunk of mic audio. The VAD decides when a turn has ended; no push-to-talk.
+
+    Returns: (audio_out, conversation_html, status, live_partial, state)
+    """
+    if state is None:
+        state = _init_state()
+    if stream_chunk is None:
+        return (None, _render_conversation(state["history"]),
+                "Listening …", state.get("partial", ""), state)
+
+    if state["sasr"] is None:
+        state["sasr"] = ai_pipeline.make_streaming_session(
+            emit_partials=SHOW_PARTIALS)
+
+    sr, chunk = stream_chunk
+    events = state["sasr"].accept_audio(chunk, sr)
+
+    audio_out = None
+    status    = None
+
+    for ev in events:
+        if ev.kind == "speech_start":
+            status = "🎙  Listening — speech detected …"
+
+        elif ev.kind == "partial":
+            state["partial"] = ev.text
+            status = f"🎙  … ({ev.duration_ms/1000:.1f}s)"
+
+        elif ev.kind == "bargein":
+            # Caller interrupted the agent: stop playback immediately
+            audio_out = None
+            status = "✋  Barge-in — you interrupted, go ahead."
+
+        elif ev.kind == "discarded":
+            state["partial"] = ""
+            status = "Listening …"
+
+        elif ev.kind == "final":
+            state["partial"] = ""
+            state["sasr"].agent_speaking = True     # arm barge-in for playback
+            audio_out, status = _handle_turn(ev.text, state,
+                                             asr_ms=ev.latency_ms)
+
+    return (audio_out,
+            _render_conversation(state["history"]),
+            status or "Listening …",
+            state.get("partial", ""),
+            state)
 
 
 def reset_session(state):
     state = _init_state()
-    return None, _render_conversation([]), "New session started.", state
+    return None, _render_conversation([]), "New session started.", "", state
 
 
 def use_demo_prompt(prompt_ha, state):
@@ -488,6 +577,12 @@ with gr.Blocks(css=CSS, title="PlotWeaver · Hausa Voice AI") as demo:
                         _render_conversation([]),
                         elem_classes=["conversation-box"]
                     )
+                    partial_box = gr.Textbox(
+                        label="Live transcript (unconfirmed)",
+                        value="",
+                        interactive=False,
+                        lines=1,
+                    )
                     status_box = gr.Textbox(
                         label="Pipeline Status",
                         value="Ready. Speak or type in Hausa.",
@@ -499,12 +594,23 @@ with gr.Blocks(css=CSS, title="PlotWeaver · Hausa Voice AI") as demo:
                 with gr.Column(scale=1):
                     gr.HTML('<div class="pw-card-title" '
                             'style="margin-bottom:10px;">Input</div>')
-                    audio_in = gr.Audio(
+
+                    # ── Streaming mic: VAD endpoints the turn, no button ──
+                    stream_in = gr.Audio(
                         sources=["microphone"],
                         type="numpy",
-                        label="Voice Input (Hausa)",
-                        streaming=False,
+                        label="🔴 Live Mic — just talk, VAD ends your turn",
+                        streaming=True,
                     )
+
+                    with gr.Accordion("Push-to-talk (fallback)", open=False):
+                        audio_in = gr.Audio(
+                            sources=["microphone", "upload"],
+                            type="numpy",
+                            label="Record or upload, then press Send",
+                            streaming=False,
+                        )
+
                     text_in = gr.Textbox(
                         label="Text fallback (Hausa)",
                         placeholder="Sannu, ina son sanin asusun kuɗin…",
@@ -534,6 +640,23 @@ with gr.Blocks(css=CSS, title="PlotWeaver · Hausa Voice AI") as demo:
                         )
 
             # ── Events ─────────────────────────────────────────────────────
+            # Streaming: fires continuously; VAD decides when a turn ends.
+            stream_in.stream(
+                fn=process_stream,
+                inputs=[stream_in, app_state],
+                outputs=[audio_out, conversation_display, status_box,
+                         partial_box, app_state],
+                stream_every=0.3,
+                show_progress="hidden",
+            )
+            # Flush any in-progress utterance when the mic stops.
+            stream_in.stop_recording(
+                fn=lambda s: reset_session(s)[1:],
+                inputs=[app_state],
+                outputs=[conversation_display, status_box, partial_box,
+                         app_state],
+            )
+
             submit_btn.click(
                 fn=process_voice,
                 inputs=[audio_in, text_in, app_state],
@@ -542,7 +665,8 @@ with gr.Blocks(css=CSS, title="PlotWeaver · Hausa Voice AI") as demo:
             reset_btn.click(
                 fn=reset_session,
                 inputs=[app_state],
-                outputs=[audio_out, conversation_display, status_box, app_state],
+                outputs=[audio_out, conversation_display, status_box,
+                         partial_box, app_state],
             )
 
         # ── Tab 2 : Architecture ────────────────────────────────────────────

pipeline.py

@@ -0,0 +1,197 @@
+"""
+HausaVoiceAIPipeline
+====================
+Full pipeline: Audio → Whisper ASR → NLLB translation → LLM NLU → NLLB → MMS-TTS → Audio
+
+Models used:
+  - ASR  : openai/whisper-large-v3          (Hausa language support)
+  - MT   : facebook/nllb-200-distilled-600M  (hau_Latn ↔ eng_Latn)
+  - TTS  : facebook/mms-tts-hau              (Hausa VITS synthesis)
+"""
+
+import os
+import time
+import logging
+import numpy as np
+import torch
+from typing import Optional, Tuple
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# ── Language codes ──────────────────────────────────────────────────────────
+HAUSA_NLLB   = "hau_Latn"
+ENGLISH_NLLB = "eng_Latn"
+FRENCH_NLLB  = "fra_Latn"
+
+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+logger.info(f"Pipeline running on: {DEVICE}")
+
+
+class HausaVoiceAIPipeline:
+    """
+    Lazy-loading pipeline.  Each component is loaded on first use to keep
+    the Space startup time reasonable on CPU.
+    """
+
+    # Dual-model ASR: large-v3 for the FINAL decode (accuracy), a small model
+    # for live partials (latency). Partials are thrown away and replaced by the
+    # final, so their error rate matters far less than their speed.
+    ASR_FINAL_MODEL   = os.getenv("ASR_FINAL_MODEL",   "openai/whisper-large-v3")
+    ASR_PARTIAL_MODEL = os.getenv("ASR_PARTIAL_MODEL", "openai/whisper-small")
+
+    def __init__(self, pivot_language: str = "english"):
+        self.pivot = pivot_language          # dialogue logic runs in English
+        self._asr   = None
+        self._asr_fast = None
+        self._nllb_model     = None
+        self._nllb_tokenizer = None
+        self._tts_model      = None
+        self._tts_tokenizer  = None
+        self.sample_rate = 16_000           # Whisper input
+        self.tts_sample_rate = 16_000       # MMS output
+
+    # ── Lazy loaders ────────────────────────────────────────────────────────
+
+    def _load_asr(self):
+        if self._asr is not None:
+            return
+        logger.info(f"Loading {self.ASR_FINAL_MODEL} …")
+        from transformers import pipeline as hf_pipeline
+        self._asr = hf_pipeline(
+            "automatic-speech-recognition",
+            model=self.ASR_FINAL_MODEL,
+            generate_kwargs={"language": "hausa", "task": "transcribe"},
+            device=0 if DEVICE == "cuda" else -1,
+            chunk_length_s=30,
+        )
+
+    def _load_asr_fast(self):
+        """Small model for live partial transcripts."""
+        if self._asr_fast is not None:
+            return
+        if self.ASR_PARTIAL_MODEL == self.ASR_FINAL_MODEL:
+            self._load_asr()
+            self._asr_fast = self._asr
+            return
+        logger.info(f"Loading {self.ASR_PARTIAL_MODEL} (partials) …")
+        from transformers import pipeline as hf_pipeline
+        self._asr_fast = hf_pipeline(
+            "automatic-speech-recognition",
+            model=self.ASR_PARTIAL_MODEL,
+            generate_kwargs={"language": "hausa", "task": "transcribe"},
+            device=0 if DEVICE == "cuda" else -1,
+            chunk_length_s=30,
+        )
+
+    def _load_nllb(self):
+        if self._nllb_model is not None:
+            return
+        logger.info("Loading NLLB-200-distilled-600M …")
+        from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+        model_id = "facebook/nllb-200-distilled-600M"
+        self._nllb_tokenizer = AutoTokenizer.from_pretrained(model_id)
+        self._nllb_model = AutoModelForSeq2SeqLM.from_pretrained(model_id).to(DEVICE)
+
+    def _load_tts(self):
+        if self._tts_model is not None:
+            return
+        logger.info("Loading MMS-TTS Hausa …")
+        from transformers import VitsModel, AutoTokenizer
+        model_id = "facebook/mms-tts-hau"
+        self._tts_tokenizer = AutoTokenizer.from_pretrained(model_id)
+        self._tts_model = VitsModel.from_pretrained(model_id).to(DEVICE)
+        self.tts_sample_rate = self._tts_model.config.sampling_rate
+
+    # ── Public API ───────────────────────────────────────────────────────────
+
+    def transcribe(self, audio_array: np.ndarray, sample_rate: int = 16_000) -> str:
+        """
+        Whisper ASR: raw audio → Hausa text.
+        audio_array : float32 numpy array, mono
+        """
+        self._load_asr()
+        t0 = time.perf_counter()
+        # Resample if needed (scipy-based; avoids pulling in librosa/numba)
+        if sample_rate != 16_000:
+            from streaming_asr import _resample
+            audio_array = _resample(audio_array, sample_rate, 16_000)
+        result = self._asr({"array": audio_array.astype(np.float32), "sampling_rate": 16_000})
+        text = result["text"].strip()
+        logger.info(f"ASR ({time.perf_counter()-t0:.2f}s): {text}")
+        return text
+
+    def transcribe_partial(self, audio_array: np.ndarray,
+                           sample_rate: int = 16_000) -> str:
+        """Fast, lower-accuracy decode for live on-screen partials."""
+        self._load_asr_fast()
+        if sample_rate != 16_000:
+            from streaming_asr import _resample
+            audio_array = _resample(audio_array, sample_rate, 16_000)
+        result = self._asr_fast({"array": audio_array.astype(np.float32),
+                                  "sampling_rate": 16_000})
+        return result["text"].strip()
+
+    def make_streaming_session(self, emit_partials: bool = True,
+                                config=None, vad_backend: str = "auto"):
+        """
+        Build a StreamingASR wired to this pipeline's two Whisper models.
+        Each caller/session needs its own instance (it holds audio state).
+        """
+        from streaming_asr import StreamingASR
+        return StreamingASR(
+            transcribe_fn=self.transcribe,
+            partial_transcribe_fn=self.transcribe_partial,
+            config=config,
+            vad_backend=vad_backend,
+            emit_partials=emit_partials,
+        )
+
+    def translate(self, text: str, src_lang: str, tgt_lang: str,
+                  max_new_tokens: int = 256) -> str:
+        """NLLB translation."""
+        self._load_nllb()
+        t0 = time.perf_counter()
+        self._nllb_tokenizer.src_lang = src_lang
+        inputs = self._nllb_tokenizer(text, return_tensors="pt", truncation=True,
+                                       max_length=512).to(DEVICE)
+        forced_id = self._nllb_tokenizer.convert_tokens_to_ids(tgt_lang)
+        with torch.no_grad():
+            tokens = self._nllb_model.generate(
+                **inputs,
+                forced_bos_token_id=forced_id,
+                max_new_tokens=max_new_tokens,
+            )
+        translated = self._nllb_tokenizer.decode(tokens[0], skip_special_tokens=True)
+        logger.info(f"NLLB {src_lang}→{tgt_lang} ({time.perf_counter()-t0:.2f}s): {translated}")
+        return translated
+
+    def synthesize(self, text: str) -> Tuple[int, np.ndarray]:
+        """
+        MMS-TTS: Hausa text → (sample_rate, audio_array int16).
+        """
+        self._load_tts()
+        t0 = time.perf_counter()
+        inputs = self._tts_tokenizer(text, return_tensors="pt").to(DEVICE)
+        with torch.no_grad():
+            output = self._tts_model(**inputs).waveform
+        audio = output.squeeze().cpu().numpy()
+        # Normalise → int16
+        audio = (audio / np.abs(audio).max() * 32767).astype(np.int16)
+        logger.info(f"TTS ({time.perf_counter()-t0:.2f}s): {len(audio)/self.tts_sample_rate:.1f}s audio")
+        return self.tts_sample_rate, audio
+
+    # ── Full round-trip helper ───────────────────────────────────────────────
+
+    def hausa_to_english(self, hausa_text: str) -> str:
+        return self.translate(hausa_text, HAUSA_NLLB, ENGLISH_NLLB)
+
+    def english_to_hausa(self, english_text: str) -> str:
+        return self.translate(english_text, ENGLISH_NLLB, HAUSA_NLLB)
+
+    def audio_to_hausa_text(self, audio_array: np.ndarray,
+                             sample_rate: int = 16_000) -> str:
+        return self.transcribe(audio_array, sample_rate)
+
+    def hausa_text_to_audio(self, hausa_text: str) -> Tuple[int, np.ndarray]:
+        return self.synthesize(hausa_text)

requirements.txt

@@ -1,10 +1,37 @@
-# PlotWeaver Voice Agent — HF Space (Gradio SDK + ZeroGPU)
-gradio>=6.0,<7.0
-spaces
-torch>=2.2
-transformers>=4.44
-sentence-transformers>=3.0
-huggingface_hub>=0.24
-accelerate
-numpy
-scipy
+# ── PlotWeaver Hausa Voice AI Agent ──────────────────────────────────────────
+# Audited against actual imports. Nothing here is speculative.
+#
+# HF Spaces reinstalls this on every rebuild, so the list is deliberately small:
+# librosa was removed in favour of the scipy resampler already in the codebase
+# (librosa pulls numba + llvmlite and adds several minutes to cold builds).
+
+# ── Core ML ──────────────────────────────────────────────────────────────────
+torch>=2.2.0                # Whisper, NLLB, MMS, Silero VAD
+transformers>=4.40.0        # >=4.40 required for VitsModel (MMS-TTS)
+sentencepiece>=0.2.0        # NLLB tokenizer — NOT optional, NLLB fails without it
+accelerate>=0.28.0          # device_map="auto" for the local NLU model
+protobuf>=4.25.0            # slow→fast tokenizer conversion for NLLB
+
+# ── Audio ────────────────────────────────────────────────────────────────────
+numpy>=1.26.0,<3.0.0        # 2.x is fine; cap guards against a future 3.0 break
+scipy>=1.12.0               # scipy.signal.resample — the only resampler used
+soundfile>=0.12.1           # Gradio audio I/O backend
+
+# ── App ──────────────────────────────────────────────────────────────────────
+gradio>=4.44.0              # streaming=True + stream_every need >=4.44
+requests>=2.31.0            # WhatsApp Cloud API, Zendesk, HF Inference API
+
+# ── Optional integrations ────────────────────────────────────────────────────
+# Only needed when SIP_PROVIDER=twilio with real credentials. integrations/sip.py
+# imports twilio lazily inside the non-demo branches, so the demo never touches
+# it and this line can be dropped to slim the image.
+twilio>=9.0.0
+
+# ── Deliberately absent ──────────────────────────────────────────────────────
+# torchaudio  — nothing imports it.
+# librosa     — resampling uses scipy.signal.resample instead.
+# silero-vad  — vad.py loads it via torch.hub and falls back to the built-in
+#               energy VAD if the download is blocked. No package needed.
+# audioop     — G.711 mu-law decoding is implemented in numpy in
+#               integrations/sip.py, because audioop was REMOVED from the
+#               stdlib in Python 3.13 and would break telephony on new images.

sip.py

@@ -0,0 +1,186 @@
+"""
+Phone / SIP Integration  (Twilio + Bandwidth stubs)
+=====================================================
+Handles inbound calls → streams audio → runs pipeline → streams TTS back.
+
+For a real deployment, use:
+  - Twilio Media Streams (WebSocket) + <Stream> TwiML verb
+  - Bandwidth BXML + WebSocket audio streaming
+  - Vonage Voice API + WebSocket
+
+Environment variables:
+  TWILIO_ACCOUNT_SID  = ACxxxx
+  TWILIO_AUTH_TOKEN   = xxxx
+  TWILIO_PHONE_NUMBER = +1234567890
+  SIP_PROVIDER        = twilio | bandwidth | demo
+"""
+
+import os
+import logging
+from typing import Callable, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class TwilioAdapter:
+    """
+    Twilio Media Streams WebSocket adapter.
+    
+    Flow:
+      1.  Inbound call → Twilio webhook → /voice endpoint
+      2.  Return TwiML with <Connect><Stream> → Twilio opens WS
+      3.  WebSocket handler receives mulaw 8kHz chunks
+      4.  Chunks accumulated → ASR → NLU → TTS → send back over WS
+    """
+
+    def __init__(self):
+        self.sid   = os.getenv("TWILIO_ACCOUNT_SID",  "DEMO")
+        self.token = os.getenv("TWILIO_AUTH_TOKEN",   "DEMO")
+        self.phone = os.getenv("TWILIO_PHONE_NUMBER", "+0000000000")
+        self._demo = self.sid == "DEMO"
+
+    def incoming_call_twiml(self, websocket_url: str) -> str:
+        """
+        Returns TwiML that Twilio will execute when a call arrives.
+        websocket_url: wss://yourserver.com/ws/audio
+        """
+        return f"""<?xml version="1.0" encoding="UTF-8"?>
+<Response>
+  <Say language="ha-NG">Sannu, barka da zuwa PlotWeaver. Muna jira…</Say>
+  <Connect>
+    <Stream url="{websocket_url}">
+      <Parameter name="language" value="hausa"/>
+    </Stream>
+  </Connect>
+</Response>"""
+
+    def handle_ws_message(self, message: dict,
+                           on_audio_chunk: Callable[[bytes], None]) -> None:
+        """
+        Called for each WebSocket message from Twilio.
+        Twilio sends: start, media (base64 mulaw), stop events.
+        """
+        import base64
+        event = message.get("event")
+        if event == "media":
+            chunk = base64.b64decode(message["media"]["payload"])
+            on_audio_chunk(chunk)
+        elif event == "stop":
+            logger.info(f"Call ended: {message.get('stop', {}).get('callSid')}")
+
+    def send_audio_twiml(self, call_sid: str, audio_url: str) -> dict:
+        """
+        Interrupt the current call and play synthesised audio.
+        Production: POST to Twilio API to update call.
+        """
+        if self._demo:
+            logger.info(f"[DEMO] Would play {audio_url} on call {call_sid}")
+            return {"status": "demo"}
+        from twilio.rest import Client
+        client = Client(self.sid, self.token)
+        call = client.calls(call_sid).update(
+            twiml=f'<Response><Play>{audio_url}</Play></Response>'
+        )
+        return {"status": call.status}
+
+    def make_outbound_call(self, to: str, message_en: str,
+                            message_ha: str = "") -> dict:
+        """Outbound IVR call with TTS message."""
+        twiml = f"""<?xml version="1.0" encoding="UTF-8"?>
+<Response>
+  <Say language="ha-NG">{message_ha or message_en}</Say>
+</Response>"""
+        if self._demo:
+            logger.info(f"[DEMO] Outbound to {to}: {message_en[:60]}…")
+            return {"status": "demo_queued", "to": to}
+        from twilio.rest import Client
+        client = Client(self.sid, self.token)
+        call = client.calls.create(
+            to=to, from_=self.phone, twiml=twiml
+        )
+        return {"sid": call.sid, "status": call.status}
+
+    @staticmethod
+    def mulaw_to_pcm(mulaw_bytes: bytes) -> bytes:
+        """
+        Convert 8kHz G.711 mu-law to 16-bit PCM at 16kHz for Whisper.
+
+        Implemented in numpy rather than the stdlib `audioop` module, which
+        was removed in Python 3.13. Keeping this dependency-free means the
+        telephony path works on any modern image.
+        """
+        import numpy as np
+
+        u = np.frombuffer(mulaw_bytes, dtype=np.uint8).astype(np.int32)
+        u = ~u & 0xFF                                  # mu-law is stored inverted
+        sign     = u & 0x80
+        exponent = (u >> 4) & 0x07
+        mantissa = u & 0x0F
+        # ITU-T G.711: t = ((mantissa << 3) + BIAS) << exponent, BIAS = 0x84
+        t = ((mantissa << 3) + 0x84) << exponent
+        pcm8k = np.where(sign != 0, 0x84 - t, t - 0x84).astype(np.int16)
+
+        # 8kHz → 16kHz (linear interpolation; the band-limited content of a
+        # phone call makes a higher-order filter unnecessary here)
+        if len(pcm8k) == 0:
+            return b""
+        x  = np.arange(len(pcm8k))
+        xi = np.arange(len(pcm8k) * 2) / 2.0      # exact 2x: 0, 0.5, 1, 1.5, …
+        pcm16k = np.interp(xi, x, pcm8k).astype(np.int16)
+        return pcm16k.tobytes()
+
+
+class BandwidthAdapter:
+    """Bandwidth BXML + WebSocket audio streaming (stub)."""
+
+    def __init__(self):
+        self.account_id   = os.getenv("BANDWIDTH_ACCOUNT_ID",   "DEMO")
+        self.api_token    = os.getenv("BANDWIDTH_API_TOKEN",     "DEMO")
+        self._demo        = self.account_id == "DEMO"
+
+    def incoming_call_bxml(self, websocket_url: str) -> str:
+        return f"""<?xml version="1.0" encoding="UTF-8"?>
+<Response>
+  <SpeakSentence locale="ha-NG">Sannu da zuwa PlotWeaver.</SpeakSentence>
+  <StartStream url="{websocket_url}" streamEventUrl="{websocket_url}/events"/>
+</Response>"""
+
+    def send_tts(self, call_id: str, text: str, locale: str = "ha-NG") -> dict:
+        if self._demo:
+            logger.info(f"[DEMO] Bandwidth TTS on call {call_id}: {text[:60]}…")
+            return {"status": "demo"}
+        # Production: PATCH /calls/{callId} with BXML
+        raise NotImplementedError
+
+
+class SIPRouter:
+    """
+    Routes a call to the correct adapter based on SIP_PROVIDER env var.
+    Also manages human-agent transfer via SIP REFER.
+    """
+
+    PROVIDERS = {"twilio": TwilioAdapter, "bandwidth": BandwidthAdapter}
+
+    def __init__(self):
+        provider = os.getenv("SIP_PROVIDER", "demo").lower()
+        if provider in self.PROVIDERS:
+            self.adapter = self.PROVIDERS[provider]()
+        else:
+            self.adapter = TwilioAdapter()   # demo mode
+        logger.info(f"SIP provider: {provider}")
+
+    def transfer_to_human(self, call_sid: str,
+                           agent_extension: str = "+0000000001") -> dict:
+        """
+        REFER / warm transfer to human agent queue.
+        In demo mode just logs.
+        """
+        logger.info(f"[SIP] Transferring {call_sid} → agent {agent_extension}")
+        if isinstance(self.adapter, TwilioAdapter) and not self.adapter._demo:
+            from twilio.rest import Client
+            client = Client(self.adapter.sid, self.adapter.token)
+            call = client.calls(call_sid).update(
+                url=f"http://twimlets.com/forward?PhoneNumber={agent_extension}"
+            )
+            return {"status": call.status, "agent": agent_extension}
+        return {"status": "demo_transfer", "agent": agent_extension}

streaming_asr.py

@@ -0,0 +1,349 @@
+"""
+Streaming ASR with VAD Endpointing
+====================================
+Turns push-to-talk into a conversation.
+
+The caller speaks; audio arrives in small chunks. This module decides — with
+no button press — when an utterance has STARTED and when it has ENDED, emits
+live partial transcripts while the caller is still talking, and detects
+barge-in so the agent stops talking when interrupted.
+
+Endpointing state machine
+─────────────────────────
+
+     ┌────────┐  speech ≥ min_speech_ms   ┌──────────┐
+     │  IDLE  │ ────────────────────────► │ SPEAKING │
+     └────────┘                           └──────────┘
+         ▲                                      │ silence detected
+         │                                      ▼
+         │                              ┌────────────────┐
+         │  silence ≥ endpoint_ms       │ TRAILING_SIL   │
+         └──────── (emit FINAL) ────────│  (may resume)  │
+                                        └────────────────┘
+
+Key behaviours:
+  - PREROLL      : a ring buffer holds ~300ms of audio from BEFORE the speech
+                   trigger fires, so the first phoneme is never clipped. This
+                   is the single most common cause of "it dropped my first
+                   word" in naive VAD implementations.
+  - HANGOVER     : brief silences inside speech (natural pauses between words,
+                   the gap before a plosive) do not end the turn. Only
+                   `endpoint_silence_ms` of continuous silence does.
+  - PARTIALS     : every `partial_interval_ms`, the audio so far is decoded
+                   with a SMALL Whisper model for a live on-screen transcript.
+                   The FINAL decode uses large-v3 for accuracy.
+  - BARGE-IN     : while `agent_speaking` is set, sustained caller speech
+                   raises a barge-in event so playback can be cut.
+  - MAX DURATION : a hard cap force-endpoints a caller who never pauses.
+
+Whisper hallucinates confidently on silence ("Thank you.", "Subtitles by…"),
+so utterances shorter than `min_speech_ms` of actual speech are discarded
+without ever reaching the model.
+"""
+
+import time
+import logging
+import numpy as np
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Optional, Callable
+
+from vad import load_vad, FRAME_SAMPLES, FRAME_MS, SAMPLE_RATE
+
+logger = logging.getLogger(__name__)
+
+
+class State(Enum):
+    IDLE           = "idle"
+    SPEAKING       = "speaking"
+    TRAILING_SIL   = "trailing_silence"
+
+
+@dataclass
+class EndpointConfig:
+    speech_threshold:     float = 0.55    # VAD prob above this = speech frame
+    silence_threshold:    float = 0.35    # below this = silence (hysteresis gap
+                                          # prevents flapping at the boundary)
+    min_speech_ms:        int   = 250     # ignore coughs, door slams, clicks
+    endpoint_silence_ms:  int   = 700     # silence that ends a turn
+    preroll_ms:           int   = 300     # audio kept from before speech onset
+    max_utterance_ms:     int   = 20_000  # hard cap
+    partial_interval_ms:  int   = 900     # how often to emit a live partial
+    bargein_speech_ms:    int   = 220     # speech needed to interrupt the agent
+
+
+@dataclass
+class UtteranceEvent:
+    kind:        str                       # 'partial' | 'final' | 'bargein' |
+                                           # 'speech_start' | 'discarded'
+    text:        str   = ""
+    audio:       Optional[np.ndarray] = None
+    duration_ms: float = 0.0
+    speech_ms:   float = 0.0
+    latency_ms:  float = 0.0
+
+
+class StreamingASR:
+    """
+    Feed audio with `accept_audio()`; consume the returned list of events.
+
+    Usage:
+        sasr = StreamingASR(transcribe_fn=pipeline.transcribe)
+        for chunk in mic_stream:
+            for ev in sasr.accept_audio(chunk, sr):
+                if ev.kind == "partial": show(ev.text)
+                if ev.kind == "final":   handle(ev.text)
+    """
+
+    def __init__(self,
+                 transcribe_fn: Callable[[np.ndarray, int], str],
+                 partial_transcribe_fn: Optional[Callable] = None,
+                 config: Optional[EndpointConfig] = None,
+                 vad_backend: str = "auto",
+                 emit_partials: bool = True):
+        self.cfg = config or EndpointConfig()
+        self.vad = load_vad(vad_backend)
+        self.transcribe_fn = transcribe_fn
+        # Partials can use a smaller/faster model; falls back to the main one
+        self.partial_transcribe_fn = partial_transcribe_fn or transcribe_fn
+        self.emit_partials = emit_partials
+
+        self.agent_speaking = False        # set True while TTS plays (barge-in)
+        self._preroll_frames = max(1, int(self.cfg.preroll_ms / FRAME_MS))
+        self.reset()
+
+    # ── Lifecycle ─────────────────────────────────────────────────────────────
+
+    def reset(self):
+        self.state           = State.IDLE
+        self._buffer         = np.zeros(0, dtype=np.float32)   # leftover samples
+        self._preroll        = []                              # ring of frames
+        self._utterance      = []                              # frames of turn
+        self._speech_ms      = 0.0
+        self._silence_ms     = 0.0
+        self._utterance_ms   = 0.0
+        self._bargein_ms     = 0.0
+        self._last_partial_ms = 0.0
+        self._partial_text   = ""
+        self.vad.reset()
+
+    # ── Main entry ────────────────────────────────────────────────────────────
+
+    def accept_audio(self, audio: np.ndarray,
+                     sample_rate: int = SAMPLE_RATE) -> list[UtteranceEvent]:
+        """
+        audio: mono float32 in [-1,1] (int16 is auto-converted) of any length.
+        Returns zero or more events produced by this chunk.
+        """
+        events: list[UtteranceEvent] = []
+        audio = _to_float_mono(audio)
+        if sample_rate != SAMPLE_RATE:
+            audio = _resample(audio, sample_rate, SAMPLE_RATE)
+
+        self._buffer = np.concatenate([self._buffer, audio])
+
+        # Consume whole frames only; remainder stays buffered for next chunk
+        while len(self._buffer) >= FRAME_SAMPLES:
+            frame = self._buffer[:FRAME_SAMPLES]
+            self._buffer = self._buffer[FRAME_SAMPLES:]
+            ev = self._process_frame(frame)
+            events.extend(ev)
+
+        return events
+
+    def flush(self) -> list[UtteranceEvent]:
+        """Force-endpoint whatever is buffered (e.g. caller hung up)."""
+        if self.state in (State.SPEAKING, State.TRAILING_SIL):
+            return self._finalize()
+        return []
+
+    # ── Frame processing ──────────────────────────────────────────────────────
+
+    def _process_frame(self, frame: np.ndarray) -> list[UtteranceEvent]:
+        events = []
+        prob = self.vad.speech_prob(frame)
+        is_speech  = prob >= self.cfg.speech_threshold
+        is_silence = prob <= self.cfg.silence_threshold
+
+        # ── Barge-in: caller talks over the agent ────────────────────────────
+        if self.agent_speaking:
+            self._bargein_ms = self._bargein_ms + FRAME_MS if is_speech else 0.0
+            if self._bargein_ms >= self.cfg.bargein_speech_ms:
+                self._bargein_ms = 0.0
+                self.agent_speaking = False
+                events.append(UtteranceEvent(kind="bargein"))
+                # fall through — this frame also starts the new utterance
+
+        # ── Preroll ring (only meaningful while IDLE) ────────────────────────
+        if self.state is State.IDLE:
+            self._preroll.append(frame)
+            if len(self._preroll) > self._preroll_frames:
+                self._preroll.pop(0)
+
+        # ── State machine ────────────────────────────────────────────────────
+        if self.state is State.IDLE:
+            if is_speech:
+                self._speech_ms += FRAME_MS
+                if self._speech_ms >= self.cfg.min_speech_ms:
+                    # Commit: open the utterance with the preroll in front
+                    self._utterance = list(self._preroll)
+                    self._utterance_ms = len(self._utterance) * FRAME_MS
+                    self._preroll = []
+                    self._silence_ms = 0.0
+                    self._last_partial_ms = 0.0
+                    self.state = State.SPEAKING
+                    events.append(UtteranceEvent(kind="speech_start"))
+            else:
+                self._speech_ms = 0.0
+            return events
+
+        # SPEAKING or TRAILING_SIL — always accumulate audio
+        self._utterance.append(frame)
+        self._utterance_ms += FRAME_MS
+
+        if self.state is State.SPEAKING:
+            if is_silence:
+                self.state = State.TRAILING_SIL
+                self._silence_ms = FRAME_MS
+            else:
+                if is_speech:
+                    self._speech_ms += FRAME_MS
+                self._silence_ms = 0.0
+
+        elif self.state is State.TRAILING_SIL:
+            if is_speech:
+                # Natural pause, not an endpoint — resume
+                self.state = State.SPEAKING
+                self._speech_ms += FRAME_MS
+                self._silence_ms = 0.0
+            else:
+                self._silence_ms += FRAME_MS
+                if self._silence_ms >= self.cfg.endpoint_silence_ms:
+                    return events + self._finalize()
+
+        # Hard cap on a caller who never pauses
+        if self._utterance_ms >= self.cfg.max_utterance_ms:
+            logger.info("Max utterance length reached — force endpoint.")
+            return events + self._finalize()
+
+        # ── Live partial transcript ──────────────────────────────────────────
+        if (self.emit_partials
+                and self.state is State.SPEAKING
+                and self._utterance_ms - self._last_partial_ms
+                    >= self.cfg.partial_interval_ms):
+            self._last_partial_ms = self._utterance_ms
+            ev = self._emit_partial()
+            if ev:
+                events.append(ev)
+
+        return events
+
+    # ── Emission ──────────────────────────────────────────────────────────────
+
+    def _emit_partial(self) -> Optional[UtteranceEvent]:
+        audio = np.concatenate(self._utterance)
+        t0 = time.perf_counter()
+        try:
+            text = self.partial_transcribe_fn(audio, SAMPLE_RATE)
+        except Exception as e:
+            logger.warning(f"Partial decode failed: {e}")
+            return None
+        text = (text or "").strip()
+        if not text or text == self._partial_text:
+            return None
+        self._partial_text = text
+        return UtteranceEvent(
+            kind="partial", text=text,
+            duration_ms=self._utterance_ms, speech_ms=self._speech_ms,
+            latency_ms=(time.perf_counter() - t0) * 1000)
+
+    def _finalize(self) -> list[UtteranceEvent]:
+        audio      = np.concatenate(self._utterance) if self._utterance else np.zeros(0)
+        speech_ms  = self._speech_ms
+        total_ms   = self._utterance_ms
+
+        # Reset BEFORE decoding so late-arriving audio starts a clean turn
+        self._utterance   = []
+        self._preroll     = []
+        self._speech_ms   = 0.0
+        self._silence_ms  = 0.0
+        self._utterance_ms = 0.0
+        self._partial_text = ""
+        self.state = State.IDLE
+        self.vad.reset()
+
+        # Guard: never send near-silence to Whisper (hallucination source)
+        if speech_ms < self.cfg.min_speech_ms or len(audio) < FRAME_SAMPLES * 4:
+            logger.info(f"Discarded short utterance ({speech_ms:.0f}ms speech).")
+            return [UtteranceEvent(kind="discarded", speech_ms=speech_ms,
+                                    duration_ms=total_ms)]
+
+        t0 = time.perf_counter()
+        try:
+            text = self.transcribe_fn(audio, SAMPLE_RATE)
+        except Exception as e:
+            logger.error(f"Final decode failed: {e}")
+            return [UtteranceEvent(kind="discarded", speech_ms=speech_ms,
+                                    duration_ms=total_ms)]
+        latency = (time.perf_counter() - t0) * 1000
+
+        text = (text or "").strip()
+        if not text or _is_hallucination(text):
+            logger.info(f"Discarded empty/hallucinated final: {text!r}")
+            return [UtteranceEvent(kind="discarded", speech_ms=speech_ms,
+                                    duration_ms=total_ms)]
+
+        logger.info(f"FINAL ({latency:.0f}ms, {speech_ms:.0f}ms speech): {text}")
+        return [UtteranceEvent(kind="final", text=text, audio=audio,
+                                duration_ms=total_ms, speech_ms=speech_ms,
+                                latency_ms=latency)]
+
+
+# ── Hallucination filter ──────────────────────────────────────────────────────
+
+_HALLUCINATIONS = {
+    "thank you.", "thanks for watching!", "thank you for watching.",
+    "you", ".", "...", "subtitles by the amara.org community",
+    "please subscribe.", "bye.", "amara.org", "sous-titrage",
+    "merci d'avoir regardé cette vidéo!", "à suivre",
+}
+
+
+def _is_hallucination(text: str) -> bool:
+    t = text.strip().lower()
+    if t in _HALLUCINATIONS:
+        return True
+    # A "sentence" of only punctuation / music tags
+    if all(c in " .,!?-–—♪[]()" for c in t):
+        return True
+    return False
+
+
+# ── Audio helpers ─────────────────────────────────────────────────────────────
+
+def _to_float_mono(audio: np.ndarray) -> np.ndarray:
+    audio = np.asarray(audio)
+    if audio.ndim > 1:
+        audio = audio.mean(axis=1)
+    if audio.dtype == np.int16:
+        audio = audio.astype(np.float32) / 32768.0
+    elif audio.dtype == np.int32:
+        audio = audio.astype(np.float32) / 2147483648.0
+    else:
+        audio = audio.astype(np.float32)
+    return audio
+
+
+def _resample(audio: np.ndarray, src: int, dst: int) -> np.ndarray:
+    if src == dst:
+        return audio
+    try:
+        import scipy.signal as ss
+        n = int(round(len(audio) * dst / src))
+        return ss.resample(audio, n).astype(np.float32)
+    except Exception:
+        # Linear interpolation fallback
+        n = int(round(len(audio) * dst / src))
+        xp = np.linspace(0, 1, len(audio), endpoint=False)
+        x  = np.linspace(0, 1, n, endpoint=False)
+        return np.interp(x, xp, audio).astype(np.float32)

test_streaming.py

@@ -0,0 +1,217 @@
+"""
+Streaming ASR / VAD endpointing tests.
+
+Uses synthetic audio and a mock transcriber so it runs with no models and
+no network. Verifies the state machine, not Whisper.
+"""
+
+import sys
+import logging
+import numpy as np
+
+logging.basicConfig(level=logging.WARNING)
+
+from vad import load_vad, FRAME_MS, SAMPLE_RATE
+from streaming_asr import StreamingASR, EndpointConfig, State, _is_hallucination
+
+PASS, FAIL = "\033[92mPASS\033[0m", "\033[91mFAIL\033[0m"
+results = []
+
+
+def check(name, cond, detail=""):
+    results.append((name, cond))
+    print(f"  [{PASS if cond else FAIL}] {name}" + (f"  — {detail}" if detail else ""))
+
+
+# ── Synthetic audio ───────────────────────────────────────────────────────────
+
+def silence(ms, noise=0.0005):
+    n = int(SAMPLE_RATE * ms / 1000)
+    return (np.random.randn(n) * noise).astype(np.float32)
+
+
+def speech(ms, amp=0.25):
+    """Voice-like: 120Hz glottal pulse + formants + amplitude modulation."""
+    n = int(SAMPLE_RATE * ms / 1000)
+    t = np.arange(n) / SAMPLE_RATE
+    sig = (np.sin(2 * np.pi * 120 * t)
+           + 0.5 * np.sin(2 * np.pi * 700 * t)
+           + 0.3 * np.sin(2 * np.pi * 1220 * t))
+    envelope = 0.6 + 0.4 * np.sin(2 * np.pi * 4 * t)      # syllable rate
+    sig = sig * envelope
+    sig += np.random.randn(n) * 0.01
+    return (sig / np.abs(sig).max() * amp).astype(np.float32)
+
+
+def feed(sasr, audio, chunk_ms=100):
+    """Feed audio in realistic small chunks, collecting all events."""
+    events = []
+    step = int(SAMPLE_RATE * chunk_ms / 1000)
+    for i in range(0, len(audio), step):
+        events.extend(sasr.accept_audio(audio[i:i + step], SAMPLE_RATE))
+    return events
+
+
+# Mock transcriber records exactly what audio it was handed
+DECODE_LOG = []
+
+
+def mock_transcribe(audio, sr):
+    DECODE_LOG.append(len(audio) / sr * 1000)      # duration in ms
+    return f"transcript_of_{len(audio)/sr:.2f}s"
+
+
+def new_asr(**kw):
+    DECODE_LOG.clear()
+    cfg = EndpointConfig(**kw) if kw else EndpointConfig()
+    return StreamingASR(transcribe_fn=mock_transcribe, config=cfg,
+                        vad_backend="energy")
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+
+print("\n══ VAD discriminates speech from silence ══")
+vad = load_vad("energy")
+for _ in range(12):                                   # let the floor settle
+    vad.speech_prob(silence(FRAME_MS)[:512])
+sil_probs = [vad.speech_prob(silence(FRAME_MS)[:512]) for _ in range(20)]
+sp = speech(1000)
+sp_probs = [vad.speech_prob(sp[i:i+512]) for i in range(0, 512*20, 512)]
+check("silence scores low", np.mean(sil_probs) < 0.3,
+      f"mean={np.mean(sil_probs):.2f}")
+check("speech scores high", np.mean(sp_probs) > 0.6,
+      f"mean={np.mean(sp_probs):.2f}")
+
+print("\n══ Basic endpointing: speech → silence → ONE final ══")
+sasr = new_asr()
+audio = np.concatenate([silence(500), speech(1200), silence(1200)])
+events = feed(sasr, audio)
+finals = [e for e in events if e.kind == "final"]
+check("exactly one final emitted", len(finals) == 1, f"got {len(finals)}")
+check("speech_start fired", any(e.kind == "speech_start" for e in events))
+check("returns to IDLE after endpoint", sasr.state is State.IDLE)
+
+print("\n══ Preroll: first phoneme is not clipped ══")
+sasr = new_asr(preroll_ms=300)
+audio = np.concatenate([silence(400), speech(1000), silence(1200)])
+feed(sasr, audio)
+check("decoded audio longer than speech alone (preroll included)",
+      DECODE_LOG and DECODE_LOG[-1] > 1000,
+      f"decoded {DECODE_LOG[-1]:.0f}ms for 1000ms of speech")
+
+print("\n══ Hangover: a natural pause does NOT end the turn ══")
+sasr = new_asr(endpoint_silence_ms=700)
+# 400ms pause is shorter than the 700ms endpoint → must stay ONE utterance
+audio = np.concatenate([silence(400), speech(700), silence(400),
+                        speech(700), silence(1200)])
+events = feed(sasr, audio)
+finals = [e for e in events if e.kind == "final"]
+check("short internal pause does not split the turn", len(finals) == 1,
+      f"got {len(finals)} finals")
+
+print("\n══ Long pause DOES split into two turns ══")
+sasr = new_asr(endpoint_silence_ms=700)
+audio = np.concatenate([silence(400), speech(700), silence(1300),
+                        speech(700), silence(1300)])
+events = feed(sasr, audio)
+finals = [e for e in events if e.kind == "final"]
+check("long pause produces two separate turns", len(finals) == 2,
+      f"got {len(finals)} finals")
+
+print("\n══ Silence is never sent to Whisper (hallucination guard) ══")
+sasr = new_asr()
+feed(sasr, silence(4000))
+check("no decode call on pure silence", len(DECODE_LOG) == 0,
+      f"{len(DECODE_LOG)} decode calls")
+check("no final event on pure silence",
+      not any(e.kind == "final" for e in feed(sasr, silence(2000))))
+
+print("\n══ Cough / click is discarded, not transcribed ══")
+sasr = new_asr(min_speech_ms=250)
+audio = np.concatenate([silence(400), speech(90), silence(1500)])
+events = feed(sasr, audio)
+check("sub-threshold blip produces no final",
+      not any(e.kind == "final" for e in events))
+check("no decode call for the blip", len(DECODE_LOG) == 0,
+      f"{len(DECODE_LOG)} calls")
+
+print("\n══ Live partials during a long utterance ══")
+sasr = new_asr(partial_interval_ms=600)
+audio = np.concatenate([silence(400), speech(3500), silence(1200)])
+events = feed(sasr, audio)
+partials = [e for e in events if e.kind == "partial"]
+check("partials emitted while speaking", len(partials) >= 2,
+      f"got {len(partials)} partials")
+check("final still emitted after partials",
+      any(e.kind == "final" for e in events))
+check("partials precede the final",
+      events.index(next(e for e in events if e.kind == "final"))
+      > events.index(partials[0]) if partials else False)
+
+print("\n══ Barge-in: caller interrupts the agent ══")
+sasr = new_asr(bargein_speech_ms=220)
+sasr.agent_speaking = True
+events = feed(sasr, np.concatenate([silence(300), speech(900)]))
+check("barge-in event raised", any(e.kind == "bargein" for e in events))
+check("agent_speaking cleared", sasr.agent_speaking is False)
+
+print("\n══ Barge-in does NOT fire on background noise ══")
+sasr = new_asr(bargein_speech_ms=220)
+sasr.agent_speaking = True
+events = feed(sasr, silence(2000, noise=0.002))
+check("no barge-in on quiet background",
+      not any(e.kind == "bargein" for e in events))
+
+print("\n══ Max-duration force endpoint ══")
+sasr = new_asr(max_utterance_ms=2000)
+events = feed(sasr, np.concatenate([silence(300), speech(6000)]))
+check("non-stop speaker is force-endpointed",
+      any(e.kind == "final" for e in events))
+
+print("\n══ flush() finalizes a turn in progress (caller hung up) ══")
+sasr = new_asr()
+feed(sasr, np.concatenate([silence(400), speech(1000)]))
+events = sasr.flush()
+check("flush emits the pending final",
+      any(e.kind == "final" for e in events))
+
+print("\n══ Chunk-size independence (frame alignment) ══")
+for chunk_ms in (20, 33, 100, 250):
+    sasr = new_asr()
+    audio = np.concatenate([silence(400), speech(1200), silence(1200)])
+    events = feed(sasr, audio, chunk_ms=chunk_ms)
+    n = len([e for e in events if e.kind == "final"])
+    check(f"chunk={chunk_ms}ms → 1 final", n == 1, f"got {n}")
+
+print("\n══ Whisper hallucination filter ══")
+check("'Thank you.' filtered", _is_hallucination("Thank you."))
+check("'Subtitles by the Amara.org community' filtered",
+      _is_hallucination("Subtitles by the Amara.org community"))
+check("'♪' filtered", _is_hallucination("♪"))
+check("real Hausa text NOT filtered",
+      not _is_hallucination("Ina son duba asusuna"))
+
+print("\n══ int16 input auto-converted ══")
+sasr = new_asr()
+audio_f = np.concatenate([silence(400), speech(1200), silence(1200)])
+audio_i16 = (audio_f * 32767).astype(np.int16)
+events = feed(sasr, audio_i16)
+check("int16 stream produces a final",
+      any(e.kind == "final" for e in events))
+
+print("\n══ Resampling from 8kHz (telephony) ══")
+sasr = new_asr()
+audio = np.concatenate([silence(400), speech(1200), silence(1200)])
+audio_8k = audio[::2]                                  # crude 8kHz
+events = []
+step = 800
+for i in range(0, len(audio_8k), step):
+    events.extend(sasr.accept_audio(audio_8k[i:i + step], 8000))
+check("8kHz telephony audio endpoints correctly",
+      any(e.kind == "final" for e in events))
+
+# ── Summary ───────────────────────────────────────────────────────────────────
+passed = sum(1 for _, ok in results if ok)
+total = len(results)
+print(f"\n{'='*62}\n  {passed}/{total} checks passed\n{'='*62}")
+sys.exit(0 if passed == total else 1)

vad.py

@@ -0,0 +1,173 @@
+"""
+Voice Activity Detection
+=========================
+Frame-level speech/silence classification, used for:
+
+  - Endpointing     : knowing when the caller has FINISHED speaking
+                      (so we don't wait for a push-to-talk button)
+  - Preroll         : keeping ~300ms of audio BEFORE speech onset so the
+                      first phoneme is never clipped
+  - Barge-in        : detecting that the caller started speaking while the
+                      agent's TTS is still playing, so we can stop it
+  - Silence culling : never sending silence to Whisper (Whisper hallucinates
+                      badly on silence — "Thank you." / "Subtitles by…")
+
+Two backends:
+  1. SileroVAD  — torch.hub snakers4/silero-vad. Small (~1.8MB), fast on CPU,
+                  robust to background noise. Requires EXACTLY 512 samples
+                  per frame at 16kHz.
+  2. EnergyVAD  — adaptive-threshold RMS + zero-crossing rate. No download,
+                  no torch.hub dependency. Degraded in noise but never fails.
+
+Both expose the same interface:
+    vad.speech_prob(frame: np.ndarray) -> float in [0, 1]
+    vad.reset()
+"""
+
+import logging
+import numpy as np
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+SAMPLE_RATE   = 16_000
+FRAME_SAMPLES = 512                       # Silero requirement at 16kHz
+FRAME_MS      = FRAME_SAMPLES / SAMPLE_RATE * 1000     # 32.0 ms
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+#  Silero
+# ──────────────────────────────────────────────────────────────────────────────
+
+class SileroVAD:
+
+    def __init__(self):
+        import torch
+        self.torch = torch
+        logger.info("Loading Silero VAD …")
+        self.model, _ = torch.hub.load(
+            repo_or_dir="snakers4/silero-vad",
+            model="silero_vad",
+            force_reload=False,
+            onnx=False,
+            trust_repo=True,
+        )
+        self.model.eval()
+        self.name = "silero"
+
+    def speech_prob(self, frame: np.ndarray) -> float:
+        """frame: float32 mono, exactly FRAME_SAMPLES long, range [-1, 1]."""
+        if len(frame) != FRAME_SAMPLES:
+            frame = _fit(frame, FRAME_SAMPLES)
+        with self.torch.no_grad():
+            t = self.torch.from_numpy(frame.astype(np.float32))
+            return float(self.model(t, SAMPLE_RATE).item())
+
+    def reset(self):
+        if hasattr(self.model, "reset_states"):
+            self.model.reset_states()
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+#  Energy fallback
+# ──────────────────────────────────────────────────────────────────────────────
+
+class EnergyVAD:
+    """
+    Adaptive-noise-floor RMS gate with a zero-crossing-rate sanity check.
+
+    The noise floor tracks the quietest recent frames, so it adapts to a
+    caller on a noisy street or a quiet office without reconfiguration.
+    """
+
+    SEED_FRAMES  = 8         # frames used to establish the initial floor
+    ABS_GATE_LO  = 0.015     # rms clearly above any plausible room noise
+    ABS_GATE_HI  = 0.060     # rms unambiguously speech at normal mic gain
+
+    def __init__(self, sensitivity: float = 3.0):
+        self.sensitivity = sensitivity      # how many x above floor = speech
+        self.reset()
+        self.name = "energy"
+
+    def reset(self):
+        self._floor      = None
+        self._floor_init = False
+        self._frames     = 0
+
+    def speech_prob(self, frame: np.ndarray) -> float:
+        frame = frame.astype(np.float32)
+        rms = float(np.sqrt(np.mean(frame ** 2)) + 1e-9)
+
+        # Zero-crossing rate — speech sits in a middling band; pure hiss is high,
+        # DC/rumble is very low.
+        zcr = float(np.mean(np.abs(np.diff(np.sign(frame)))) / 2.0)
+
+        self._frames += 1
+
+        # ── Seeding ──────────────────────────────────────────────────────────
+        # Seed from the MINIMUM, never a running average. If the caller starts
+        # talking part-way through the seed window (or is already talking when
+        # the stream opens), an averaged seed is dragged up to speech level and
+        # the detector goes deaf for the rest of the session.
+        if not self._floor_init:
+            self._floor = rms if self._floor is None else min(self._floor, rms)
+            if self._frames >= self.SEED_FRAMES:
+                self._floor_init = True
+            # Still answer using the absolute gate, so speech during the seed
+            # window is not silently swallowed.
+            return self._absolute_prob(rms) * self._zcr_penalty(zcr)
+
+        # ── Asymmetric adaptation ────────────────────────────────────────────
+        # Down fast (room got quiet → recover in ~7 frames).
+        # Up very slowly (~60s time constant), so a long unbroken utterance
+        # cannot drag the floor up to its own level and mute itself.
+        self._floor = max(
+            0.85 * self._floor + 0.15 * rms if rms < self._floor
+            else 0.9995 * self._floor + 0.0005 * rms,
+            1e-6,
+        )
+
+        ratio    = rms / (self._floor * self.sensitivity)
+        rel_prob = float(np.clip((ratio - 0.6) / 1.4, 0.0, 1.0))
+
+        # Absolute gate is a floor on confidence, not a cap: if the signal is
+        # loud in absolute terms it is speech regardless of what the adaptive
+        # estimate believes.
+        prob = max(rel_prob, self._absolute_prob(rms))
+        return prob * self._zcr_penalty(zcr)
+
+    # ── Helpers ───────────────────────────────────────────────────────────────
+
+    def _absolute_prob(self, rms: float) -> float:
+        return float(np.clip(
+            (rms - self.ABS_GATE_LO) / (self.ABS_GATE_HI - self.ABS_GATE_LO),
+            0.0, 1.0))
+
+    @staticmethod
+    def _zcr_penalty(zcr: float) -> float:
+        """Penalise implausible zero-crossing rates (hiss, rumble, DC)."""
+        return 0.4 if (zcr > 0.35 or zcr < 0.005) else 1.0
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+
+def _fit(frame: np.ndarray, n: int) -> np.ndarray:
+    """Pad or truncate a frame to exactly n samples."""
+    if len(frame) >= n:
+        return frame[:n]
+    return np.pad(frame, (0, n - len(frame)))
+
+
+def load_vad(prefer: str = "auto"):
+    """
+    prefer: 'auto' | 'silero' | 'energy'
+    Never raises — falls back to EnergyVAD if Silero cannot be loaded
+    (no network in the Space, torch.hub blocked, etc).
+    """
+    if prefer == "energy":
+        return EnergyVAD()
+    try:
+        return SileroVAD()
+    except Exception as e:
+        logger.warning(f"Silero VAD unavailable ({e}); using energy VAD.")
+        return EnergyVAD()