deploy to Hugging Face Spaces

.gitignore

@@ -0,0 +1,55 @@
+# Python
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+*.pdb
+*.egg
+*.egg-info/
+dist/
+build/
+*.whl
+*.so
+*.dylib
+
+# Virtual environments
+.venv/
+venv/
+env/
+ENV/
+
+# Testing / Coverage
+.pytest_cache/
+.coverage
+htmlcov/
+*.cover
+coverage.xml
+nosetests.xml
+junit*.xml
+
+# IDEs & editors
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db
+desktop.ini
+
+# Model cache / Hugging Face cache
+.cache/
+models/
+*.bin
+*.safetensors
+*.gguf
+
+# Logs
+*.log
+logs/
+
+# Misc
+*.tmp
+*.bak

README.md

@@ -0,0 +1,41 @@
+---
+title: SmartSight AI
+emoji: 🌐
+colorFrom: blue
+colorTo: indigo
+sdk: gradio
+sdk_version: 4.44.1
+app_file: app.py
+pinned: false
+license: apache-2.0
+python_version: "3.10"
+---
+
+# 🌐 SmartSight AI — Hugging Face Space Deployment
+
+Bản cập nhật cấu hình cho Hugging Face Spaces chạy mô phỏng trợ lý mô tả hình ảnh hỗ trợ tiếp cận (Accessibility System) bằng giọng nói Tiếng Việt dành cho người khiếm thị.///
+
+## 🚀 Cấu hình Hugging Face Spaces
+
+Ứng dụng này được thiết kế để triển khai trực tiếp trên Hugging Face Spaces bằng các cài đặt sau:
+
+- **SDK:** Gradio
+- **Phiên bản SDK:** 4.44.1
+- **Phiên bản Python:** 3.10
+- **Tệp khởi chạy chính:** `app.py` (tệp wrapper ở thư mục gốc)
+
+---
+
+## 🛠️ Hướng dẫn Triển khai trên Hugging Face Space
+
+1. Tạo một Space mới trên **Hugging Face**.
+2. Chọn SDK là **Gradio**.
+3. Kết nối Space của bạn với kho chứa mã nguồn GitHub này.
+4. Thiết lập nhánh triển khai là `huggingface-deployment`.
+5. Hugging Face sẽ tự động cài đặt các thư viện trong `requirements.txt`, khởi chạy `app.py` và chạy ứng dụng.
+
+---
+
+## 📐 Kiến trúc nhánh Hugging Face Deployment
+
+Sự khác biệt chính của nhánh này là việc đưa tệp khởi chạy chính `app.py` ra thư mục gốc dưới dạng wrapper để tương thích hoàn toàn với cơ chế phát hiện tự động của Hugging Face Spaces, đồng thời tích hợp Metadata YAML ở phần đầu của tệp `README.md`.

app.py

@@ -0,0 +1,13 @@
+from src.app import demo
+
+if __name__ == "__main__":
+    # Model loads lazily on first "Run" click via ModelRegistry singleton.
+    # This lets the UI become available in ~5-10s instead of waiting ~40s
+    # to download the 3.85GB Moondream2 model before Gradio even starts.
+    demo.queue().launch(
+        # Required for Hugging Face Spaces containers:
+        server_name="0.0.0.0",   # bind all interfaces, not just localhost
+        server_port=7860,         # standard HF Spaces port
+        show_api=False,           # disable API schema generation (prevents TypeError
+                                  # in gradio_client/utils.py get_type() on gradio 4.40.x)
+    )

packages.txt

@@ -0,0 +1,4 @@
+libvips-dev
+espeak-ng
+libespeak1
+ffmpeg

requirements-local.txt

@@ -0,0 +1,25 @@
+gradio==4.44.1
+transformers==4.40.0
+torch==2.2.2
+numpy==1.26.4
+pillow==10.3.0
+deep-translator==1.11.4
+gTTS==2.5.1
+pyttsx3==2.90
+psutil==5.9.8
+sentencepiece==0.2.0
+sacremoses==0.1.1
+pytest==8.2.0
+huggingface_hub==0.23.0
+pyvips==2.2.3
+pyvips-binary==8.18.2
+einops==0.8.0
+timm==0.9.16
+accelerate==0.30.0
+# Pin pydantic to avoid gradio_client JSON schema TypeError (bool-not-iterable bug)
+pydantic==2.10.6
+# Pin jinja2 to avoid unhashable type: 'dict' crash in Gradio 4.x template cache
+jinja2==3.1.4
+# Pin fastapi and starlette to avoid unhashable type: 'dict' crash in TemplateResponse
+fastapi==0.112.4
+starlette==0.38.2

requirements.txt

@@ -0,0 +1,25 @@
+gradio==4.44.1
+transformers==4.40.0
+torch==2.2.2
+numpy==1.26.4
+pillow==10.3.0
+deep-translator==1.11.4
+gTTS==2.5.1
+pyttsx3==2.90
+psutil==5.9.8
+sentencepiece==0.2.0
+sacremoses==0.1.1
+pytest==8.2.0
+huggingface_hub==0.23.0
+# Required by Moondream2 VLM model
+pyvips==2.2.3
+einops==0.8.0
+timm==0.9.16
+accelerate==0.30.0
+# Pin pydantic to avoid gradio_client JSON schema TypeError (bool-not-iterable bug)
+pydantic==2.10.6
+# Pin jinja2 to avoid unhashable type: 'dict' crash in Gradio 4.x template cache
+jinja2==3.1.4
+# Pin fastapi and starlette to avoid unhashable type: 'dict' crash in TemplateResponse
+fastapi==0.112.4
+starlette==0.38.2

scripts/test_cpu_patch.py

@@ -0,0 +1,39 @@
+"""
+Quick smoke-test for the CPU float32 monkey-patch.
+Run with: .venv/Scripts/python scripts/test_cpu_patch.py
+NOTE: Make sure app.py is NOT running at the same time (pagefile limit on Windows).
+"""
+import sys, os
+sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
+
+from PIL import Image
+import urllib.request, io
+
+print("=== SmartSight CPU Patch Smoke Test ===\n")
+
+# 1. Download a tiny test image (100x100 public domain)
+print("1. Loading test image...")
+url = "https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/320px-Cat03.jpg"
+try:
+    with urllib.request.urlopen(url, timeout=10) as r:
+        img_bytes = r.read()
+    image = Image.open(io.BytesIO(img_bytes)).convert("RGB")
+    print(f"   Image loaded: {image.size}, mode={image.mode}")
+except Exception as e:
+    # Fallback: create a solid-colour test image
+    print(f"   Download failed ({e}), using synthetic image.")
+    image = Image.new("RGB", (320, 240), color=(128, 64, 32))
+
+# 2. Load model via registry (triggers the patch)
+print("\n2. Loading Moondream2 (2B) via ModelRegistry...")
+from src.registry import ModelRegistry
+registry = ModelRegistry()
+model, processor = registry.get_vlm("Moondream2 (2B)")
+print("   Model loaded ✓")
+
+# 3. Run inference
+print("\n3. Running VLM inference on CPU...")
+from src.pipeline.vision_model import run_vlm_inference
+result = run_vlm_inference(image, "Moondream2 (2B)", model, processor)
+print(f"\n=== RESULT ===\n{result}\n")
+print("✅ Patch working correctly — no LayerNormKernelImpl error!")

src/__init__.py

@@ -0,0 +1 @@
+

src/app.py

@@ -0,0 +1,198 @@
+import os
+import sys
+import tempfile
+import time
+import uuid
+import logging
+import gradio as gr
+from PIL import Image
+from src.registry import ModelRegistry
+from src.pipeline.preprocess import preprocess_image
+from src.pipeline.vision_model import run_vlm_inference
+from src.pipeline.tts import TTSModule
+from src.utils.monitor import ExecutionMonitor
+
+# ─── Logging setup ────────────────────────────────────────────────────────────
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+    handlers=[logging.StreamHandler(sys.stdout)],
+    force=True,
+)
+logger = logging.getLogger("smartsight")
+
+logger.info("=" * 60)
+logger.info("SmartSight AI — startup")
+logger.info(f"  Python  : {sys.version.split()[0]}")
+logger.info(f"  Gradio  : {gr.__version__}")
+logger.info(f"  Platform: {sys.platform}")
+logger.info("=" * 60)
+
+# ─── Global singletons ────────────────────────────────────────────────────────
+logger.info("Initialising ModelRegistry and TTSModule singletons...")
+try:
+    registry = ModelRegistry()
+    tts_module = TTSModule()
+    logger.info("Singletons ready (models will lazy-load on first Run).")
+except Exception:
+    logger.critical("Failed to initialise singletons!", exc_info=True)
+    raise
+
+
+# ─── Helpers ──────────────────────────────────────────────────────────────────
+def get_performance_html(durations: dict) -> str:
+    total = sum(durations.values())
+    if total == 0:
+        return "<p>Chưa có dữ liệu hiệu năng.</p>"
+
+    html = "<div style='font-family: monospace; background: #1e1e1e; padding: 10px; border-radius: 5px; color: #fff;'>"
+    html += "<h4 style='margin-top:0; color:#58a6ff;'>Timing Breakdown:</h4>"
+    for stage, duration in durations.items():
+        pct = (duration / total) * 100 if total > 0 else 0
+        bar_count = int(pct / 5)
+        bar = "█" * bar_count + "░" * (20 - bar_count)
+        html += f"<div style='margin-bottom: 5px;'><b>{stage.capitalize()}:</b> {duration:.3f}s <span style='color: #8b949e;'>[{bar}]</span> {pct:.1f}%</div>"
+    html += "</div>"
+    return html
+
+
+# ─── Main pipeline ────────────────────────────────────────────────────────────
+def run_pipeline(image, vlm_version, translate_mode, tts_mode, custom_prompt):
+    logger.info(
+        "run_pipeline called | vlm=%s | translate=%s | tts=%s",
+        vlm_version, translate_mode, tts_mode,
+    )
+    monitor = ExecutionMonitor()
+
+    if image is None:
+        logger.warning("run_pipeline: no image provided")
+        raise gr.Error("Vui lòng chụp ảnh hoặc tải ảnh lên trước!")
+
+    # Preprocessing
+    with monitor.track("preprocess"):
+        try:
+            img = preprocess_image(image)
+            logger.info("Preprocess OK: %s → %s", image.size if hasattr(image, 'size') else '?', img.size)
+        except Exception as e:
+            logger.error("Preprocess failed: %s", e, exc_info=True)
+            raise gr.Error(f"Lỗi xử lý ảnh: {str(e)}")
+
+    # Load VLM and Inference
+    with monitor.track("vlm_inference"):
+        try:
+            logger.info("Loading VLM: %s", vlm_version)
+            vlm_model, vlm_processor = registry.get_vlm(vlm_version)
+            eng_desc = run_vlm_inference(img, vlm_version, vlm_model, vlm_processor, custom_prompt)
+            logger.info("VLM inference OK (%d chars)", len(eng_desc))
+        except Exception as e:
+            logger.error("VLM inference failed: %s", e, exc_info=True)
+            raise gr.Error(f"Lỗi VLM Inference: {str(e)}")
+
+    # Translate
+    with monitor.track("translation"):
+        try:
+            translator = registry.get_translator_module(translate_mode)
+            vi_desc, is_offline_trans = translator.translate(eng_desc, translate_mode)
+            logger.info("Translation OK (offline=%s)", is_offline_trans)
+        except Exception as e:
+            logger.warning("Translation failed: %s", e, exc_info=True)
+            vi_desc = f"[Lỗi dịch] {eng_desc}"
+            is_offline_trans = False
+            gr.Warning(f"Dịch thuật thất bại: {str(e)}")
+
+    # TTS
+    with monitor.track("tts"):
+        try:
+            temp_path = os.path.join(tempfile.gettempdir(), f"output_{uuid.uuid4().hex}.mp3")
+            audio_path = tts_module.generate_speech(vi_desc, tts_mode, filename=temp_path)
+            logger.info("TTS OK: %s", audio_path)
+        except Exception as e:
+            logger.warning("TTS failed: %s", e, exc_info=True)
+            audio_path = None
+            gr.Warning(f"Không thể tạo giọng đọc: {str(e)}")
+
+    total_time = sum(monitor.get_durations().values())
+    ram_usage = monitor.get_ram_usage()
+    timing_html = get_performance_html(monitor.get_durations())
+
+    logger.info("Pipeline complete: %.3fs | RAM %.1fMB", total_time, ram_usage)
+
+    if "Auto-Detect" in translate_mode and is_offline_trans:
+        gr.Warning("Mất kết nối Internet - Tự động chuyển sang dịch Offline (Helsinki-NLP)")
+
+    return (
+        eng_desc,
+        vi_desc,
+        audio_path,
+        f"{total_time:.3f} s",
+        f"{ram_usage:.1f} MB",
+        timing_html
+    )
+
+
+# ─── Gradio UI ────────────────────────────────────────────────────────────────
+logger.info("Building Gradio Blocks UI...")
+try:
+    with gr.Blocks(theme=gr.themes.Default(primary_hue="blue", secondary_hue="indigo")) as demo:
+        gr.HTML("<h1 style='text-align: center; color: #1f6feb;'>🌐 SmartSight AI — Hỗ Trợ Người Khiếm Thị</h1>")
+        gr.HTML("<p style='text-align: center;'>Hệ thống mô tả hình ảnh tự động bằng giọng nói Tiếng Việt</p>")
+
+        with gr.Row():
+            with gr.Column(scale=1):
+                input_image = gr.Image(sources=["webcam", "upload"], type="pil", label="Đầu vào hình ảnh")
+                vlm_version = gr.Radio(
+                    choices=["Moondream2 (2B)", "Moondream2 (0.5B)"],
+                    value="Moondream2 (2B)",
+                    label="Mô hình VLM"
+                )
+                with gr.Row():
+                    run_btn = gr.Button("Run Pipeline", variant="primary")
+                    cancel_btn = gr.Button("Cancel", variant="stop")
+
+                with gr.Accordion("Parameters & Thresholds (Cấu hình nâng cao)", open=False):
+                    translate_mode = gr.Dropdown(
+                        choices=["Auto-Detect (Online)", "Offline (Helsinki-NLP)"],
+                        value="Auto-Detect (Online)",
+                        label="Chế độ dịch"
+                    )
+                    tts_mode = gr.Dropdown(
+                        choices=["Auto-Detect (Online)", "Offline (pyttsx3)"],
+                        value="Auto-Detect (Online)",
+                        label="Chế độ TTS"
+                    )
+                    custom_prompt = gr.Textbox(
+                        lines=2,
+                        label="VLM Prompt Template",
+                        placeholder="Mặc định: Describe what you see..."
+                    )
+
+            with gr.Column(scale=1):
+                eng_out = gr.Textbox(label="Mô tả Tiếng Anh (VLM Output)", interactive=False)
+                vi_out = gr.Textbox(label="Mô tả Tiếng Việt (Dịch)", interactive=False)
+                audio_out = gr.Audio(label="Giọng đọc Tiếng Việt", autoplay=True, interactive=False)
+
+                with gr.Group():
+                    gr.Markdown("### 📊 Performance Dashboard")
+                    with gr.Row():
+                        total_time_lbl = gr.Textbox(label="TOTAL TIME", value="0.000 s", interactive=False)
+                        ram_usage_lbl = gr.Textbox(label="RAM USAGE", value="0.0 MB", interactive=False)
+                    timing_chart = gr.HTML(value="<p>Chưa chạy xử lý.</p>")
+
+        run_event = run_btn.click(
+            fn=run_pipeline,
+            inputs=[input_image, vlm_version, translate_mode, tts_mode, custom_prompt],
+            outputs=[eng_out, vi_out, audio_out, total_time_lbl, ram_usage_lbl, timing_chart]
+        )
+        cancel_btn.click(fn=None, cancels=[run_event])
+
+    logger.info("Gradio Blocks UI built successfully.")
+
+except Exception:
+    logger.critical("Failed to build Gradio UI!", exc_info=True)
+    raise
+
+
+if __name__ == "__main__":
+    # For local development only — HF Spaces uses the root app.py instead.
+    # Model loads lazily on first Run click (no warm-start blocking the UI).
+    demo.queue().launch()

src/pipeline/__init__.py

@@ -0,0 +1 @@
+

src/pipeline/preprocess.py

@@ -0,0 +1,26 @@
+from PIL import Image
+
+def preprocess_image(image: Image.Image) -> Image.Image:
+    if image is None:
+        raise ValueError("No image provided")
+        
+    w, h = image.size
+    if w < 224 or h < 224:
+        raise ValueError(f"Image is too small ({w}x{h}). Minimum size is 224x224 px.")
+        
+    # Convert RGBA or L to RGB
+    if image.mode != "RGB":
+        image = image.convert("RGB")
+        
+    # Center crop / Resize if too large (> 1280px on any dimension)
+    max_size = 1280
+    if w > max_size or h > max_size:
+        if w > h:
+            new_w = max_size
+            new_h = int(h * (max_size / w))
+        else:
+            new_h = max_size
+            new_w = int(w * (max_size / h))
+        image = image.resize((new_w, new_h), Image.Resampling.LANCZOS)
+        
+    return image

src/pipeline/translate.py

@@ -0,0 +1,38 @@
+import logging
+from deep_translator import GoogleTranslator
+
+class TranslatorModule:
+    def __init__(self, offline_model=None, offline_tokenizer=None):
+        self.offline_model = offline_model
+        self.offline_tokenizer = offline_tokenizer
+        self.google_translator = GoogleTranslator(source="auto", target="vi")
+        
+    def translate(self, text: str, mode: str = "Auto-Detect (Online)") -> tuple[str, bool]:
+        if not text.strip():
+            return "", False
+            
+        if mode == "Offline (Helsinki-NLP)":
+            return self._translate_offline(text), True
+            
+        # Try Online translation
+        try:
+            translated_text = self.google_translator.translate(text)
+            return translated_text, False
+        except Exception as e:
+            logging.warning(f"Online translation failed: {e}. Falling back to offline translation.")
+            if self.offline_model and self.offline_tokenizer:
+                return self._translate_offline(text), True
+            else:
+                return f"[Lỗi mạng - Không dịch được] {text}", False
+                
+    def _translate_offline(self, text: str) -> str:
+        if not self.offline_model or not self.offline_tokenizer:
+            return f"[Chưa load model offline] {text}"
+            
+        inputs = self.offline_tokenizer(text, return_tensors="pt")
+        # Move tensors to same device as the model (handles CPU/GPU HF Spaces)
+        if hasattr(self.offline_model, "device") and hasattr(inputs, "to"):
+            inputs = inputs.to(self.offline_model.device)
+        generated_ids = self.offline_model.generate(**inputs)
+        translated_text = self.offline_tokenizer.decode(generated_ids[0], skip_special_tokens=True)
+        return translated_text

src/pipeline/tts.py

@@ -0,0 +1,68 @@
+import os
+import tempfile
+import logging
+from gtts import gTTS
+import pyttsx3
+
+class TTSModule:
+    def __init__(self):
+        self.offline_engine = None
+        
+    def _init_offline(self):
+        if self.offline_engine is None:
+            try:
+                self.offline_engine = pyttsx3.init()
+                voices = self.offline_engine.getProperty("voices")
+                for voice in voices:
+                    if voice.languages and any("vi" in lang for lang in voice.languages):
+                        self.offline_engine.setProperty("voice", voice.id)
+                        break
+                    elif "vietnam" in voice.name.lower():
+                        self.offline_engine.setProperty("voice", voice.id)
+                        break
+            except Exception as e:
+                logging.error(f"Failed to initialize pyttsx3 offline TTS: {e}")
+                
+    def generate_speech(self, text: str, mode: str = "Auto-Detect (Online)", filename: str = None) -> str:
+        if not text or not text.strip():
+            return None
+            
+        if filename is None:
+            filename = os.path.join(tempfile.gettempdir(), "output.mp3")
+            
+        # Clean existing file to avoid permission or handle lock issues
+        if os.path.exists(filename):
+            try:
+                os.remove(filename)
+            except Exception:
+                pass
+                
+        if mode == "Offline (pyttsx3)":
+            try:
+                self._generate_offline(text, filename)
+                return filename
+            except Exception as e:
+                logging.error(f"Offline TTS failed: {e}")
+                return None
+            
+        # Try Online gTTS
+        try:
+            tts = gTTS(text=text, lang="vi", slow=False)
+            tts.save(filename)
+            return filename
+        except Exception as e:
+            logging.warning(f"Online gTTS failed: {e}. Falling back to pyttsx3.")
+            try:
+                self._generate_offline(text, filename)
+                return filename
+            except Exception as e_off:
+                logging.error(f"Offline TTS failed: {e_off}")
+                return None
+            
+    def _generate_offline(self, text: str, filename: str):
+        self._init_offline()
+        if self.offline_engine:
+            self.offline_engine.save_to_file(text, filename)
+            self.offline_engine.runAndWait()
+        else:
+            raise RuntimeError("Offline TTS engine pyttsx3 is not available.")

src/pipeline/vision_model.py

@@ -0,0 +1,23 @@
+import torch
+from PIL import Image
+
+
+def run_vlm_inference(image: Image.Image, version: str, model, processor, prompt: str = "") -> str:
+    if version not in ["Moondream2 (2B)", "Moondream2 (0.5B)"]:
+        raise ValueError(f"Unsupported model version: {version}")
+
+    if not prompt or not prompt.strip():
+        prompt = (
+            "Describe what you see in this image briefly and clearly. "
+            "Focus on the main subject, people, objects, and any important context. "
+            "Keep it under 3 sentences."
+        )
+
+    # Check for the newer API (model.query) or fallback to older API
+    if hasattr(model, "query"):
+        response = model.query(image, prompt)
+        return response["answer"]
+    else:
+        # Fallback to older Moondream version API
+        enc_image = model.encode_image(image)
+        return model.answer_question(enc_image, prompt, processor)

src/registry.py

@@ -0,0 +1,171 @@
+import sys
+import types
+import logging
+import threading
+import torch
+import torch.nn as nn
+from transformers import AutoModelForCausalLM, AutoTokenizer, MarianMTModel, MarianTokenizer
+from src.pipeline.translate import TranslatorModule
+
+logger = logging.getLogger("smartsight")
+
+
+def _patch_moondream_for_cpu(model: nn.Module) -> nn.Module:
+    """
+    Patch Moondream2 (revision 2025-01-09) for correct float32 CPU inference.
+
+    The upstream cached files have two hardcoded float16 issues that break CPU:
+
+      [Bug 1] vision.py line 45:
+          all_crops = torch.from_numpy(...).to(device=device, dtype=torch.float16)
+          → image crop tensor is Half, but model weights are Float32 after .float()
+          → RuntimeError: mat1 and mat2 must have the same dtype, but got Half and Float
+
+      [Bug 2] moondream.py encode_image():
+          kv_cache = torch.zeros(..., dtype=torch.float16)
+          → kv_cache is Half, but text model activations are Float32
+          → Potential dtype mismatch in prefill/decode_one_token ops
+
+    These files are cached by HuggingFace Hub and cannot be edited at deploy time.
+    We fix both bugs by monkey-patching the relevant functions after model load.
+
+    On GPU (CUDA / MPS) the model is left completely untouched — float16 is fine there.
+    """
+    # HfMoondream wraps MoondreamModel as .model
+    inner: nn.Module = getattr(model, "model", model)
+
+    try:
+        device = next(inner.parameters()).device
+    except StopIteration:
+        device = next(model.parameters()).device
+
+    if device.type != "cpu":
+        logger.info("GPU detected: skipping CPU float32 patch for Moondream2.")
+        return model
+
+    logger.info("CPU detected: applying Moondream2 float32 patches for HF Space.")
+
+    # ── Step 1: Convert all nn.Parameters and registered buffers to float32 ────
+    inner.float()
+    logger.info("Step 1 done: model weights/buffers converted to float32.")
+
+    # ── Step 2: Patch ops["vision_encoder"] ─────────────────────────────────────
+    # Bug 1 fix: vision.py hardcodes .to(dtype=torch.float16) for image crops.
+    # We wrap the vision_encoder op to convert the crop tensor to float32 first.
+    _orig_vision_encoder = inner.ops["vision_encoder"]
+
+    def _float32_vision_encoder(crops: torch.Tensor, w, cfg):
+        return _orig_vision_encoder(crops.to(torch.float32), w, cfg)
+
+    inner.ops["vision_encoder"] = _float32_vision_encoder
+    logger.info("Step 2 done: vision_encoder patched — image crops cast float16→float32.")
+
+    # ── Step 3: Patch encode_image to use float32 kv_cache ──────────────────────
+    # Bug 2 fix: encode_image hardcodes dtype=torch.float16 for kv_cache.
+    # We rewrite encode_image to use float32 for the kv_cache tensor.
+    moondream_mod = sys.modules.get(type(inner).__module__)
+    if moondream_mod is None:
+        logger.warning("Step 3 skipped: could not locate moondream module in sys.modules.")
+    else:
+        _EncodedImage = getattr(moondream_mod, "EncodedImage", None)
+        _text_encoder = getattr(moondream_mod, "text_encoder", None)
+
+        if _EncodedImage is None or _text_encoder is None:
+            logger.warning(
+                "Step 3 skipped: EncodedImage or text_encoder not found in moondream module."
+            )
+        else:
+            def _patched_encode_image(self, image):
+                # Pass-through if image is already encoded
+                if isinstance(image, _EncodedImage):
+                    return image
+                if not hasattr(image, "size"):
+                    raise ValueError("image must be a PIL Image or EncodedImage")
+
+                # Use float32 instead of the hardcoded float16
+                kv_cache = torch.zeros(
+                    self.config.text.n_layers,
+                    2,   # k, v
+                    1,   # batch size
+                    self.config.text.n_heads,
+                    self.config.text.max_context,
+                    self.config.text.dim // self.config.text.n_heads,
+                    device=self.device,
+                    dtype=torch.float32,  # ← patched from float16
+                )
+
+                with torch.no_grad():
+                    img_emb = self._run_vision_encoder(image)
+                    bos_emb = _text_encoder(
+                        torch.tensor(
+                            [[self.config.tokenizer.bos_id]], device=self.device
+                        ),
+                        self.text,
+                    )
+                    inputs_embeds = torch.cat([bos_emb, img_emb[None]], dim=1)
+                    self.ops["prefill"](
+                        inputs_embeds, kv_cache, 0, self.text, self.config.text
+                    )
+
+                return _EncodedImage(pos=inputs_embeds.size(1), kv_cache=kv_cache)
+
+            inner.encode_image = types.MethodType(_patched_encode_image, inner)
+            logger.info(
+                "Step 3 done: encode_image patched — kv_cache dtype float16→float32."
+            )
+
+    logger.info("Moondream2 CPU patches applied. Ready for float32 inference.")
+    return model
+
+
+class ModelRegistry:
+    _instance = None
+    _lock = threading.Lock()
+
+    def __new__(cls):
+        with cls._lock:
+            if cls._instance is None:
+                cls._instance = super(ModelRegistry, cls).__new__(cls)
+                cls._instance.vlm_models = {"Moondream2 (2B)": None, "Moondream2 (0.5B)": None}
+                cls._instance.vlm_processors = {"Moondream2 (2B)": None, "Moondream2 (0.5B)": None}
+                cls._instance.translation_model = None
+                cls._instance.translation_tokenizer = None
+                cls._instance.translator_instance = None
+        return cls._instance
+
+    def get_vlm(self, version: str) -> tuple:
+        if version not in self.vlm_models:
+            raise ValueError(f"Unknown VLM model version: {version}")
+
+        if self.vlm_models[version] is None:
+            if version == "Moondream2 (0.5B)":
+                model_id = "andito/moondream05"  # Community transformers 0.5B model
+                processor = AutoTokenizer.from_pretrained(model_id)
+                model = AutoModelForCausalLM.from_pretrained(
+                    model_id,
+                    trust_remote_code=True,
+                    torch_dtype=torch.float32,
+                )
+            else:  # Moondream2 (2B)
+                model_id = "vikhyatk/moondream2"
+                revision = "2025-01-09"  # API 2025 compatible
+                processor = AutoTokenizer.from_pretrained(model_id, revision=revision)
+                model = AutoModelForCausalLM.from_pretrained(
+                    model_id,
+                    revision=revision,
+                    trust_remote_code=True,
+                    torch_dtype=torch.float32,  # load weights as float32
+                )
+                model = _patch_moondream_for_cpu(model)
+            self.vlm_models[version] = model
+            self.vlm_processors[version] = processor
+
+        return self.vlm_models[version], self.vlm_processors[version]
+
+    def get_translator_module(self, mode: str) -> TranslatorModule:
+        if self.translator_instance is None:
+            model_id = "Helsinki-NLP/opus-mt-en-vi"
+            self.translation_tokenizer = MarianTokenizer.from_pretrained(model_id)
+            self.translation_model = MarianMTModel.from_pretrained(model_id)
+            self.translator_instance = TranslatorModule(self.translation_model, self.translation_tokenizer)
+        return self.translator_instance

src/utils/__init__.py

@@ -0,0 +1 @@
+

src/utils/monitor.py

@@ -0,0 +1,22 @@
+import psutil
+import time
+from contextlib import contextmanager
+
+class ExecutionMonitor:
+    def __init__(self):
+        self.durations = {}
+        
+    def get_ram_usage(self) -> float:
+        process = psutil.Process()
+        return process.memory_info().rss / (1024 * 1024)  # MB
+        
+    @contextmanager
+    def track(self, stage_name: str):
+        start = time.perf_counter()
+        try:
+            yield
+        finally:
+            self.durations[stage_name] = time.perf_counter() - start
+            
+    def get_durations(self) -> dict:
+        return self.durations