core/lang.py

"""
core/lang.py — Language detection and chunk translation utilities.

If a source chunk is not in English, the LLM struggles to follow
"write in English only" instructions because the French/Spanish tokens
in the context dominate the probability distribution at each decoding step.
Pre-translating the chunk removes this gravitational pull entirely.
"""

import re
from functools import lru_cache

# Common function words that appear in French but rarely in English text
_FR_INDICATORS = frozenset([
    "le", "la", "les", "de", "du", "des", "un", "une", "est", "en", "et",
    "avec", "dans", "sur", "pour", "que", "qui", "se", "au", "aux", "par",
    "ou", "ne", "pas", "plus", "son", "sa", "ses", "leur", "leurs", "lui",
    "ils", "elles", "nous", "vous", "je", "tu", "il", "elle", "ce", "cet",
    "cette", "ces", "mon", "ma", "ta", "sont", "ont", "une", "comme", "aussi",
    "mais", "donc", "car", "si", "tout", "tous", "toute", "toutes", "quel",
    "quelle", "quels", "quelles", "dont", "très", "aussi", "puis",
])


def is_english(text: str) -> bool:
    """Return True if *text* appears to be in English."""
    words = set(re.findall(r'\b[a-zA-ZÀ-ÿ]{2,}\b', text.lower()))
    french_hits = len(words & _FR_INDICATORS)
    # Heuristic: ≥5 French function words → almost certainly French
    return french_hits < 5


@lru_cache(maxsize=64)
def _cached_translate(chunk: str) -> str:
    from model.llm import get_llm
    llm = get_llm()
    prompt = (
        "Translate the following text to English. "
        "Output ONLY the English translation, nothing else.\n\n"
        "Text:\n" + chunk + "\n\nTranslation:"
    )
    max_tok = min(600, max(80, len(chunk.split()) * 2))
    result = llm.generate(prompt, max_new_tokens=max_tok, temperature=0.1).strip()
    # Sanity-check: if translation looks empty or too short, return original
    if len(result) < 20:
        return chunk
    return result


def ensure_english(chunk: str) -> str:
    """Return an English version of *chunk*, translating only if necessary."""
    if is_english(chunk):
        return chunk
    return _cached_translate(chunk)