engine/chunking.py

"""Whitespace/punctuation-based text segmentation.

There is no subtitle timing here — segments are just chunks of source text
sized for one translation call each. Splits on blank-line paragraph breaks
first, then on Tibetan sentence-ending shad marks (། ༎) for any paragraph
still longer than max_chars, then falls back to plain whitespace chunking
for any single "sentence" that is still too long on its own.
"""

import re

_SHAD_SPLIT_RE = re.compile(r"(?<=[།༎])\s*")


def _chunk_by_words(text: str, max_chars: int) -> list[str]:
    words = text.split(" ")
    chunks, current = [], ""
    for word in words:
        if current and len(current) + 1 + len(word) > max_chars:
            chunks.append(current)
            current = word
        else:
            current = f"{current} {word}".strip()
    if current:
        chunks.append(current)
    return chunks


def _chunk_paragraph(paragraph: str, max_chars: int) -> list[str]:
    if len(paragraph) <= max_chars:
        return [paragraph]

    pieces = [p.strip() for p in _SHAD_SPLIT_RE.split(paragraph) if p.strip()]
    chunks, current = [], ""
    for piece in pieces:
        if len(piece) > max_chars:
            if current:
                chunks.append(current)
                current = ""
            chunks.extend(_chunk_by_words(piece, max_chars))
        elif current and len(current) + 1 + len(piece) > max_chars:
            chunks.append(current)
            current = piece
        else:
            current = f"{current} {piece}".strip()
    if current:
        chunks.append(current)
    return chunks


def split_into_segments(text: str, max_chars: int = 400) -> list[str]:
    """Split raw text into a list of segment strings."""
    paragraphs = re.split(r"\n\s*\n", text.strip())
    segments = []
    for para in paragraphs:
        para = " ".join(para.split())
        if not para:
            continue
        segments.extend(_chunk_paragraph(para, max_chars))
    return segments