src/evaluation/benchmarks.py

from __future__ import annotations

import json
from pathlib import Path
from typing import Dict, List

from src.coherence.msci import compute_msci_v0
from src.embeddings.aligned_embeddings import AlignedEmbedder
from src.planner.schema_to_text import plan_to_canonical_text

GOLD_PATH = Path("evaluation/gold_dataset/samples.jsonl")


def load_gold_samples() -> List[Dict]:
    with GOLD_PATH.open("r", encoding="utf-8") as handle:
        return [json.loads(line) for line in handle]


def evaluate_gold_samples():
    embedder = AlignedEmbedder(target_dim=512)

    results = []

    for sample in load_gold_samples():
        plan_text = plan_to_canonical_text(sample["semantic_plan"])
        e_text = embedder.embed_text(plan_text)

        e_image = None
        e_audio = None

        if sample["image_path"]:
            e_image = embedder.embed_image(sample["image_path"])
        if sample["audio_path"]:
            e_audio = embedder.embed_audio(sample["audio_path"])

        msci = compute_msci_v0(
            e_text,
            e_image,
            e_audio,
            include_image_audio=True,
        )

        results.append(
            {
                "id": sample["id"],
                "msci": msci.msci,
                "st_i": msci.st_i,
                "st_a": msci.st_a,
                "si_a": msci.si_a,
                "human": sample["human_criteria"],
            }
        )

    return results