src/data/wikimedia_loader.py

import json
from pathlib import Path


class WikimediaSampleLoader:
    def __init__(self, base_dir: str = "data/wikimedia"):
        self.base_dir = Path(base_dir)
        self.samples_file = self.base_dir / "samples.json"

    def load(self):
        print("Wikimedia loader starting")
        if not self.samples_file.exists():
            raise FileNotFoundError(f"Missing {self.samples_file}")

        with self.samples_file.open("r", encoding="utf-8") as f:
            data = json.load(f)

        if isinstance(data, dict) and "samples" in data:
            data = data["samples"]
        elif isinstance(data, dict):
            data = list(data.values())

        samples = []
        for item in data:
            print("Checking item:", item.get("id"))
            samples.append(
                {
                    "id": item["id"],
                    "caption": item["caption"],
                    "image_path": self.base_dir / item["image"],
                    "audio_path": (
                        self.base_dir / item["audio"] if item.get("audio") else None
                    ),
                }
            )

        print("Wikimedia loader finished. Samples:", len(samples))
        return samples