| import os | |
| from typing import Any | |
| from data.vector_db import load_chunks_with_local_cache | |
| # cacheing logic here | |
| # note cacheing just useful in dev environment | |
| # not really needed in hf, not even sure if hf memory is persistent | |
| def get_cache_settings() -> tuple[str, bool]: | |
| project_root = os.path.dirname(os.path.abspath(__file__)) | |
| cache_dir = os.getenv("BM25_CACHE_DIR", os.path.join(project_root, "..", ".cache")) | |
| force_cache_refresh = os.getenv("BM25_CACHE_REFRESH", "0").lower() in {"1", "true", "yes"} | |
| return cache_dir, force_cache_refresh | |
| def load_cached_chunks( | |
| index: Any, | |
| index_name: str, | |
| cache_dir: str, | |
| force_cache_refresh: bool, | |
| batch_size: int = 100, | |
| ) -> tuple[list[dict[str, Any]], str]: | |
| return load_chunks_with_local_cache( | |
| index=index, | |
| index_name=index_name, | |
| cache_dir=cache_dir, | |
| batch_size=batch_size, | |
| force_refresh=force_cache_refresh, | |
| ) | |