Rename Lodestar -> PRIMO

README.md

@@ -1,5 +1,5 @@
 ---
-title: Lodestar Benchmark
+title: PRIMO Benchmark
 emoji: 🧬
 colorFrom: indigo
 colorTo: blue
@@ -10,7 +10,9 @@ app_file: app.py
 pinned: false
 ---
 
-# Lodestar — blind benchmark for transcriptomic foundation models
+# PRIMO — Patient Representations in Multi-Omics
+
+**A blind benchmark for transcriptomic foundation models.**
 
 Embed **every** dataset with your model and upload **one** file. A fixed linear
 probe grades each hidden task (a dataset may be scored on several targets) over
@@ -56,16 +58,16 @@ feedback, they just aren't ranked.
 
 ```bash
 pip install -r requirements.txt
-export HF_TOKEN=...   # read access to the Lodestar datasets
+export HF_TOKEN=...   # read access to the PRIMO datasets
 python evaluator.py --submission my_embeddings.parquet
 ```
 
 ## Space configuration
 
 - Set an **`HF_TOKEN`** Space secret (fine-grained) with: **read** on
-  `ScientaLab/lodestar` (the public `datasets.yaml` manifest) and
-  `ScientaLab/lodestar-labels` (the private `tasks.yaml` registry +
-  `<task_id>/labels.csv`), and **write** on `ScientaLab/lodestar-results` (the
+  `ScientaLab/primo` (the public `datasets.yaml` manifest) and
+  `ScientaLab/primo-labels` (the private `tasks.yaml` registry +
+  `<task_id>/labels.csv`), and **write** on `ScientaLab/primo-results` (the
   persisted leaderboard).
 - Results persist as one normalized `task_results.csv` (`model_name, task_id,
   score, submitted_at`) in the results dataset; the leaderboard is recomputed

app.py

@@ -1,4 +1,4 @@
-"""Gradio front-end for the Lodestar public benchmark.
+"""Gradio front-end for the PRIMO public benchmark.
 
 Upload one embedding file spanning every dataset (rows keyed by ``dataset_id``
 + ``sample_id``); a fixed linear probe scores each task (a dataset may carry
@@ -259,15 +259,15 @@ def evaluate(submission_path: str, model_name: str):
 
 
 def build_demo() -> gr.Blocks:
-    with gr.Blocks(title="Lodestar Benchmark") as demo:
+    with gr.Blocks(title="PRIMO Benchmark") as demo:
         gr.Markdown(
-            "# 🧬 Lodestar\n\n"
+            "# 🧬 PRIMO\n\n"
             "A **blind benchmark for transcriptomic foundation models** — grade "
             "your model's patient-level embeddings against real clinical signal, "
             "without ever seeing the labels.\n\n"
             "**Submit in 3 steps:**\n"
             "1. **Get the data** → download the opaque datasets from "
-            "[ScientaLab/lodestar](https://huggingface.co/datasets/ScientaLab/lodestar)"
+            "[ScientaLab/primo](https://huggingface.co/datasets/ScientaLab/primo)"
             " (start with its `datasets.yaml`).\n"
             "2. **Embed every dataset** → build **one** file: `dataset_id`, "
             "`sample_id`, then one column per embedding dim (`e0`, `e1`, …). "

evaluator.py

@@ -1,4 +1,4 @@
-"""Standalone probe for the Lodestar public benchmark.
+"""Standalone probe for the PRIMO public benchmark.
 
 Loads one embedding submission that spans every dataset (rows keyed by
 ``dataset_id`` + ``sample_id``), then scores each TASK = (dataset, target): a
@@ -22,6 +22,7 @@ the fetch helpers / CLI, imported lazily.
 
 import argparse
 import logging
+import os
 import time
 from collections import defaultdict
 from dataclasses import dataclass
@@ -41,9 +42,9 @@ TASK_ID = "task_id"
 LABEL = "label"
 FOLD = "fold"
 
-PUBLIC_REPO = "ScientaLab/lodestar"
-LABELS_REPO = "ScientaLab/lodestar-labels"
-RESULTS_REPO = "ScientaLab/lodestar-results"
+PUBLIC_REPO = "ScientaLab/primo"
+LABELS_REPO = "ScientaLab/primo-labels"
+RESULTS_REPO = "ScientaLab/primo-results"
 MANIFEST_FILENAME = "datasets.yaml"
 TASKS_FILENAME = "tasks.yaml"
 LABELS_FILENAME = "labels.csv"
@@ -98,7 +99,7 @@ def load_tasks_registry(path: str | Path) -> list[dict]:
     with open(path) as handle:
         data = yaml.safe_load(handle)
     tasks = data.get("tasks", []) if isinstance(data, dict) else (data or [])
-    ids = [_norm_id(t["task_id"]) for t in tasks]
+    ids = [_norm_id(t[TASK_ID]) for t in tasks]
     if len(set(ids)) != len(ids):
         raise EvaluatorError(f"duplicate task_id in the registry: {ids}")
     return tasks
@@ -109,7 +110,7 @@ def load_labels(path: str | Path) -> pd.DataFrame:
     df = pd.read_csv(path)
     missing = {SAMPLE_ID, LABEL, FOLD} - set(df.columns)
     if missing:
-        raise SubmissionError(f"labels.csv missing columns: {sorted(missing)}")
+        raise EvaluatorError(f"labels.csv missing columns: {sorted(missing)}")
     return df
 
 
@@ -497,13 +498,11 @@ def _rollup_datasets(outcomes: list[TaskOutcome]) -> list[dict]:
 
 
 def _cli() -> None:
-    parser = argparse.ArgumentParser(description="Score a Lodestar submission locally.")
+    parser = argparse.ArgumentParser(description="Score a PRIMO submission locally.")
     parser.add_argument("--submission", required=True, help="CSV/TSV/Parquet/NPZ file")
     parser.add_argument("--token", default=None, help="HF token (else env HF_TOKEN)")
     args = parser.parse_args()
 
-    import os
-
     token = args.token or os.environ.get("HF_TOKEN")
     result = score_all(args.submission, token)
     print(

scoring.py

@@ -1,4 +1,4 @@
-"""Scoring policy for the Lodestar benchmark: predictions -> final scores.
+"""Scoring policy for the PRIMO benchmark: predictions -> final scores.
 
 Everything that turns a model's out-of-fold predictions into leaderboard
 numbers lives here, kept apart from the probe and from any I/O so it stays easy