Generalization-gap metric: held-out split in run_eval + leaderboard

The stated goal is 'generalize on reasoning'; the literature
(Procgen, SMACv2, lmgame-Bench) is emphatic that the public-vs-held-
out generalization gap — not aggregate score — is THE anti-
memorization metric. evaluate() now takes held_out_seeds: those run
as split='held_out'; report adds overall_held_out +
generalization_gap (public − held-out composite). run_eval gains
--held-out-seeds; leaderboard ingest records held_out_composite +
generalization_gap; app.py surfaces both columns. Backward compatible
(no held-out → unchanged shape). tests: gap math, split tagging,
leaderboard capture, back-compat. Full bench suite 269 passed.

app.py

@@ -173,6 +173,7 @@ def load_capability_leaderboard() -> pd.DataFrame:
     cols = [
         "rank", "model", "episodes", "win_rate", "composite",
         "perception", "reasoning", "action", "weakest_link",
+        "held_out_composite", "generalization_gap",
     ]
     if not rows:
         return pd.DataFrame(columns=cols)

openra_bench/leaderboard.py

@@ -63,6 +63,12 @@ def ingest_run(
         "reasoning": overall.get("reasoning_mean", 0.0),
         "action": overall.get("action_mean", 0.0),
         "weakest_link_hist": overall.get("weakest_link_hist", {}),
+        # Anti-memorization: held-out composite + generalization gap
+        # (public − held-out). None when the run had no held-out split.
+        "held_out_composite": stats.get("overall_held_out", {}).get(
+            "composite_mean"
+        ),
+        "generalization_gap": stats.get("generalization_gap"),
         "by_capability": _capability_breakdown(eps),
         "cells": sorted(stats.get("summary", {}).keys()),
     }

openra_bench/run_eval.py

@@ -70,11 +70,21 @@ def evaluate(
     seeds: list[int],
     provider_cfg=None,
     agent_factory: AgentFactory | None = None,
+    held_out_seeds: list[int] | None = None,
 ) -> dict:
+    """Run packs×levels×seeds. If `held_out_seeds` is given, those are
+    run too and tagged split='held_out'; the report adds
+    `overall_held_out` and `generalization_gap` (public composite −
+    held-out composite) — the anti-memorization metric the
+    generalization literature (Procgen/SMACv2/lmgame-Bench) requires.
+    """
     factory = agent_factory or _default_agent_factory(provider_cfg)
     by_cell: dict[str, list] = {}
     episodes: list[dict] = []
     skipped: list[str] = []
+    public_scores: list = []
+    held_scores: list = []
+    held_out_seeds = held_out_seeds or []
 
     for pack_path in packs:
         pack = load_pack(pack_path)
@@ -84,32 +94,47 @@ def evaluate(
                 skipped.append(f"{pack.meta.id}:{level} (map not Rust-loadable)")
                 continue
             cell = f"{pack.meta.id}:{level}"
-            for seed in seeds:
-                res = run_level(compiled, factory(compiled), seed=seed)
-                sc = score_episode(compiled, res)
-                by_cell.setdefault(cell, []).append(sc)
-                episodes.append(
-                    {
-                        "cell": cell,
-                        "capability": compiled.meta.capability,
-                        "seed": seed,
-                        "outcome": sc.outcome,
-                        "composite": sc.composite,
-                        "perception": sc.perception,
-                        "reasoning": sc.reasoning,
-                        "action": sc.action,
-                        "weakest_link": sc.weakest_link,
-                        "turns": res.turns,
-                        "notes": sc.notes,
-                    }
-                )
-
-    return {
+            for split, slist in (("public", seeds), ("held_out", held_out_seeds)):
+                for seed in slist:
+                    res = run_level(compiled, factory(compiled), seed=seed)
+                    sc = score_episode(compiled, res)
+                    if split == "public":
+                        by_cell.setdefault(cell, []).append(sc)
+                        public_scores.append(sc)
+                    else:
+                        held_scores.append(sc)
+                    episodes.append(
+                        {
+                            "cell": cell,
+                            "capability": compiled.meta.capability,
+                            "split": split,
+                            "seed": seed,
+                            "outcome": sc.outcome,
+                            "composite": sc.composite,
+                            "perception": sc.perception,
+                            "reasoning": sc.reasoning,
+                            "action": sc.action,
+                            "weakest_link": sc.weakest_link,
+                            "turns": res.turns,
+                            "notes": sc.notes,
+                        }
+                    )
+
+    out = {
         "summary": {cell: _agg(scs) for cell, scs in by_cell.items()},
-        "overall": _agg([s for scs in by_cell.values() for s in scs]),
+        "overall": _agg(public_scores),
         "episodes": episodes,
         "skipped": skipped,
     }
+    if held_scores:
+        ho = _agg(held_scores)
+        out["overall_held_out"] = ho
+        out["generalization_gap"] = round(
+            out["overall"].get("composite_mean", 0.0)
+            - ho.get("composite_mean", 0.0),
+            4,
+        )
+    return out
 
 
 def write_report(stats: dict, path: str | Path) -> None:
@@ -132,6 +157,12 @@ def main(argv: list[str]) -> int:
     ap.add_argument("--packs", help="pack file or dir (default: bundled packs/)")
     ap.add_argument("--levels", default="easy,medium,hard")
     ap.add_argument("--seeds", default="1,2,3")
+    ap.add_argument(
+        "--held-out-seeds",
+        default="",
+        help="comma seeds run as a held-out split; reports the "
+        "generalization gap (anti-memorization metric)",
+    )
     ap.add_argument("--provider", help="openrouter|vllm|openai (omit = scripted baseline)")
     ap.add_argument("--model", default="anthropic/claude-3.5-sonnet")
     ap.add_argument("--base-url")
@@ -162,6 +193,7 @@ def main(argv: list[str]) -> int:
         a.levels.split(","),
         [int(s) for s in a.seeds.split(",")],
         provider_cfg=cfg,
+        held_out_seeds=[int(s) for s in a.held_out_seeds.split(",") if s.strip()],
     )
     write_report(stats, a.out)
     o = stats["overall"]

tests/test_run_eval.py

@@ -44,6 +44,47 @@ def test_evaluate_aggregates_and_reports(tmp_path):
     assert loaded["overall"]["n"] == 2
 
 
+def test_held_out_split_reports_generalization_gap(tmp_path):
+    stats = evaluate(
+        packs=[PACKS / "perception-frontier-reading.yaml"],
+        levels=["easy"],
+        seeds=[1, 2],
+        held_out_seeds=[7, 8],
+    )
+    assert "overall_held_out" in stats
+    assert stats["overall_held_out"]["n"] == 2
+    assert isinstance(stats["generalization_gap"], float)
+    splits = {e["split"] for e in stats["episodes"]}
+    assert splits == {"public", "held_out"}
+    assert sum(e["split"] == "held_out" for e in stats["episodes"]) == 2
+    # gap == public composite − held-out composite (sign can be ±).
+    g = round(
+        stats["overall"]["composite_mean"]
+        - stats["overall_held_out"]["composite_mean"],
+        4,
+    )
+    assert stats["generalization_gap"] == g
+
+    # Leaderboard captures the gap.
+    from openra_bench.leaderboard import build_table, ingest_run
+
+    s = tmp_path / "lb.jsonl"
+    ingest_run(stats, "m", s)
+    row = build_table(s, min_episodes=1)[0]
+    assert row["generalization_gap"] == stats["generalization_gap"]
+    assert row["held_out_composite"] == stats["overall_held_out"]["composite_mean"]
+
+
+def test_no_held_out_keeps_backward_compatible_shape():
+    stats = evaluate(
+        packs=[PACKS / "perception-frontier-reading.yaml"],
+        levels=["easy"],
+        seeds=[1],
+    )
+    assert "overall_held_out" not in stats and "generalization_gap" not in stats
+    assert all(e["split"] == "public" for e in stats["episodes"])
+
+
 def test_unsupported_map_is_skipped_not_crashed(tmp_path):
     """A pack on a non-Rust map must be reported as skipped, not raise."""
     pack = (PACKS / "perception-frontier-reading.yaml").read_text()