File size: 4,328 Bytes
a484e22 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 | """Cross-model generalization summary (Phase 3).
Reads the per-model acceptance result JSONs (gpt-oss Phase-2 headline + any
Phase-3 models) and emits:
* results/phase3_crossmodel.json -- machine-readable comparison
* paper/figures/crossmodel.{pdf,png} -- grouped bar of personal vs static MAT
* a LaTeX-ready table printed to stdout
No model is re-run here; this only aggregates existing result files. Missing
model files are skipped (so it works whether or not Nemotron served).
"""
from __future__ import annotations
import json
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
RESULTS = ROOT / "results"
FIGS = ROOT / "figures"
# (display name, results json). row order = table order (gpt-oss first).
MODELS = [
("gpt-oss-120b", RESULTS / "phase2_accept_results.json"),
("gemma-4-31B-it", RESULTS / "phase3_gemma_accept_results.json"),
("Nemotron-3-Super-120B", RESULTS / "phase3_nemotron_accept_results.json"),
]
def _row(path: Path):
d = json.loads(path.read_text())
ow = d["overall_post_warmup"]
per = d["summary"] # per-session, for tail gap
sessions = sorted(int(s) for s in per["personal_memory"].keys())
last = str(sessions[-1])
def mat(arm, block=ow, key=None):
return block[arm]["MAT"] if key is None else block[arm][key]["MAT"]
stat, pers, nomem = mat("static_global"), mat("personal_memory"), mat("no_memory")
gap = 100.0 * (pers - stat) / stat
tail_stat = per["static_global"][last]["MAT"]
tail_pers = per["personal_memory"][last]["MAT"]
tail_gap = 100.0 * (tail_pers - tail_stat) / tail_stat
return {
"no_memory": round(nomem, 2),
"static_global": round(stat, 2),
"personal_memory": round(pers, 2),
"gap_pct": round(gap, 1),
"tail_gap_pct": round(tail_gap, 1),
"personal_seed_std": ow["personal_memory"].get("MAT_seed_std"),
"n": ow["personal_memory"]["n"],
}
def main():
out = {}
for name, path in MODELS:
if path.exists():
out[name] = _row(path)
print(f"[ok] {name}: {out[name]}")
else:
print(f"[skip] {name}: {path.name} not found")
(RESULTS / "phase3_crossmodel.json").write_text(json.dumps(out, indent=2))
# LaTeX table body
print("\n% --- LaTeX table rows (personal vs static vs none, +gap) ---")
for name, r in out.items():
print(f"{name} & {r['no_memory']:.2f} & {r['static_global']:.2f} & "
f"{r['personal_memory']:.2f} & $+{r['gap_pct']:.0f}\\%$ & "
f"$+{r['tail_gap_pct']:.0f}\\%$ \\\\")
# grouped bar figure
try:
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
names = list(out.keys())
x = np.arange(len(names))
w = 0.26
nomem = [out[n]["no_memory"] for n in names]
stat = [out[n]["static_global"] for n in names]
pers = [out[n]["personal_memory"] for n in names]
fig, ax = plt.subplots(figsize=(7.2, 3.6))
ax.bar(x - w, nomem, w, label="No memory", color="#9e9e9e")
ax.bar(x, stat, w, label="Static datastore", color="#4C72B0")
ax.bar(x + w, pers, w, label="Personal evicting (ours)", color="#C44E52")
ymax = max(pers) * 1.30 # headroom for labels + legend
ax.set_ylim(0, ymax)
for xi, n in zip(x, names):
ax.text(xi + w, out[n]["personal_memory"] + ymax * 0.015,
f"+{out[n]['gap_pct']:.0f}%", ha="center", fontsize=8,
color="#C44E52", fontweight="bold")
ax.set_xticks(x)
ax.set_xticklabels(names, fontsize=9)
ax.set_ylabel("Mean accepted tokens (post-warmup)")
ax.set_title("Personalized evicting memory generalizes across served models")
ax.legend(fontsize=8, loc="upper center", ncol=3, frameon=False,
bbox_to_anchor=(0.5, 1.0))
ax.grid(axis="y", alpha=0.3)
fig.tight_layout()
FIGS.mkdir(parents=True, exist_ok=True)
fig.savefig(FIGS / "crossmodel.pdf")
fig.savefig(FIGS / "crossmodel.png", dpi=150)
print(f"\n[fig] wrote {FIGS/'crossmodel.pdf'}")
except Exception as e: # noqa: BLE001
print(f"[fig] skipped: {e}")
if __name__ == "__main__":
main()
|