Buckets:

cmpatino
/

flint-programbench

Files

xet

cmpatino/flint-programbench / scripts /plot_scores.py

cmpatino

7 days ago

download

raw

3.8 kB

"""Per-task ProgramBench reward, solo PIV vs team PIV, from sweep/manifest.json.
Run: python -I scripts/plot_scores.py [--exp-dir D]  -> results/scores.png
Dots: solo = filled blue circle, team = hollow orange diamond; a grey connector joins a task's pair;
tasks ordered easy / medium / hard (the benchmark's label); ungraded entries are simply absent."""
import argparse, csv, json
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt

def main() -> None:
    ap = argparse.ArgumentParser(); ap.add_argument("--exp-dir", default="/data/workspaces/swarmlab/experiments/flint-programbench")
    E = ap.parse_args().exp_dir
    m = json.load(open(f"{E}/sweep/manifest.json")); rs = m["runs"] if isinstance(m.get("runs"), list) else list(m["runs"].values())
    rows = list(csv.DictReader(open(f"{E}/tasks/programbench-tasks.csv")))
    idk = next(k for k in rows[0] if k in ("id", "task_id", "instance_id", "task"))
    diff = {r[idk]: r.get("difficulty", "") for r in rows}
    order = [l.strip() for l in open(f"{E}/tasks/tasks-balanced-30.txt") if l.strip() and not l.startswith("#")]
    scores = {(r["condition"], r["task"]): r.get("score") for r in rs if r.get("status") == "graded" and r.get("score") is not None}
    labels = [t.split("__", 1)[1].rsplit(".", 1)[0] for t in order]
    fig, ax = plt.subplots(figsize=(14, 5.4), dpi=150); fig.patch.set_facecolor("#fcfcfb"); ax.set_facecolor("#fcfcfb")
    for i, t in enumerate(order):
        s, g = scores.get(("solo-piv", t)), scores.get(("team-piv", t))
        if s is not None and g is not None: ax.plot([i, i], [s, g], color="#c3c2b7", lw=1, zorder=1)
        if s is not None: ax.scatter([i], [s], s=64, color="#2a78d6", marker="o", zorder=3)
        if g is not None: ax.scatter([i], [g], s=70, facecolor="none", edgecolor="#eb6834", marker="D", linewidths=1.8, zorder=3)
    n_s = sum(1 for t in order if ("solo-piv", t) in scores); n_t = sum(1 for t in order if ("team-piv", t) in scores)
    bounds = [0]; prev = diff.get(order[0], "")
    for i, t in enumerate(order[1:], 1):
        if diff.get(t, "") != prev: bounds.append(i); prev = diff.get(t, "")
    bounds.append(len(order))
    for b0, b1 in zip(bounds[:-1], bounds[1:]):
        ax.text((b0 + b1 - 1) / 2, 1.05, diff.get(order[b0], "") or "?", ha="center", va="bottom", fontsize=9, color="#6b6a63")
        if b0: ax.axvline(b0 - 0.5, color="#e4e3dd", lw=1, zorder=0)
    ax.set_xticks(range(len(order))); ax.set_xticklabels(labels, rotation=60, ha="right", fontsize=8, color="#3a3935")
    ax.set_ylim(-0.03, 1.12); ax.set_yticks([0, 0.25, 0.5, 0.75, 1.0]); ax.set_ylabel("ProgramBench reward (tests passed / total)", fontsize=9, color="#3a3935")
    ax.grid(axis="y", color="#e4e3dd", lw=0.8); ax.set_axisbelow(True)
    for sp in ("top", "right"): ax.spines[sp].set_visible(False)
    for sp in ("left", "bottom"): ax.spines[sp].set_color("#c3c2b7")
    ax.tick_params(colors="#6b6a63", length=3)
    handles = [plt.Line2D([], [], marker="o", color="#2a78d6", ls="", ms=8, label=f"solo PIV ({n_s} of 30 graded)"),
               plt.Line2D([], [], marker="D", markerfacecolor="none", markeredgecolor="#eb6834", markeredgewidth=1.8, color="#eb6834", ls="", ms=8, label=f"team PIV ({n_t} of 30 graded)")]
    ax.legend(handles=handles, loc="lower left", bbox_to_anchor=(0.0, 1.07), ncol=2, frameon=False, fontsize=9, borderaxespad=0)
    ax.set_title("Flint on ProgramBench, Balanced-30: solo PIV vs team PIV" + ("  (grading in progress; a missing dot is not yet graded)" if n_s + n_t < 60 else ""), fontsize=10, color="#1a1a19", loc="left", pad=34)
    fig.tight_layout(); out = f"{E}/results/scores.png"; fig.savefig(out, facecolor=fig.get_facecolor()); print("wrote", out, "| graded: solo", n_s, "team", n_t)

if __name__ == "__main__":
    main()

Xet Storage Details

Size:: 3.8 kB
Xet hash:: 60dd50810b9d4d01218a58d58df149b7b41c1f66d924866ee95ac1ca16c30dda

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.

	"""Per-task ProgramBench reward, solo PIV vs team PIV, from sweep/manifest.json.
	Run: python -I scripts/plot_scores.py [--exp-dir D] -> results/scores.png
	Dots: solo = filled blue circle, team = hollow orange diamond; a grey connector joins a task's pair;
	tasks ordered easy / medium / hard (the benchmark's label); ungraded entries are simply absent."""
	import argparse, csv, json
	import matplotlib
	matplotlib.use("Agg")
	import matplotlib.pyplot as plt

	def main() -> None:
	ap = argparse.ArgumentParser(); ap.add_argument("--exp-dir", default="/data/workspaces/swarmlab/experiments/flint-programbench")
	E = ap.parse_args().exp_dir
	m = json.load(open(f"{E}/sweep/manifest.json")); rs = m["runs"] if isinstance(m.get("runs"), list) else list(m["runs"].values())
	rows = list(csv.DictReader(open(f"{E}/tasks/programbench-tasks.csv")))
	idk = next(k for k in rows[0] if k in ("id", "task_id", "instance_id", "task"))
	diff = {r[idk]: r.get("difficulty", "") for r in rows}
	order = [l.strip() for l in open(f"{E}/tasks/tasks-balanced-30.txt") if l.strip() and not l.startswith("#")]
	scores = {(r["condition"], r["task"]): r.get("score") for r in rs if r.get("status") == "graded" and r.get("score") is not None}
	labels = [t.split("__", 1)[1].rsplit(".", 1)[0] for t in order]
	fig, ax = plt.subplots(figsize=(14, 5.4), dpi=150); fig.patch.set_facecolor("#fcfcfb"); ax.set_facecolor("#fcfcfb")
	for i, t in enumerate(order):
	s, g = scores.get(("solo-piv", t)), scores.get(("team-piv", t))
	if s is not None and g is not None: ax.plot([i, i], [s, g], color="#c3c2b7", lw=1, zorder=1)
	if s is not None: ax.scatter([i], [s], s=64, color="#2a78d6", marker="o", zorder=3)
	if g is not None: ax.scatter([i], [g], s=70, facecolor="none", edgecolor="#eb6834", marker="D", linewidths=1.8, zorder=3)
	n_s = sum(1 for t in order if ("solo-piv", t) in scores); n_t = sum(1 for t in order if ("team-piv", t) in scores)
	bounds = [0]; prev = diff.get(order[0], "")
	for i, t in enumerate(order[1:], 1):
	if diff.get(t, "") != prev: bounds.append(i); prev = diff.get(t, "")
	bounds.append(len(order))
	for b0, b1 in zip(bounds[:-1], bounds[1:]):
	ax.text((b0 + b1 - 1) / 2, 1.05, diff.get(order[b0], "") or "?", ha="center", va="bottom", fontsize=9, color="#6b6a63")
	if b0: ax.axvline(b0 - 0.5, color="#e4e3dd", lw=1, zorder=0)
	ax.set_xticks(range(len(order))); ax.set_xticklabels(labels, rotation=60, ha="right", fontsize=8, color="#3a3935")
	ax.set_ylim(-0.03, 1.12); ax.set_yticks([0, 0.25, 0.5, 0.75, 1.0]); ax.set_ylabel("ProgramBench reward (tests passed / total)", fontsize=9, color="#3a3935")
	ax.grid(axis="y", color="#e4e3dd", lw=0.8); ax.set_axisbelow(True)
	for sp in ("top", "right"): ax.spines[sp].set_visible(False)
	for sp in ("left", "bottom"): ax.spines[sp].set_color("#c3c2b7")
	ax.tick_params(colors="#6b6a63", length=3)
	handles = [plt.Line2D([], [], marker="o", color="#2a78d6", ls="", ms=8, label=f"solo PIV ({n_s} of 30 graded)"),
	plt.Line2D([], [], marker="D", markerfacecolor="none", markeredgecolor="#eb6834", markeredgewidth=1.8, color="#eb6834", ls="", ms=8, label=f"team PIV ({n_t} of 30 graded)")]
	ax.legend(handles=handles, loc="lower left", bbox_to_anchor=(0.0, 1.07), ncol=2, frameon=False, fontsize=9, borderaxespad=0)
	ax.set_title("Flint on ProgramBench, Balanced-30: solo PIV vs team PIV" + (" (grading in progress; a missing dot is not yet graded)" if n_s + n_t < 60 else ""), fontsize=10, color="#1a1a19", loc="left", pad=34)
	fig.tight_layout(); out = f"{E}/results/scores.png"; fig.savefig(out, facecolor=fig.get_facecolor()); print("wrote", out, "\| graded: solo", n_s, "team", n_t)

	if __name__ == "__main__":
	main()