"""Compare a fine-tuned model against the base gpt-5-nano baseline. Runs the exact same 10-record smoke eval used in `run_multimodel_benchmark.py` against two models — the production baseline (gpt-5-nano @ minimal) and the fine-tuned model whose id you pass with `--ft-model`. Produces a side-by-side comparison table. The interesting output is not just "which one has higher F1" — it\'s also the cost delta. Fine-tuned models bill at higher per-token rates than base (gpt-4o-mini fine-tunes cost ~$0.30/$1.20 per 1M in/out tokens vs. $0.15/$0.60 for base). If the base beats the fine-tune on F1, or ties within the noise band, the fine-tune isn\'t worth shipping. Usage ----- python scripts/compare_finetune.py --ft-model ft:gpt-4o-mini:aditya-p:receipts:abc123 """ from __future__ import annotations import argparse import csv import json import subprocess import sys from datetime import datetime, timezone from pathlib import Path ROOT = Path(__file__).resolve().parents[1] DATASETS = [ ("receipt", ROOT / "evaluation" / "smoke_sroie_sample.jsonl", "sroie"), ("receipt", ROOT / "evaluation" / "smoke_cord_sample.jsonl", "cord"), ] def run_one_eval(model: str, doc_type: str, dataset: Path, out_dir: Path, reasoning_effort: str | None = None) -> dict: """Fire the eval CLI once. Returns the summary dict.""" cmd = [ sys.executable, "-m", "src.eval.cli", "--dataset", str(dataset), "--doc-type", doc_type, "--mode", "live", "--model", model, "--output-dir", str(out_dir), ] if reasoning_effort: cmd += ["--reasoning-effort", reasoning_effort] cmd_str = " ".join(cmd) print(f" $ {cmd_str}", flush=True) r = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True) if r.returncode != 0: print(r.stdout) print(r.stderr, file=sys.stderr) raise SystemExit(f"eval CLI failed: rc={r.returncode}") summary_paths = sorted(out_dir.glob("*_summary.json")) if not summary_paths: raise RuntimeError(f"no summary.json in {out_dir}") with summary_paths[-1].open() as f: return json.load(f)["summary"] def aggregate(rows: list[dict]) -> dict: """Weighted aggregate of per-dataset runs into one row.""" n = sum(r["n_docs"] for r in rows) if n == 0: return {} def w(k): return sum(r[k] * r["n_docs"] for r in rows) / n return { "n_docs": n, "errors": sum(r["errors"] for r in rows), "micro_f1": round(w("micro_f1"), 4), "macro_f1": round(w("macro_f1"), 4), "doc_exact_match": round(w("doc_exact_match"), 4), "mean_latency_ms": round(w("mean_latency_ms"), 0), "mean_cost_usd": round(w("mean_cost_usd"), 6), "total_cost_usd": round(sum(r["total_cost_usd"] for r in rows), 4), } def write_markdown(rows: list[dict], out: Path) -> Path: lines = [ "# Fine-tuning comparison", "", f"_Generated: {datetime.now(timezone.utc).isoformat(timespec='seconds')}_", "", "10 receipts (5 SROIE + 5 CORD), same prompts, same schemas, same", "eval harness. Only the model changes.", "", "| Model | Micro F1 | Macro F1 | Doc-exact | Latency | Cost / doc |", "|---|---:|---:|---:|---:|---:|", ] for r in rows: label = r["label"] mf1 = r["micro_f1"] mac = r["macro_f1"] de = r["doc_exact_match"] lat = r["mean_latency_ms"] cd = r["mean_cost_usd"] lines.append( f"| `{label}` | {mf1:.3f} | {mac:.3f} | {de:.0%} | {lat:.0f} ms | ${cd:.5f} |" ) lines.append("") lines.append("**Read the numbers:** if the fine-tuned F1 is within noise (a few points)") lines.append("of the base and its cost/doc is higher, do NOT ship the fine-tune —") lines.append("ongoing cost + schema lock-in isn\'t justified. If F1 is materially") lines.append("higher and cost is comparable, the fine-tune is a real win.") out.write_text("\n".join(lines), encoding="utf-8") return out def main(argv=None) -> int: ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--ft-model", required=True, help="Fine-tuned model id, e.g. ft:gpt-4o-mini:you:receipts:abc123") ap.add_argument("--base-model", default="gpt-5-nano", help="Baseline model. Default: gpt-5-nano (our production choice).") ap.add_argument("--base-effort", default="minimal", help="Reasoning effort for the base model. Default: minimal.") args = ap.parse_args(argv) stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") out_root = ROOT / "evaluation" / "finetuning" / stamp out_root.mkdir(parents=True, exist_ok=True) print(f"Fine-tuning comparison ({out_root.relative_to(ROOT)})") print(f" base: {args.base_model} (effort={args.base_effort})") print(f" fine-tune: {args.ft_model}") print() matrix = [ (args.base_model, args.base_effort, args.base_model), (args.ft_model, None, "fine-tuned"), ] rollup: list[dict] = [] for model, effort, label in matrix: print(f"=== {label} ({model}) ===") per_dataset: list[dict] = [] for doc_type, dataset, tag in DATASETS: run_dir = out_root / f"{label}_{tag}" run_dir.mkdir(parents=True, exist_ok=True) summary = run_one_eval(model, doc_type, dataset, run_dir, reasoning_effort=effort) per_dataset.append(summary) mf1 = summary["micro_f1"] cd = summary["mean_cost_usd"] print(f" [{tag}] micro_f1={mf1:.3f} cost/doc=${cd:.5f}") agg = aggregate(per_dataset) agg["label"] = label rollup.append(agg) print() # Roll-up files. (out_root / "comparison.json").write_text( json.dumps({ "generated_at": datetime.now(timezone.utc).isoformat(), "base_model": args.base_model, "ft_model": args.ft_model, "rows": rollup, }, indent=2) ) with (out_root / "comparison.csv").open("w", newline="") as f: cols = ["label", "micro_f1", "macro_f1", "doc_exact_match", "mean_latency_ms", "mean_cost_usd", "total_cost_usd", "n_docs"] w = csv.DictWriter(f, fieldnames=cols) w.writeheader() for r in rollup: w.writerow({c: r.get(c, "") for c in cols}) write_markdown(rollup, out_root / "comparison.md") print(f"Done. Comparison in {out_root.relative_to(ROOT)}/comparison.{{md,csv,json}}") print("\n" + (out_root / "comparison.md").read_text()) return 0 if __name__ == "__main__": raise SystemExit(main())