File size: 4,330 Bytes
a484e22 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 | """Bootstrap/paired CIs + variance decomposition (external-review item 5).
Deterministically replays the phase-2 seed-0 stream against CACHED targets
(no model calls) to obtain per-instance paired (static, personal) accepts,
then reports:
- paired bootstrap 95% CI for the post-warmup MAT gap and % gap
- per-user, per-session, per-signature-task gap distributions
Writes results/phase2_bootstrap_ci.json.
"""
from __future__ import annotations
import json
import random
from collections import defaultdict
from pathlib import Path
from . import metrics
from .data import load_bfcl
from .memory import Embedder, PersonalMemory, StaticGlobal
from .run_accept import MODEL_PATH, _parse_target
from .simulate import build_users
ROOT = Path(__file__).resolve().parent.parent
RESULTS = ROOT / "results"
def main():
metrics.get_tokenizer(MODEL_PATH)
tasks = load_bfcl()
embedder = Embedder()
instances = build_users(tasks, n_users=40, tasks_per_user=15,
n_sessions=12, queries_per_session=6, seed=0)
instances.sort(key=lambda x: (x.session, x.user_id))
targets = json.loads((RESULTS / "phase2_targets_seed0.json").read_text())
static, personal = StaticGlobal(), PersonalMemory(capacity=48,
eviction="lru")
rows = [] # (user, session, sig, a_static, a_personal)
cur = -1
for ins in instances:
tgt = targets.get(ins.query)
if tgt is None:
continue
if ins.session != cur:
cur = ins.session
if cur == 1:
static.freeze()
a_s = metrics.score(static.draft(ins.query, ins.functions,
ins.user_id, embedder),
tgt)["accept_length"]
a_p = metrics.score(personal.draft(ins.query, ins.functions,
ins.user_id, embedder),
tgt)["accept_length"]
rows.append((ins.user_id, ins.session, ins.signature_id, a_s, a_p))
cname, cargs = _parse_target(tgt)
for a in (static, personal):
a.observe(ins.query, ins.functions, ins.user_id, cname, cargs,
embedder)
if ins.session == 0:
personal.seed_shared(ins.query, cname, cargs, embedder)
post = [r for r in rows if r[1] > 0]
n = len(post)
mat_s = sum(r[3] for r in post) / n
mat_p = sum(r[4] for r in post) / n
rng = random.Random(0)
B = 10_000
gaps, pct = [], []
for _ in range(B):
idx = [rng.randrange(n) for _ in range(n)]
s = sum(post[i][3] for i in idx) / n
p = sum(post[i][4] for i in idx) / n
gaps.append(p - s)
pct.append(100 * (p - s) / s)
gaps.sort(); pct.sort()
ci = lambda xs: (round(xs[int(0.025 * B)], 3), round(xs[int(0.975 * B)], 3))
def group_gaps(key):
g = defaultdict(lambda: [0.0, 0.0, 0])
for r in post:
k = key(r); g[k][0] += r[3]; g[k][1] += r[4]; g[k][2] += 1
vals = sorted((v[1] - v[0]) / v[2] for v in g.values())
m = len(vals)
return {"n_groups": m,
"mean_gap": round(sum(vals) / m, 3),
"min": round(vals[0], 3), "p25": round(vals[m // 4], 3),
"median": round(vals[m // 2], 3),
"p75": round(vals[3 * m // 4], 3), "max": round(vals[-1], 3),
"groups_with_negative_gap": sum(1 for v in vals if v < 0)}
out = {
"config": {"seed": 0, "targets": "phase2_targets_seed0.json (cached)",
"n_post_warmup_paired": n, "bootstrap_resamples": B},
"MAT": {"static": round(mat_s, 3), "personal": round(mat_p, 3),
"gap": round(mat_p - mat_s, 3),
"gap_pct": round(100 * (mat_p - mat_s) / mat_s, 2)},
"paired_bootstrap_95CI": {"gap_MAT": ci(gaps), "gap_pct": ci(pct)},
"per_user_gap": group_gaps(lambda r: r[0]),
"per_session_gap": group_gaps(lambda r: r[1]),
"per_task_gap": group_gaps(lambda r: r[2]),
}
(RESULTS / "phase2_bootstrap_ci.json").write_text(json.dumps(out,
indent=2))
print(json.dumps(out, indent=2))
if __name__ == "__main__":
main()
|