File size: 3,591 Bytes
a484e22 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 | """Scoped wall-clock micro-benchmark (external-review item 3).
Not a production deployment study. Measures, on the real infra:
(a) draft-side overhead: embed(query) + nearest-neighbor retrieval against a
realistically-sized personal store (~48 entries) — timed on this CPU node;
(b) target-side decode speed of the served gpt-oss-120b: per-token decode
time from paired short/long generations (sequential, unbatched);
then compares mean retrieval overhead against the decode time represented by
the accepted-token gap (personal 19.96 vs static 15.70 MAT, headline run).
Writes results/wallclock_micro.json.
"""
from __future__ import annotations
import json
import time
from pathlib import Path
import numpy as np
import requests
from .data import load_bfcl
from .memory import Embedder, _best_match, Entry
ROOT = Path(__file__).resolve().parent.parent
RESULTS = ROOT / "results"
URL = "http://localhost:30000/v1/chat/completions"
N_RETRIEVAL = 200
N_DECODE_PAIRS = 30
def main():
tasks = load_bfcl()[:N_RETRIEVAL]
emb = Embedder()
# warm the embedder, then build a 48-entry store
store = [Entry(emb.embed(t.query), "x") for t in tasks[:48]]
for t in tasks[:20]:
emb.embed(t.query + " warm")
lat = []
for t in tasks:
q = t.query + " ?" # bust the embed cache
t0 = time.perf_counter()
e = emb.embed(q)
_best_match(e, store)
lat.append((time.perf_counter() - t0) * 1000)
lat.sort()
retr = {"n": len(lat), "mean_ms": round(float(np.mean(lat)), 2),
"median_ms": round(lat[len(lat) // 2], 2),
"p95_ms": round(lat[int(0.95 * len(lat))], 2)}
def timed_gen(prompt, max_tok):
t0 = time.perf_counter()
r = requests.post(URL, json={
"model": "gpt-oss-120b", "temperature": 0.0,
"max_tokens": max_tok,
"messages": [{"role": "user", "content": prompt}]}, timeout=120)
dt = time.perf_counter() - t0
u = r.json().get("usage", {})
return dt, u.get("completion_tokens", max_tok)
per_tok = []
for t in tasks[:N_DECODE_PAIRS]:
p = "Write a long detailed paragraph about: " + t.query[:200]
d1, n1 = timed_gen(p, 8)
d2, n2 = timed_gen(p, 168)
if n2 > n1:
per_tok.append((d2 - d1) / (n2 - n1) * 1000)
per_tok.sort()
dec = {"n_pairs": len(per_tok),
"per_token_ms_median": round(per_tok[len(per_tok) // 2], 2),
"per_token_ms_mean": round(float(np.mean(per_tok)), 2)}
ptok = dec["per_token_ms_median"]
out = {
"retrieval_overhead": retr,
"decode": dec,
"comparison": {
"personal_MAT_19.96_decode_ms": round(19.96 * ptok, 1),
"static_MAT_15.70_decode_ms": round(15.70 * ptok, 1),
"gap_4.26_tokens_decode_ms": round(4.26 * ptok, 1),
"retrieval_overhead_mean_ms": retr["mean_ms"],
"overhead_over_gap_pct": round(
100 * retr["mean_ms"] / (4.26 * ptok), 1),
},
"caveats": ("Sequential unbatched decode on the live server (batching "
"changes per-token time); retrieval timed on the CPU node,"
" single-threaded; no end-to-end speculative decoder is "
"integrated — this bounds the overhead-vs-savings ratio, "
"not deployed speedup."),
}
(RESULTS / "wallclock_micro.json").write_text(json.dumps(out, indent=2))
print(json.dumps(out, indent=2))
if __name__ == "__main__":
main()
|