| """Regenerate results/REPORT.md from raw final benchmark files.""" |
| import json |
| import statistics |
| from pathlib import Path |
| from rlcd.benchmark import summarize |
| from rlcd.tasks import CASES |
| from rlcd.stress_tasks import CASES as STRESS_CASES |
|
|
| root = Path("results") |
| lines = ["# Measured benchmark results", "", "FP16 on all devices; unchanged pinned base weights; PyTorch eager attention and reference convolution kernels. Lower latency is better. Speedup is autoregressive mean / constrained mean. These are request latencies after warmup, excluding network, provisioning, model loading and validation.", "", "## Diagnostic suite", "", "12 hand-authored cases, 3 fields each; 2 warmups per case/method and 3 measured repetitions (36 requests per method). Repetitions measure timing variability, not independent accuracy samples.", "", "| Device | Method | Mean ms | Median ms | p95 ms | JSON valid | Schema compliant | Field accuracy | Exact objects |", "|---|---|---:|---:|---:|---:|---:|---:|---:|"] |
| for name in ["m2-max", "l40s", "h100"]: |
| d = json.loads((root / f"{name}.json").read_text()) |
| assert d["summary"] == summarize(d["rows"]) |
| for method in ["constrained", "autoregressive"]: |
| s = d["summary"][method] |
| lines.append(f'| {name} | {method} | {s["latency_mean_ms"]:.2f} | {s["latency_median_ms"]:.2f} | {s["latency_p95_ms"]:.2f} | {s["syntax_valid_rate"]:.1%} | {s["schema_compliant_rate"]:.1%} | {s["field_accuracy"]:.1%} | {s["exact_match_rate"]:.1%} |') |
| lines += ["", "| Device | Diagnostic speedup |", "|---|---:|"] |
| for name in ["m2-max", "l40s", "h100"]: |
| d = json.loads((root / f"{name}.json").read_text()) |
| lines.append(f'| {name} | {d["summary"]["speedup_ratio_of_mean_latency"]:.2f}× |') |
| lines += ["", "## Scaling probes", "", "Synthetic probes are reported individually, with 2 warmups and 3 measured repeats. The autoregressive token cap is 768 (versus 192 for diagnostics). Accuracy below is field accuracy; each probe is one unique example. This is not a broad high-cardinality accuracy evaluation. Both methods achieved 100% measured JSON syntax and schema compliance on all scaling probes. No diagnostic or scaling output hit its generation token cap.", "", "| Device | Probe | Constrained mean ms | AR mean ms | Speedup | Constrained / AR field accuracy |", "|---|---|---:|---:|---:|---:|"] |
| for name in ["m2-max", "l40s", "h100"]: |
| d = json.loads((root / f"{name}-stress.json").read_text()) |
| for case_id, *_ in STRESS_CASES: |
| rows = [r for r in d["rows"] if r["case"] == case_id] |
| s = summarize(rows) |
| c, a = s["constrained"], s["autoregressive"] |
| lines.append(f'| {name} | {case_id} | {c["latency_mean_ms"]:.2f} | {a["latency_mean_ms"]:.2f} | {s["speedup_ratio_of_mean_latency"]:.2f}× | {c["field_accuracy"]:.1%} / {a["field_accuracy"]:.1%} |') |
| lines += ["", "## Incorrect diagnostic outputs (first measured repeat)", "", "These show actual model decision errors even when the output passes the schema. Full outputs and candidate log-likelihoods are in the raw JSON files.", "", "| Device | Case | Method | Expected | Actual |", "|---|---|---|---|---|"] |
| expected = {c[0]: c[3] for c in CASES} |
| for name in ["m2-max", "l40s", "h100"]: |
| d = json.loads((root / f"{name}.json").read_text()) |
| for r in d["rows"]: |
| if r["repeat"] == 0 and not r["exact_match"]: |
| lines.append(f'| {name} | {r["case"]} | {r["method"]} | `{json.dumps(expected[r["case"]], ensure_ascii=False)}` | `{r["text"].replace(chr(10), " ")}` |') |
| lines += ["", "## Environment and limitations", ""] |
| for name in ["m2-max", "l40s", "h100"]: |
| d = json.loads((root / f"{name}.json").read_text()) |
| m = d["metadata"] |
| lines.append(f'- {name}: {m["hardware"]}; {m["platform"]}; Python {m["python"]}; CUDA runtime {m["cuda_runtime"]}; {m.get("gpu_info", "12 CPU cores, 30 GPU cores, 32 GiB unified memory")}.') |
| lines += ["", "All runs use torch 2.14.0, transformers 5.17.0, FP16 weights/activations and FP32 log-softmax accumulation. No quantization, FlashAttention, torch.compile, optimized causal-conv1d, MLX, or serving framework is enabled. GPU matmul TF32 is disabled. This controlled implementation comparison is not the maximum performance of each device or a comparison against optimized grammar-constrained serving.", "", "P95 estimates are descriptive with only 36 diagnostic measurements. The local Mac was not thermally isolated or reserved exclusively for this job; allocator and system contention can affect larger probes. Candidate batch expansion grows memory with context length and candidate count. No request throughput, concurrency, OOM boundary, cold-start, energy, cost, calibration, or production-distribution study is claimed.", "", "Accuracy uses fixed hand-authored labels. The test cases were not changed to improve scores. The prompted token sequence differs after the common prefix: the constrained branches place each field first, while autoregressive decoding conditions later fields on earlier generated values. Independent decisions may violate relationships not expressible by the supported flat schema. Both methods get every allowed value in the same shared schema.", "", "Syntax is strict json.loads over the entire reconstructed answer, without extracting braces or stripping code fences. Schema checks use Draft202012Validator (required fields, types, enums, no additional properties). Invalid/missing values count as wrong; exact match requires compliance and all fields correct. Syntax/schema guarantees for the constrained method come from programmatic assembly, not learned reliability.", "", "Raw JSON files contain every measured output, latency, score, token count, dependency versions and source hashes. Earlier development runs are kept separately under exploratory/ and are not used in the tables above."] |
| (root / "REPORT.md").write_text("\n".join(lines) + "\n") |
|
|
