Spaces:
Running
Running
Update logbook: Repro — CapBencher: Give Your LLM Benchmark a Built-in Alarm for Test-Set Overfitting (formerly “How Can I Publish My LLM Benchmark Without Giving the True Answers Away?”)
Browse files- README.md +2 -2
- index.html +1 -1
- logbook.json +43 -6
- pages/claim-a-detecting-deliberate-contamination/page.md +73 -0
- pages/claim-b-the-bayes-accuracy-cap/page.md +105 -0
- pages/conclusion/page.md +0 -0
- pages/further-evidence-ranking-and-merge/page.md +615 -0
- pages/index.md +9 -4
- pages/methods-and-provenance/page.md +301 -0
- pages/run-history-and-compute/page.md +0 -0
README.md
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
---
|
| 2 |
-
title: "Repro: How Can I Publish My LLM Benchmark Without Giving the True Answers Away?"
|
| 3 |
emoji: 🎯
|
| 4 |
colorFrom: yellow
|
| 5 |
colorTo: red
|
|
@@ -13,6 +13,6 @@ tags:
|
|
| 13 |
- paper-oCNT5PcMSQ
|
| 14 |
---
|
| 15 |
|
| 16 |
-
# Repro: How Can I Publish My LLM Benchmark Without Giving the True Answers Away?
|
| 17 |
|
| 18 |
An open experiment logbook, published with [Trackio](https://github.com/gradio-app/trackio).
|
|
|
|
| 1 |
---
|
| 2 |
+
title: "Repro — CapBencher: Give Your LLM Benchmark a Built-in Alarm for Test-Set Overfitting (formerly “How Can I Publish My LLM Benchmark Without Giving the True Answers Away?”)"
|
| 3 |
emoji: 🎯
|
| 4 |
colorFrom: yellow
|
| 5 |
colorTo: red
|
|
|
|
| 13 |
- paper-oCNT5PcMSQ
|
| 14 |
---
|
| 15 |
|
| 16 |
+
# Repro — CapBencher: Give Your LLM Benchmark a Built-in Alarm for Test-Set Overfitting (formerly “How Can I Publish My LLM Benchmark Without Giving the True Answers Away?”)
|
| 17 |
|
| 18 |
An open experiment logbook, published with [Trackio](https://github.com/gradio-app/trackio).
|
index.html
CHANGED
|
@@ -3,7 +3,7 @@
|
|
| 3 |
<head>
|
| 4 |
<meta charset="utf-8" />
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
| 6 |
-
<title>Repro: How Can I Publish My LLM Benchmark Without Giving the True Answers Away?</title>
|
| 7 |
<link rel="stylesheet" href="./logbook.css" />
|
| 8 |
</head>
|
| 9 |
<body>
|
|
|
|
| 3 |
<head>
|
| 4 |
<meta charset="utf-8" />
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
| 6 |
+
<title>Repro — CapBencher: Give Your LLM Benchmark a Built-in Alarm for Test-Set Overfitting (formerly “How Can I Publish My LLM Benchmark Without Giving the True Answers Away?”)</title>
|
| 7 |
<link rel="stylesheet" href="./logbook.css" />
|
| 8 |
</head>
|
| 9 |
<body>
|
logbook.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
"schema_version": 1,
|
| 3 |
-
"title": "Repro: How Can I Publish My LLM Benchmark Without Giving the True Answers Away?",
|
| 4 |
"emoji": "🎯",
|
| 5 |
"space_id": "Boopster/oCNT5PcMSQ",
|
| 6 |
"paper": {
|
|
@@ -10,13 +10,50 @@
|
|
| 10 |
"icml2026-repro",
|
| 11 |
"paper-oCNT5PcMSQ"
|
| 12 |
],
|
| 13 |
-
"updated_at": "2026-07-
|
| 14 |
"root": {
|
| 15 |
"slug": "index",
|
| 16 |
-
"title": "Repro: How Can I Publish My LLM Benchmark Without Giving the True Answers Away?",
|
| 17 |
"file": "pages/index.md",
|
| 18 |
-
"children": [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
},
|
| 20 |
-
"agent_view_tokens":
|
| 21 |
-
"revision": "
|
| 22 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"schema_version": 1,
|
| 3 |
+
"title": "Repro — CapBencher: Give Your LLM Benchmark a Built-in Alarm for Test-Set Overfitting (formerly “How Can I Publish My LLM Benchmark Without Giving the True Answers Away?”)",
|
| 4 |
"emoji": "🎯",
|
| 5 |
"space_id": "Boopster/oCNT5PcMSQ",
|
| 6 |
"paper": {
|
|
|
|
| 10 |
"icml2026-repro",
|
| 11 |
"paper-oCNT5PcMSQ"
|
| 12 |
],
|
| 13 |
+
"updated_at": "2026-07-17T23:21:31+00:00",
|
| 14 |
"root": {
|
| 15 |
"slug": "index",
|
| 16 |
+
"title": "Repro — CapBencher: Give Your LLM Benchmark a Built-in Alarm for Test-Set Overfitting (formerly “How Can I Publish My LLM Benchmark Without Giving the True Answers Away?”)",
|
| 17 |
"file": "pages/index.md",
|
| 18 |
+
"children": [
|
| 19 |
+
{
|
| 20 |
+
"slug": "claim-a-detecting-deliberate-contamination",
|
| 21 |
+
"title": "Official claim 1 — detecting deliberate contamination",
|
| 22 |
+
"file": "pages/claim-a-detecting-deliberate-contamination/page.md",
|
| 23 |
+
"children": []
|
| 24 |
+
},
|
| 25 |
+
{
|
| 26 |
+
"slug": "claim-b-the-bayes-accuracy-cap",
|
| 27 |
+
"title": "Official claim 2 — the Bayes-accuracy cap",
|
| 28 |
+
"file": "pages/claim-b-the-bayes-accuracy-cap/page.md",
|
| 29 |
+
"children": []
|
| 30 |
+
},
|
| 31 |
+
{
|
| 32 |
+
"slug": "conclusion",
|
| 33 |
+
"title": "Conclusion",
|
| 34 |
+
"file": "pages/conclusion/page.md",
|
| 35 |
+
"children": []
|
| 36 |
+
},
|
| 37 |
+
{
|
| 38 |
+
"slug": "further-evidence-ranking-and-merge",
|
| 39 |
+
"title": "Further evidence — BF16 ranking ladder and merge detection",
|
| 40 |
+
"file": "pages/further-evidence-ranking-and-merge/page.md",
|
| 41 |
+
"children": []
|
| 42 |
+
},
|
| 43 |
+
{
|
| 44 |
+
"slug": "methods-and-provenance",
|
| 45 |
+
"title": "Methods and provenance",
|
| 46 |
+
"file": "pages/methods-and-provenance/page.md",
|
| 47 |
+
"children": []
|
| 48 |
+
},
|
| 49 |
+
{
|
| 50 |
+
"slug": "run-history-and-compute",
|
| 51 |
+
"title": "Run history and compute",
|
| 52 |
+
"file": "pages/run-history-and-compute/page.md",
|
| 53 |
+
"children": []
|
| 54 |
+
}
|
| 55 |
+
]
|
| 56 |
},
|
| 57 |
+
"agent_view_tokens": 19484,
|
| 58 |
+
"revision": "1784330491786477000"
|
| 59 |
}
|
pages/claim-a-detecting-deliberate-contamination/page.md
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Official claim 1 — detecting deliberate contamination
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
---
|
| 5 |
+
<!-- trackio-cell
|
| 6 |
+
{"type": "markdown", "id": "cell_f92361869465", "created_at": "2026-07-15T17:28:20+00:00", "title": "Claim and test"}
|
| 7 |
+
-->
|
| 8 |
+
**Official challenge claim (verbatim):** “Can accurately detect data contamination or hacking across wide range of benchmarks, models, and training methodologies.”
|
| 9 |
+
|
| 10 |
+
Our scoped test uses the cap mechanism to detect deliberate exposure: independent random target choices cap clean-model accuracy at 50%, while a model exposed to the stored targets can exceed the cap. We compare the same pinned BF16 Qwen2.5-3B lineage before training and after the preregistered eighth epoch on all 1,319 capped GSM8K rows.
|
| 11 |
+
|
| 12 |
+
**Challenge scope label: `toy` relative to the official breadth claim.** This is a substantive full-dataset A100 experiment, but it covers one benchmark, one deliberately exposed model and one training family rather than the claimed wide range of benchmarks, models and methodologies.
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
---
|
| 16 |
+
<!-- trackio-cell
|
| 17 |
+
{"type": "markdown", "id": "cell_aa803bad0b5c", "created_at": "2026-07-16T15:02:00+00:00", "title": "What the paper reports"}
|
| 18 |
+
-->
|
| 19 |
+
The paper's contamination experiments (Figure 4; Figure 13 and Appendix C) intentionally train several models on phished benchmark data, then apply a one-sided binomial test at α = 0.05. It reports above-cap detection across all tested models and benchmarks and says larger models need fewer epochs. Our scoped adaptation matches GSM8K, its full 1,319 rows, Qwen2.5-3B, the 50% null and an eight-epoch exposure; it does not reproduce the paper's breadth.
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
---
|
| 23 |
+
<!-- trackio-cell
|
| 24 |
+
{"type": "markdown", "id": "cell_a743d16c93c5", "created_at": "2026-07-16T15:02:00+00:00", "title": "Fixed decision rule and two formal implementations"}
|
| 25 |
+
-->
|
| 26 |
+
The statistical rule was fixed before V4 and retained for V5: clean below 690/1,319, epoch 8 at least 690, and significant paired improvement. V5 freshly generated checkpoint 0 and checkpoint 8 in one A100 Job from the same model object; checkpoint 0 was verified byte-identical to the preregistered V4 clean artifact. Both endpoints use greedy cache-on generation and the same strict final `#### <integer>` parser.
|
| 27 |
+
|
| 28 |
+
V4 ([Job `6a58edd9…`](https://huggingface.co/jobs/Boopster/6a58edd9b1669a49bf077ba6)) trained raw prompt/completion text and needed hash-audited recovery after a telemetry-only failure. V5 ([Job `6a5a584e…`](https://huggingface.co/jobs/Boopster/6a5a584ebee6ee1cf4ecd9ed)) changed training-record rendering to Qwen's native chat template; revisions, seed, scorer and boundary stayed pinned.
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
---
|
| 32 |
+
<!-- trackio-cell
|
| 33 |
+
{"type": "markdown", "id": "cell_71f5d8c2a904", "created_at": "2026-07-17T17:28:00+00:00", "title": "Measured V4/V5 result — the alarm fired in the chat-aligned sensitivity"}
|
| 34 |
+
-->
|
| 35 |
+
| Endpoint | Correct | Capped accuracy | Strict-invalid |
|
| 36 |
+
| --- | ---: | ---: | ---: |
|
| 37 |
+
| Clean, byte-identical in V4/V5 | 98/1,319 | 7.43% | 1,057 |
|
| 38 |
+
| V4 exposed, raw training format | 258/1,319 | 19.56% | 987 |
|
| 39 |
+
| V5 exposed, chat-aligned | **1,263/1,319** | **95.75%** | **0** |
|
| 40 |
+
|
| 41 |
+
V4's alarm stayed silent because only 332 outputs were parseable. V5 exceeded the fixed line by 573 answers (`p = 2.13×10⁻²⁹⁸`); 1,167 rows improved and 2 regressed (exact paired tail ≈8.53×10⁻³⁴⁷, below binary64 range). The hash-bound statistics record says `alarm_fired = true` and `controlled_replication_support = supported` after 8/8 epochs and 792/792 steps, with zero retries. The network-free audit independently verified the 43-artifact chain, recomputed the central verdict and sensitivity, and confirmed the byte-identical clean baseline.
|
| 42 |
+
|
| 43 |
+
Across exposed endpoints, validity rose from 25.17% in V4 to 100% in V5 (+74.83 points; 987 improvements, 0 regressions, `p = 7.65×10⁻²⁹⁸`). V5's rendered sequences also packed into 792 optimizer steps rather than V4's 712, so this contrast does not isolate a pure chat-template causal effect. It supports the alarm mechanism in one benchmark/model/training setup and shows material formatting-and-packing sensitivity. It does **not** establish the official claim's breadth across many benchmarks, models and methods.
|
| 44 |
+
|
| 45 |
+
The prespecified data-quality sensitivity excludes the 14 published targets
|
| 46 |
+
affected by the authors' comma-parsing behavior. V5 still scores
|
| 47 |
+
1,251/1,305 (95.86%) with zero strict-invalid outputs, well above the
|
| 48 |
+
recomputed 683/1,305 alarm boundary, so the scoped conclusion is unchanged.
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
---
|
| 52 |
+
<!-- trackio-cell
|
| 53 |
+
{"type": "figure", "id": "cell_d82bc136a4f7", "created_at": "2026-07-17T17:29:00+00:00", "title": "Figure: one fixed tripwire across clean, V4 and V5"}
|
| 54 |
+
-->
|
| 55 |
+
````html
|
| 56 |
+
<svg viewBox="0 0 680 292" xmlns="http://www.w3.org/2000/svg">
|
| 57 |
+
<text x="10" y="24" style="font:700 15px Georgia,serif" fill="#24355e">One fixed alarm line; one preregistered formatting intervention</text>
|
| 58 |
+
<text x="10" y="68" style="font:600 12px monospace" fill="#24355e">clean · identical</text><rect x="215" y="52" width="420" height="24" rx="5" fill="#edf2fa"/><rect x="215" y="52" width="31.2" height="24" rx="5" fill="#3d87d8"/><text x="254" y="68" style="font:600 11.5px monospace" fill="#24355e">98 · 7.4%</text>
|
| 59 |
+
<text x="10" y="120" style="font:600 12px monospace" fill="#24355e">V4 · raw format</text><rect x="215" y="104" width="420" height="24" rx="5" fill="#edf2fa"/><rect x="215" y="104" width="82.2" height="24" rx="5" fill="#c9972b"/><text x="305" y="120" style="font:600 11.5px monospace" fill="#24355e">258 · 19.6%</text>
|
| 60 |
+
<text x="10" y="172" style="font:600 12px monospace" fill="#24355e">V5 · chat aligned</text><rect x="215" y="156" width="420" height="24" rx="5" fill="#edf2fa"/><rect x="215" y="156" width="402.2" height="24" rx="5" fill="#1f9d73"/><text x="505" y="172" style="font:700 11.5px monospace" fill="#fff">1,263 · 95.8%</text>
|
| 61 |
+
<line x1="434.7" y1="42" x2="434.7" y2="192" stroke="#24355e" stroke-width="2"/><text x="434.7" y="210" text-anchor="middle" style="font:600 11px monospace" fill="#24355e">fixed tripwire · 690/1,319</text>
|
| 62 |
+
<text x="10" y="244" style="font:500 11.5px Georgia,serif" fill="#555">Validity: 25.2% in exposed V4 → 100% in exposed V5. Clean endpoint bytes unchanged.</text>
|
| 63 |
+
<text x="10" y="264" style="font:500 11.5px Georgia,serif" fill="#555">V5 crossed the line by 573 answers; 1,167 rows improved and 2 regressed.</text>
|
| 64 |
+
</svg>
|
| 65 |
+
````
|
| 66 |
+
|
| 67 |
+
````raw
|
| 68 |
+
run,endpoint,correct,trials,capped_accuracy,strict_invalid,valid_outputs,alarm_threshold
|
| 69 |
+
formal_v4_v5,checkpoint_0_clean,98,1319,0.07429871114480667,1057,262,690
|
| 70 |
+
formal_v4,checkpoint_8_raw_format,258,1319,0.1956027293404094,987,332,690
|
| 71 |
+
formal_v5,checkpoint_8_chat_aligned,1263,1319,0.9575435936315391,0,1319,690
|
| 72 |
+
|
| 73 |
+
````
|
pages/claim-b-the-bayes-accuracy-cap/page.md
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Official claim 2 — the Bayes-accuracy cap
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
---
|
| 5 |
+
<!-- trackio-cell
|
| 6 |
+
{"type": "markdown", "id": "cell_f61ed3808f63", "created_at": "2026-07-15T17:28:21+00:00", "title": "Claim and test"}
|
| 7 |
+
-->
|
| 8 |
+
**Official challenge claim (verbatim):** “Fully capable models should not surpass the Bayes accuracy; surpassing it signals leakage or gaming.”
|
| 9 |
+
|
| 10 |
+
We test both directions of that claim and its stated decision machinery. Above-cap performance is tested with a one-sided exact binomial test rather than an asymptotic approximation; for 1,319 examples and a 50 percent null, 690 correct is significant and 689 is not. A capable unexposed 7B model supplies the below-cap witness, while the deliberately exposed V5 model on [Official claim 1](#/claim-a-detecting-deliberate-contamination) supplies the above-cap leakage witness.
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
---
|
| 14 |
+
<!-- trackio-cell
|
| 15 |
+
{"type": "markdown", "id": "cell_3e7f0c8a9b21", "created_at": "2026-07-16T15:02:00+00:00", "title": "What the paper reports"}
|
| 16 |
+
-->
|
| 17 |
+
The paper's abstract states the claim directly: "In principle, even fully capable models should not surpass the Bayes accuracy. If a model surpasses this ceiling despite this expectation, this is a strong signal of data contamination." Section 3 defines the detection machinery, and the baseline-comparison section states that "our binomial test computes exact p-values (see Section 3) rather than relying on asymptotic approximations, making it valid even for small samples." The paper uses GSM8K with n = 1,319 (Section 3 variance example; appendix size table). The exact numeric threshold is implied by the test definition rather than printed, which makes an independent recomputation a genuine check.
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
---
|
| 21 |
+
<!-- trackio-cell
|
| 22 |
+
{"type": "markdown", "id": "cell_bb7b62e1e3ae", "created_at": "2026-07-15T17:57:45+00:00", "title": "Retired mutable path reference: early model metrics"}
|
| 23 |
+
-->
|
| 24 |
+
This auto-captured cell originally described 335 bytes at `outputs/analysis/model_metrics.csv`. Later analyses overwrote that mutable path and no immutable 335-byte copy survives, so the path reference is retired and is **not evidence**. The final five-model metrics are preserved separately under the immutable ranking run directory.
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
---
|
| 28 |
+
<!-- trackio-cell
|
| 29 |
+
{"type": "markdown", "id": "cell_4902e3ed439d", "created_at": "2026-07-15T18:52:28+00:00", "title": "Verified exact boundary"}
|
| 30 |
+
-->
|
| 31 |
+
For n=1,319 and the preregistered one-sided exact binomial test against p0=0.5, the smallest significant count is 690: accuracy 52.3124%, p=0.0492416. A count of 689 gives p=0.0551151 and is not significant at alpha=0.05. This reproduces the stated boundary exactly.
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
---
|
| 35 |
+
<!-- trackio-cell
|
| 36 |
+
{"type": "markdown", "id": "cell_a68783032f49", "created_at": "2026-07-16T15:02:00+00:00", "title": "A capable unexposed BF16 model stays below the cap"}
|
| 37 |
+
-->
|
| 38 |
+
The same-job BF16 ladder evaluated five pinned Qwen2.5 checkpoints on all 1,319 original and capped GSM8K rows (13,190 generations). Its strongest model, Qwen2.5-7B-Instruct, scored 1,150/1,319 = **87.2%** on original GSM8K with 94.5% strict-format validity, yet only 481/1,319 = **36.5%** against the published capped targets (exact 95% CI 33.9–39.1%; one-sided above-cap p = 1.0). It also stayed below 50% on both frozen synthetic target assignments, and all 15 model-by-assignment cap tests had Holm-adjusted p = 1.0. This is substantive support for the claim's below-cap prediction in one benchmark/model family; a large upper-tail p-value means no evidence of exceeding the cap, not proof of a universal ceiling. The complementary exposure test reached **1,263/1,319 = 95.75%** and crossed the fixed alarm with zero strict-invalid outputs; its full paired evidence and scope limits are on [Official claim 1](#/claim-a-detecting-deliberate-contamination). The complete five-model table and inconclusive ranking test are on [Further evidence](#/further-evidence-ranking-and-merge).
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
---
|
| 42 |
+
<!-- trackio-cell
|
| 43 |
+
{"type": "figure", "id": "cell_9a7d9365cc61", "created_at": "2026-07-16T14:21:33+00:00", "title": "Figure: the significance cliff at 690/1,319"}
|
| 44 |
+
-->
|
| 45 |
+
````html
|
| 46 |
+
<svg viewBox="0 0 720 380" xmlns="http://www.w3.org/2000/svg" font-family="Georgia,serif">
|
| 47 |
+
<text x="360.0" y="22" text-anchor="middle" font-size="16" fill="#222">One-sided exact binomial p-value vs correct count (n = 1,319, null = 50%)</text>
|
| 48 |
+
<rect x="70" y="232.5" width="630" height="87.5" fill="#fdecea"/>
|
| 49 |
+
<line x1="70" y1="232.5" x2="700" y2="232.5" stroke="#c0392b" stroke-width="1.5" stroke-dasharray="6 4"/>
|
| 50 |
+
<text x="76" y="226.5" font-size="12" fill="#c0392b">α = 0.05</text>
|
| 51 |
+
<polyline points="70.0,40.0 85.8,40.0 101.5,40.0 117.2,40.0 133.0,40.0 148.8,40.0 164.5,40.0 180.2,40.0 196.0,40.0 211.8,61.5 227.5,83.1 243.2,103.4 259.0,122.5 274.8,140.4 290.5,157.0 306.2,172.5 322.0,186.8 337.8,200.1 353.5,212.3 369.2,223.5 385.0,233.8 400.8,243.2 416.5,251.8 432.2,259.5 448.0,266.5 463.8,272.8 479.5,278.5 495.2,283.6 511.0,288.2 526.8,292.3 542.5,295.9 558.2,299.1 574.0,301.9 589.8,304.4 605.5,306.6 621.2,308.5 637.0,310.1 652.8,311.6 668.5,312.8 684.2,313.9 700.0,314.9" fill="none" stroke="#456" stroke-width="2.5" stroke-linejoin="round"/>
|
| 52 |
+
<circle cx="369.25" cy="223.5485057140751" r="6" fill="#3d87d8" stroke="#fff" stroke-width="2"/>
|
| 53 |
+
<circle cx="385.0" cy="233.8272269093784" r="6" fill="#c0392b" stroke="#fff" stroke-width="2"/>
|
| 54 |
+
<text x="359.25" y="209.5485057140751" text-anchor="end" font-size="13" fill="#3d87d8">689 → p = 0.0551 (not significant)</text>
|
| 55 |
+
<text x="395.0" y="255.8272269093784" font-size="13" fill="#c0392b">690 → p = 0.0492 (flagged)</text>
|
| 56 |
+
<line x1="70" y1="320" x2="700" y2="320" stroke="#999"/>
|
| 57 |
+
<line x1="70" y1="40" x2="70" y2="320" stroke="#999"/><text x="70.0" y="340" text-anchor="middle" font-size="12" fill="#555">670</text><text x="227.5" y="340" text-anchor="middle" font-size="12" fill="#555">680</text><text x="385.0" y="340" text-anchor="middle" font-size="12" fill="#555">690</text><text x="542.5" y="340" text-anchor="middle" font-size="12" fill="#555">700</text><text x="700.0" y="340" text-anchor="middle" font-size="12" fill="#555">710</text><text x="62" y="324.0" text-anchor="end" font-size="12" fill="#555">0.00</text><text x="62" y="236.5" text-anchor="end" font-size="12" fill="#555">0.05</text><text x="62" y="149.0" text-anchor="end" font-size="12" fill="#555">0.10</text><text x="62" y="61.5" text-anchor="end" font-size="12" fill="#555">0.15</text><text x="385.0" y="364" text-anchor="middle" font-size="13" fill="#333">correct answers out of 1,319</text>
|
| 58 |
+
<text x="18" y="180.0" font-size="13" fill="#333" transform="rotate(-90 18 180.0)">p-value</text></svg>
|
| 59 |
+
````
|
| 60 |
+
|
| 61 |
+
````raw
|
| 62 |
+
correct,accuracy,one_sided_exact_p
|
| 63 |
+
670,0.507961,0.2909312
|
| 64 |
+
671,0.508719,0.2723453
|
| 65 |
+
672,0.509477,0.2543687
|
| 66 |
+
673,0.510235,0.2370341
|
| 67 |
+
674,0.510993,0.2203692
|
| 68 |
+
675,0.511751,0.2043966
|
| 69 |
+
676,0.512509,0.1891339
|
| 70 |
+
677,0.513268,0.1745937
|
| 71 |
+
678,0.514026,0.1607838
|
| 72 |
+
679,0.514784,0.1477071
|
| 73 |
+
680,0.515542,0.1353622
|
| 74 |
+
681,0.516300,0.1237435
|
| 75 |
+
682,0.517058,0.1128414
|
| 76 |
+
683,0.517817,0.1026426
|
| 77 |
+
684,0.518575,0.0931308
|
| 78 |
+
685,0.519333,0.0842864
|
| 79 |
+
686,0.520091,0.0760876
|
| 80 |
+
687,0.520849,0.0685103
|
| 81 |
+
688,0.521607,0.0615286
|
| 82 |
+
689,0.522365,0.0551151
|
| 83 |
+
690,0.523124,0.0492416
|
| 84 |
+
691,0.523882,0.0438788
|
| 85 |
+
692,0.524640,0.0389971
|
| 86 |
+
693,0.525398,0.0345670
|
| 87 |
+
694,0.526156,0.0305588
|
| 88 |
+
695,0.526914,0.0269433
|
| 89 |
+
696,0.527672,0.0236919
|
| 90 |
+
697,0.528431,0.0207769
|
| 91 |
+
698,0.529189,0.0181714
|
| 92 |
+
699,0.529947,0.0158496
|
| 93 |
+
700,0.530705,0.0137868
|
| 94 |
+
701,0.531463,0.0119599
|
| 95 |
+
702,0.532221,0.0103466
|
| 96 |
+
703,0.532980,0.0089263
|
| 97 |
+
704,0.533738,0.0076798
|
| 98 |
+
705,0.534496,0.0065892
|
| 99 |
+
706,0.535254,0.0056377
|
| 100 |
+
707,0.536012,0.0048103
|
| 101 |
+
708,0.536770,0.0040928
|
| 102 |
+
709,0.537528,0.0034726
|
| 103 |
+
710,0.538287,0.0029382
|
| 104 |
+
|
| 105 |
+
````
|
pages/conclusion/page.md
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
pages/further-evidence-ranking-and-merge/page.md
ADDED
|
@@ -0,0 +1,615 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Further evidence — BF16 ranking ladder and merge detection
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
---
|
| 5 |
+
<!-- trackio-cell
|
| 6 |
+
{"type": "markdown", "id": "cell_1b021ce5d144", "created_at": "2026-07-16T15:02:00+00:00", "title": "What this page is"}
|
| 7 |
+
-->
|
| 8 |
+
Supporting evidence for the Bayes-cap claim and the paper's model-ranking result. The formal five-model BF16 ladder is primary here; the earlier Mac Q8 run remains as a labelled pipeline precursor. The paper's evolutionary merge experiment remains outside scope.
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
---
|
| 12 |
+
<!-- trackio-cell
|
| 13 |
+
{"type": "markdown", "id": "cell_44893eda1431", "created_at": "2026-07-15T17:28:20+00:00", "title": "Claim and test"}
|
| 14 |
+
-->
|
| 15 |
+
Claim tested: capped accuracy remains monotonically related to original accuracy. Five pinned official Qwen2.5 checkpoints (0.5B, 1.5B, Math-1.5B, 3B and 7B) were evaluated in BF16 inside one A100 Job on all 1,319 original and capped GSM8K rows. The preregistered primary statistic is exact two-sided permutation Kendall τ-b between strict original accuracy and per-row mean strict capped accuracy across the published assignment plus two frozen synthetic assignments; all five models are required for a verdict.
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
---
|
| 19 |
+
<!-- trackio-cell
|
| 20 |
+
{"type": "markdown", "id": "cell_2a0a81023027", "created_at": "2026-07-16T15:02:00+00:00", "title": "What the paper reports (ranking)"}
|
| 21 |
+
-->
|
| 22 |
+
The camera-ready paper reports Kendall's τ = 0.92 for its aggregate progress-tracking plot and **τ = 0.96 for GSM8K** in Table 6; capped accuracy rises with original accuracy while remaining lower overall. Our comparison target is the GSM8K result. This ladder is a scoped extension rather than an exact Table 6 recreation: five related Qwen2.5 checkpoints, a strict numeric parser, three frozen target assignments and all 1,319 rows, versus the paper's broader model set, three-seed reporting and 1,024-row tracking sample.
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
---
|
| 26 |
+
<!-- trackio-cell
|
| 27 |
+
{"type": "markdown", "id": "cell_16d67dd86cb8", "created_at": "2026-07-15T23:52:59+00:00", "title": "Historical precursor — exploratory Mac Q8 pilot"}
|
| 28 |
+
-->
|
| 29 |
+
Measured exploratory Mac Q8 pilot result (2026-07-15): both Qwen2.5 0.5B and 1.5B completed all 2,638 expected original/capped generations with zero inference errors and one pinned Ollama digest per model.
|
| 30 |
+
|
| 31 |
+
| Model | Strict original | Strict mean capped | Strict-invalid capped | Fallback original | Fallback mean capped | Fallback returned unshifted reference on capped prompts |
|
| 32 |
+
| --- | ---: | ---: | ---: | ---: | ---: | ---: |
|
| 33 |
+
| Qwen2.5 0.5B Q8 | 9.10% | 0.33% | 76.72% | 41.47% | 1.59% | 37.91% |
|
| 34 |
+
| Qwen2.5 1.5B Q8 | 0.00% | 0.05% | 99.62% | 67.32% | 3.46% | 59.51% |
|
| 35 |
+
|
| 36 |
+
The preregistered strict `#### <integer>` scorer remains primary. The fallback parser is a sensitivity analysis only and does not overwrite the official scores. It shows that the near-zero strict scores are dominated by output-format and instruction-adherence failures rather than an actual collapse in maths ability: 1.5B has higher fallback original accuracy, but usually answers with the true value instead of applying the requested random plus-or-minus-one shift.
|
| 37 |
+
|
| 38 |
+
Both two-model rankings have exact Kendall tau = 1.0 with p = 1.0 (two permutations), so **this pilot alone was inconclusive**. It validated resumability, provenance capture, scoring and audit paths and anticipated the format-adherence limitation later measured in BF16; it is not used for the formal ladder verdict.
|
| 39 |
+
|
| 40 |
+
Machine-readable evidence: `outputs/analysis/statistics.json`, `outputs/analysis/model_metrics.csv`, `outputs/audit/audit_summary.json`, and the raw JSONL files under `outputs/ollama/mac-q8/`. The fixed audit sampled 80 responses across strict-correct, strict-incorrect, strict-invalid and parser-disagreement categories; the credential scan found zero secrets.
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
---
|
| 44 |
+
<!-- trackio-cell
|
| 45 |
+
{"type": "markdown", "id": "cell_5cddd3646ed2", "created_at": "2026-07-15T23:54:07+00:00", "title": "Fixed response audit - observed failure modes"}
|
| 46 |
+
-->
|
| 47 |
+
Inspection of the fixed 80-response audit sample identified recurring failure modes that explain the aggregate parser statistics:
|
| 48 |
+
|
| 49 |
+
- `gsm8k-test-0801`, Qwen2.5 0.5B original: correctly concludes 1,240 but omits the required `####` marker; strict is invalid and fallback recovers 1,240.
|
| 50 |
+
- `gsm8k-test-0081`, Qwen2.5 0.5B capped: emits `#### 17`, which is the true reference rather than the published shifted target 16; this is an instruction-adherence failure, not a parser failure.
|
| 51 |
+
- `gsm8k-test-0563`, Qwen2.5 1.5B capped: states both possible shifted answers, 13 and 11, without making one random choice; fallback selects the last integer and can therefore over-credit an ambiguous response.
|
| 52 |
+
- `gsm8k-test-0153`, Qwen2.5 0.5B original: the completion truncates during algebra and fallback selects an intermediate 10, illustrating that a permissive last-integer parse can be misleading.
|
| 53 |
+
|
| 54 |
+
The sample also contains ordinary reasoning errors and unit-format mismatches. These observations support retaining the strict scorer as primary, reporting fallback only as sensitivity, and treating the pilot as a pipeline/instruction-adherence finding rather than evidence for or against the paper-level ranking claim.
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
---
|
| 58 |
+
<!-- trackio-cell
|
| 59 |
+
{"type": "markdown", "id": "cell_521a15ad209a", "created_at": "2026-07-16T15:02:00+00:00", "title": "Completed five-model BF16 ladder — measured result"}
|
| 60 |
+
-->
|
| 61 |
+
All five pinned checkpoints and **13,190/13,190** records completed in one A100 Job with zero reported failures. “Valid” means the response contained the preregistered final `#### <integer>` form; the primary capped score is the per-row mean across three frozen target assignments.
|
| 62 |
+
|
| 63 |
+
| BF16 model | Original strict (valid) | Mean capped strict (valid) | Published target, exact 95% CI | Above-cap p / Holm p |
|
| 64 |
+
| --- | ---: | ---: | ---: | ---: |
|
| 65 |
+
| Qwen2.5-0.5B | 0.91% (1.7%) | 0.00% (1.2%) | 0/1,319 = 0.00% [0.00, 0.28] | 1.0 / 1.0 |
|
| 66 |
+
| Qwen2.5-1.5B | 0.83% (1.1%) | 0.13% (3.0%) | 2/1,319 = 0.15% [0.02, 0.55] | 1.0 / 1.0 |
|
| 67 |
+
| Qwen2.5-Math-1.5B | 0.00% (0.0%) | 0.05% (0.3%) | 1/1,319 = 0.08% [0.00, 0.42] | 1.0 / 1.0 |
|
| 68 |
+
| Qwen2.5-3B | 1.06% (1.2%) | 8.06% (20.4%) | 103/1,319 = 7.81% [6.42, 9.39] | 1.0 / 1.0 |
|
| 69 |
+
| **Qwen2.5-7B** | **87.19% (94.5%)** | **35.53% (97.4%)** | **481/1,319 = 36.47% [33.86, 39.13]** | **1.0 / 1.0** |
|
| 70 |
+
|
| 71 |
+
The 7B row is the cleanest below-cap witness: a capable, format-compliant model remains decisively below 50%. Across the ladder, the preregistered exact-permutation result is **Kendall τ-b = 0.60, two-sided p = 0.2333** (28/120 permutations at least as extreme). With five untied models this means eight of ten model pairs preserved their order and two reversed; both reversals involved 0.5B versus 1.5B/Math-1.5B. The association is positive but not statistically significant, so ranking preservation is **inconclusive**. This neither reproduces the paper's GSM8K τ = 0.96 nor significantly contradicts a positive relation. Published-only, synthetic-only, fallback-parser and 1,305-row semantic sensitivities all returned the same τ = 0.60 and p = 0.2333. The very low strict validity of four smaller checkpoints makes their near-zero ordering format-sensitive; it does not affect the 7B below-cap finding. Large upper-tail p-values mean no evidence of exceeding the cap, not proof that every capable model must stay below it.
|
| 72 |
+
|
| 73 |
+
Canonical evidence: [Hugging Face Job `6a59e48e…`](https://huggingface.co/jobs/Boopster/6a59e48eb1669a49bf07969f), run `ranking-bf16-v1-20260716-a100-01`, with terminal completion, response hashes, `statistics.json`, `model_metrics.csv`, artifact inventory and remote-byte verification recorded before Job completion.
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
---
|
| 77 |
+
<!-- trackio-cell
|
| 78 |
+
{"type": "artifact", "id": "cell_418f6daf1751", "created_at": "2026-07-17T12:18:00+00:00", "title": "Artifact: independently verified ladder audit", "path": "artifacts/ranking/ranking-bf16-v1-20260716-a100-01/ladder_audit.json", "size": 9695, "artifact_type": "dataset", "auto": true}
|
| 79 |
+
-->
|
| 80 |
+
**📦 Artifact** `artifacts/ranking/ranking-bf16-v1-20260716-a100-01/ladder_audit.json` · dataset · 9.7 kB
|
| 81 |
+
|
| 82 |
+
https://huggingface.co/buckets/Boopster/oCNT5PcMSQ-artifacts#logbook-files/artifacts/ranking/ranking-bf16-v1-20260716-a100-01/ladder_audit.json
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
---
|
| 86 |
+
<!-- trackio-cell
|
| 87 |
+
{"type": "artifact", "id": "cell_588d811b7e54", "created_at": "2026-07-17T12:18:00+00:00", "title": "Artifact: public raw-response evidence bundle", "path": "outputs/ranking/ranking-bf16-v1-20260716-a100-01/public_evidence_bundle.tar.gz", "size": 4599691, "artifact_type": "dataset", "auto": true}
|
| 88 |
+
-->
|
| 89 |
+
**📦 Artifact** `outputs/ranking/ranking-bf16-v1-20260716-a100-01/public_evidence_bundle.tar.gz` · dataset · 4.6 MB
|
| 90 |
+
|
| 91 |
+
https://huggingface.co/buckets/Boopster/oCNT5PcMSQ-artifacts#logbook-files/outputs/ranking/ranking-bf16-v1-20260716-a100-01/public_evidence_bundle.tar.gz
|
| 92 |
+
|
| 93 |
+
SHA-256: `e1d3c88656ef7c80201eb357aaafba2db81bd6b13f16dbe13b9c2f5fc2c8dfe7`. The archive contains the five raw-response files, terminal model records, run-level evidence, `audit/unload_evidence.json`, and the network-free auditor, tests and lockfile; it self-audits from extracted bytes.
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
---
|
| 97 |
+
<!-- trackio-cell
|
| 98 |
+
{"type": "markdown", "id": "cell_40ef37c566a9", "created_at": "2026-07-16T15:02:00+00:00", "title": "Merge detection \u2014 what the paper reports, and why it is not attempted"}
|
| 99 |
+
-->
|
| 100 |
+
The paper's merge experiment optimises a model merge for a phished GSM8K (50% Bayes accuracy) with evolutionary optimisation following Akiba et al. (2025), using Task Arithmetic (Ilharco et al., 2023) via the Mergenetic library; Table 2 shows the merged model exceeding 50%, so test-set overfitting introduced by merging is detected.
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
---
|
| 104 |
+
<!-- trackio-cell
|
| 105 |
+
{"type": "markdown", "id": "cell_2b3fc94c18f7", "created_at": "2026-07-15T17:28:21+00:00", "title": "Scope decision"}
|
| 106 |
+
-->
|
| 107 |
+
The paper reports an evolutionary merge of three 7B models. This claim is preregistered as **not attempted** for two reasons. First, the authors' evolutionary-merge protocol and its controlling artifacts (the exact parent checkpoints, merge search configuration and per-generation acceptance criteria) are not fully specified in the public materials, so a faithful reconstruction is not currently reproducible. Second, a compute-intensive evolutionary search over multiple 7B parents is beyond the scoped single-A100 budget used for the rest of this reproduction. Rather than present a weaker proxy as if it were the paper's experiment, we document the blocker and leave the claim unaddressed.
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
---
|
| 111 |
+
<!-- trackio-cell
|
| 112 |
+
{"type": "markdown", "id": "cell_ece1b305c148", "created_at": "2026-07-16T15:02:00+00:00", "title": "Provenance runs for the pilot"}
|
| 113 |
+
-->
|
| 114 |
+
One exemplar of each command class is kept below; the complete run log, including interrupted and duplicate runs, is preserved on [Run history and compute](#/run-history-and-compute).
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
---
|
| 118 |
+
<!-- trackio-cell
|
| 119 |
+
{"type": "markdown", "id": "cell_0b9bbe2a8c04", "created_at": "2026-07-15T17:42:06+00:00", "title": "Retired local pilot dashboard reference"}
|
| 120 |
+
-->
|
| 121 |
+
The exploratory Ollama pilot originally referenced a local Trackio dashboard,
|
| 122 |
+
but no publishable dashboard database survives. That reference is retired
|
| 123 |
+
rather than promoted as a dead dependency. The raw pilot responses, commands
|
| 124 |
+
and final analyses remain available in the registered artifacts and run
|
| 125 |
+
history; the formal conclusions rely on the audited A100 evidence bundles.
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
---
|
| 129 |
+
<!-- trackio-cell
|
| 130 |
+
{"type": "code", "id": "cell_dce6a5480103", "created_at": "2026-07-15T23:50:43+00:00", "title": "Run: cap-eval repro.yaml (exit 0)", "command": [".venv/bin/cap-eval", "--config", "configs/repro.yaml", "--model-key", "qwen2.5-1.5b", "--backend-label", "mac-q8", "--pull", "--remove-after"], "exit_code": 0, "duration_s": 4525.62}
|
| 131 |
+
-->
|
| 132 |
+
````bash
|
| 133 |
+
$ .venv/bin/cap-eval --config configs/repro.yaml --model-key qwen2.5-1.5b --backend-label mac-q8 --pull --remove-after
|
| 134 |
+
````
|
| 135 |
+
|
| 136 |
+
exit 0 · 4525.6s
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
````yaml title=repro.yaml
|
| 140 |
+
project:
|
| 141 |
+
name: capbencher-icml2026-repro
|
| 142 |
+
owner: Boopster
|
| 143 |
+
paper_title: "How Can I Publish My LLM Benchmark Without Giving the True Answers Away?"
|
| 144 |
+
arxiv_id: "2505.18102"
|
| 145 |
+
openreview_id: oCNT5PcMSQ
|
| 146 |
+
logbook_space: Boopster/oCNT5PcMSQ
|
| 147 |
+
|
| 148 |
+
datasets:
|
| 149 |
+
capped:
|
| 150 |
+
id: ishidalab/capbencher
|
| 151 |
+
config: gsm8k-capped
|
| 152 |
+
split: test
|
| 153 |
+
revision: 62ab88c06dd78400f8092e61e6b23eff257cd2b8
|
| 154 |
+
original:
|
| 155 |
+
id: openai/gsm8k
|
| 156 |
+
config: main
|
| 157 |
+
split: test
|
| 158 |
+
revision: 740312add88f781978c0658806c59bc2815b9866
|
| 159 |
+
|
| 160 |
+
models:
|
| 161 |
+
- key: qwen2.5-0.5b
|
| 162 |
+
hf_id: Qwen/Qwen2.5-0.5B-Instruct
|
| 163 |
+
hf_revision: 7ae557604adf67be50417f59c2c2f167def9a775
|
| 164 |
+
ollama_q8: qwen2.5:0.5b-instruct-q8_0
|
| 165 |
+
- key: qwen2.5-1.5b
|
| 166 |
+
hf_id: Qwen/Qwen2.5-1.5B-Instruct
|
| 167 |
+
hf_revision: 989aa7980e4cf806f80c7fef2b1adb7bc71aa306
|
| 168 |
+
ollama_q8: qwen2.5:1.5b-instruct-q8_0
|
| 169 |
+
- key: qwen2.5-math-1.5b
|
| 170 |
+
hf_id: Qwen/Qwen2.5-Math-1.5B-Instruct
|
| 171 |
+
hf_revision: aafeb0fc6f22cbf0eaeed126eff8be45b0360a35
|
| 172 |
+
ollama_q8: capbencher-qwen2.5-math-1.5b-q8
|
| 173 |
+
- key: qwen2.5-3b
|
| 174 |
+
hf_id: Qwen/Qwen2.5-3B-Instruct
|
| 175 |
+
hf_revision: aa8e72537993ba99e69dfaafa59ed015b17504d1
|
| 176 |
+
ollama_q8: qwen2.5:3b-instruct-q8_0
|
| 177 |
+
- key: qwen2.5-7b
|
| 178 |
+
hf_id: Qwen/Qwen2.5-7B-Instruct
|
| 179 |
+
hf_revision: a09a35458c702b33eeacc393d103063234e8bc28
|
| 180 |
+
ollama_q8: qwen2.5:7b-instruct-q8_0
|
| 181 |
+
|
| 182 |
+
targets:
|
| 183 |
+
synthetic_seeds: [202601, 202602]
|
| 184 |
+
|
| 185 |
+
inference:
|
| 186 |
+
temperature: 0
|
| 187 |
+
seed: 42
|
| 188 |
+
num_ctx: 2048
|
| 189 |
+
num_predict: 512
|
| 190 |
+
request_timeout_seconds: 900
|
| 191 |
+
keep_alive: 5m
|
| 192 |
+
|
| 193 |
+
statistics:
|
| 194 |
+
alpha: 0.05
|
| 195 |
+
null_accuracy: 0.5
|
| 196 |
+
bootstrap_samples: 10000
|
| 197 |
+
bootstrap_seed: 202603
|
| 198 |
+
|
| 199 |
+
training:
|
| 200 |
+
model_key: qwen2.5-3b
|
| 201 |
+
seed: 42
|
| 202 |
+
epochs: 8
|
| 203 |
+
evaluation_epochs: [0, 1, 2, 4, 8]
|
| 204 |
+
learning_rate: 0.00005
|
| 205 |
+
per_device_train_batch_size: 2
|
| 206 |
+
gradient_accumulation_steps: 1
|
| 207 |
+
max_length: 2048
|
| 208 |
+
flavor: a100-large
|
| 209 |
+
timeout: 5h
|
| 210 |
+
private_model_repo: Boopster/capbencher-qwen2.5-3b-contaminated-repro
|
| 211 |
+
private_results_repo: Boopster/capbencher-repro-staging
|
| 212 |
+
private_trackio_space: Boopster/capbencher-repro-training-private
|
| 213 |
+
|
| 214 |
+
paths:
|
| 215 |
+
prepared_data: artifacts/data/gsm8k_repro.jsonl
|
| 216 |
+
training_data: artifacts/data/gsm8k_contamination_sft.jsonl
|
| 217 |
+
data_manifest: artifacts/data/manifest.json
|
| 218 |
+
outputs: outputs
|
| 219 |
+
|
| 220 |
+
````
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
````output
|
| 224 |
+
[?2026h[?25l[1Gpulling manifest ⠋ [K[?25h[?2026l[?2026h[?25l[1Gpulling manifest ⠙ [K[?25h[?2026l[?2026h[?25l[1Gpulling manifest ⠹ [K[?25h[?2026l[?2026h[?25l[1Gpulling manifest ⠸ [K[?25h[?2026l[?2026h[?25l[1Gpulling manifest ⠼ [K[?25h[?2026l[?2026h[?25l[1Gpulling manifest ⠴ [K[?25h[?2026l[?2026h[?25l[1Gpulling manifest ⠦ [K[?25h[?2026l[?2026h[?25l[1Gpulling manifest [K
|
| 225 |
+
pulling 144ea9981a90: 100% ▕██████████████████▏ 1.6 GB [K
|
| 226 |
+
pulling 66b9ea09bd5b: 100% ▕██████████████████▏ 68 B [K
|
| 227 |
+
pulling eb4402837c78: 100% ▕██████████████████▏ 1.5 KB [K
|
| 228 |
+
pulling 832dd9e00a68: 100% ▕██████████████████▏ 11 KB [K
|
| 229 |
+
pulling f573a68391c5: 100% ▕██████████████████▏ 485 B [K
|
| 230 |
+
verifying sha256 digest [K
|
| 231 |
+
writing manifest [K
|
| 232 |
+
success [K[?25h[?2026l
|
| 233 |
+
* Trackio project initialized: capbencher-ollama-eval
|
| 234 |
+
* Trackio metrics logged to: <user-cache>/huggingface/trackio
|
| 235 |
+
* View dashboard by running in your terminal:
|
| 236 |
+
[1m[38;5;208mtrackio show --project "capbencher-ollama-eval"[0m
|
| 237 |
+
* or by running in Python: trackio.show(project="capbencher-ollama-eval")
|
| 238 |
+
* Created new run: mac-q8-qwen2.5-1.5b
|
| 239 |
+
qwen2.5-1.5b: generated 25/753 new responses (0 errors)
|
| 240 |
+
qwen2.5-1.5b: generated 50/753 new responses (0 errors)
|
| 241 |
+
qwen2.5-1.5b: generated 75/753 new responses (0 errors)
|
| 242 |
+
qwen2.5-1.5b: generated 100/753 new responses (0 errors)
|
| 243 |
+
qwen2.5-1.5b: generated 125/753 new responses (0 errors)
|
| 244 |
+
qwen2.5-1.5b: generated 150/753 new responses (0 errors)
|
| 245 |
+
qwen2.5-1.5b: generated 175/753 new responses (0 errors)
|
| 246 |
+
qwen2.5-1.5b: generated 200/753 new responses (0 errors)
|
| 247 |
+
qwen2.5-1.5b: generated 225/753 new responses (0 errors)
|
| 248 |
+
qwen2.5-1.5b: generated 250/753 new responses (0 errors)
|
| 249 |
+
qwen2.5-1.5b: generated 275/753 new responses (0 errors)
|
| 250 |
+
qwen2.5-1.5b: generated 300/753 new responses (0 errors)
|
| 251 |
+
qwen2.5-1.5b: generated 325/753 new responses (0 errors)
|
| 252 |
+
qwen2.5-1.5b: generated 350/753 new responses (0 errors)
|
| 253 |
+
qwen2.5-1.5b: generated 375/753 new responses (0 errors)
|
| 254 |
+
qwen2.5-1.5b: generated 400/753 new responses (0 errors)
|
| 255 |
+
qwen2.5-1.5b: generated 425/753 new responses (0 errors)
|
| 256 |
+
qwen2.5-1.5b: generated 450/753 new responses (0 errors)
|
| 257 |
+
qwen2.5-1.5b: generated 475/753 new responses (0 errors)
|
| 258 |
+
qwen2.5-1.5b: generated 500/753 new responses (0 errors)
|
| 259 |
+
qwen2.5-1.5b: generated 525/753 new responses (0 errors)
|
| 260 |
+
qwen2.5-1.5b: generated 550/753 new responses (0 errors)
|
| 261 |
+
qwen2.5-1.5b: generated 575/753 new responses (0 errors)
|
| 262 |
+
qwen2.5-1.5b: generated 600/753 new responses (0 errors)
|
| 263 |
+
qwen2.5-1.5b: generated 625/753 new responses (0 errors)
|
| 264 |
+
qwen2.5-1.5b: generated 650/753 new responses (0 errors)
|
| 265 |
+
qwen2.5-1.5b: generated 675/753 new responses (0 errors)
|
| 266 |
+
qwen2.5-1.5b: generated 700/753 new responses (0 errors)
|
| 267 |
+
qwen2.5-1.5b: generated 725/753 new responses (0 errors)
|
| 268 |
+
qwen2.5-1.5b: generated 750/753 new responses (0 errors)
|
| 269 |
+
[?25l[?2026h[?25l[1G[K[?25h[?2026l[2K[1G[?25hdeleted 'qwen2.5:1.5b-instruct-q8_0'
|
| 270 |
+
* Run finished. Uploading logs to Trackio (please wait...)
|
| 271 |
+
{
|
| 272 |
+
"backend": "mac-q8",
|
| 273 |
+
"complete": true,
|
| 274 |
+
"errors_in_this_run": 0,
|
| 275 |
+
"expected_responses": 2638,
|
| 276 |
+
"model_key": "qwen2.5-1.5b",
|
| 277 |
+
"model_name": "qwen2.5:1.5b-instruct-q8_0",
|
| 278 |
+
"output": "outputs/ollama/mac-q8/qwen2.5-1.5b.jsonl",
|
| 279 |
+
"successful_responses": 2638
|
| 280 |
+
}
|
| 281 |
+
|
| 282 |
+
````
|
| 283 |
+
|
| 284 |
+
|
| 285 |
+
---
|
| 286 |
+
<!-- trackio-cell
|
| 287 |
+
{"type": "artifact", "id": "cell_71c615c9c98d", "created_at": "2026-07-15T23:50:43+00:00", "title": "Artifact: qwen2.5-1.5b.jsonl", "path": "outputs/ollama/mac-q8/qwen2.5-1.5b.jsonl", "size": 4291805, "artifact_type": "dataset", "auto": true}
|
| 288 |
+
-->
|
| 289 |
+
**📦 Artifact** `outputs/ollama/mac-q8/qwen2.5-1.5b.jsonl` · dataset · 4.3 MB
|
| 290 |
+
|
| 291 |
+
https://huggingface.co/buckets/Boopster/oCNT5PcMSQ-artifacts#logbook-files/outputs/ollama/mac-q8/qwen2.5-1.5b.jsonl
|
| 292 |
+
|
| 293 |
+
|
| 294 |
+
---
|
| 295 |
+
<!-- trackio-cell
|
| 296 |
+
{"type": "code", "id": "cell_caac6657fa67", "created_at": "2026-07-15T23:50:45+00:00", "title": "Run: cap-analyze repro.yaml (exit 0)", "command": [".venv/bin/cap-analyze", "--config", "configs/repro.yaml", "--backend", "mac-q8", "--allow-partial"], "exit_code": 0, "duration_s": 1.431}
|
| 297 |
+
-->
|
| 298 |
+
````bash
|
| 299 |
+
$ .venv/bin/cap-analyze --config configs/repro.yaml --backend mac-q8 --allow-partial
|
| 300 |
+
````
|
| 301 |
+
|
| 302 |
+
exit 0 · 1.4s
|
| 303 |
+
|
| 304 |
+
|
| 305 |
+
````yaml title=repro.yaml
|
| 306 |
+
project:
|
| 307 |
+
name: capbencher-icml2026-repro
|
| 308 |
+
owner: Boopster
|
| 309 |
+
paper_title: "How Can I Publish My LLM Benchmark Without Giving the True Answers Away?"
|
| 310 |
+
arxiv_id: "2505.18102"
|
| 311 |
+
openreview_id: oCNT5PcMSQ
|
| 312 |
+
logbook_space: Boopster/oCNT5PcMSQ
|
| 313 |
+
|
| 314 |
+
datasets:
|
| 315 |
+
capped:
|
| 316 |
+
id: ishidalab/capbencher
|
| 317 |
+
config: gsm8k-capped
|
| 318 |
+
split: test
|
| 319 |
+
revision: 62ab88c06dd78400f8092e61e6b23eff257cd2b8
|
| 320 |
+
original:
|
| 321 |
+
id: openai/gsm8k
|
| 322 |
+
config: main
|
| 323 |
+
split: test
|
| 324 |
+
revision: 740312add88f781978c0658806c59bc2815b9866
|
| 325 |
+
|
| 326 |
+
models:
|
| 327 |
+
- key: qwen2.5-0.5b
|
| 328 |
+
hf_id: Qwen/Qwen2.5-0.5B-Instruct
|
| 329 |
+
hf_revision: 7ae557604adf67be50417f59c2c2f167def9a775
|
| 330 |
+
ollama_q8: qwen2.5:0.5b-instruct-q8_0
|
| 331 |
+
- key: qwen2.5-1.5b
|
| 332 |
+
hf_id: Qwen/Qwen2.5-1.5B-Instruct
|
| 333 |
+
hf_revision: 989aa7980e4cf806f80c7fef2b1adb7bc71aa306
|
| 334 |
+
ollama_q8: qwen2.5:1.5b-instruct-q8_0
|
| 335 |
+
- key: qwen2.5-math-1.5b
|
| 336 |
+
hf_id: Qwen/Qwen2.5-Math-1.5B-Instruct
|
| 337 |
+
hf_revision: aafeb0fc6f22cbf0eaeed126eff8be45b0360a35
|
| 338 |
+
ollama_q8: capbencher-qwen2.5-math-1.5b-q8
|
| 339 |
+
- key: qwen2.5-3b
|
| 340 |
+
hf_id: Qwen/Qwen2.5-3B-Instruct
|
| 341 |
+
hf_revision: aa8e72537993ba99e69dfaafa59ed015b17504d1
|
| 342 |
+
ollama_q8: qwen2.5:3b-instruct-q8_0
|
| 343 |
+
- key: qwen2.5-7b
|
| 344 |
+
hf_id: Qwen/Qwen2.5-7B-Instruct
|
| 345 |
+
hf_revision: a09a35458c702b33eeacc393d103063234e8bc28
|
| 346 |
+
ollama_q8: qwen2.5:7b-instruct-q8_0
|
| 347 |
+
|
| 348 |
+
targets:
|
| 349 |
+
synthetic_seeds: [202601, 202602]
|
| 350 |
+
|
| 351 |
+
inference:
|
| 352 |
+
temperature: 0
|
| 353 |
+
seed: 42
|
| 354 |
+
num_ctx: 2048
|
| 355 |
+
num_predict: 512
|
| 356 |
+
request_timeout_seconds: 900
|
| 357 |
+
keep_alive: 5m
|
| 358 |
+
|
| 359 |
+
statistics:
|
| 360 |
+
alpha: 0.05
|
| 361 |
+
null_accuracy: 0.5
|
| 362 |
+
bootstrap_samples: 10000
|
| 363 |
+
bootstrap_seed: 202603
|
| 364 |
+
|
| 365 |
+
training:
|
| 366 |
+
model_key: qwen2.5-3b
|
| 367 |
+
seed: 42
|
| 368 |
+
epochs: 8
|
| 369 |
+
evaluation_epochs: [0, 1, 2, 4, 8]
|
| 370 |
+
learning_rate: 0.00005
|
| 371 |
+
per_device_train_batch_size: 2
|
| 372 |
+
gradient_accumulation_steps: 1
|
| 373 |
+
max_length: 2048
|
| 374 |
+
flavor: a100-large
|
| 375 |
+
timeout: 5h
|
| 376 |
+
private_model_repo: Boopster/capbencher-qwen2.5-3b-contaminated-repro
|
| 377 |
+
private_results_repo: Boopster/capbencher-repro-staging
|
| 378 |
+
private_trackio_space: Boopster/capbencher-repro-training-private
|
| 379 |
+
|
| 380 |
+
paths:
|
| 381 |
+
prepared_data: artifacts/data/gsm8k_repro.jsonl
|
| 382 |
+
training_data: artifacts/data/gsm8k_contamination_sft.jsonl
|
| 383 |
+
data_manifest: artifacts/data/manifest.json
|
| 384 |
+
outputs: outputs
|
| 385 |
+
|
| 386 |
+
````
|
| 387 |
+
|
| 388 |
+
|
| 389 |
+
````output
|
| 390 |
+
{
|
| 391 |
+
"claim_1": "pending",
|
| 392 |
+
"claim_2": "inconclusive",
|
| 393 |
+
"claim_3": "verified",
|
| 394 |
+
"claim_4": "not_attempted"
|
| 395 |
+
}
|
| 396 |
+
Kendall result: {"permutations": 2, "pvalue": 1.0, "tau": 1.0}
|
| 397 |
+
|
| 398 |
+
````
|
| 399 |
+
|
| 400 |
+
---
|
| 401 |
+
<!-- trackio-cell
|
| 402 |
+
{"type": "markdown", "id": "cell_89bc5bc6df0f", "created_at": "2026-07-17T18:30:00+00:00", "title": "Note: superseded early self-assessment"}
|
| 403 |
+
-->
|
| 404 |
+
The block above is an **early self-assessment snapshot** from a 15 July `cap-analyze` run: the `claim_1..4` labels use pre-restructure numbering, and the `tau = 1.0` line is the two-model Q8 exploratory pilot — **both superseded**. Final measured results: Claim 1 alarm fired under the V5 sensitivity; Claim 2 supported by the exact boundary and the five-model BF16 ladder, whose canonical ranking is Kendall τ-b = 0.60 (p = 0.2333). Self-assigned labels are not evidence — only the measured results are.
|
| 405 |
+
|
| 406 |
+
|
| 407 |
+
|
| 408 |
+
---
|
| 409 |
+
<!-- trackio-cell
|
| 410 |
+
{"type": "markdown", "id": "cell_d0a69fb0a5d0", "created_at": "2026-07-15T23:50:45+00:00", "title": "Retired mutable path reference: exploratory model metrics"}
|
| 411 |
+
-->
|
| 412 |
+
This auto-captured cell originally described 665 bytes at `outputs/analysis/model_metrics.csv`. Later analyses overwrote that mutable path and no immutable 665-byte copy survives, so the path reference is retired and is **not evidence**. The final ladder metrics use the immutable run-specific artifact registered elsewhere on this page.
|
| 413 |
+
|
| 414 |
+
|
| 415 |
+
---
|
| 416 |
+
<!-- trackio-cell
|
| 417 |
+
{"type": "code", "id": "cell_00fbe61555a9", "created_at": "2026-07-15T23:51:11+00:00", "title": "Run: cap-audit repro.yaml (exit 0)", "command": [".venv/bin/cap-audit", "--config", "configs/repro.yaml", "--backend", "mac-q8", "--per-category", "20", "--seed", "202604"], "exit_code": 0, "duration_s": 3.783}
|
| 418 |
+
-->
|
| 419 |
+
````bash
|
| 420 |
+
$ .venv/bin/cap-audit --config configs/repro.yaml --backend mac-q8 --per-category 20 --seed 202604
|
| 421 |
+
````
|
| 422 |
+
|
| 423 |
+
exit 0 · 3.8s
|
| 424 |
+
|
| 425 |
+
|
| 426 |
+
````yaml title=repro.yaml
|
| 427 |
+
project:
|
| 428 |
+
name: capbencher-icml2026-repro
|
| 429 |
+
owner: Boopster
|
| 430 |
+
paper_title: "How Can I Publish My LLM Benchmark Without Giving the True Answers Away?"
|
| 431 |
+
arxiv_id: "2505.18102"
|
| 432 |
+
openreview_id: oCNT5PcMSQ
|
| 433 |
+
logbook_space: Boopster/oCNT5PcMSQ
|
| 434 |
+
|
| 435 |
+
datasets:
|
| 436 |
+
capped:
|
| 437 |
+
id: ishidalab/capbencher
|
| 438 |
+
config: gsm8k-capped
|
| 439 |
+
split: test
|
| 440 |
+
revision: 62ab88c06dd78400f8092e61e6b23eff257cd2b8
|
| 441 |
+
original:
|
| 442 |
+
id: openai/gsm8k
|
| 443 |
+
config: main
|
| 444 |
+
split: test
|
| 445 |
+
revision: 740312add88f781978c0658806c59bc2815b9866
|
| 446 |
+
|
| 447 |
+
models:
|
| 448 |
+
- key: qwen2.5-0.5b
|
| 449 |
+
hf_id: Qwen/Qwen2.5-0.5B-Instruct
|
| 450 |
+
hf_revision: 7ae557604adf67be50417f59c2c2f167def9a775
|
| 451 |
+
ollama_q8: qwen2.5:0.5b-instruct-q8_0
|
| 452 |
+
- key: qwen2.5-1.5b
|
| 453 |
+
hf_id: Qwen/Qwen2.5-1.5B-Instruct
|
| 454 |
+
hf_revision: 989aa7980e4cf806f80c7fef2b1adb7bc71aa306
|
| 455 |
+
ollama_q8: qwen2.5:1.5b-instruct-q8_0
|
| 456 |
+
- key: qwen2.5-math-1.5b
|
| 457 |
+
hf_id: Qwen/Qwen2.5-Math-1.5B-Instruct
|
| 458 |
+
hf_revision: aafeb0fc6f22cbf0eaeed126eff8be45b0360a35
|
| 459 |
+
ollama_q8: capbencher-qwen2.5-math-1.5b-q8
|
| 460 |
+
- key: qwen2.5-3b
|
| 461 |
+
hf_id: Qwen/Qwen2.5-3B-Instruct
|
| 462 |
+
hf_revision: aa8e72537993ba99e69dfaafa59ed015b17504d1
|
| 463 |
+
ollama_q8: qwen2.5:3b-instruct-q8_0
|
| 464 |
+
- key: qwen2.5-7b
|
| 465 |
+
hf_id: Qwen/Qwen2.5-7B-Instruct
|
| 466 |
+
hf_revision: a09a35458c702b33eeacc393d103063234e8bc28
|
| 467 |
+
ollama_q8: qwen2.5:7b-instruct-q8_0
|
| 468 |
+
|
| 469 |
+
targets:
|
| 470 |
+
synthetic_seeds: [202601, 202602]
|
| 471 |
+
|
| 472 |
+
inference:
|
| 473 |
+
temperature: 0
|
| 474 |
+
seed: 42
|
| 475 |
+
num_ctx: 2048
|
| 476 |
+
num_predict: 512
|
| 477 |
+
request_timeout_seconds: 900
|
| 478 |
+
keep_alive: 5m
|
| 479 |
+
|
| 480 |
+
statistics:
|
| 481 |
+
alpha: 0.05
|
| 482 |
+
null_accuracy: 0.5
|
| 483 |
+
bootstrap_samples: 10000
|
| 484 |
+
bootstrap_seed: 202603
|
| 485 |
+
|
| 486 |
+
training:
|
| 487 |
+
model_key: qwen2.5-3b
|
| 488 |
+
seed: 42
|
| 489 |
+
epochs: 8
|
| 490 |
+
evaluation_epochs: [0, 1, 2, 4, 8]
|
| 491 |
+
learning_rate: 0.00005
|
| 492 |
+
per_device_train_batch_size: 2
|
| 493 |
+
gradient_accumulation_steps: 1
|
| 494 |
+
max_length: 2048
|
| 495 |
+
flavor: a100-large
|
| 496 |
+
timeout: 5h
|
| 497 |
+
private_model_repo: Boopster/capbencher-qwen2.5-3b-contaminated-repro
|
| 498 |
+
private_results_repo: Boopster/capbencher-repro-staging
|
| 499 |
+
private_trackio_space: Boopster/capbencher-repro-training-private
|
| 500 |
+
|
| 501 |
+
paths:
|
| 502 |
+
prepared_data: artifacts/data/gsm8k_repro.jsonl
|
| 503 |
+
training_data: artifacts/data/gsm8k_contamination_sft.jsonl
|
| 504 |
+
data_manifest: artifacts/data/manifest.json
|
| 505 |
+
outputs: outputs
|
| 506 |
+
|
| 507 |
+
````
|
| 508 |
+
|
| 509 |
+
|
| 510 |
+
````output
|
| 511 |
+
{
|
| 512 |
+
"backend": "mac-q8",
|
| 513 |
+
"category_counts": {
|
| 514 |
+
"parser_disagreement": 4720,
|
| 515 |
+
"strict_correct": 123,
|
| 516 |
+
"strict_incorrect": 432,
|
| 517 |
+
"strict_invalid": 4721
|
| 518 |
+
},
|
| 519 |
+
"created_at": "2026-07-15T23:51:11.259072+00:00",
|
| 520 |
+
"per_category": 20,
|
| 521 |
+
"sampled_rows": 80,
|
| 522 |
+
"secret_scan_findings": [],
|
| 523 |
+
"seed": 202604
|
| 524 |
+
}
|
| 525 |
+
|
| 526 |
+
````
|
| 527 |
+
|
| 528 |
+
|
| 529 |
+
---
|
| 530 |
+
<!-- trackio-cell
|
| 531 |
+
{"type": "artifact", "id": "cell_bce7aea73ce7", "created_at": "2026-07-15T23:51:11+00:00", "title": "Artifact: stratified_sample.jsonl", "path": "outputs/audit/stratified_sample.jsonl", "size": 100327, "artifact_type": "dataset", "auto": true}
|
| 532 |
+
-->
|
| 533 |
+
**📦 Artifact** `outputs/audit/stratified_sample.jsonl` · dataset · 0.1 MB
|
| 534 |
+
|
| 535 |
+
https://huggingface.co/buckets/Boopster/oCNT5PcMSQ-artifacts#logbook-files/outputs/audit/stratified_sample.jsonl
|
| 536 |
+
|
| 537 |
+
|
| 538 |
+
---
|
| 539 |
+
<!-- trackio-cell
|
| 540 |
+
{"type": "figure", "id": "cell_a8994e77b900", "created_at": "2026-07-16T14:21:33+00:00", "title": "Figure: strict vs fallback accuracy in the toy pilot"}
|
| 541 |
+
-->
|
| 542 |
+
````html
|
| 543 |
+
<svg viewBox="0 0 720 300" xmlns="http://www.w3.org/2000/svg" font-family="Georgia,serif">
|
| 544 |
+
<text x="360.0" y="24" text-anchor="middle" font-size="16" fill="#222">Original-task accuracy: strict scorer (primary) vs fallback parser (sensitivity)</text><text x="220" y="71" text-anchor="end" font-size="13" fill="#333">Qwen2.5-0.5B · strict</text><rect x="230" y="56" width="36.4" height="22" rx="4" fill="#3d87d8" opacity="1.0"/><text x="274.4" y="71" font-size="13" fill="#222">9.10%</text><text x="220" y="111" text-anchor="end" font-size="13" fill="#333">Qwen2.5-0.5B · fallback</text><rect x="230" y="96" width="165.9" height="22" rx="4" fill="#3d87d8" opacity="0.45"/><text x="403.9" y="111" font-size="13" fill="#222">41.47%</text><text x="220" y="151" text-anchor="end" font-size="13" fill="#333">Qwen2.5-1.5B · strict</text><rect x="230" y="136" width="1.5" height="22" rx="4" fill="#c0392b" opacity="1.0"/><text x="239.5" y="151" font-size="13" fill="#222">0.00%</text><text x="220" y="191" text-anchor="end" font-size="13" fill="#333">Qwen2.5-1.5B · fallback</text><rect x="230" y="176" width="269.3" height="22" rx="4" fill="#c0392b" opacity="0.45"/><text x="507.3" y="191" font-size="13" fill="#222">67.32%</text><text x="230" y="234" font-size="12.5" fill="#555">Strict `#### integer` scores are dominated by format failures (strict-invalid: 76.7–99.6% on capped,</text>
|
| 545 |
+
<text x="230" y="252" font-size="12.5" fill="#555">81.7–99.9% on original prompts). The fallback parser is sensitivity-only and never the official score.</text></svg>
|
| 546 |
+
````
|
| 547 |
+
|
| 548 |
+
````raw
|
| 549 |
+
model_key,hf_id,examples,original_accuracy,original_ci_low,original_ci_high,mean_capped_accuracy,mean_capped_ci_low,mean_capped_ci_high,invalid_rate,strict_invalid_rate_original,strict_invalid_rate_capped,fallback_invalid_rate_original,fallback_invalid_rate_capped,fallback_original_accuracy,fallback_mean_capped_accuracy,fallback_returns_reference_on_capped_rate,parser_disagreement_rate,capped_accuracy_published,fallback_capped_accuracy_published,capped_accuracy_synthetic_202601,fallback_capped_accuracy_synthetic_202601,capped_accuracy_synthetic_202602,fallback_capped_accuracy_synthetic_202602
|
| 550 |
+
qwen2.5-0.5b,Qwen/Qwen2.5-0.5B-Instruct,1319,0.09097801364670205,0.0758150113722517,0.1068991660348749,0.0032853171594642403,0.0010108668182966893,0.006065200909780136,0.7672479150871873,0.8165276724791509,0.7672479150871873,0.0,0.000758150113722517,0.41470811220621684,0.01592115238817286,0.37907505686125853,0.7915087187263078,0.002274450341167551,0.015163002274450341,0.0037907505686125853,0.014404852160727824,0.0037907505686125853,0.01819560272934041
|
| 551 |
+
qwen2.5-1.5b,Qwen/Qwen2.5-1.5B-Instruct,1319,0.0,0.0,0.0,0.0005054334091483447,0.0,0.001516300227445034,0.9962092494313874,0.9992418498862775,0.9962092494313874,0.0,0.0,0.6732373009855952,0.03462218852666161,0.5951478392721758,0.9977255496588324,0.0,0.03184230477634572,0.000758150113722517,0.037149355572403335,0.000758150113722517,0.034874905231235785
|
| 552 |
+
|
| 553 |
+
````
|
| 554 |
+
|
| 555 |
+
|
| 556 |
+
---
|
| 557 |
+
<!-- trackio-cell
|
| 558 |
+
{"type": "figure", "id": "cell_c60586e2b83c", "created_at": "2026-07-17T12:17:02+00:00", "title": "Figure: five-model BF16 ranking"}
|
| 559 |
+
-->
|
| 560 |
+
````html
|
| 561 |
+
<svg class="ranking-chart" viewBox="0 0 800 480"
|
| 562 |
+
xmlns="http://www.w3.org/2000/svg" role="img" aria-labelledby="ranking-title ranking-desc">
|
| 563 |
+
<title id="ranking-title">Original versus capped model ranking</title>
|
| 564 |
+
<desc id="ranking-desc">Scatter plot of original and mean capped accuracy for 5 models. Observed Kendall tau-b is 0.6000 with exact two-sided permutation p-value 0.233333. The paper reported tau 0.96; this is a comparison, not a verification verdict.</desc>
|
| 565 |
+
<style>
|
| 566 |
+
.ranking-chart { color: currentColor; font-family: ui-sans-serif, system-ui, sans-serif; }
|
| 567 |
+
.ranking-grid { stroke: color-mix(in srgb, currentColor 16%, transparent); stroke-width: 1; }
|
| 568 |
+
.ranking-axis { stroke: currentColor; stroke-width: 1; }
|
| 569 |
+
.ranking-tick, .ranking-detail { fill: currentColor; opacity: .72; font-size: 11px; }
|
| 570 |
+
.ranking-heading { fill: currentColor; font-size: 16px; font-weight: 500; }
|
| 571 |
+
.ranking-summary { fill: currentColor; font-size: 13px; font-weight: 500; }
|
| 572 |
+
.ranking-comparison { fill: currentColor; opacity: .75; font-size: 11px; }
|
| 573 |
+
.ranking-point, .ranking-key-dot { fill: var(--viz-series-1, currentColor); }
|
| 574 |
+
.ranking-point-number, .ranking-key-number {
|
| 575 |
+
fill: var(--background, Canvas); font-size: 10px; font-weight: 500;
|
| 576 |
+
}
|
| 577 |
+
.ranking-model { fill: currentColor; font-size: 11px; font-weight: 500; }
|
| 578 |
+
.ranking-cap {
|
| 579 |
+
stroke: var(--viz-series-2, currentColor); stroke-width: 1.5; stroke-dasharray: 6 5;
|
| 580 |
+
}
|
| 581 |
+
.ranking-cap-label { fill: currentColor; font-size: 10px; font-weight: 500; }
|
| 582 |
+
.ranking-axis-label { fill: currentColor; font-size: 12px; }
|
| 583 |
+
</style>
|
| 584 |
+
<text class="ranking-heading" x="76" y="24">Original vs capped accuracy across the model ladder</text>
|
| 585 |
+
<text class="ranking-summary" x="76" y="47">Kendall τ-b = 0.60; exact two-sided permutation p = 0.233</text>
|
| 586 |
+
<text class="ranking-comparison" x="76" y="68">Paper reported τ = 0.96; descriptive comparison only—not a verification verdict.</text>
|
| 587 |
+
<line class="ranking-grid" x1="76.0" y1="104" x2="76.0" y2="416"/><text class="ranking-tick" x="76.0" y="438" text-anchor="middle">0%</text><line class="ranking-grid" x1="199.5" y1="104" x2="199.5" y2="416"/><text class="ranking-tick" x="199.5" y="438" text-anchor="middle">25%</text><line class="ranking-grid" x1="323.0" y1="104" x2="323.0" y2="416"/><text class="ranking-tick" x="323.0" y="438" text-anchor="middle">50%</text><line class="ranking-grid" x1="446.5" y1="104" x2="446.5" y2="416"/><text class="ranking-tick" x="446.5" y="438" text-anchor="middle">75%</text><line class="ranking-grid" x1="570.0" y1="104" x2="570.0" y2="416"/><text class="ranking-tick" x="570.0" y="438" text-anchor="middle">100%</text><line class="ranking-grid" x1="76" y1="416.0" x2="570" y2="416.0"/><text class="ranking-tick" x="65" y="420.0" text-anchor="end">0%</text><line class="ranking-grid" x1="76" y1="359.3" x2="570" y2="359.3"/><text class="ranking-tick" x="65" y="363.3" text-anchor="end">10%</text><line class="ranking-grid" x1="76" y1="302.5" x2="570" y2="302.5"/><text class="ranking-tick" x="65" y="306.5" text-anchor="end">20%</text><line class="ranking-grid" x1="76" y1="245.8" x2="570" y2="245.8"/><text class="ranking-tick" x="65" y="249.8" text-anchor="end">30%</text><line class="ranking-grid" x1="76" y1="189.1" x2="570" y2="189.1"/><text class="ranking-tick" x="65" y="193.1" text-anchor="end">40%</text><line class="ranking-grid" x1="76" y1="132.4" x2="570" y2="132.4"/><text class="ranking-tick" x="65" y="136.4" text-anchor="end">50%</text>
|
| 588 |
+
<line class="ranking-axis" x1="76" y1="416"
|
| 589 |
+
x2="570" y2="416"/>
|
| 590 |
+
<line class="ranking-axis" x1="76" y1="104"
|
| 591 |
+
x2="76" y2="416"/>
|
| 592 |
+
<line class="ranking-cap" x1="76" y1="132.4"
|
| 593 |
+
x2="570" y2="132.4"/>
|
| 594 |
+
<text class="ranking-cap-label" x="563" y="124.4"
|
| 595 |
+
text-anchor="end">50% Bayes cap</text>
|
| 596 |
+
<g aria-label="qwen2.5-0.5b: original 0.91%, mean capped 0.00%"><circle class="ranking-point" cx="80.5" cy="416.0" r="10"/><text class="ranking-point-number" x="80.5" y="420.0" text-anchor="middle">1</text></g><g aria-label="qwen2.5-1.5b: original 0.83%, mean capped 0.13%"><circle class="ranking-point" cx="80.1" cy="415.3" r="10"/><text class="ranking-point-number" x="80.1" y="419.3" text-anchor="middle">2</text></g><g aria-label="qwen2.5-math-1.5b: original 0.00%, mean capped 0.05%"><circle class="ranking-point" cx="76.0" cy="415.7" r="10"/><text class="ranking-point-number" x="76.0" y="419.7" text-anchor="middle">3</text></g><g aria-label="qwen2.5-3b: original 1.06%, mean capped 8.06%"><circle class="ranking-point" cx="81.2" cy="370.3" r="10"/><text class="ranking-point-number" x="81.2" y="374.3" text-anchor="middle">4</text></g><g aria-label="qwen2.5-7b: original 87.19%, mean capped 35.53%"><circle class="ranking-point" cx="506.7" cy="214.4" r="10"/><text class="ranking-point-number" x="506.7" y="218.4" text-anchor="middle">5</text></g>
|
| 597 |
+
<circle class="ranking-key-dot" cx="596" cy="128" r="9"/><text class="ranking-key-number" x="596" y="132" text-anchor="middle">1</text><text class="ranking-model" x="613" y="126">qwen2.5-0.5b</text><text class="ranking-detail" x="613" y="144">original 0.9% · capped 0.0%</text><circle class="ranking-key-dot" cx="596" cy="185" r="9"/><text class="ranking-key-number" x="596" y="189" text-anchor="middle">2</text><text class="ranking-model" x="613" y="183">qwen2.5-1.5b</text><text class="ranking-detail" x="613" y="201">original 0.8% · capped 0.1%</text><circle class="ranking-key-dot" cx="596" cy="242" r="9"/><text class="ranking-key-number" x="596" y="246" text-anchor="middle">3</text><text class="ranking-model" x="613" y="240">qwen2.5-math-1.5b</text><text class="ranking-detail" x="613" y="258">original 0.0% · capped 0.1%</text><circle class="ranking-key-dot" cx="596" cy="299" r="9"/><text class="ranking-key-number" x="596" y="303" text-anchor="middle">4</text><text class="ranking-model" x="613" y="297">qwen2.5-3b</text><text class="ranking-detail" x="613" y="315">original 1.1% · capped 8.1%</text><circle class="ranking-key-dot" cx="596" cy="356" r="9"/><text class="ranking-key-number" x="596" y="360" text-anchor="middle">5</text><text class="ranking-model" x="613" y="354">qwen2.5-7b</text><text class="ranking-detail" x="613" y="372">original 87.2% · capped 35.5%</text>
|
| 598 |
+
<text class="ranking-axis-label" x="323.0" y="464"
|
| 599 |
+
text-anchor="middle">Original-benchmark accuracy</text>
|
| 600 |
+
<text class="ranking-axis-label" x="18" y="260.0"
|
| 601 |
+
text-anchor="middle" transform="rotate(-90 18 260.0)">
|
| 602 |
+
Mean capped accuracy</text>
|
| 603 |
+
</svg>
|
| 604 |
+
|
| 605 |
+
````
|
| 606 |
+
|
| 607 |
+
````raw
|
| 608 |
+
model_key,original_accuracy,mean_capped_accuracy
|
| 609 |
+
qwen2.5-0.5b,0.009097801364670205,0.0
|
| 610 |
+
qwen2.5-1.5b,0.008339651250947688,0.0012635835228708616
|
| 611 |
+
qwen2.5-math-1.5b,0.0,0.0005054334091483447
|
| 612 |
+
qwen2.5-3b,0.01061410159211524,0.08061662875916098
|
| 613 |
+
qwen2.5-7b,0.8718726307808946,0.3553196866312863
|
| 614 |
+
|
| 615 |
+
````
|
pages/index.md
CHANGED
|
@@ -1,7 +1,12 @@
|
|
| 1 |
-
# Repro: How Can I Publish My LLM Benchmark Without Giving the True Answers Away?
|
| 2 |
|
| 3 |
## Pages
|
| 4 |
|
| 5 |
-
| Page |
|
| 6 |
-
| --- |
|
| 7 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Repro — CapBencher: Give Your LLM Benchmark a Built-in Alarm for Test-Set Overfitting (formerly “How Can I Publish My LLM Benchmark Without Giving the True Answers Away?”)
|
| 2 |
|
| 3 |
## Pages
|
| 4 |
|
| 5 |
+
| Page | Scope |
|
| 6 |
+
| --- | --- |
|
| 7 |
+
| [Official claim 1 — detecting deliberate contamination](#/claim-a-detecting-deliberate-contamination) | `toy` relative to the official breadth claim |
|
| 8 |
+
| [Official claim 2 — the Bayes-accuracy cap](#/claim-b-the-bayes-accuracy-cap) | scoped empirical reproduction |
|
| 9 |
+
| [Conclusion](#/conclusion) | executive summary, scope, cost and downloads |
|
| 10 |
+
| [Further evidence — BF16 ranking ladder and merge detection](#/further-evidence-ranking-and-merge) | supporting and not-attempted experiments |
|
| 11 |
+
| [Methods and provenance](#/methods-and-provenance) | protocol and audit trail |
|
| 12 |
+
| [Run history and compute](#/run-history-and-compute) | chronological operations ledger |
|
pages/methods-and-provenance/page.md
ADDED
|
@@ -0,0 +1,301 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Methods and provenance
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
---
|
| 5 |
+
<!-- trackio-cell
|
| 6 |
+
{"type": "markdown", "id": "cell_3a112112bef2", "created_at": "2026-07-15T17:28:20+00:00", "title": "Preregistered method"}
|
| 7 |
+
-->
|
| 8 |
+
This page records immutable source revisions, dataset validation, model digests, hardware, prompts, parser rules, exact commands and terminal evidence. The challenge contract was rechecked on 17 July 2026 at pinned Space revision [`40062b5a744ba2c069639f5e7e8b7c21c702bc3d`](https://huggingface.co/spaces/ICML-2026-agent-repro/challenge/blob/40062b5a744ba2c069639f5e7e8b7c21c702bc3d/README.md); the official claim wording in this logbook is taken verbatim from that challenge record. The Mac Q8 run is an exploratory precursor. Formal evidence comprises the V4 same-lineage BF16 Qwen2.5-3B contamination comparison, the preregistered V5 chat-template-alignment sensitivity, and a separate same-job five-checkpoint BF16 ranking ladder, all approval-gated on Hugging Face A100 hardware. Note on Job timeouts: the pinned `repro.yaml` shown below carries the original `timeout: 5h` used by the V4-era A100 Jobs; the formal V5 sensitivity run used a tightened 75-minute (4,500-second) timeout as a documented protocol amendment. The 5-hour value in the config snapshots is historical, not the V5 protocol.
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
---
|
| 12 |
+
<!-- trackio-cell
|
| 13 |
+
{"type": "markdown", "id": "cell_f1de117417d4", "created_at": "2026-07-17T12:24:00+00:00", "title": "Reproduction fidelity — scoped claim test, not an exact rerun"}
|
| 14 |
+
-->
|
| 15 |
+
This project is a rigorous **independent, scoped reproduction of the paper's claims**, not a bit-for-bit rerun of every author experiment. That distinction is material.
|
| 16 |
+
|
| 17 |
+
| Component | Fidelity to the pinned authors' release |
|
| 18 |
+
| --- | --- |
|
| 19 |
+
| GSM8K source, published capped dataset and question templates | Matched and revision-pinned; every row was independently aligned. |
|
| 20 |
+
| 50% Bayes cap and one-sided exact binomial test | Reproduced exactly, including the 690/1,319 significance boundary. |
|
| 21 |
+
| Controlled contamination | One Qwen2.5-3B, eight-epoch adaptation with one model/seed. V4 statistics were hash-audited after a post-endpoint telemetry failure. V5 generated both endpoints in one Job/model object; checkpoint 0 matched the preregistered V4 clean artifact byte-for-byte. |
|
| 22 |
+
| Progress-tracking model population | Material deviation: five related pinned Qwen2.5 checkpoints rather than the paper's broader ladder. |
|
| 23 |
+
| Response scoring | Material deviation: deterministic strict `#### <integer>` parsing was primary; the authors' public `evaluate_qa.py` sends both response and reference to a GPT-4.1 judge. |
|
| 24 |
+
| Tracking sample and repetitions | Material deviation: all 1,319 rows plus three frozen target assignments, rather than the paper's 1,024-row tracking sample and three-run reporting. |
|
| 25 |
+
| Execution environment | Material deviation: local Hugging Face BF16 inference/training in approval-gated A100 Jobs rather than the authors' API/OpenRouter evaluation path. |
|
| 26 |
+
|
| 27 |
+
The authors' public repository does not include the exact Table 6 model roster, frozen 1,024 row IDs, per-seed outputs or plotting inputs. In addition, unsnapshotted API aliases can change. A historically exact Table 6 rerun is therefore not fully specified by the released artifacts. The completed ladder should be read as new evidence about whether the ranking claim generalises to this pinned five-model family—not as a recreation of the paper's reported GSM8K τ = 0.96.
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
---
|
| 31 |
+
<!-- trackio-cell
|
| 32 |
+
{"type": "markdown", "id": "cell_8e37c1a5f294", "created_at": "2026-07-17T17:31:00+00:00", "title": "Formal V5 protocol and trust chain"}
|
| 33 |
+
-->
|
| 34 |
+
Formal V5 (`formal-v5-20260717-a100-01`) rendered all 1,319 training examples through the exact pinned Qwen2.5 chat template before tokenisation, then packed the audited 391,655-token sequence multiset into 197 bins. It trained the pinned `Qwen/Qwen2.5-3B-Instruct` revision `aa8e7253…04d1` for 8 epochs, 99 steps per epoch and 792/792 total steps in BF16 with batch size 2 and seed 42. V4 used 712 steps; the 792-step V5 count follows from its different rendered-sequence packing, so the contrast does not isolate a pure template-only causal effect. Both endpoints used greedy cache-on generation and the unchanged strict final `#### <integer>` parser.
|
| 35 |
+
|
| 36 |
+
The [A100 Job](https://huggingface.co/jobs/Boopster/6a5a584ebee6ee1cf4ecd9ed) completed in approximately 3,100 seconds (51 minutes 40 seconds) with zero retries. Its terminal marker binds both freshly generated endpoints from the same model object, checkpoint-0 byte equivalence to the preregistered V4 clean artifact, training state, code and remote inventory. The network-free audit verified the 43-artifact chain and recomputed the alarm verdict, paired table, exact boundaries and 1,305-row sensitivity. V5 remains a preregistered sensitivity—not a silent V4 replacement—and its single-model scope does not establish the official claim's breadth.
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
---
|
| 40 |
+
<!-- trackio-cell
|
| 41 |
+
{"type": "markdown", "id": "cell_7f2b3c4d5e6a", "created_at": "2026-07-17T12:06:43+00:00", "title": "Five-model BF16 ranking protocol and trust chain"}
|
| 42 |
+
-->
|
| 43 |
+
The formal ladder pinned Qwen2.5-Instruct 0.5B, 1.5B, 3B and 7B plus Qwen2.5-Math-1.5B to the revisions recorded in `preregistration.json`. One `a100-large` Job generated 5 models × 1,319 rows × 2 prompt kinds = 13,190 responses in BF16, using each checkpoint's native chat template, greedy cache-on decoding, a 2,048-token total context and at most 512 new tokens. The strict final `#### <integer>` parser was primary; fallback parsing was sensitivity-only.
|
| 44 |
+
|
| 45 |
+
Original strict accuracy was ranked against each model's per-row mean strict capped accuracy over the published target assignment and two target assignments frozen from seeds 202601 and 202602. The decision rule required all five models and a two-sided exact-permutation Kendall τ-b p < 0.05; fifteen secondary above-cap tests were Holm-adjusted. The terminal marker was written only after all records, per-model manifests, global statistics, metrics, artifact inventory and remote bytes were verified. Executed source was bound to SHA-256 `71219a692e5aa717707d88af949d6ff96100d0f0c8ed5bc934979eaf73cbb5de`. See [Job `6a59e48eb1669a49bf07969f`](https://huggingface.co/jobs/Boopster/6a59e48eb1669a49bf07969f).
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
---
|
| 49 |
+
<!-- trackio-cell
|
| 50 |
+
{"type": "markdown", "id": "cell_7b169ad08e0c", "created_at": "2026-07-16T15:02:00+00:00", "title": "Terminology note"}
|
| 51 |
+
-->
|
| 52 |
+
The paper calls the shifted benchmarks "phished benchmarks" (method name PhishBencher). These pages use "capped"; the terms are equivalent: a published target randomly shifted ±1 from the true answer, giving a 50% Bayes accuracy for models that have not seen the stored targets.
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
---
|
| 56 |
+
<!-- trackio-cell
|
| 57 |
+
{"type": "code", "id": "cell_9832540c6220", "created_at": "2026-07-15T17:57:33+00:00", "title": "Run: cap-prepare repro.yaml (exit 0)", "command": [".venv/bin/cap-prepare", "--config", "configs/repro.yaml"], "exit_code": 0, "duration_s": 4.312}
|
| 58 |
+
-->
|
| 59 |
+
````bash
|
| 60 |
+
$ .venv/bin/cap-prepare --config configs/repro.yaml
|
| 61 |
+
````
|
| 62 |
+
|
| 63 |
+
exit 0 · 4.3s
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
````yaml title=repro.yaml
|
| 67 |
+
project:
|
| 68 |
+
name: capbencher-icml2026-repro
|
| 69 |
+
owner: Boopster
|
| 70 |
+
paper_title: "How Can I Publish My LLM Benchmark Without Giving the True Answers Away?"
|
| 71 |
+
arxiv_id: "2505.18102"
|
| 72 |
+
openreview_id: oCNT5PcMSQ
|
| 73 |
+
logbook_space: Boopster/oCNT5PcMSQ
|
| 74 |
+
|
| 75 |
+
datasets:
|
| 76 |
+
capped:
|
| 77 |
+
id: ishidalab/capbencher
|
| 78 |
+
config: gsm8k-capped
|
| 79 |
+
split: test
|
| 80 |
+
revision: 62ab88c06dd78400f8092e61e6b23eff257cd2b8
|
| 81 |
+
original:
|
| 82 |
+
id: openai/gsm8k
|
| 83 |
+
config: main
|
| 84 |
+
split: test
|
| 85 |
+
revision: 740312add88f781978c0658806c59bc2815b9866
|
| 86 |
+
|
| 87 |
+
models:
|
| 88 |
+
- key: qwen2.5-0.5b
|
| 89 |
+
hf_id: Qwen/Qwen2.5-0.5B-Instruct
|
| 90 |
+
hf_revision: 7ae557604adf67be50417f59c2c2f167def9a775
|
| 91 |
+
ollama_q8: qwen2.5:0.5b-instruct-q8_0
|
| 92 |
+
- key: qwen2.5-1.5b
|
| 93 |
+
hf_id: Qwen/Qwen2.5-1.5B-Instruct
|
| 94 |
+
hf_revision: 989aa7980e4cf806f80c7fef2b1adb7bc71aa306
|
| 95 |
+
ollama_q8: qwen2.5:1.5b-instruct-q8_0
|
| 96 |
+
- key: qwen2.5-math-1.5b
|
| 97 |
+
hf_id: Qwen/Qwen2.5-Math-1.5B-Instruct
|
| 98 |
+
hf_revision: aafeb0fc6f22cbf0eaeed126eff8be45b0360a35
|
| 99 |
+
ollama_q8: capbencher-qwen2.5-math-1.5b-q8
|
| 100 |
+
- key: qwen2.5-3b
|
| 101 |
+
hf_id: Qwen/Qwen2.5-3B-Instruct
|
| 102 |
+
hf_revision: aa8e72537993ba99e69dfaafa59ed015b17504d1
|
| 103 |
+
ollama_q8: qwen2.5:3b-instruct-q8_0
|
| 104 |
+
- key: qwen2.5-7b
|
| 105 |
+
hf_id: Qwen/Qwen2.5-7B-Instruct
|
| 106 |
+
hf_revision: a09a35458c702b33eeacc393d103063234e8bc28
|
| 107 |
+
ollama_q8: qwen2.5:7b-instruct-q8_0
|
| 108 |
+
|
| 109 |
+
targets:
|
| 110 |
+
synthetic_seeds: [202601, 202602]
|
| 111 |
+
|
| 112 |
+
inference:
|
| 113 |
+
temperature: 0
|
| 114 |
+
seed: 42
|
| 115 |
+
num_ctx: 2048
|
| 116 |
+
num_predict: 512
|
| 117 |
+
request_timeout_seconds: 900
|
| 118 |
+
keep_alive: 5m
|
| 119 |
+
|
| 120 |
+
statistics:
|
| 121 |
+
alpha: 0.05
|
| 122 |
+
null_accuracy: 0.5
|
| 123 |
+
bootstrap_samples: 10000
|
| 124 |
+
bootstrap_seed: 202603
|
| 125 |
+
|
| 126 |
+
training:
|
| 127 |
+
model_key: qwen2.5-3b
|
| 128 |
+
seed: 42
|
| 129 |
+
epochs: 8
|
| 130 |
+
evaluation_epochs: [0, 1, 2, 4, 8]
|
| 131 |
+
learning_rate: 0.00005
|
| 132 |
+
per_device_train_batch_size: 2
|
| 133 |
+
gradient_accumulation_steps: 1
|
| 134 |
+
max_length: 2048
|
| 135 |
+
flavor: a100-large
|
| 136 |
+
timeout: 5h
|
| 137 |
+
private_model_repo: Boopster/capbencher-qwen2.5-3b-contaminated-repro
|
| 138 |
+
private_results_repo: Boopster/capbencher-repro-staging
|
| 139 |
+
private_trackio_space: Boopster/capbencher-repro-training-private
|
| 140 |
+
|
| 141 |
+
paths:
|
| 142 |
+
prepared_data: artifacts/data/gsm8k_repro.jsonl
|
| 143 |
+
training_data: artifacts/data/gsm8k_contamination_sft.jsonl
|
| 144 |
+
data_manifest: artifacts/data/manifest.json
|
| 145 |
+
outputs: outputs
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
````
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
````output
|
| 152 |
+
Validated 1319/1319 rows; author-parser signs={'minus_one': 683, 'plus_one': 636}; semantic cap anomalies=14
|
| 153 |
+
|
| 154 |
+
````
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
---
|
| 158 |
+
<!-- trackio-cell
|
| 159 |
+
{"type": "artifact", "id": "cell_7ac29556dd4c", "created_at": "2026-07-15T17:57:33+00:00", "title": "Artifact: gsm8k_repro.jsonl", "path": "artifacts/data/gsm8k_repro.jsonl", "size": 2588383, "artifact_type": "dataset", "auto": true}
|
| 160 |
+
-->
|
| 161 |
+
**📦 Artifact** `artifacts/data/gsm8k_repro.jsonl` · dataset · 2.6 MB
|
| 162 |
+
|
| 163 |
+
https://huggingface.co/buckets/Boopster/oCNT5PcMSQ-artifacts#logbook-files/artifacts/data/gsm8k_repro.jsonl
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
---
|
| 167 |
+
<!-- trackio-cell
|
| 168 |
+
{"type": "artifact", "id": "cell_0aac627372f1", "created_at": "2026-07-15T17:57:33+00:00", "title": "Artifact: gsm8k_contamination_sft.jsonl", "path": "artifacts/data/gsm8k_contamination_sft.jsonl", "size": 1284437, "artifact_type": "dataset", "auto": true}
|
| 169 |
+
-->
|
| 170 |
+
**📦 Artifact** `artifacts/data/gsm8k_contamination_sft.jsonl` · dataset · 1.3 MB
|
| 171 |
+
|
| 172 |
+
https://huggingface.co/buckets/Boopster/oCNT5PcMSQ-artifacts#logbook-files/artifacts/data/gsm8k_contamination_sft.jsonl
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
---
|
| 176 |
+
<!-- trackio-cell
|
| 177 |
+
{"type": "markdown", "id": "cell_cb303f6d1a46", "created_at": "2026-07-15T17:39:16+00:00", "title": "Dataset quality finding"}
|
| 178 |
+
-->
|
| 179 |
+
All 1,319 questions align exactly. The published target column obeys the pinned authors’ parser ±1 rule for every row, but 14 targets are not semantically the true GSM8K answer ±1 because the authors’ regular expression stops at the first comma in thousands-formatted answers. The primary published-target reproduction preserves those rows exactly; two independently seeded assignments use correctly normalized integer answers. Full row IDs and deltas are recorded in artifacts/data/manifest.json.
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
---
|
| 183 |
+
<!-- trackio-cell
|
| 184 |
+
{"type": "code", "id": "cell_37daba060c2f", "created_at": "2026-07-15T18:51:56+00:00", "title": "Run: cap-audit repro.yaml (exit 0)", "command": [".venv/bin/cap-audit", "--config", "configs/repro.yaml", "--backend", "mac-q8-smoke", "--per-category", "5", "--seed", "202604"], "exit_code": 0, "duration_s": 0.548}
|
| 185 |
+
-->
|
| 186 |
+
````bash
|
| 187 |
+
$ .venv/bin/cap-audit --config configs/repro.yaml --backend mac-q8-smoke --per-category 5 --seed 202604
|
| 188 |
+
````
|
| 189 |
+
|
| 190 |
+
exit 0 · 0.5s
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
````yaml title=repro.yaml
|
| 194 |
+
project:
|
| 195 |
+
name: capbencher-icml2026-repro
|
| 196 |
+
owner: Boopster
|
| 197 |
+
paper_title: "How Can I Publish My LLM Benchmark Without Giving the True Answers Away?"
|
| 198 |
+
arxiv_id: "2505.18102"
|
| 199 |
+
openreview_id: oCNT5PcMSQ
|
| 200 |
+
logbook_space: Boopster/oCNT5PcMSQ
|
| 201 |
+
|
| 202 |
+
datasets:
|
| 203 |
+
capped:
|
| 204 |
+
id: ishidalab/capbencher
|
| 205 |
+
config: gsm8k-capped
|
| 206 |
+
split: test
|
| 207 |
+
revision: 62ab88c06dd78400f8092e61e6b23eff257cd2b8
|
| 208 |
+
original:
|
| 209 |
+
id: openai/gsm8k
|
| 210 |
+
config: main
|
| 211 |
+
split: test
|
| 212 |
+
revision: 740312add88f781978c0658806c59bc2815b9866
|
| 213 |
+
|
| 214 |
+
models:
|
| 215 |
+
- key: qwen2.5-0.5b
|
| 216 |
+
hf_id: Qwen/Qwen2.5-0.5B-Instruct
|
| 217 |
+
hf_revision: 7ae557604adf67be50417f59c2c2f167def9a775
|
| 218 |
+
ollama_q8: qwen2.5:0.5b-instruct-q8_0
|
| 219 |
+
- key: qwen2.5-1.5b
|
| 220 |
+
hf_id: Qwen/Qwen2.5-1.5B-Instruct
|
| 221 |
+
hf_revision: 989aa7980e4cf806f80c7fef2b1adb7bc71aa306
|
| 222 |
+
ollama_q8: qwen2.5:1.5b-instruct-q8_0
|
| 223 |
+
- key: qwen2.5-math-1.5b
|
| 224 |
+
hf_id: Qwen/Qwen2.5-Math-1.5B-Instruct
|
| 225 |
+
hf_revision: aafeb0fc6f22cbf0eaeed126eff8be45b0360a35
|
| 226 |
+
ollama_q8: capbencher-qwen2.5-math-1.5b-q8
|
| 227 |
+
- key: qwen2.5-3b
|
| 228 |
+
hf_id: Qwen/Qwen2.5-3B-Instruct
|
| 229 |
+
hf_revision: aa8e72537993ba99e69dfaafa59ed015b17504d1
|
| 230 |
+
ollama_q8: qwen2.5:3b-instruct-q8_0
|
| 231 |
+
- key: qwen2.5-7b
|
| 232 |
+
hf_id: Qwen/Qwen2.5-7B-Instruct
|
| 233 |
+
hf_revision: a09a35458c702b33eeacc393d103063234e8bc28
|
| 234 |
+
ollama_q8: qwen2.5:7b-instruct-q8_0
|
| 235 |
+
|
| 236 |
+
targets:
|
| 237 |
+
synthetic_seeds: [202601, 202602]
|
| 238 |
+
|
| 239 |
+
inference:
|
| 240 |
+
temperature: 0
|
| 241 |
+
seed: 42
|
| 242 |
+
num_ctx: 2048
|
| 243 |
+
num_predict: 512
|
| 244 |
+
request_timeout_seconds: 900
|
| 245 |
+
keep_alive: 5m
|
| 246 |
+
|
| 247 |
+
statistics:
|
| 248 |
+
alpha: 0.05
|
| 249 |
+
null_accuracy: 0.5
|
| 250 |
+
bootstrap_samples: 10000
|
| 251 |
+
bootstrap_seed: 202603
|
| 252 |
+
|
| 253 |
+
training:
|
| 254 |
+
model_key: qwen2.5-3b
|
| 255 |
+
seed: 42
|
| 256 |
+
epochs: 8
|
| 257 |
+
evaluation_epochs: [0, 1, 2, 4, 8]
|
| 258 |
+
learning_rate: 0.00005
|
| 259 |
+
per_device_train_batch_size: 2
|
| 260 |
+
gradient_accumulation_steps: 1
|
| 261 |
+
max_length: 2048
|
| 262 |
+
flavor: a100-large
|
| 263 |
+
timeout: 5h
|
| 264 |
+
private_model_repo: Boopster/capbencher-qwen2.5-3b-contaminated-repro
|
| 265 |
+
private_results_repo: Boopster/capbencher-repro-staging
|
| 266 |
+
private_trackio_space: Boopster/capbencher-repro-training-private
|
| 267 |
+
|
| 268 |
+
paths:
|
| 269 |
+
prepared_data: artifacts/data/gsm8k_repro.jsonl
|
| 270 |
+
training_data: artifacts/data/gsm8k_contamination_sft.jsonl
|
| 271 |
+
data_manifest: artifacts/data/manifest.json
|
| 272 |
+
outputs: outputs
|
| 273 |
+
|
| 274 |
+
|
| 275 |
+
````
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
````output
|
| 279 |
+
{
|
| 280 |
+
"backend": "mac-q8-smoke",
|
| 281 |
+
"category_counts": {
|
| 282 |
+
"parser_disagreement": 27,
|
| 283 |
+
"strict_correct": 1,
|
| 284 |
+
"strict_incorrect": 4,
|
| 285 |
+
"strict_invalid": 27
|
| 286 |
+
},
|
| 287 |
+
"created_at": "2026-07-15T18:51:56.180123+00:00",
|
| 288 |
+
"per_category": 5,
|
| 289 |
+
"sampled_rows": 15,
|
| 290 |
+
"secret_scan_findings": [],
|
| 291 |
+
"seed": 202604
|
| 292 |
+
}
|
| 293 |
+
|
| 294 |
+
````
|
| 295 |
+
|
| 296 |
+
|
| 297 |
+
---
|
| 298 |
+
<!-- trackio-cell
|
| 299 |
+
{"type": "markdown", "id": "cell_65cfa44f42f4", "created_at": "2026-07-15T18:51:56+00:00", "title": "Retired mutable path reference: early audit sample"}
|
| 300 |
+
-->
|
| 301 |
+
This auto-captured cell originally described a 17,935-byte sample at `outputs/audit/stratified_sample.jsonl`. A later audit overwrote the mutable path and no immutable copy of those bytes survives, so this path reference is retired and is **not evidence**. The code cell immediately above retains the contemporaneous printed audit summary.
|
pages/run-history-and-compute/page.md
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|