File size: 1,971 Bytes
7233de0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | {
"suite": "slm-benchmark-v1.1",
"generated_at": "Illustrative data — not an evaluation",
"models": [
{
"id": "demo-110m", "name": "Illustrative Base 110M", "params_m": 110,
"track": "base", "complete": true, "rank_range": [1, 2],
"composite": {"score": 16.8, "ci": [15.1, 18.6], "classification": "reliably above chance"},
"benchmarks": {
"hellaswag": {"score": 12.4, "ci": [10.7, 14.2]},
"piqa": {"score": 20.6, "ci": [17.1, 24.0]},
"arc_easy": {"score": 15.0, "ci": [11.5, 18.5]},
"arc_challenge": {"score": 7.1, "ci": [3.4, 10.7]},
"commonsense_qa": {"score": 8.9, "ci": [5.2, 12.6]},
"blimp": {"score": 27.2, "ci": [24.8, 29.6]}
}
},
{
"id": "demo-160m", "name": "Illustrative Base 160M", "params_m": 160,
"track": "base", "complete": true, "rank_range": [1, 2],
"composite": {"score": 17.7, "ci": [16.0, 19.5], "classification": "reliably above chance"},
"benchmarks": {
"hellaswag": {"score": 13.9, "ci": [12.1, 15.6]},
"piqa": {"score": 22.1, "ci": [18.7, 25.4]},
"arc_easy": {"score": 16.3, "ci": [12.9, 19.7]},
"arc_challenge": {"score": 9.8, "ci": [6.1, 13.5]},
"commonsense_qa": {"score": 10.6, "ci": [7.1, 14.0]},
"blimp": {"score": 25.8, "ci": [23.5, 28.1]}
}
},
{
"id": "demo-60m", "name": "Illustrative Base 60M", "params_m": 60,
"track": "base", "complete": true, "rank_range": [3, 3],
"composite": {"score": 5.6, "ci": [3.9, 7.4], "classification": "reliably above chance"},
"benchmarks": {
"hellaswag": {"score": 2.1, "ci": [0.5, 3.8]},
"piqa": {"score": 7.3, "ci": [4.0, 10.5]},
"arc_easy": {"score": 3.8, "ci": [0.8, 6.7]},
"arc_challenge": {"score": 1.4, "ci": [-2.4, 5.2]},
"commonsense_qa": {"score": 1.2, "ci": [-2.1, 4.4]},
"blimp": {"score": 13.6, "ci": [11.2, 16.0]}
}
}
]
}
|