| { | |
| "acc": 35.699999999999996, | |
| "acc_norm": 36.0, | |
| "categories": { | |
| "elementary_school_math_continuation::addition::grades_1_2::easy": { | |
| "acc": 28.125, | |
| "acc_norm": 28.125, | |
| "norm_correct": 36, | |
| "raw_correct": 36, | |
| "total": 128 | |
| }, | |
| "elementary_school_math_continuation::comparison::grades_2_3::medium": { | |
| "acc": 27.27272727272727, | |
| "acc_norm": 27.27272727272727, | |
| "norm_correct": 12, | |
| "raw_correct": 12, | |
| "total": 44 | |
| }, | |
| "elementary_school_math_continuation::comparison_difference::grades_2_3::medium": { | |
| "acc": 25.0, | |
| "acc_norm": 25.0, | |
| "norm_correct": 12, | |
| "raw_correct": 12, | |
| "total": 48 | |
| }, | |
| "elementary_school_math_continuation::data::grades_2_3::easy": { | |
| "acc": 23.25581395348837, | |
| "acc_norm": 25.581395348837212, | |
| "norm_correct": 11, | |
| "raw_correct": 10, | |
| "total": 43 | |
| }, | |
| "elementary_school_math_continuation::division::grades_3_4::medium": { | |
| "acc": 25.925925925925924, | |
| "acc_norm": 24.074074074074073, | |
| "norm_correct": 13, | |
| "raw_correct": 14, | |
| "total": 54 | |
| }, | |
| "elementary_school_math_continuation::fractions_counting::grades_3_4::medium": { | |
| "acc": 20.0, | |
| "acc_norm": 20.0, | |
| "norm_correct": 10, | |
| "raw_correct": 10, | |
| "total": 50 | |
| }, | |
| "elementary_school_math_continuation::geometry_area::grades_4_5::medium": { | |
| "acc": 51.92307692307693, | |
| "acc_norm": 53.84615384615385, | |
| "norm_correct": 28, | |
| "raw_correct": 27, | |
| "total": 52 | |
| }, | |
| "elementary_school_math_continuation::geometry_perimeter::grades_4_5::medium": { | |
| "acc": 62.22222222222222, | |
| "acc_norm": 62.22222222222222, | |
| "norm_correct": 28, | |
| "raw_correct": 28, | |
| "total": 45 | |
| }, | |
| "elementary_school_math_continuation::measurement::grades_2_3::easy": { | |
| "acc": 32.89473684210527, | |
| "acc_norm": 32.89473684210527, | |
| "norm_correct": 25, | |
| "raw_correct": 25, | |
| "total": 76 | |
| }, | |
| "elementary_school_math_continuation::money::grades_3_4::medium": { | |
| "acc": 29.6875, | |
| "acc_norm": 29.6875, | |
| "norm_correct": 19, | |
| "raw_correct": 19, | |
| "total": 64 | |
| }, | |
| "elementary_school_math_continuation::multiplication::grades_3_4::medium": { | |
| "acc": 51.35135135135135, | |
| "acc_norm": 52.702702702702695, | |
| "norm_correct": 39, | |
| "raw_correct": 38, | |
| "total": 74 | |
| }, | |
| "elementary_school_math_continuation::patterns::grades_3_4::medium": { | |
| "acc": 24.528301886792452, | |
| "acc_norm": 24.528301886792452, | |
| "norm_correct": 13, | |
| "raw_correct": 13, | |
| "total": 53 | |
| }, | |
| "elementary_school_math_continuation::subtraction::grades_1_2::easy": { | |
| "acc": 30.76923076923077, | |
| "acc_norm": 31.62393162393162, | |
| "norm_correct": 37, | |
| "raw_correct": 36, | |
| "total": 117 | |
| }, | |
| "elementary_school_math_continuation::time::grades_2_3::easy": { | |
| "acc": 100.0, | |
| "acc_norm": 100.0, | |
| "norm_correct": 55, | |
| "raw_correct": 55, | |
| "total": 55 | |
| }, | |
| "elementary_school_math_continuation::two_step_add_subtract::grades_2_3::medium": { | |
| "acc": 21.73913043478261, | |
| "acc_norm": 21.73913043478261, | |
| "norm_correct": 10, | |
| "raw_correct": 10, | |
| "total": 46 | |
| }, | |
| "elementary_school_math_continuation::two_step_addition::grades_2_3::medium": { | |
| "acc": 15.789473684210526, | |
| "acc_norm": 15.789473684210526, | |
| "norm_correct": 3, | |
| "raw_correct": 3, | |
| "total": 19 | |
| }, | |
| "elementary_school_math_continuation::two_step_subtraction::grades_2_3::medium": { | |
| "acc": 28.125, | |
| "acc_norm": 28.125, | |
| "norm_correct": 9, | |
| "raw_correct": 9, | |
| "total": 32 | |
| } | |
| }, | |
| "frost_evaluation": { | |
| "checkpoint": "D:\\llm\\hummingbird\\artifacts\\runs\\flightmix_10b_balanced_10m_muon_lrfix_20260921\\checkpoint-tokens-10000000000", | |
| "data_sha256": "bf8ab1a5193d52cdf0e05ff0b3ca226bdfcf416cb6e75562dcbe72e7e4559435", | |
| "dataset": "AxiomicLabs/Arithmark-3.0", | |
| "dataset_revision": "6f6e59dd9b7e2c63455f7af7f838f9ecc3d0a746", | |
| "device": "cuda", | |
| "evaluator_sha256": "09667e1ea05ecd8b91dbfea14a47365b555c72622afed9fc284c127ecd5e1bcd", | |
| "limit": null, | |
| "loop_count": 1, | |
| "numeric_serialization": "none", | |
| "precision": "bfloat16", | |
| "qk_norm_position": null, | |
| "tokenizer": "D:\\llm\\hummingbird\\artifacts\\runs\\e3_tokenizers\\tok_4k_digit" | |
| }, | |
| "norm_correct": 360, | |
| "raw_correct": 357, | |
| "timing": { | |
| "evaluation_seconds": 0.8658304999989923, | |
| "examples_per_second": 1154.9604685919055, | |
| "tokenization_seconds": 0.30124549998436123 | |
| }, | |
| "total": 1000 | |
| } |