| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.9813333333333332, | |
| "eval_steps": 10, | |
| "global_step": 186, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "completion_length": 535.1041831970215, | |
| "epoch": 0.010666666666666666, | |
| "grad_norm": 0.10447478294372559, | |
| "kl": 0.0, | |
| "learning_rate": 1.5789473684210525e-07, | |
| "loss": -0.0025, | |
| "reward": 0.031250000931322575, | |
| "reward_std": 0.10825317353010178, | |
| "rewards/accuracy_reward": 0.031250000931322575, | |
| "rewards/format_reward": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "completion_length": 555.5937728881836, | |
| "epoch": 0.021333333333333333, | |
| "grad_norm": 0.05305163562297821, | |
| "kl": 0.0, | |
| "learning_rate": 3.157894736842105e-07, | |
| "loss": 0.009, | |
| "reward": 0.02083333395421505, | |
| "reward_std": 0.04865618050098419, | |
| "rewards/accuracy_reward": 0.02083333395421505, | |
| "rewards/format_reward": 0.0, | |
| "step": 2 | |
| }, | |
| { | |
| "completion_length": 510.41668701171875, | |
| "epoch": 0.032, | |
| "grad_norm": 0.0009671024745330215, | |
| "kl": 0.00044345855712890625, | |
| "learning_rate": 4.736842105263158e-07, | |
| "loss": 0.0, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "step": 3 | |
| }, | |
| { | |
| "completion_length": 570.4166831970215, | |
| "epoch": 0.042666666666666665, | |
| "grad_norm": 0.06929787248373032, | |
| "kl": 0.00043070316314697266, | |
| "learning_rate": 6.31578947368421e-07, | |
| "loss": -0.0023, | |
| "reward": 0.02083333395421505, | |
| "reward_std": 0.04865618050098419, | |
| "rewards/accuracy_reward": 0.02083333395421505, | |
| "rewards/format_reward": 0.0, | |
| "step": 4 | |
| }, | |
| { | |
| "completion_length": 564.0833435058594, | |
| "epoch": 0.05333333333333334, | |
| "grad_norm": 0.03304513171315193, | |
| "kl": 0.0004966259002685547, | |
| "learning_rate": 7.894736842105263e-07, | |
| "loss": 0.0287, | |
| "reward": 0.010416666977107525, | |
| "reward_std": 0.03608439117670059, | |
| "rewards/accuracy_reward": 0.010416666977107525, | |
| "rewards/format_reward": 0.0, | |
| "step": 5 | |
| }, | |
| { | |
| "completion_length": 434.66668701171875, | |
| "epoch": 0.064, | |
| "grad_norm": 0.0016080428613349795, | |
| "kl": 0.0005583763122558594, | |
| "learning_rate": 9.473684210526316e-07, | |
| "loss": 0.0, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "step": 6 | |
| }, | |
| { | |
| "completion_length": 617.7396087646484, | |
| "epoch": 0.07466666666666667, | |
| "grad_norm": 0.08006145060062408, | |
| "kl": 0.000385284423828125, | |
| "learning_rate": 1.1052631578947369e-06, | |
| "loss": 0.0445, | |
| "reward": 0.052083334885537624, | |
| "reward_std": 0.13339675962924957, | |
| "rewards/accuracy_reward": 0.052083334885537624, | |
| "rewards/format_reward": 0.0, | |
| "step": 7 | |
| }, | |
| { | |
| "completion_length": 527.3333511352539, | |
| "epoch": 0.08533333333333333, | |
| "grad_norm": 0.0587327815592289, | |
| "kl": 0.0003993511199951172, | |
| "learning_rate": 1.263157894736842e-06, | |
| "loss": 0.0507, | |
| "reward": 0.031250000931322575, | |
| "reward_std": 0.10825317353010178, | |
| "rewards/accuracy_reward": 0.031250000931322575, | |
| "rewards/format_reward": 0.0, | |
| "step": 8 | |
| }, | |
| { | |
| "completion_length": 551.2083435058594, | |
| "epoch": 0.096, | |
| "grad_norm": 0.001157123246230185, | |
| "kl": 0.0004718303680419922, | |
| "learning_rate": 1.4210526315789473e-06, | |
| "loss": 0.0, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "step": 9 | |
| }, | |
| { | |
| "completion_length": 486.7291793823242, | |
| "epoch": 0.10666666666666667, | |
| "grad_norm": 0.12886527180671692, | |
| "kl": 0.00043964385986328125, | |
| "learning_rate": 1.5789473684210526e-06, | |
| "loss": -0.0082, | |
| "reward": 0.08333333395421505, | |
| "reward_std": 0.13744790107011795, | |
| "rewards/accuracy_reward": 0.08333333395421505, | |
| "rewards/format_reward": 0.0, | |
| "step": 10 | |
| }, | |
| { | |
| "completion_length": 512.5000114440918, | |
| "epoch": 0.11733333333333333, | |
| "grad_norm": 0.10466578602790833, | |
| "kl": 0.0008594989776611328, | |
| "learning_rate": 1.736842105263158e-06, | |
| "loss": 0.0013, | |
| "reward": 0.031250000931322575, | |
| "reward_std": 0.10825317353010178, | |
| "rewards/accuracy_reward": 0.031250000931322575, | |
| "rewards/format_reward": 0.0, | |
| "step": 11 | |
| }, | |
| { | |
| "completion_length": 485.0000114440918, | |
| "epoch": 0.128, | |
| "grad_norm": 0.17413121461868286, | |
| "kl": 0.0007164478302001953, | |
| "learning_rate": 1.8947368421052632e-06, | |
| "loss": -0.0509, | |
| "reward": 0.031250000931322575, | |
| "reward_std": 0.10825317353010178, | |
| "rewards/accuracy_reward": 0.031250000931322575, | |
| "rewards/format_reward": 0.0, | |
| "step": 12 | |
| }, | |
| { | |
| "completion_length": 495.39584732055664, | |
| "epoch": 0.13866666666666666, | |
| "grad_norm": 0.0025116761680692434, | |
| "kl": 0.0007200241088867188, | |
| "learning_rate": 2.0526315789473687e-06, | |
| "loss": 0.0, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "step": 13 | |
| }, | |
| { | |
| "completion_length": 579.197925567627, | |
| "epoch": 0.14933333333333335, | |
| "grad_norm": 0.10607028752565384, | |
| "kl": 0.001117706298828125, | |
| "learning_rate": 2.2105263157894738e-06, | |
| "loss": 0.0997, | |
| "reward": 0.05208333395421505, | |
| "reward_std": 0.12870215997099876, | |
| "rewards/accuracy_reward": 0.05208333395421505, | |
| "rewards/format_reward": 0.0, | |
| "step": 14 | |
| }, | |
| { | |
| "completion_length": 618.0729370117188, | |
| "epoch": 0.16, | |
| "grad_norm": 0.04636254161596298, | |
| "kl": 0.00128936767578125, | |
| "learning_rate": 2.368421052631579e-06, | |
| "loss": -0.0278, | |
| "reward": 0.010416666977107525, | |
| "reward_std": 0.03608439117670059, | |
| "rewards/accuracy_reward": 0.010416666977107525, | |
| "rewards/format_reward": 0.0, | |
| "step": 15 | |
| }, | |
| { | |
| "completion_length": 529.8020973205566, | |
| "epoch": 0.17066666666666666, | |
| "grad_norm": 0.12151255458593369, | |
| "kl": 0.0032758712768554688, | |
| "learning_rate": 2.526315789473684e-06, | |
| "loss": 0.0198, | |
| "reward": 0.0416666679084301, | |
| "reward_std": 0.14433756470680237, | |
| "rewards/accuracy_reward": 0.0416666679084301, | |
| "rewards/format_reward": 0.0, | |
| "step": 16 | |
| }, | |
| { | |
| "completion_length": 583.6979370117188, | |
| "epoch": 0.18133333333333335, | |
| "grad_norm": 0.09600550681352615, | |
| "kl": 0.0034093856811523438, | |
| "learning_rate": 2.6842105263157895e-06, | |
| "loss": 0.0187, | |
| "reward": 0.0729166679084301, | |
| "reward_std": 0.17735834047198296, | |
| "rewards/accuracy_reward": 0.0729166679084301, | |
| "rewards/format_reward": 0.0, | |
| "step": 17 | |
| }, | |
| { | |
| "completion_length": 526.9583473205566, | |
| "epoch": 0.192, | |
| "grad_norm": 0.09340275824069977, | |
| "kl": 0.005596160888671875, | |
| "learning_rate": 2.8421052631578946e-06, | |
| "loss": 0.026, | |
| "reward": 0.0416666679084301, | |
| "reward_std": 0.12082496285438538, | |
| "rewards/accuracy_reward": 0.0416666679084301, | |
| "rewards/format_reward": 0.0, | |
| "step": 18 | |
| }, | |
| { | |
| "completion_length": 465.8645935058594, | |
| "epoch": 0.20266666666666666, | |
| "grad_norm": 0.08133361488580704, | |
| "kl": 0.006252288818359375, | |
| "learning_rate": 3e-06, | |
| "loss": -0.0449, | |
| "reward": 0.052083334885537624, | |
| "reward_std": 0.13339675962924957, | |
| "rewards/accuracy_reward": 0.052083334885537624, | |
| "rewards/format_reward": 0.0, | |
| "step": 19 | |
| }, | |
| { | |
| "completion_length": 525.0937690734863, | |
| "epoch": 0.21333333333333335, | |
| "grad_norm": 0.1193581074476242, | |
| "kl": 0.008209228515625, | |
| "learning_rate": 2.9997345912364375e-06, | |
| "loss": -0.0093, | |
| "reward": 0.07291666883975267, | |
| "reward_std": 0.16979892551898956, | |
| "rewards/accuracy_reward": 0.07291666883975267, | |
| "rewards/format_reward": 0.0, | |
| "step": 20 | |
| }, | |
| { | |
| "completion_length": 504.42709732055664, | |
| "epoch": 0.224, | |
| "grad_norm": 0.1320876032114029, | |
| "kl": 0.011646270751953125, | |
| "learning_rate": 2.9989384588681657e-06, | |
| "loss": -0.027, | |
| "reward": 0.13541666697710752, | |
| "reward_std": 0.21443026512861252, | |
| "rewards/accuracy_reward": 0.13541666697710752, | |
| "rewards/format_reward": 0.0, | |
| "step": 21 | |
| }, | |
| { | |
| "completion_length": 566.2187690734863, | |
| "epoch": 0.23466666666666666, | |
| "grad_norm": 0.11368165165185928, | |
| "kl": 0.012992382049560547, | |
| "learning_rate": 2.9976118846291945e-06, | |
| "loss": 0.0194, | |
| "reward": 0.06250000186264515, | |
| "reward_std": 0.19299374520778656, | |
| "rewards/accuracy_reward": 0.06250000186264515, | |
| "rewards/format_reward": 0.0, | |
| "step": 22 | |
| }, | |
| { | |
| "completion_length": 457.18751525878906, | |
| "epoch": 0.24533333333333332, | |
| "grad_norm": 0.21248622238636017, | |
| "kl": 0.023479461669921875, | |
| "learning_rate": 2.995755337965428e-06, | |
| "loss": 0.0218, | |
| "reward": 0.11458333767950535, | |
| "reward_std": 0.2338472306728363, | |
| "rewards/accuracy_reward": 0.11458333767950535, | |
| "rewards/format_reward": 0.0, | |
| "step": 23 | |
| }, | |
| { | |
| "completion_length": 559.9166831970215, | |
| "epoch": 0.256, | |
| "grad_norm": 0.129845529794693, | |
| "kl": 0.02849578857421875, | |
| "learning_rate": 2.99336947586854e-06, | |
| "loss": -0.035, | |
| "reward": 0.12500000186264515, | |
| "reward_std": 0.28254789859056473, | |
| "rewards/accuracy_reward": 0.12500000186264515, | |
| "rewards/format_reward": 0.0, | |
| "step": 24 | |
| }, | |
| { | |
| "completion_length": 564.6979446411133, | |
| "epoch": 0.26666666666666666, | |
| "grad_norm": 0.10117102414369583, | |
| "kl": 0.0623321533203125, | |
| "learning_rate": 2.9904551426434754e-06, | |
| "loss": 0.0704, | |
| "reward": 0.1145833358168602, | |
| "reward_std": 0.23890409246087074, | |
| "rewards/accuracy_reward": 0.1145833358168602, | |
| "rewards/format_reward": 0.0, | |
| "step": 25 | |
| }, | |
| { | |
| "completion_length": 605.0937690734863, | |
| "epoch": 0.2773333333333333, | |
| "grad_norm": 0.1382395625114441, | |
| "kl": 0.037479400634765625, | |
| "learning_rate": 2.987013369609672e-06, | |
| "loss": -0.0203, | |
| "reward": 0.10416666977107525, | |
| "reward_std": 0.26679351180791855, | |
| "rewards/accuracy_reward": 0.10416666977107525, | |
| "rewards/format_reward": 0.0, | |
| "step": 26 | |
| }, | |
| { | |
| "completion_length": 517.604190826416, | |
| "epoch": 0.288, | |
| "grad_norm": 0.21559005975723267, | |
| "kl": 0.061981201171875, | |
| "learning_rate": 2.983045374736096e-06, | |
| "loss": 0.0576, | |
| "reward": 0.14583333767950535, | |
| "reward_std": 0.17174768447875977, | |
| "rewards/accuracy_reward": 0.14583333767950535, | |
| "rewards/format_reward": 0.0, | |
| "step": 27 | |
| }, | |
| { | |
| "completion_length": 594.8958473205566, | |
| "epoch": 0.2986666666666667, | |
| "grad_norm": 0.1260552555322647, | |
| "kl": 0.02332305908203125, | |
| "learning_rate": 2.978552562210232e-06, | |
| "loss": 0.0939, | |
| "reward": 0.12500000279396772, | |
| "reward_std": 0.25147588923573494, | |
| "rewards/accuracy_reward": 0.12500000279396772, | |
| "rewards/format_reward": 0.0, | |
| "step": 28 | |
| }, | |
| { | |
| "completion_length": 546.0312690734863, | |
| "epoch": 0.30933333333333335, | |
| "grad_norm": 0.12078198045492172, | |
| "kl": 0.0504302978515625, | |
| "learning_rate": 2.9735365219411707e-06, | |
| "loss": -0.0113, | |
| "reward": 0.18750001024454832, | |
| "reward_std": 0.14910665899515152, | |
| "rewards/accuracy_reward": 0.18750001024454832, | |
| "rewards/format_reward": 0.0, | |
| "step": 29 | |
| }, | |
| { | |
| "completion_length": 469.4896049499512, | |
| "epoch": 0.32, | |
| "grad_norm": 0.16322949528694153, | |
| "kl": 0.0821685791015625, | |
| "learning_rate": 2.9679990289969723e-06, | |
| "loss": -0.011, | |
| "reward": 0.15625000186264515, | |
| "reward_std": 0.2555270306766033, | |
| "rewards/accuracy_reward": 0.15625000186264515, | |
| "rewards/format_reward": 0.0, | |
| "step": 30 | |
| }, | |
| { | |
| "completion_length": 556.3125152587891, | |
| "epoch": 0.33066666666666666, | |
| "grad_norm": 0.12340069562196732, | |
| "kl": 0.03859710693359375, | |
| "learning_rate": 2.9619420429765106e-06, | |
| "loss": -0.0312, | |
| "reward": 0.18750000093132257, | |
| "reward_std": 0.2543407045304775, | |
| "rewards/accuracy_reward": 0.18750000093132257, | |
| "rewards/format_reward": 0.0, | |
| "step": 31 | |
| }, | |
| { | |
| "completion_length": 469.6770896911621, | |
| "epoch": 0.3413333333333333, | |
| "grad_norm": 0.1940806806087494, | |
| "kl": 0.06516265869140625, | |
| "learning_rate": 2.955367707316014e-06, | |
| "loss": -0.0537, | |
| "reward": 0.1979166716337204, | |
| "reward_std": 0.2885033190250397, | |
| "rewards/accuracy_reward": 0.1979166716337204, | |
| "rewards/format_reward": 0.0, | |
| "step": 32 | |
| }, | |
| { | |
| "completion_length": 602.7396011352539, | |
| "epoch": 0.352, | |
| "grad_norm": 0.09024544060230255, | |
| "kl": 0.02783203125, | |
| "learning_rate": 2.948278348530547e-06, | |
| "loss": 0.0558, | |
| "reward": 0.12500000186264515, | |
| "reward_std": 0.21124766021966934, | |
| "rewards/accuracy_reward": 0.12500000186264515, | |
| "rewards/format_reward": 0.0, | |
| "step": 33 | |
| }, | |
| { | |
| "completion_length": 593.6770935058594, | |
| "epoch": 0.3626666666666667, | |
| "grad_norm": 0.08488824963569641, | |
| "kl": 0.05257415771484375, | |
| "learning_rate": 2.9406764753907096e-06, | |
| "loss": 0.0303, | |
| "reward": 0.07291666883975267, | |
| "reward_std": 0.18205294013023376, | |
| "rewards/accuracy_reward": 0.07291666883975267, | |
| "rewards/format_reward": 0.0, | |
| "step": 34 | |
| }, | |
| { | |
| "completion_length": 574.5312728881836, | |
| "epoch": 0.37333333333333335, | |
| "grad_norm": 0.12150894105434418, | |
| "kl": 0.03229522705078125, | |
| "learning_rate": 2.9325647780348364e-06, | |
| "loss": 0.0854, | |
| "reward": 0.1041666679084301, | |
| "reward_std": 0.23389172554016113, | |
| "rewards/accuracy_reward": 0.1041666679084301, | |
| "rewards/format_reward": 0.0, | |
| "step": 35 | |
| }, | |
| { | |
| "completion_length": 626.3750152587891, | |
| "epoch": 0.384, | |
| "grad_norm": 0.12855271995067596, | |
| "kl": 0.02007293701171875, | |
| "learning_rate": 2.9239461270170137e-06, | |
| "loss": 0.0565, | |
| "reward": 0.17708333674818277, | |
| "reward_std": 0.3208988308906555, | |
| "rewards/accuracy_reward": 0.17708333674818277, | |
| "rewards/format_reward": 0.0, | |
| "step": 36 | |
| }, | |
| { | |
| "completion_length": 628.3437652587891, | |
| "epoch": 0.39466666666666667, | |
| "grad_norm": 0.14817272126674652, | |
| "kl": 0.03232574462890625, | |
| "learning_rate": 2.914823572291257e-06, | |
| "loss": 0.0374, | |
| "reward": 0.20833333767950535, | |
| "reward_std": 0.2605394050478935, | |
| "rewards/accuracy_reward": 0.20833333767950535, | |
| "rewards/format_reward": 0.0, | |
| "step": 37 | |
| }, | |
| { | |
| "completion_length": 580.7604331970215, | |
| "epoch": 0.4053333333333333, | |
| "grad_norm": 0.17469142377376556, | |
| "kl": 0.0690155029296875, | |
| "learning_rate": 2.9052003421321917e-06, | |
| "loss": 0.038, | |
| "reward": 0.0729166679084301, | |
| "reward_std": 0.15384574607014656, | |
| "rewards/accuracy_reward": 0.0729166679084301, | |
| "rewards/format_reward": 0.0, | |
| "step": 38 | |
| }, | |
| { | |
| "completion_length": 609.6666831970215, | |
| "epoch": 0.416, | |
| "grad_norm": 0.16530297696590424, | |
| "kl": 0.06507110595703125, | |
| "learning_rate": 2.8950798419926422e-06, | |
| "loss": 0.0371, | |
| "reward": 0.22916667629033327, | |
| "reward_std": 0.37515120208263397, | |
| "rewards/accuracy_reward": 0.22916667629033327, | |
| "rewards/format_reward": 0.0, | |
| "step": 39 | |
| }, | |
| { | |
| "completion_length": 589.1979293823242, | |
| "epoch": 0.4266666666666667, | |
| "grad_norm": 0.16562184691429138, | |
| "kl": 0.0455322265625, | |
| "learning_rate": 2.884465653298514e-06, | |
| "loss": 0.1025, | |
| "reward": 0.17708334047347307, | |
| "reward_std": 0.2671857886016369, | |
| "rewards/accuracy_reward": 0.17708334047347307, | |
| "rewards/format_reward": 0.0, | |
| "step": 40 | |
| }, | |
| { | |
| "completion_length": 597.1875190734863, | |
| "epoch": 0.43733333333333335, | |
| "grad_norm": 0.46513137221336365, | |
| "kl": 0.05785369873046875, | |
| "learning_rate": 2.8733615321814036e-06, | |
| "loss": 0.1034, | |
| "reward": 0.2291666753590107, | |
| "reward_std": 0.339111290872097, | |
| "rewards/accuracy_reward": 0.2291666753590107, | |
| "rewards/format_reward": 0.0, | |
| "step": 41 | |
| }, | |
| { | |
| "completion_length": 553.8333435058594, | |
| "epoch": 0.448, | |
| "grad_norm": 0.12673671543598175, | |
| "kl": 0.06561279296875, | |
| "learning_rate": 2.8617714081493856e-06, | |
| "loss": 0.0195, | |
| "reward": 0.19791667815297842, | |
| "reward_std": 0.28564153611660004, | |
| "rewards/accuracy_reward": 0.19791667815297842, | |
| "rewards/format_reward": 0.0, | |
| "step": 42 | |
| }, | |
| { | |
| "completion_length": 502.4479331970215, | |
| "epoch": 0.45866666666666667, | |
| "grad_norm": 0.20127542316913605, | |
| "kl": 0.103759765625, | |
| "learning_rate": 2.849699382696445e-06, | |
| "loss": -0.0076, | |
| "reward": 0.22916667256504297, | |
| "reward_std": 0.2759315297007561, | |
| "rewards/accuracy_reward": 0.22916667256504297, | |
| "rewards/format_reward": 0.0, | |
| "step": 43 | |
| }, | |
| { | |
| "completion_length": 490.50000762939453, | |
| "epoch": 0.4693333333333333, | |
| "grad_norm": 0.231169655919075, | |
| "kl": 0.0522918701171875, | |
| "learning_rate": 2.8371497278510474e-06, | |
| "loss": 0.1514, | |
| "reward": 0.3645833432674408, | |
| "reward_std": 0.427349217236042, | |
| "rewards/accuracy_reward": 0.3645833432674408, | |
| "rewards/format_reward": 0.0, | |
| "step": 44 | |
| }, | |
| { | |
| "completion_length": 597.1875228881836, | |
| "epoch": 0.48, | |
| "grad_norm": 0.48629194498062134, | |
| "kl": 0.05603790283203125, | |
| "learning_rate": 2.8241268846643613e-06, | |
| "loss": 0.0505, | |
| "reward": 0.15625000279396772, | |
| "reward_std": 0.22226297482848167, | |
| "rewards/accuracy_reward": 0.15625000279396772, | |
| "rewards/format_reward": 0.0, | |
| "step": 45 | |
| }, | |
| { | |
| "completion_length": 529.3333587646484, | |
| "epoch": 0.49066666666666664, | |
| "grad_norm": 0.14516475796699524, | |
| "kl": 0.0585784912109375, | |
| "learning_rate": 2.8106354616386633e-06, | |
| "loss": 0.0372, | |
| "reward": 0.3229166744276881, | |
| "reward_std": 0.3968754820525646, | |
| "rewards/accuracy_reward": 0.3229166744276881, | |
| "rewards/format_reward": 0.0, | |
| "step": 46 | |
| }, | |
| { | |
| "completion_length": 586.8333549499512, | |
| "epoch": 0.5013333333333333, | |
| "grad_norm": 0.13861708343029022, | |
| "kl": 0.0889892578125, | |
| "learning_rate": 2.796680233096492e-06, | |
| "loss": -0.0095, | |
| "reward": 0.1770833358168602, | |
| "reward_std": 0.21539149805903435, | |
| "rewards/accuracy_reward": 0.1770833358168602, | |
| "rewards/format_reward": 0.0, | |
| "step": 47 | |
| }, | |
| { | |
| "completion_length": 578.0312614440918, | |
| "epoch": 0.512, | |
| "grad_norm": 0.14956602454185486, | |
| "kl": 0.1368408203125, | |
| "learning_rate": 2.7822661374911177e-06, | |
| "loss": 0.0516, | |
| "reward": 0.2708333395421505, | |
| "reward_std": 0.30700354650616646, | |
| "rewards/accuracy_reward": 0.2708333395421505, | |
| "rewards/format_reward": 0.0, | |
| "step": 48 | |
| }, | |
| { | |
| "completion_length": 623.2916870117188, | |
| "epoch": 0.5226666666666666, | |
| "grad_norm": 0.18586260080337524, | |
| "kl": 0.120880126953125, | |
| "learning_rate": 2.7673982756589312e-06, | |
| "loss": 0.0158, | |
| "reward": 0.22916667070239782, | |
| "reward_std": 0.3167550005018711, | |
| "rewards/accuracy_reward": 0.22916667070239782, | |
| "rewards/format_reward": 0.0, | |
| "step": 49 | |
| }, | |
| { | |
| "completion_length": 697.7291793823242, | |
| "epoch": 0.5333333333333333, | |
| "grad_norm": 6.404502868652344, | |
| "kl": 0.319976806640625, | |
| "learning_rate": 2.7520819090143655e-06, | |
| "loss": 0.0299, | |
| "reward": 0.1562500037252903, | |
| "reward_std": 0.2504701688885689, | |
| "rewards/accuracy_reward": 0.1562500037252903, | |
| "rewards/format_reward": 0.0, | |
| "step": 50 | |
| }, | |
| { | |
| "completion_length": 584.5208435058594, | |
| "epoch": 0.544, | |
| "grad_norm": 0.21096980571746826, | |
| "kl": 0.162750244140625, | |
| "learning_rate": 2.736322457687999e-06, | |
| "loss": -0.0173, | |
| "reward": 0.19791667070239782, | |
| "reward_std": 0.3315218612551689, | |
| "rewards/accuracy_reward": 0.19791667070239782, | |
| "rewards/format_reward": 0.0, | |
| "step": 51 | |
| }, | |
| { | |
| "completion_length": 646.1875152587891, | |
| "epoch": 0.5546666666666666, | |
| "grad_norm": 0.34582799673080444, | |
| "kl": 0.1904296875, | |
| "learning_rate": 2.720125498608487e-06, | |
| "loss": 0.0753, | |
| "reward": 0.10416666883975267, | |
| "reward_std": 0.20281970128417015, | |
| "rewards/accuracy_reward": 0.10416666883975267, | |
| "rewards/format_reward": 0.0, | |
| "step": 52 | |
| }, | |
| { | |
| "completion_length": 722.7604370117188, | |
| "epoch": 0.5653333333333334, | |
| "grad_norm": 0.16253501176834106, | |
| "kl": 0.1534423828125, | |
| "learning_rate": 2.7034967635290054e-06, | |
| "loss": 0.1153, | |
| "reward": 0.23958333674818277, | |
| "reward_std": 0.25935307890176773, | |
| "rewards/accuracy_reward": 0.23958333674818277, | |
| "rewards/format_reward": 0.0, | |
| "step": 53 | |
| }, | |
| { | |
| "completion_length": 713.0312652587891, | |
| "epoch": 0.576, | |
| "grad_norm": 0.1711745262145996, | |
| "kl": 0.2108154296875, | |
| "learning_rate": 2.6864421369989112e-06, | |
| "loss": 0.0778, | |
| "reward": 0.16666667070239782, | |
| "reward_std": 0.27498848363757133, | |
| "rewards/accuracy_reward": 0.16666667070239782, | |
| "rewards/format_reward": 0.0, | |
| "step": 54 | |
| }, | |
| { | |
| "completion_length": 709.7916946411133, | |
| "epoch": 0.5866666666666667, | |
| "grad_norm": 0.21810775995254517, | |
| "kl": 0.18017578125, | |
| "learning_rate": 2.668967654281324e-06, | |
| "loss": 0.0342, | |
| "reward": 0.2395833469927311, | |
| "reward_std": 0.254644013941288, | |
| "rewards/accuracy_reward": 0.2395833469927311, | |
| "rewards/format_reward": 0.0, | |
| "step": 55 | |
| }, | |
| { | |
| "completion_length": 673.8229293823242, | |
| "epoch": 0.5973333333333334, | |
| "grad_norm": 0.3457491099834442, | |
| "kl": 0.2532958984375, | |
| "learning_rate": 2.65107949921738e-06, | |
| "loss": 0.1691, | |
| "reward": 0.20833333674818277, | |
| "reward_std": 0.3889237828552723, | |
| "rewards/accuracy_reward": 0.20833333674818277, | |
| "rewards/format_reward": 0.0, | |
| "step": 56 | |
| }, | |
| { | |
| "completion_length": 575.7500152587891, | |
| "epoch": 0.608, | |
| "grad_norm": 0.8664973974227905, | |
| "kl": 0.773681640625, | |
| "learning_rate": 2.632784002037904e-06, | |
| "loss": 0.1391, | |
| "reward": 0.3229166716337204, | |
| "reward_std": 0.4173244684934616, | |
| "rewards/accuracy_reward": 0.3229166716337204, | |
| "rewards/format_reward": 0.0, | |
| "step": 57 | |
| }, | |
| { | |
| "completion_length": 687.2500152587891, | |
| "epoch": 0.6186666666666667, | |
| "grad_norm": 0.33583614230155945, | |
| "kl": 0.62158203125, | |
| "learning_rate": 2.6140876371232755e-06, | |
| "loss": 0.051, | |
| "reward": 0.17708334047347307, | |
| "reward_std": 0.2671857886016369, | |
| "rewards/accuracy_reward": 0.17708334047347307, | |
| "rewards/format_reward": 0.0, | |
| "step": 58 | |
| }, | |
| { | |
| "completion_length": 726.1875076293945, | |
| "epoch": 0.6293333333333333, | |
| "grad_norm": 2.140490770339966, | |
| "kl": 1.1827392578125, | |
| "learning_rate": 2.594997020712288e-06, | |
| "loss": 0.043, | |
| "reward": 0.08333333395421505, | |
| "reward_std": 0.13744790107011795, | |
| "rewards/accuracy_reward": 0.08333333395421505, | |
| "rewards/format_reward": 0.0, | |
| "step": 59 | |
| }, | |
| { | |
| "completion_length": 749.4895935058594, | |
| "epoch": 0.64, | |
| "grad_norm": 0.2817237377166748, | |
| "kl": 0.57421875, | |
| "learning_rate": 2.5755189085608046e-06, | |
| "loss": 0.0747, | |
| "reward": 0.09375000279396772, | |
| "reward_std": 0.24196770787239075, | |
| "rewards/accuracy_reward": 0.09375000279396772, | |
| "rewards/format_reward": 0.0, | |
| "step": 60 | |
| }, | |
| { | |
| "completion_length": 735.6354370117188, | |
| "epoch": 0.6506666666666666, | |
| "grad_norm": 0.3483006954193115, | |
| "kl": 0.52978515625, | |
| "learning_rate": 2.5556601935510422e-06, | |
| "loss": 0.0503, | |
| "reward": 0.15625000186264515, | |
| "reward_std": 0.2912936396896839, | |
| "rewards/accuracy_reward": 0.15625000186264515, | |
| "rewards/format_reward": 0.0, | |
| "step": 61 | |
| }, | |
| { | |
| "completion_length": 692.6041793823242, | |
| "epoch": 0.6613333333333333, | |
| "grad_norm": 0.3014882206916809, | |
| "kl": 0.6661376953125, | |
| "learning_rate": 2.53542790325233e-06, | |
| "loss": 0.0723, | |
| "reward": 0.0416666679084301, | |
| "reward_std": 0.06154575198888779, | |
| "rewards/accuracy_reward": 0.0416666679084301, | |
| "rewards/format_reward": 0.0, | |
| "step": 62 | |
| }, | |
| { | |
| "completion_length": 617.5937728881836, | |
| "epoch": 0.672, | |
| "grad_norm": 0.887993335723877, | |
| "kl": 0.4677734375, | |
| "learning_rate": 2.5148291974342035e-06, | |
| "loss": 0.1084, | |
| "reward": 0.322916672565043, | |
| "reward_std": 0.41752326115965843, | |
| "rewards/accuracy_reward": 0.322916672565043, | |
| "rewards/format_reward": 0.0, | |
| "step": 63 | |
| }, | |
| { | |
| "completion_length": 653.7395973205566, | |
| "epoch": 0.6826666666666666, | |
| "grad_norm": 0.3804032802581787, | |
| "kl": 0.9503173828125, | |
| "learning_rate": 2.493871365532719e-06, | |
| "loss": 0.0645, | |
| "reward": 0.17708333767950535, | |
| "reward_std": 0.2633597403764725, | |
| "rewards/accuracy_reward": 0.17708333767950535, | |
| "rewards/format_reward": 0.0, | |
| "step": 64 | |
| }, | |
| { | |
| "completion_length": 558.3646011352539, | |
| "epoch": 0.6933333333333334, | |
| "grad_norm": 0.8641887903213501, | |
| "kl": 0.542236328125, | |
| "learning_rate": 2.4725618240708804e-06, | |
| "loss": 0.1947, | |
| "reward": 0.3125000037252903, | |
| "reward_std": 0.3014373779296875, | |
| "rewards/accuracy_reward": 0.3125000037252903, | |
| "rewards/format_reward": 0.0, | |
| "step": 65 | |
| }, | |
| { | |
| "completion_length": 554.1041793823242, | |
| "epoch": 0.704, | |
| "grad_norm": 0.7259989380836487, | |
| "kl": 1.319580078125, | |
| "learning_rate": 2.4509081140340877e-06, | |
| "loss": 0.0516, | |
| "reward": 0.3020833423361182, | |
| "reward_std": 0.33246491476893425, | |
| "rewards/accuracy_reward": 0.3020833423361182, | |
| "rewards/format_reward": 0.0, | |
| "step": 66 | |
| }, | |
| { | |
| "completion_length": 534.1146011352539, | |
| "epoch": 0.7146666666666667, | |
| "grad_norm": 0.7615457773208618, | |
| "kl": 1.759307861328125, | |
| "learning_rate": 2.428917898201553e-06, | |
| "loss": 0.1123, | |
| "reward": 0.2812500027939677, | |
| "reward_std": 0.2837342247366905, | |
| "rewards/accuracy_reward": 0.2812500027939677, | |
| "rewards/format_reward": 0.0, | |
| "step": 67 | |
| }, | |
| { | |
| "completion_length": 536.3125152587891, | |
| "epoch": 0.7253333333333334, | |
| "grad_norm": 0.9113590121269226, | |
| "kl": 0.890380859375, | |
| "learning_rate": 2.4065989584346e-06, | |
| "loss": 0.1048, | |
| "reward": 0.22916667629033327, | |
| "reward_std": 0.3314773738384247, | |
| "rewards/accuracy_reward": 0.22916667629033327, | |
| "rewards/format_reward": 0.0, | |
| "step": 68 | |
| }, | |
| { | |
| "completion_length": 524.9895935058594, | |
| "epoch": 0.736, | |
| "grad_norm": 0.4094133973121643, | |
| "kl": 0.77490234375, | |
| "learning_rate": 2.3839591929228413e-06, | |
| "loss": 0.0492, | |
| "reward": 0.2708333423361182, | |
| "reward_std": 0.41816671192646027, | |
| "rewards/accuracy_reward": 0.2708333423361182, | |
| "rewards/format_reward": 0.0, | |
| "step": 69 | |
| }, | |
| { | |
| "completion_length": 532.0625133514404, | |
| "epoch": 0.7466666666666667, | |
| "grad_norm": 2.149420976638794, | |
| "kl": 0.80133056640625, | |
| "learning_rate": 2.3610066133891706e-06, | |
| "loss": 0.0904, | |
| "reward": 0.260416672565043, | |
| "reward_std": 0.35224414244294167, | |
| "rewards/accuracy_reward": 0.260416672565043, | |
| "rewards/format_reward": 0.0, | |
| "step": 70 | |
| }, | |
| { | |
| "completion_length": 639.7916870117188, | |
| "epoch": 0.7573333333333333, | |
| "grad_norm": 0.29616329073905945, | |
| "kl": 0.350982666015625, | |
| "learning_rate": 2.337749342254595e-06, | |
| "loss": 0.0271, | |
| "reward": 0.2708333395421505, | |
| "reward_std": 0.2885478064417839, | |
| "rewards/accuracy_reward": 0.2708333395421505, | |
| "rewards/format_reward": 0.0, | |
| "step": 71 | |
| }, | |
| { | |
| "completion_length": 584.4166831970215, | |
| "epoch": 0.768, | |
| "grad_norm": 0.42845651507377625, | |
| "kl": 0.38970947265625, | |
| "learning_rate": 2.3141956097638817e-06, | |
| "loss": 0.0633, | |
| "reward": 0.17708334047347307, | |
| "reward_std": 0.3029524050652981, | |
| "rewards/accuracy_reward": 0.17708334047347307, | |
| "rewards/format_reward": 0.0, | |
| "step": 72 | |
| }, | |
| { | |
| "completion_length": 543.7083435058594, | |
| "epoch": 0.7786666666666666, | |
| "grad_norm": 0.6256530284881592, | |
| "kl": 0.306884765625, | |
| "learning_rate": 2.2903537510730547e-06, | |
| "loss": 0.1157, | |
| "reward": 0.4583333535119891, | |
| "reward_std": 0.4731736220419407, | |
| "rewards/accuracy_reward": 0.4583333535119891, | |
| "rewards/format_reward": 0.0, | |
| "step": 73 | |
| }, | |
| { | |
| "completion_length": 560.2812614440918, | |
| "epoch": 0.7893333333333333, | |
| "grad_norm": 0.5944956541061401, | |
| "kl": 0.662353515625, | |
| "learning_rate": 2.2662322032997616e-06, | |
| "loss": 0.1079, | |
| "reward": 0.19791667349636555, | |
| "reward_std": 0.29912634938955307, | |
| "rewards/accuracy_reward": 0.19791667349636555, | |
| "rewards/format_reward": 0.0, | |
| "step": 74 | |
| }, | |
| { | |
| "completion_length": 699.5521011352539, | |
| "epoch": 0.8, | |
| "grad_norm": 0.554081380367279, | |
| "kl": 0.7374267578125, | |
| "learning_rate": 2.241839502537563e-06, | |
| "loss": 0.0715, | |
| "reward": 0.15625000651925802, | |
| "reward_std": 0.22636231780052185, | |
| "rewards/accuracy_reward": 0.15625000651925802, | |
| "rewards/format_reward": 0.0, | |
| "step": 75 | |
| }, | |
| { | |
| "completion_length": 646.3437652587891, | |
| "epoch": 0.8106666666666666, | |
| "grad_norm": 1.3772621154785156, | |
| "kl": 1.7783203125, | |
| "learning_rate": 2.21718428083519e-06, | |
| "loss": 0.0255, | |
| "reward": 0.17708333767950535, | |
| "reward_std": 0.28982681408524513, | |
| "rewards/accuracy_reward": 0.17708333767950535, | |
| "rewards/format_reward": 0.0, | |
| "step": 76 | |
| }, | |
| { | |
| "completion_length": 632.2500152587891, | |
| "epoch": 0.8213333333333334, | |
| "grad_norm": 0.865872859954834, | |
| "kl": 0.7518310546875, | |
| "learning_rate": 2.1922752631418534e-06, | |
| "loss": 0.035, | |
| "reward": 0.2500000074505806, | |
| "reward_std": 0.28194961696863174, | |
| "rewards/accuracy_reward": 0.2500000074505806, | |
| "rewards/format_reward": 0.0, | |
| "step": 77 | |
| }, | |
| { | |
| "completion_length": 702.2395935058594, | |
| "epoch": 0.832, | |
| "grad_norm": 0.4784860908985138, | |
| "kl": 0.572998046875, | |
| "learning_rate": 2.167121264219669e-06, | |
| "loss": 0.0299, | |
| "reward": 0.15625000186264515, | |
| "reward_std": 0.23201442882418633, | |
| "rewards/accuracy_reward": 0.15625000186264515, | |
| "rewards/format_reward": 0.0, | |
| "step": 78 | |
| }, | |
| { | |
| "completion_length": 673.2396087646484, | |
| "epoch": 0.8426666666666667, | |
| "grad_norm": 1.1683677434921265, | |
| "kl": 0.584228515625, | |
| "learning_rate": 2.1417311855243084e-06, | |
| "loss": 0.1041, | |
| "reward": 0.18750000465661287, | |
| "reward_std": 0.29944413155317307, | |
| "rewards/accuracy_reward": 0.18750000465661287, | |
| "rewards/format_reward": 0.0, | |
| "step": 79 | |
| }, | |
| { | |
| "completion_length": 561.9270973205566, | |
| "epoch": 0.8533333333333334, | |
| "grad_norm": 0.7934542894363403, | |
| "kl": 1.1396484375, | |
| "learning_rate": 2.116114012054961e-06, | |
| "loss": 0.1309, | |
| "reward": 0.27083334047347307, | |
| "reward_std": 0.38526491820812225, | |
| "rewards/accuracy_reward": 0.27083334047347307, | |
| "rewards/format_reward": 0.0, | |
| "step": 80 | |
| }, | |
| { | |
| "completion_length": 486.2395935058594, | |
| "epoch": 0.864, | |
| "grad_norm": 1.099777340888977, | |
| "kl": 1.2906494140625, | |
| "learning_rate": 2.0902788091747436e-06, | |
| "loss": 0.0744, | |
| "reward": 0.22916667256504297, | |
| "reward_std": 0.3494238071143627, | |
| "rewards/accuracy_reward": 0.22916667256504297, | |
| "rewards/format_reward": 0.0, | |
| "step": 81 | |
| }, | |
| { | |
| "completion_length": 573.5729293823242, | |
| "epoch": 0.8746666666666667, | |
| "grad_norm": 0.3417358994483948, | |
| "kl": 0.5443115234375, | |
| "learning_rate": 2.064234719402658e-06, | |
| "loss": 0.1249, | |
| "reward": 0.2812500037252903, | |
| "reward_std": 0.30425771325826645, | |
| "rewards/accuracy_reward": 0.2812500037252903, | |
| "rewards/format_reward": 0.0, | |
| "step": 82 | |
| }, | |
| { | |
| "completion_length": 600.3646011352539, | |
| "epoch": 0.8853333333333333, | |
| "grad_norm": 0.3621276021003723, | |
| "kl": 0.4794921875, | |
| "learning_rate": 2.037990959178259e-06, | |
| "loss": 0.0338, | |
| "reward": 0.281250006519258, | |
| "reward_std": 0.2800753600895405, | |
| "rewards/accuracy_reward": 0.281250006519258, | |
| "rewards/format_reward": 0.0, | |
| "step": 83 | |
| }, | |
| { | |
| "completion_length": 584.6458473205566, | |
| "epoch": 0.896, | |
| "grad_norm": 0.5323257446289062, | |
| "kl": 0.467041015625, | |
| "learning_rate": 2.0115568156001488e-06, | |
| "loss": 0.0821, | |
| "reward": 0.27083334513008595, | |
| "reward_std": 0.3042132258415222, | |
| "rewards/accuracy_reward": 0.27083334513008595, | |
| "rewards/format_reward": 0.0, | |
| "step": 84 | |
| }, | |
| { | |
| "completion_length": 585.9166870117188, | |
| "epoch": 0.9066666666666666, | |
| "grad_norm": 0.467734694480896, | |
| "kl": 0.3864898681640625, | |
| "learning_rate": 1.984941643139478e-06, | |
| "loss": 0.1324, | |
| "reward": 0.3229166753590107, | |
| "reward_std": 0.1980806216597557, | |
| "rewards/accuracy_reward": 0.3229166753590107, | |
| "rewards/format_reward": 0.0, | |
| "step": 85 | |
| }, | |
| { | |
| "completion_length": 619.7187652587891, | |
| "epoch": 0.9173333333333333, | |
| "grad_norm": 0.3862552344799042, | |
| "kl": 0.523406982421875, | |
| "learning_rate": 1.9581548603295995e-06, | |
| "loss": 0.0401, | |
| "reward": 0.19791667256504297, | |
| "reward_std": 0.2671857886016369, | |
| "rewards/accuracy_reward": 0.19791667256504297, | |
| "rewards/format_reward": 0.0, | |
| "step": 86 | |
| }, | |
| { | |
| "completion_length": 613.7083511352539, | |
| "epoch": 0.928, | |
| "grad_norm": 0.27887099981307983, | |
| "kl": 0.801605224609375, | |
| "learning_rate": 1.9312059464330545e-06, | |
| "loss": 0.0648, | |
| "reward": 0.1875000037252903, | |
| "reward_std": 0.2856829836964607, | |
| "rewards/accuracy_reward": 0.1875000037252903, | |
| "rewards/format_reward": 0.0, | |
| "step": 87 | |
| }, | |
| { | |
| "completion_length": 588.0521049499512, | |
| "epoch": 0.9386666666666666, | |
| "grad_norm": 0.4169585704803467, | |
| "kl": 0.687591552734375, | |
| "learning_rate": 1.9041044380870643e-06, | |
| "loss": 0.0083, | |
| "reward": 0.1979166716337204, | |
| "reward_std": 0.32426992803812027, | |
| "rewards/accuracy_reward": 0.1979166716337204, | |
| "rewards/format_reward": 0.0, | |
| "step": 88 | |
| }, | |
| { | |
| "completion_length": 581.4687614440918, | |
| "epoch": 0.9493333333333334, | |
| "grad_norm": 0.6219002604484558, | |
| "kl": 0.71435546875, | |
| "learning_rate": 1.8768599259287235e-06, | |
| "loss": 0.0248, | |
| "reward": 0.26041667349636555, | |
| "reward_std": 0.20783207565546036, | |
| "rewards/accuracy_reward": 0.26041667349636555, | |
| "rewards/format_reward": 0.0, | |
| "step": 89 | |
| }, | |
| { | |
| "completion_length": 576.2916831970215, | |
| "epoch": 0.96, | |
| "grad_norm": 0.5098808407783508, | |
| "kl": 0.950927734375, | |
| "learning_rate": 1.8494820512010797e-06, | |
| "loss": 0.1095, | |
| "reward": 0.2291666753590107, | |
| "reward_std": 0.2797575779259205, | |
| "rewards/accuracy_reward": 0.2291666753590107, | |
| "rewards/format_reward": 0.0, | |
| "step": 90 | |
| }, | |
| { | |
| "completion_length": 624.4271087646484, | |
| "epoch": 0.9706666666666667, | |
| "grad_norm": 0.6733171343803406, | |
| "kl": 0.8636474609375, | |
| "learning_rate": 1.821980502341308e-06, | |
| "loss": 0.0614, | |
| "reward": 0.15625000651925802, | |
| "reward_std": 0.2185296081006527, | |
| "rewards/accuracy_reward": 0.15625000651925802, | |
| "rewards/format_reward": 0.0, | |
| "step": 91 | |
| }, | |
| { | |
| "completion_length": 587.5208587646484, | |
| "epoch": 0.9813333333333333, | |
| "grad_norm": 1.0600099563598633, | |
| "kl": 0.5804443359375, | |
| "learning_rate": 1.7943650115521788e-06, | |
| "loss": 0.1491, | |
| "reward": 0.281250006519258, | |
| "reward_std": 0.2633597403764725, | |
| "rewards/accuracy_reward": 0.281250006519258, | |
| "rewards/format_reward": 0.0, | |
| "step": 92 | |
| }, | |
| { | |
| "completion_length": 635.9375152587891, | |
| "epoch": 0.992, | |
| "grad_norm": 0.22917693853378296, | |
| "kl": 0.6182861328125, | |
| "learning_rate": 1.766645351358047e-06, | |
| "loss": 0.085, | |
| "reward": 0.2395833395421505, | |
| "reward_std": 0.24681130424141884, | |
| "rewards/accuracy_reward": 0.2395833395421505, | |
| "rewards/format_reward": 0.0, | |
| "step": 93 | |
| }, | |
| { | |
| "completion_length": 693.3055826822916, | |
| "epoch": 1.0, | |
| "grad_norm": 0.22917693853378296, | |
| "kl": 1.232421875, | |
| "learning_rate": 1.7388313311465638e-06, | |
| "loss": 0.07, | |
| "reward": 0.22222223257025084, | |
| "reward_std": 0.2980559766292572, | |
| "rewards/accuracy_reward": 0.22222223257025084, | |
| "rewards/format_reward": 0.0, | |
| "step": 94 | |
| }, | |
| { | |
| "completion_length": 568.6562652587891, | |
| "epoch": 1.0106666666666666, | |
| "grad_norm": 0.39828255772590637, | |
| "kl": 1.0860595703125, | |
| "learning_rate": 1.7109327936973479e-06, | |
| "loss": 0.1009, | |
| "reward": 0.22916667349636555, | |
| "reward_std": 0.2341650128364563, | |
| "rewards/accuracy_reward": 0.22916667349636555, | |
| "rewards/format_reward": 0.0, | |
| "step": 95 | |
| }, | |
| { | |
| "completion_length": 568.7083549499512, | |
| "epoch": 1.0213333333333334, | |
| "grad_norm": 0.3863210380077362, | |
| "kl": 0.499755859375, | |
| "learning_rate": 1.6829596116988381e-06, | |
| "loss": 0.0364, | |
| "reward": 0.43750001676380634, | |
| "reward_std": 0.44166112691164017, | |
| "rewards/accuracy_reward": 0.43750001676380634, | |
| "rewards/format_reward": 0.0, | |
| "step": 96 | |
| }, | |
| { | |
| "completion_length": 576.3958549499512, | |
| "epoch": 1.032, | |
| "grad_norm": 0.621791660785675, | |
| "kl": 0.8270263671875, | |
| "learning_rate": 1.6549216842545649e-06, | |
| "loss": -0.0011, | |
| "reward": 0.18750000465661287, | |
| "reward_std": 0.35948578268289566, | |
| "rewards/accuracy_reward": 0.18750000465661287, | |
| "rewards/format_reward": 0.0, | |
| "step": 97 | |
| }, | |
| { | |
| "completion_length": 618.697925567627, | |
| "epoch": 1.0426666666666666, | |
| "grad_norm": 0.27708661556243896, | |
| "kl": 0.358612060546875, | |
| "learning_rate": 1.6268289333800686e-06, | |
| "loss": 0.1116, | |
| "reward": 0.41666668001562357, | |
| "reward_std": 0.35948577523231506, | |
| "rewards/accuracy_reward": 0.41666668001562357, | |
| "rewards/format_reward": 0.0, | |
| "step": 98 | |
| }, | |
| { | |
| "completion_length": 602.5625152587891, | |
| "epoch": 1.0533333333333332, | |
| "grad_norm": 1.2479599714279175, | |
| "kl": 0.67333984375, | |
| "learning_rate": 1.598691300491716e-06, | |
| "loss": 0.1111, | |
| "reward": 0.33333334513008595, | |
| "reward_std": 0.28194961696863174, | |
| "rewards/accuracy_reward": 0.33333334513008595, | |
| "rewards/format_reward": 0.0, | |
| "step": 99 | |
| }, | |
| { | |
| "completion_length": 664.4062576293945, | |
| "epoch": 1.064, | |
| "grad_norm": 0.31423419713974, | |
| "kl": 0.4830322265625, | |
| "learning_rate": 1.5705187428886465e-06, | |
| "loss": 0.0806, | |
| "reward": 0.3437500139698386, | |
| "reward_std": 0.3402976170182228, | |
| "rewards/accuracy_reward": 0.3437500139698386, | |
| "rewards/format_reward": 0.0, | |
| "step": 100 | |
| }, | |
| { | |
| "completion_length": 620.0312652587891, | |
| "epoch": 1.0746666666666667, | |
| "grad_norm": 0.2487853467464447, | |
| "kl": 0.7161865234375, | |
| "learning_rate": 1.542321230229099e-06, | |
| "loss": 0.1072, | |
| "reward": 0.36458333767950535, | |
| "reward_std": 0.35291389375925064, | |
| "rewards/accuracy_reward": 0.36458333767950535, | |
| "rewards/format_reward": 0.0, | |
| "step": 101 | |
| }, | |
| { | |
| "completion_length": 554.6875152587891, | |
| "epoch": 1.0853333333333333, | |
| "grad_norm": 0.3372454047203064, | |
| "kl": 0.9202880859375, | |
| "learning_rate": 1.5141087410023688e-06, | |
| "loss": 0.1058, | |
| "reward": 0.27083334140479565, | |
| "reward_std": 0.3755134642124176, | |
| "rewards/accuracy_reward": 0.27083334140479565, | |
| "rewards/format_reward": 0.0, | |
| "step": 102 | |
| }, | |
| { | |
| "completion_length": 661.9583587646484, | |
| "epoch": 1.096, | |
| "grad_norm": 0.3665924668312073, | |
| "kl": 0.68231201171875, | |
| "learning_rate": 1.485891258997632e-06, | |
| "loss": 0.071, | |
| "reward": 0.20833333767950535, | |
| "reward_std": 0.29638051614165306, | |
| "rewards/accuracy_reward": 0.20833333767950535, | |
| "rewards/format_reward": 0.0, | |
| "step": 103 | |
| }, | |
| { | |
| "completion_length": 570.8541793823242, | |
| "epoch": 1.1066666666666667, | |
| "grad_norm": 0.4261507987976074, | |
| "kl": 0.9154052734375, | |
| "learning_rate": 1.4576787697709012e-06, | |
| "loss": 0.0776, | |
| "reward": 0.27083334047347307, | |
| "reward_std": 0.4203435964882374, | |
| "rewards/accuracy_reward": 0.27083334047347307, | |
| "rewards/format_reward": 0.0, | |
| "step": 104 | |
| }, | |
| { | |
| "completion_length": 570.9791793823242, | |
| "epoch": 1.1173333333333333, | |
| "grad_norm": 1.0135829448699951, | |
| "kl": 0.7562255859375, | |
| "learning_rate": 1.429481257111354e-06, | |
| "loss": 0.1382, | |
| "reward": 0.5000000102445483, | |
| "reward_std": 0.39492369070649147, | |
| "rewards/accuracy_reward": 0.5000000102445483, | |
| "rewards/format_reward": 0.0, | |
| "step": 105 | |
| }, | |
| { | |
| "completion_length": 485.27084732055664, | |
| "epoch": 1.1280000000000001, | |
| "grad_norm": 0.5100330710411072, | |
| "kl": 0.30596923828125, | |
| "learning_rate": 1.401308699508284e-06, | |
| "loss": 0.139, | |
| "reward": 0.5729166828095913, | |
| "reward_std": 0.3619955964386463, | |
| "rewards/accuracy_reward": 0.5729166828095913, | |
| "rewards/format_reward": 0.0, | |
| "step": 106 | |
| }, | |
| { | |
| "completion_length": 745.677116394043, | |
| "epoch": 1.1386666666666667, | |
| "grad_norm": 0.6321167349815369, | |
| "kl": 0.743896484375, | |
| "learning_rate": 1.3731710666199315e-06, | |
| "loss": 0.1902, | |
| "reward": 0.29166667349636555, | |
| "reward_std": 0.39369287714362144, | |
| "rewards/accuracy_reward": 0.29166667349636555, | |
| "rewards/format_reward": 0.0, | |
| "step": 107 | |
| }, | |
| { | |
| "completion_length": 683.0833587646484, | |
| "epoch": 1.1493333333333333, | |
| "grad_norm": 0.19921176135540009, | |
| "kl": 0.631439208984375, | |
| "learning_rate": 1.3450783157454354e-06, | |
| "loss": 0.0708, | |
| "reward": 0.3437500102445483, | |
| "reward_std": 0.29825781285762787, | |
| "rewards/accuracy_reward": 0.3437500102445483, | |
| "rewards/format_reward": 0.0, | |
| "step": 108 | |
| }, | |
| { | |
| "completion_length": 543.0416831970215, | |
| "epoch": 1.16, | |
| "grad_norm": 0.653483510017395, | |
| "kl": 0.3345947265625, | |
| "learning_rate": 1.317040388301162e-06, | |
| "loss": 0.2456, | |
| "reward": 0.45833335164934397, | |
| "reward_std": 0.34686949849128723, | |
| "rewards/accuracy_reward": 0.45833335164934397, | |
| "rewards/format_reward": 0.0, | |
| "step": 109 | |
| }, | |
| { | |
| "completion_length": 568.3125114440918, | |
| "epoch": 1.1706666666666667, | |
| "grad_norm": 1.6327720880508423, | |
| "kl": 0.2816009521484375, | |
| "learning_rate": 1.289067206302653e-06, | |
| "loss": 0.0651, | |
| "reward": 0.45833334885537624, | |
| "reward_std": 0.3930976279079914, | |
| "rewards/accuracy_reward": 0.45833334885537624, | |
| "rewards/format_reward": 0.0, | |
| "step": 110 | |
| }, | |
| { | |
| "completion_length": 613.6250114440918, | |
| "epoch": 1.1813333333333333, | |
| "grad_norm": 0.16312555968761444, | |
| "kl": 0.50360107421875, | |
| "learning_rate": 1.2611686688534365e-06, | |
| "loss": 0.0214, | |
| "reward": 0.19791667349636555, | |
| "reward_std": 0.27467070892453194, | |
| "rewards/accuracy_reward": 0.19791667349636555, | |
| "rewards/format_reward": 0.0, | |
| "step": 111 | |
| }, | |
| { | |
| "completion_length": 548.9583587646484, | |
| "epoch": 1.192, | |
| "grad_norm": 0.5374172329902649, | |
| "kl": 0.38507080078125, | |
| "learning_rate": 1.2333546486419533e-06, | |
| "loss": 0.0715, | |
| "reward": 0.42708334047347307, | |
| "reward_std": 0.3080092594027519, | |
| "rewards/accuracy_reward": 0.42708334047347307, | |
| "rewards/format_reward": 0.0, | |
| "step": 112 | |
| }, | |
| { | |
| "completion_length": 542.7083549499512, | |
| "epoch": 1.2026666666666666, | |
| "grad_norm": 0.6037431955337524, | |
| "kl": 0.6873931884765625, | |
| "learning_rate": 1.2056349884478213e-06, | |
| "loss": 0.1474, | |
| "reward": 0.4375000149011612, | |
| "reward_std": 0.35419290140271187, | |
| "rewards/accuracy_reward": 0.4375000149011612, | |
| "rewards/format_reward": 0.0, | |
| "step": 113 | |
| }, | |
| { | |
| "completion_length": 563.3645935058594, | |
| "epoch": 1.2133333333333334, | |
| "grad_norm": 0.7209116220474243, | |
| "kl": 0.3206329345703125, | |
| "learning_rate": 1.178019497658692e-06, | |
| "loss": 0.0499, | |
| "reward": 0.3854166744276881, | |
| "reward_std": 0.31488074362277985, | |
| "rewards/accuracy_reward": 0.3854166744276881, | |
| "rewards/format_reward": 0.0, | |
| "step": 114 | |
| }, | |
| { | |
| "completion_length": 641.1041831970215, | |
| "epoch": 1.224, | |
| "grad_norm": 0.1957225352525711, | |
| "kl": 0.39288330078125, | |
| "learning_rate": 1.1505179487989203e-06, | |
| "loss": 0.038, | |
| "reward": 0.30208333395421505, | |
| "reward_std": 0.2357477955520153, | |
| "rewards/accuracy_reward": 0.30208333395421505, | |
| "rewards/format_reward": 0.0, | |
| "step": 115 | |
| }, | |
| { | |
| "completion_length": 507.43751525878906, | |
| "epoch": 1.2346666666666666, | |
| "grad_norm": 0.23401759564876556, | |
| "kl": 0.369598388671875, | |
| "learning_rate": 1.123140074071277e-06, | |
| "loss": 0.0856, | |
| "reward": 0.4479166818782687, | |
| "reward_std": 0.47189461439847946, | |
| "rewards/accuracy_reward": 0.4479166818782687, | |
| "rewards/format_reward": 0.0, | |
| "step": 116 | |
| }, | |
| { | |
| "completion_length": 629.677116394043, | |
| "epoch": 1.2453333333333334, | |
| "grad_norm": 0.4538763463497162, | |
| "kl": 0.764892578125, | |
| "learning_rate": 1.0958955619129364e-06, | |
| "loss": -0.0148, | |
| "reward": 0.09375000279396772, | |
| "reward_std": 0.1569838561117649, | |
| "rewards/accuracy_reward": 0.09375000279396772, | |
| "rewards/format_reward": 0.0, | |
| "step": 117 | |
| }, | |
| { | |
| "completion_length": 507.5937614440918, | |
| "epoch": 1.256, | |
| "grad_norm": 0.48793312907218933, | |
| "kl": 0.26214599609375, | |
| "learning_rate": 1.068794053566946e-06, | |
| "loss": 0.0633, | |
| "reward": 0.5000000102445483, | |
| "reward_std": 0.2546139992773533, | |
| "rewards/accuracy_reward": 0.5000000102445483, | |
| "rewards/format_reward": 0.0, | |
| "step": 118 | |
| }, | |
| { | |
| "completion_length": 633.2291946411133, | |
| "epoch": 1.2666666666666666, | |
| "grad_norm": 0.45710209012031555, | |
| "kl": 0.2803955078125, | |
| "learning_rate": 1.0418451396704008e-06, | |
| "loss": 0.0682, | |
| "reward": 0.2395833395421505, | |
| "reward_std": 0.22601452842354774, | |
| "rewards/accuracy_reward": 0.2395833395421505, | |
| "rewards/format_reward": 0.0, | |
| "step": 119 | |
| }, | |
| { | |
| "completion_length": 471.57293701171875, | |
| "epoch": 1.2773333333333334, | |
| "grad_norm": 0.4903794229030609, | |
| "kl": 0.21380615234375, | |
| "learning_rate": 1.0150583568605221e-06, | |
| "loss": 0.1971, | |
| "reward": 0.5833333507180214, | |
| "reward_std": 0.4508173316717148, | |
| "rewards/accuracy_reward": 0.5833333507180214, | |
| "rewards/format_reward": 0.0, | |
| "step": 120 | |
| }, | |
| { | |
| "completion_length": 668.8020935058594, | |
| "epoch": 1.288, | |
| "grad_norm": 0.42050671577453613, | |
| "kl": 0.277130126953125, | |
| "learning_rate": 9.884431843998509e-07, | |
| "loss": 0.1015, | |
| "reward": 0.34375000558793545, | |
| "reward_std": 0.2768445611000061, | |
| "rewards/accuracy_reward": 0.34375000558793545, | |
| "rewards/format_reward": 0.0, | |
| "step": 121 | |
| }, | |
| { | |
| "completion_length": 497.5000228881836, | |
| "epoch": 1.2986666666666666, | |
| "grad_norm": 0.38155919313430786, | |
| "kl": 0.33343505859375, | |
| "learning_rate": 9.620090408217412e-07, | |
| "loss": 0.0934, | |
| "reward": 0.593750013038516, | |
| "reward_std": 0.3603988066315651, | |
| "rewards/accuracy_reward": 0.593750013038516, | |
| "rewards/format_reward": 0.0, | |
| "step": 122 | |
| }, | |
| { | |
| "completion_length": 613.1666870117188, | |
| "epoch": 1.3093333333333335, | |
| "grad_norm": 2.1156508922576904, | |
| "kl": 0.541748046875, | |
| "learning_rate": 9.357652805973417e-07, | |
| "loss": 0.0729, | |
| "reward": 0.2291666716337204, | |
| "reward_std": 0.24179892987012863, | |
| "rewards/accuracy_reward": 0.2291666716337204, | |
| "rewards/format_reward": 0.0, | |
| "step": 123 | |
| }, | |
| { | |
| "completion_length": 648.8229331970215, | |
| "epoch": 1.32, | |
| "grad_norm": 0.4442153871059418, | |
| "kl": 0.259307861328125, | |
| "learning_rate": 9.097211908252569e-07, | |
| "loss": 0.0742, | |
| "reward": 0.41666668467223644, | |
| "reward_std": 0.3726486414670944, | |
| "rewards/accuracy_reward": 0.41666668467223644, | |
| "rewards/format_reward": 0.0, | |
| "step": 124 | |
| }, | |
| { | |
| "completion_length": 593.2708511352539, | |
| "epoch": 1.3306666666666667, | |
| "grad_norm": 0.5561993718147278, | |
| "kl": 0.417083740234375, | |
| "learning_rate": 8.838859879450389e-07, | |
| "loss": 0.1325, | |
| "reward": 0.36458334140479565, | |
| "reward_std": 0.4144596457481384, | |
| "rewards/accuracy_reward": 0.36458334140479565, | |
| "rewards/format_reward": 0.0, | |
| "step": 125 | |
| }, | |
| { | |
| "completion_length": 622.2083511352539, | |
| "epoch": 1.3413333333333333, | |
| "grad_norm": 0.29431644082069397, | |
| "kl": 0.33892822265625, | |
| "learning_rate": 8.582688144756916e-07, | |
| "loss": -0.0189, | |
| "reward": 0.3229166781529784, | |
| "reward_std": 0.2596263736486435, | |
| "rewards/accuracy_reward": 0.3229166781529784, | |
| "rewards/format_reward": 0.0, | |
| "step": 126 | |
| }, | |
| { | |
| "completion_length": 553.5104370117188, | |
| "epoch": 1.3519999999999999, | |
| "grad_norm": 0.35601741075515747, | |
| "kl": 0.4847412109375, | |
| "learning_rate": 8.328787357803312e-07, | |
| "loss": 0.0578, | |
| "reward": 0.21875000186264515, | |
| "reward_std": 0.3450811840593815, | |
| "rewards/accuracy_reward": 0.21875000186264515, | |
| "rewards/format_reward": 0.0, | |
| "step": 127 | |
| }, | |
| { | |
| "completion_length": 556.8958396911621, | |
| "epoch": 1.3626666666666667, | |
| "grad_norm": 0.2601093351840973, | |
| "kl": 0.44805908203125, | |
| "learning_rate": 8.077247368581468e-07, | |
| "loss": 0.0774, | |
| "reward": 0.43750000931322575, | |
| "reward_std": 0.2957107648253441, | |
| "rewards/accuracy_reward": 0.43750000931322575, | |
| "rewards/format_reward": 0.0, | |
| "step": 128 | |
| }, | |
| { | |
| "completion_length": 671.4896011352539, | |
| "epoch": 1.3733333333333333, | |
| "grad_norm": 0.3361031711101532, | |
| "kl": 0.432769775390625, | |
| "learning_rate": 7.828157191648098e-07, | |
| "loss": 0.0428, | |
| "reward": 0.07291666977107525, | |
| "reward_std": 0.13653486967086792, | |
| "rewards/accuracy_reward": 0.07291666977107525, | |
| "rewards/format_reward": 0.0, | |
| "step": 129 | |
| }, | |
| { | |
| "completion_length": 533.1354370117188, | |
| "epoch": 1.384, | |
| "grad_norm": 0.4666847288608551, | |
| "kl": 0.45465087890625, | |
| "learning_rate": 7.581604974624371e-07, | |
| "loss": 0.1254, | |
| "reward": 0.3750000027939677, | |
| "reward_std": 0.21944263949990273, | |
| "rewards/accuracy_reward": 0.3750000027939677, | |
| "rewards/format_reward": 0.0, | |
| "step": 130 | |
| }, | |
| { | |
| "completion_length": 493.0937614440918, | |
| "epoch": 1.3946666666666667, | |
| "grad_norm": 0.47746792435646057, | |
| "kl": 0.385833740234375, | |
| "learning_rate": 7.337677967002385e-07, | |
| "loss": 0.0853, | |
| "reward": 0.4166666781529784, | |
| "reward_std": 0.3755134642124176, | |
| "rewards/accuracy_reward": 0.4166666781529784, | |
| "rewards/format_reward": 0.0, | |
| "step": 131 | |
| }, | |
| { | |
| "completion_length": 625.0104331970215, | |
| "epoch": 1.4053333333333333, | |
| "grad_norm": 0.8601658940315247, | |
| "kl": 0.3341064453125, | |
| "learning_rate": 7.096462489269454e-07, | |
| "loss": 0.1341, | |
| "reward": 0.28125000931322575, | |
| "reward_std": 0.32490549236536026, | |
| "rewards/accuracy_reward": 0.28125000931322575, | |
| "rewards/format_reward": 0.0, | |
| "step": 132 | |
| }, | |
| { | |
| "completion_length": 595.4583473205566, | |
| "epoch": 1.416, | |
| "grad_norm": 0.45403310656547546, | |
| "kl": 0.3372802734375, | |
| "learning_rate": 6.858043902361181e-07, | |
| "loss": 0.0947, | |
| "reward": 0.32291667629033327, | |
| "reward_std": 0.4055767357349396, | |
| "rewards/accuracy_reward": 0.32291667629033327, | |
| "rewards/format_reward": 0.0, | |
| "step": 133 | |
| }, | |
| { | |
| "completion_length": 478.6666717529297, | |
| "epoch": 1.4266666666666667, | |
| "grad_norm": 1.0305501222610474, | |
| "kl": 0.44744873046875, | |
| "learning_rate": 6.622506577454054e-07, | |
| "loss": 0.2416, | |
| "reward": 0.5416666809469461, | |
| "reward_std": 0.3745673857629299, | |
| "rewards/accuracy_reward": 0.5416666809469461, | |
| "rewards/format_reward": 0.0, | |
| "step": 134 | |
| }, | |
| { | |
| "completion_length": 591.9895973205566, | |
| "epoch": 1.4373333333333334, | |
| "grad_norm": 1.416900873184204, | |
| "kl": 0.714111328125, | |
| "learning_rate": 6.389933866108296e-07, | |
| "loss": 0.1579, | |
| "reward": 0.3020833395421505, | |
| "reward_std": 0.42134930938482285, | |
| "rewards/accuracy_reward": 0.3020833395421505, | |
| "rewards/format_reward": 0.0, | |
| "step": 135 | |
| }, | |
| { | |
| "completion_length": 578.6146011352539, | |
| "epoch": 1.448, | |
| "grad_norm": 0.6322238445281982, | |
| "kl": 1.034423828125, | |
| "learning_rate": 6.160408070771595e-07, | |
| "loss": 0.1152, | |
| "reward": 0.18750000651925802, | |
| "reward_std": 0.28289569541811943, | |
| "rewards/accuracy_reward": 0.18750000651925802, | |
| "rewards/format_reward": 0.0, | |
| "step": 136 | |
| }, | |
| { | |
| "completion_length": 669.4375228881836, | |
| "epoch": 1.4586666666666668, | |
| "grad_norm": 1.085103154182434, | |
| "kl": 1.0677490234375, | |
| "learning_rate": 5.934010415653995e-07, | |
| "loss": 0.0643, | |
| "reward": 0.25000001210719347, | |
| "reward_std": 0.3393845856189728, | |
| "rewards/accuracy_reward": 0.25000001210719347, | |
| "rewards/format_reward": 0.0, | |
| "step": 137 | |
| }, | |
| { | |
| "completion_length": 716.8229293823242, | |
| "epoch": 1.4693333333333334, | |
| "grad_norm": 0.7747601270675659, | |
| "kl": 1.439208984375, | |
| "learning_rate": 5.710821017984476e-07, | |
| "loss": 0.1124, | |
| "reward": 0.21875001024454832, | |
| "reward_std": 0.22636231780052185, | |
| "rewards/accuracy_reward": 0.21875001024454832, | |
| "rewards/format_reward": 0.0, | |
| "step": 138 | |
| }, | |
| { | |
| "completion_length": 531.7916793823242, | |
| "epoch": 1.48, | |
| "grad_norm": 1.2720061540603638, | |
| "kl": 0.59039306640625, | |
| "learning_rate": 5.490918859659122e-07, | |
| "loss": 0.1796, | |
| "reward": 0.38541667722165585, | |
| "reward_std": 0.3657289706170559, | |
| "rewards/accuracy_reward": 0.38541667722165585, | |
| "rewards/format_reward": 0.0, | |
| "step": 139 | |
| }, | |
| { | |
| "completion_length": 592.1666831970215, | |
| "epoch": 1.4906666666666666, | |
| "grad_norm": 0.8733373284339905, | |
| "kl": 0.7615966796875, | |
| "learning_rate": 5.2743817592912e-07, | |
| "loss": 0.119, | |
| "reward": 0.27083334140479565, | |
| "reward_std": 0.2904551103711128, | |
| "rewards/accuracy_reward": 0.27083334140479565, | |
| "rewards/format_reward": 0.0, | |
| "step": 140 | |
| }, | |
| { | |
| "completion_length": 509.00001525878906, | |
| "epoch": 1.5013333333333332, | |
| "grad_norm": 0.5739810466766357, | |
| "kl": 0.782470703125, | |
| "learning_rate": 5.061286344672812e-07, | |
| "loss": 0.2072, | |
| "reward": 0.4270833460614085, | |
| "reward_std": 0.3695550188422203, | |
| "rewards/accuracy_reward": 0.4270833460614085, | |
| "rewards/format_reward": 0.0, | |
| "step": 141 | |
| }, | |
| { | |
| "completion_length": 632.4166793823242, | |
| "epoch": 1.512, | |
| "grad_norm": 0.5129138231277466, | |
| "kl": 0.471435546875, | |
| "learning_rate": 4.85170802565797e-07, | |
| "loss": 0.0212, | |
| "reward": 0.2083333395421505, | |
| "reward_std": 0.30546221882104874, | |
| "rewards/accuracy_reward": 0.2083333395421505, | |
| "rewards/format_reward": 0.0, | |
| "step": 142 | |
| }, | |
| { | |
| "completion_length": 675.6354293823242, | |
| "epoch": 1.5226666666666666, | |
| "grad_norm": 0.6775838136672974, | |
| "kl": 1.49658203125, | |
| "learning_rate": 4.645720967476709e-07, | |
| "loss": 0.1271, | |
| "reward": 0.16666666977107525, | |
| "reward_std": 0.3798602595925331, | |
| "rewards/accuracy_reward": 0.16666666977107525, | |
| "rewards/format_reward": 0.0, | |
| "step": 143 | |
| }, | |
| { | |
| "completion_length": 599.3333473205566, | |
| "epoch": 1.5333333333333332, | |
| "grad_norm": 0.7139673233032227, | |
| "kl": 1.02490234375, | |
| "learning_rate": 4.443398064489581e-07, | |
| "loss": 0.1713, | |
| "reward": 0.32291667629033327, | |
| "reward_std": 0.4055767357349396, | |
| "rewards/accuracy_reward": 0.32291667629033327, | |
| "rewards/format_reward": 0.0, | |
| "step": 144 | |
| }, | |
| { | |
| "completion_length": 571.4687652587891, | |
| "epoch": 1.544, | |
| "grad_norm": 0.5391110181808472, | |
| "kl": 0.8048095703125, | |
| "learning_rate": 4.244810914391956e-07, | |
| "loss": 0.0891, | |
| "reward": 0.3020833348855376, | |
| "reward_std": 0.2281883805990219, | |
| "rewards/accuracy_reward": 0.3020833348855376, | |
| "rewards/format_reward": 0.0, | |
| "step": 145 | |
| }, | |
| { | |
| "completion_length": 616.6250267028809, | |
| "epoch": 1.5546666666666666, | |
| "grad_norm": 0.23279787600040436, | |
| "kl": 0.5025634765625, | |
| "learning_rate": 4.050029792877119e-07, | |
| "loss": 0.1257, | |
| "reward": 0.17708333767950535, | |
| "reward_std": 0.22828106209635735, | |
| "rewards/accuracy_reward": 0.17708333767950535, | |
| "rewards/format_reward": 0.0, | |
| "step": 146 | |
| }, | |
| { | |
| "completion_length": 546.5104331970215, | |
| "epoch": 1.5653333333333332, | |
| "grad_norm": 0.29375898838043213, | |
| "kl": 0.83935546875, | |
| "learning_rate": 3.8591236287672435e-07, | |
| "loss": 0.161, | |
| "reward": 0.40625001210719347, | |
| "reward_std": 0.44266683980822563, | |
| "rewards/accuracy_reward": 0.40625001210719347, | |
| "rewards/format_reward": 0.0, | |
| "step": 147 | |
| }, | |
| { | |
| "completion_length": 602.1250190734863, | |
| "epoch": 1.576, | |
| "grad_norm": 0.38167208433151245, | |
| "kl": 0.68280029296875, | |
| "learning_rate": 3.6721599796209593e-07, | |
| "loss": 0.0954, | |
| "reward": 0.2916666688397527, | |
| "reward_std": 0.3333779387176037, | |
| "rewards/accuracy_reward": 0.2916666688397527, | |
| "rewards/format_reward": 0.0, | |
| "step": 148 | |
| }, | |
| { | |
| "completion_length": 528.9479370117188, | |
| "epoch": 1.5866666666666667, | |
| "grad_norm": 0.2693575322628021, | |
| "kl": 0.5400390625, | |
| "learning_rate": 3.489205007826196e-07, | |
| "loss": 0.1159, | |
| "reward": 0.4166666818782687, | |
| "reward_std": 0.31082959473133087, | |
| "rewards/accuracy_reward": 0.4166666818782687, | |
| "rewards/format_reward": 0.0, | |
| "step": 149 | |
| }, | |
| { | |
| "completion_length": 623.6250152587891, | |
| "epoch": 1.5973333333333333, | |
| "grad_norm": 0.4524155557155609, | |
| "kl": 0.508544921875, | |
| "learning_rate": 3.3103234571867633e-07, | |
| "loss": 0.1098, | |
| "reward": 0.22916666977107525, | |
| "reward_std": 0.22326868772506714, | |
| "rewards/accuracy_reward": 0.22916666977107525, | |
| "rewards/format_reward": 0.0, | |
| "step": 150 | |
| }, | |
| { | |
| "completion_length": 543.322925567627, | |
| "epoch": 1.608, | |
| "grad_norm": 0.3804190456867218, | |
| "kl": 0.6044921875, | |
| "learning_rate": 3.135578630010894e-07, | |
| "loss": 0.139, | |
| "reward": 0.3020833469927311, | |
| "reward_std": 0.2718803808093071, | |
| "rewards/accuracy_reward": 0.3020833469927311, | |
| "rewards/format_reward": 0.0, | |
| "step": 151 | |
| }, | |
| { | |
| "completion_length": 605.7708473205566, | |
| "epoch": 1.6186666666666667, | |
| "grad_norm": 0.2589690685272217, | |
| "kl": 0.49359130859375, | |
| "learning_rate": 2.965032364709947e-07, | |
| "loss": 0.1063, | |
| "reward": 0.3020833367481828, | |
| "reward_std": 0.3481447994709015, | |
| "rewards/accuracy_reward": 0.3020833367481828, | |
| "rewards/format_reward": 0.0, | |
| "step": 152 | |
| }, | |
| { | |
| "completion_length": 596.3541793823242, | |
| "epoch": 1.6293333333333333, | |
| "grad_norm": 0.31066620349884033, | |
| "kl": 0.743408203125, | |
| "learning_rate": 2.798745013915132e-07, | |
| "loss": 0.1456, | |
| "reward": 0.3333333497866988, | |
| "reward_std": 0.3522741571068764, | |
| "rewards/accuracy_reward": 0.3333333497866988, | |
| "rewards/format_reward": 0.0, | |
| "step": 153 | |
| }, | |
| { | |
| "completion_length": 630.0312614440918, | |
| "epoch": 1.6400000000000001, | |
| "grad_norm": 0.30095869302749634, | |
| "kl": 0.455352783203125, | |
| "learning_rate": 2.636775423120007e-07, | |
| "loss": 0.0601, | |
| "reward": 0.27083334140479565, | |
| "reward_std": 0.2546139992773533, | |
| "rewards/accuracy_reward": 0.27083334140479565, | |
| "rewards/format_reward": 0.0, | |
| "step": 154 | |
| }, | |
| { | |
| "completion_length": 626.2812652587891, | |
| "epoch": 1.6506666666666665, | |
| "grad_norm": 0.4043548107147217, | |
| "kl": 0.3909912109375, | |
| "learning_rate": 2.479180909856347e-07, | |
| "loss": 0.0545, | |
| "reward": 0.18750000465661287, | |
| "reward_std": 0.28787805512547493, | |
| "rewards/accuracy_reward": 0.18750000465661287, | |
| "rewards/format_reward": 0.0, | |
| "step": 155 | |
| }, | |
| { | |
| "completion_length": 611.9791870117188, | |
| "epoch": 1.6613333333333333, | |
| "grad_norm": 0.22528155148029327, | |
| "kl": 0.381103515625, | |
| "learning_rate": 2.3260172434106913e-07, | |
| "loss": 0.0343, | |
| "reward": 0.2708333386108279, | |
| "reward_std": 0.19899365305900574, | |
| "rewards/accuracy_reward": 0.2708333386108279, | |
| "rewards/format_reward": 0.0, | |
| "step": 156 | |
| }, | |
| { | |
| "completion_length": 681.979175567627, | |
| "epoch": 1.6720000000000002, | |
| "grad_norm": 0.22992321848869324, | |
| "kl": 0.5267333984375, | |
| "learning_rate": 2.177338625088826e-07, | |
| "loss": 0.2209, | |
| "reward": 0.3020833395421505, | |
| "reward_std": 0.33629096299409866, | |
| "rewards/accuracy_reward": 0.3020833395421505, | |
| "rewards/format_reward": 0.0, | |
| "step": 157 | |
| }, | |
| { | |
| "completion_length": 550.0833511352539, | |
| "epoch": 1.6826666666666665, | |
| "grad_norm": 1.0294519662857056, | |
| "kl": 0.4268798828125, | |
| "learning_rate": 2.0331976690350844e-07, | |
| "loss": 0.0676, | |
| "reward": 0.30208334885537624, | |
| "reward_std": 0.2103346362709999, | |
| "rewards/accuracy_reward": 0.30208334885537624, | |
| "rewards/format_reward": 0.0, | |
| "step": 158 | |
| }, | |
| { | |
| "completion_length": 541.1458396911621, | |
| "epoch": 1.6933333333333334, | |
| "grad_norm": 0.44960758090019226, | |
| "kl": 0.64111328125, | |
| "learning_rate": 1.893645383613371e-07, | |
| "loss": 0.1856, | |
| "reward": 0.4583333423361182, | |
| "reward_std": 0.3755134642124176, | |
| "rewards/accuracy_reward": 0.4583333423361182, | |
| "rewards/format_reward": 0.0, | |
| "step": 159 | |
| }, | |
| { | |
| "completion_length": 585.1354293823242, | |
| "epoch": 1.704, | |
| "grad_norm": 0.2397642582654953, | |
| "kl": 0.6611480712890625, | |
| "learning_rate": 1.7587311533563887e-07, | |
| "loss": 0.1011, | |
| "reward": 0.34375000186264515, | |
| "reward_std": 0.3033146671950817, | |
| "rewards/accuracy_reward": 0.34375000186264515, | |
| "rewards/format_reward": 0.0, | |
| "step": 160 | |
| }, | |
| { | |
| "completion_length": 618.8854446411133, | |
| "epoch": 1.7146666666666666, | |
| "grad_norm": 0.30194056034088135, | |
| "kl": 0.41375732421875, | |
| "learning_rate": 1.6285027214895227e-07, | |
| "loss": 0.0246, | |
| "reward": 0.21875000558793545, | |
| "reward_std": 0.32490549236536026, | |
| "rewards/accuracy_reward": 0.21875000558793545, | |
| "rewards/format_reward": 0.0, | |
| "step": 161 | |
| }, | |
| { | |
| "completion_length": 480.3020896911621, | |
| "epoch": 1.7253333333333334, | |
| "grad_norm": 0.3240108788013458, | |
| "kl": 0.6272735595703125, | |
| "learning_rate": 1.503006173035552e-07, | |
| "loss": 0.0577, | |
| "reward": 0.3020833348855376, | |
| "reward_std": 0.21069689840078354, | |
| "rewards/accuracy_reward": 0.3020833348855376, | |
| "rewards/format_reward": 0.0, | |
| "step": 162 | |
| }, | |
| { | |
| "completion_length": 655.1562614440918, | |
| "epoch": 1.736, | |
| "grad_norm": 0.16485954821109772, | |
| "kl": 0.4539642333984375, | |
| "learning_rate": 1.3822859185061453e-07, | |
| "loss": 0.0482, | |
| "reward": 0.34375000558793545, | |
| "reward_std": 0.3551059402525425, | |
| "rewards/accuracy_reward": 0.34375000558793545, | |
| "rewards/format_reward": 0.0, | |
| "step": 163 | |
| }, | |
| { | |
| "completion_length": 551.5833549499512, | |
| "epoch": 1.7466666666666666, | |
| "grad_norm": 0.33394524455070496, | |
| "kl": 0.3665771484375, | |
| "learning_rate": 1.2663846781859673e-07, | |
| "loss": 0.0863, | |
| "reward": 0.43750001303851604, | |
| "reward_std": 0.2447880432009697, | |
| "rewards/accuracy_reward": 0.43750001303851604, | |
| "rewards/format_reward": 0.0, | |
| "step": 164 | |
| }, | |
| { | |
| "completion_length": 633.9687728881836, | |
| "epoch": 1.7573333333333334, | |
| "grad_norm": 0.5149164795875549, | |
| "kl": 0.45458984375, | |
| "learning_rate": 1.1553434670148605e-07, | |
| "loss": 0.0203, | |
| "reward": 0.1666666716337204, | |
| "reward_std": 0.18463725596666336, | |
| "rewards/accuracy_reward": 0.1666666716337204, | |
| "rewards/format_reward": 0.0, | |
| "step": 165 | |
| }, | |
| { | |
| "completion_length": 551.354175567627, | |
| "epoch": 1.768, | |
| "grad_norm": 0.5073640942573547, | |
| "kl": 0.592529296875, | |
| "learning_rate": 1.049201580073576e-07, | |
| "loss": 0.1616, | |
| "reward": 0.3125000111758709, | |
| "reward_std": 0.35565972700715065, | |
| "rewards/accuracy_reward": 0.3125000111758709, | |
| "rewards/format_reward": 0.0, | |
| "step": 166 | |
| }, | |
| { | |
| "completion_length": 652.8750228881836, | |
| "epoch": 1.7786666666666666, | |
| "grad_norm": 0.38101547956466675, | |
| "kl": 0.4947509765625, | |
| "learning_rate": 9.479965786780847e-08, | |
| "loss": 0.0703, | |
| "reward": 0.28125000838190317, | |
| "reward_std": 0.31957534328103065, | |
| "rewards/accuracy_reward": 0.28125000838190317, | |
| "rewards/format_reward": 0.0, | |
| "step": 167 | |
| }, | |
| { | |
| "completion_length": 664.114616394043, | |
| "epoch": 1.7893333333333334, | |
| "grad_norm": 0.29761403799057007, | |
| "kl": 0.537353515625, | |
| "learning_rate": 8.517642770874318e-08, | |
| "loss": 0.0996, | |
| "reward": 0.4062500111758709, | |
| "reward_std": 0.2633597403764725, | |
| "rewards/accuracy_reward": 0.4062500111758709, | |
| "rewards/format_reward": 0.0, | |
| "step": 168 | |
| }, | |
| { | |
| "completion_length": 566.6146087646484, | |
| "epoch": 1.8, | |
| "grad_norm": 0.5472803115844727, | |
| "kl": 0.47998046875, | |
| "learning_rate": 7.60538729829861e-08, | |
| "loss": 0.1289, | |
| "reward": 0.36458334885537624, | |
| "reward_std": 0.27501849830150604, | |
| "rewards/accuracy_reward": 0.36458334885537624, | |
| "rewards/format_reward": 0.0, | |
| "step": 169 | |
| }, | |
| { | |
| "completion_length": 566.8750228881836, | |
| "epoch": 1.8106666666666666, | |
| "grad_norm": 0.4136505722999573, | |
| "kl": 0.71148681640625, | |
| "learning_rate": 6.743522196516388e-08, | |
| "loss": 0.1041, | |
| "reward": 0.21875000838190317, | |
| "reward_std": 0.2671857886016369, | |
| "rewards/accuracy_reward": 0.21875000838190317, | |
| "rewards/format_reward": 0.0, | |
| "step": 170 | |
| }, | |
| { | |
| "completion_length": 616.1458587646484, | |
| "epoch": 1.8213333333333335, | |
| "grad_norm": 0.42460325360298157, | |
| "kl": 0.53466796875, | |
| "learning_rate": 5.932352460929052e-08, | |
| "loss": 0.1008, | |
| "reward": 0.2604166688397527, | |
| "reward_std": 0.2951196953654289, | |
| "rewards/accuracy_reward": 0.2604166688397527, | |
| "rewards/format_reward": 0.0, | |
| "step": 171 | |
| }, | |
| { | |
| "completion_length": 569.2083435058594, | |
| "epoch": 1.8319999999999999, | |
| "grad_norm": 0.3544805347919464, | |
| "kl": 0.1856689453125, | |
| "learning_rate": 5.172165146945296e-08, | |
| "loss": 0.1447, | |
| "reward": 0.3645833395421505, | |
| "reward_std": 0.359878059476614, | |
| "rewards/accuracy_reward": 0.3645833395421505, | |
| "rewards/format_reward": 0.0, | |
| "step": 172 | |
| }, | |
| { | |
| "completion_length": 539.2812652587891, | |
| "epoch": 1.8426666666666667, | |
| "grad_norm": 0.19653554260730743, | |
| "kl": 0.3319091796875, | |
| "learning_rate": 4.463229268398611e-08, | |
| "loss": 0.1411, | |
| "reward": 0.3125000139698386, | |
| "reward_std": 0.3444414474070072, | |
| "rewards/accuracy_reward": 0.3125000139698386, | |
| "rewards/format_reward": 0.0, | |
| "step": 173 | |
| }, | |
| { | |
| "completion_length": 485.46876525878906, | |
| "epoch": 1.8533333333333335, | |
| "grad_norm": 0.3276241421699524, | |
| "kl": 0.443328857421875, | |
| "learning_rate": 3.805795702348941e-08, | |
| "loss": 0.0521, | |
| "reward": 0.41666667722165585, | |
| "reward_std": 0.3042132332921028, | |
| "rewards/accuracy_reward": 0.41666667722165585, | |
| "rewards/format_reward": 0.0, | |
| "step": 174 | |
| }, | |
| { | |
| "completion_length": 619.3125076293945, | |
| "epoch": 1.8639999999999999, | |
| "grad_norm": 0.6277369856834412, | |
| "kl": 0.71484375, | |
| "learning_rate": 3.200097100302812e-08, | |
| "loss": 0.0719, | |
| "reward": 0.17708333488553762, | |
| "reward_std": 0.21069689840078354, | |
| "rewards/accuracy_reward": 0.17708333488553762, | |
| "rewards/format_reward": 0.0, | |
| "step": 175 | |
| }, | |
| { | |
| "completion_length": 574.572940826416, | |
| "epoch": 1.8746666666666667, | |
| "grad_norm": 0.5137563347816467, | |
| "kl": 0.8292236328125, | |
| "learning_rate": 2.6463478058829426e-08, | |
| "loss": 0.0123, | |
| "reward": 0.11458333861082792, | |
| "reward_std": 0.1569838561117649, | |
| "rewards/accuracy_reward": 0.11458333861082792, | |
| "rewards/format_reward": 0.0, | |
| "step": 176 | |
| }, | |
| { | |
| "completion_length": 495.9583511352539, | |
| "epoch": 1.8853333333333333, | |
| "grad_norm": 0.7497807145118713, | |
| "kl": 0.44970703125, | |
| "learning_rate": 2.144743778976782e-08, | |
| "loss": 0.0592, | |
| "reward": 0.43750000931322575, | |
| "reward_std": 0.3692372292280197, | |
| "rewards/accuracy_reward": 0.43750000931322575, | |
| "rewards/format_reward": 0.0, | |
| "step": 177 | |
| }, | |
| { | |
| "completion_length": 630.9271049499512, | |
| "epoch": 1.896, | |
| "grad_norm": 0.29420143365859985, | |
| "kl": 0.35113525390625, | |
| "learning_rate": 1.6954625263904177e-08, | |
| "loss": 0.1362, | |
| "reward": 0.312500006519258, | |
| "reward_std": 0.37610871344804764, | |
| "rewards/accuracy_reward": 0.312500006519258, | |
| "rewards/format_reward": 0.0, | |
| "step": 178 | |
| }, | |
| { | |
| "completion_length": 545.3750114440918, | |
| "epoch": 1.9066666666666667, | |
| "grad_norm": 0.38894397020339966, | |
| "kl": 0.6158599853515625, | |
| "learning_rate": 1.2986630390328214e-08, | |
| "loss": 0.0661, | |
| "reward": 0.44791667722165585, | |
| "reward_std": 0.31957533583045006, | |
| "rewards/accuracy_reward": 0.44791667722165585, | |
| "rewards/format_reward": 0.0, | |
| "step": 179 | |
| }, | |
| { | |
| "completion_length": 590.3020973205566, | |
| "epoch": 1.9173333333333333, | |
| "grad_norm": 0.2821875810623169, | |
| "kl": 0.53765869140625, | |
| "learning_rate": 9.54485735652455e-09, | |
| "loss": 0.0518, | |
| "reward": 0.3125000102445483, | |
| "reward_std": 0.18237072229385376, | |
| "rewards/accuracy_reward": 0.3125000102445483, | |
| "rewards/format_reward": 0.0, | |
| "step": 180 | |
| }, | |
| { | |
| "completion_length": 635.7187690734863, | |
| "epoch": 1.928, | |
| "grad_norm": 0.24759963154792786, | |
| "kl": 0.51806640625, | |
| "learning_rate": 6.630524131460037e-09, | |
| "loss": 0.0553, | |
| "reward": 0.23958333674818277, | |
| "reward_std": 0.24263745918869972, | |
| "rewards/accuracy_reward": 0.23958333674818277, | |
| "rewards/format_reward": 0.0, | |
| "step": 181 | |
| }, | |
| { | |
| "completion_length": 558.8333377838135, | |
| "epoch": 1.9386666666666668, | |
| "grad_norm": 0.2948830723762512, | |
| "kl": 0.4090576171875, | |
| "learning_rate": 4.244662034572022e-09, | |
| "loss": 0.0335, | |
| "reward": 0.29166667722165585, | |
| "reward_std": 0.298287820070982, | |
| "rewards/accuracy_reward": 0.29166667722165585, | |
| "rewards/format_reward": 0.0, | |
| "step": 182 | |
| }, | |
| { | |
| "completion_length": 596.3646049499512, | |
| "epoch": 1.9493333333333334, | |
| "grad_norm": 0.23650887608528137, | |
| "kl": 0.533660888671875, | |
| "learning_rate": 2.3881153708059656e-09, | |
| "loss": 0.1084, | |
| "reward": 0.3229166669771075, | |
| "reward_std": 0.2319217473268509, | |
| "rewards/accuracy_reward": 0.3229166669771075, | |
| "rewards/format_reward": 0.0, | |
| "step": 183 | |
| }, | |
| { | |
| "completion_length": 461.8020896911621, | |
| "epoch": 1.96, | |
| "grad_norm": 0.697127640247345, | |
| "kl": 0.49359130859375, | |
| "learning_rate": 1.0615411318344315e-09, | |
| "loss": 0.1386, | |
| "reward": 0.4895833460614085, | |
| "reward_std": 0.45709774643182755, | |
| "rewards/accuracy_reward": 0.4895833460614085, | |
| "rewards/format_reward": 0.0, | |
| "step": 184 | |
| }, | |
| { | |
| "completion_length": 533.010425567627, | |
| "epoch": 1.9706666666666668, | |
| "grad_norm": 0.9241780042648315, | |
| "kl": 0.367156982421875, | |
| "learning_rate": 2.6540876356256906e-10, | |
| "loss": 0.1469, | |
| "reward": 0.5104166744276881, | |
| "reward_std": 0.3491060249507427, | |
| "rewards/accuracy_reward": 0.5104166744276881, | |
| "rewards/format_reward": 0.0, | |
| "step": 185 | |
| }, | |
| { | |
| "completion_length": 609.9062728881836, | |
| "epoch": 1.9813333333333332, | |
| "grad_norm": 0.3692702353000641, | |
| "kl": 0.4107666015625, | |
| "learning_rate": 0.0, | |
| "loss": 0.1591, | |
| "reward": 0.31250001303851604, | |
| "reward_std": 0.34694400057196617, | |
| "rewards/accuracy_reward": 0.31250001303851604, | |
| "rewards/format_reward": 0.0, | |
| "step": 186 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 186, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 2, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 4, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |