| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.8333333333333334, |
| "eval_steps": 500, |
| "global_step": 80, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 153.1875, |
| "epoch": 0.010416666666666666, |
| "grad_norm": 2.2964696301344203, |
| "kl": 0.0008754730224609375, |
| "learning_rate": 1e-06, |
| "loss": -0.0104, |
| "num_tokens": 44706.0, |
| "reward": 0.46666670590639114, |
| "reward_std": 0.7099685594439507, |
| "rewards/warm_up_reward/mean": 0.3888888955116272, |
| "rewards/warm_up_reward/std": 0.7168056517839432, |
| "step": 1 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 147.95833587646484, |
| "epoch": 0.020833333333333332, |
| "grad_norm": 2.332332033549255, |
| "kl": 0.0011844635009765625, |
| "learning_rate": 1e-06, |
| "loss": 0.0071, |
| "num_tokens": 88904.0, |
| "reward": 0.5250000506639481, |
| "reward_std": 0.748512014746666, |
| "rewards/warm_up_reward/mean": 0.4375, |
| "rewards/warm_up_reward/std": 0.6997176110744476, |
| "step": 2 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 154.00000381469727, |
| "epoch": 0.03125, |
| "grad_norm": 2.04671487874671, |
| "kl": 0.0013523101806640625, |
| "learning_rate": 1e-06, |
| "loss": 0.0124, |
| "num_tokens": 133634.0, |
| "reward": 0.500000037252903, |
| "reward_std": 0.667382538318634, |
| "rewards/warm_up_reward/mean": 0.416666679084301, |
| "rewards/warm_up_reward/std": 0.7197580486536026, |
| "step": 3 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 151.69792556762695, |
| "epoch": 0.041666666666666664, |
| "grad_norm": 2.2153093232246226, |
| "kl": 0.00243377685546875, |
| "learning_rate": 1e-06, |
| "loss": 0.0232, |
| "num_tokens": 178089.0, |
| "reward": 0.43125002086162567, |
| "reward_std": 0.662388876080513, |
| "rewards/warm_up_reward/mean": 0.359375, |
| "rewards/warm_up_reward/std": 0.6697845309972763, |
| "step": 4 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 148.84375762939453, |
| "epoch": 0.052083333333333336, |
| "grad_norm": 2.4723944423129782, |
| "kl": 0.004669189453125, |
| "learning_rate": 1e-06, |
| "loss": 0.0002, |
| "num_tokens": 222360.0, |
| "reward": 0.6250000596046448, |
| "reward_std": 0.8341160118579865, |
| "rewards/warm_up_reward/mean": 0.5208333358168602, |
| "rewards/warm_up_reward/std": 0.7749323397874832, |
| "step": 5 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 147.4791717529297, |
| "epoch": 0.0625, |
| "grad_norm": 2.452268566541841, |
| "kl": 0.00980377197265625, |
| "learning_rate": 1e-06, |
| "loss": 0.05, |
| "num_tokens": 266434.0, |
| "reward": 0.5625000298023224, |
| "reward_std": 0.8674589395523071, |
| "rewards/warm_up_reward/mean": 0.46875, |
| "rewards/warm_up_reward/std": 0.7468476742506027, |
| "step": 6 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 147.79166793823242, |
| "epoch": 0.07291666666666667, |
| "grad_norm": 55.84129867431237, |
| "kl": 0.194091796875, |
| "learning_rate": 1e-06, |
| "loss": 0.0511, |
| "num_tokens": 310532.0, |
| "reward": 0.9333333820104599, |
| "reward_std": 0.9551109671592712, |
| "rewards/warm_up_reward/mean": 0.7777777910232544, |
| "rewards/warm_up_reward/std": 0.8151216059923172, |
| "step": 7 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 157.65625381469727, |
| "epoch": 0.08333333333333333, |
| "grad_norm": 29.072228981795064, |
| "kl": 0.1204376220703125, |
| "learning_rate": 1e-06, |
| "loss": 0.0653, |
| "num_tokens": 355673.0, |
| "reward": 0.8281250894069672, |
| "reward_std": 0.9019797444343567, |
| "rewards/warm_up_reward/mean": 0.6901041716337204, |
| "rewards/warm_up_reward/std": 0.800490528345108, |
| "step": 8 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 147.62500381469727, |
| "epoch": 0.09375, |
| "grad_norm": 6.698990666563141, |
| "kl": 0.04058837890625, |
| "learning_rate": 1e-06, |
| "loss": 0.0742, |
| "num_tokens": 399929.0, |
| "reward": 0.6916667073965073, |
| "reward_std": 0.7857400476932526, |
| "rewards/warm_up_reward/mean": 0.5763888955116272, |
| "rewards/warm_up_reward/std": 0.7836765646934509, |
| "step": 9 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 150.2291717529297, |
| "epoch": 0.10416666666666667, |
| "grad_norm": 5.758187436239667, |
| "kl": 0.1109619140625, |
| "learning_rate": 1e-06, |
| "loss": 0.0303, |
| "num_tokens": 444279.0, |
| "reward": 0.9000000357627869, |
| "reward_std": 0.9381224364042282, |
| "rewards/warm_up_reward/mean": 0.75, |
| "rewards/warm_up_reward/std": 0.820982426404953, |
| "step": 10 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 150.67708587646484, |
| "epoch": 0.11458333333333333, |
| "grad_norm": 24.363046434997802, |
| "kl": 0.2352294921875, |
| "learning_rate": 1e-06, |
| "loss": 0.0567, |
| "num_tokens": 488708.0, |
| "reward": 0.8645834177732468, |
| "reward_std": 0.9693308770656586, |
| "rewards/warm_up_reward/mean": 0.720486119389534, |
| "rewards/warm_up_reward/std": 0.8197668194770813, |
| "step": 11 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 155.2916717529297, |
| "epoch": 0.125, |
| "grad_norm": 3.1714199544174053, |
| "kl": 0.08984375, |
| "learning_rate": 1e-06, |
| "loss": 0.0481, |
| "num_tokens": 533526.0, |
| "reward": 1.0781250596046448, |
| "reward_std": 0.9790745824575424, |
| "rewards/warm_up_reward/mean": 0.8984375, |
| "rewards/warm_up_reward/std": 0.8148495256900787, |
| "step": 12 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 140.2395896911621, |
| "epoch": 0.13541666666666666, |
| "grad_norm": 5.687529927842251, |
| "kl": 0.12872314453125, |
| "learning_rate": 1e-06, |
| "loss": 0.0633, |
| "num_tokens": 576965.0, |
| "reward": 0.9635417610406876, |
| "reward_std": 0.9496043026447296, |
| "rewards/warm_up_reward/mean": 0.802951380610466, |
| "rewards/warm_up_reward/std": 0.7969174236059189, |
| "step": 13 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 137.87500762939453, |
| "epoch": 0.14583333333333334, |
| "grad_norm": 4.785248899065895, |
| "kl": 0.09088134765625, |
| "learning_rate": 1e-06, |
| "loss": 0.0956, |
| "num_tokens": 620003.0, |
| "reward": 0.8687500655651093, |
| "reward_std": 0.8496406525373459, |
| "rewards/warm_up_reward/mean": 0.7239583283662796, |
| "rewards/warm_up_reward/std": 0.7836050242185593, |
| "step": 14 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 140.87500381469727, |
| "epoch": 0.15625, |
| "grad_norm": 3.043938278706536, |
| "kl": 0.06243896484375, |
| "learning_rate": 1e-06, |
| "loss": 0.048, |
| "num_tokens": 663347.0, |
| "reward": 0.8843750804662704, |
| "reward_std": 0.9135490357875824, |
| "rewards/warm_up_reward/mean": 0.7369791716337204, |
| "rewards/warm_up_reward/std": 0.8226732462644577, |
| "step": 15 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 145.0833396911621, |
| "epoch": 0.16666666666666666, |
| "grad_norm": 4.964409464782977, |
| "kl": 0.0794677734375, |
| "learning_rate": 1e-06, |
| "loss": 0.0574, |
| "num_tokens": 707341.0, |
| "reward": 0.9437500536441803, |
| "reward_std": 0.8649309277534485, |
| "rewards/warm_up_reward/mean": 0.7864583283662796, |
| "rewards/warm_up_reward/std": 0.795333594083786, |
| "step": 16 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 143.76041793823242, |
| "epoch": 0.17708333333333334, |
| "grad_norm": 2.5933725635254965, |
| "kl": 0.0601806640625, |
| "learning_rate": 1e-06, |
| "loss": 0.0293, |
| "num_tokens": 751028.0, |
| "reward": 0.7000000327825546, |
| "reward_std": 0.9639299660921097, |
| "rewards/warm_up_reward/mean": 0.5833333283662796, |
| "rewards/warm_up_reward/std": 0.7868598401546478, |
| "step": 17 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 140.90625381469727, |
| "epoch": 0.1875, |
| "grad_norm": 2.560263758795748, |
| "kl": 0.0513916015625, |
| "learning_rate": 1e-06, |
| "loss": 0.0333, |
| "num_tokens": 794597.0, |
| "reward": 0.947916716337204, |
| "reward_std": 0.9378172904253006, |
| "rewards/warm_up_reward/mean": 0.7899305373430252, |
| "rewards/warm_up_reward/std": 0.8166805952787399, |
| "step": 18 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 134.42708587646484, |
| "epoch": 0.19791666666666666, |
| "grad_norm": 2.827678010439988, |
| "kl": 0.05694580078125, |
| "learning_rate": 1e-06, |
| "loss": 0.0819, |
| "num_tokens": 837364.0, |
| "reward": 0.9250000715255737, |
| "reward_std": 0.9560818523168564, |
| "rewards/warm_up_reward/mean": 0.7708333283662796, |
| "rewards/warm_up_reward/std": 0.8091117739677429, |
| "step": 19 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 137.16666793823242, |
| "epoch": 0.20833333333333334, |
| "grad_norm": 2.92256682920272, |
| "kl": 0.05963134765625, |
| "learning_rate": 1e-06, |
| "loss": 0.0522, |
| "num_tokens": 880442.0, |
| "reward": 0.9791667610406876, |
| "reward_std": 0.9076904356479645, |
| "rewards/warm_up_reward/mean": 0.8159722089767456, |
| "rewards/warm_up_reward/std": 0.8102934062480927, |
| "step": 20 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 137.84375381469727, |
| "epoch": 0.21875, |
| "grad_norm": 2.864931852177023, |
| "kl": 0.04974365234375, |
| "learning_rate": 1e-06, |
| "loss": 0.0248, |
| "num_tokens": 923615.0, |
| "reward": 0.9281250536441803, |
| "reward_std": 0.9918985664844513, |
| "rewards/warm_up_reward/mean": 0.7734375, |
| "rewards/warm_up_reward/std": 0.8185379058122635, |
| "step": 21 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 130.8750057220459, |
| "epoch": 0.22916666666666666, |
| "grad_norm": 2.8348309851740456, |
| "kl": 0.05242919921875, |
| "learning_rate": 1e-06, |
| "loss": 0.0427, |
| "num_tokens": 966107.0, |
| "reward": 1.0531250834465027, |
| "reward_std": 0.9989801347255707, |
| "rewards/warm_up_reward/mean": 0.8776041567325592, |
| "rewards/warm_up_reward/std": 0.8102044314146042, |
| "step": 22 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 144.26041793823242, |
| "epoch": 0.23958333333333334, |
| "grad_norm": 2.6071380741149004, |
| "kl": 0.06549072265625, |
| "learning_rate": 1e-06, |
| "loss": 0.0523, |
| "num_tokens": 1009998.0, |
| "reward": 0.9625000357627869, |
| "reward_std": 0.9378929734230042, |
| "rewards/warm_up_reward/mean": 0.8020833432674408, |
| "rewards/warm_up_reward/std": 0.8023640215396881, |
| "step": 23 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 139.73958587646484, |
| "epoch": 0.25, |
| "grad_norm": 2.697375275936146, |
| "kl": 0.05670166015625, |
| "learning_rate": 1e-06, |
| "loss": 0.0705, |
| "num_tokens": 1053401.0, |
| "reward": 1.031250074505806, |
| "reward_std": 0.9829376488924026, |
| "rewards/warm_up_reward/mean": 0.859375, |
| "rewards/warm_up_reward/std": 0.813440352678299, |
| "step": 24 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 138.8854217529297, |
| "epoch": 0.2604166666666667, |
| "grad_norm": 85.52835351864486, |
| "kl": 0.198486328125, |
| "learning_rate": 1e-06, |
| "loss": 0.0333, |
| "num_tokens": 1096650.0, |
| "reward": 1.089583471417427, |
| "reward_std": 0.9481612741947174, |
| "rewards/warm_up_reward/mean": 0.9079861044883728, |
| "rewards/warm_up_reward/std": 0.7854074388742447, |
| "step": 25 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 130.62500381469727, |
| "epoch": 0.2708333333333333, |
| "grad_norm": 3.4064495832019297, |
| "kl": 0.07513427734375, |
| "learning_rate": 1e-06, |
| "loss": 0.0544, |
| "num_tokens": 1139058.0, |
| "reward": 0.9885417520999908, |
| "reward_std": 0.990489736199379, |
| "rewards/warm_up_reward/mean": 0.8237847238779068, |
| "rewards/warm_up_reward/std": 0.8235566318035126, |
| "step": 26 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 134.50000381469727, |
| "epoch": 0.28125, |
| "grad_norm": 118.32186343464612, |
| "kl": 0.2930908203125, |
| "learning_rate": 1e-06, |
| "loss": 0.011, |
| "num_tokens": 1181802.0, |
| "reward": 1.0552084296941757, |
| "reward_std": 0.868858814239502, |
| "rewards/warm_up_reward/mean": 0.8793402910232544, |
| "rewards/warm_up_reward/std": 0.8186527788639069, |
| "step": 27 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 140.8541717529297, |
| "epoch": 0.2916666666666667, |
| "grad_norm": 6.711195724360143, |
| "kl": 0.1092529296875, |
| "learning_rate": 1e-06, |
| "loss": 0.0187, |
| "num_tokens": 1225408.0, |
| "reward": 0.9666667431592941, |
| "reward_std": 0.9556048065423965, |
| "rewards/warm_up_reward/mean": 0.8055555671453476, |
| "rewards/warm_up_reward/std": 0.8192583322525024, |
| "step": 28 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 137.48958778381348, |
| "epoch": 0.3020833333333333, |
| "grad_norm": 4.712858469896548, |
| "kl": 0.13214111328125, |
| "learning_rate": 1e-06, |
| "loss": 0.0356, |
| "num_tokens": 1268481.0, |
| "reward": 0.9000000655651093, |
| "reward_std": 1.0170713812112808, |
| "rewards/warm_up_reward/mean": 0.7500000149011612, |
| "rewards/warm_up_reward/std": 0.8409183472394943, |
| "step": 29 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 142.09375381469727, |
| "epoch": 0.3125, |
| "grad_norm": 2.604660112727859, |
| "kl": 0.063720703125, |
| "learning_rate": 1e-06, |
| "loss": -0.0023, |
| "num_tokens": 1312008.0, |
| "reward": 0.9000000059604645, |
| "reward_std": 0.9869166016578674, |
| "rewards/warm_up_reward/mean": 0.75, |
| "rewards/warm_up_reward/std": 0.8337783664464951, |
| "step": 30 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 145.59375381469727, |
| "epoch": 0.3229166666666667, |
| "grad_norm": 6.185404235919849, |
| "kl": 0.157470703125, |
| "learning_rate": 1e-06, |
| "loss": 0.0097, |
| "num_tokens": 1356039.0, |
| "reward": 1.0770834237337112, |
| "reward_std": 0.9777155965566635, |
| "rewards/warm_up_reward/mean": 0.8975694477558136, |
| "rewards/warm_up_reward/std": 0.7948237210512161, |
| "step": 31 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 146.37500762939453, |
| "epoch": 0.3333333333333333, |
| "grad_norm": 2.4537091049295956, |
| "kl": 0.06231689453125, |
| "learning_rate": 1e-06, |
| "loss": 0.0046, |
| "num_tokens": 1400007.0, |
| "reward": 0.9812500327825546, |
| "reward_std": 0.911426916718483, |
| "rewards/warm_up_reward/mean": 0.8177083283662796, |
| "rewards/warm_up_reward/std": 0.7836276739835739, |
| "step": 32 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 145.19792556762695, |
| "epoch": 0.34375, |
| "grad_norm": 3.0265092714690702, |
| "kl": 0.061767578125, |
| "learning_rate": 1e-06, |
| "loss": 0.0423, |
| "num_tokens": 1443874.0, |
| "reward": 1.0354167222976685, |
| "reward_std": 0.9364243745803833, |
| "rewards/warm_up_reward/mean": 0.8628472238779068, |
| "rewards/warm_up_reward/std": 0.8118415027856827, |
| "step": 33 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 143.8645896911621, |
| "epoch": 0.3541666666666667, |
| "grad_norm": 2.4770416085087477, |
| "kl": 0.05682373046875, |
| "learning_rate": 1e-06, |
| "loss": 0.0242, |
| "num_tokens": 1487601.0, |
| "reward": 0.9906250834465027, |
| "reward_std": 1.0336193144321442, |
| "rewards/warm_up_reward/mean": 0.8255208432674408, |
| "rewards/warm_up_reward/std": 0.8268236815929413, |
| "step": 34 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 137.34375762939453, |
| "epoch": 0.3645833333333333, |
| "grad_norm": 2.787024849258649, |
| "kl": 0.051513671875, |
| "learning_rate": 1e-06, |
| "loss": 0.0409, |
| "num_tokens": 1530594.0, |
| "reward": 1.0822917073965073, |
| "reward_std": 0.8545732349157333, |
| "rewards/warm_up_reward/mean": 0.9019097238779068, |
| "rewards/warm_up_reward/std": 0.7748938798904419, |
| "step": 35 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 145.71875381469727, |
| "epoch": 0.375, |
| "grad_norm": 2.515335560097886, |
| "kl": 0.0687255859375, |
| "learning_rate": 1e-06, |
| "loss": 0.0284, |
| "num_tokens": 1574409.0, |
| "reward": 0.9875000715255737, |
| "reward_std": 0.9168877303600311, |
| "rewards/warm_up_reward/mean": 0.8229166567325592, |
| "rewards/warm_up_reward/std": 0.8267286717891693, |
| "step": 36 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 136.58333778381348, |
| "epoch": 0.3854166666666667, |
| "grad_norm": 11.790654099301383, |
| "kl": 0.1207275390625, |
| "learning_rate": 1e-06, |
| "loss": 0.0652, |
| "num_tokens": 1617503.0, |
| "reward": 1.031250074505806, |
| "reward_std": 0.9142753481864929, |
| "rewards/warm_up_reward/mean": 0.859375, |
| "rewards/warm_up_reward/std": 0.7878352403640747, |
| "step": 37 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 151.8645896911621, |
| "epoch": 0.3958333333333333, |
| "grad_norm": 3.4202024786499643, |
| "kl": 0.08099365234375, |
| "learning_rate": 1e-06, |
| "loss": 0.0212, |
| "num_tokens": 1661980.0, |
| "reward": 0.6229167133569717, |
| "reward_std": 0.8097958564758301, |
| "rewards/warm_up_reward/mean": 0.5190972313284874, |
| "rewards/warm_up_reward/std": 0.7617596387863159, |
| "step": 38 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 142.8854217529297, |
| "epoch": 0.40625, |
| "grad_norm": 2.4994048524913364, |
| "kl": 0.06414794921875, |
| "learning_rate": 1e-06, |
| "loss": 0.0569, |
| "num_tokens": 1705637.0, |
| "reward": 1.0656251087784767, |
| "reward_std": 0.8034301698207855, |
| "rewards/warm_up_reward/mean": 0.8880208432674408, |
| "rewards/warm_up_reward/std": 0.7390912175178528, |
| "step": 39 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 146.18750762939453, |
| "epoch": 0.4166666666666667, |
| "grad_norm": 2.4999575203414186, |
| "kl": 0.0628662109375, |
| "learning_rate": 1e-06, |
| "loss": 0.0137, |
| "num_tokens": 1749719.0, |
| "reward": 1.0104167312383652, |
| "reward_std": 0.9736887365579605, |
| "rewards/warm_up_reward/mean": 0.8420138955116272, |
| "rewards/warm_up_reward/std": 0.822531133890152, |
| "step": 40 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 143.9479217529297, |
| "epoch": 0.4270833333333333, |
| "grad_norm": 2.2639443093591205, |
| "kl": 0.0675048828125, |
| "learning_rate": 1e-06, |
| "loss": 0.0087, |
| "num_tokens": 1793436.0, |
| "reward": 0.9229167401790619, |
| "reward_std": 0.9153347015380859, |
| "rewards/warm_up_reward/mean": 0.7690972089767456, |
| "rewards/warm_up_reward/std": 0.8183709383010864, |
| "step": 41 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 147.58333587646484, |
| "epoch": 0.4375, |
| "grad_norm": 2.5193978009218965, |
| "kl": 0.07177734375, |
| "learning_rate": 1e-06, |
| "loss": 0.0392, |
| "num_tokens": 1837562.0, |
| "reward": 1.006250038743019, |
| "reward_std": 0.8919505327939987, |
| "rewards/warm_up_reward/mean": 0.8385416716337204, |
| "rewards/warm_up_reward/std": 0.8046689182519913, |
| "step": 42 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 141.37500381469727, |
| "epoch": 0.4479166666666667, |
| "grad_norm": 2.5496207567669034, |
| "kl": 0.06622314453125, |
| "learning_rate": 1e-06, |
| "loss": 0.0452, |
| "num_tokens": 1880972.0, |
| "reward": 1.068750038743019, |
| "reward_std": 0.9111972749233246, |
| "rewards/warm_up_reward/mean": 0.890625, |
| "rewards/warm_up_reward/std": 0.7989336252212524, |
| "step": 43 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 130.83333587646484, |
| "epoch": 0.4583333333333333, |
| "grad_norm": 5.245713134644536, |
| "kl": 0.0999755859375, |
| "learning_rate": 1e-06, |
| "loss": -0.0051, |
| "num_tokens": 1923460.0, |
| "reward": 0.929166704416275, |
| "reward_std": 0.8137651234865189, |
| "rewards/warm_up_reward/mean": 0.7743055373430252, |
| "rewards/warm_up_reward/std": 0.8136637955904007, |
| "step": 44 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 144.2604217529297, |
| "epoch": 0.46875, |
| "grad_norm": 6.317144379422631, |
| "kl": 0.10107421875, |
| "learning_rate": 1e-06, |
| "loss": -0.0175, |
| "num_tokens": 1967315.0, |
| "reward": 1.1012500673532486, |
| "reward_std": 0.7948171943426132, |
| "rewards/warm_up_reward/mean": 0.9177083224058151, |
| "rewards/warm_up_reward/std": 0.7791551500558853, |
| "step": 45 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 141.5104217529297, |
| "epoch": 0.4791666666666667, |
| "grad_norm": 2.8932825846308172, |
| "kl": 0.0859375, |
| "learning_rate": 1e-06, |
| "loss": 0.0029, |
| "num_tokens": 2010882.0, |
| "reward": 1.0656251162290573, |
| "reward_std": 0.9644656330347061, |
| "rewards/warm_up_reward/mean": 0.8880208432674408, |
| "rewards/warm_up_reward/std": 0.8194199502468109, |
| "step": 46 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 140.21875762939453, |
| "epoch": 0.4895833333333333, |
| "grad_norm": 2.9093719889125174, |
| "kl": 0.1007080078125, |
| "learning_rate": 1e-06, |
| "loss": -0.0275, |
| "num_tokens": 2054307.0, |
| "reward": 0.991666704416275, |
| "reward_std": 0.9324973523616791, |
| "rewards/warm_up_reward/mean": 0.8263888955116272, |
| "rewards/warm_up_reward/std": 0.8247981667518616, |
| "step": 47 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 136.5729217529297, |
| "epoch": 0.5, |
| "grad_norm": 2.3209209113606146, |
| "kl": 0.08441162109375, |
| "learning_rate": 1e-06, |
| "loss": 0.001, |
| "num_tokens": 2097424.0, |
| "reward": 1.1010417491197586, |
| "reward_std": 0.8305719494819641, |
| "rewards/warm_up_reward/mean": 0.9175347238779068, |
| "rewards/warm_up_reward/std": 0.7910451591014862, |
| "step": 48 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 142.50000762939453, |
| "epoch": 0.5104166666666666, |
| "grad_norm": 2.470827147583925, |
| "kl": 0.08355712890625, |
| "learning_rate": 1e-06, |
| "loss": 0.0433, |
| "num_tokens": 2141092.0, |
| "reward": 1.0208334028720856, |
| "reward_std": 0.9052031934261322, |
| "rewards/warm_up_reward/mean": 0.8506944477558136, |
| "rewards/warm_up_reward/std": 0.8133653849363327, |
| "step": 49 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 138.58333587646484, |
| "epoch": 0.5208333333333334, |
| "grad_norm": 2.314915736053297, |
| "kl": 0.0765380859375, |
| "learning_rate": 1e-06, |
| "loss": 0.0188, |
| "num_tokens": 2184306.0, |
| "reward": 1.050000086426735, |
| "reward_std": 0.9242848604917526, |
| "rewards/warm_up_reward/mean": 0.8749999850988388, |
| "rewards/warm_up_reward/std": 0.7892495840787888, |
| "step": 50 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 138.92708778381348, |
| "epoch": 0.53125, |
| "grad_norm": 16.28250872141934, |
| "kl": 0.26171875, |
| "learning_rate": 1e-06, |
| "loss": 0.0328, |
| "num_tokens": 2227673.0, |
| "reward": 1.1312500685453415, |
| "reward_std": 1.0009342432022095, |
| "rewards/warm_up_reward/mean": 0.9427083432674408, |
| "rewards/warm_up_reward/std": 0.8114291131496429, |
| "step": 51 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 140.92708587646484, |
| "epoch": 0.5416666666666666, |
| "grad_norm": 2.4052567403889364, |
| "kl": 0.0528564453125, |
| "learning_rate": 1e-06, |
| "loss": 0.0201, |
| "num_tokens": 2271064.0, |
| "reward": 1.2375000417232513, |
| "reward_std": 0.8487301468849182, |
| "rewards/warm_up_reward/mean": 1.03125, |
| "rewards/warm_up_reward/std": 0.7739475220441818, |
| "step": 52 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 146.1458396911621, |
| "epoch": 0.5520833333333334, |
| "grad_norm": 2.4582483635661467, |
| "kl": 0.05413818359375, |
| "learning_rate": 1e-06, |
| "loss": 0.0429, |
| "num_tokens": 2315052.0, |
| "reward": 1.1104167699813843, |
| "reward_std": 0.9700468927621841, |
| "rewards/warm_up_reward/mean": 0.9253472238779068, |
| "rewards/warm_up_reward/std": 0.7955830246210098, |
| "step": 53 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 145.2083396911621, |
| "epoch": 0.5625, |
| "grad_norm": 2.4577305981193165, |
| "kl": 0.0728759765625, |
| "learning_rate": 1e-06, |
| "loss": -0.0131, |
| "num_tokens": 2358962.0, |
| "reward": 1.1750001013278961, |
| "reward_std": 0.9326367676258087, |
| "rewards/warm_up_reward/mean": 0.9791666716337204, |
| "rewards/warm_up_reward/std": 0.7897387892007828, |
| "step": 54 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 145.7916717529297, |
| "epoch": 0.5729166666666666, |
| "grad_norm": 2.302326467124225, |
| "kl": 0.0810546875, |
| "learning_rate": 1e-06, |
| "loss": 0.0291, |
| "num_tokens": 2402922.0, |
| "reward": 0.9875000566244125, |
| "reward_std": 1.0013651847839355, |
| "rewards/warm_up_reward/mean": 0.8229166716337204, |
| "rewards/warm_up_reward/std": 0.8247637003660202, |
| "step": 55 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 147.9479217529297, |
| "epoch": 0.5833333333333334, |
| "grad_norm": 2.580316913788417, |
| "kl": 0.068115234375, |
| "learning_rate": 1e-06, |
| "loss": 0.0509, |
| "num_tokens": 2447065.0, |
| "reward": 1.183750033378601, |
| "reward_std": 0.9039967954158783, |
| "rewards/warm_up_reward/mean": 0.9864583313465118, |
| "rewards/warm_up_reward/std": 0.787983849644661, |
| "step": 56 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 142.05208587646484, |
| "epoch": 0.59375, |
| "grad_norm": 2.748934001160214, |
| "kl": 0.06414794921875, |
| "learning_rate": 1e-06, |
| "loss": -0.0078, |
| "num_tokens": 2490552.0, |
| "reward": 1.07750004529953, |
| "reward_std": 0.9763932228088379, |
| "rewards/warm_up_reward/mean": 0.8979166746139526, |
| "rewards/warm_up_reward/std": 0.8105349242687225, |
| "step": 57 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 146.0729217529297, |
| "epoch": 0.6041666666666666, |
| "grad_norm": 2.279728805159882, |
| "kl": 0.05682373046875, |
| "learning_rate": 1e-06, |
| "loss": 0.0054, |
| "num_tokens": 2534503.0, |
| "reward": 1.2562500685453415, |
| "reward_std": 0.8728772848844528, |
| "rewards/warm_up_reward/mean": 1.046875, |
| "rewards/warm_up_reward/std": 0.715716764330864, |
| "step": 58 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 136.77083587646484, |
| "epoch": 0.6145833333333334, |
| "grad_norm": 2.301940905359112, |
| "kl": 0.0634765625, |
| "learning_rate": 1e-06, |
| "loss": 0.0198, |
| "num_tokens": 2577603.0, |
| "reward": 1.2200001031160355, |
| "reward_std": 0.9238942861557007, |
| "rewards/warm_up_reward/mean": 1.0166666805744171, |
| "rewards/warm_up_reward/std": 0.7837828695774078, |
| "step": 59 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 149.3645896911621, |
| "epoch": 0.625, |
| "grad_norm": 2.3088797777576535, |
| "kl": 0.0701904296875, |
| "learning_rate": 1e-06, |
| "loss": 0.0668, |
| "num_tokens": 2621852.0, |
| "reward": 1.0885417461395264, |
| "reward_std": 0.9921838045120239, |
| "rewards/warm_up_reward/mean": 0.9071180820465088, |
| "rewards/warm_up_reward/std": 0.8187949508428574, |
| "step": 60 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 144.9791717529297, |
| "epoch": 0.6354166666666666, |
| "grad_norm": 2.3531788106957063, |
| "kl": 0.0830078125, |
| "learning_rate": 1e-06, |
| "loss": 0.0715, |
| "num_tokens": 2665584.0, |
| "reward": 1.103541761636734, |
| "reward_std": 0.8904776722192764, |
| "rewards/warm_up_reward/mean": 0.9196180552244186, |
| "rewards/warm_up_reward/std": 0.7996000051498413, |
| "step": 61 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 140.5416717529297, |
| "epoch": 0.6458333333333334, |
| "grad_norm": 2.3780245058749894, |
| "kl": 0.0545654296875, |
| "learning_rate": 1e-06, |
| "loss": 0.0144, |
| "num_tokens": 2709058.0, |
| "reward": 1.2650001347064972, |
| "reward_std": 0.8583473563194275, |
| "rewards/warm_up_reward/mean": 1.0541666597127914, |
| "rewards/warm_up_reward/std": 0.753357321023941, |
| "step": 62 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 150.5208396911621, |
| "epoch": 0.65625, |
| "grad_norm": 2.291724896507479, |
| "kl": 0.05914306640625, |
| "learning_rate": 1e-06, |
| "loss": 0.0102, |
| "num_tokens": 2753538.0, |
| "reward": 1.2431251406669617, |
| "reward_std": 0.8321643471717834, |
| "rewards/warm_up_reward/mean": 1.0359375476837158, |
| "rewards/warm_up_reward/std": 0.74222831428051, |
| "step": 63 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 143.2395896911621, |
| "epoch": 0.6666666666666666, |
| "grad_norm": 3.018275289311917, |
| "kl": 0.085693359375, |
| "learning_rate": 1e-06, |
| "loss": 0.0078, |
| "num_tokens": 2797253.0, |
| "reward": 1.1802085041999817, |
| "reward_std": 0.948539987206459, |
| "rewards/warm_up_reward/mean": 0.9835069626569748, |
| "rewards/warm_up_reward/std": 0.7748740911483765, |
| "step": 64 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 138.9791717529297, |
| "epoch": 0.6770833333333334, |
| "grad_norm": 2.219343433713507, |
| "kl": 0.06298828125, |
| "learning_rate": 1e-06, |
| "loss": 0.0203, |
| "num_tokens": 2840571.0, |
| "reward": 1.2229167520999908, |
| "reward_std": 0.783911868929863, |
| "rewards/warm_up_reward/mean": 1.0190972089767456, |
| "rewards/warm_up_reward/std": 0.7600821256637573, |
| "step": 65 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 152.8958396911621, |
| "epoch": 0.6875, |
| "grad_norm": 2.2126312475293877, |
| "kl": 0.0704345703125, |
| "learning_rate": 1e-06, |
| "loss": 0.0425, |
| "num_tokens": 2885201.0, |
| "reward": 1.1691668182611465, |
| "reward_std": 0.8853475451469421, |
| "rewards/warm_up_reward/mean": 0.9743055552244186, |
| "rewards/warm_up_reward/std": 0.7726792246103287, |
| "step": 66 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 147.61459350585938, |
| "epoch": 0.6979166666666666, |
| "grad_norm": 2.3729988144079748, |
| "kl": 0.05316162109375, |
| "learning_rate": 1e-06, |
| "loss": 0.0235, |
| "num_tokens": 2929396.0, |
| "reward": 1.3312501311302185, |
| "reward_std": 0.7987091541290283, |
| "rewards/warm_up_reward/mean": 1.109375, |
| "rewards/warm_up_reward/std": 0.7549550831317902, |
| "step": 67 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 144.1041717529297, |
| "epoch": 0.7083333333333334, |
| "grad_norm": 2.268946790543885, |
| "kl": 0.04766845703125, |
| "learning_rate": 1e-06, |
| "loss": -0.0027, |
| "num_tokens": 2973200.0, |
| "reward": 1.325416773557663, |
| "reward_std": 0.8379913568496704, |
| "rewards/warm_up_reward/mean": 1.1045138835906982, |
| "rewards/warm_up_reward/std": 0.7519785463809967, |
| "step": 68 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 140.4791717529297, |
| "epoch": 0.71875, |
| "grad_norm": 2.40410834715932, |
| "kl": 0.0625, |
| "learning_rate": 1e-06, |
| "loss": 0.0508, |
| "num_tokens": 3016728.0, |
| "reward": 1.143750011920929, |
| "reward_std": 0.8826231509447098, |
| "rewards/warm_up_reward/mean": 0.953125, |
| "rewards/warm_up_reward/std": 0.7572390139102936, |
| "step": 69 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 145.9791717529297, |
| "epoch": 0.7291666666666666, |
| "grad_norm": 2.377991017189631, |
| "kl": 0.06512451171875, |
| "learning_rate": 1e-06, |
| "loss": 0.0505, |
| "num_tokens": 3060790.0, |
| "reward": 1.2854167819023132, |
| "reward_std": 0.9141092300415039, |
| "rewards/warm_up_reward/mean": 1.0711805671453476, |
| "rewards/warm_up_reward/std": 0.7661919444799423, |
| "step": 70 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 144.7916717529297, |
| "epoch": 0.7395833333333334, |
| "grad_norm": 5.490360584080418, |
| "kl": 0.068115234375, |
| "learning_rate": 1e-06, |
| "loss": 0.0061, |
| "num_tokens": 3104672.0, |
| "reward": 1.2937501072883606, |
| "reward_std": 0.9385685622692108, |
| "rewards/warm_up_reward/mean": 1.078125, |
| "rewards/warm_up_reward/std": 0.7564428001642227, |
| "step": 71 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 132.94792366027832, |
| "epoch": 0.75, |
| "grad_norm": 2.705450393270237, |
| "kl": 0.0748291015625, |
| "learning_rate": 1e-06, |
| "loss": 0.0032, |
| "num_tokens": 3147441.0, |
| "reward": 1.2687500715255737, |
| "reward_std": 0.9319685697555542, |
| "rewards/warm_up_reward/mean": 1.0572916567325592, |
| "rewards/warm_up_reward/std": 0.7771977633237839, |
| "step": 72 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 151.91667556762695, |
| "epoch": 0.7604166666666666, |
| "grad_norm": 2.254735299608175, |
| "kl": 0.07171630859375, |
| "learning_rate": 1e-06, |
| "loss": 0.0105, |
| "num_tokens": 3192121.0, |
| "reward": 1.0045834183692932, |
| "reward_std": 0.8630332052707672, |
| "rewards/warm_up_reward/mean": 0.8371527940034866, |
| "rewards/warm_up_reward/std": 0.7969614416360855, |
| "step": 73 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 140.31250190734863, |
| "epoch": 0.7708333333333334, |
| "grad_norm": 2.4523220833578345, |
| "kl": 0.05816650390625, |
| "learning_rate": 1e-06, |
| "loss": 0.0158, |
| "num_tokens": 3235453.0, |
| "reward": 1.1837501227855682, |
| "reward_std": 0.9126418828964233, |
| "rewards/warm_up_reward/mean": 0.9864583015441895, |
| "rewards/warm_up_reward/std": 0.7863775044679642, |
| "step": 74 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 141.7395896911621, |
| "epoch": 0.78125, |
| "grad_norm": 2.2478331336635513, |
| "kl": 0.0609130859375, |
| "learning_rate": 1e-06, |
| "loss": 0.0284, |
| "num_tokens": 3278964.0, |
| "reward": 1.2054167687892914, |
| "reward_std": 0.9118891954421997, |
| "rewards/warm_up_reward/mean": 1.0045138746500015, |
| "rewards/warm_up_reward/std": 0.7829048186540604, |
| "step": 75 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 139.71875381469727, |
| "epoch": 0.7916666666666666, |
| "grad_norm": 2.6147718994418203, |
| "kl": 0.0904541015625, |
| "learning_rate": 1e-06, |
| "loss": 0.0143, |
| "num_tokens": 3322407.0, |
| "reward": 1.2660417556762695, |
| "reward_std": 0.9197860509157181, |
| "rewards/warm_up_reward/mean": 1.0550346970558167, |
| "rewards/warm_up_reward/std": 0.7622723281383514, |
| "step": 76 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 141.53125762939453, |
| "epoch": 0.8020833333333334, |
| "grad_norm": 2.530432648354832, |
| "kl": 0.093505859375, |
| "learning_rate": 1e-06, |
| "loss": -0.0104, |
| "num_tokens": 3366036.0, |
| "reward": 1.1020834147930145, |
| "reward_std": 0.902558371424675, |
| "rewards/warm_up_reward/mean": 0.9184028059244156, |
| "rewards/warm_up_reward/std": 0.7975014746189117, |
| "step": 77 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 146.0416717529297, |
| "epoch": 0.8125, |
| "grad_norm": 2.2386503327740015, |
| "kl": 0.0811767578125, |
| "learning_rate": 1e-06, |
| "loss": 0.0184, |
| "num_tokens": 3410026.0, |
| "reward": 1.3406251072883606, |
| "reward_std": 0.8862900286912918, |
| "rewards/warm_up_reward/mean": 1.1171874850988388, |
| "rewards/warm_up_reward/std": 0.7199237793684006, |
| "step": 78 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 133.6041717529297, |
| "epoch": 0.8229166666666666, |
| "grad_norm": 2.9736551256376744, |
| "kl": 0.0992431640625, |
| "learning_rate": 1e-06, |
| "loss": 0.0029, |
| "num_tokens": 3452750.0, |
| "reward": 1.3556251227855682, |
| "reward_std": 0.8310635536909103, |
| "rewards/warm_up_reward/mean": 1.129687488079071, |
| "rewards/warm_up_reward/std": 0.7288718819618225, |
| "step": 79 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 139.45834350585938, |
| "epoch": 0.8333333333333334, |
| "grad_norm": 4.081234898460602, |
| "kl": 0.15234375, |
| "learning_rate": 1e-06, |
| "loss": -0.0154, |
| "num_tokens": 3496018.0, |
| "reward": 1.0906250923871994, |
| "reward_std": 0.8768025040626526, |
| "rewards/warm_up_reward/mean": 0.9088541716337204, |
| "rewards/warm_up_reward/std": 0.8029628545045853, |
| "step": 80 |
| } |
| ], |
| "logging_steps": 1.0, |
| "max_steps": 96, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 16, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 6, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
