{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.30012004801920766, "eval_steps": 500, "global_step": 250, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 17003.0, "completions/max_terminated_length": 17003.0, "completions/mean_length": 10512.40625, "completions/mean_terminated_length": 10512.40625, "completions/min_length": 6733.0, "completions/min_terminated_length": 6733.0, "entropy": 0.2287780987098813, "epoch": 0.0012004801920768306, "frac_reward_zero_std": 0.25, "grad_norm": 0.3057364225387573, "learning_rate": 1e-06, "loss": 0.0187, "num_tokens": 377149.0, "reward": 2.172255277633667, "reward_std": 2.8551955223083496, "rewards/reward_func/mean": 2.1769778728485107, "rewards/reward_func/std": 2.863737106323242, "rewards/soft_overlong_punishment_reward/mean": -0.00472259521484375, "rewards/soft_overlong_punishment_reward/std": 0.026715034618973732, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9920079708099365, "sampling/importance_sampling_ratio/min": 0.06831115484237671, "sampling/sampling_logp_difference/max": 2.6836822032928467, "sampling/sampling_logp_difference/mean": 0.016660798341035843, "step": 1, "step_time": 260.7638783827424 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13544.0, "completions/max_terminated_length": 13544.0, "completions/mean_length": 9758.78125, "completions/mean_terminated_length": 9758.78125, "completions/min_length": 7933.0, "completions/min_terminated_length": 7933.0, "entropy": 0.26108355820178986, "epoch": 0.0024009603841536613, "frac_reward_zero_std": 0.0, "grad_norm": 0.45485132932662964, "learning_rate": 9.959999999999999e-07, "loss": -0.0342, "num_tokens": 724854.0, "reward": 2.3083786964416504, "reward_std": 3.1393442153930664, "rewards/reward_func/mean": 2.3083786964416504, "rewards/reward_func/std": 3.1393442153930664, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.9068528413772583, "sampling/importance_sampling_ratio/mean": 0.9908677339553833, "sampling/importance_sampling_ratio/min": 0.009255149401724339, "sampling/sampling_logp_difference/max": 4.682575225830078, "sampling/sampling_logp_difference/mean": 0.01871836744248867, "step": 2, "step_time": 251.4262609881116 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14033.0, "completions/max_terminated_length": 14033.0, "completions/mean_length": 9934.3125, "completions/mean_terminated_length": 9934.3125, "completions/min_length": 4489.0, "completions/min_terminated_length": 4489.0, "entropy": 0.26445347908884287, "epoch": 0.003601440576230492, "frac_reward_zero_std": 0.0, "grad_norm": 0.3962293267250061, "learning_rate": 9.92e-07, "loss": -0.0314, "num_tokens": 1075280.0, "reward": 2.6239943504333496, "reward_std": 2.9723362922668457, "rewards/reward_func/mean": 2.6239943504333496, "rewards/reward_func/std": 2.972336769104004, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.1135287284851074, "sampling/importance_sampling_ratio/mean": 0.9906726479530334, "sampling/importance_sampling_ratio/min": 0.20889143645763397, "sampling/sampling_logp_difference/max": 1.5659406185150146, "sampling/sampling_logp_difference/mean": 0.01895604282617569, "step": 3, "step_time": 210.0828171017347 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15270.0, "completions/max_terminated_length": 15270.0, "completions/mean_length": 10381.21875, "completions/mean_terminated_length": 10381.21875, "completions/min_length": 5544.0, "completions/min_terminated_length": 5544.0, "entropy": 0.2782252887263894, "epoch": 0.004801920768307323, "frac_reward_zero_std": 0.25, "grad_norm": 0.33767879009246826, "learning_rate": 9.88e-07, "loss": -0.0169, "num_tokens": 1430327.0, "reward": 1775.4732666015625, "reward_std": 4829.59228515625, "rewards/reward_func/mean": 1775.4732666015625, "rewards/reward_func/std": 4829.5927734375, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.093236207962036, "sampling/importance_sampling_ratio/mean": 0.9902609586715698, "sampling/importance_sampling_ratio/min": 0.18372339010238647, "sampling/sampling_logp_difference/max": 1.694324016571045, "sampling/sampling_logp_difference/mean": 0.019832005724310875, "step": 4, "step_time": 419.7083154738648 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16885.0, "completions/max_terminated_length": 16885.0, "completions/mean_length": 12227.375, "completions/mean_terminated_length": 12227.375, "completions/min_length": 9054.0, "completions/min_terminated_length": 9054.0, "entropy": 0.2566723832860589, "epoch": 0.006002400960384154, "frac_reward_zero_std": 0.0, "grad_norm": 0.3329413831233978, "learning_rate": 9.84e-07, "loss": 0.021, "num_tokens": 1853891.0, "reward": 1.8860834836959839, "reward_std": 2.3241159915924072, "rewards/reward_func/mean": 1.8933619260787964, "rewards/reward_func/std": 2.3200485706329346, "rewards/soft_overlong_punishment_reward/mean": -0.0072784423828125, "rewards/soft_overlong_punishment_reward/std": 0.028263933956623077, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9909947514533997, "sampling/importance_sampling_ratio/min": 0.17708024382591248, "sampling/sampling_logp_difference/max": 1.7311522960662842, "sampling/sampling_logp_difference/mean": 0.018662026152014732, "step": 5, "step_time": 244.05006601125933 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16596.0, "completions/max_terminated_length": 16596.0, "completions/mean_length": 10645.0625, "completions/mean_terminated_length": 10645.0625, "completions/min_length": 7554.0, "completions/min_terminated_length": 7554.0, "entropy": 0.27365364972501993, "epoch": 0.007202881152460984, "frac_reward_zero_std": 0.0, "grad_norm": 0.40234512090682983, "learning_rate": 9.8e-07, "loss": 0.0528, "num_tokens": 2223725.0, "reward": 1.9188847541809082, "reward_std": 2.996480703353882, "rewards/reward_func/mean": 1.9205021858215332, "rewards/reward_func/std": 2.995396375656128, "rewards/soft_overlong_punishment_reward/mean": -0.001617431640625, "rewards/soft_overlong_punishment_reward/std": 0.009149574674665928, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9904695749282837, "sampling/importance_sampling_ratio/min": 0.10882613807916641, "sampling/sampling_logp_difference/max": 2.218003749847412, "sampling/sampling_logp_difference/mean": 0.019550971686840057, "step": 6, "step_time": 222.7570838360116 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16016.0, "completions/max_terminated_length": 16016.0, "completions/mean_length": 10611.75, "completions/mean_terminated_length": 10611.75, "completions/min_length": 3615.0, "completions/min_terminated_length": 3615.0, "entropy": 0.2584681212902069, "epoch": 0.008403361344537815, "frac_reward_zero_std": 0.0, "grad_norm": 0.37323054671287537, "learning_rate": 9.759999999999998e-07, "loss": -0.034, "num_tokens": 2600909.0, "reward": 2.0009255409240723, "reward_std": 2.1709108352661133, "rewards/reward_func/mean": 2.0009255409240723, "rewards/reward_func/std": 2.1709108352661133, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9910842776298523, "sampling/importance_sampling_ratio/min": 0.12746387720108032, "sampling/sampling_logp_difference/max": 2.059922218322754, "sampling/sampling_logp_difference/mean": 0.018402449786663055, "step": 7, "step_time": 236.24530657555442 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15769.0, "completions/max_terminated_length": 15769.0, "completions/mean_length": 11196.375, "completions/mean_terminated_length": 11196.375, "completions/min_length": 7706.0, "completions/min_terminated_length": 7706.0, "entropy": 0.2538974490016699, "epoch": 0.009603841536614645, "frac_reward_zero_std": 0.5, "grad_norm": 0.23562167584896088, "learning_rate": 9.72e-07, "loss": 0.0366, "num_tokens": 2999913.0, "reward": 0.09837960451841354, "reward_std": 0.23358193039894104, "rewards/reward_func/mean": 0.09837960451841354, "rewards/reward_func/std": 0.23358193039894104, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9912012815475464, "sampling/importance_sampling_ratio/min": 0.16663774847984314, "sampling/sampling_logp_difference/max": 1.953153371810913, "sampling/sampling_logp_difference/mean": 0.018111014738678932, "step": 8, "step_time": 234.31651349924505 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15866.0, "completions/max_terminated_length": 15866.0, "completions/mean_length": 12010.46875, "completions/mean_terminated_length": 12010.46875, "completions/min_length": 8409.0, "completions/min_terminated_length": 8409.0, "entropy": 0.27975638769567013, "epoch": 0.010804321728691477, "frac_reward_zero_std": 0.0, "grad_norm": 0.376257061958313, "learning_rate": 9.679999999999999e-07, "loss": -0.0141, "num_tokens": 3418312.0, "reward": 3.2278504371643066, "reward_std": 11.484737396240234, "rewards/reward_func/mean": 3.2278504371643066, "rewards/reward_func/std": 11.484737396240234, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.738037109375, "sampling/importance_sampling_ratio/mean": 0.9900665879249573, "sampling/importance_sampling_ratio/min": 0.0053693195804953575, "sampling/sampling_logp_difference/max": 5.227054119110107, "sampling/sampling_logp_difference/mean": 0.02012288197875023, "step": 9, "step_time": 239.36658373998944 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 17639.0, "completions/max_terminated_length": 17639.0, "completions/mean_length": 11850.59375, "completions/mean_terminated_length": 11850.59375, "completions/min_length": 8366.0, "completions/min_terminated_length": 8366.0, "entropy": 0.27660834789276123, "epoch": 0.012004801920768308, "frac_reward_zero_std": 0.0, "grad_norm": 0.3692915141582489, "learning_rate": 9.64e-07, "loss": 0.0535, "num_tokens": 3824779.0, "reward": 0.4762687683105469, "reward_std": 0.7091255784034729, "rewards/reward_func/mean": 0.4858436584472656, "rewards/reward_func/std": 0.7002295255661011, "rewards/soft_overlong_punishment_reward/mean": -0.00957489013671875, "rewards/soft_overlong_punishment_reward/std": 0.05416375771164894, "sampling/importance_sampling_ratio/max": 2.2065083980560303, "sampling/importance_sampling_ratio/mean": 0.9904176592826843, "sampling/importance_sampling_ratio/min": 0.13561852276325226, "sampling/sampling_logp_difference/max": 1.9979093074798584, "sampling/sampling_logp_difference/mean": 0.019777128472924232, "step": 10, "step_time": 242.8805887659546 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 17650.0, "completions/max_terminated_length": 17650.0, "completions/mean_length": 11499.65625, "completions/mean_terminated_length": 11499.65625, "completions/min_length": 7873.0, "completions/min_terminated_length": 7873.0, "entropy": 0.24676325172185898, "epoch": 0.013205282112845138, "frac_reward_zero_std": 0.0, "grad_norm": 0.41307520866394043, "learning_rate": 9.6e-07, "loss": 0.0019, "num_tokens": 4228592.0, "reward": 1.2467007637023926, "reward_std": 2.456725597381592, "rewards/reward_func/mean": 1.256359577178955, "rewards/reward_func/std": 2.451215982437134, "rewards/soft_overlong_punishment_reward/mean": -0.0096588134765625, "rewards/soft_overlong_punishment_reward/std": 0.05463850125670433, "sampling/importance_sampling_ratio/max": 2.828012466430664, "sampling/importance_sampling_ratio/mean": 0.9914248585700989, "sampling/importance_sampling_ratio/min": 0.36373454332351685, "sampling/sampling_logp_difference/max": 1.039574146270752, "sampling/sampling_logp_difference/mean": 0.01774602197110653, "step": 11, "step_time": 265.6139151197858 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14906.0, "completions/max_terminated_length": 14906.0, "completions/mean_length": 11677.15625, "completions/mean_terminated_length": 11677.15625, "completions/min_length": 7384.0, "completions/min_terminated_length": 7384.0, "entropy": 0.23969209752976894, "epoch": 0.014405762304921969, "frac_reward_zero_std": 0.0, "grad_norm": 0.3561864495277405, "learning_rate": 9.559999999999998e-07, "loss": 0.0084, "num_tokens": 4638621.0, "reward": 0.6092424392700195, "reward_std": 1.582262396812439, "rewards/reward_func/mean": 0.6092424392700195, "rewards/reward_func/std": 1.5822621583938599, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.7359044551849365, "sampling/importance_sampling_ratio/mean": 0.9916994571685791, "sampling/importance_sampling_ratio/min": 0.26353466510772705, "sampling/sampling_logp_difference/max": 1.3335703611373901, "sampling/sampling_logp_difference/mean": 0.017739253118634224, "step": 12, "step_time": 233.1278730876511 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 17077.0, "completions/max_terminated_length": 17077.0, "completions/mean_length": 10545.90625, "completions/mean_terminated_length": 10545.90625, "completions/min_length": 7467.0, "completions/min_terminated_length": 7467.0, "entropy": 0.2735821781679988, "epoch": 0.015606242496998799, "frac_reward_zero_std": 0.0, "grad_norm": 0.3848613202571869, "learning_rate": 9.52e-07, "loss": 0.0584, "num_tokens": 5005954.0, "reward": 2.2471165657043457, "reward_std": 3.020482301712036, "rewards/reward_func/mean": 2.253136157989502, "rewards/reward_func/std": 3.0157971382141113, "rewards/soft_overlong_punishment_reward/mean": -0.00601959228515625, "rewards/soft_overlong_punishment_reward/std": 0.030061684548854828, "sampling/importance_sampling_ratio/max": 2.850409746170044, "sampling/importance_sampling_ratio/mean": 0.9906243085861206, "sampling/importance_sampling_ratio/min": 6.962251973163802e-06, "sampling/sampling_logp_difference/max": 11.875007629394531, "sampling/sampling_logp_difference/mean": 0.019261419773101807, "step": 13, "step_time": 243.16750305576716 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14631.0, "completions/max_terminated_length": 14631.0, "completions/mean_length": 10139.75, "completions/mean_terminated_length": 10139.75, "completions/min_length": 5047.0, "completions/min_terminated_length": 5047.0, "entropy": 0.24060232006013393, "epoch": 0.01680672268907563, "frac_reward_zero_std": 0.0, "grad_norm": 0.383498877286911, "learning_rate": 9.479999999999999e-07, "loss": 0.0478, "num_tokens": 5374362.0, "reward": 0.869640588760376, "reward_std": 1.4982999563217163, "rewards/reward_func/mean": 0.869640588760376, "rewards/reward_func/std": 1.4982999563217163, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.358623743057251, "sampling/importance_sampling_ratio/mean": 0.9915570616722107, "sampling/importance_sampling_ratio/min": 0.07955317199230194, "sampling/sampling_logp_difference/max": 2.53132963180542, "sampling/sampling_logp_difference/mean": 0.017563849687576294, "step": 14, "step_time": 217.46682468871586 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 17053.0, "completions/max_terminated_length": 17053.0, "completions/mean_length": 13305.40625, "completions/mean_terminated_length": 13305.40625, "completions/min_length": 10198.0, "completions/min_terminated_length": 10198.0, "entropy": 0.2278358107432723, "epoch": 0.01800720288115246, "frac_reward_zero_std": 0.25, "grad_norm": 0.29915085434913635, "learning_rate": 9.439999999999999e-07, "loss": 0.0318, "num_tokens": 5840119.0, "reward": 0.04119223356246948, "reward_std": 0.2189895212650299, "rewards/reward_func/mean": 0.04761618375778198, "rewards/reward_func/std": 0.21552126109600067, "rewards/soft_overlong_punishment_reward/mean": -0.0064239501953125, "rewards/soft_overlong_punishment_reward/std": 0.029588615521788597, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9919904470443726, "sampling/importance_sampling_ratio/min": 0.06755267828702927, "sampling/sampling_logp_difference/max": 2.694847583770752, "sampling/sampling_logp_difference/mean": 0.01704443246126175, "step": 15, "step_time": 473.42477186385076 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16258.0, "completions/max_terminated_length": 16258.0, "completions/mean_length": 10728.21875, "completions/mean_terminated_length": 10728.21875, "completions/min_length": 6511.0, "completions/min_terminated_length": 6511.0, "entropy": 0.2541895778849721, "epoch": 0.01920768307322929, "frac_reward_zero_std": 0.25, "grad_norm": 0.357921302318573, "learning_rate": 9.399999999999999e-07, "loss": -0.0349, "num_tokens": 6216630.0, "reward": 3.3305792808532715, "reward_std": 7.61651611328125, "rewards/reward_func/mean": 3.3305792808532715, "rewards/reward_func/std": 7.61651611328125, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.522019624710083, "sampling/importance_sampling_ratio/mean": 0.9909427762031555, "sampling/importance_sampling_ratio/min": 0.14655600488185883, "sampling/sampling_logp_difference/max": 1.9203475713729858, "sampling/sampling_logp_difference/mean": 0.018465207889676094, "step": 16, "step_time": 229.7186487779254 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14954.0, "completions/max_terminated_length": 14954.0, "completions/mean_length": 10770.53125, "completions/mean_terminated_length": 10770.53125, "completions/min_length": 5937.0, "completions/min_terminated_length": 5937.0, "entropy": 0.2734638024121523, "epoch": 0.02040816326530612, "frac_reward_zero_std": 0.25, "grad_norm": 0.32195746898651123, "learning_rate": 9.36e-07, "loss": 0.0259, "num_tokens": 6590607.0, "reward": 0.6464040875434875, "reward_std": 0.9011892676353455, "rewards/reward_func/mean": 0.6464040875434875, "rewards/reward_func/std": 0.9011892676353455, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.1569271087646484, "sampling/importance_sampling_ratio/mean": 0.9905099868774414, "sampling/importance_sampling_ratio/min": 0.02047974243760109, "sampling/sampling_logp_difference/max": 3.8883190155029297, "sampling/sampling_logp_difference/mean": 0.019440319389104843, "step": 17, "step_time": 433.71242073585745 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16352.0, "completions/max_terminated_length": 16352.0, "completions/mean_length": 9984.46875, "completions/mean_terminated_length": 9984.46875, "completions/min_length": 4147.0, "completions/min_terminated_length": 4147.0, "entropy": 0.2726652082055807, "epoch": 0.021608643457382955, "frac_reward_zero_std": 0.0, "grad_norm": 0.3855082392692566, "learning_rate": 9.32e-07, "loss": 0.0234, "num_tokens": 6944518.0, "reward": 1.9245760440826416, "reward_std": 2.4882936477661133, "rewards/reward_func/mean": 1.9245760440826416, "rewards/reward_func/std": 2.4882936477661133, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0238037109375, "sampling/importance_sampling_ratio/mean": 0.9904158115386963, "sampling/importance_sampling_ratio/min": 0.19708161056041718, "sampling/sampling_logp_difference/max": 1.6241374015808105, "sampling/sampling_logp_difference/mean": 0.01931804046034813, "step": 18, "step_time": 261.609183562221 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 19890.0, "completions/max_terminated_length": 19890.0, "completions/mean_length": 12736.40625, "completions/mean_terminated_length": 12736.40625, "completions/min_length": 8136.0, "completions/min_terminated_length": 8136.0, "entropy": 0.22564240265637636, "epoch": 0.022809123649459785, "frac_reward_zero_std": 0.0, "grad_norm": 0.3456736207008362, "learning_rate": 9.28e-07, "loss": 0.0822, "num_tokens": 7391411.0, "reward": 55.75565719604492, "reward_std": 127.06681823730469, "rewards/reward_func/mean": 55.820682525634766, "rewards/reward_func/std": 127.06536102294922, "rewards/soft_overlong_punishment_reward/mean": -0.06502532958984375, "rewards/soft_overlong_punishment_reward/std": 0.21553081274032593, "sampling/importance_sampling_ratio/max": 2.4781954288482666, "sampling/importance_sampling_ratio/mean": 0.9922029376029968, "sampling/importance_sampling_ratio/min": 0.18941795825958252, "sampling/sampling_logp_difference/max": 1.6637992858886719, "sampling/sampling_logp_difference/mean": 0.016782237216830254, "step": 19, "step_time": 274.4253786377376 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 18974.0, "completions/max_terminated_length": 18974.0, "completions/mean_length": 11451.9375, "completions/mean_terminated_length": 11451.9375, "completions/min_length": 8223.0, "completions/min_terminated_length": 8223.0, "entropy": 0.24693564232438803, "epoch": 0.024009603841536616, "frac_reward_zero_std": 0.0, "grad_norm": 0.37484318017959595, "learning_rate": 9.24e-07, "loss": 0.0537, "num_tokens": 7801249.0, "reward": 1.6996723413467407, "reward_std": 2.6643435955047607, "rewards/reward_func/mean": 1.7194324731826782, "rewards/reward_func/std": 2.648789882659912, "rewards/soft_overlong_punishment_reward/mean": -0.0197601318359375, "rewards/soft_overlong_punishment_reward/std": 0.11178018897771835, "sampling/importance_sampling_ratio/max": 2.0502030849456787, "sampling/importance_sampling_ratio/mean": 0.991265594959259, "sampling/importance_sampling_ratio/min": 0.03500676155090332, "sampling/sampling_logp_difference/max": 3.3522140979766846, "sampling/sampling_logp_difference/mean": 0.017859291285276413, "step": 20, "step_time": 254.13871470477898 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14921.0, "completions/max_terminated_length": 14921.0, "completions/mean_length": 11261.625, "completions/mean_terminated_length": 11261.625, "completions/min_length": 8921.0, "completions/min_terminated_length": 8921.0, "entropy": 0.21324953343719244, "epoch": 0.025210084033613446, "frac_reward_zero_std": 0.0, "grad_norm": 0.37738341093063354, "learning_rate": 9.2e-07, "loss": 0.0006, "num_tokens": 8209717.0, "reward": 5.487376689910889, "reward_std": 5.091497421264648, "rewards/reward_func/mean": 5.487376689910889, "rewards/reward_func/std": 5.091497421264648, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.863360643386841, "sampling/importance_sampling_ratio/mean": 0.9924338459968567, "sampling/importance_sampling_ratio/min": 0.2230137288570404, "sampling/sampling_logp_difference/max": 1.5005220174789429, "sampling/sampling_logp_difference/mean": 0.01615520380437374, "step": 21, "step_time": 226.67027153016534 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15897.0, "completions/max_terminated_length": 15897.0, "completions/mean_length": 9916.03125, "completions/mean_terminated_length": 9916.03125, "completions/min_length": 6559.0, "completions/min_terminated_length": 6559.0, "entropy": 0.2823543632403016, "epoch": 0.026410564225690276, "frac_reward_zero_std": 0.0, "grad_norm": 0.41728851199150085, "learning_rate": 9.16e-07, "loss": 0.0357, "num_tokens": 8558022.0, "reward": 1.0093857049942017, "reward_std": 1.6929666996002197, "rewards/reward_func/mean": 1.0093857049942017, "rewards/reward_func/std": 1.6929665803909302, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9902706146240234, "sampling/importance_sampling_ratio/min": 0.29672637581825256, "sampling/sampling_logp_difference/max": 1.2743182182312012, "sampling/sampling_logp_difference/mean": 0.020027777180075645, "step": 22, "step_time": 210.78112597286236 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15085.0, "completions/max_terminated_length": 15085.0, "completions/mean_length": 9004.78125, "completions/mean_terminated_length": 9004.78125, "completions/min_length": 5019.0, "completions/min_terminated_length": 5019.0, "entropy": 0.26619420014321804, "epoch": 0.027611044417767107, "frac_reward_zero_std": 0.0, "grad_norm": 0.4362182021141052, "learning_rate": 9.12e-07, "loss": -0.0028, "num_tokens": 8867615.0, "reward": 3.192936420440674, "reward_std": 3.843594551086426, "rewards/reward_func/mean": 3.192936420440674, "rewards/reward_func/std": 3.8435938358306885, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.684079170227051, "sampling/importance_sampling_ratio/mean": 0.9908168315887451, "sampling/importance_sampling_ratio/min": 0.24921053647994995, "sampling/sampling_logp_difference/max": 1.3894572257995605, "sampling/sampling_logp_difference/mean": 0.019214916974306107, "step": 23, "step_time": 200.50691007694695 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15050.0, "completions/max_terminated_length": 15050.0, "completions/mean_length": 10668.5625, "completions/mean_terminated_length": 10668.5625, "completions/min_length": 6005.0, "completions/min_terminated_length": 6005.0, "entropy": 0.2756733503192663, "epoch": 0.028811524609843937, "frac_reward_zero_std": 0.0, "grad_norm": 0.37513336539268494, "learning_rate": 9.08e-07, "loss": 0.0388, "num_tokens": 9237721.0, "reward": 0.4544358551502228, "reward_std": 0.5964015126228333, "rewards/reward_func/mean": 0.4544358551502228, "rewards/reward_func/std": 0.5964015126228333, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.185565233230591, "sampling/importance_sampling_ratio/mean": 0.9904376268386841, "sampling/importance_sampling_ratio/min": 0.3395426869392395, "sampling/sampling_logp_difference/max": 1.080155611038208, "sampling/sampling_logp_difference/mean": 0.019677145406603813, "step": 24, "step_time": 225.78321035497356 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14237.0, "completions/max_terminated_length": 14237.0, "completions/mean_length": 10287.375, "completions/mean_terminated_length": 10287.375, "completions/min_length": 6909.0, "completions/min_terminated_length": 6909.0, "entropy": 0.2614977052435279, "epoch": 0.030012004801920768, "frac_reward_zero_std": 0.25, "grad_norm": 0.3477057218551636, "learning_rate": 9.039999999999999e-07, "loss": 0.0355, "num_tokens": 9600461.0, "reward": 1.3245511054992676, "reward_std": 2.4857571125030518, "rewards/reward_func/mean": 1.3245511054992676, "rewards/reward_func/std": 2.4857571125030518, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.117020606994629, "sampling/importance_sampling_ratio/mean": 0.9908631443977356, "sampling/importance_sampling_ratio/min": 0.12998507916927338, "sampling/sampling_logp_difference/max": 2.0403356552124023, "sampling/sampling_logp_difference/mean": 0.018762774765491486, "step": 25, "step_time": 207.05053522088565 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14980.0, "completions/max_terminated_length": 14980.0, "completions/mean_length": 9842.53125, "completions/mean_terminated_length": 9842.53125, "completions/min_length": 6382.0, "completions/min_terminated_length": 6382.0, "entropy": 0.2648027054965496, "epoch": 0.031212484993997598, "frac_reward_zero_std": 0.0, "grad_norm": 0.3818010091781616, "learning_rate": 9e-07, "loss": 0.0235, "num_tokens": 9945774.0, "reward": 1.9594688415527344, "reward_std": 1.9573324918746948, "rewards/reward_func/mean": 1.9594688415527344, "rewards/reward_func/std": 1.9573323726654053, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.115938425064087, "sampling/importance_sampling_ratio/mean": 0.9907746315002441, "sampling/importance_sampling_ratio/min": 0.16936549544334412, "sampling/sampling_logp_difference/max": 1.7756961584091187, "sampling/sampling_logp_difference/mean": 0.018964044749736786, "step": 26, "step_time": 210.5417184109101 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14752.0, "completions/max_terminated_length": 14752.0, "completions/mean_length": 10530.3125, "completions/mean_terminated_length": 10530.3125, "completions/min_length": 6272.0, "completions/min_terminated_length": 6272.0, "entropy": 0.24934661202132702, "epoch": 0.03241296518607443, "frac_reward_zero_std": 0.0, "grad_norm": 0.40029430389404297, "learning_rate": 8.96e-07, "loss": -0.0036, "num_tokens": 10312320.0, "reward": 3.0405454635620117, "reward_std": 3.8312883377075195, "rewards/reward_func/mean": 3.0405454635620117, "rewards/reward_func/std": 3.8312880992889404, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9914329648017883, "sampling/importance_sampling_ratio/min": 0.09976457804441452, "sampling/sampling_logp_difference/max": 2.3049421310424805, "sampling/sampling_logp_difference/mean": 0.017998939380049706, "step": 27, "step_time": 431.60967801115476 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 17459.0, "completions/max_terminated_length": 17459.0, "completions/mean_length": 11231.8125, "completions/mean_terminated_length": 11231.8125, "completions/min_length": 5633.0, "completions/min_terminated_length": 5633.0, "entropy": 0.25841675139963627, "epoch": 0.03361344537815126, "frac_reward_zero_std": 0.0, "grad_norm": 0.3741910755634308, "learning_rate": 8.92e-07, "loss": 0.0071, "num_tokens": 10709450.0, "reward": 1.4909887313842773, "reward_std": 2.263838291168213, "rewards/reward_func/mean": 1.499190330505371, "rewards/reward_func/std": 2.2577483654022217, "rewards/soft_overlong_punishment_reward/mean": -0.00820159912109375, "rewards/soft_overlong_punishment_reward/std": 0.04639524966478348, "sampling/importance_sampling_ratio/max": 2.4496078491210938, "sampling/importance_sampling_ratio/mean": 0.99090176820755, "sampling/importance_sampling_ratio/min": 0.13220876455307007, "sampling/sampling_logp_difference/max": 2.0233731269836426, "sampling/sampling_logp_difference/mean": 0.01868780329823494, "step": 28, "step_time": 426.57934639882296 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16356.0, "completions/max_terminated_length": 16356.0, "completions/mean_length": 9573.6875, "completions/mean_terminated_length": 9573.6875, "completions/min_length": 2818.0, "completions/min_terminated_length": 2818.0, "entropy": 0.2775294054299593, "epoch": 0.03481392557022809, "frac_reward_zero_std": 0.25, "grad_norm": 0.3293408453464508, "learning_rate": 8.88e-07, "loss": 0.0192, "num_tokens": 11044352.0, "reward": 1.3649942874908447, "reward_std": 1.9040188789367676, "rewards/reward_func/mean": 1.3649942874908447, "rewards/reward_func/std": 1.9040188789367676, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.990324854850769, "sampling/importance_sampling_ratio/min": 0.37744393944740295, "sampling/sampling_logp_difference/max": 1.3116214275360107, "sampling/sampling_logp_difference/mean": 0.019651668146252632, "step": 29, "step_time": 238.1490348140942 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14524.0, "completions/max_terminated_length": 14524.0, "completions/mean_length": 11151.8125, "completions/mean_terminated_length": 11151.8125, "completions/min_length": 7531.0, "completions/min_terminated_length": 7531.0, "entropy": 0.24554751068353653, "epoch": 0.03601440576230492, "frac_reward_zero_std": 0.0, "grad_norm": 0.36097458004951477, "learning_rate": 8.839999999999999e-07, "loss": -0.0026, "num_tokens": 11442354.0, "reward": 2.1457722187042236, "reward_std": 2.174523115158081, "rewards/reward_func/mean": 2.1457722187042236, "rewards/reward_func/std": 2.17452335357666, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9914038777351379, "sampling/importance_sampling_ratio/min": 0.1909109503030777, "sampling/sampling_logp_difference/max": 1.6559481620788574, "sampling/sampling_logp_difference/mean": 0.0179743692278862, "step": 30, "step_time": 251.35512228007428 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13663.0, "completions/max_terminated_length": 13663.0, "completions/mean_length": 8797.78125, "completions/mean_terminated_length": 8797.78125, "completions/min_length": 3457.0, "completions/min_terminated_length": 3457.0, "entropy": 0.26490489952266216, "epoch": 0.03721488595438175, "frac_reward_zero_std": 0.0, "grad_norm": 0.45504486560821533, "learning_rate": 8.799999999999999e-07, "loss": 0.0903, "num_tokens": 11752779.0, "reward": 2.032346725463867, "reward_std": 2.6423678398132324, "rewards/reward_func/mean": 2.032346725463867, "rewards/reward_func/std": 2.6423676013946533, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9907644987106323, "sampling/importance_sampling_ratio/min": 0.12091671675443649, "sampling/sampling_logp_difference/max": 2.1126532554626465, "sampling/sampling_logp_difference/mean": 0.01906166598200798, "step": 31, "step_time": 210.08964343706612 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 20480.0, "completions/max_terminated_length": 17167.0, "completions/mean_length": 10623.3125, "completions/mean_terminated_length": 10305.3544921875, "completions/min_length": 4879.0, "completions/min_terminated_length": 4879.0, "entropy": 0.2628009468317032, "epoch": 0.03841536614645858, "frac_reward_zero_std": 0.25, "grad_norm": 0.2900553047657013, "learning_rate": 8.76e-07, "loss": -0.0423, "num_tokens": 12126181.0, "reward": 1.0021374225616455, "reward_std": 1.9135971069335938, "rewards/reward_func/mean": 1.0393612384796143, "rewards/reward_func/std": 1.8841371536254883, "rewards/soft_overlong_punishment_reward/mean": -0.03722381591796875, "rewards/soft_overlong_punishment_reward/std": 0.17890377342700958, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9907552599906921, "sampling/importance_sampling_ratio/min": 0.2805570363998413, "sampling/sampling_logp_difference/max": 1.2783308029174805, "sampling/sampling_logp_difference/mean": 0.018829116597771645, "step": 32, "step_time": 280.71025710599497 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14347.0, "completions/max_terminated_length": 14347.0, "completions/mean_length": 10457.4375, "completions/mean_terminated_length": 10457.4375, "completions/min_length": 7484.0, "completions/min_terminated_length": 7484.0, "entropy": 0.28439216688275337, "epoch": 0.03961584633853541, "frac_reward_zero_std": 0.25, "grad_norm": 0.3339356482028961, "learning_rate": 8.72e-07, "loss": 0.031, "num_tokens": 12492219.0, "reward": 0.7305504083633423, "reward_std": 0.7552641034126282, "rewards/reward_func/mean": 0.7305504083633423, "rewards/reward_func/std": 0.7552641034126282, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9899792671203613, "sampling/importance_sampling_ratio/min": 0.15883956849575043, "sampling/sampling_logp_difference/max": 1.8398605585098267, "sampling/sampling_logp_difference/mean": 0.020156892016530037, "step": 33, "step_time": 222.89753303222824 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14362.0, "completions/max_terminated_length": 14362.0, "completions/mean_length": 10071.5, "completions/mean_terminated_length": 10071.5, "completions/min_length": 6369.0, "completions/min_terminated_length": 6369.0, "entropy": 0.24872859101742506, "epoch": 0.04081632653061224, "frac_reward_zero_std": 0.0, "grad_norm": 0.4110364019870758, "learning_rate": 8.68e-07, "loss": -0.0135, "num_tokens": 12855851.0, "reward": 2.7421875, "reward_std": 3.5190269947052, "rewards/reward_func/mean": 2.7421875, "rewards/reward_func/std": 3.5190269947052, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9915586113929749, "sampling/importance_sampling_ratio/min": 0.27868548035621643, "sampling/sampling_logp_difference/max": 1.2776713371276855, "sampling/sampling_logp_difference/mean": 0.01771131530404091, "step": 34, "step_time": 224.99640086409636 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13259.0, "completions/max_terminated_length": 13259.0, "completions/mean_length": 8307.40625, "completions/mean_terminated_length": 8307.40625, "completions/min_length": 3245.0, "completions/min_terminated_length": 3245.0, "entropy": 0.2815095093101263, "epoch": 0.04201680672268908, "frac_reward_zero_std": 0.0, "grad_norm": 0.4219931364059448, "learning_rate": 8.639999999999999e-07, "loss": 0.02, "num_tokens": 13148672.0, "reward": 1.6012442111968994, "reward_std": 2.0349299907684326, "rewards/reward_func/mean": 1.6012442111968994, "rewards/reward_func/std": 2.0349299907684326, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.3349249362945557, "sampling/importance_sampling_ratio/mean": 0.990214467048645, "sampling/importance_sampling_ratio/min": 0.10321995615959167, "sampling/sampling_logp_difference/max": 2.270893096923828, "sampling/sampling_logp_difference/mean": 0.019673598930239677, "step": 35, "step_time": 197.53019676276017 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13594.0, "completions/max_terminated_length": 13594.0, "completions/mean_length": 10127.0625, "completions/mean_terminated_length": 10127.0625, "completions/min_length": 6623.0, "completions/min_terminated_length": 6623.0, "entropy": 0.2668821206316352, "epoch": 0.04321728691476591, "frac_reward_zero_std": 0.0, "grad_norm": 0.37350013852119446, "learning_rate": 8.599999999999999e-07, "loss": 0.0004, "num_tokens": 13502514.0, "reward": 2.3392128944396973, "reward_std": 3.019270896911621, "rewards/reward_func/mean": 2.3392128944396973, "rewards/reward_func/std": 3.019270896911621, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.516148328781128, "sampling/importance_sampling_ratio/mean": 0.9906859397888184, "sampling/importance_sampling_ratio/min": 0.2636016309261322, "sampling/sampling_logp_difference/max": 1.3333163261413574, "sampling/sampling_logp_difference/mean": 0.01926989108324051, "step": 36, "step_time": 219.1794932379853 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15015.0, "completions/max_terminated_length": 15015.0, "completions/mean_length": 10019.1875, "completions/mean_terminated_length": 10019.1875, "completions/min_length": 6810.0, "completions/min_terminated_length": 6810.0, "entropy": 0.24516229517757893, "epoch": 0.04441776710684274, "frac_reward_zero_std": 0.0, "grad_norm": 0.37556055188179016, "learning_rate": 8.559999999999999e-07, "loss": -0.0232, "num_tokens": 13860184.0, "reward": 5.546761512756348, "reward_std": 4.9547014236450195, "rewards/reward_func/mean": 5.546761512756348, "rewards/reward_func/std": 4.954700946807861, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9915067553520203, "sampling/importance_sampling_ratio/min": 0.3167104423046112, "sampling/sampling_logp_difference/max": 1.2001211643218994, "sampling/sampling_logp_difference/mean": 0.017740583047270775, "step": 37, "step_time": 429.79983439377975 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15132.0, "completions/max_terminated_length": 15132.0, "completions/mean_length": 10126.53125, "completions/mean_terminated_length": 10126.53125, "completions/min_length": 3623.0, "completions/min_terminated_length": 3623.0, "entropy": 0.2536043291911483, "epoch": 0.04561824729891957, "frac_reward_zero_std": 0.0, "grad_norm": 0.4053056538105011, "learning_rate": 8.52e-07, "loss": 0.0016, "num_tokens": 14219801.0, "reward": 1.1494144201278687, "reward_std": 1.515267252922058, "rewards/reward_func/mean": 1.1494144201278687, "rewards/reward_func/std": 1.515267252922058, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.504192590713501, "sampling/importance_sampling_ratio/mean": 0.9913269281387329, "sampling/importance_sampling_ratio/min": 0.11853170394897461, "sampling/sampling_logp_difference/max": 2.1325747966766357, "sampling/sampling_logp_difference/mean": 0.018079929053783417, "step": 38, "step_time": 231.09112938609906 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13953.0, "completions/max_terminated_length": 13953.0, "completions/mean_length": 9400.75, "completions/mean_terminated_length": 9400.75, "completions/min_length": 3400.0, "completions/min_terminated_length": 3400.0, "entropy": 0.2656275276094675, "epoch": 0.0468187274909964, "frac_reward_zero_std": 0.25, "grad_norm": 0.35668453574180603, "learning_rate": 8.48e-07, "loss": 0.0444, "num_tokens": 14552409.0, "reward": 1.375227689743042, "reward_std": 2.0771355628967285, "rewards/reward_func/mean": 1.375227689743042, "rewards/reward_func/std": 2.0771355628967285, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.8366079330444336, "sampling/importance_sampling_ratio/mean": 0.9909170269966125, "sampling/importance_sampling_ratio/min": 0.021294279024004936, "sampling/sampling_logp_difference/max": 3.8493168354034424, "sampling/sampling_logp_difference/mean": 0.018923122435808182, "step": 39, "step_time": 205.99593690189067 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14127.0, "completions/max_terminated_length": 14127.0, "completions/mean_length": 9623.75, "completions/mean_terminated_length": 9623.75, "completions/min_length": 5045.0, "completions/min_terminated_length": 5045.0, "entropy": 0.22837793547660112, "epoch": 0.04801920768307323, "frac_reward_zero_std": 0.25, "grad_norm": 0.344384104013443, "learning_rate": 8.439999999999999e-07, "loss": 0.0052, "num_tokens": 14898929.0, "reward": 0.5646069049835205, "reward_std": 1.1903877258300781, "rewards/reward_func/mean": 0.5646069049835205, "rewards/reward_func/std": 1.1903877258300781, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.5692195892333984, "sampling/importance_sampling_ratio/mean": 0.9922388195991516, "sampling/importance_sampling_ratio/min": 0.14704354107379913, "sampling/sampling_logp_difference/max": 1.9170265197753906, "sampling/sampling_logp_difference/mean": 0.01625021919608116, "step": 40, "step_time": 228.8796401922591 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16700.0, "completions/max_terminated_length": 16700.0, "completions/mean_length": 11196.0, "completions/mean_terminated_length": 11196.0, "completions/min_length": 7400.0, "completions/min_terminated_length": 7400.0, "entropy": 0.23109178990125656, "epoch": 0.04921968787515006, "frac_reward_zero_std": 0.0, "grad_norm": 0.35686036944389343, "learning_rate": 8.399999999999999e-07, "loss": -0.025, "num_tokens": 15295537.0, "reward": 1.7151178121566772, "reward_std": 2.868417501449585, "rewards/reward_func/mean": 1.7175287008285522, "rewards/reward_func/std": 2.868037700653076, "rewards/soft_overlong_punishment_reward/mean": -0.002410888671875, "rewards/soft_overlong_punishment_reward/std": 0.013638045638799667, "sampling/importance_sampling_ratio/max": 2.9357402324676514, "sampling/importance_sampling_ratio/mean": 0.9918723106384277, "sampling/importance_sampling_ratio/min": 0.3307071328163147, "sampling/sampling_logp_difference/max": 1.1065220832824707, "sampling/sampling_logp_difference/mean": 0.016960658133029938, "step": 41, "step_time": 247.38727062719408 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11833.0, "completions/max_terminated_length": 11833.0, "completions/mean_length": 9130.28125, "completions/mean_terminated_length": 9130.28125, "completions/min_length": 6003.0, "completions/min_terminated_length": 6003.0, "entropy": 0.27010741643607616, "epoch": 0.05042016806722689, "frac_reward_zero_std": 0.25, "grad_norm": 0.36788010597229004, "learning_rate": 8.359999999999999e-07, "loss": 0.0177, "num_tokens": 15624346.0, "reward": 0.4658914804458618, "reward_std": 0.9191152453422546, "rewards/reward_func/mean": 0.4658914804458618, "rewards/reward_func/std": 0.9191153049468994, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9905910491943359, "sampling/importance_sampling_ratio/min": 0.02883448265492916, "sampling/sampling_logp_difference/max": 3.5461833477020264, "sampling/sampling_logp_difference/mean": 0.01935204118490219, "step": 42, "step_time": 186.53937864280306 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15715.0, "completions/max_terminated_length": 15715.0, "completions/mean_length": 11385.625, "completions/mean_terminated_length": 11385.625, "completions/min_length": 5736.0, "completions/min_terminated_length": 5736.0, "entropy": 0.2562467288225889, "epoch": 0.05162064825930372, "frac_reward_zero_std": 0.25, "grad_norm": 0.31052911281585693, "learning_rate": 8.319999999999999e-07, "loss": -0.0051, "num_tokens": 16022038.0, "reward": 1.6798204183578491, "reward_std": 2.83245587348938, "rewards/reward_func/mean": 1.6798204183578491, "rewards/reward_func/std": 2.83245587348938, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.5458321571350098, "sampling/importance_sampling_ratio/mean": 0.9909483790397644, "sampling/importance_sampling_ratio/min": 0.2335936576128006, "sampling/sampling_logp_difference/max": 1.454172134399414, "sampling/sampling_logp_difference/mean": 0.01847730576992035, "step": 43, "step_time": 446.1300635159714 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 17778.0, "completions/max_terminated_length": 17778.0, "completions/mean_length": 9443.03125, "completions/mean_terminated_length": 9443.03125, "completions/min_length": 4794.0, "completions/min_terminated_length": 4794.0, "entropy": 0.25649653095752, "epoch": 0.05282112845138055, "frac_reward_zero_std": 0.0, "grad_norm": 0.4464053809642792, "learning_rate": 8.28e-07, "loss": 0.0453, "num_tokens": 16358935.0, "reward": 1.2445961236953735, "reward_std": 2.735797643661499, "rewards/reward_func/mean": 1.255231499671936, "rewards/reward_func/std": 2.730092763900757, "rewards/soft_overlong_punishment_reward/mean": -0.0106353759765625, "rewards/soft_overlong_punishment_reward/std": 0.060162775218486786, "sampling/importance_sampling_ratio/max": 2.382478713989258, "sampling/importance_sampling_ratio/mean": 0.9911012649536133, "sampling/importance_sampling_ratio/min": 0.2831081748008728, "sampling/sampling_logp_difference/max": 1.2619261741638184, "sampling/sampling_logp_difference/mean": 0.018203921616077423, "step": 44, "step_time": 238.45520115306135 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 18164.0, "completions/max_terminated_length": 18164.0, "completions/mean_length": 11039.0625, "completions/mean_terminated_length": 11039.0625, "completions/min_length": 6961.0, "completions/min_terminated_length": 6961.0, "entropy": 0.26013822108507156, "epoch": 0.05402160864345738, "frac_reward_zero_std": 0.25, "grad_norm": 0.7753751873970032, "learning_rate": 8.24e-07, "loss": 0.0627, "num_tokens": 16746577.0, "reward": 0.7348491549491882, "reward_std": 1.1186299324035645, "rewards/reward_func/mean": 0.756905734539032, "rewards/reward_func/std": 1.0995012521743774, "rewards/soft_overlong_punishment_reward/mean": -0.02205657958984375, "rewards/soft_overlong_punishment_reward/std": 0.08923598378896713, "sampling/importance_sampling_ratio/max": 2.8484528064727783, "sampling/importance_sampling_ratio/mean": 0.9910010695457458, "sampling/importance_sampling_ratio/min": 0.23139524459838867, "sampling/sampling_logp_difference/max": 1.4636280536651611, "sampling/sampling_logp_difference/mean": 0.01876596175134182, "step": 45, "step_time": 266.24963577114977 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16171.0, "completions/max_terminated_length": 16171.0, "completions/mean_length": 10262.96875, "completions/mean_terminated_length": 10262.96875, "completions/min_length": 5913.0, "completions/min_terminated_length": 5913.0, "entropy": 0.25811031460762024, "epoch": 0.055222088835534214, "frac_reward_zero_std": 0.0, "grad_norm": 0.378570556640625, "learning_rate": 8.199999999999999e-07, "loss": 0.0386, "num_tokens": 17105832.0, "reward": 2.625810146331787, "reward_std": 3.4443087577819824, "rewards/reward_func/mean": 2.625810146331787, "rewards/reward_func/std": 3.4443087577819824, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0112507343292236, "sampling/importance_sampling_ratio/mean": 0.9910373091697693, "sampling/importance_sampling_ratio/min": 0.15773901343345642, "sampling/sampling_logp_difference/max": 1.846813440322876, "sampling/sampling_logp_difference/mean": 0.01831601746380329, "step": 46, "step_time": 237.72815251280554 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15380.0, "completions/max_terminated_length": 15380.0, "completions/mean_length": 8343.25, "completions/mean_terminated_length": 8343.25, "completions/min_length": 4085.0, "completions/min_terminated_length": 4085.0, "entropy": 0.264944227412343, "epoch": 0.056422569027611044, "frac_reward_zero_std": 0.0, "grad_norm": 0.4716252088546753, "learning_rate": 8.159999999999999e-07, "loss": 0.0628, "num_tokens": 17399992.0, "reward": 2.4003467559814453, "reward_std": 3.171684741973877, "rewards/reward_func/mean": 2.4003467559814453, "rewards/reward_func/std": 3.171684503555298, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.402794599533081, "sampling/importance_sampling_ratio/mean": 0.9908208250999451, "sampling/importance_sampling_ratio/min": 0.3649902939796448, "sampling/sampling_logp_difference/max": 1.0078845024108887, "sampling/sampling_logp_difference/mean": 0.018845025449991226, "step": 47, "step_time": 209.52517059491947 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14507.0, "completions/max_terminated_length": 14507.0, "completions/mean_length": 10999.65625, "completions/mean_terminated_length": 10999.65625, "completions/min_length": 7392.0, "completions/min_terminated_length": 7392.0, "entropy": 0.2484130971133709, "epoch": 0.057623049219687875, "frac_reward_zero_std": 0.0, "grad_norm": 0.3668311834335327, "learning_rate": 8.12e-07, "loss": -0.0103, "num_tokens": 17789653.0, "reward": 0.8897507786750793, "reward_std": 0.6408941149711609, "rewards/reward_func/mean": 0.8897507786750793, "rewards/reward_func/std": 0.6408940553665161, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.341449022293091, "sampling/importance_sampling_ratio/mean": 0.9913567304611206, "sampling/importance_sampling_ratio/min": 0.022933995351195335, "sampling/sampling_logp_difference/max": 3.775135040283203, "sampling/sampling_logp_difference/mean": 0.017700741067528725, "step": 48, "step_time": 227.09850537101738 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16482.0, "completions/max_terminated_length": 16482.0, "completions/mean_length": 11000.375, "completions/mean_terminated_length": 11000.375, "completions/min_length": 6194.0, "completions/min_terminated_length": 6194.0, "entropy": 0.2574823070317507, "epoch": 0.058823529411764705, "frac_reward_zero_std": 0.0, "grad_norm": 0.37714579701423645, "learning_rate": 8.08e-07, "loss": 0.039, "num_tokens": 18186617.0, "reward": 1.1769545078277588, "reward_std": 0.6496437788009644, "rewards/reward_func/mean": 1.1777021884918213, "rewards/reward_func/std": 0.6509500741958618, "rewards/soft_overlong_punishment_reward/mean": -0.0007476806640625, "rewards/soft_overlong_punishment_reward/std": 0.004229520447552204, "sampling/importance_sampling_ratio/max": 2.0076770782470703, "sampling/importance_sampling_ratio/mean": 0.9911171197891235, "sampling/importance_sampling_ratio/min": 0.015267138369381428, "sampling/sampling_logp_difference/max": 4.1820526123046875, "sampling/sampling_logp_difference/mean": 0.017983591184020042, "step": 49, "step_time": 450.96922948525753 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12987.0, "completions/max_terminated_length": 12987.0, "completions/mean_length": 9108.96875, "completions/mean_terminated_length": 9108.96875, "completions/min_length": 5579.0, "completions/min_terminated_length": 5579.0, "entropy": 0.29258614405989647, "epoch": 0.060024009603841535, "frac_reward_zero_std": 0.0, "grad_norm": 0.4107678234577179, "learning_rate": 8.04e-07, "loss": -0.0154, "num_tokens": 18501752.0, "reward": 1.3277008533477783, "reward_std": 2.548473596572876, "rewards/reward_func/mean": 1.3277008533477783, "rewards/reward_func/std": 2.548473358154297, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.7192769050598145, "sampling/importance_sampling_ratio/mean": 0.9897809624671936, "sampling/importance_sampling_ratio/min": 0.033950600773096085, "sampling/sampling_logp_difference/max": 3.3828487396240234, "sampling/sampling_logp_difference/mean": 0.020536189898848534, "step": 50, "step_time": 202.79201423272025 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15226.0, "completions/max_terminated_length": 15226.0, "completions/mean_length": 8895.78125, "completions/mean_terminated_length": 8895.78125, "completions/min_length": 3790.0, "completions/min_terminated_length": 3790.0, "entropy": 0.24668313097208738, "epoch": 0.061224489795918366, "frac_reward_zero_std": 0.0, "grad_norm": 0.4362691640853882, "learning_rate": 8e-07, "loss": -0.015, "num_tokens": 18819793.0, "reward": 4.881908893585205, "reward_std": 4.377172946929932, "rewards/reward_func/mean": 4.881908893585205, "rewards/reward_func/std": 4.377172470092773, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.1437125205993652, "sampling/importance_sampling_ratio/mean": 0.991713285446167, "sampling/importance_sampling_ratio/min": 0.0913717970252037, "sampling/sampling_logp_difference/max": 2.3928184509277344, "sampling/sampling_logp_difference/mean": 0.0172872394323349, "step": 51, "step_time": 230.47769821400288 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10907.0, "completions/max_terminated_length": 10907.0, "completions/mean_length": 8155.625, "completions/mean_terminated_length": 8155.625, "completions/min_length": 4708.0, "completions/min_terminated_length": 4708.0, "entropy": 0.2947898115962744, "epoch": 0.062424969987995196, "frac_reward_zero_std": 0.25, "grad_norm": 0.3665950894355774, "learning_rate": 7.96e-07, "loss": 0.0084, "num_tokens": 19104357.0, "reward": 2.1094980239868164, "reward_std": 2.361452102661133, "rewards/reward_func/mean": 2.1094980239868164, "rewards/reward_func/std": 2.361452102661133, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.565842866897583, "sampling/importance_sampling_ratio/mean": 0.9897834062576294, "sampling/importance_sampling_ratio/min": 0.222982257604599, "sampling/sampling_logp_difference/max": 1.500663161277771, "sampling/sampling_logp_difference/mean": 0.02075599506497383, "step": 52, "step_time": 177.9754405770218 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 19155.0, "completions/max_terminated_length": 19155.0, "completions/mean_length": 10726.125, "completions/mean_terminated_length": 10726.125, "completions/min_length": 5744.0, "completions/min_terminated_length": 5744.0, "entropy": 0.2794426204636693, "epoch": 0.06362545018007203, "frac_reward_zero_std": 0.25, "grad_norm": 0.34014397859573364, "learning_rate": 7.92e-07, "loss": 0.051, "num_tokens": 19475873.0, "reward": 0.08617197722196579, "reward_std": 0.35582923889160156, "rewards/reward_func/mean": 0.10731302201747894, "rewards/reward_func/std": 0.3313770294189453, "rewards/soft_overlong_punishment_reward/mean": -0.02114105224609375, "rewards/soft_overlong_punishment_reward/std": 0.1195918545126915, "sampling/importance_sampling_ratio/max": 2.9801175594329834, "sampling/importance_sampling_ratio/mean": 0.990292489528656, "sampling/importance_sampling_ratio/min": 0.1588689088821411, "sampling/sampling_logp_difference/max": 1.8396759033203125, "sampling/sampling_logp_difference/mean": 0.019603673368692398, "step": 53, "step_time": 264.9613647001097 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14962.0, "completions/max_terminated_length": 14962.0, "completions/mean_length": 10665.8125, "completions/mean_terminated_length": 10665.8125, "completions/min_length": 5778.0, "completions/min_terminated_length": 5778.0, "entropy": 0.2630910240113735, "epoch": 0.06482593037214886, "frac_reward_zero_std": 0.0, "grad_norm": 0.36794134974479675, "learning_rate": 7.88e-07, "loss": 0.0013, "num_tokens": 19851875.0, "reward": 1.1953718662261963, "reward_std": 2.0580642223358154, "rewards/reward_func/mean": 1.1953718662261963, "rewards/reward_func/std": 2.0580642223358154, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.635934829711914, "sampling/importance_sampling_ratio/mean": 0.9908843040466309, "sampling/importance_sampling_ratio/min": 0.25040531158447266, "sampling/sampling_logp_difference/max": 1.3846744298934937, "sampling/sampling_logp_difference/mean": 0.01858539693057537, "step": 54, "step_time": 230.69495908927638 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13381.0, "completions/max_terminated_length": 13381.0, "completions/mean_length": 8781.25, "completions/mean_terminated_length": 8781.25, "completions/min_length": 3175.0, "completions/min_terminated_length": 3175.0, "entropy": 0.2748223738744855, "epoch": 0.06602641056422569, "frac_reward_zero_std": 0.0, "grad_norm": 0.42096376419067383, "learning_rate": 7.84e-07, "loss": -0.0035, "num_tokens": 20161883.0, "reward": 0.9055551290512085, "reward_std": 0.9041584134101868, "rewards/reward_func/mean": 0.9055551290512085, "rewards/reward_func/std": 0.9041584730148315, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.9826617240905762, "sampling/importance_sampling_ratio/mean": 0.9905050992965698, "sampling/importance_sampling_ratio/min": 0.18111246824264526, "sampling/sampling_logp_difference/max": 1.708636999130249, "sampling/sampling_logp_difference/mean": 0.01938163861632347, "step": 55, "step_time": 206.5735895021353 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12658.0, "completions/max_terminated_length": 12658.0, "completions/mean_length": 8712.09375, "completions/mean_terminated_length": 8712.09375, "completions/min_length": 5680.0, "completions/min_terminated_length": 5680.0, "entropy": 0.2725402005016804, "epoch": 0.06722689075630252, "frac_reward_zero_std": 0.0, "grad_norm": 0.4232986569404602, "learning_rate": 7.799999999999999e-07, "loss": 0.0346, "num_tokens": 20467166.0, "reward": 1.345833659172058, "reward_std": 2.5527396202087402, "rewards/reward_func/mean": 1.345833659172058, "rewards/reward_func/std": 2.552739381790161, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.3792266845703125, "sampling/importance_sampling_ratio/mean": 0.9905150532722473, "sampling/importance_sampling_ratio/min": 0.2472153902053833, "sampling/sampling_logp_difference/max": 1.3974952697753906, "sampling/sampling_logp_difference/mean": 0.01930988021194935, "step": 56, "step_time": 218.33462092967238 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12566.0, "completions/max_terminated_length": 12566.0, "completions/mean_length": 9388.78125, "completions/mean_terminated_length": 9388.78125, "completions/min_length": 6542.0, "completions/min_terminated_length": 6542.0, "entropy": 0.2632417492568493, "epoch": 0.06842737094837935, "frac_reward_zero_std": 0.25, "grad_norm": 0.3933756649494171, "learning_rate": 7.76e-07, "loss": -0.0044, "num_tokens": 20799727.0, "reward": 0.655514121055603, "reward_std": 1.1067219972610474, "rewards/reward_func/mean": 0.655514121055603, "rewards/reward_func/std": 1.1067217588424683, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.9161028861999512, "sampling/importance_sampling_ratio/mean": 0.9906187653541565, "sampling/importance_sampling_ratio/min": 0.03224591165781021, "sampling/sampling_logp_difference/max": 3.434364080429077, "sampling/sampling_logp_difference/mean": 0.018842609599232674, "step": 57, "step_time": 200.1914525568718 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13388.0, "completions/max_terminated_length": 13388.0, "completions/mean_length": 9069.125, "completions/mean_terminated_length": 9069.125, "completions/min_length": 3500.0, "completions/min_terminated_length": 3500.0, "entropy": 0.26544647198170424, "epoch": 0.06962785114045618, "frac_reward_zero_std": 0.5, "grad_norm": 0.34516435861587524, "learning_rate": 7.72e-07, "loss": 0.028, "num_tokens": 21118195.0, "reward": 1.8895032405853271, "reward_std": 2.862610101699829, "rewards/reward_func/mean": 1.8895032405853271, "rewards/reward_func/std": 2.862610101699829, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9907073378562927, "sampling/importance_sampling_ratio/min": 0.20175053179264069, "sampling/sampling_logp_difference/max": 1.6007232666015625, "sampling/sampling_logp_difference/mean": 0.0191123615950346, "step": 58, "step_time": 416.02178934798576 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16766.0, "completions/max_terminated_length": 16766.0, "completions/mean_length": 9156.90625, "completions/mean_terminated_length": 9156.90625, "completions/min_length": 6183.0, "completions/min_terminated_length": 6183.0, "entropy": 0.2845827713608742, "epoch": 0.07082833133253301, "frac_reward_zero_std": 0.0, "grad_norm": 0.4064350128173828, "learning_rate": 7.68e-07, "loss": -0.0212, "num_tokens": 21436904.0, "reward": 3.7760183811187744, "reward_std": 4.1162896156311035, "rewards/reward_func/mean": 3.778932809829712, "rewards/reward_func/std": 4.113493919372559, "rewards/soft_overlong_punishment_reward/mean": -0.0029144287109375, "rewards/soft_overlong_punishment_reward/std": 0.016486497595906258, "sampling/importance_sampling_ratio/max": 2.561572790145874, "sampling/importance_sampling_ratio/mean": 0.9900461435317993, "sampling/importance_sampling_ratio/min": 0.3691723942756653, "sampling/sampling_logp_difference/max": 0.996491551399231, "sampling/sampling_logp_difference/mean": 0.020054107531905174, "step": 59, "step_time": 232.3352104361402 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15562.0, "completions/max_terminated_length": 15562.0, "completions/mean_length": 8621.25, "completions/mean_terminated_length": 8621.25, "completions/min_length": 3889.0, "completions/min_terminated_length": 3889.0, "entropy": 0.2679576650261879, "epoch": 0.07202881152460984, "frac_reward_zero_std": 0.0, "grad_norm": 0.4170191287994385, "learning_rate": 7.64e-07, "loss": -0.014, "num_tokens": 21749208.0, "reward": 2.277270555496216, "reward_std": 3.568782329559326, "rewards/reward_func/mean": 2.277270555496216, "rewards/reward_func/std": 3.568782329559326, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.22825288772583, "sampling/importance_sampling_ratio/mean": 0.9907122254371643, "sampling/importance_sampling_ratio/min": 0.2719807028770447, "sampling/sampling_logp_difference/max": 1.302024245262146, "sampling/sampling_logp_difference/mean": 0.019024543464183807, "step": 60, "step_time": 241.59232664620504 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16602.0, "completions/max_terminated_length": 16602.0, "completions/mean_length": 11490.8125, "completions/mean_terminated_length": 11490.8125, "completions/min_length": 7465.0, "completions/min_terminated_length": 7465.0, "entropy": 0.24971914291381836, "epoch": 0.07322929171668667, "frac_reward_zero_std": 0.0, "grad_norm": 0.36966225504875183, "learning_rate": 7.599999999999999e-07, "loss": 0.0258, "num_tokens": 22150906.0, "reward": 2.009342670440674, "reward_std": 2.9576900005340576, "rewards/reward_func/mean": 2.0110058784484863, "rewards/reward_func/std": 2.956507444381714, "rewards/soft_overlong_punishment_reward/mean": -0.0016632080078125, "rewards/soft_overlong_punishment_reward/std": 0.009408525191247463, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9912000894546509, "sampling/importance_sampling_ratio/min": 0.0796288475394249, "sampling/sampling_logp_difference/max": 2.530378818511963, "sampling/sampling_logp_difference/mean": 0.018482467159628868, "step": 61, "step_time": 262.58794342912734 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16208.0, "completions/max_terminated_length": 16208.0, "completions/mean_length": 9912.53125, "completions/mean_terminated_length": 9912.53125, "completions/min_length": 3451.0, "completions/min_terminated_length": 3451.0, "entropy": 0.2602381184697151, "epoch": 0.0744297719087635, "frac_reward_zero_std": 0.0, "grad_norm": 0.45621025562286377, "learning_rate": 7.559999999999999e-07, "loss": -0.005, "num_tokens": 22506499.0, "reward": 0.5048784017562866, "reward_std": 0.6571192145347595, "rewards/reward_func/mean": 0.5048784017562866, "rewards/reward_func/std": 0.6571192145347595, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.990957498550415, "sampling/importance_sampling_ratio/min": 0.23703482747077942, "sampling/sampling_logp_difference/max": 1.4395482540130615, "sampling/sampling_logp_difference/mean": 0.01870187371969223, "step": 62, "step_time": 227.91837301501073 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13957.0, "completions/max_terminated_length": 13957.0, "completions/mean_length": 10636.0625, "completions/mean_terminated_length": 10636.0625, "completions/min_length": 8297.0, "completions/min_terminated_length": 8297.0, "entropy": 0.24767167586833239, "epoch": 0.07563025210084033, "frac_reward_zero_std": 0.0, "grad_norm": 0.3626573383808136, "learning_rate": 7.52e-07, "loss": 0.0097, "num_tokens": 22889037.0, "reward": 2.379978656768799, "reward_std": 2.5725669860839844, "rewards/reward_func/mean": 2.379978656768799, "rewards/reward_func/std": 2.5725667476654053, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.4255101680755615, "sampling/importance_sampling_ratio/mean": 0.991312563419342, "sampling/importance_sampling_ratio/min": 0.31686195731163025, "sampling/sampling_logp_difference/max": 1.1492891311645508, "sampling/sampling_logp_difference/mean": 0.017962327226996422, "step": 63, "step_time": 220.6857718299143 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14398.0, "completions/max_terminated_length": 14398.0, "completions/mean_length": 10088.125, "completions/mean_terminated_length": 10088.125, "completions/min_length": 6941.0, "completions/min_terminated_length": 6941.0, "entropy": 0.25979764945805073, "epoch": 0.07683073229291716, "frac_reward_zero_std": 0.0, "grad_norm": 0.3685627281665802, "learning_rate": 7.48e-07, "loss": 0.0522, "num_tokens": 23250569.0, "reward": 3681.080322265625, "reward_std": 20817.853515625, "rewards/reward_func/mean": 3681.080322265625, "rewards/reward_func/std": 20817.853515625, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9908851981163025, "sampling/importance_sampling_ratio/min": 0.15633250772953033, "sampling/sampling_logp_difference/max": 1.8557701110839844, "sampling/sampling_logp_difference/mean": 0.01882169209420681, "step": 64, "step_time": 207.90169405727647 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15345.0, "completions/max_terminated_length": 15345.0, "completions/mean_length": 10208.5625, "completions/mean_terminated_length": 10208.5625, "completions/min_length": 4160.0, "completions/min_terminated_length": 4160.0, "entropy": 0.25587698817253113, "epoch": 0.07803121248499399, "frac_reward_zero_std": 0.0, "grad_norm": 0.37043145298957825, "learning_rate": 7.44e-07, "loss": -0.0128, "num_tokens": 23616123.0, "reward": 115.96958923339844, "reward_std": 450.455322265625, "rewards/reward_func/mean": 115.96958923339844, "rewards/reward_func/std": 450.455322265625, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.3009722232818604, "sampling/importance_sampling_ratio/mean": 0.9911485314369202, "sampling/importance_sampling_ratio/min": 0.3561326265335083, "sampling/sampling_logp_difference/max": 1.03245210647583, "sampling/sampling_logp_difference/mean": 0.018131505697965622, "step": 65, "step_time": 247.2077819444239 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15775.0, "completions/max_terminated_length": 15775.0, "completions/mean_length": 10089.09375, "completions/mean_terminated_length": 10089.09375, "completions/min_length": 5130.0, "completions/min_terminated_length": 5130.0, "entropy": 0.25899219047278166, "epoch": 0.07923169267707082, "frac_reward_zero_std": 0.0, "grad_norm": 0.38848623633384705, "learning_rate": 7.4e-07, "loss": -0.0311, "num_tokens": 23972966.0, "reward": 2.4493942260742188, "reward_std": 3.161067247390747, "rewards/reward_func/mean": 2.4493942260742188, "rewards/reward_func/std": 3.161067247390747, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.568775177001953, "sampling/importance_sampling_ratio/mean": 0.9909378290176392, "sampling/importance_sampling_ratio/min": 0.3397059142589569, "sampling/sampling_logp_difference/max": 1.0796749591827393, "sampling/sampling_logp_difference/mean": 0.018584787845611572, "step": 66, "step_time": 235.83589306659997 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 17692.0, "completions/max_terminated_length": 17692.0, "completions/mean_length": 11849.34375, "completions/mean_terminated_length": 11849.34375, "completions/min_length": 6275.0, "completions/min_terminated_length": 6275.0, "entropy": 0.2503598555922508, "epoch": 0.08043217286914765, "frac_reward_zero_std": 0.0, "grad_norm": 0.36103734374046326, "learning_rate": 7.359999999999999e-07, "loss": 0.0653, "num_tokens": 24390121.0, "reward": 0.5631309747695923, "reward_std": 0.7355284690856934, "rewards/reward_func/mean": 0.5829674005508423, "rewards/reward_func/std": 0.7148671746253967, "rewards/soft_overlong_punishment_reward/mean": -0.01983642578125, "rewards/soft_overlong_punishment_reward/std": 0.07805701345205307, "sampling/importance_sampling_ratio/max": 2.5908403396606445, "sampling/importance_sampling_ratio/mean": 0.9912521243095398, "sampling/importance_sampling_ratio/min": 0.20392736792564392, "sampling/sampling_logp_difference/max": 1.5899913311004639, "sampling/sampling_logp_difference/mean": 0.017768753692507744, "step": 67, "step_time": 273.55189195496496 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13621.0, "completions/max_terminated_length": 13621.0, "completions/mean_length": 10604.21875, "completions/mean_terminated_length": 10604.21875, "completions/min_length": 7394.0, "completions/min_terminated_length": 7394.0, "entropy": 0.2723994627594948, "epoch": 0.08163265306122448, "frac_reward_zero_std": 0.0, "grad_norm": 0.40947461128234863, "learning_rate": 7.319999999999999e-07, "loss": 0.017, "num_tokens": 24763560.0, "reward": 15.798975944519043, "reward_std": 37.360469818115234, "rewards/reward_func/mean": 15.798975944519043, "rewards/reward_func/std": 37.360469818115234, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.846003293991089, "sampling/importance_sampling_ratio/mean": 0.9903964996337891, "sampling/importance_sampling_ratio/min": 0.025304390117526054, "sampling/sampling_logp_difference/max": 3.6767773628234863, "sampling/sampling_logp_difference/mean": 0.01946277916431427, "step": 68, "step_time": 208.66941105970182 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12376.0, "completions/max_terminated_length": 12376.0, "completions/mean_length": 9659.3125, "completions/mean_terminated_length": 9659.3125, "completions/min_length": 7514.0, "completions/min_terminated_length": 7514.0, "entropy": 0.2647173348814249, "epoch": 0.08283313325330131, "frac_reward_zero_std": 0.0, "grad_norm": 0.3932405710220337, "learning_rate": 7.28e-07, "loss": -0.0129, "num_tokens": 25105250.0, "reward": 1.1635289192199707, "reward_std": 1.419382929801941, "rewards/reward_func/mean": 1.1635289192199707, "rewards/reward_func/std": 1.4193830490112305, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.187065362930298, "sampling/importance_sampling_ratio/mean": 0.9907566905021667, "sampling/importance_sampling_ratio/min": 0.29801252484321594, "sampling/sampling_logp_difference/max": 1.2106198072433472, "sampling/sampling_logp_difference/mean": 0.018872233107686043, "step": 69, "step_time": 308.8531298188027 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15954.0, "completions/max_terminated_length": 15954.0, "completions/mean_length": 10261.5625, "completions/mean_terminated_length": 10261.5625, "completions/min_length": 6635.0, "completions/min_terminated_length": 6635.0, "entropy": 0.25401466619223356, "epoch": 0.08403361344537816, "frac_reward_zero_std": 0.25, "grad_norm": 0.32909703254699707, "learning_rate": 7.24e-07, "loss": 0.0189, "num_tokens": 25472484.0, "reward": 0.7801880836486816, "reward_std": 0.7046358585357666, "rewards/reward_func/mean": 0.7801880836486816, "rewards/reward_func/std": 0.7046358585357666, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9910746812820435, "sampling/importance_sampling_ratio/min": 0.15575462579727173, "sampling/sampling_logp_difference/max": 1.8594733476638794, "sampling/sampling_logp_difference/mean": 0.018227702006697655, "step": 70, "step_time": 458.7579943599412 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 17229.0, "completions/max_terminated_length": 17229.0, "completions/mean_length": 11900.3125, "completions/mean_terminated_length": 11900.3125, "completions/min_length": 8069.0, "completions/min_terminated_length": 8069.0, "entropy": 0.23596489429473877, "epoch": 0.08523409363745499, "frac_reward_zero_std": 0.25, "grad_norm": 0.28990110754966736, "learning_rate": 7.2e-07, "loss": 0.0406, "num_tokens": 25902358.0, "reward": 1.1131587028503418, "reward_std": 2.4644935131073, "rewards/reward_func/mean": 1.119605541229248, "rewards/reward_func/std": 2.461198329925537, "rewards/soft_overlong_punishment_reward/mean": -0.00644683837890625, "rewards/soft_overlong_punishment_reward/std": 0.03646882623434067, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9916808605194092, "sampling/importance_sampling_ratio/min": 0.07585223764181137, "sampling/sampling_logp_difference/max": 2.578968048095703, "sampling/sampling_logp_difference/mean": 0.017171792685985565, "step": 71, "step_time": 256.0422782639507 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15647.0, "completions/max_terminated_length": 15647.0, "completions/mean_length": 10677.8125, "completions/mean_terminated_length": 10677.8125, "completions/min_length": 6661.0, "completions/min_terminated_length": 6661.0, "entropy": 0.24026651121675968, "epoch": 0.08643457382953182, "frac_reward_zero_std": 0.0, "grad_norm": 0.36302992701530457, "learning_rate": 7.159999999999999e-07, "loss": 0.0069, "num_tokens": 26286872.0, "reward": 6.745024681091309, "reward_std": 27.16277313232422, "rewards/reward_func/mean": 6.745024681091309, "rewards/reward_func/std": 27.162776947021484, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.476977825164795, "sampling/importance_sampling_ratio/mean": 0.991825520992279, "sampling/importance_sampling_ratio/min": 0.006692860741168261, "sampling/sampling_logp_difference/max": 5.0067138671875, "sampling/sampling_logp_difference/mean": 0.01745101995766163, "step": 72, "step_time": 454.2569543699501 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14315.0, "completions/max_terminated_length": 14315.0, "completions/mean_length": 6956.90625, "completions/mean_terminated_length": 6956.90625, "completions/min_length": 3062.0, "completions/min_terminated_length": 3062.0, "entropy": 0.28412431478500366, "epoch": 0.08763505402160865, "frac_reward_zero_std": 0.0, "grad_norm": 0.5102128386497498, "learning_rate": 7.119999999999999e-07, "loss": 0.0493, "num_tokens": 26533853.0, "reward": 1.6690137386322021, "reward_std": 2.0055153369903564, "rewards/reward_func/mean": 1.6690137386322021, "rewards/reward_func/std": 2.0055153369903564, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9905214905738831, "sampling/importance_sampling_ratio/min": 0.027704769745469093, "sampling/sampling_logp_difference/max": 3.586150646209717, "sampling/sampling_logp_difference/mean": 0.019451972097158432, "step": 73, "step_time": 203.57053557073232 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11038.0, "completions/max_terminated_length": 11038.0, "completions/mean_length": 7485.53125, "completions/mean_terminated_length": 7485.53125, "completions/min_length": 3940.0, "completions/min_terminated_length": 3940.0, "entropy": 0.27887804340571165, "epoch": 0.08883553421368548, "frac_reward_zero_std": 0.25, "grad_norm": 0.3595903813838959, "learning_rate": 7.079999999999999e-07, "loss": 0.0016, "num_tokens": 26797934.0, "reward": 0.582500159740448, "reward_std": 0.6758586764335632, "rewards/reward_func/mean": 0.582500159740448, "rewards/reward_func/std": 0.6758586168289185, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.3009722232818604, "sampling/importance_sampling_ratio/mean": 0.9900974035263062, "sampling/importance_sampling_ratio/min": 0.3130251169204712, "sampling/sampling_logp_difference/max": 1.1614718437194824, "sampling/sampling_logp_difference/mean": 0.01946306601166725, "step": 74, "step_time": 394.7669193340698 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12980.0, "completions/max_terminated_length": 12980.0, "completions/mean_length": 9189.96875, "completions/mean_terminated_length": 9189.96875, "completions/min_length": 4670.0, "completions/min_terminated_length": 4670.0, "entropy": 0.27417559176683426, "epoch": 0.09003601440576231, "frac_reward_zero_std": 0.0, "grad_norm": 0.41591066122055054, "learning_rate": 7.04e-07, "loss": 0.0263, "num_tokens": 27116477.0, "reward": 1.5546950101852417, "reward_std": 2.706759452819824, "rewards/reward_func/mean": 1.5546950101852417, "rewards/reward_func/std": 2.706759214401245, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9904885292053223, "sampling/importance_sampling_ratio/min": 0.23027251660823822, "sampling/sampling_logp_difference/max": 1.468491792678833, "sampling/sampling_logp_difference/mean": 0.019385498017072678, "step": 75, "step_time": 212.57274886220694 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15570.0, "completions/max_terminated_length": 15570.0, "completions/mean_length": 9589.8125, "completions/mean_terminated_length": 9589.8125, "completions/min_length": 3431.0, "completions/min_terminated_length": 3431.0, "entropy": 0.24109713081270456, "epoch": 0.09123649459783914, "frac_reward_zero_std": 0.0, "grad_norm": 0.39267539978027344, "learning_rate": 7e-07, "loss": -0.0101, "num_tokens": 27469983.0, "reward": 1.6705238819122314, "reward_std": 1.578829288482666, "rewards/reward_func/mean": 1.6705238819122314, "rewards/reward_func/std": 1.578829288482666, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9916650056838989, "sampling/importance_sampling_ratio/min": 0.3055680990219116, "sampling/sampling_logp_difference/max": 1.5075340270996094, "sampling/sampling_logp_difference/mean": 0.01719709485769272, "step": 76, "step_time": 434.341842096881 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12644.0, "completions/max_terminated_length": 12644.0, "completions/mean_length": 8874.84375, "completions/mean_terminated_length": 8874.84375, "completions/min_length": 4780.0, "completions/min_terminated_length": 4780.0, "entropy": 0.28068294655531645, "epoch": 0.09243697478991597, "frac_reward_zero_std": 0.0, "grad_norm": 0.4375174045562744, "learning_rate": 6.959999999999999e-07, "loss": 0.0539, "num_tokens": 27781946.0, "reward": 19.96912956237793, "reward_std": 36.33182144165039, "rewards/reward_func/mean": 19.96912956237793, "rewards/reward_func/std": 36.33182144165039, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.198267698287964, "sampling/importance_sampling_ratio/mean": 0.990230143070221, "sampling/importance_sampling_ratio/min": 0.1569797843694687, "sampling/sampling_logp_difference/max": 1.8516383171081543, "sampling/sampling_logp_difference/mean": 0.01989983581006527, "step": 77, "step_time": 399.6355443559587 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12663.0, "completions/max_terminated_length": 12663.0, "completions/mean_length": 9384.59375, "completions/mean_terminated_length": 9384.59375, "completions/min_length": 5366.0, "completions/min_terminated_length": 5366.0, "entropy": 0.23270147573202848, "epoch": 0.0936374549819928, "frac_reward_zero_std": 0.0, "grad_norm": 0.41008490324020386, "learning_rate": 6.919999999999999e-07, "loss": 0.0169, "num_tokens": 28131637.0, "reward": 0.34000474214553833, "reward_std": 0.4872087836265564, "rewards/reward_func/mean": 0.34000474214553833, "rewards/reward_func/std": 0.4872088134288788, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9919241666793823, "sampling/importance_sampling_ratio/min": 0.012431665323674679, "sampling/sampling_logp_difference/max": 4.387508392333984, "sampling/sampling_logp_difference/mean": 0.016751548275351524, "step": 78, "step_time": 213.20301025058143 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13605.0, "completions/max_terminated_length": 13605.0, "completions/mean_length": 9480.09375, "completions/mean_terminated_length": 9480.09375, "completions/min_length": 6576.0, "completions/min_terminated_length": 6576.0, "entropy": 0.2560074757784605, "epoch": 0.09483793517406963, "frac_reward_zero_std": 0.0, "grad_norm": 0.4534052312374115, "learning_rate": 6.879999999999999e-07, "loss": 0.0027, "num_tokens": 28473544.0, "reward": 1.1451480388641357, "reward_std": 2.02068829536438, "rewards/reward_func/mean": 1.1451480388641357, "rewards/reward_func/std": 2.02068829536438, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9910561442375183, "sampling/importance_sampling_ratio/min": 0.28057095408439636, "sampling/sampling_logp_difference/max": 1.2709286212921143, "sampling/sampling_logp_difference/mean": 0.018182622268795967, "step": 79, "step_time": 198.80667186016217 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14034.0, "completions/max_terminated_length": 14034.0, "completions/mean_length": 10575.3125, "completions/mean_terminated_length": 10575.3125, "completions/min_length": 7172.0, "completions/min_terminated_length": 7172.0, "entropy": 0.25964460615068674, "epoch": 0.09603841536614646, "frac_reward_zero_std": 0.0, "grad_norm": 0.4087451696395874, "learning_rate": 6.84e-07, "loss": 0.0462, "num_tokens": 28857042.0, "reward": 0.8121464252471924, "reward_std": 1.5952792167663574, "rewards/reward_func/mean": 0.8121464252471924, "rewards/reward_func/std": 1.595279335975647, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.889493227005005, "sampling/importance_sampling_ratio/mean": 0.9910570383071899, "sampling/importance_sampling_ratio/min": 0.0975920781493187, "sampling/sampling_logp_difference/max": 2.3269588947296143, "sampling/sampling_logp_difference/mean": 0.01859731785953045, "step": 80, "step_time": 237.49234476010315 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 17704.0, "completions/max_terminated_length": 17704.0, "completions/mean_length": 10183.125, "completions/mean_terminated_length": 10183.125, "completions/min_length": 5525.0, "completions/min_terminated_length": 5525.0, "entropy": 0.25165214855223894, "epoch": 0.09723889555822329, "frac_reward_zero_std": 0.0, "grad_norm": 0.3784518241882324, "learning_rate": 6.800000000000001e-07, "loss": -0.0141, "num_tokens": 29213526.0, "reward": 8.427579879760742, "reward_std": 24.21912956237793, "rewards/reward_func/mean": 8.437650680541992, "rewards/reward_func/std": 24.21601676940918, "rewards/soft_overlong_punishment_reward/mean": -0.01007080078125, "rewards/soft_overlong_punishment_reward/std": 0.0569690503180027, "sampling/importance_sampling_ratio/max": 2.1546497344970703, "sampling/importance_sampling_ratio/mean": 0.9912103414535522, "sampling/importance_sampling_ratio/min": 0.3988558351993561, "sampling/sampling_logp_difference/max": 0.9191552400588989, "sampling/sampling_logp_difference/mean": 0.018073324114084244, "step": 81, "step_time": 258.5655134237604 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12930.0, "completions/max_terminated_length": 12930.0, "completions/mean_length": 8081.375, "completions/mean_terminated_length": 8081.375, "completions/min_length": 4585.0, "completions/min_terminated_length": 4585.0, "entropy": 0.2729193214327097, "epoch": 0.09843937575030012, "frac_reward_zero_std": 0.0, "grad_norm": 0.43891701102256775, "learning_rate": 6.76e-07, "loss": 0.0029, "num_tokens": 29504138.0, "reward": 2.3440794944763184, "reward_std": 2.2432892322540283, "rewards/reward_func/mean": 2.3440794944763184, "rewards/reward_func/std": 2.2432892322540283, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9903400540351868, "sampling/importance_sampling_ratio/min": 0.15401123464107513, "sampling/sampling_logp_difference/max": 1.8707298040390015, "sampling/sampling_logp_difference/mean": 0.01947890594601631, "step": 82, "step_time": 202.33901768375654 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14334.0, "completions/max_terminated_length": 14334.0, "completions/mean_length": 8477.21875, "completions/mean_terminated_length": 8477.21875, "completions/min_length": 2572.0, "completions/min_terminated_length": 2572.0, "entropy": 0.26816452480852604, "epoch": 0.09963985594237695, "frac_reward_zero_std": 0.0, "grad_norm": 0.48063552379608154, "learning_rate": 6.72e-07, "loss": 0.0714, "num_tokens": 29807561.0, "reward": 2.8314692974090576, "reward_std": 3.5692901611328125, "rewards/reward_func/mean": 2.8314692974090576, "rewards/reward_func/std": 3.5692899227142334, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.565842866897583, "sampling/importance_sampling_ratio/mean": 0.9907183051109314, "sampling/importance_sampling_ratio/min": 0.19031807780265808, "sampling/sampling_logp_difference/max": 1.6590585708618164, "sampling/sampling_logp_difference/mean": 0.018856924027204514, "step": 83, "step_time": 217.18472903524525 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 17204.0, "completions/max_terminated_length": 17204.0, "completions/mean_length": 8830.375, "completions/mean_terminated_length": 8830.375, "completions/min_length": 4996.0, "completions/min_terminated_length": 4996.0, "entropy": 0.27092269994318485, "epoch": 0.10084033613445378, "frac_reward_zero_std": 0.0, "grad_norm": 0.41215991973876953, "learning_rate": 6.68e-07, "loss": -0.0409, "num_tokens": 30123829.0, "reward": 2.2067344188690186, "reward_std": 3.047388792037964, "rewards/reward_func/mean": 2.2129905223846436, "rewards/reward_func/std": 3.0453271865844727, "rewards/soft_overlong_punishment_reward/mean": -0.006256103515625, "rewards/soft_overlong_punishment_reward/std": 0.03538986295461655, "sampling/importance_sampling_ratio/max": 2.052222490310669, "sampling/importance_sampling_ratio/mean": 0.990820586681366, "sampling/importance_sampling_ratio/min": 0.3114754259586334, "sampling/sampling_logp_difference/max": 1.1664347648620605, "sampling/sampling_logp_difference/mean": 0.0186629518866539, "step": 84, "step_time": 211.40592918510083 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12600.0, "completions/max_terminated_length": 12600.0, "completions/mean_length": 8298.3125, "completions/mean_terminated_length": 8298.3125, "completions/min_length": 4949.0, "completions/min_terminated_length": 4949.0, "entropy": 0.2829645574092865, "epoch": 0.10204081632653061, "frac_reward_zero_std": 0.0, "grad_norm": 0.5601394772529602, "learning_rate": 6.64e-07, "loss": 0.0305, "num_tokens": 30421735.0, "reward": 3.7544779777526855, "reward_std": 3.1711204051971436, "rewards/reward_func/mean": 3.7544779777526855, "rewards/reward_func/std": 3.1711206436157227, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9901652932167053, "sampling/importance_sampling_ratio/min": 0.12367851287126541, "sampling/sampling_logp_difference/max": 2.0900697708129883, "sampling/sampling_logp_difference/mean": 0.020197300240397453, "step": 85, "step_time": 266.99491723254323 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 20480.0, "completions/max_terminated_length": 15285.0, "completions/mean_length": 10550.75, "completions/mean_terminated_length": 10230.451171875, "completions/min_length": 5820.0, "completions/min_terminated_length": 5820.0, "entropy": 0.24555738922208548, "epoch": 0.10324129651860744, "frac_reward_zero_std": 0.0, "grad_norm": 0.314035564661026, "learning_rate": 6.6e-07, "loss": -0.0973, "num_tokens": 30792879.0, "reward": 1.717525601387024, "reward_std": 3.4412992000579834, "rewards/reward_func/mean": 1.805187702178955, "rewards/reward_func/std": 3.4616737365722656, "rewards/soft_overlong_punishment_reward/mean": -0.03125, "rewards/soft_overlong_punishment_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9913591742515564, "sampling/importance_sampling_ratio/min": 0.16628147661685944, "sampling/sampling_logp_difference/max": 1.7940733432769775, "sampling/sampling_logp_difference/mean": 0.017975792288780212, "step": 86, "step_time": 266.66299685207196 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13159.0, "completions/max_terminated_length": 13159.0, "completions/mean_length": 9776.40625, "completions/mean_terminated_length": 9776.40625, "completions/min_length": 7210.0, "completions/min_terminated_length": 7210.0, "entropy": 0.24749416019767523, "epoch": 0.10444177671068428, "frac_reward_zero_std": 0.0, "grad_norm": 0.37279677391052246, "learning_rate": 6.56e-07, "loss": -0.0527, "num_tokens": 31140836.0, "reward": 2.521646738052368, "reward_std": 3.81414794921875, "rewards/reward_func/mean": 2.521646738052368, "rewards/reward_func/std": 3.81414794921875, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.4072625637054443, "sampling/importance_sampling_ratio/mean": 0.9911715984344482, "sampling/importance_sampling_ratio/min": 0.19457262754440308, "sampling/sampling_logp_difference/max": 1.6369497776031494, "sampling/sampling_logp_difference/mean": 0.017878280952572823, "step": 87, "step_time": 313.04910568264313 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16942.0, "completions/max_terminated_length": 16942.0, "completions/mean_length": 9632.09375, "completions/mean_terminated_length": 9632.09375, "completions/min_length": 3954.0, "completions/min_terminated_length": 3954.0, "entropy": 0.28111381735652685, "epoch": 0.1056422569027611, "frac_reward_zero_std": 0.0, "grad_norm": 0.41607415676116943, "learning_rate": 6.52e-07, "loss": 0.0437, "num_tokens": 31474263.0, "reward": 1.1277384757995605, "reward_std": 1.8439369201660156, "rewards/reward_func/mean": 1.131995677947998, "rewards/reward_func/std": 1.8411990404129028, "rewards/soft_overlong_punishment_reward/mean": -0.0042572021484375, "rewards/soft_overlong_punishment_reward/std": 0.024082370102405548, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9902504682540894, "sampling/importance_sampling_ratio/min": 0.01312999241054058, "sampling/sampling_logp_difference/max": 4.332856178283691, "sampling/sampling_logp_difference/mean": 0.01968482695519924, "step": 88, "step_time": 231.826713241986 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16728.0, "completions/max_terminated_length": 16728.0, "completions/mean_length": 10530.4375, "completions/mean_terminated_length": 10530.4375, "completions/min_length": 6271.0, "completions/min_terminated_length": 6271.0, "entropy": 0.25707424618303776, "epoch": 0.10684273709483794, "frac_reward_zero_std": 0.0, "grad_norm": 0.38471195101737976, "learning_rate": 6.48e-07, "loss": 0.0606, "num_tokens": 31852261.0, "reward": 1.277016282081604, "reward_std": 2.039738893508911, "rewards/reward_func/mean": 1.279640793800354, "rewards/reward_func/std": 2.038050889968872, "rewards/soft_overlong_punishment_reward/mean": -0.00262451171875, "rewards/soft_overlong_punishment_reward/std": 0.014846480451524258, "sampling/importance_sampling_ratio/max": 2.706266164779663, "sampling/importance_sampling_ratio/mean": 0.9911405444145203, "sampling/importance_sampling_ratio/min": 0.22103802859783173, "sampling/sampling_logp_difference/max": 1.5094205141067505, "sampling/sampling_logp_difference/mean": 0.0181552916765213, "step": 89, "step_time": 452.72294589015655 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15561.0, "completions/max_terminated_length": 15561.0, "completions/mean_length": 10859.5, "completions/mean_terminated_length": 10859.5, "completions/min_length": 4952.0, "completions/min_terminated_length": 4952.0, "entropy": 0.24442417547106743, "epoch": 0.10804321728691477, "frac_reward_zero_std": 0.0, "grad_norm": 0.37132179737091064, "learning_rate": 6.44e-07, "loss": 0.025, "num_tokens": 32237653.0, "reward": 0.5021224617958069, "reward_std": 0.7325555682182312, "rewards/reward_func/mean": 0.5021224617958069, "rewards/reward_func/std": 0.7325555086135864, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.3009634017944336, "sampling/importance_sampling_ratio/mean": 0.9916141033172607, "sampling/importance_sampling_ratio/min": 0.0029834972228854895, "sampling/sampling_logp_difference/max": 5.814659118652344, "sampling/sampling_logp_difference/mean": 0.017731234431266785, "step": 90, "step_time": 248.43443835526705 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14837.0, "completions/max_terminated_length": 14837.0, "completions/mean_length": 10019.59375, "completions/mean_terminated_length": 10019.59375, "completions/min_length": 6964.0, "completions/min_terminated_length": 6964.0, "entropy": 0.279639994725585, "epoch": 0.1092436974789916, "frac_reward_zero_std": 0.0, "grad_norm": 0.3926539421081543, "learning_rate": 6.4e-07, "loss": 0.0568, "num_tokens": 32596080.0, "reward": 2894.755615234375, "reward_std": 7884.30615234375, "rewards/reward_func/mean": 2894.755615234375, "rewards/reward_func/std": 7884.30615234375, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9901866316795349, "sampling/importance_sampling_ratio/min": 0.2964800298213959, "sampling/sampling_logp_difference/max": 1.3845736980438232, "sampling/sampling_logp_difference/mean": 0.019792938604950905, "step": 91, "step_time": 230.6299822588917 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12330.0, "completions/max_terminated_length": 12330.0, "completions/mean_length": 9052.78125, "completions/mean_terminated_length": 9052.78125, "completions/min_length": 5906.0, "completions/min_terminated_length": 5906.0, "entropy": 0.31160302832722664, "epoch": 0.11044417767106843, "frac_reward_zero_std": 0.0, "grad_norm": 0.41976141929626465, "learning_rate": 6.36e-07, "loss": -0.0106, "num_tokens": 32908137.0, "reward": 1.0270764827728271, "reward_std": 1.5837911367416382, "rewards/reward_func/mean": 1.0270764827728271, "rewards/reward_func/std": 1.5837911367416382, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9889901280403137, "sampling/importance_sampling_ratio/min": 0.251454621553421, "sampling/sampling_logp_difference/max": 1.3804928064346313, "sampling/sampling_logp_difference/mean": 0.02152213454246521, "step": 92, "step_time": 209.22184566501528 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13164.0, "completions/max_terminated_length": 13164.0, "completions/mean_length": 7722.125, "completions/mean_terminated_length": 7722.125, "completions/min_length": 3235.0, "completions/min_terminated_length": 3235.0, "entropy": 0.27631482668220997, "epoch": 0.11164465786314526, "frac_reward_zero_std": 0.0, "grad_norm": 0.43816474080085754, "learning_rate": 6.319999999999999e-07, "loss": -0.0276, "num_tokens": 33179125.0, "reward": 3.121471881866455, "reward_std": 3.0470423698425293, "rewards/reward_func/mean": 3.121471881866455, "rewards/reward_func/std": 3.0470426082611084, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.2446866035461426, "sampling/importance_sampling_ratio/mean": 0.990494430065155, "sampling/importance_sampling_ratio/min": 0.3229256570339203, "sampling/sampling_logp_difference/max": 1.1303331851959229, "sampling/sampling_logp_difference/mean": 0.0194493867456913, "step": 93, "step_time": 403.52912656613626 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9834.0, "completions/max_terminated_length": 9834.0, "completions/mean_length": 6429.0, "completions/mean_terminated_length": 6429.0, "completions/min_length": 3298.0, "completions/min_terminated_length": 3298.0, "entropy": 0.3079283908009529, "epoch": 0.11284513805522209, "frac_reward_zero_std": 0.0, "grad_norm": 0.512400209903717, "learning_rate": 6.28e-07, "loss": -0.0118, "num_tokens": 33409485.0, "reward": 4.05039119720459, "reward_std": 6.237717628479004, "rewards/reward_func/mean": 4.05039119720459, "rewards/reward_func/std": 6.237717151641846, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.9068455696105957, "sampling/importance_sampling_ratio/mean": 0.9891414046287537, "sampling/importance_sampling_ratio/min": 0.28006723523139954, "sampling/sampling_logp_difference/max": 1.2727255821228027, "sampling/sampling_logp_difference/mean": 0.021337833255529404, "step": 94, "step_time": 150.58229626482353 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 20480.0, "completions/max_terminated_length": 13641.0, "completions/mean_length": 8809.875, "completions/mean_terminated_length": 8433.4189453125, "completions/min_length": 2494.0, "completions/min_terminated_length": 2494.0, "entropy": 0.28533732518553734, "epoch": 0.11404561824729892, "frac_reward_zero_std": 0.0, "grad_norm": 0.34757664799690247, "learning_rate": 6.24e-07, "loss": -0.1059, "num_tokens": 33722393.0, "reward": 2.1445155143737793, "reward_std": 2.832806348800659, "rewards/reward_func/mean": 2.2459514141082764, "rewards/reward_func/std": 2.8199384212493896, "rewards/soft_overlong_punishment_reward/mean": -0.03125, "rewards/soft_overlong_punishment_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9901010990142822, "sampling/importance_sampling_ratio/min": 0.0004452175635378808, "sampling/sampling_logp_difference/max": 7.716947555541992, "sampling/sampling_logp_difference/mean": 0.01991894841194153, "step": 95, "step_time": 253.43176098098047 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15697.0, "completions/max_terminated_length": 15697.0, "completions/mean_length": 10009.0625, "completions/mean_terminated_length": 10009.0625, "completions/min_length": 5990.0, "completions/min_terminated_length": 5990.0, "entropy": 0.24590188823640347, "epoch": 0.11524609843937575, "frac_reward_zero_std": 0.25, "grad_norm": 0.316346138715744, "learning_rate": 6.2e-07, "loss": -0.0135, "num_tokens": 34078643.0, "reward": 0.39841264486312866, "reward_std": 0.5363882184028625, "rewards/reward_func/mean": 0.39841264486312866, "rewards/reward_func/std": 0.5363882184028625, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.871351718902588, "sampling/importance_sampling_ratio/mean": 0.9914729595184326, "sampling/importance_sampling_ratio/min": 0.09377329051494598, "sampling/sampling_logp_difference/max": 2.366875171661377, "sampling/sampling_logp_difference/mean": 0.017338084056973457, "step": 96, "step_time": 435.25480223365594 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14781.0, "completions/max_terminated_length": 14781.0, "completions/mean_length": 10244.75, "completions/mean_terminated_length": 10244.75, "completions/min_length": 5004.0, "completions/min_terminated_length": 5004.0, "entropy": 0.3078562542796135, "epoch": 0.11644657863145258, "frac_reward_zero_std": 0.0, "grad_norm": 0.4103779196739197, "learning_rate": 6.16e-07, "loss": 0.038, "num_tokens": 34434571.0, "reward": 0.8614925146102905, "reward_std": 0.5891639590263367, "rewards/reward_func/mean": 0.8614925146102905, "rewards/reward_func/std": 0.5891639590263367, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.2111990451812744, "sampling/importance_sampling_ratio/mean": 0.9891133308410645, "sampling/importance_sampling_ratio/min": 0.1545713096857071, "sampling/sampling_logp_difference/max": 1.8670997619628906, "sampling/sampling_logp_difference/mean": 0.021383928135037422, "step": 97, "step_time": 212.63684044289403 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15325.0, "completions/max_terminated_length": 15325.0, "completions/mean_length": 11469.84375, "completions/mean_terminated_length": 11469.84375, "completions/min_length": 6985.0, "completions/min_terminated_length": 6985.0, "entropy": 0.26262489426881075, "epoch": 0.11764705882352941, "frac_reward_zero_std": 0.0, "grad_norm": 0.37121981382369995, "learning_rate": 6.119999999999999e-07, "loss": 0.0254, "num_tokens": 34846182.0, "reward": 1.391387701034546, "reward_std": 2.258481025695801, "rewards/reward_func/mean": 1.391387701034546, "rewards/reward_func/std": 2.258481025695801, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.9727016687393188, "sampling/importance_sampling_ratio/mean": 0.9908169507980347, "sampling/importance_sampling_ratio/min": 0.24390530586242676, "sampling/sampling_logp_difference/max": 1.4109752178192139, "sampling/sampling_logp_difference/mean": 0.018506620079278946, "step": 98, "step_time": 251.3301967199659 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12299.0, "completions/max_terminated_length": 12299.0, "completions/mean_length": 8463.15625, "completions/mean_terminated_length": 8463.15625, "completions/min_length": 3478.0, "completions/min_terminated_length": 3478.0, "entropy": 0.273245669901371, "epoch": 0.11884753901560624, "frac_reward_zero_std": 0.25, "grad_norm": 0.365919291973114, "learning_rate": 6.079999999999999e-07, "loss": 0.0383, "num_tokens": 35145083.0, "reward": 3.128838539123535, "reward_std": 3.8530807495117188, "rewards/reward_func/mean": 3.128838539123535, "rewards/reward_func/std": 3.853080987930298, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.477442979812622, "sampling/importance_sampling_ratio/mean": 0.9905266761779785, "sampling/importance_sampling_ratio/min": 0.38133692741394043, "sampling/sampling_logp_difference/max": 0.9640719890594482, "sampling/sampling_logp_difference/mean": 0.0193619467318058, "step": 99, "step_time": 214.26986349211074 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14368.0, "completions/max_terminated_length": 14368.0, "completions/mean_length": 8256.4375, "completions/mean_terminated_length": 8256.4375, "completions/min_length": 4534.0, "completions/min_terminated_length": 4534.0, "entropy": 0.3049591239541769, "epoch": 0.12004801920768307, "frac_reward_zero_std": 0.25, "grad_norm": 0.38011953234672546, "learning_rate": 6.04e-07, "loss": -0.0081, "num_tokens": 35432593.0, "reward": 2.3642165660858154, "reward_std": 2.706695556640625, "rewards/reward_func/mean": 2.3642165660858154, "rewards/reward_func/std": 2.706695556640625, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.9068528413772583, "sampling/importance_sampling_ratio/mean": 0.9893739223480225, "sampling/importance_sampling_ratio/min": 0.1614706665277481, "sampling/sampling_logp_difference/max": 1.8234318494796753, "sampling/sampling_logp_difference/mean": 0.02130398340523243, "step": 100, "step_time": 184.41112436982803 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 17413.0, "completions/max_terminated_length": 17413.0, "completions/mean_length": 11627.0625, "completions/mean_terminated_length": 11627.0625, "completions/min_length": 6505.0, "completions/min_terminated_length": 6505.0, "entropy": 0.24142704904079437, "epoch": 0.1212484993997599, "frac_reward_zero_std": 0.0, "grad_norm": 0.36239883303642273, "learning_rate": 6e-07, "loss": 0.0495, "num_tokens": 35847699.0, "reward": 0.8143461346626282, "reward_std": 0.6679160594940186, "rewards/reward_func/mean": 0.8221967816352844, "rewards/reward_func/std": 0.6563639044761658, "rewards/soft_overlong_punishment_reward/mean": -0.00785064697265625, "rewards/soft_overlong_punishment_reward/std": 0.044409964233636856, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9916003346443176, "sampling/importance_sampling_ratio/min": 0.05457422509789467, "sampling/sampling_logp_difference/max": 2.908193588256836, "sampling/sampling_logp_difference/mean": 0.01743597723543644, "step": 101, "step_time": 247.9607744079549 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12313.0, "completions/max_terminated_length": 12313.0, "completions/mean_length": 9030.0625, "completions/mean_terminated_length": 9030.0625, "completions/min_length": 6142.0, "completions/min_terminated_length": 6142.0, "entropy": 0.2866959013044834, "epoch": 0.12244897959183673, "frac_reward_zero_std": 0.0, "grad_norm": 0.4205459654331207, "learning_rate": 5.96e-07, "loss": 0.0247, "num_tokens": 36166773.0, "reward": 1.333907127380371, "reward_std": 1.1282483339309692, "rewards/reward_func/mean": 1.333907127380371, "rewards/reward_func/std": 1.1282483339309692, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.730395555496216, "sampling/importance_sampling_ratio/mean": 0.9900311231613159, "sampling/importance_sampling_ratio/min": 0.05878521502017975, "sampling/sampling_logp_difference/max": 2.83386492729187, "sampling/sampling_logp_difference/mean": 0.020093757659196854, "step": 102, "step_time": 215.10454945149831 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12475.0, "completions/max_terminated_length": 12475.0, "completions/mean_length": 8976.34375, "completions/mean_terminated_length": 8976.34375, "completions/min_length": 3906.0, "completions/min_terminated_length": 3906.0, "entropy": 0.24582395143806934, "epoch": 0.12364945978391356, "frac_reward_zero_std": 0.0, "grad_norm": 0.3737013339996338, "learning_rate": 5.919999999999999e-07, "loss": 0.0012, "num_tokens": 36486264.0, "reward": 0.4444224536418915, "reward_std": 0.6303268671035767, "rewards/reward_func/mean": 0.4444224536418915, "rewards/reward_func/std": 0.6303268074989319, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9914492964744568, "sampling/importance_sampling_ratio/min": 0.18756607174873352, "sampling/sampling_logp_difference/max": 1.673624038696289, "sampling/sampling_logp_difference/mean": 0.01754648983478546, "step": 103, "step_time": 206.92123383819126 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13655.0, "completions/max_terminated_length": 13655.0, "completions/mean_length": 9150.15625, "completions/mean_terminated_length": 9150.15625, "completions/min_length": 4648.0, "completions/min_terminated_length": 4648.0, "entropy": 0.26344494987279177, "epoch": 0.12484993997599039, "frac_reward_zero_std": 0.25, "grad_norm": 0.34451353549957275, "learning_rate": 5.879999999999999e-07, "loss": -0.0295, "num_tokens": 36814093.0, "reward": 5.4827399253845215, "reward_std": 10.697521209716797, "rewards/reward_func/mean": 5.4827399253845215, "rewards/reward_func/std": 10.697521209716797, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.3717105388641357, "sampling/importance_sampling_ratio/mean": 0.9908387064933777, "sampling/importance_sampling_ratio/min": 0.23612551391124725, "sampling/sampling_logp_difference/max": 1.4433917999267578, "sampling/sampling_logp_difference/mean": 0.018703076988458633, "step": 104, "step_time": 213.1645632191794 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14144.0, "completions/max_terminated_length": 14144.0, "completions/mean_length": 9819.1875, "completions/mean_terminated_length": 9819.1875, "completions/min_length": 5625.0, "completions/min_terminated_length": 5625.0, "entropy": 0.24409340787678957, "epoch": 0.12605042016806722, "frac_reward_zero_std": 0.25, "grad_norm": 0.33012041449546814, "learning_rate": 5.839999999999999e-07, "loss": 0.029, "num_tokens": 37167963.0, "reward": 1.2769612073898315, "reward_std": 2.2972137928009033, "rewards/reward_func/mean": 1.2769612073898315, "rewards/reward_func/std": 2.2972140312194824, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.3744382858276367, "sampling/importance_sampling_ratio/mean": 0.9916211366653442, "sampling/importance_sampling_ratio/min": 0.18144047260284424, "sampling/sampling_logp_difference/max": 1.7068276405334473, "sampling/sampling_logp_difference/mean": 0.017610425129532814, "step": 105, "step_time": 233.00164715701248 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 19045.0, "completions/max_terminated_length": 19045.0, "completions/mean_length": 10475.28125, "completions/mean_terminated_length": 10475.28125, "completions/min_length": 6236.0, "completions/min_terminated_length": 6236.0, "entropy": 0.2810442354530096, "epoch": 0.12725090036014405, "frac_reward_zero_std": 0.0, "grad_norm": 0.4047479033470154, "learning_rate": 5.8e-07, "loss": 0.0844, "num_tokens": 37540988.0, "reward": 1.971191644668579, "reward_std": 3.9106719493865967, "rewards/reward_func/mean": 1.9987413883209229, "rewards/reward_func/std": 3.894240140914917, "rewards/soft_overlong_punishment_reward/mean": -0.02754974365234375, "rewards/soft_overlong_punishment_reward/std": 0.12069173902273178, "sampling/importance_sampling_ratio/max": 2.3556199073791504, "sampling/importance_sampling_ratio/mean": 0.990192711353302, "sampling/importance_sampling_ratio/min": 0.1058955043554306, "sampling/sampling_logp_difference/max": 2.245302438735962, "sampling/sampling_logp_difference/mean": 0.019729377701878548, "step": 106, "step_time": 259.3078554049134 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11035.0, "completions/max_terminated_length": 11035.0, "completions/mean_length": 8085.09375, "completions/mean_terminated_length": 8085.09375, "completions/min_length": 5843.0, "completions/min_terminated_length": 5843.0, "entropy": 0.28292676340788603, "epoch": 0.12845138055222088, "frac_reward_zero_std": 0.0, "grad_norm": 0.44497084617614746, "learning_rate": 5.76e-07, "loss": -0.0027, "num_tokens": 37824127.0, "reward": 5.807962417602539, "reward_std": 3.6478431224823, "rewards/reward_func/mean": 5.807962417602539, "rewards/reward_func/std": 3.6478431224823, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9902195930480957, "sampling/importance_sampling_ratio/min": 0.2602680027484894, "sampling/sampling_logp_difference/max": 1.6943206787109375, "sampling/sampling_logp_difference/mean": 0.01987529546022415, "step": 107, "step_time": 164.33568983571604 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16656.0, "completions/max_terminated_length": 16656.0, "completions/mean_length": 10496.1875, "completions/mean_terminated_length": 10496.1875, "completions/min_length": 6152.0, "completions/min_terminated_length": 6152.0, "entropy": 0.31788551434874535, "epoch": 0.12965186074429771, "frac_reward_zero_std": 0.0, "grad_norm": 0.36823001503944397, "learning_rate": 5.719999999999999e-07, "loss": 0.0434, "num_tokens": 38184413.0, "reward": 0.822684109210968, "reward_std": 0.7153370380401611, "rewards/reward_func/mean": 0.824759304523468, "rewards/reward_func/std": 0.7129164934158325, "rewards/soft_overlong_punishment_reward/mean": -0.0020751953125, "rewards/soft_overlong_punishment_reward/std": 0.011739077046513557, "sampling/importance_sampling_ratio/max": 2.7742605209350586, "sampling/importance_sampling_ratio/mean": 0.9888496994972229, "sampling/importance_sampling_ratio/min": 0.3614434599876404, "sampling/sampling_logp_difference/max": 1.0203841924667358, "sampling/sampling_logp_difference/mean": 0.0220322422683239, "step": 108, "step_time": 238.20601075387094 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14831.0, "completions/max_terminated_length": 14831.0, "completions/mean_length": 10238.46875, "completions/mean_terminated_length": 10238.46875, "completions/min_length": 6359.0, "completions/min_terminated_length": 6359.0, "entropy": 0.28053270652890205, "epoch": 0.13085234093637454, "frac_reward_zero_std": 0.0, "grad_norm": 0.39827480912208557, "learning_rate": 5.679999999999999e-07, "loss": -0.0661, "num_tokens": 38553132.0, "reward": 0.5400049686431885, "reward_std": 0.8026726245880127, "rewards/reward_func/mean": 0.5400049686431885, "rewards/reward_func/std": 0.8026726841926575, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.3182525634765625, "sampling/importance_sampling_ratio/mean": 0.9902611374855042, "sampling/importance_sampling_ratio/min": 0.3386656939983368, "sampling/sampling_logp_difference/max": 1.0827418565750122, "sampling/sampling_logp_difference/mean": 0.01999467983841896, "step": 109, "step_time": 251.36417699581943 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12513.0, "completions/max_terminated_length": 12513.0, "completions/mean_length": 9081.59375, "completions/mean_terminated_length": 9081.59375, "completions/min_length": 4968.0, "completions/min_terminated_length": 4968.0, "entropy": 0.2736341943964362, "epoch": 0.13205282112845138, "frac_reward_zero_std": 0.0, "grad_norm": 0.4176849126815796, "learning_rate": 5.639999999999999e-07, "loss": 0.0632, "num_tokens": 38883943.0, "reward": 1.397516131401062, "reward_std": 1.4397801160812378, "rewards/reward_func/mean": 1.397516131401062, "rewards/reward_func/std": 1.4397801160812378, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.864140510559082, "sampling/importance_sampling_ratio/mean": 0.9903908371925354, "sampling/importance_sampling_ratio/min": 0.30289867520332336, "sampling/sampling_logp_difference/max": 1.194356918334961, "sampling/sampling_logp_difference/mean": 0.019294166937470436, "step": 110, "step_time": 432.76785584609024 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15302.0, "completions/max_terminated_length": 15302.0, "completions/mean_length": 10490.46875, "completions/mean_terminated_length": 10490.46875, "completions/min_length": 7700.0, "completions/min_terminated_length": 7700.0, "entropy": 0.2960231024771929, "epoch": 0.1332533013205282, "frac_reward_zero_std": 0.0, "grad_norm": 0.4143314063549042, "learning_rate": 5.6e-07, "loss": 0.0137, "num_tokens": 39249678.0, "reward": 1.0926156044006348, "reward_std": 0.6936445832252502, "rewards/reward_func/mean": 1.0926156044006348, "rewards/reward_func/std": 0.6936445236206055, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.972447633743286, "sampling/importance_sampling_ratio/mean": 0.9898686408996582, "sampling/importance_sampling_ratio/min": 0.15995651483535767, "sampling/sampling_logp_difference/max": 1.8328533172607422, "sampling/sampling_logp_difference/mean": 0.02054767683148384, "step": 111, "step_time": 220.7248961061705 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 17000.0, "completions/max_terminated_length": 17000.0, "completions/mean_length": 9145.03125, "completions/mean_terminated_length": 9145.03125, "completions/min_length": 3641.0, "completions/min_terminated_length": 3641.0, "entropy": 0.2596113607287407, "epoch": 0.13445378151260504, "frac_reward_zero_std": 0.0, "grad_norm": 0.42476722598075867, "learning_rate": 5.560000000000001e-07, "loss": 0.0171, "num_tokens": 39581495.0, "reward": 0.21374335885047913, "reward_std": 0.40230268239974976, "rewards/reward_func/mean": 0.21844306588172913, "rewards/reward_func/std": 0.39877456426620483, "rewards/soft_overlong_punishment_reward/mean": -0.00469970703125, "rewards/soft_overlong_punishment_reward/std": 0.02658555842936039, "sampling/importance_sampling_ratio/max": 2.445462942123413, "sampling/importance_sampling_ratio/mean": 0.9910917282104492, "sampling/importance_sampling_ratio/min": 0.0019266947638243437, "sampling/sampling_logp_difference/max": 6.251949310302734, "sampling/sampling_logp_difference/mean": 0.018219787627458572, "step": 112, "step_time": 280.31022595707327 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13890.0, "completions/max_terminated_length": 13890.0, "completions/mean_length": 10680.34375, "completions/mean_terminated_length": 10680.34375, "completions/min_length": 6448.0, "completions/min_terminated_length": 6448.0, "entropy": 0.28790590912103653, "epoch": 0.13565426170468187, "frac_reward_zero_std": 0.0, "grad_norm": 0.3604806065559387, "learning_rate": 5.520000000000001e-07, "loss": -0.0006, "num_tokens": 39955530.0, "reward": 0.5530512928962708, "reward_std": 0.9826109409332275, "rewards/reward_func/mean": 0.5530512928962708, "rewards/reward_func/std": 0.9826108813285828, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.565842866897583, "sampling/importance_sampling_ratio/mean": 0.9898175597190857, "sampling/importance_sampling_ratio/min": 0.2070624679327011, "sampling/sampling_logp_difference/max": 1.5747346878051758, "sampling/sampling_logp_difference/mean": 0.02044212445616722, "step": 113, "step_time": 420.17655016179197 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13642.0, "completions/max_terminated_length": 13642.0, "completions/mean_length": 9351.1875, "completions/mean_terminated_length": 9351.1875, "completions/min_length": 5824.0, "completions/min_terminated_length": 5824.0, "entropy": 0.2932744771242142, "epoch": 0.1368547418967587, "frac_reward_zero_std": 0.0, "grad_norm": 0.4188636839389801, "learning_rate": 5.48e-07, "loss": 0.004, "num_tokens": 40283608.0, "reward": 1.3525290489196777, "reward_std": 1.7022446393966675, "rewards/reward_func/mean": 1.3525290489196777, "rewards/reward_func/std": 1.702244520187378, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.989858090877533, "sampling/importance_sampling_ratio/min": 0.2756859362125397, "sampling/sampling_logp_difference/max": 2.190220355987549, "sampling/sampling_logp_difference/mean": 0.020323438569903374, "step": 114, "step_time": 226.98730401217472 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 17342.0, "completions/max_terminated_length": 17342.0, "completions/mean_length": 8931.0, "completions/mean_terminated_length": 8931.0, "completions/min_length": 3315.0, "completions/min_terminated_length": 3315.0, "entropy": 0.24458013847470284, "epoch": 0.13805522208883553, "frac_reward_zero_std": 0.0, "grad_norm": 0.3886120617389679, "learning_rate": 5.44e-07, "loss": -0.0341, "num_tokens": 40610928.0, "reward": 0.9550597667694092, "reward_std": 1.4101365804672241, "rewards/reward_func/mean": 0.9623687267303467, "rewards/reward_func/std": 1.4116019010543823, "rewards/soft_overlong_punishment_reward/mean": -0.0073089599609375, "rewards/soft_overlong_punishment_reward/std": 0.04134571924805641, "sampling/importance_sampling_ratio/max": 2.6611251831054688, "sampling/importance_sampling_ratio/mean": 0.9916718006134033, "sampling/importance_sampling_ratio/min": 0.16927804052829742, "sampling/sampling_logp_difference/max": 1.7762126922607422, "sampling/sampling_logp_difference/mean": 0.017050782218575478, "step": 115, "step_time": 229.30203108501155 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15122.0, "completions/max_terminated_length": 15122.0, "completions/mean_length": 10591.75, "completions/mean_terminated_length": 10591.75, "completions/min_length": 7260.0, "completions/min_terminated_length": 7260.0, "entropy": 0.2582763684913516, "epoch": 0.13925570228091236, "frac_reward_zero_std": 0.0, "grad_norm": 0.3612559139728546, "learning_rate": 5.4e-07, "loss": 0.0453, "num_tokens": 40996336.0, "reward": 2.7374861240386963, "reward_std": 3.0827701091766357, "rewards/reward_func/mean": 2.7374861240386963, "rewards/reward_func/std": 3.0827698707580566, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.93190598487854, "sampling/importance_sampling_ratio/mean": 0.9909803867340088, "sampling/importance_sampling_ratio/min": 0.031009899452328682, "sampling/sampling_logp_difference/max": 3.4734487533569336, "sampling/sampling_logp_difference/mean": 0.018270010128617287, "step": 116, "step_time": 227.7563733382849 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13868.0, "completions/max_terminated_length": 13868.0, "completions/mean_length": 9498.53125, "completions/mean_terminated_length": 9498.53125, "completions/min_length": 5313.0, "completions/min_terminated_length": 5313.0, "entropy": 0.2784241568297148, "epoch": 0.1404561824729892, "frac_reward_zero_std": 0.0, "grad_norm": 0.5930799841880798, "learning_rate": 5.36e-07, "loss": -0.033, "num_tokens": 41336345.0, "reward": 0.9384326338768005, "reward_std": 1.186398983001709, "rewards/reward_func/mean": 0.9384326338768005, "rewards/reward_func/std": 1.1863987445831299, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.455018997192383, "sampling/importance_sampling_ratio/mean": 0.9903002977371216, "sampling/importance_sampling_ratio/min": 0.015267138369381428, "sampling/sampling_logp_difference/max": 4.1820526123046875, "sampling/sampling_logp_difference/mean": 0.019709795713424683, "step": 117, "step_time": 243.17280280101113 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14484.0, "completions/max_terminated_length": 14484.0, "completions/mean_length": 9971.8125, "completions/mean_terminated_length": 9971.8125, "completions/min_length": 7100.0, "completions/min_terminated_length": 7100.0, "entropy": 0.29973538406193256, "epoch": 0.14165666266506602, "frac_reward_zero_std": 0.0, "grad_norm": 0.5883257985115051, "learning_rate": 5.32e-07, "loss": 0.0281, "num_tokens": 41688307.0, "reward": 0.6539914608001709, "reward_std": 0.7319720983505249, "rewards/reward_func/mean": 0.6539914608001709, "rewards/reward_func/std": 0.7319720387458801, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.818315029144287, "sampling/importance_sampling_ratio/mean": 0.9895184636116028, "sampling/importance_sampling_ratio/min": 0.4023731052875519, "sampling/sampling_logp_difference/max": 1.0361392498016357, "sampling/sampling_logp_difference/mean": 0.021014517173171043, "step": 118, "step_time": 231.1251015268499 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 19530.0, "completions/max_terminated_length": 19530.0, "completions/mean_length": 11520.625, "completions/mean_terminated_length": 11520.625, "completions/min_length": 3825.0, "completions/min_terminated_length": 3825.0, "entropy": 0.24881694186478853, "epoch": 0.14285714285714285, "frac_reward_zero_std": 0.0, "grad_norm": 0.3857250213623047, "learning_rate": 5.28e-07, "loss": -0.0142, "num_tokens": 42102007.0, "reward": 1.7721220254898071, "reward_std": 2.1013307571411133, "rewards/reward_func/mean": 1.8261228799819946, "rewards/reward_func/std": 2.050947427749634, "rewards/soft_overlong_punishment_reward/mean": -0.0540008544921875, "rewards/soft_overlong_punishment_reward/std": 0.18218861520290375, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9912911653518677, "sampling/importance_sampling_ratio/min": 0.07585831731557846, "sampling/sampling_logp_difference/max": 2.578887939453125, "sampling/sampling_logp_difference/mean": 0.017922591418027878, "step": 119, "step_time": 291.4575341643067 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15462.0, "completions/max_terminated_length": 15462.0, "completions/mean_length": 8598.9375, "completions/mean_terminated_length": 8598.9375, "completions/min_length": 3214.0, "completions/min_terminated_length": 3214.0, "entropy": 0.25942704640328884, "epoch": 0.14405762304921968, "frac_reward_zero_std": 0.25, "grad_norm": 0.37554553151130676, "learning_rate": 5.24e-07, "loss": 0.0008, "num_tokens": 42406077.0, "reward": 1.3100762367248535, "reward_std": 1.914520263671875, "rewards/reward_func/mean": 1.3100762367248535, "rewards/reward_func/std": 1.914520263671875, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.4972586631774902, "sampling/importance_sampling_ratio/mean": 0.9908623099327087, "sampling/importance_sampling_ratio/min": 0.1831483691930771, "sampling/sampling_logp_difference/max": 1.6974586248397827, "sampling/sampling_logp_difference/mean": 0.01822926476597786, "step": 120, "step_time": 209.00005609192885 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16215.0, "completions/max_terminated_length": 16215.0, "completions/mean_length": 9589.25, "completions/mean_terminated_length": 9589.25, "completions/min_length": 5802.0, "completions/min_terminated_length": 5802.0, "entropy": 0.3083428144454956, "epoch": 0.1452581032412965, "frac_reward_zero_std": 0.0, "grad_norm": 0.4060857594013214, "learning_rate": 5.2e-07, "loss": -0.0524, "num_tokens": 42737701.0, "reward": 0.49280887842178345, "reward_std": 0.6724991798400879, "rewards/reward_func/mean": 0.49280887842178345, "rewards/reward_func/std": 0.6724991798400879, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.6253695487976074, "sampling/importance_sampling_ratio/mean": 0.9890679717063904, "sampling/importance_sampling_ratio/min": 0.00019750210049096495, "sampling/sampling_logp_difference/max": 8.52976131439209, "sampling/sampling_logp_difference/mean": 0.021406318992376328, "step": 121, "step_time": 241.5949507113546 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15308.0, "completions/max_terminated_length": 15308.0, "completions/mean_length": 10594.34375, "completions/mean_terminated_length": 10594.34375, "completions/min_length": 7324.0, "completions/min_terminated_length": 7324.0, "entropy": 0.28266207315027714, "epoch": 0.14645858343337334, "frac_reward_zero_std": 0.5, "grad_norm": 0.2750304937362671, "learning_rate": 5.16e-07, "loss": 0.0195, "num_tokens": 43113840.0, "reward": 0.5235170722007751, "reward_std": 0.633671224117279, "rewards/reward_func/mean": 0.5235170722007751, "rewards/reward_func/std": 0.633671224117279, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.4920129776000977, "sampling/importance_sampling_ratio/mean": 0.9901369214057922, "sampling/importance_sampling_ratio/min": 0.10062485188245773, "sampling/sampling_logp_difference/max": 2.296355962753296, "sampling/sampling_logp_difference/mean": 0.01979176700115204, "step": 122, "step_time": 225.33721455116756 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15019.0, "completions/max_terminated_length": 15019.0, "completions/mean_length": 11032.53125, "completions/mean_terminated_length": 11032.53125, "completions/min_length": 5304.0, "completions/min_terminated_length": 5304.0, "entropy": 0.26705341041088104, "epoch": 0.14765906362545017, "frac_reward_zero_std": 0.0, "grad_norm": 0.3584786653518677, "learning_rate": 5.12e-07, "loss": 0.0368, "num_tokens": 43507401.0, "reward": 1.9813694953918457, "reward_std": 2.857267141342163, "rewards/reward_func/mean": 1.9813694953918457, "rewards/reward_func/std": 2.857267379760742, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.2451746463775635, "sampling/importance_sampling_ratio/mean": 0.9905773997306824, "sampling/importance_sampling_ratio/min": 0.07104846835136414, "sampling/sampling_logp_difference/max": 2.644392967224121, "sampling/sampling_logp_difference/mean": 0.019043972715735435, "step": 123, "step_time": 460.36738448997494 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 20032.0, "completions/max_terminated_length": 20032.0, "completions/mean_length": 11611.53125, "completions/mean_terminated_length": 11611.53125, "completions/min_length": 6363.0, "completions/min_terminated_length": 6363.0, "entropy": 0.2338911509141326, "epoch": 0.148859543817527, "frac_reward_zero_std": 0.0, "grad_norm": 0.3491296172142029, "learning_rate": 5.079999999999999e-07, "loss": 0.0364, "num_tokens": 43926578.0, "reward": 1.0934959650039673, "reward_std": 1.0090538263320923, "rewards/reward_func/mean": 1.1581779718399048, "rewards/reward_func/std": 0.9206379055976868, "rewards/soft_overlong_punishment_reward/mean": -0.0646820068359375, "rewards/soft_overlong_punishment_reward/std": 0.220370814204216, "sampling/importance_sampling_ratio/max": 1.99778151512146, "sampling/importance_sampling_ratio/mean": 0.9919837713241577, "sampling/importance_sampling_ratio/min": 0.01377395261079073, "sampling/sampling_logp_difference/max": 4.284976005554199, "sampling/sampling_logp_difference/mean": 0.016360996291041374, "step": 124, "step_time": 275.2218638029881 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16157.0, "completions/max_terminated_length": 16157.0, "completions/mean_length": 10606.0625, "completions/mean_terminated_length": 10606.0625, "completions/min_length": 4477.0, "completions/min_terminated_length": 4477.0, "entropy": 0.2876010574400425, "epoch": 0.15006002400960383, "frac_reward_zero_std": 0.25, "grad_norm": 0.4879244565963745, "learning_rate": 5.04e-07, "loss": 0.0299, "num_tokens": 44300284.0, "reward": 8.492034912109375, "reward_std": 24.353479385375977, "rewards/reward_func/mean": 8.492034912109375, "rewards/reward_func/std": 24.35348129272461, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.82053542137146, "sampling/importance_sampling_ratio/mean": 0.9900239109992981, "sampling/importance_sampling_ratio/min": 0.2887732684612274, "sampling/sampling_logp_difference/max": 1.242113471031189, "sampling/sampling_logp_difference/mean": 0.020131349563598633, "step": 125, "step_time": 262.3409889435861 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15917.0, "completions/max_terminated_length": 15917.0, "completions/mean_length": 9516.4375, "completions/mean_terminated_length": 9516.4375, "completions/min_length": 4847.0, "completions/min_terminated_length": 4847.0, "entropy": 0.2908124439418316, "epoch": 0.15126050420168066, "frac_reward_zero_std": 0.0, "grad_norm": 0.38954854011535645, "learning_rate": 5e-07, "loss": 0.0245, "num_tokens": 44634890.0, "reward": 1.9146395921707153, "reward_std": 2.8896992206573486, "rewards/reward_func/mean": 1.9146395921707153, "rewards/reward_func/std": 2.8896987438201904, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.4463016986846924, "sampling/importance_sampling_ratio/mean": 0.9897171854972839, "sampling/importance_sampling_ratio/min": 0.3122645914554596, "sampling/sampling_logp_difference/max": 1.1639044284820557, "sampling/sampling_logp_difference/mean": 0.020406031981110573, "step": 126, "step_time": 262.93810594966635 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15624.0, "completions/max_terminated_length": 15624.0, "completions/mean_length": 9908.875, "completions/mean_terminated_length": 9908.875, "completions/min_length": 6398.0, "completions/min_terminated_length": 6398.0, "entropy": 0.3054868131875992, "epoch": 0.1524609843937575, "frac_reward_zero_std": 0.25, "grad_norm": 0.3650784492492676, "learning_rate": 4.96e-07, "loss": 0.0082, "num_tokens": 44975030.0, "reward": 1.5860934257507324, "reward_std": 2.869269847869873, "rewards/reward_func/mean": 1.5860934257507324, "rewards/reward_func/std": 2.869270086288452, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.127547025680542, "sampling/importance_sampling_ratio/mean": 0.9892829060554504, "sampling/importance_sampling_ratio/min": 0.21681442856788635, "sampling/sampling_logp_difference/max": 1.5287134647369385, "sampling/sampling_logp_difference/mean": 0.02112971805036068, "step": 127, "step_time": 216.3248087487882 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14126.0, "completions/max_terminated_length": 14126.0, "completions/mean_length": 7167.53125, "completions/mean_terminated_length": 7167.53125, "completions/min_length": 5186.0, "completions/min_terminated_length": 5186.0, "entropy": 0.2729322211816907, "epoch": 0.15366146458583432, "frac_reward_zero_std": 0.0, "grad_norm": 0.44890305399894714, "learning_rate": 4.92e-07, "loss": 0.0657, "num_tokens": 45233063.0, "reward": 3.381237745285034, "reward_std": 3.5361828804016113, "rewards/reward_func/mean": 3.381237745285034, "rewards/reward_func/std": 3.5361826419830322, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.002379894256592, "sampling/importance_sampling_ratio/mean": 0.9904590845108032, "sampling/importance_sampling_ratio/min": 0.3677864670753479, "sampling/sampling_logp_difference/max": 1.0002528429031372, "sampling/sampling_logp_difference/mean": 0.019061625003814697, "step": 128, "step_time": 196.9817350585945 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15338.0, "completions/max_terminated_length": 15338.0, "completions/mean_length": 7408.9375, "completions/mean_terminated_length": 7408.9375, "completions/min_length": 2486.0, "completions/min_terminated_length": 2486.0, "entropy": 0.28559087216854095, "epoch": 0.15486194477791115, "frac_reward_zero_std": 0.0, "grad_norm": 0.4474518895149231, "learning_rate": 4.879999999999999e-07, "loss": 0.0143, "num_tokens": 45501781.0, "reward": 1.7653834819793701, "reward_std": 1.865670919418335, "rewards/reward_func/mean": 1.7653834819793701, "rewards/reward_func/std": 1.865670919418335, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.251772165298462, "sampling/importance_sampling_ratio/mean": 0.9898481369018555, "sampling/importance_sampling_ratio/min": 0.33571237325668335, "sampling/sampling_logp_difference/max": 1.0915005207061768, "sampling/sampling_logp_difference/mean": 0.020369449630379677, "step": 129, "step_time": 213.23672278015874 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 20480.0, "completions/max_terminated_length": 13311.0, "completions/mean_length": 10419.03125, "completions/mean_terminated_length": 9378.2412109375, "completions/min_length": 6891.0, "completions/min_terminated_length": 6891.0, "entropy": 0.288710068911314, "epoch": 0.15606242496998798, "frac_reward_zero_std": 0.0, "grad_norm": 0.37506240606307983, "learning_rate": 4.839999999999999e-07, "loss": -0.1003, "num_tokens": 45864014.0, "reward": 1.1006951332092285, "reward_std": 1.99079167842865, "rewards/reward_func/mean": 1.3180084228515625, "rewards/reward_func/std": 1.9662554264068604, "rewards/soft_overlong_punishment_reward/mean": -0.09375, "rewards/soft_overlong_punishment_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 2.094200849533081, "sampling/importance_sampling_ratio/mean": 0.9896755218505859, "sampling/importance_sampling_ratio/min": 3.3655967854429036e-05, "sampling/sampling_logp_difference/max": 10.299320220947266, "sampling/sampling_logp_difference/mean": 0.02032674476504326, "step": 130, "step_time": 249.37841368792579 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13651.0, "completions/max_terminated_length": 13651.0, "completions/mean_length": 10875.9375, "completions/mean_terminated_length": 10875.9375, "completions/min_length": 6579.0, "completions/min_terminated_length": 6579.0, "entropy": 0.27954767271876335, "epoch": 0.15726290516206481, "frac_reward_zero_std": 0.0, "grad_norm": 0.4141541123390198, "learning_rate": 4.8e-07, "loss": -0.0105, "num_tokens": 46248332.0, "reward": 0.5992684364318848, "reward_std": 0.9581432342529297, "rewards/reward_func/mean": 0.5992684364318848, "rewards/reward_func/std": 0.9581431746482849, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.2382140159606934, "sampling/importance_sampling_ratio/mean": 0.990227997303009, "sampling/importance_sampling_ratio/min": 0.3830912113189697, "sampling/sampling_logp_difference/max": 0.9594821929931641, "sampling/sampling_logp_difference/mean": 0.019494500011205673, "step": 131, "step_time": 450.12081778328866 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11920.0, "completions/max_terminated_length": 11920.0, "completions/mean_length": 7238.625, "completions/mean_terminated_length": 7238.625, "completions/min_length": 3557.0, "completions/min_terminated_length": 3557.0, "entropy": 0.2874604295939207, "epoch": 0.15846338535414164, "frac_reward_zero_std": 0.0, "grad_norm": 0.5339971780776978, "learning_rate": 4.76e-07, "loss": 0.0164, "num_tokens": 46507880.0, "reward": 1.9809516668319702, "reward_std": 1.8525370359420776, "rewards/reward_func/mean": 1.9809516668319702, "rewards/reward_func/std": 1.8525370359420776, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.424567699432373, "sampling/importance_sampling_ratio/mean": 0.9898762702941895, "sampling/importance_sampling_ratio/min": 0.3163217008113861, "sampling/sampling_logp_difference/max": 1.1509954929351807, "sampling/sampling_logp_difference/mean": 0.020033420994877815, "step": 132, "step_time": 163.30515668692533 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10824.0, "completions/max_terminated_length": 10824.0, "completions/mean_length": 8501.71875, "completions/mean_terminated_length": 8501.71875, "completions/min_length": 5532.0, "completions/min_terminated_length": 5532.0, "entropy": 0.2722770730033517, "epoch": 0.15966386554621848, "frac_reward_zero_std": 0.0, "grad_norm": 0.4209204614162445, "learning_rate": 4.7199999999999994e-07, "loss": 0.0654, "num_tokens": 46813647.0, "reward": 5.072177886962891, "reward_std": 4.542591094970703, "rewards/reward_func/mean": 5.072177886962891, "rewards/reward_func/std": 4.542591094970703, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.7433615922927856, "sampling/importance_sampling_ratio/mean": 0.9904061555862427, "sampling/importance_sampling_ratio/min": 0.09293912351131439, "sampling/sampling_logp_difference/max": 2.3758106231689453, "sampling/sampling_logp_difference/mean": 0.01888871379196644, "step": 133, "step_time": 183.62756524700671 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16072.0, "completions/max_terminated_length": 16072.0, "completions/mean_length": 11215.71875, "completions/mean_terminated_length": 11215.71875, "completions/min_length": 7240.0, "completions/min_terminated_length": 7240.0, "entropy": 0.22843076288700104, "epoch": 0.1608643457382953, "frac_reward_zero_std": 0.0, "grad_norm": 0.3479200005531311, "learning_rate": 4.68e-07, "loss": -0.0415, "num_tokens": 47226070.0, "reward": 0.25698286294937134, "reward_std": 0.8653408288955688, "rewards/reward_func/mean": 0.25698286294937134, "rewards/reward_func/std": 0.8653408288955688, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9921608567237854, "sampling/importance_sampling_ratio/min": 8.29739210672642e-09, "sampling/sampling_logp_difference/max": 18.607324600219727, "sampling/sampling_logp_difference/mean": 0.0165218785405159, "step": 134, "step_time": 278.68348298675846 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15346.0, "completions/max_terminated_length": 15346.0, "completions/mean_length": 9651.1875, "completions/mean_terminated_length": 9651.1875, "completions/min_length": 4686.0, "completions/min_terminated_length": 4686.0, "entropy": 0.2711868965998292, "epoch": 0.16206482593037214, "frac_reward_zero_std": 0.0, "grad_norm": 0.3969678580760956, "learning_rate": 4.64e-07, "loss": -0.0081, "num_tokens": 47570140.0, "reward": 2.622737407684326, "reward_std": 3.003697633743286, "rewards/reward_func/mean": 2.622737407684326, "rewards/reward_func/std": 3.003697395324707, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.7643277645111084, "sampling/importance_sampling_ratio/mean": 0.990485668182373, "sampling/importance_sampling_ratio/min": 0.2230183184146881, "sampling/sampling_logp_difference/max": 1.5005013942718506, "sampling/sampling_logp_difference/mean": 0.018908120691776276, "step": 135, "step_time": 213.85006742121186 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11030.0, "completions/max_terminated_length": 11030.0, "completions/mean_length": 6975.90625, "completions/mean_terminated_length": 6975.90625, "completions/min_length": 3032.0, "completions/min_terminated_length": 3032.0, "entropy": 0.307786175981164, "epoch": 0.16326530612244897, "frac_reward_zero_std": 0.0, "grad_norm": 0.47706398367881775, "learning_rate": 4.6e-07, "loss": 0.0439, "num_tokens": 47817641.0, "reward": 2.0658512115478516, "reward_std": 1.9194847345352173, "rewards/reward_func/mean": 2.0658512115478516, "rewards/reward_func/std": 1.9194847345352173, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.136054754257202, "sampling/importance_sampling_ratio/mean": 0.9892805218696594, "sampling/importance_sampling_ratio/min": 1.11404536667876e-15, "sampling/sampling_logp_difference/max": 34.43077850341797, "sampling/sampling_logp_difference/mean": 0.021540788933634758, "step": 136, "step_time": 174.63315520831384 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15303.0, "completions/max_terminated_length": 15303.0, "completions/mean_length": 10812.3125, "completions/mean_terminated_length": 10812.3125, "completions/min_length": 7491.0, "completions/min_terminated_length": 7491.0, "entropy": 0.2458400446921587, "epoch": 0.1644657863145258, "frac_reward_zero_std": 0.0, "grad_norm": 0.35999420285224915, "learning_rate": 4.56e-07, "loss": 0.0237, "num_tokens": 48214651.0, "reward": 2.3839621543884277, "reward_std": 3.7139806747436523, "rewards/reward_func/mean": 2.3839621543884277, "rewards/reward_func/std": 3.7139809131622314, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.1656391620635986, "sampling/importance_sampling_ratio/mean": 0.9914739727973938, "sampling/importance_sampling_ratio/min": 0.28416091203689575, "sampling/sampling_logp_difference/max": 1.2582145929336548, "sampling/sampling_logp_difference/mean": 0.017416341230273247, "step": 137, "step_time": 483.0681917546317 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15823.0, "completions/max_terminated_length": 15823.0, "completions/mean_length": 9706.625, "completions/mean_terminated_length": 9706.625, "completions/min_length": 3619.0, "completions/min_terminated_length": 3619.0, "entropy": 0.2981583010405302, "epoch": 0.16566626650660263, "frac_reward_zero_std": 0.25, "grad_norm": 0.3441935181617737, "learning_rate": 4.5199999999999997e-07, "loss": 0.0477, "num_tokens": 48558839.0, "reward": 1027.4560546875, "reward_std": 3014.18603515625, "rewards/reward_func/mean": 1027.4560546875, "rewards/reward_func/std": 3014.18603515625, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.989443838596344, "sampling/importance_sampling_ratio/min": 0.0003842052537947893, "sampling/sampling_logp_difference/max": 7.864333629608154, "sampling/sampling_logp_difference/mean": 0.020724810659885406, "step": 138, "step_time": 429.2948167860741 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11754.0, "completions/max_terminated_length": 11754.0, "completions/mean_length": 9398.84375, "completions/mean_terminated_length": 9398.84375, "completions/min_length": 6889.0, "completions/min_terminated_length": 6889.0, "entropy": 0.24972048588097095, "epoch": 0.16686674669867949, "frac_reward_zero_std": 0.0, "grad_norm": 0.5226235389709473, "learning_rate": 4.48e-07, "loss": 0.0647, "num_tokens": 48899354.0, "reward": 1.3123189210891724, "reward_std": 1.0608717203140259, "rewards/reward_func/mean": 1.3123189210891724, "rewards/reward_func/std": 1.0608716011047363, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.4963085651397705, "sampling/importance_sampling_ratio/mean": 0.9913288354873657, "sampling/importance_sampling_ratio/min": 0.014635493978857994, "sampling/sampling_logp_difference/max": 4.224305629730225, "sampling/sampling_logp_difference/mean": 0.017568625509738922, "step": 139, "step_time": 214.9732713351259 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14780.0, "completions/max_terminated_length": 14780.0, "completions/mean_length": 10126.9375, "completions/mean_terminated_length": 10126.9375, "completions/min_length": 5722.0, "completions/min_terminated_length": 5722.0, "entropy": 0.30815622955560684, "epoch": 0.16806722689075632, "frac_reward_zero_std": 0.0, "grad_norm": 0.43311142921447754, "learning_rate": 4.44e-07, "loss": 0.0193, "num_tokens": 49251000.0, "reward": 2.31791353225708, "reward_std": 2.4861373901367188, "rewards/reward_func/mean": 2.31791353225708, "rewards/reward_func/std": 2.4861373901367188, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9891684055328369, "sampling/importance_sampling_ratio/min": 0.002064452273771167, "sampling/sampling_logp_difference/max": 6.18289041519165, "sampling/sampling_logp_difference/mean": 0.02135949209332466, "step": 140, "step_time": 213.63927030388732 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15453.0, "completions/max_terminated_length": 15453.0, "completions/mean_length": 9498.9375, "completions/mean_terminated_length": 9498.9375, "completions/min_length": 5658.0, "completions/min_terminated_length": 5658.0, "entropy": 0.23229695297777653, "epoch": 0.16926770708283315, "frac_reward_zero_std": 0.25, "grad_norm": 0.30538368225097656, "learning_rate": 4.3999999999999997e-07, "loss": -0.0339, "num_tokens": 49601214.0, "reward": 0.6548795700073242, "reward_std": 0.8164750337600708, "rewards/reward_func/mean": 0.6548795700073242, "rewards/reward_func/std": 0.8164750337600708, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.2676289081573486, "sampling/importance_sampling_ratio/mean": 0.9919700622558594, "sampling/importance_sampling_ratio/min": 0.0972149670124054, "sampling/sampling_logp_difference/max": 2.3308305740356445, "sampling/sampling_logp_difference/mean": 0.016680430620908737, "step": 141, "step_time": 231.1800272183027 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14022.0, "completions/max_terminated_length": 14022.0, "completions/mean_length": 8557.125, "completions/mean_terminated_length": 8557.125, "completions/min_length": 4786.0, "completions/min_terminated_length": 4786.0, "entropy": 0.27586540672928095, "epoch": 0.17046818727490998, "frac_reward_zero_std": 0.0, "grad_norm": 0.3966992199420929, "learning_rate": 4.36e-07, "loss": -0.0055, "num_tokens": 49905418.0, "reward": 0.8080017566680908, "reward_std": 1.4851527214050293, "rewards/reward_func/mean": 0.8080017566680908, "rewards/reward_func/std": 1.4851524829864502, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.329014539718628, "sampling/importance_sampling_ratio/mean": 0.9903725981712341, "sampling/importance_sampling_ratio/min": 0.3647924065589905, "sampling/sampling_logp_difference/max": 1.0084269046783447, "sampling/sampling_logp_difference/mean": 0.01935010962188244, "step": 142, "step_time": 201.8291045529768 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10552.0, "completions/max_terminated_length": 10552.0, "completions/mean_length": 8153.9375, "completions/mean_terminated_length": 8153.9375, "completions/min_length": 5045.0, "completions/min_terminated_length": 5045.0, "entropy": 0.260047709569335, "epoch": 0.1716686674669868, "frac_reward_zero_std": 0.25, "grad_norm": 0.3805393874645233, "learning_rate": 4.3199999999999995e-07, "loss": -0.0278, "num_tokens": 50202872.0, "reward": 0.9525775909423828, "reward_std": 1.2389754056930542, "rewards/reward_func/mean": 0.9525775909423828, "rewards/reward_func/std": 1.2389752864837646, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.504868507385254, "sampling/importance_sampling_ratio/mean": 0.9909122586250305, "sampling/importance_sampling_ratio/min": 0.055922865867614746, "sampling/sampling_logp_difference/max": 2.883781909942627, "sampling/sampling_logp_difference/mean": 0.018256302922964096, "step": 143, "step_time": 171.05576233169995 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 19289.0, "completions/max_terminated_length": 19289.0, "completions/mean_length": 10663.5625, "completions/mean_terminated_length": 10663.5625, "completions/min_length": 5378.0, "completions/min_terminated_length": 5378.0, "entropy": 0.2466154731810093, "epoch": 0.17286914765906364, "frac_reward_zero_std": 0.0, "grad_norm": 0.34512680768966675, "learning_rate": 4.2799999999999997e-07, "loss": 0.0035, "num_tokens": 50586930.0, "reward": 2.0170817375183105, "reward_std": 5.622786998748779, "rewards/reward_func/mean": 2.039245128631592, "rewards/reward_func/std": 5.613083362579346, "rewards/soft_overlong_punishment_reward/mean": -0.02216339111328125, "rewards/soft_overlong_punishment_reward/std": 0.12537507712841034, "sampling/importance_sampling_ratio/max": 2.3612794876098633, "sampling/importance_sampling_ratio/mean": 0.9915375113487244, "sampling/importance_sampling_ratio/min": 0.20737797021865845, "sampling/sampling_logp_difference/max": 1.5732121467590332, "sampling/sampling_logp_difference/mean": 0.017183100804686546, "step": 144, "step_time": 274.66985757567454 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14072.0, "completions/max_terminated_length": 14072.0, "completions/mean_length": 10114.15625, "completions/mean_terminated_length": 10114.15625, "completions/min_length": 5891.0, "completions/min_terminated_length": 5891.0, "entropy": 0.2593291876837611, "epoch": 0.17406962785114047, "frac_reward_zero_std": 0.0, "grad_norm": 0.4165084958076477, "learning_rate": 4.24e-07, "loss": -0.0155, "num_tokens": 50952575.0, "reward": 0.6612014174461365, "reward_std": 0.6770195364952087, "rewards/reward_func/mean": 0.6612014174461365, "rewards/reward_func/std": 0.6770195364952087, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.3497302532196045, "sampling/importance_sampling_ratio/mean": 0.9909900426864624, "sampling/importance_sampling_ratio/min": 0.00037760011036880314, "sampling/sampling_logp_difference/max": 7.881674766540527, "sampling/sampling_logp_difference/mean": 0.018293218687176704, "step": 145, "step_time": 220.56290533288848 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15097.0, "completions/max_terminated_length": 15097.0, "completions/mean_length": 10543.625, "completions/mean_terminated_length": 10543.625, "completions/min_length": 5699.0, "completions/min_terminated_length": 5699.0, "entropy": 0.2614777237176895, "epoch": 0.1752701080432173, "frac_reward_zero_std": 0.25, "grad_norm": 0.3072114884853363, "learning_rate": 4.1999999999999995e-07, "loss": 0.0314, "num_tokens": 51330571.0, "reward": 0.5684531927108765, "reward_std": 0.9393928647041321, "rewards/reward_func/mean": 0.5684531927108765, "rewards/reward_func/std": 0.9393928050994873, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.8473550081253052, "sampling/importance_sampling_ratio/mean": 0.9907210469245911, "sampling/importance_sampling_ratio/min": 0.07700582593679428, "sampling/sampling_logp_difference/max": 2.5638742446899414, "sampling/sampling_logp_difference/mean": 0.01846243068575859, "step": 146, "step_time": 453.85375829122495 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13158.0, "completions/max_terminated_length": 13158.0, "completions/mean_length": 8469.71875, "completions/mean_terminated_length": 8469.71875, "completions/min_length": 3331.0, "completions/min_terminated_length": 3331.0, "entropy": 0.29093294590711594, "epoch": 0.17647058823529413, "frac_reward_zero_std": 0.0, "grad_norm": 0.4070035517215729, "learning_rate": 4.1599999999999997e-07, "loss": 0.0461, "num_tokens": 51637050.0, "reward": 2.5153932571411133, "reward_std": 2.769888401031494, "rewards/reward_func/mean": 2.5153932571411133, "rewards/reward_func/std": 2.769888401031494, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0668880939483643, "sampling/importance_sampling_ratio/mean": 0.9898422956466675, "sampling/importance_sampling_ratio/min": 0.24710887670516968, "sampling/sampling_logp_difference/max": 1.3979263305664062, "sampling/sampling_logp_difference/mean": 0.0202447809278965, "step": 147, "step_time": 189.9260320410831 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14203.0, "completions/max_terminated_length": 14203.0, "completions/mean_length": 8437.125, "completions/mean_terminated_length": 8437.125, "completions/min_length": 4104.0, "completions/min_terminated_length": 4104.0, "entropy": 0.2681101858615875, "epoch": 0.17767106842737096, "frac_reward_zero_std": 0.25, "grad_norm": 0.39123448729515076, "learning_rate": 4.12e-07, "loss": 0.022, "num_tokens": 51937598.0, "reward": 2.7620649337768555, "reward_std": 3.608797073364258, "rewards/reward_func/mean": 2.7620649337768555, "rewards/reward_func/std": 3.608797073364258, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.5222227573394775, "sampling/importance_sampling_ratio/mean": 0.9908010959625244, "sampling/importance_sampling_ratio/min": 0.23630328476428986, "sampling/sampling_logp_difference/max": 1.4426392316818237, "sampling/sampling_logp_difference/mean": 0.018574317917227745, "step": 148, "step_time": 205.4168942749966 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 17113.0, "completions/max_terminated_length": 17113.0, "completions/mean_length": 9732.375, "completions/mean_terminated_length": 9732.375, "completions/min_length": 5745.0, "completions/min_terminated_length": 5745.0, "entropy": 0.28236152697354555, "epoch": 0.1788715486194478, "frac_reward_zero_std": 0.0, "grad_norm": 0.41433754563331604, "learning_rate": 4.0799999999999995e-07, "loss": -0.0364, "num_tokens": 52280586.0, "reward": 1.7899069786071777, "reward_std": 2.3258073329925537, "rewards/reward_func/mean": 1.799375057220459, "rewards/reward_func/std": 2.324876070022583, "rewards/soft_overlong_punishment_reward/mean": -0.00946807861328125, "rewards/soft_overlong_punishment_reward/std": 0.03785913065075874, "sampling/importance_sampling_ratio/max": 1.7942250967025757, "sampling/importance_sampling_ratio/mean": 0.9903931021690369, "sampling/importance_sampling_ratio/min": 0.028741903603076935, "sampling/sampling_logp_difference/max": 3.5493991374969482, "sampling/sampling_logp_difference/mean": 0.01913137547671795, "step": 149, "step_time": 246.32968526915647 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11989.0, "completions/max_terminated_length": 11989.0, "completions/mean_length": 6893.03125, "completions/mean_terminated_length": 6893.03125, "completions/min_length": 2952.0, "completions/min_terminated_length": 2952.0, "entropy": 0.3159897867590189, "epoch": 0.18007202881152462, "frac_reward_zero_std": 0.0, "grad_norm": 0.4671184718608856, "learning_rate": 4.04e-07, "loss": 0.022, "num_tokens": 52520571.0, "reward": 2.152081251144409, "reward_std": 2.514601707458496, "rewards/reward_func/mean": 2.152081251144409, "rewards/reward_func/std": 2.514601707458496, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.3258213996887207, "sampling/importance_sampling_ratio/mean": 0.9888448715209961, "sampling/importance_sampling_ratio/min": 0.2504069209098816, "sampling/sampling_logp_difference/max": 1.384667992591858, "sampling/sampling_logp_difference/mean": 0.021897951140999794, "step": 150, "step_time": 195.26920710073318 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12887.0, "completions/max_terminated_length": 12887.0, "completions/mean_length": 8146.09375, "completions/mean_terminated_length": 8146.09375, "completions/min_length": 4733.0, "completions/min_terminated_length": 4733.0, "entropy": 0.2663646712899208, "epoch": 0.18127250900360145, "frac_reward_zero_std": 0.0, "grad_norm": 0.4627015292644501, "learning_rate": 4e-07, "loss": -0.0405, "num_tokens": 52819142.0, "reward": 1.819735050201416, "reward_std": 2.4615325927734375, "rewards/reward_func/mean": 1.819735050201416, "rewards/reward_func/std": 2.4615325927734375, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.3554580211639404, "sampling/importance_sampling_ratio/mean": 0.9907447695732117, "sampling/importance_sampling_ratio/min": 0.12350412458181381, "sampling/sampling_logp_difference/max": 2.0914807319641113, "sampling/sampling_logp_difference/mean": 0.01854807324707508, "step": 151, "step_time": 201.55244890286122 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16359.0, "completions/max_terminated_length": 16359.0, "completions/mean_length": 9608.84375, "completions/mean_terminated_length": 9608.84375, "completions/min_length": 4375.0, "completions/min_terminated_length": 4375.0, "entropy": 0.2643621703609824, "epoch": 0.18247298919567828, "frac_reward_zero_std": 0.0, "grad_norm": 0.4149344563484192, "learning_rate": 3.96e-07, "loss": 0.0057, "num_tokens": 53164609.0, "reward": 0.8813707828521729, "reward_std": 1.7989546060562134, "rewards/reward_func/mean": 0.8813707828521729, "rewards/reward_func/std": 1.7989544868469238, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.488729476928711, "sampling/importance_sampling_ratio/mean": 0.9908096790313721, "sampling/importance_sampling_ratio/min": 0.10281366854906082, "sampling/sampling_logp_difference/max": 2.274837017059326, "sampling/sampling_logp_difference/mean": 0.018455805256962776, "step": 152, "step_time": 226.86713050387334 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 20480.0, "completions/max_terminated_length": 10407.0, "completions/mean_length": 8275.34375, "completions/mean_terminated_length": 7881.64501953125, "completions/min_length": 5786.0, "completions/min_terminated_length": 5786.0, "entropy": 0.2822192329913378, "epoch": 0.1836734693877551, "frac_reward_zero_std": 0.0, "grad_norm": 0.38609081506729126, "learning_rate": 3.92e-07, "loss": -0.0484, "num_tokens": 53458380.0, "reward": 2.3562521934509277, "reward_std": 3.1254520416259766, "rewards/reward_func/mean": 2.4645183086395264, "rewards/reward_func/std": 3.11552095413208, "rewards/soft_overlong_punishment_reward/mean": -0.03125, "rewards/soft_overlong_punishment_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.7898873090744019, "sampling/importance_sampling_ratio/mean": 0.990231454372406, "sampling/importance_sampling_ratio/min": 0.25040626525878906, "sampling/sampling_logp_difference/max": 1.384670615196228, "sampling/sampling_logp_difference/mean": 0.019599847495555878, "step": 153, "step_time": 236.71146991685964 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14804.0, "completions/max_terminated_length": 14804.0, "completions/mean_length": 9698.5625, "completions/mean_terminated_length": 9698.5625, "completions/min_length": 4843.0, "completions/min_terminated_length": 4843.0, "entropy": 0.2661178121343255, "epoch": 0.18487394957983194, "frac_reward_zero_std": 0.0, "grad_norm": 0.38603511452674866, "learning_rate": 3.88e-07, "loss": -0.0049, "num_tokens": 53810886.0, "reward": 0.7766771912574768, "reward_std": 1.2446486949920654, "rewards/reward_func/mean": 0.7766771912574768, "rewards/reward_func/std": 1.2446486949920654, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.604177951812744, "sampling/importance_sampling_ratio/mean": 0.9907273054122925, "sampling/importance_sampling_ratio/min": 2.992864028783515e-05, "sampling/sampling_logp_difference/max": 10.416694641113281, "sampling/sampling_logp_difference/mean": 0.018732445314526558, "step": 154, "step_time": 311.80545009789057 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13282.0, "completions/max_terminated_length": 13282.0, "completions/mean_length": 10236.65625, "completions/mean_terminated_length": 10236.65625, "completions/min_length": 6620.0, "completions/min_terminated_length": 6620.0, "entropy": 0.23385445773601532, "epoch": 0.18607442977190877, "frac_reward_zero_std": 0.0, "grad_norm": 0.34655675292015076, "learning_rate": 3.84e-07, "loss": 0.0323, "num_tokens": 54183971.0, "reward": 1.4783434867858887, "reward_std": 4.052839756011963, "rewards/reward_func/mean": 1.4783434867858887, "rewards/reward_func/std": 4.052839756011963, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.1445651054382324, "sampling/importance_sampling_ratio/mean": 0.9918794631958008, "sampling/importance_sampling_ratio/min": 0.05829664319753647, "sampling/sampling_logp_difference/max": 2.8422107696533203, "sampling/sampling_logp_difference/mean": 0.0164381992071867, "step": 155, "step_time": 215.27548193011899 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13371.0, "completions/max_terminated_length": 13371.0, "completions/mean_length": 8997.625, "completions/mean_terminated_length": 8997.625, "completions/min_length": 5840.0, "completions/min_terminated_length": 5840.0, "entropy": 0.23230466898530722, "epoch": 0.1872749099639856, "frac_reward_zero_std": 0.0, "grad_norm": 0.3713797628879547, "learning_rate": 3.7999999999999996e-07, "loss": 0.0149, "num_tokens": 54512903.0, "reward": 1.508521318435669, "reward_std": 2.5881083011627197, "rewards/reward_func/mean": 1.508521318435669, "rewards/reward_func/std": 2.5881083011627197, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9918398857116699, "sampling/importance_sampling_ratio/min": 0.12671518325805664, "sampling/sampling_logp_difference/max": 2.0658133029937744, "sampling/sampling_logp_difference/mean": 0.016411125659942627, "step": 156, "step_time": 255.3710762731498 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12916.0, "completions/max_terminated_length": 12916.0, "completions/mean_length": 8303.0625, "completions/mean_terminated_length": 8303.0625, "completions/min_length": 3105.0, "completions/min_terminated_length": 3105.0, "entropy": 0.2881615702062845, "epoch": 0.18847539015606243, "frac_reward_zero_std": 0.0, "grad_norm": 0.39465340971946716, "learning_rate": 3.76e-07, "loss": 0.0027, "num_tokens": 54806233.0, "reward": 2.848686456680298, "reward_std": 3.52091646194458, "rewards/reward_func/mean": 2.848686456680298, "rewards/reward_func/std": 3.520916700363159, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.2623817920684814, "sampling/importance_sampling_ratio/mean": 0.9897094964981079, "sampling/importance_sampling_ratio/min": 0.13919180631637573, "sampling/sampling_logp_difference/max": 1.9719023704528809, "sampling/sampling_logp_difference/mean": 0.020325226709246635, "step": 157, "step_time": 223.82761657808442 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13680.0, "completions/max_terminated_length": 13680.0, "completions/mean_length": 8154.1875, "completions/mean_terminated_length": 8154.1875, "completions/min_length": 4297.0, "completions/min_terminated_length": 4297.0, "entropy": 0.31936432234942913, "epoch": 0.18967587034813926, "frac_reward_zero_std": 0.25, "grad_norm": 0.40818971395492554, "learning_rate": 3.72e-07, "loss": -0.0041, "num_tokens": 55087247.0, "reward": 0.5234838724136353, "reward_std": 0.6490806937217712, "rewards/reward_func/mean": 0.5234838724136353, "rewards/reward_func/std": 0.6490806341171265, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9888770580291748, "sampling/importance_sampling_ratio/min": 0.18931546807289124, "sampling/sampling_logp_difference/max": 1.6643404960632324, "sampling/sampling_logp_difference/mean": 0.021932730451226234, "step": 158, "step_time": 218.13758707989473 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15527.0, "completions/max_terminated_length": 15527.0, "completions/mean_length": 10728.09375, "completions/mean_terminated_length": 10728.09375, "completions/min_length": 4513.0, "completions/min_terminated_length": 4513.0, "entropy": 0.3004476949572563, "epoch": 0.1908763505402161, "frac_reward_zero_std": 0.0, "grad_norm": 0.3766877353191376, "learning_rate": 3.6799999999999996e-07, "loss": 0.0295, "num_tokens": 55460634.0, "reward": 18.52477264404297, "reward_std": 40.702205657958984, "rewards/reward_func/mean": 18.52477264404297, "rewards/reward_func/std": 40.702205657958984, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.10101580619812, "sampling/importance_sampling_ratio/mean": 0.9893911480903625, "sampling/importance_sampling_ratio/min": 0.34575095772743225, "sampling/sampling_logp_difference/max": 1.0620365142822266, "sampling/sampling_logp_difference/mean": 0.021128207445144653, "step": 159, "step_time": 454.3971255911747 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16856.0, "completions/max_terminated_length": 16856.0, "completions/mean_length": 10528.46875, "completions/mean_terminated_length": 10528.46875, "completions/min_length": 7584.0, "completions/min_terminated_length": 7584.0, "entropy": 0.24533494841307402, "epoch": 0.19207683073229292, "frac_reward_zero_std": 0.0, "grad_norm": 0.3707125782966614, "learning_rate": 3.64e-07, "loss": 0.04, "num_tokens": 55848537.0, "reward": 1.3989126682281494, "reward_std": 0.9870246648788452, "rewards/reward_func/mean": 1.4025137424468994, "rewards/reward_func/std": 0.9815170168876648, "rewards/soft_overlong_punishment_reward/mean": -0.00360107421875, "rewards/soft_overlong_punishment_reward/std": 0.02037075161933899, "sampling/importance_sampling_ratio/max": 2.11053729057312, "sampling/importance_sampling_ratio/mean": 0.9915527105331421, "sampling/importance_sampling_ratio/min": 0.2704220414161682, "sampling/sampling_logp_difference/max": 1.3077714443206787, "sampling/sampling_logp_difference/mean": 0.017412563785910606, "step": 160, "step_time": 272.99579531908967 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10456.0, "completions/max_terminated_length": 10456.0, "completions/mean_length": 6813.90625, "completions/mean_terminated_length": 6813.90625, "completions/min_length": 4551.0, "completions/min_terminated_length": 4551.0, "entropy": 0.27734176721423864, "epoch": 0.19327731092436976, "frac_reward_zero_std": 0.0, "grad_norm": 0.48373842239379883, "learning_rate": 3.6e-07, "loss": -0.0433, "num_tokens": 56091246.0, "reward": 0.9251768589019775, "reward_std": 1.2734801769256592, "rewards/reward_func/mean": 0.9251768589019775, "rewards/reward_func/std": 1.2734800577163696, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.068244218826294, "sampling/importance_sampling_ratio/mean": 0.990391194820404, "sampling/importance_sampling_ratio/min": 0.18318888545036316, "sampling/sampling_logp_difference/max": 1.697237491607666, "sampling/sampling_logp_difference/mean": 0.01954927295446396, "step": 161, "step_time": 222.0121338494355 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15822.0, "completions/max_terminated_length": 15822.0, "completions/mean_length": 9338.15625, "completions/mean_terminated_length": 9338.15625, "completions/min_length": 4551.0, "completions/min_terminated_length": 4551.0, "entropy": 0.25283763464540243, "epoch": 0.19447779111644659, "frac_reward_zero_std": 0.25, "grad_norm": 0.3540862202644348, "learning_rate": 3.5599999999999996e-07, "loss": -0.0236, "num_tokens": 56431251.0, "reward": 2.1091599464416504, "reward_std": 3.2207794189453125, "rewards/reward_func/mean": 2.1091599464416504, "rewards/reward_func/std": 3.2207798957824707, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.2128798961639404, "sampling/importance_sampling_ratio/mean": 0.9914829730987549, "sampling/importance_sampling_ratio/min": 0.43283000588417053, "sampling/sampling_logp_difference/max": 0.8374102115631104, "sampling/sampling_logp_difference/mean": 0.017802974209189415, "step": 162, "step_time": 242.35264018306043 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13122.0, "completions/max_terminated_length": 13122.0, "completions/mean_length": 9301.21875, "completions/mean_terminated_length": 9301.21875, "completions/min_length": 6263.0, "completions/min_terminated_length": 6263.0, "entropy": 0.2940525095909834, "epoch": 0.19567827130852342, "frac_reward_zero_std": 0.0, "grad_norm": 0.39872726798057556, "learning_rate": 3.52e-07, "loss": 0.0262, "num_tokens": 56764610.0, "reward": 1.4544447660446167, "reward_std": 1.857795238494873, "rewards/reward_func/mean": 1.4544447660446167, "rewards/reward_func/std": 1.857795238494873, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9896678924560547, "sampling/importance_sampling_ratio/min": 0.36473485827445984, "sampling/sampling_logp_difference/max": 1.1077795028686523, "sampling/sampling_logp_difference/mean": 0.020575741305947304, "step": 163, "step_time": 434.3237003698014 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13825.0, "completions/max_terminated_length": 13825.0, "completions/mean_length": 9424.21875, "completions/mean_terminated_length": 9424.21875, "completions/min_length": 5781.0, "completions/min_terminated_length": 5781.0, "entropy": 0.3086765222251415, "epoch": 0.19687875150060025, "frac_reward_zero_std": 0.0, "grad_norm": 0.40642544627189636, "learning_rate": 3.4799999999999994e-07, "loss": 0.034, "num_tokens": 57091961.0, "reward": 66378.796875, "reward_std": 375489.625, "rewards/reward_func/mean": 66378.796875, "rewards/reward_func/std": 375489.625, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.98918217420578, "sampling/importance_sampling_ratio/min": 0.039938684552907944, "sampling/sampling_logp_difference/max": 3.220409870147705, "sampling/sampling_logp_difference/mean": 0.02147614024579525, "step": 164, "step_time": 258.5584741720231 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16400.0, "completions/max_terminated_length": 16400.0, "completions/mean_length": 9196.0, "completions/mean_terminated_length": 9196.0, "completions/min_length": 3739.0, "completions/min_terminated_length": 3739.0, "entropy": 0.2833999376744032, "epoch": 0.19807923169267708, "frac_reward_zero_std": 0.0, "grad_norm": 1.0096226930618286, "learning_rate": 3.4399999999999996e-07, "loss": -0.0008, "num_tokens": 57423017.0, "reward": 0.45536255836486816, "reward_std": 0.5211973190307617, "rewards/reward_func/mean": 0.45548462867736816, "rewards/reward_func/std": 0.521098792552948, "rewards/soft_overlong_punishment_reward/mean": -0.0001220703125, "rewards/soft_overlong_punishment_reward/std": 0.0006905339541845024, "sampling/importance_sampling_ratio/max": 2.0532639026641846, "sampling/importance_sampling_ratio/mean": 0.9901537895202637, "sampling/importance_sampling_ratio/min": 0.2596132457256317, "sampling/sampling_logp_difference/max": 1.348562240600586, "sampling/sampling_logp_difference/mean": 0.019828852266073227, "step": 165, "step_time": 268.0153879421996 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15331.0, "completions/max_terminated_length": 15331.0, "completions/mean_length": 10289.25, "completions/mean_terminated_length": 10289.25, "completions/min_length": 7699.0, "completions/min_terminated_length": 7699.0, "entropy": 0.2959921294823289, "epoch": 0.1992797118847539, "frac_reward_zero_std": 0.0, "grad_norm": 0.3890758454799652, "learning_rate": 3.4000000000000003e-07, "loss": -0.0129, "num_tokens": 57786057.0, "reward": 0.2713179886341095, "reward_std": 0.6150152087211609, "rewards/reward_func/mean": 0.2713179886341095, "rewards/reward_func/std": 0.6150152087211609, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9894802570343018, "sampling/importance_sampling_ratio/min": 0.03638508915901184, "sampling/sampling_logp_difference/max": 3.313596248626709, "sampling/sampling_logp_difference/mean": 0.02085329219698906, "step": 166, "step_time": 255.01892217074055 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16600.0, "completions/max_terminated_length": 16600.0, "completions/mean_length": 9496.1875, "completions/mean_terminated_length": 9496.1875, "completions/min_length": 5800.0, "completions/min_terminated_length": 5800.0, "entropy": 0.2720292443409562, "epoch": 0.20048019207683074, "frac_reward_zero_std": 0.0, "grad_norm": 0.3967978358268738, "learning_rate": 3.36e-07, "loss": 0.0574, "num_tokens": 58128367.0, "reward": 2.866912841796875, "reward_std": 3.3676812648773193, "rewards/reward_func/mean": 2.868560791015625, "rewards/reward_func/std": 3.367079496383667, "rewards/soft_overlong_punishment_reward/mean": -0.00164794921875, "rewards/soft_overlong_punishment_reward/std": 0.009322208352386951, "sampling/importance_sampling_ratio/max": 2.074486494064331, "sampling/importance_sampling_ratio/mean": 0.9905571937561035, "sampling/importance_sampling_ratio/min": 0.25333765149116516, "sampling/sampling_logp_difference/max": 1.3730320930480957, "sampling/sampling_logp_difference/mean": 0.019127877429127693, "step": 167, "step_time": 239.48420156200882 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12687.0, "completions/max_terminated_length": 12687.0, "completions/mean_length": 8358.1875, "completions/mean_terminated_length": 8358.1875, "completions/min_length": 5265.0, "completions/min_terminated_length": 5265.0, "entropy": 0.28812102414667606, "epoch": 0.20168067226890757, "frac_reward_zero_std": 0.0, "grad_norm": 0.41675513982772827, "learning_rate": 3.32e-07, "loss": 0.0333, "num_tokens": 58430437.0, "reward": 2.597444534301758, "reward_std": 3.35589861869812, "rewards/reward_func/mean": 2.597444534301758, "rewards/reward_func/std": 3.355898857116699, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.186401128768921, "sampling/importance_sampling_ratio/mean": 0.9898570775985718, "sampling/importance_sampling_ratio/min": 0.055154554545879364, "sampling/sampling_logp_difference/max": 2.897615909576416, "sampling/sampling_logp_difference/mean": 0.020029492676258087, "step": 168, "step_time": 226.04919587192126 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12104.0, "completions/max_terminated_length": 12104.0, "completions/mean_length": 8018.03125, "completions/mean_terminated_length": 8018.03125, "completions/min_length": 3307.0, "completions/min_terminated_length": 3307.0, "entropy": 0.3021853733807802, "epoch": 0.2028811524609844, "frac_reward_zero_std": 0.0, "grad_norm": 0.4631645083427429, "learning_rate": 3.28e-07, "loss": 0.0106, "num_tokens": 58713550.0, "reward": 0.6359751224517822, "reward_std": 0.6328838467597961, "rewards/reward_func/mean": 0.6359751224517822, "rewards/reward_func/std": 0.6328837871551514, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.1259548664093018, "sampling/importance_sampling_ratio/mean": 0.9894533157348633, "sampling/importance_sampling_ratio/min": 0.005165865179151297, "sampling/sampling_logp_difference/max": 5.265682697296143, "sampling/sampling_logp_difference/mean": 0.020912254229187965, "step": 169, "step_time": 182.4748994258698 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12901.0, "completions/max_terminated_length": 12901.0, "completions/mean_length": 9577.5625, "completions/mean_terminated_length": 9577.5625, "completions/min_length": 5991.0, "completions/min_terminated_length": 5991.0, "entropy": 0.2918419409543276, "epoch": 0.20408163265306123, "frac_reward_zero_std": 0.0, "grad_norm": 0.4238099157810211, "learning_rate": 3.24e-07, "loss": 0.0158, "num_tokens": 59055152.0, "reward": 1.1994872093200684, "reward_std": 1.4579551219940186, "rewards/reward_func/mean": 1.1994872093200684, "rewards/reward_func/std": 1.457955241203308, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.3555917739868164, "sampling/importance_sampling_ratio/mean": 0.9899040460586548, "sampling/importance_sampling_ratio/min": 0.15884944796562195, "sampling/sampling_logp_difference/max": 1.8397983312606812, "sampling/sampling_logp_difference/mean": 0.020075272768735886, "step": 170, "step_time": 205.09224270377308 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11374.0, "completions/max_terminated_length": 11374.0, "completions/mean_length": 7372.3125, "completions/mean_terminated_length": 7372.3125, "completions/min_length": 2447.0, "completions/min_terminated_length": 2447.0, "entropy": 0.2957879761233926, "epoch": 0.20528211284513806, "frac_reward_zero_std": 0.0, "grad_norm": 0.4720025956630707, "learning_rate": 3.2e-07, "loss": 0.0587, "num_tokens": 59316858.0, "reward": 2.2769434452056885, "reward_std": 2.0495495796203613, "rewards/reward_func/mean": 2.2769434452056885, "rewards/reward_func/std": 2.0495495796203613, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.989801287651062, "sampling/importance_sampling_ratio/min": 0.36141398549079895, "sampling/sampling_logp_difference/max": 1.1415172815322876, "sampling/sampling_logp_difference/mean": 0.020276790484786034, "step": 171, "step_time": 171.338580377982 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13035.0, "completions/max_terminated_length": 13035.0, "completions/mean_length": 9139.9375, "completions/mean_terminated_length": 9139.9375, "completions/min_length": 6085.0, "completions/min_terminated_length": 6085.0, "entropy": 0.265663193538785, "epoch": 0.2064825930372149, "frac_reward_zero_std": 0.0, "grad_norm": 0.4693741202354431, "learning_rate": 3.1599999999999997e-07, "loss": 0.0361, "num_tokens": 59651160.0, "reward": 1.905058741569519, "reward_std": 3.157362699508667, "rewards/reward_func/mean": 1.905058741569519, "rewards/reward_func/std": 3.157362699508667, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.7062559127807617, "sampling/importance_sampling_ratio/mean": 0.9907447695732117, "sampling/importance_sampling_ratio/min": 0.05720826983451843, "sampling/sampling_logp_difference/max": 2.8610568046569824, "sampling/sampling_logp_difference/mean": 0.018653716892004013, "step": 172, "step_time": 397.120246386854 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14177.0, "completions/max_terminated_length": 14177.0, "completions/mean_length": 9426.0625, "completions/mean_terminated_length": 9426.0625, "completions/min_length": 5614.0, "completions/min_terminated_length": 5614.0, "entropy": 0.2500729188323021, "epoch": 0.20768307322929172, "frac_reward_zero_std": 0.5, "grad_norm": 0.2702878415584564, "learning_rate": 3.12e-07, "loss": 0.0171, "num_tokens": 59992778.0, "reward": 1.3135590553283691, "reward_std": 2.5076005458831787, "rewards/reward_func/mean": 1.3135590553283691, "rewards/reward_func/std": 2.5076003074645996, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.058269500732422, "sampling/importance_sampling_ratio/mean": 0.9914534687995911, "sampling/importance_sampling_ratio/min": 0.10936449468135834, "sampling/sampling_logp_difference/max": 2.213068962097168, "sampling/sampling_logp_difference/mean": 0.01742243394255638, "step": 173, "step_time": 239.5976502839476 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12497.0, "completions/max_terminated_length": 12497.0, "completions/mean_length": 8828.0, "completions/mean_terminated_length": 8828.0, "completions/min_length": 6421.0, "completions/min_terminated_length": 6421.0, "entropy": 0.27684462163597345, "epoch": 0.20888355342136855, "frac_reward_zero_std": 0.0, "grad_norm": 0.3942298889160156, "learning_rate": 3.08e-07, "loss": 0.0059, "num_tokens": 60315146.0, "reward": 0.6179296970367432, "reward_std": 0.6792400479316711, "rewards/reward_func/mean": 0.6179296970367432, "rewards/reward_func/std": 0.6792401075363159, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9904294013977051, "sampling/importance_sampling_ratio/min": 0.2878604829311371, "sampling/sampling_logp_difference/max": 1.245279312133789, "sampling/sampling_logp_difference/mean": 0.01934416964650154, "step": 174, "step_time": 214.77978121873457 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 17360.0, "completions/max_terminated_length": 17360.0, "completions/mean_length": 9081.84375, "completions/mean_terminated_length": 9081.84375, "completions/min_length": 4942.0, "completions/min_terminated_length": 4942.0, "entropy": 0.2940607778728008, "epoch": 0.21008403361344538, "frac_reward_zero_std": 0.0, "grad_norm": 0.4100255072116852, "learning_rate": 3.0399999999999997e-07, "loss": 0.0503, "num_tokens": 60635101.0, "reward": 1.2948029041290283, "reward_std": 1.4059300422668457, "rewards/reward_func/mean": 1.3022491931915283, "rewards/reward_func/std": 1.3984326124191284, "rewards/soft_overlong_punishment_reward/mean": -0.0074462890625, "rewards/soft_overlong_punishment_reward/std": 0.04212257266044617, "sampling/importance_sampling_ratio/max": 2.523259162902832, "sampling/importance_sampling_ratio/mean": 0.9897422790527344, "sampling/importance_sampling_ratio/min": 0.2878084182739258, "sampling/sampling_logp_difference/max": 1.2454602718353271, "sampling/sampling_logp_difference/mean": 0.020703017711639404, "step": 175, "step_time": 467.0385398848448 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 20480.0, "completions/max_terminated_length": 20183.0, "completions/mean_length": 11330.46875, "completions/mean_terminated_length": 9219.0390625, "completions/min_length": 6454.0, "completions/min_terminated_length": 6454.0, "entropy": 0.2567200744524598, "epoch": 0.2112845138055222, "frac_reward_zero_std": 0.0, "grad_norm": 0.33059075474739075, "learning_rate": 3e-07, "loss": -0.0585, "num_tokens": 61020372.0, "reward": 1.0467814207077026, "reward_std": 1.7194699048995972, "rewards/reward_func/mean": 1.55478835105896, "rewards/reward_func/std": 1.5100483894348145, "rewards/soft_overlong_punishment_reward/mean": -0.21648406982421875, "rewards/soft_overlong_punishment_reward/std": 0.41583752632141113, "sampling/importance_sampling_ratio/max": 2.49721097946167, "sampling/importance_sampling_ratio/mean": 0.9905225038528442, "sampling/importance_sampling_ratio/min": 0.2589978873729706, "sampling/sampling_logp_difference/max": 1.3509353399276733, "sampling/sampling_logp_difference/mean": 0.019211364910006523, "step": 176, "step_time": 262.3387938350206 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13843.0, "completions/max_terminated_length": 13843.0, "completions/mean_length": 9019.90625, "completions/mean_terminated_length": 9019.90625, "completions/min_length": 5026.0, "completions/min_terminated_length": 5026.0, "entropy": 0.26270918548107147, "epoch": 0.21248499399759904, "frac_reward_zero_std": 0.0, "grad_norm": 0.40801480412483215, "learning_rate": 2.9599999999999995e-07, "loss": -0.0516, "num_tokens": 61349169.0, "reward": 0.7610185146331787, "reward_std": 0.6794523000717163, "rewards/reward_func/mean": 0.7610185146331787, "rewards/reward_func/std": 0.6794522404670715, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9909648895263672, "sampling/importance_sampling_ratio/min": 0.45922839641571045, "sampling/sampling_logp_difference/max": 1.4038057327270508, "sampling/sampling_logp_difference/mean": 0.0184706449508667, "step": 177, "step_time": 210.60857710335404 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13779.0, "completions/max_terminated_length": 13779.0, "completions/mean_length": 8056.8125, "completions/mean_terminated_length": 8056.8125, "completions/min_length": 4695.0, "completions/min_terminated_length": 4695.0, "entropy": 0.2336973724886775, "epoch": 0.21368547418967587, "frac_reward_zero_std": 0.0, "grad_norm": 0.4138389527797699, "learning_rate": 2.9199999999999997e-07, "loss": 0.0636, "num_tokens": 61643739.0, "reward": 0.7230340242385864, "reward_std": 1.2502714395523071, "rewards/reward_func/mean": 0.7230340242385864, "rewards/reward_func/std": 1.2502713203430176, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.6206603050231934, "sampling/importance_sampling_ratio/mean": 0.9920225143432617, "sampling/importance_sampling_ratio/min": 0.001480705919675529, "sampling/sampling_logp_difference/max": 6.5152363777160645, "sampling/sampling_logp_difference/mean": 0.016540315002202988, "step": 178, "step_time": 223.01532334729563 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 20367.0, "completions/max_terminated_length": 20367.0, "completions/mean_length": 9921.0625, "completions/mean_terminated_length": 9921.0625, "completions/min_length": 6193.0, "completions/min_terminated_length": 6193.0, "entropy": 0.2881326600909233, "epoch": 0.2148859543817527, "frac_reward_zero_std": 0.25, "grad_norm": 0.34104132652282715, "learning_rate": 2.88e-07, "loss": 0.05, "num_tokens": 61997757.0, "reward": 1.612834095954895, "reward_std": 2.867422580718994, "rewards/reward_func/mean": 1.6432219743728638, "rewards/reward_func/std": 2.8589067459106445, "rewards/soft_overlong_punishment_reward/mean": -0.03038787841796875, "rewards/soft_overlong_punishment_reward/std": 0.17189979553222656, "sampling/importance_sampling_ratio/max": 2.565842866897583, "sampling/importance_sampling_ratio/mean": 0.9899250268936157, "sampling/importance_sampling_ratio/min": 0.11633654683828354, "sampling/sampling_logp_difference/max": 2.1512680053710938, "sampling/sampling_logp_difference/mean": 0.020425235852599144, "step": 179, "step_time": 249.0168090411462 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12388.0, "completions/max_terminated_length": 12388.0, "completions/mean_length": 6812.59375, "completions/mean_terminated_length": 6812.59375, "completions/min_length": 2401.0, "completions/min_terminated_length": 2401.0, "entropy": 0.3213646560907364, "epoch": 0.21608643457382953, "frac_reward_zero_std": 0.0, "grad_norm": 0.510503888130188, "learning_rate": 2.8399999999999995e-07, "loss": -0.0106, "num_tokens": 62243096.0, "reward": 1.7355564832687378, "reward_std": 1.7355234622955322, "rewards/reward_func/mean": 1.7355564832687378, "rewards/reward_func/std": 1.7355234622955322, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.2783377170562744, "sampling/importance_sampling_ratio/mean": 0.9888811707496643, "sampling/importance_sampling_ratio/min": 0.27378326654434204, "sampling/sampling_logp_difference/max": 1.2954185009002686, "sampling/sampling_logp_difference/mean": 0.022066859528422356, "step": 180, "step_time": 233.19768030114938 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 20480.0, "completions/max_terminated_length": 17498.0, "completions/mean_length": 10577.9375, "completions/mean_terminated_length": 10258.515625, "completions/min_length": 5877.0, "completions/min_terminated_length": 5877.0, "entropy": 0.2779244324192405, "epoch": 0.21728691476590636, "frac_reward_zero_std": 0.0, "grad_norm": 0.350612074136734, "learning_rate": 2.8e-07, "loss": -0.0024, "num_tokens": 62611454.0, "reward": 2.4395365715026855, "reward_std": 4.4411797523498535, "rewards/reward_func/mean": 2.479285717010498, "rewards/reward_func/std": 4.416739463806152, "rewards/soft_overlong_punishment_reward/mean": -0.0397491455078125, "rewards/soft_overlong_punishment_reward/std": 0.1816953718662262, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9902195930480957, "sampling/importance_sampling_ratio/min": 0.2922537326812744, "sampling/sampling_logp_difference/max": 1.230132818222046, "sampling/sampling_logp_difference/mean": 0.01920732483267784, "step": 181, "step_time": 265.4475303606596 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12516.0, "completions/max_terminated_length": 12516.0, "completions/mean_length": 8191.75, "completions/mean_terminated_length": 8191.75, "completions/min_length": 4992.0, "completions/min_terminated_length": 4992.0, "entropy": 0.26552946865558624, "epoch": 0.2184873949579832, "frac_reward_zero_std": 0.0, "grad_norm": 0.4106624126434326, "learning_rate": 2.7600000000000004e-07, "loss": 0.0802, "num_tokens": 62906998.0, "reward": 2.137417793273926, "reward_std": 2.385087013244629, "rewards/reward_func/mean": 2.137417793273926, "rewards/reward_func/std": 2.385087013244629, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.4886276721954346, "sampling/importance_sampling_ratio/mean": 0.9908239245414734, "sampling/importance_sampling_ratio/min": 0.23087278008460999, "sampling/sampling_logp_difference/max": 1.465888500213623, "sampling/sampling_logp_difference/mean": 0.018649548292160034, "step": 182, "step_time": 180.85359141172376 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11870.0, "completions/max_terminated_length": 11870.0, "completions/mean_length": 8648.4375, "completions/mean_terminated_length": 8648.4375, "completions/min_length": 5557.0, "completions/min_terminated_length": 5557.0, "entropy": 0.2885238602757454, "epoch": 0.21968787515006002, "frac_reward_zero_std": 0.0, "grad_norm": 0.4420488178730011, "learning_rate": 2.72e-07, "loss": -0.0295, "num_tokens": 63213556.0, "reward": 0.8226311802864075, "reward_std": 0.6850433945655823, "rewards/reward_func/mean": 0.8226311802864075, "rewards/reward_func/std": 0.6850433945655823, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.508232593536377, "sampling/importance_sampling_ratio/mean": 0.989946722984314, "sampling/importance_sampling_ratio/min": 0.0002403110993327573, "sampling/sampling_logp_difference/max": 8.333576202392578, "sampling/sampling_logp_difference/mean": 0.02029946818947792, "step": 183, "step_time": 198.74373724497855 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15138.0, "completions/max_terminated_length": 15138.0, "completions/mean_length": 10048.90625, "completions/mean_terminated_length": 10048.90625, "completions/min_length": 5452.0, "completions/min_terminated_length": 5452.0, "entropy": 0.24506080709397793, "epoch": 0.22088835534213686, "frac_reward_zero_std": 0.0, "grad_norm": 0.3524847626686096, "learning_rate": 2.68e-07, "loss": 0.0155, "num_tokens": 63572233.0, "reward": 3.185716152191162, "reward_std": 4.615874290466309, "rewards/reward_func/mean": 3.185716152191162, "rewards/reward_func/std": 4.61587381362915, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9916706085205078, "sampling/importance_sampling_ratio/min": 0.018803074955940247, "sampling/sampling_logp_difference/max": 3.9737348556518555, "sampling/sampling_logp_difference/mean": 0.01703486405313015, "step": 184, "step_time": 224.22839038609527 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11046.0, "completions/max_terminated_length": 11046.0, "completions/mean_length": 7622.9375, "completions/mean_terminated_length": 7622.9375, "completions/min_length": 4460.0, "completions/min_terminated_length": 4460.0, "entropy": 0.2557719862088561, "epoch": 0.22208883553421369, "frac_reward_zero_std": 0.0, "grad_norm": 0.42476049065589905, "learning_rate": 2.64e-07, "loss": -0.0013, "num_tokens": 63848983.0, "reward": 4.9202985763549805, "reward_std": 5.301763534545898, "rewards/reward_func/mean": 4.9202985763549805, "rewards/reward_func/std": 5.301763534545898, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.9607212543487549, "sampling/importance_sampling_ratio/mean": 0.9911612868309021, "sampling/importance_sampling_ratio/min": 0.019983841106295586, "sampling/sampling_logp_difference/max": 3.9128313064575195, "sampling/sampling_logp_difference/mean": 0.017659198492765427, "step": 185, "step_time": 181.76806327118538 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13441.0, "completions/max_terminated_length": 13441.0, "completions/mean_length": 9717.0, "completions/mean_terminated_length": 9717.0, "completions/min_length": 5815.0, "completions/min_terminated_length": 5815.0, "entropy": 0.24156292621046305, "epoch": 0.22328931572629052, "frac_reward_zero_std": 0.0, "grad_norm": 0.442635715007782, "learning_rate": 2.6e-07, "loss": -0.0102, "num_tokens": 64210287.0, "reward": 1.863988995552063, "reward_std": 2.8825900554656982, "rewards/reward_func/mean": 1.863988995552063, "rewards/reward_func/std": 2.8825902938842773, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.648536443710327, "sampling/importance_sampling_ratio/mean": 0.9914799928665161, "sampling/importance_sampling_ratio/min": 0.027922935783863068, "sampling/sampling_logp_difference/max": 3.5783069133758545, "sampling/sampling_logp_difference/mean": 0.017204025760293007, "step": 186, "step_time": 238.74721054232214 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13385.0, "completions/max_terminated_length": 13385.0, "completions/mean_length": 10068.6875, "completions/mean_terminated_length": 10068.6875, "completions/min_length": 7343.0, "completions/min_terminated_length": 7343.0, "entropy": 0.2825509188696742, "epoch": 0.22448979591836735, "frac_reward_zero_std": 0.0, "grad_norm": 0.4277617335319519, "learning_rate": 2.56e-07, "loss": 0.0193, "num_tokens": 64569965.0, "reward": 0.8854573965072632, "reward_std": 0.6112152934074402, "rewards/reward_func/mean": 0.8854573965072632, "rewards/reward_func/std": 0.6112152934074402, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0112133026123047, "sampling/importance_sampling_ratio/mean": 0.9901423454284668, "sampling/importance_sampling_ratio/min": 0.16564348340034485, "sampling/sampling_logp_difference/max": 1.7979174852371216, "sampling/sampling_logp_difference/mean": 0.019815031439065933, "step": 187, "step_time": 228.7444448468741 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13373.0, "completions/max_terminated_length": 13373.0, "completions/mean_length": 9405.59375, "completions/mean_terminated_length": 9405.59375, "completions/min_length": 5967.0, "completions/min_terminated_length": 5967.0, "entropy": 0.318213177844882, "epoch": 0.22569027611044418, "frac_reward_zero_std": 0.25, "grad_norm": 0.33108896017074585, "learning_rate": 2.52e-07, "loss": 0.0167, "num_tokens": 64897776.0, "reward": 0.5405811071395874, "reward_std": 0.6606644988059998, "rewards/reward_func/mean": 0.5405811071395874, "rewards/reward_func/std": 0.6606644988059998, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9889894723892212, "sampling/importance_sampling_ratio/min": 0.19092479348182678, "sampling/sampling_logp_difference/max": 1.9465055465698242, "sampling/sampling_logp_difference/mean": 0.02206978015601635, "step": 188, "step_time": 415.85145999782253 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11114.0, "completions/max_terminated_length": 11114.0, "completions/mean_length": 8580.78125, "completions/mean_terminated_length": 8580.78125, "completions/min_length": 4516.0, "completions/min_terminated_length": 4516.0, "entropy": 0.3377720918506384, "epoch": 0.226890756302521, "frac_reward_zero_std": 0.0, "grad_norm": 0.42674168944358826, "learning_rate": 2.48e-07, "loss": 0.0088, "num_tokens": 65198441.0, "reward": 1.591444730758667, "reward_std": 2.4525747299194336, "rewards/reward_func/mean": 1.591444730758667, "rewards/reward_func/std": 2.4525744915008545, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6534841060638428, "sampling/importance_sampling_ratio/mean": 0.9881213903427124, "sampling/importance_sampling_ratio/min": 0.33775991201400757, "sampling/sampling_logp_difference/max": 1.0854198932647705, "sampling/sampling_logp_difference/mean": 0.022877736017107964, "step": 189, "step_time": 285.5526141769951 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10786.0, "completions/max_terminated_length": 10786.0, "completions/mean_length": 7476.28125, "completions/mean_terminated_length": 7476.28125, "completions/min_length": 4472.0, "completions/min_terminated_length": 4472.0, "entropy": 0.3017116552218795, "epoch": 0.22809123649459784, "frac_reward_zero_std": 0.0, "grad_norm": 0.41298624873161316, "learning_rate": 2.4399999999999996e-07, "loss": 0.049, "num_tokens": 65463538.0, "reward": 1.0010404586791992, "reward_std": 0.7883979678153992, "rewards/reward_func/mean": 1.0010404586791992, "rewards/reward_func/std": 0.7883979082107544, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.1348612308502197, "sampling/importance_sampling_ratio/mean": 0.989497184753418, "sampling/importance_sampling_ratio/min": 0.0010880796471610665, "sampling/sampling_logp_difference/max": 6.823340892791748, "sampling/sampling_logp_difference/mean": 0.020549368113279343, "step": 190, "step_time": 186.38824158336502 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11019.0, "completions/max_terminated_length": 11019.0, "completions/mean_length": 7058.15625, "completions/mean_terminated_length": 7058.15625, "completions/min_length": 2793.0, "completions/min_terminated_length": 2793.0, "entropy": 0.29681937769055367, "epoch": 0.22929171668667467, "frac_reward_zero_std": 0.0, "grad_norm": 0.48941799998283386, "learning_rate": 2.4e-07, "loss": -0.0289, "num_tokens": 65715367.0, "reward": 2.424999475479126, "reward_std": 2.926403045654297, "rewards/reward_func/mean": 2.424999475479126, "rewards/reward_func/std": 2.926403045654297, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.788496494293213, "sampling/importance_sampling_ratio/mean": 0.9895862936973572, "sampling/importance_sampling_ratio/min": 0.3045606315135956, "sampling/sampling_logp_difference/max": 1.1888850927352905, "sampling/sampling_logp_difference/mean": 0.02054070681333542, "step": 191, "step_time": 202.2660656850785 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15425.0, "completions/max_terminated_length": 15425.0, "completions/mean_length": 10237.875, "completions/mean_terminated_length": 10237.875, "completions/min_length": 4844.0, "completions/min_terminated_length": 4844.0, "entropy": 0.28149267937988043, "epoch": 0.2304921968787515, "frac_reward_zero_std": 0.0, "grad_norm": 0.5577535033226013, "learning_rate": 2.3599999999999997e-07, "loss": 0.0313, "num_tokens": 66083091.0, "reward": 1.852272391319275, "reward_std": 3.534353494644165, "rewards/reward_func/mean": 1.852272391319275, "rewards/reward_func/std": 3.534353494644165, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.9435347318649292, "sampling/importance_sampling_ratio/mean": 0.9903722405433655, "sampling/importance_sampling_ratio/min": 0.42083677649497986, "sampling/sampling_logp_difference/max": 0.8655102252960205, "sampling/sampling_logp_difference/mean": 0.019453980028629303, "step": 192, "step_time": 476.26090911997017 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12826.0, "completions/max_terminated_length": 12826.0, "completions/mean_length": 9010.1875, "completions/mean_terminated_length": 9010.1875, "completions/min_length": 5762.0, "completions/min_terminated_length": 5762.0, "entropy": 0.2871004734188318, "epoch": 0.23169267707082833, "frac_reward_zero_std": 0.0, "grad_norm": 0.4089401364326477, "learning_rate": 2.32e-07, "loss": -0.0462, "num_tokens": 66413569.0, "reward": 34.56924057006836, "reward_std": 156.94326782226562, "rewards/reward_func/mean": 34.56924057006836, "rewards/reward_func/std": 156.9432830810547, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.8682339191436768, "sampling/importance_sampling_ratio/mean": 0.989984929561615, "sampling/importance_sampling_ratio/min": 0.0005362548981793225, "sampling/sampling_logp_difference/max": 7.530900955200195, "sampling/sampling_logp_difference/mean": 0.020170990377664566, "step": 193, "step_time": 259.0703094872879 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10244.0, "completions/max_terminated_length": 10244.0, "completions/mean_length": 7021.78125, "completions/mean_terminated_length": 7021.78125, "completions/min_length": 4505.0, "completions/min_terminated_length": 4505.0, "entropy": 0.281132142059505, "epoch": 0.23289315726290516, "frac_reward_zero_std": 0.0, "grad_norm": 0.45947787165641785, "learning_rate": 2.28e-07, "loss": 0.0177, "num_tokens": 66662410.0, "reward": 2.6954410076141357, "reward_std": 2.3101069927215576, "rewards/reward_func/mean": 2.6954410076141357, "rewards/reward_func/std": 2.3101069927215576, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.2453787326812744, "sampling/importance_sampling_ratio/mean": 0.9901114702224731, "sampling/importance_sampling_ratio/min": 0.13286785781383514, "sampling/sampling_logp_difference/max": 2.018400192260742, "sampling/sampling_logp_difference/mean": 0.019676901400089264, "step": 194, "step_time": 158.03437204903457 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 20480.0, "completions/max_terminated_length": 14415.0, "completions/mean_length": 10107.21875, "completions/mean_terminated_length": 9772.6123046875, "completions/min_length": 5721.0, "completions/min_terminated_length": 5721.0, "entropy": 0.26805521361529827, "epoch": 0.234093637454982, "frac_reward_zero_std": 0.0, "grad_norm": 0.3630945682525635, "learning_rate": 2.24e-07, "loss": -0.0655, "num_tokens": 67019769.0, "reward": 0.5480448007583618, "reward_std": 0.7183238863945007, "rewards/reward_func/mean": 0.5979817509651184, "rewards/reward_func/std": 0.6713647246360779, "rewards/soft_overlong_punishment_reward/mean": -0.03125, "rewards/soft_overlong_punishment_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 2.4275918006896973, "sampling/importance_sampling_ratio/mean": 0.9905480146408081, "sampling/importance_sampling_ratio/min": 0.3468351662158966, "sampling/sampling_logp_difference/max": 1.0589056015014648, "sampling/sampling_logp_difference/mean": 0.018695417791604996, "step": 195, "step_time": 256.8099763797363 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11044.0, "completions/max_terminated_length": 11044.0, "completions/mean_length": 8487.1875, "completions/mean_terminated_length": 8487.1875, "completions/min_length": 4974.0, "completions/min_terminated_length": 4974.0, "entropy": 0.2542698457837105, "epoch": 0.23529411764705882, "frac_reward_zero_std": 0.0, "grad_norm": 0.40941476821899414, "learning_rate": 2.1999999999999998e-07, "loss": 0.0108, "num_tokens": 67330087.0, "reward": 0.9295157790184021, "reward_std": 1.449187159538269, "rewards/reward_func/mean": 0.9295157790184021, "rewards/reward_func/std": 1.449187159538269, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.435007333755493, "sampling/importance_sampling_ratio/mean": 0.9911128282546997, "sampling/importance_sampling_ratio/min": 0.0041382331401109695, "sampling/sampling_logp_difference/max": 5.487486362457275, "sampling/sampling_logp_difference/mean": 0.01788972318172455, "step": 196, "step_time": 230.38768620893825 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11561.0, "completions/max_terminated_length": 11561.0, "completions/mean_length": 8563.40625, "completions/mean_terminated_length": 8563.40625, "completions/min_length": 3335.0, "completions/min_terminated_length": 3335.0, "entropy": 0.29917307291179895, "epoch": 0.23649459783913565, "frac_reward_zero_std": 0.25, "grad_norm": 0.4290471374988556, "learning_rate": 2.1599999999999998e-07, "loss": -0.0069, "num_tokens": 67638532.0, "reward": 0.5732239484786987, "reward_std": 0.6618029475212097, "rewards/reward_func/mean": 0.5732239484786987, "rewards/reward_func/std": 0.6618028879165649, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.300884246826172, "sampling/importance_sampling_ratio/mean": 0.9896308779716492, "sampling/importance_sampling_ratio/min": 0.13782455027103424, "sampling/sampling_logp_difference/max": 1.9817737340927124, "sampling/sampling_logp_difference/mean": 0.020700525492429733, "step": 197, "step_time": 230.13961980782915 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14545.0, "completions/max_terminated_length": 14545.0, "completions/mean_length": 8384.875, "completions/mean_terminated_length": 8384.875, "completions/min_length": 4210.0, "completions/min_terminated_length": 4210.0, "entropy": 0.27590967528522015, "epoch": 0.23769507803121248, "frac_reward_zero_std": 0.0, "grad_norm": 0.4398875832557678, "learning_rate": 2.12e-07, "loss": 0.019, "num_tokens": 67938368.0, "reward": 29078.322265625, "reward_std": 164479.546875, "rewards/reward_func/mean": 29078.322265625, "rewards/reward_func/std": 164479.546875, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.5150890350341797, "sampling/importance_sampling_ratio/mean": 0.9903858304023743, "sampling/importance_sampling_ratio/min": 0.07026353478431702, "sampling/sampling_logp_difference/max": 2.6555023193359375, "sampling/sampling_logp_difference/mean": 0.019529715180397034, "step": 198, "step_time": 445.3559492408531 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15397.0, "completions/max_terminated_length": 15397.0, "completions/mean_length": 10389.4375, "completions/mean_terminated_length": 10389.4375, "completions/min_length": 6805.0, "completions/min_terminated_length": 6805.0, "entropy": 0.2911730855703354, "epoch": 0.2388955582232893, "frac_reward_zero_std": 0.0, "grad_norm": 0.42178842425346375, "learning_rate": 2.0799999999999998e-07, "loss": 0.0221, "num_tokens": 68309710.0, "reward": 0.8557122945785522, "reward_std": 0.7761005163192749, "rewards/reward_func/mean": 0.8557122945785522, "rewards/reward_func/std": 0.7761005759239197, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.1360371112823486, "sampling/importance_sampling_ratio/mean": 0.989779531955719, "sampling/importance_sampling_ratio/min": 0.04221712425351143, "sampling/sampling_logp_difference/max": 3.1649293899536133, "sampling/sampling_logp_difference/mean": 0.02039753459393978, "step": 199, "step_time": 232.6083577099489 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13338.0, "completions/max_terminated_length": 13338.0, "completions/mean_length": 9125.3125, "completions/mean_terminated_length": 9125.3125, "completions/min_length": 5379.0, "completions/min_terminated_length": 5379.0, "entropy": 0.3312236089259386, "epoch": 0.24009603841536614, "frac_reward_zero_std": 0.0, "grad_norm": 0.40419086813926697, "learning_rate": 2.0399999999999997e-07, "loss": -0.0259, "num_tokens": 68628320.0, "reward": 0.9386237859725952, "reward_std": 0.6489390730857849, "rewards/reward_func/mean": 0.9386237859725952, "rewards/reward_func/std": 0.6489390134811401, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.085390090942383, "sampling/importance_sampling_ratio/mean": 0.9884665608406067, "sampling/importance_sampling_ratio/min": 0.2759179472923279, "sampling/sampling_logp_difference/max": 1.287651777267456, "sampling/sampling_logp_difference/mean": 0.022430500015616417, "step": 200, "step_time": 201.43533559609205 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11893.0, "completions/max_terminated_length": 11893.0, "completions/mean_length": 7656.96875, "completions/mean_terminated_length": 7656.96875, "completions/min_length": 3211.0, "completions/min_terminated_length": 3211.0, "entropy": 0.2885604901239276, "epoch": 0.24129651860744297, "frac_reward_zero_std": 0.0, "grad_norm": 0.4403934180736542, "learning_rate": 2e-07, "loss": -0.0085, "num_tokens": 68899535.0, "reward": 2.592362880706787, "reward_std": 2.72981858253479, "rewards/reward_func/mean": 2.592362880706787, "rewards/reward_func/std": 2.72981858253479, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9901413917541504, "sampling/importance_sampling_ratio/min": 0.06037893146276474, "sampling/sampling_logp_difference/max": 2.807115077972412, "sampling/sampling_logp_difference/mean": 0.02000146359205246, "step": 201, "step_time": 182.84680638380814 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13383.0, "completions/max_terminated_length": 13383.0, "completions/mean_length": 8786.125, "completions/mean_terminated_length": 8786.125, "completions/min_length": 4108.0, "completions/min_terminated_length": 4108.0, "entropy": 0.30553513020277023, "epoch": 0.2424969987995198, "frac_reward_zero_std": 0.0, "grad_norm": 0.4264076352119446, "learning_rate": 1.96e-07, "loss": 0.0064, "num_tokens": 69208147.0, "reward": 1.068824291229248, "reward_std": 0.6439973711967468, "rewards/reward_func/mean": 1.068824291229248, "rewards/reward_func/std": 0.643997311592102, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.9403151273727417, "sampling/importance_sampling_ratio/mean": 0.9893855452537537, "sampling/importance_sampling_ratio/min": 0.1719096601009369, "sampling/sampling_logp_difference/max": 1.7607861757278442, "sampling/sampling_logp_difference/mean": 0.02125393971800804, "step": 202, "step_time": 202.14960491785314 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13249.0, "completions/max_terminated_length": 13249.0, "completions/mean_length": 8494.1875, "completions/mean_terminated_length": 8494.1875, "completions/min_length": 2688.0, "completions/min_terminated_length": 2688.0, "entropy": 0.2879168214276433, "epoch": 0.24369747899159663, "frac_reward_zero_std": 0.0, "grad_norm": 0.39069902896881104, "learning_rate": 1.92e-07, "loss": 0.0649, "num_tokens": 69509457.0, "reward": 1.7704880237579346, "reward_std": 2.810868978500366, "rewards/reward_func/mean": 1.7704880237579346, "rewards/reward_func/std": 2.810868978500366, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9898999333381653, "sampling/importance_sampling_ratio/min": 0.003532076720148325, "sampling/sampling_logp_difference/max": 5.645869255065918, "sampling/sampling_logp_difference/mean": 0.020249199122190475, "step": 203, "step_time": 183.80464980110992 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13377.0, "completions/max_terminated_length": 13377.0, "completions/mean_length": 9125.28125, "completions/mean_terminated_length": 9125.28125, "completions/min_length": 4439.0, "completions/min_terminated_length": 4439.0, "entropy": 0.24132216069847345, "epoch": 0.24489795918367346, "frac_reward_zero_std": 0.0, "grad_norm": 0.39876803755760193, "learning_rate": 1.88e-07, "loss": 0.0231, "num_tokens": 69851202.0, "reward": 0.5941991209983826, "reward_std": 0.7350529432296753, "rewards/reward_func/mean": 0.5941991209983826, "rewards/reward_func/std": 0.7350529432296753, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.8217852115631104, "sampling/importance_sampling_ratio/mean": 0.991719663143158, "sampling/importance_sampling_ratio/min": 0.302060604095459, "sampling/sampling_logp_difference/max": 1.1971275806427002, "sampling/sampling_logp_difference/mean": 0.016849037259817123, "step": 204, "step_time": 229.6054396291729 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12452.0, "completions/max_terminated_length": 12452.0, "completions/mean_length": 8173.46875, "completions/mean_terminated_length": 8173.46875, "completions/min_length": 4740.0, "completions/min_terminated_length": 4740.0, "entropy": 0.2661965722218156, "epoch": 0.2460984393757503, "frac_reward_zero_std": 0.0, "grad_norm": 0.3894799053668976, "learning_rate": 1.8399999999999998e-07, "loss": -0.0405, "num_tokens": 70146561.0, "reward": 8.144660949707031, "reward_std": 13.748440742492676, "rewards/reward_func/mean": 8.144660949707031, "rewards/reward_func/std": 13.748440742492676, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9909324049949646, "sampling/importance_sampling_ratio/min": 0.17160779237747192, "sampling/sampling_logp_difference/max": 1.7625436782836914, "sampling/sampling_logp_difference/mean": 0.018414584919810295, "step": 205, "step_time": 194.27041996375192 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 20480.0, "completions/max_terminated_length": 14681.0, "completions/mean_length": 9778.9375, "completions/mean_terminated_length": 9433.7412109375, "completions/min_length": 5841.0, "completions/min_terminated_length": 5841.0, "entropy": 0.2878849096596241, "epoch": 0.24729891956782712, "frac_reward_zero_std": 0.5, "grad_norm": 0.23984801769256592, "learning_rate": 1.8e-07, "loss": -0.0947, "num_tokens": 70496111.0, "reward": 0.5641666650772095, "reward_std": 0.7118456363677979, "rewards/reward_func/mean": 0.6146236658096313, "rewards/reward_func/std": 0.6628953814506531, "rewards/soft_overlong_punishment_reward/mean": -0.03125, "rewards/soft_overlong_punishment_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.989820122718811, "sampling/importance_sampling_ratio/min": 0.3761327564716339, "sampling/sampling_logp_difference/max": 1.2785313129425049, "sampling/sampling_logp_difference/mean": 0.020214585587382317, "step": 206, "step_time": 256.00012533087283 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12854.0, "completions/max_terminated_length": 12854.0, "completions/mean_length": 7986.3125, "completions/mean_terminated_length": 7986.3125, "completions/min_length": 5362.0, "completions/min_terminated_length": 5362.0, "entropy": 0.26751279924064875, "epoch": 0.24849939975990396, "frac_reward_zero_std": 0.0, "grad_norm": 0.44527071714401245, "learning_rate": 1.76e-07, "loss": -0.0467, "num_tokens": 70784649.0, "reward": 1.1738154888153076, "reward_std": 0.8459872603416443, "rewards/reward_func/mean": 1.1738154888153076, "rewards/reward_func/std": 0.8459872603416443, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.8473902940750122, "sampling/importance_sampling_ratio/mean": 0.9906308650970459, "sampling/importance_sampling_ratio/min": 0.3182202875614166, "sampling/sampling_logp_difference/max": 1.1450114250183105, "sampling/sampling_logp_difference/mean": 0.018941214308142662, "step": 207, "step_time": 183.06572339008562 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 17590.0, "completions/max_terminated_length": 17590.0, "completions/mean_length": 10755.09375, "completions/mean_terminated_length": 10755.09375, "completions/min_length": 7289.0, "completions/min_terminated_length": 7289.0, "entropy": 0.28774886950850487, "epoch": 0.24969987995198079, "frac_reward_zero_std": 0.0, "grad_norm": 0.3857475817203522, "learning_rate": 1.7199999999999998e-07, "loss": 0.0468, "num_tokens": 71168340.0, "reward": 1.0103178024291992, "reward_std": 1.2723941802978516, "rewards/reward_func/mean": 1.0195188522338867, "rewards/reward_func/std": 1.2721405029296875, "rewards/soft_overlong_punishment_reward/mean": -0.0092010498046875, "rewards/soft_overlong_punishment_reward/std": 0.052048999816179276, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9900601506233215, "sampling/importance_sampling_ratio/min": 0.051096703857183456, "sampling/sampling_logp_difference/max": 2.9740352630615234, "sampling/sampling_logp_difference/mean": 0.01990516111254692, "step": 208, "step_time": 276.0685674578417 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16527.0, "completions/max_terminated_length": 16527.0, "completions/mean_length": 10267.90625, "completions/mean_terminated_length": 10267.90625, "completions/min_length": 5650.0, "completions/min_terminated_length": 5650.0, "entropy": 0.2723705815151334, "epoch": 0.25090036014405764, "frac_reward_zero_std": 0.0, "grad_norm": 0.3713222146034241, "learning_rate": 1.68e-07, "loss": -0.014, "num_tokens": 71532585.0, "reward": 1.7478203773498535, "reward_std": 2.0512068271636963, "rewards/reward_func/mean": 1.7489113807678223, "rewards/reward_func/std": 2.0506913661956787, "rewards/soft_overlong_punishment_reward/mean": -0.00109100341796875, "rewards/soft_overlong_punishment_reward/std": 0.006171646993607283, "sampling/importance_sampling_ratio/max": 2.043245553970337, "sampling/importance_sampling_ratio/mean": 0.9906348586082458, "sampling/importance_sampling_ratio/min": 0.11972002685070038, "sampling/sampling_logp_difference/max": 2.1225993633270264, "sampling/sampling_logp_difference/mean": 0.019026953727006912, "step": 209, "step_time": 249.78715924406424 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14411.0, "completions/max_terminated_length": 14411.0, "completions/mean_length": 8568.375, "completions/mean_terminated_length": 8568.375, "completions/min_length": 4104.0, "completions/min_terminated_length": 4104.0, "entropy": 0.3192238248884678, "epoch": 0.25210084033613445, "frac_reward_zero_std": 0.0, "grad_norm": 0.4139728844165802, "learning_rate": 1.64e-07, "loss": -0.0156, "num_tokens": 71830213.0, "reward": 0.5381001830101013, "reward_std": 0.655325710773468, "rewards/reward_func/mean": 0.5381001830101013, "rewards/reward_func/std": 0.655325710773468, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.4862608909606934, "sampling/importance_sampling_ratio/mean": 0.9889300465583801, "sampling/importance_sampling_ratio/min": 0.2908362150192261, "sampling/sampling_logp_difference/max": 1.234994888305664, "sampling/sampling_logp_difference/mean": 0.021744508296251297, "step": 210, "step_time": 233.44613694911823 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15175.0, "completions/max_terminated_length": 15175.0, "completions/mean_length": 6835.25, "completions/mean_terminated_length": 6835.25, "completions/min_length": 2738.0, "completions/min_terminated_length": 2738.0, "entropy": 0.2472194992005825, "epoch": 0.2533013205282113, "frac_reward_zero_std": 0.25, "grad_norm": 0.3917876183986664, "learning_rate": 1.6e-07, "loss": 0.0568, "num_tokens": 72084285.0, "reward": 2.039097785949707, "reward_std": 2.948822498321533, "rewards/reward_func/mean": 2.039097785949707, "rewards/reward_func/std": 2.9488227367401123, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.354682207107544, "sampling/importance_sampling_ratio/mean": 0.991452157497406, "sampling/importance_sampling_ratio/min": 0.037272509187459946, "sampling/sampling_logp_difference/max": 3.289499282836914, "sampling/sampling_logp_difference/mean": 0.017462395131587982, "step": 211, "step_time": 213.77216677484103 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13379.0, "completions/max_terminated_length": 13379.0, "completions/mean_length": 8146.84375, "completions/mean_terminated_length": 8146.84375, "completions/min_length": 4187.0, "completions/min_terminated_length": 4187.0, "entropy": 0.27302129566669464, "epoch": 0.2545018007202881, "frac_reward_zero_std": 0.25, "grad_norm": 0.3473767638206482, "learning_rate": 1.56e-07, "loss": 0.0159, "num_tokens": 72380368.0, "reward": 2.9426181316375732, "reward_std": 3.704005718231201, "rewards/reward_func/mean": 2.9426181316375732, "rewards/reward_func/std": 3.704005479812622, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.7764674425125122, "sampling/importance_sampling_ratio/mean": 0.9906426668167114, "sampling/importance_sampling_ratio/min": 0.03155454993247986, "sampling/sampling_logp_difference/max": 3.4560375213623047, "sampling/sampling_logp_difference/mean": 0.01900378428399563, "step": 212, "step_time": 210.75419173389673 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14846.0, "completions/max_terminated_length": 14846.0, "completions/mean_length": 11225.4375, "completions/mean_terminated_length": 11225.4375, "completions/min_length": 7437.0, "completions/min_terminated_length": 7437.0, "entropy": 0.26801333110779524, "epoch": 0.25570228091236497, "frac_reward_zero_std": 0.0, "grad_norm": 0.34166890382766724, "learning_rate": 1.5199999999999998e-07, "loss": -0.0198, "num_tokens": 72773310.0, "reward": 0.42110854387283325, "reward_std": 0.5873985290527344, "rewards/reward_func/mean": 0.42110854387283325, "rewards/reward_func/std": 0.5873985290527344, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9907779693603516, "sampling/importance_sampling_ratio/min": 0.34792667627334595, "sampling/sampling_logp_difference/max": 1.1829512119293213, "sampling/sampling_logp_difference/mean": 0.018622785806655884, "step": 213, "step_time": 239.31595174130052 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11657.0, "completions/max_terminated_length": 11657.0, "completions/mean_length": 7578.28125, "completions/mean_terminated_length": 7578.28125, "completions/min_length": 2938.0, "completions/min_terminated_length": 2938.0, "entropy": 0.28475786093622446, "epoch": 0.25690276110444177, "frac_reward_zero_std": 0.0, "grad_norm": 0.4476838707923889, "learning_rate": 1.4799999999999998e-07, "loss": -0.0071, "num_tokens": 73049535.0, "reward": 1.1447203159332275, "reward_std": 1.569128155708313, "rewards/reward_func/mean": 1.1447203159332275, "rewards/reward_func/std": 1.569128155708313, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6962001323699951, "sampling/importance_sampling_ratio/mean": 0.9899388551712036, "sampling/importance_sampling_ratio/min": 0.2515447735786438, "sampling/sampling_logp_difference/max": 1.3801343441009521, "sampling/sampling_logp_difference/mean": 0.019816022366285324, "step": 214, "step_time": 192.09097037103493 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15818.0, "completions/max_terminated_length": 15818.0, "completions/mean_length": 9813.6875, "completions/mean_terminated_length": 9813.6875, "completions/min_length": 5428.0, "completions/min_terminated_length": 5428.0, "entropy": 0.2705380003899336, "epoch": 0.2581032412965186, "frac_reward_zero_std": 0.0, "grad_norm": 0.38898560404777527, "learning_rate": 1.44e-07, "loss": 0.0382, "num_tokens": 73403117.0, "reward": 0.6952791810035706, "reward_std": 0.6618348360061646, "rewards/reward_func/mean": 0.6952791810035706, "rewards/reward_func/std": 0.6618348360061646, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.8969855308532715, "sampling/importance_sampling_ratio/mean": 0.9905439615249634, "sampling/importance_sampling_ratio/min": 0.05445502698421478, "sampling/sampling_logp_difference/max": 2.9103801250457764, "sampling/sampling_logp_difference/mean": 0.018809445202350616, "step": 215, "step_time": 219.26422131690197 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12131.0, "completions/max_terminated_length": 12131.0, "completions/mean_length": 9191.46875, "completions/mean_terminated_length": 9191.46875, "completions/min_length": 4309.0, "completions/min_terminated_length": 4309.0, "entropy": 0.29348371736705303, "epoch": 0.25930372148859543, "frac_reward_zero_std": 0.0, "grad_norm": 0.4402393102645874, "learning_rate": 1.4e-07, "loss": 0.0291, "num_tokens": 73730012.0, "reward": 2.512660264968872, "reward_std": 2.8215131759643555, "rewards/reward_func/mean": 2.512660264968872, "rewards/reward_func/std": 2.8215129375457764, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.989812970161438, "sampling/importance_sampling_ratio/min": 0.16945770382881165, "sampling/sampling_logp_difference/max": 1.7751518487930298, "sampling/sampling_logp_difference/mean": 0.020123887807130814, "step": 216, "step_time": 192.42200359026901 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14410.0, "completions/max_terminated_length": 14410.0, "completions/mean_length": 10286.53125, "completions/mean_terminated_length": 10286.53125, "completions/min_length": 6044.0, "completions/min_terminated_length": 6044.0, "entropy": 0.2660679593682289, "epoch": 0.2605042016806723, "frac_reward_zero_std": 0.0, "grad_norm": 0.3704564571380615, "learning_rate": 1.36e-07, "loss": 0.0008, "num_tokens": 74100413.0, "reward": 0.5737283229827881, "reward_std": 0.6282573342323303, "rewards/reward_func/mean": 0.5737283229827881, "rewards/reward_func/std": 0.6282573342323303, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.249643325805664, "sampling/importance_sampling_ratio/mean": 0.9908007979393005, "sampling/importance_sampling_ratio/min": 0.00029296314460225403, "sampling/sampling_logp_difference/max": 8.13546371459961, "sampling/sampling_logp_difference/mean": 0.018784763291478157, "step": 217, "step_time": 440.4557863563532 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14001.0, "completions/max_terminated_length": 14001.0, "completions/mean_length": 10060.25, "completions/mean_terminated_length": 10060.25, "completions/min_length": 6824.0, "completions/min_terminated_length": 6824.0, "entropy": 0.3022372853010893, "epoch": 0.2617046818727491, "frac_reward_zero_std": 0.0, "grad_norm": 0.43306854367256165, "learning_rate": 1.32e-07, "loss": 0.0028, "num_tokens": 74455253.0, "reward": 16.12359619140625, "reward_std": 59.91655731201172, "rewards/reward_func/mean": 16.12359619140625, "rewards/reward_func/std": 59.91655349731445, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.2559473514556885, "sampling/importance_sampling_ratio/mean": 0.9893619418144226, "sampling/importance_sampling_ratio/min": 0.22430028021335602, "sampling/sampling_logp_difference/max": 1.49476957321167, "sampling/sampling_logp_difference/mean": 0.021203139796853065, "step": 218, "step_time": 414.3311550068902 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11622.0, "completions/max_terminated_length": 11622.0, "completions/mean_length": 6517.96875, "completions/mean_terminated_length": 6517.96875, "completions/min_length": 3112.0, "completions/min_terminated_length": 3112.0, "entropy": 0.28310599736869335, "epoch": 0.26290516206482595, "frac_reward_zero_std": 0.25, "grad_norm": 0.447173535823822, "learning_rate": 1.28e-07, "loss": -0.0289, "num_tokens": 74686092.0, "reward": 0.6452304720878601, "reward_std": 2.197897434234619, "rewards/reward_func/mean": 0.6452304720878601, "rewards/reward_func/std": 2.197897434234619, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.134779691696167, "sampling/importance_sampling_ratio/mean": 0.9900719523429871, "sampling/importance_sampling_ratio/min": 0.36751604080200195, "sampling/sampling_logp_difference/max": 1.0009883642196655, "sampling/sampling_logp_difference/mean": 0.0198367852717638, "step": 219, "step_time": 177.24385808466468 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11380.0, "completions/max_terminated_length": 11380.0, "completions/mean_length": 6471.75, "completions/mean_terminated_length": 6471.75, "completions/min_length": 4281.0, "completions/min_terminated_length": 4281.0, "entropy": 0.3177992310374975, "epoch": 0.26410564225690275, "frac_reward_zero_std": 0.0, "grad_norm": 0.5168129205703735, "learning_rate": 1.24e-07, "loss": 0.0395, "num_tokens": 74912764.0, "reward": 2.1183838844299316, "reward_std": 1.930953025817871, "rewards/reward_func/mean": 2.1183838844299316, "rewards/reward_func/std": 1.930953025817871, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0708396434783936, "sampling/importance_sampling_ratio/mean": 0.9886492490768433, "sampling/importance_sampling_ratio/min": 0.3355446755886078, "sampling/sampling_logp_difference/max": 1.0920002460479736, "sampling/sampling_logp_difference/mean": 0.022046584635972977, "step": 220, "step_time": 158.16589547123294 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14608.0, "completions/max_terminated_length": 14608.0, "completions/mean_length": 9121.625, "completions/mean_terminated_length": 9121.625, "completions/min_length": 2448.0, "completions/min_terminated_length": 2448.0, "entropy": 0.2602370521053672, "epoch": 0.2653061224489796, "frac_reward_zero_std": 0.0, "grad_norm": 0.39733272790908813, "learning_rate": 1.2e-07, "loss": 0.0234, "num_tokens": 75242344.0, "reward": 0.6069552898406982, "reward_std": 0.6848835945129395, "rewards/reward_func/mean": 0.6069552898406982, "rewards/reward_func/std": 0.6848835349082947, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.775014877319336, "sampling/importance_sampling_ratio/mean": 0.9909803867340088, "sampling/importance_sampling_ratio/min": 0.14724299311637878, "sampling/sampling_logp_difference/max": 1.9156711101531982, "sampling/sampling_logp_difference/mean": 0.01835060492157936, "step": 221, "step_time": 228.61106476001441 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12311.0, "completions/max_terminated_length": 12311.0, "completions/mean_length": 8921.34375, "completions/mean_terminated_length": 8921.34375, "completions/min_length": 5336.0, "completions/min_terminated_length": 5336.0, "entropy": 0.2625180771574378, "epoch": 0.2665066026410564, "frac_reward_zero_std": 0.0, "grad_norm": 0.3746626079082489, "learning_rate": 1.16e-07, "loss": -0.0205, "num_tokens": 75562803.0, "reward": 1.2791639566421509, "reward_std": 1.4882169961929321, "rewards/reward_func/mean": 1.2791639566421509, "rewards/reward_func/std": 1.4882171154022217, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.9303572177886963, "sampling/importance_sampling_ratio/mean": 0.9908260107040405, "sampling/importance_sampling_ratio/min": 0.32418113946914673, "sampling/sampling_logp_difference/max": 1.1264528036117554, "sampling/sampling_logp_difference/mean": 0.01839890144765377, "step": 222, "step_time": 331.951701607788 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15993.0, "completions/max_terminated_length": 15993.0, "completions/mean_length": 9919.09375, "completions/mean_terminated_length": 9919.09375, "completions/min_length": 2850.0, "completions/min_terminated_length": 2850.0, "entropy": 0.27178078424185514, "epoch": 0.26770708283313327, "frac_reward_zero_std": 0.25, "grad_norm": 0.32960885763168335, "learning_rate": 1.12e-07, "loss": -0.0391, "num_tokens": 75917510.0, "reward": 0.9057698845863342, "reward_std": 1.4171531200408936, "rewards/reward_func/mean": 0.9057698845863342, "rewards/reward_func/std": 1.4171531200408936, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9905833005905151, "sampling/importance_sampling_ratio/min": 0.12444481998682022, "sampling/sampling_logp_difference/max": 2.083892822265625, "sampling/sampling_logp_difference/mean": 0.019010471180081367, "step": 223, "step_time": 452.7820453970926 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15207.0, "completions/max_terminated_length": 15207.0, "completions/mean_length": 8989.75, "completions/mean_terminated_length": 8989.75, "completions/min_length": 5085.0, "completions/min_terminated_length": 5085.0, "entropy": 0.27119907177984715, "epoch": 0.2689075630252101, "frac_reward_zero_std": 0.25, "grad_norm": 0.3286755681037903, "learning_rate": 1.0799999999999999e-07, "loss": -0.01, "num_tokens": 76241470.0, "reward": 2.0997040271759033, "reward_std": 2.725891351699829, "rewards/reward_func/mean": 2.0997040271759033, "rewards/reward_func/std": 2.72589111328125, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.770100474357605, "sampling/importance_sampling_ratio/mean": 0.9906846284866333, "sampling/importance_sampling_ratio/min": 0.03444494679570198, "sampling/sampling_logp_difference/max": 3.3683929443359375, "sampling/sampling_logp_difference/mean": 0.018905045464634895, "step": 224, "step_time": 434.58480245189276 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14341.0, "completions/max_terminated_length": 14341.0, "completions/mean_length": 9169.65625, "completions/mean_terminated_length": 9169.65625, "completions/min_length": 4655.0, "completions/min_terminated_length": 4655.0, "entropy": 0.2545336727052927, "epoch": 0.27010804321728693, "frac_reward_zero_std": 0.0, "grad_norm": 0.43626582622528076, "learning_rate": 1.0399999999999999e-07, "loss": -0.0511, "num_tokens": 76579227.0, "reward": 0.7725225687026978, "reward_std": 0.8143731951713562, "rewards/reward_func/mean": 0.7725225687026978, "rewards/reward_func/std": 0.8143731951713562, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.39496111869812, "sampling/importance_sampling_ratio/mean": 0.9909839630126953, "sampling/importance_sampling_ratio/min": 0.17538554966449738, "sampling/sampling_logp_difference/max": 1.7407686710357666, "sampling/sampling_logp_difference/mean": 0.018118245527148247, "step": 225, "step_time": 256.96223465609364 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11969.0, "completions/max_terminated_length": 11969.0, "completions/mean_length": 7255.71875, "completions/mean_terminated_length": 7255.71875, "completions/min_length": 3750.0, "completions/min_terminated_length": 3750.0, "entropy": 0.2862818920984864, "epoch": 0.27130852340936373, "frac_reward_zero_std": 0.0, "grad_norm": 0.5091444253921509, "learning_rate": 1e-07, "loss": 0.0555, "num_tokens": 76838802.0, "reward": 3.338810443878174, "reward_std": 3.9317619800567627, "rewards/reward_func/mean": 3.338810443878174, "rewards/reward_func/std": 3.9317619800567627, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.1975717544555664, "sampling/importance_sampling_ratio/mean": 0.9902316331863403, "sampling/importance_sampling_ratio/min": 0.10265523940324783, "sampling/sampling_logp_difference/max": 2.276379108428955, "sampling/sampling_logp_difference/mean": 0.01985924504697323, "step": 226, "step_time": 177.48551015171688 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14337.0, "completions/max_terminated_length": 14337.0, "completions/mean_length": 8913.25, "completions/mean_terminated_length": 8913.25, "completions/min_length": 3138.0, "completions/min_terminated_length": 3138.0, "entropy": 0.26585716381669044, "epoch": 0.2725090036014406, "frac_reward_zero_std": 0.0, "grad_norm": 0.3888947069644928, "learning_rate": 9.6e-08, "loss": -0.0538, "num_tokens": 77161562.0, "reward": 2.5214858055114746, "reward_std": 2.8809354305267334, "rewards/reward_func/mean": 2.5214858055114746, "rewards/reward_func/std": 2.8809354305267334, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.9848451614379883, "sampling/importance_sampling_ratio/mean": 0.9906054139137268, "sampling/importance_sampling_ratio/min": 0.12535132467746735, "sampling/sampling_logp_difference/max": 2.0766348838806152, "sampling/sampling_logp_difference/mean": 0.01874399371445179, "step": 227, "step_time": 423.3902858378133 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12350.0, "completions/max_terminated_length": 12350.0, "completions/mean_length": 9451.5625, "completions/mean_terminated_length": 9451.5625, "completions/min_length": 6397.0, "completions/min_terminated_length": 6397.0, "entropy": 0.2693768171593547, "epoch": 0.2737094837935174, "frac_reward_zero_std": 0.0, "grad_norm": 0.3820766508579254, "learning_rate": 9.199999999999999e-08, "loss": 0.033, "num_tokens": 77498220.0, "reward": 1.154233455657959, "reward_std": 1.959659218788147, "rewards/reward_func/mean": 1.154233455657959, "rewards/reward_func/std": 1.959659218788147, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9909024238586426, "sampling/importance_sampling_ratio/min": 0.20406676828861237, "sampling/sampling_logp_difference/max": 1.5893080234527588, "sampling/sampling_logp_difference/mean": 0.018469782546162605, "step": 228, "step_time": 217.56979901902378 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12107.0, "completions/max_terminated_length": 12107.0, "completions/mean_length": 8378.96875, "completions/mean_terminated_length": 8378.96875, "completions/min_length": 5003.0, "completions/min_terminated_length": 5003.0, "entropy": 0.2748628966510296, "epoch": 0.27490996398559425, "frac_reward_zero_std": 0.0, "grad_norm": 0.4501209557056427, "learning_rate": 8.8e-08, "loss": -0.0042, "num_tokens": 77802059.0, "reward": 1.3091723918914795, "reward_std": 0.47337406873703003, "rewards/reward_func/mean": 1.3091723918914795, "rewards/reward_func/std": 0.47337406873703003, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0756125450134277, "sampling/importance_sampling_ratio/mean": 0.9905352592468262, "sampling/importance_sampling_ratio/min": 0.22848102450370789, "sampling/sampling_logp_difference/max": 1.476302146911621, "sampling/sampling_logp_difference/mean": 0.01908484473824501, "step": 229, "step_time": 173.2410452858312 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11253.0, "completions/max_terminated_length": 11253.0, "completions/mean_length": 8427.15625, "completions/mean_terminated_length": 8427.15625, "completions/min_length": 5975.0, "completions/min_terminated_length": 5975.0, "entropy": 0.28478993009775877, "epoch": 0.27611044417767105, "frac_reward_zero_std": 0.0, "grad_norm": 0.42257431149482727, "learning_rate": 8.4e-08, "loss": 0.0338, "num_tokens": 78102984.0, "reward": 0.6847608089447021, "reward_std": 0.6159871220588684, "rewards/reward_func/mean": 0.6847608089447021, "rewards/reward_func/std": 0.6159871220588684, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.810465693473816, "sampling/importance_sampling_ratio/mean": 0.9901672601699829, "sampling/importance_sampling_ratio/min": 0.21681459248065948, "sampling/sampling_logp_difference/max": 1.5287127494812012, "sampling/sampling_logp_difference/mean": 0.01989746280014515, "step": 230, "step_time": 376.1443422808079 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13502.0, "completions/max_terminated_length": 13502.0, "completions/mean_length": 9104.8125, "completions/mean_terminated_length": 9104.8125, "completions/min_length": 5037.0, "completions/min_terminated_length": 5037.0, "entropy": 0.28373472206294537, "epoch": 0.2773109243697479, "frac_reward_zero_std": 0.25, "grad_norm": 0.3297131657600403, "learning_rate": 8e-08, "loss": -0.004, "num_tokens": 78423906.0, "reward": 3.833698272705078, "reward_std": 8.968595504760742, "rewards/reward_func/mean": 3.833698272705078, "rewards/reward_func/std": 8.968595504760742, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.7473583221435547, "sampling/importance_sampling_ratio/mean": 0.9901688694953918, "sampling/importance_sampling_ratio/min": 0.3091135621070862, "sampling/sampling_logp_difference/max": 1.174046516418457, "sampling/sampling_logp_difference/mean": 0.019583595916628838, "step": 231, "step_time": 192.75546575232875 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13344.0, "completions/max_terminated_length": 13344.0, "completions/mean_length": 9607.15625, "completions/mean_terminated_length": 9607.15625, "completions/min_length": 6523.0, "completions/min_terminated_length": 6523.0, "entropy": 0.258233230561018, "epoch": 0.2785114045618247, "frac_reward_zero_std": 0.75, "grad_norm": 0.22545225918293, "learning_rate": 7.599999999999999e-08, "loss": 0.0098, "num_tokens": 78774783.0, "reward": 1.9562370777130127, "reward_std": 3.413482666015625, "rewards/reward_func/mean": 1.9562370777130127, "rewards/reward_func/std": 3.413482666015625, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9909834861755371, "sampling/importance_sampling_ratio/min": 0.03608707711100578, "sampling/sampling_logp_difference/max": 3.3218204975128174, "sampling/sampling_logp_difference/mean": 0.01814298704266548, "step": 232, "step_time": 441.14181373117026 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14557.0, "completions/max_terminated_length": 14557.0, "completions/mean_length": 8880.46875, "completions/mean_terminated_length": 8880.46875, "completions/min_length": 3949.0, "completions/min_terminated_length": 3949.0, "entropy": 0.2796018850058317, "epoch": 0.2797118847539016, "frac_reward_zero_std": 0.0, "grad_norm": 0.41359373927116394, "learning_rate": 7.2e-08, "loss": 0.0204, "num_tokens": 79094742.0, "reward": 2.0865750312805176, "reward_std": 3.248898506164551, "rewards/reward_func/mean": 2.0865750312805176, "rewards/reward_func/std": 3.248898506164551, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.355583906173706, "sampling/importance_sampling_ratio/mean": 0.9904192090034485, "sampling/importance_sampling_ratio/min": 0.22791339457035065, "sampling/sampling_logp_difference/max": 1.4787895679473877, "sampling/sampling_logp_difference/mean": 0.019396543502807617, "step": 233, "step_time": 433.1994162177434 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13023.0, "completions/max_terminated_length": 13023.0, "completions/mean_length": 9278.125, "completions/mean_terminated_length": 9278.125, "completions/min_length": 6046.0, "completions/min_terminated_length": 6046.0, "entropy": 0.23018638603389263, "epoch": 0.2809123649459784, "frac_reward_zero_std": 0.0, "grad_norm": 0.40194013714790344, "learning_rate": 6.8e-08, "loss": -0.0207, "num_tokens": 79432034.0, "reward": 1.2649976015090942, "reward_std": 2.4887611865997314, "rewards/reward_func/mean": 1.2649976015090942, "rewards/reward_func/std": 2.4887611865997314, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.066953420639038, "sampling/importance_sampling_ratio/mean": 0.9919387698173523, "sampling/importance_sampling_ratio/min": 0.07596759498119354, "sampling/sampling_logp_difference/max": 2.5774483680725098, "sampling/sampling_logp_difference/mean": 0.01637454889714718, "step": 234, "step_time": 200.44299350422807 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11484.0, "completions/max_terminated_length": 11484.0, "completions/mean_length": 7951.46875, "completions/mean_terminated_length": 7951.46875, "completions/min_length": 4951.0, "completions/min_terminated_length": 4951.0, "entropy": 0.28384537622332573, "epoch": 0.28211284513805523, "frac_reward_zero_std": 0.0, "grad_norm": 0.4639303684234619, "learning_rate": 6.4e-08, "loss": 0.0062, "num_tokens": 79715057.0, "reward": 1.722654938697815, "reward_std": 2.3874762058258057, "rewards/reward_func/mean": 1.722654938697815, "rewards/reward_func/std": 2.3874759674072266, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9900355339050293, "sampling/importance_sampling_ratio/min": 0.16387537121772766, "sampling/sampling_logp_difference/max": 1.8086490631103516, "sampling/sampling_logp_difference/mean": 0.019974499940872192, "step": 235, "step_time": 188.83989403117448 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15359.0, "completions/max_terminated_length": 15359.0, "completions/mean_length": 9800.75, "completions/mean_terminated_length": 9800.75, "completions/min_length": 6034.0, "completions/min_terminated_length": 6034.0, "entropy": 0.3060651607811451, "epoch": 0.28331332533013204, "frac_reward_zero_std": 0.0, "grad_norm": 0.4038175940513611, "learning_rate": 6e-08, "loss": 0.0295, "num_tokens": 80063393.0, "reward": 0.5499008893966675, "reward_std": 0.9594078660011292, "rewards/reward_func/mean": 0.5499008893966675, "rewards/reward_func/std": 0.9594078660011292, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.989382803440094, "sampling/importance_sampling_ratio/min": 0.10317613184452057, "sampling/sampling_logp_difference/max": 2.271317720413208, "sampling/sampling_logp_difference/mean": 0.021366208791732788, "step": 236, "step_time": 246.472208352061 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14811.0, "completions/max_terminated_length": 14811.0, "completions/mean_length": 10420.625, "completions/mean_terminated_length": 10420.625, "completions/min_length": 7528.0, "completions/min_terminated_length": 7528.0, "entropy": 0.26841907389461994, "epoch": 0.2845138055222089, "frac_reward_zero_std": 0.0, "grad_norm": 0.3900371789932251, "learning_rate": 5.6e-08, "loss": 0.0085, "num_tokens": 80437933.0, "reward": 0.7362344264984131, "reward_std": 0.9423819780349731, "rewards/reward_func/mean": 0.7362344264984131, "rewards/reward_func/std": 0.9423819184303284, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9905475378036499, "sampling/importance_sampling_ratio/min": 0.06430970132350922, "sampling/sampling_logp_difference/max": 2.744044780731201, "sampling/sampling_logp_difference/mean": 0.018762534484267235, "step": 237, "step_time": 224.29740512394346 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 19168.0, "completions/max_terminated_length": 19168.0, "completions/mean_length": 10909.96875, "completions/mean_terminated_length": 10909.96875, "completions/min_length": 7484.0, "completions/min_terminated_length": 7484.0, "entropy": 0.2649214630946517, "epoch": 0.2857142857142857, "frac_reward_zero_std": 0.0, "grad_norm": 0.3601086735725403, "learning_rate": 5.1999999999999996e-08, "loss": 0.0136, "num_tokens": 80832676.0, "reward": 1.2694827318191528, "reward_std": 1.707560420036316, "rewards/reward_func/mean": 1.2907229661941528, "rewards/reward_func/std": 1.7039649486541748, "rewards/soft_overlong_punishment_reward/mean": -0.021240234375, "rewards/soft_overlong_punishment_reward/std": 0.12015291303396225, "sampling/importance_sampling_ratio/max": 2.5833005905151367, "sampling/importance_sampling_ratio/mean": 0.990875244140625, "sampling/importance_sampling_ratio/min": 0.3051411211490631, "sampling/sampling_logp_difference/max": 1.186980962753296, "sampling/sampling_logp_difference/mean": 0.018429221585392952, "step": 238, "step_time": 254.5579416519031 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13040.0, "completions/max_terminated_length": 13040.0, "completions/mean_length": 7543.53125, "completions/mean_terminated_length": 7543.53125, "completions/min_length": 4494.0, "completions/min_terminated_length": 4494.0, "entropy": 0.2839345559477806, "epoch": 0.28691476590636256, "frac_reward_zero_std": 0.0, "grad_norm": 0.487249493598938, "learning_rate": 4.8e-08, "loss": 0.028, "num_tokens": 81106021.0, "reward": 0.7453573942184448, "reward_std": 0.7949322462081909, "rewards/reward_func/mean": 0.7453573942184448, "rewards/reward_func/std": 0.7949321269989014, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.3466830253601074, "sampling/importance_sampling_ratio/mean": 0.9900220036506653, "sampling/importance_sampling_ratio/min": 0.012635735794901848, "sampling/sampling_logp_difference/max": 4.3712263107299805, "sampling/sampling_logp_difference/mean": 0.019755955785512924, "step": 239, "step_time": 204.47343361668754 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13324.0, "completions/max_terminated_length": 13324.0, "completions/mean_length": 8653.3125, "completions/mean_terminated_length": 8653.3125, "completions/min_length": 4673.0, "completions/min_terminated_length": 4673.0, "entropy": 0.2574413791298866, "epoch": 0.28811524609843936, "frac_reward_zero_std": 0.25, "grad_norm": 0.3604678213596344, "learning_rate": 4.4e-08, "loss": -0.0105, "num_tokens": 81421999.0, "reward": 1.671258568763733, "reward_std": 2.822765350341797, "rewards/reward_func/mean": 1.671258568763733, "rewards/reward_func/std": 2.822765588760376, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.407453775405884, "sampling/importance_sampling_ratio/mean": 0.9911367893218994, "sampling/importance_sampling_ratio/min": 0.35551854968070984, "sampling/sampling_logp_difference/max": 1.0341778993606567, "sampling/sampling_logp_difference/mean": 0.018147394061088562, "step": 240, "step_time": 216.4728114741156 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15334.0, "completions/max_terminated_length": 15334.0, "completions/mean_length": 10050.375, "completions/mean_terminated_length": 10050.375, "completions/min_length": 3177.0, "completions/min_terminated_length": 3177.0, "entropy": 0.25266273971647024, "epoch": 0.2893157262905162, "frac_reward_zero_std": 0.25, "grad_norm": 0.34388092160224915, "learning_rate": 4e-08, "loss": -0.0302, "num_tokens": 81783875.0, "reward": 2.149714469909668, "reward_std": 2.923807144165039, "rewards/reward_func/mean": 2.149714469909668, "rewards/reward_func/std": 2.92380690574646, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9911876916885376, "sampling/importance_sampling_ratio/min": 0.00033268670085817575, "sampling/sampling_logp_difference/max": 8.008309364318848, "sampling/sampling_logp_difference/mean": 0.01783919706940651, "step": 241, "step_time": 258.6670099237235 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10994.0, "completions/max_terminated_length": 10994.0, "completions/mean_length": 8268.15625, "completions/mean_terminated_length": 8268.15625, "completions/min_length": 5467.0, "completions/min_terminated_length": 5467.0, "entropy": 0.2947918279096484, "epoch": 0.290516206482593, "frac_reward_zero_std": 0.25, "grad_norm": 0.37617602944374084, "learning_rate": 3.6e-08, "loss": 0.045, "num_tokens": 82076200.0, "reward": 1.3230187892913818, "reward_std": 1.936329960823059, "rewards/reward_func/mean": 1.3230187892913818, "rewards/reward_func/std": 1.9363298416137695, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.1945412158966064, "sampling/importance_sampling_ratio/mean": 0.98971027135849, "sampling/importance_sampling_ratio/min": 0.0968865156173706, "sampling/sampling_logp_difference/max": 2.334214925765991, "sampling/sampling_logp_difference/mean": 0.020341265946626663, "step": 242, "step_time": 192.7506662090309 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12898.0, "completions/max_terminated_length": 12898.0, "completions/mean_length": 9256.78125, "completions/mean_terminated_length": 9256.78125, "completions/min_length": 4686.0, "completions/min_terminated_length": 4686.0, "entropy": 0.30538202077150345, "epoch": 0.2917166866746699, "frac_reward_zero_std": 0.25, "grad_norm": 0.3372436761856079, "learning_rate": 3.2e-08, "loss": 0.0148, "num_tokens": 82395961.0, "reward": 0.8563218712806702, "reward_std": 1.9926910400390625, "rewards/reward_func/mean": 0.8563218712806702, "rewards/reward_func/std": 1.9926910400390625, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9894178509712219, "sampling/importance_sampling_ratio/min": 0.13205546140670776, "sampling/sampling_logp_difference/max": 2.024533271789551, "sampling/sampling_logp_difference/mean": 0.02108946070075035, "step": 243, "step_time": 222.2967141327681 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13215.0, "completions/max_terminated_length": 13215.0, "completions/mean_length": 9513.03125, "completions/mean_terminated_length": 9513.03125, "completions/min_length": 6086.0, "completions/min_terminated_length": 6086.0, "entropy": 0.26438521407544613, "epoch": 0.2929171668667467, "frac_reward_zero_std": 0.0, "grad_norm": 0.40192142128944397, "learning_rate": 2.8e-08, "loss": 0.0154, "num_tokens": 82738746.0, "reward": 3.0766522884368896, "reward_std": 2.7076849937438965, "rewards/reward_func/mean": 3.0766522884368896, "rewards/reward_func/std": 2.7076849937438965, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9906589388847351, "sampling/importance_sampling_ratio/min": 0.27606433629989624, "sampling/sampling_logp_difference/max": 1.287121295928955, "sampling/sampling_logp_difference/mean": 0.018659161403775215, "step": 244, "step_time": 206.91322167252656 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 17517.0, "completions/max_terminated_length": 17517.0, "completions/mean_length": 9881.0625, "completions/mean_terminated_length": 9881.0625, "completions/min_length": 2381.0, "completions/min_terminated_length": 2381.0, "entropy": 0.25762535352259874, "epoch": 0.29411764705882354, "frac_reward_zero_std": 0.0, "grad_norm": 0.40251660346984863, "learning_rate": 2.4e-08, "loss": 0.0311, "num_tokens": 83097732.0, "reward": 3.15523099899292, "reward_std": 5.307023525238037, "rewards/reward_func/mean": 3.163875102996826, "rewards/reward_func/std": 5.301476001739502, "rewards/soft_overlong_punishment_reward/mean": -0.00864410400390625, "rewards/soft_overlong_punishment_reward/std": 0.04889843612909317, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9913111925125122, "sampling/importance_sampling_ratio/min": 0.0033161414321511984, "sampling/sampling_logp_difference/max": 5.708953380584717, "sampling/sampling_logp_difference/mean": 0.01810595765709877, "step": 245, "step_time": 277.0211471380899 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16853.0, "completions/max_terminated_length": 16853.0, "completions/mean_length": 10218.5, "completions/mean_terminated_length": 10218.5, "completions/min_length": 6606.0, "completions/min_terminated_length": 6606.0, "entropy": 0.28045162558555603, "epoch": 0.29531812725090034, "frac_reward_zero_std": 0.0, "grad_norm": 0.3734090030193329, "learning_rate": 2e-08, "loss": 0.001, "num_tokens": 83459092.0, "reward": 0.8230442404747009, "reward_std": 0.6862863898277283, "rewards/reward_func/mean": 0.8266224265098572, "rewards/reward_func/std": 0.6817933917045593, "rewards/soft_overlong_punishment_reward/mean": -0.00357818603515625, "rewards/soft_overlong_punishment_reward/std": 0.020241277292370796, "sampling/importance_sampling_ratio/max": 2.0942885875701904, "sampling/importance_sampling_ratio/mean": 0.990305483341217, "sampling/importance_sampling_ratio/min": 0.018712347373366356, "sampling/sampling_logp_difference/max": 3.978571653366089, "sampling/sampling_logp_difference/mean": 0.019516419619321823, "step": 246, "step_time": 472.73078032024205 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 19484.0, "completions/max_terminated_length": 19484.0, "completions/mean_length": 10279.5625, "completions/mean_terminated_length": 10279.5625, "completions/min_length": 5641.0, "completions/min_terminated_length": 5641.0, "entropy": 0.2904065754264593, "epoch": 0.2965186074429772, "frac_reward_zero_std": 0.0, "grad_norm": 0.3862820565700531, "learning_rate": 1.6e-08, "loss": 0.032, "num_tokens": 83814838.0, "reward": 1.4464482069015503, "reward_std": 2.287731885910034, "rewards/reward_func/mean": 1.4815586805343628, "rewards/reward_func/std": 2.2599167823791504, "rewards/soft_overlong_punishment_reward/mean": -0.0351104736328125, "rewards/soft_overlong_punishment_reward/std": 0.146774023771286, "sampling/importance_sampling_ratio/max": 2.4637815952301025, "sampling/importance_sampling_ratio/mean": 0.989745557308197, "sampling/importance_sampling_ratio/min": 0.15747244656085968, "sampling/sampling_logp_difference/max": 1.8485047817230225, "sampling/sampling_logp_difference/mean": 0.020400259643793106, "step": 247, "step_time": 332.2391231590882 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12802.0, "completions/max_terminated_length": 12802.0, "completions/mean_length": 7361.375, "completions/mean_terminated_length": 7361.375, "completions/min_length": 5259.0, "completions/min_terminated_length": 5259.0, "entropy": 0.3144523575901985, "epoch": 0.297719087635054, "frac_reward_zero_std": 0.0, "grad_norm": 0.45082876086235046, "learning_rate": 1.2e-08, "loss": 0.0151, "num_tokens": 84073074.0, "reward": 1.048694372177124, "reward_std": 0.5158096551895142, "rewards/reward_func/mean": 1.048694372177124, "rewards/reward_func/std": 0.5158095955848694, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.7785451412200928, "sampling/importance_sampling_ratio/mean": 0.9890033602714539, "sampling/importance_sampling_ratio/min": 0.31890588998794556, "sampling/sampling_logp_difference/max": 1.1428592205047607, "sampling/sampling_logp_difference/mean": 0.02159862592816353, "step": 248, "step_time": 215.50420774670783 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14278.0, "completions/max_terminated_length": 14278.0, "completions/mean_length": 9695.75, "completions/mean_terminated_length": 9695.75, "completions/min_length": 4891.0, "completions/min_terminated_length": 4891.0, "entropy": 0.2848109360784292, "epoch": 0.29891956782713086, "frac_reward_zero_std": 0.0, "grad_norm": 0.38538479804992676, "learning_rate": 8e-09, "loss": -0.0234, "num_tokens": 84425578.0, "reward": 1.7004798650741577, "reward_std": 2.567974805831909, "rewards/reward_func/mean": 1.7004798650741577, "rewards/reward_func/std": 2.567974805831909, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.873475432395935, "sampling/importance_sampling_ratio/mean": 0.9899736642837524, "sampling/importance_sampling_ratio/min": 0.29256924986839294, "sampling/sampling_logp_difference/max": 1.2290538549423218, "sampling/sampling_logp_difference/mean": 0.01992029696702957, "step": 249, "step_time": 247.963375205989 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11532.0, "completions/max_terminated_length": 11532.0, "completions/mean_length": 9036.21875, "completions/mean_terminated_length": 9036.21875, "completions/min_length": 4886.0, "completions/min_terminated_length": 4886.0, "entropy": 0.27648542169481516, "epoch": 0.30012004801920766, "frac_reward_zero_std": 0.0, "grad_norm": 0.8751509189605713, "learning_rate": 4e-09, "loss": -0.0151, "num_tokens": 84746073.0, "reward": 3.147611618041992, "reward_std": 4.971174716949463, "rewards/reward_func/mean": 3.147611618041992, "rewards/reward_func/std": 4.971174716949463, "rewards/soft_overlong_punishment_reward/mean": 0.0, "rewards/soft_overlong_punishment_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.4234373569488525, "sampling/importance_sampling_ratio/mean": 0.9902631640434265, "sampling/importance_sampling_ratio/min": 0.09392388164997101, "sampling/sampling_logp_difference/max": 2.3652706146240234, "sampling/sampling_logp_difference/mean": 0.019402729347348213, "step": 250, "step_time": 203.76658588682767 } ], "logging_steps": 1, "max_steps": 250, "num_input_tokens_seen": 84746073, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 2, "trial_name": null, "trial_params": null }