diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,9343 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 620, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0032258064516129032, + "grad_norm": 1.0198665857315063, + "learning_rate": 5e-06, + "logits/chosen": -3.4055747985839844, + "logits/rejected": -3.2500038146972656, + "logps/chosen": -85.92189025878906, + "logps/rejected": -115.17630004882812, + "loss": 0.6931471824645996, + "rewards/accuracies": 0.0, + "rewards/chosen": 0.0, + "rewards/margins": 0.0, + "rewards/rejected": 0.0, + "step": 1 + }, + { + "epoch": 0.0064516129032258064, + "grad_norm": 1.12330162525177, + "learning_rate": 4.999967905881621e-06, + "logits/chosen": -3.139765739440918, + "logits/rejected": -2.9487788677215576, + "logps/chosen": -107.27665710449219, + "logps/rejected": -138.18399047851562, + "loss": 0.7021108269691467, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.01106848195195198, + "rewards/margins": -0.01585940644145012, + "rewards/rejected": 0.004790925420820713, + "step": 2 + }, + { + "epoch": 0.00967741935483871, + "grad_norm": 1.035811424255371, + "learning_rate": 4.99987162435051e-06, + "logits/chosen": -3.148529052734375, + "logits/rejected": -3.0062148571014404, + "logps/chosen": -103.82777404785156, + "logps/rejected": -128.03005981445312, + "loss": 0.668667197227478, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.03935158997774124, + "rewards/margins": 0.05168798193335533, + "rewards/rejected": -0.012336388230323792, + "step": 3 + }, + { + "epoch": 0.012903225806451613, + "grad_norm": 1.1256532669067383, + "learning_rate": 4.999711157878722e-06, + "logits/chosen": -2.938189744949341, + "logits/rejected": -2.779592514038086, + "logps/chosen": -100.74897003173828, + "logps/rejected": -134.83859252929688, + "loss": 0.6965321898460388, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.0025862313341349363, + "rewards/margins": -0.00128110870718956, + "rewards/rejected": 0.0038673398084938526, + "step": 4 + }, + { + "epoch": 0.016129032258064516, + "grad_norm": 1.0809041261672974, + "learning_rate": 4.999486510586282e-06, + "logits/chosen": -2.927344799041748, + "logits/rejected": -2.8265066146850586, + "logps/chosen": -109.85308837890625, + "logps/rejected": -142.212646484375, + "loss": 0.648101806640625, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.011704119853675365, + "rewards/margins": 0.09639576077461243, + "rewards/rejected": -0.08469165116548538, + "step": 5 + }, + { + "epoch": 0.01935483870967742, + "grad_norm": 1.0711078643798828, + "learning_rate": 4.999197688241076e-06, + "logits/chosen": -3.085522413253784, + "logits/rejected": -2.979081153869629, + "logps/chosen": -111.31352233886719, + "logps/rejected": -131.05545043945312, + "loss": 0.6814134120941162, + "rewards/accuracies": 0.5625, + "rewards/chosen": 0.013973352499306202, + "rewards/margins": 0.031338244676589966, + "rewards/rejected": -0.01736488938331604, + "step": 6 + }, + { + "epoch": 0.02258064516129032, + "grad_norm": 1.1095834970474243, + "learning_rate": 4.998844698258704e-06, + "logits/chosen": -2.7590060234069824, + "logits/rejected": -2.5014760494232178, + "logps/chosen": -102.21183776855469, + "logps/rejected": -141.93539428710938, + "loss": 0.6776243448257446, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.013474291190505028, + "rewards/margins": 0.0413118414580822, + "rewards/rejected": -0.054786134511232376, + "step": 7 + }, + { + "epoch": 0.025806451612903226, + "grad_norm": 0.9900579452514648, + "learning_rate": 4.998427549702284e-06, + "logits/chosen": -3.2602813243865967, + "logits/rejected": -3.1628756523132324, + "logps/chosen": -93.34729766845703, + "logps/rejected": -111.89982604980469, + "loss": 0.673986554145813, + "rewards/accuracies": 0.59375, + "rewards/chosen": -0.03970911726355553, + "rewards/margins": 0.04514574632048607, + "rewards/rejected": -0.08485487103462219, + "step": 8 + }, + { + "epoch": 0.02903225806451613, + "grad_norm": 1.0532326698303223, + "learning_rate": 4.9979462532822305e-06, + "logits/chosen": -3.270956516265869, + "logits/rejected": -3.086606502532959, + "logps/chosen": -101.70204162597656, + "logps/rejected": -145.21560668945312, + "loss": 0.6543633937835693, + "rewards/accuracies": 0.59375, + "rewards/chosen": -0.07679509371519089, + "rewards/margins": 0.0888080894947052, + "rewards/rejected": -0.16560319066047668, + "step": 9 + }, + { + "epoch": 0.03225806451612903, + "grad_norm": 1.2684348821640015, + "learning_rate": 4.9974008213559725e-06, + "logits/chosen": -3.132490396499634, + "logits/rejected": -3.018911838531494, + "logps/chosen": -116.5405502319336, + "logps/rejected": -137.92234802246094, + "loss": 0.6893508434295654, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.13492733240127563, + "rewards/margins": 0.019388491287827492, + "rewards/rejected": -0.15431582927703857, + "step": 10 + }, + { + "epoch": 0.035483870967741936, + "grad_norm": 1.0866929292678833, + "learning_rate": 4.996791267927632e-06, + "logits/chosen": -2.9289159774780273, + "logits/rejected": -2.7686662673950195, + "logps/chosen": -116.60456085205078, + "logps/rejected": -143.89141845703125, + "loss": 0.6853089332580566, + "rewards/accuracies": 0.53125, + "rewards/chosen": -0.15715695917606354, + "rewards/margins": 0.03740931302309036, + "rewards/rejected": -0.1945662796497345, + "step": 11 + }, + { + "epoch": 0.03870967741935484, + "grad_norm": 1.0992016792297363, + "learning_rate": 4.996117608647676e-06, + "logits/chosen": -2.9300031661987305, + "logits/rejected": -2.8163886070251465, + "logps/chosen": -125.79206085205078, + "logps/rejected": -144.17791748046875, + "loss": 0.6614134907722473, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1761740744113922, + "rewards/margins": 0.08016557991504669, + "rewards/rejected": -0.2563396394252777, + "step": 12 + }, + { + "epoch": 0.041935483870967745, + "grad_norm": 1.0385782718658447, + "learning_rate": 4.995379860812503e-06, + "logits/chosen": -3.0234384536743164, + "logits/rejected": -2.918104648590088, + "logps/chosen": -88.80581665039062, + "logps/rejected": -118.65660095214844, + "loss": 0.6563053131103516, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.11144424974918365, + "rewards/margins": 0.09995441138744354, + "rewards/rejected": -0.2113986611366272, + "step": 13 + }, + { + "epoch": 0.04516129032258064, + "grad_norm": 1.0566327571868896, + "learning_rate": 4.994578043364007e-06, + "logits/chosen": -3.111154556274414, + "logits/rejected": -2.9898476600646973, + "logps/chosen": -108.92998504638672, + "logps/rejected": -138.91552734375, + "loss": 0.6367413997650146, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.17835265398025513, + "rewards/margins": 0.1417873203754425, + "rewards/rejected": -0.32013997435569763, + "step": 14 + }, + { + "epoch": 0.04838709677419355, + "grad_norm": 0.9933570623397827, + "learning_rate": 4.993712176889086e-06, + "logits/chosen": -3.0580408573150635, + "logits/rejected": -2.8851232528686523, + "logps/chosen": -98.05117797851562, + "logps/rejected": -131.82333374023438, + "loss": 0.6436130404472351, + "rewards/accuracies": 0.53125, + "rewards/chosen": -0.2019367814064026, + "rewards/margins": 0.1264682561159134, + "rewards/rejected": -0.3284050226211548, + "step": 15 + }, + { + "epoch": 0.05161290322580645, + "grad_norm": 0.9798956513404846, + "learning_rate": 4.9927822836191185e-06, + "logits/chosen": -3.128474235534668, + "logits/rejected": -2.990847110748291, + "logps/chosen": -111.88795471191406, + "logps/rejected": -138.34701538085938, + "loss": 0.6226807832717896, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.21347865462303162, + "rewards/margins": 0.1850709319114685, + "rewards/rejected": -0.39854955673217773, + "step": 16 + }, + { + "epoch": 0.054838709677419356, + "grad_norm": 1.058609962463379, + "learning_rate": 4.991788387429388e-06, + "logits/chosen": -2.850104331970215, + "logits/rejected": -2.7300033569335938, + "logps/chosen": -113.05485534667969, + "logps/rejected": -142.72119140625, + "loss": 0.6618618369102478, + "rewards/accuracies": 0.59375, + "rewards/chosen": -0.37465834617614746, + "rewards/margins": 0.10391812026500702, + "rewards/rejected": -0.4785764813423157, + "step": 17 + }, + { + "epoch": 0.05806451612903226, + "grad_norm": 1.1318104267120361, + "learning_rate": 4.990730513838472e-06, + "logits/chosen": -3.082528591156006, + "logits/rejected": -2.8970086574554443, + "logps/chosen": -97.33485412597656, + "logps/rejected": -129.76806640625, + "loss": 0.605815589427948, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.34302327036857605, + "rewards/margins": 0.24319365620613098, + "rewards/rejected": -0.586216926574707, + "step": 18 + }, + { + "epoch": 0.06129032258064516, + "grad_norm": 0.9978364706039429, + "learning_rate": 4.9896086900075865e-06, + "logits/chosen": -2.8832178115844727, + "logits/rejected": -2.745048761367798, + "logps/chosen": -112.83102416992188, + "logps/rejected": -140.89207458496094, + "loss": 0.6403385400772095, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.4615880846977234, + "rewards/margins": 0.1403249055147171, + "rewards/rejected": -0.6019130349159241, + "step": 19 + }, + { + "epoch": 0.06451612903225806, + "grad_norm": 1.0464746952056885, + "learning_rate": 4.988422944739889e-06, + "logits/chosen": -2.8241257667541504, + "logits/rejected": -2.7562663555145264, + "logps/chosen": -109.80278015136719, + "logps/rejected": -139.4794921875, + "loss": 0.6105231046676636, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.40774962306022644, + "rewards/margins": 0.19766247272491455, + "rewards/rejected": -0.6054120659828186, + "step": 20 + }, + { + "epoch": 0.06774193548387097, + "grad_norm": 1.0746631622314453, + "learning_rate": 4.987173308479738e-06, + "logits/chosen": -3.0489606857299805, + "logits/rejected": -2.9206061363220215, + "logps/chosen": -107.85366821289062, + "logps/rejected": -138.60345458984375, + "loss": 0.6688461899757385, + "rewards/accuracies": 0.59375, + "rewards/chosen": -0.4516957104206085, + "rewards/margins": 0.10383725166320801, + "rewards/rejected": -0.5555329322814941, + "step": 21 + }, + { + "epoch": 0.07096774193548387, + "grad_norm": 1.0326505899429321, + "learning_rate": 4.985859813311914e-06, + "logits/chosen": -3.1783857345581055, + "logits/rejected": -3.0726547241210938, + "logps/chosen": -115.14910888671875, + "logps/rejected": -140.83590698242188, + "loss": 0.6399143934249878, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.5430137515068054, + "rewards/margins": 0.1574849635362625, + "rewards/rejected": -0.7004987001419067, + "step": 22 + }, + { + "epoch": 0.07419354838709677, + "grad_norm": 1.038557767868042, + "learning_rate": 4.984482492960792e-06, + "logits/chosen": -2.906726360321045, + "logits/rejected": -2.7238097190856934, + "logps/chosen": -105.02792358398438, + "logps/rejected": -135.632568359375, + "loss": 0.633417546749115, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.7342426180839539, + "rewards/margins": 0.15591225028038025, + "rewards/rejected": -0.8901548385620117, + "step": 23 + }, + { + "epoch": 0.07741935483870968, + "grad_norm": 0.9582953453063965, + "learning_rate": 4.983041382789478e-06, + "logits/chosen": -3.3154075145721436, + "logits/rejected": -3.2139768600463867, + "logps/chosen": -92.43376159667969, + "logps/rejected": -118.47042083740234, + "loss": 0.6151463985443115, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.6794711947441101, + "rewards/margins": 0.23817875981330872, + "rewards/rejected": -0.9176499843597412, + "step": 24 + }, + { + "epoch": 0.08064516129032258, + "grad_norm": 1.001360535621643, + "learning_rate": 4.981536519798899e-06, + "logits/chosen": -2.97209095954895, + "logits/rejected": -2.7885656356811523, + "logps/chosen": -111.38399505615234, + "logps/rejected": -144.8062744140625, + "loss": 0.6042543649673462, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.7312175035476685, + "rewards/margins": 0.26750829815864563, + "rewards/rejected": -0.9987257122993469, + "step": 25 + }, + { + "epoch": 0.08387096774193549, + "grad_norm": 1.0800671577453613, + "learning_rate": 4.9799679426268575e-06, + "logits/chosen": -2.918426990509033, + "logits/rejected": -2.8081374168395996, + "logps/chosen": -121.90701293945312, + "logps/rejected": -149.9063720703125, + "loss": 0.6208748817443848, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.9017692804336548, + "rewards/margins": 0.21920597553253174, + "rewards/rejected": -1.1209752559661865, + "step": 26 + }, + { + "epoch": 0.08709677419354839, + "grad_norm": 1.1123878955841064, + "learning_rate": 4.978335691547035e-06, + "logits/chosen": -3.031327247619629, + "logits/rejected": -2.892502784729004, + "logps/chosen": -109.16461181640625, + "logps/rejected": -138.34585571289062, + "loss": 0.6401875019073486, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.7755411267280579, + "rewards/margins": 0.2256021499633789, + "rewards/rejected": -1.001143217086792, + "step": 27 + }, + { + "epoch": 0.09032258064516129, + "grad_norm": 1.013746738433838, + "learning_rate": 4.976639808467957e-06, + "logits/chosen": -2.7579987049102783, + "logits/rejected": -2.5802083015441895, + "logps/chosen": -113.30269622802734, + "logps/rejected": -141.89920043945312, + "loss": 0.6180602312088013, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.8010274171829224, + "rewards/margins": 0.23785457015037537, + "rewards/rejected": -1.0388820171356201, + "step": 28 + }, + { + "epoch": 0.0935483870967742, + "grad_norm": 1.4393163919448853, + "learning_rate": 4.974880336931923e-06, + "logits/chosen": -2.799999952316284, + "logits/rejected": -2.6729190349578857, + "logps/chosen": -136.82716369628906, + "logps/rejected": -157.08663940429688, + "loss": 0.7176676988601685, + "rewards/accuracies": 0.46875, + "rewards/chosen": -1.0027340650558472, + "rewards/margins": 0.07778176665306091, + "rewards/rejected": -1.0805158615112305, + "step": 29 + }, + { + "epoch": 0.0967741935483871, + "grad_norm": 1.0653221607208252, + "learning_rate": 4.973057322113883e-06, + "logits/chosen": -3.0367493629455566, + "logits/rejected": -2.8530149459838867, + "logps/chosen": -102.79076385498047, + "logps/rejected": -136.4600830078125, + "loss": 0.6130280494689941, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.9152212142944336, + "rewards/margins": 0.359441876411438, + "rewards/rejected": -1.2746630907058716, + "step": 30 + }, + { + "epoch": 0.1, + "grad_norm": 0.9103254675865173, + "learning_rate": 4.971170810820279e-06, + "logits/chosen": -2.9878039360046387, + "logits/rejected": -2.897019863128662, + "logps/chosen": -113.10630798339844, + "logps/rejected": -142.175537109375, + "loss": 0.5397008657455444, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.9760595560073853, + "rewards/margins": 0.45220765471458435, + "rewards/rejected": -1.428267240524292, + "step": 31 + }, + { + "epoch": 0.1032258064516129, + "grad_norm": 1.028403639793396, + "learning_rate": 4.9692208514878445e-06, + "logits/chosen": -3.098134994506836, + "logits/rejected": -3.0521764755249023, + "logps/chosen": -111.02950286865234, + "logps/rejected": -137.7850341796875, + "loss": 0.6879711747169495, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.1832095384597778, + "rewards/margins": 0.1095898449420929, + "rewards/rejected": -1.2927993535995483, + "step": 32 + }, + { + "epoch": 0.1064516129032258, + "grad_norm": 0.9742422103881836, + "learning_rate": 4.967207494182361e-06, + "logits/chosen": -2.9038455486297607, + "logits/rejected": -2.777085542678833, + "logps/chosen": -114.38245391845703, + "logps/rejected": -147.81060791015625, + "loss": 0.5629678964614868, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.9888625144958496, + "rewards/margins": 0.44450241327285767, + "rewards/rejected": -1.4333648681640625, + "step": 33 + }, + { + "epoch": 0.10967741935483871, + "grad_norm": 0.9404062628746033, + "learning_rate": 4.965130790597369e-06, + "logits/chosen": -3.026500701904297, + "logits/rejected": -2.8468196392059326, + "logps/chosen": -110.82350158691406, + "logps/rejected": -139.73025512695312, + "loss": 0.5878911018371582, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.8643066883087158, + "rewards/margins": 0.43531858921051025, + "rewards/rejected": -1.2996253967285156, + "step": 34 + }, + { + "epoch": 0.11290322580645161, + "grad_norm": 0.9421868920326233, + "learning_rate": 4.962990794052847e-06, + "logits/chosen": -2.985588550567627, + "logits/rejected": -2.7967123985290527, + "logps/chosen": -114.51589965820312, + "logps/rejected": -140.2834930419922, + "loss": 0.5792871713638306, + "rewards/accuracies": 0.71875, + "rewards/chosen": -1.0735702514648438, + "rewards/margins": 0.41116511821746826, + "rewards/rejected": -1.484735369682312, + "step": 35 + }, + { + "epoch": 0.11612903225806452, + "grad_norm": 0.9833240509033203, + "learning_rate": 4.960787559493836e-06, + "logits/chosen": -2.819369316101074, + "logits/rejected": -2.6581039428710938, + "logps/chosen": -126.71798706054688, + "logps/rejected": -154.09063720703125, + "loss": 0.5701221227645874, + "rewards/accuracies": 0.78125, + "rewards/chosen": -1.0069059133529663, + "rewards/margins": 0.3965708017349243, + "rewards/rejected": -1.4034767150878906, + "step": 36 + }, + { + "epoch": 0.11935483870967742, + "grad_norm": 0.9587810039520264, + "learning_rate": 4.958521143489032e-06, + "logits/chosen": -2.9700632095336914, + "logits/rejected": -2.7929463386535645, + "logps/chosen": -117.53889465332031, + "logps/rejected": -141.08412170410156, + "loss": 0.5579148530960083, + "rewards/accuracies": 0.71875, + "rewards/chosen": -1.2454460859298706, + "rewards/margins": 0.46080076694488525, + "rewards/rejected": -1.7062468528747559, + "step": 37 + }, + { + "epoch": 0.12258064516129032, + "grad_norm": 0.952377200126648, + "learning_rate": 4.956191604229335e-06, + "logits/chosen": -2.7670180797576904, + "logits/rejected": -2.653087615966797, + "logps/chosen": -122.13555908203125, + "logps/rejected": -154.33908081054688, + "loss": 0.5420794486999512, + "rewards/accuracies": 0.71875, + "rewards/chosen": -1.417367935180664, + "rewards/margins": 0.5341382026672363, + "rewards/rejected": -1.9515061378479004, + "step": 38 + }, + { + "epoch": 0.12580645161290321, + "grad_norm": 0.9858971834182739, + "learning_rate": 4.9537990015263505e-06, + "logits/chosen": -2.846860647201538, + "logits/rejected": -2.6801350116729736, + "logps/chosen": -109.1124267578125, + "logps/rejected": -141.81492614746094, + "loss": 0.5824028253555298, + "rewards/accuracies": 0.65625, + "rewards/chosen": -1.316831350326538, + "rewards/margins": 0.42448168992996216, + "rewards/rejected": -1.7413129806518555, + "step": 39 + }, + { + "epoch": 0.12903225806451613, + "grad_norm": 1.1429812908172607, + "learning_rate": 4.95134339681086e-06, + "logits/chosen": -3.008500337600708, + "logits/rejected": -2.8374381065368652, + "logps/chosen": -111.08134460449219, + "logps/rejected": -140.59872436523438, + "loss": 0.6458199620246887, + "rewards/accuracies": 0.65625, + "rewards/chosen": -1.5981895923614502, + "rewards/margins": 0.2978651523590088, + "rewards/rejected": -1.896054744720459, + "step": 40 + }, + { + "epoch": 0.13225806451612904, + "grad_norm": 0.8448069095611572, + "learning_rate": 4.948824853131237e-06, + "logits/chosen": -2.907655715942383, + "logits/rejected": -2.7536003589630127, + "logps/chosen": -113.13584899902344, + "logps/rejected": -144.23391723632812, + "loss": 0.4713582396507263, + "rewards/accuracies": 0.84375, + "rewards/chosen": -1.285538673400879, + "rewards/margins": 0.6696875095367432, + "rewards/rejected": -1.955226182937622, + "step": 41 + }, + { + "epoch": 0.13548387096774195, + "grad_norm": 1.1781965494155884, + "learning_rate": 4.946243435151833e-06, + "logits/chosen": -2.7036216259002686, + "logits/rejected": -2.5935046672821045, + "logps/chosen": -133.15252685546875, + "logps/rejected": -154.00750732421875, + "loss": 0.7647944688796997, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.6891489028930664, + "rewards/margins": 0.04440240561962128, + "rewards/rejected": -1.733551263809204, + "step": 42 + }, + { + "epoch": 0.13870967741935483, + "grad_norm": 0.9805737733840942, + "learning_rate": 4.943599209151314e-06, + "logits/chosen": -2.7226176261901855, + "logits/rejected": -2.517169237136841, + "logps/chosen": -125.3690185546875, + "logps/rejected": -162.36477661132812, + "loss": 0.5878539085388184, + "rewards/accuracies": 0.6875, + "rewards/chosen": -1.4429672956466675, + "rewards/margins": 0.4025247097015381, + "rewards/rejected": -1.8454921245574951, + "step": 43 + }, + { + "epoch": 0.14193548387096774, + "grad_norm": 0.8965495228767395, + "learning_rate": 4.940892243020964e-06, + "logits/chosen": -2.879836082458496, + "logits/rejected": -2.6803863048553467, + "logps/chosen": -111.92822265625, + "logps/rejected": -144.03843688964844, + "loss": 0.5402083396911621, + "rewards/accuracies": 0.71875, + "rewards/chosen": -1.189293384552002, + "rewards/margins": 0.5704534649848938, + "rewards/rejected": -1.759746789932251, + "step": 44 + }, + { + "epoch": 0.14516129032258066, + "grad_norm": 1.0602669715881348, + "learning_rate": 4.938122606262935e-06, + "logits/chosen": -2.8707027435302734, + "logits/rejected": -2.67893123626709, + "logps/chosen": -125.02201080322266, + "logps/rejected": -153.2523956298828, + "loss": 0.6121110916137695, + "rewards/accuracies": 0.6875, + "rewards/chosen": -1.6958320140838623, + "rewards/margins": 0.445503294467926, + "rewards/rejected": -2.1413354873657227, + "step": 45 + }, + { + "epoch": 0.14838709677419354, + "grad_norm": 0.927438497543335, + "learning_rate": 4.935290369988468e-06, + "logits/chosen": -2.7243499755859375, + "logits/rejected": -2.563385486602783, + "logps/chosen": -132.1870880126953, + "logps/rejected": -171.1417236328125, + "loss": 0.4953867793083191, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.4100818634033203, + "rewards/margins": 0.6493280529975891, + "rewards/rejected": -2.0594100952148438, + "step": 46 + }, + { + "epoch": 0.15161290322580645, + "grad_norm": 0.8959330916404724, + "learning_rate": 4.932395606916063e-06, + "logits/chosen": -2.4819681644439697, + "logits/rejected": -2.2971127033233643, + "logps/chosen": -122.25642395019531, + "logps/rejected": -159.7163543701172, + "loss": 0.4567532241344452, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.6163623332977295, + "rewards/margins": 0.7664045691490173, + "rewards/rejected": -2.3827669620513916, + "step": 47 + }, + { + "epoch": 0.15483870967741936, + "grad_norm": 0.854138970375061, + "learning_rate": 4.9294383913696145e-06, + "logits/chosen": -2.6553072929382324, + "logits/rejected": -2.431748628616333, + "logps/chosen": -122.9141616821289, + "logps/rejected": -171.38320922851562, + "loss": 0.4372290372848511, + "rewards/accuracies": 0.78125, + "rewards/chosen": -1.455812692642212, + "rewards/margins": 0.9929529428482056, + "rewards/rejected": -2.448765754699707, + "step": 48 + }, + { + "epoch": 0.15806451612903225, + "grad_norm": 0.7815476655960083, + "learning_rate": 4.926418799276505e-06, + "logits/chosen": -2.765216827392578, + "logits/rejected": -2.5981082916259766, + "logps/chosen": -111.628173828125, + "logps/rejected": -138.0706024169922, + "loss": 0.479827880859375, + "rewards/accuracies": 0.84375, + "rewards/chosen": -1.5760267972946167, + "rewards/margins": 0.758277177810669, + "rewards/rejected": -2.334303855895996, + "step": 49 + }, + { + "epoch": 0.16129032258064516, + "grad_norm": 1.0033049583435059, + "learning_rate": 4.923336908165649e-06, + "logits/chosen": -2.8565001487731934, + "logits/rejected": -2.618370294570923, + "logps/chosen": -108.14005279541016, + "logps/rejected": -148.12445068359375, + "loss": 0.42941758036613464, + "rewards/accuracies": 0.90625, + "rewards/chosen": -1.5442157983779907, + "rewards/margins": 0.8375117182731628, + "rewards/rejected": -2.381727695465088, + "step": 50 + }, + { + "epoch": 0.16451612903225807, + "grad_norm": 0.904968798160553, + "learning_rate": 4.920192797165511e-06, + "logits/chosen": -2.844186782836914, + "logits/rejected": -2.6398918628692627, + "logps/chosen": -125.80914306640625, + "logps/rejected": -163.47320556640625, + "loss": 0.471732497215271, + "rewards/accuracies": 0.71875, + "rewards/chosen": -1.6446901559829712, + "rewards/margins": 0.8008061647415161, + "rewards/rejected": -2.4454965591430664, + "step": 51 + }, + { + "epoch": 0.16774193548387098, + "grad_norm": 0.9294782280921936, + "learning_rate": 4.916986547002067e-06, + "logits/chosen": -2.8216707706451416, + "logits/rejected": -2.7185850143432617, + "logps/chosen": -124.8201904296875, + "logps/rejected": -155.616455078125, + "loss": 0.4811817705631256, + "rewards/accuracies": 0.71875, + "rewards/chosen": -2.0028302669525146, + "rewards/margins": 0.6963652968406677, + "rewards/rejected": -2.699195623397827, + "step": 52 + }, + { + "epoch": 0.17096774193548386, + "grad_norm": 0.8786033391952515, + "learning_rate": 4.913718239996734e-06, + "logits/chosen": -2.832566261291504, + "logits/rejected": -2.6430118083953857, + "logps/chosen": -122.54075622558594, + "logps/rejected": -158.80088806152344, + "loss": 0.5157533884048462, + "rewards/accuracies": 0.6875, + "rewards/chosen": -1.8400042057037354, + "rewards/margins": 0.6210693717002869, + "rewards/rejected": -2.461073398590088, + "step": 53 + }, + { + "epoch": 0.17419354838709677, + "grad_norm": 0.8750291466712952, + "learning_rate": 4.910387960064259e-06, + "logits/chosen": -2.6718342304229736, + "logits/rejected": -2.5271155834198, + "logps/chosen": -131.72975158691406, + "logps/rejected": -167.03482055664062, + "loss": 0.4826596975326538, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.0635039806365967, + "rewards/margins": 0.7045244574546814, + "rewards/rejected": -2.768028736114502, + "step": 54 + }, + { + "epoch": 0.1774193548387097, + "grad_norm": 0.8463932275772095, + "learning_rate": 4.906995792710559e-06, + "logits/chosen": -2.793056011199951, + "logits/rejected": -2.6334168910980225, + "logps/chosen": -108.91849517822266, + "logps/rejected": -152.1488800048828, + "loss": 0.4304446578025818, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.7403039932250977, + "rewards/margins": 1.1434907913208008, + "rewards/rejected": -2.8837947845458984, + "step": 55 + }, + { + "epoch": 0.18064516129032257, + "grad_norm": 0.8992181420326233, + "learning_rate": 4.9035418250305314e-06, + "logits/chosen": -2.832119941711426, + "logits/rejected": -2.666569709777832, + "logps/chosen": -107.66205596923828, + "logps/rejected": -141.9404754638672, + "loss": 0.5210002660751343, + "rewards/accuracies": 0.65625, + "rewards/chosen": -2.036715030670166, + "rewards/margins": 0.9046896696090698, + "rewards/rejected": -2.9414048194885254, + "step": 56 + }, + { + "epoch": 0.18387096774193548, + "grad_norm": 1.4827739000320435, + "learning_rate": 4.900026145705815e-06, + "logits/chosen": -2.7140512466430664, + "logits/rejected": -2.4837052822113037, + "logps/chosen": -121.64815521240234, + "logps/rejected": -159.62066650390625, + "loss": 0.4381827414035797, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.80775785446167, + "rewards/margins": 1.076915979385376, + "rewards/rejected": -2.884673833847046, + "step": 57 + }, + { + "epoch": 0.1870967741935484, + "grad_norm": 1.0054007768630981, + "learning_rate": 4.896448845002511e-06, + "logits/chosen": -2.706526517868042, + "logits/rejected": -2.487896680831909, + "logps/chosen": -122.99580383300781, + "logps/rejected": -156.40049743652344, + "loss": 0.5580032467842102, + "rewards/accuracies": 0.71875, + "rewards/chosen": -2.2970235347747803, + "rewards/margins": 0.7499445080757141, + "rewards/rejected": -3.0469677448272705, + "step": 58 + }, + { + "epoch": 0.19032258064516128, + "grad_norm": 1.0605281591415405, + "learning_rate": 4.8928100147688705e-06, + "logits/chosen": -2.9178082942962646, + "logits/rejected": -2.6102988719940186, + "logps/chosen": -114.86691284179688, + "logps/rejected": -154.53656005859375, + "loss": 0.6092868447303772, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.3286643028259277, + "rewards/margins": 0.5103977918624878, + "rewards/rejected": -2.839062213897705, + "step": 59 + }, + { + "epoch": 0.1935483870967742, + "grad_norm": 0.7592489123344421, + "learning_rate": 4.889109748432932e-06, + "logits/chosen": -2.9385297298431396, + "logits/rejected": -2.6411538124084473, + "logps/chosen": -115.66314697265625, + "logps/rejected": -149.95323181152344, + "loss": 0.4233652651309967, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.9713090658187866, + "rewards/margins": 0.8833580613136292, + "rewards/rejected": -2.8546671867370605, + "step": 60 + }, + { + "epoch": 0.1967741935483871, + "grad_norm": 0.7794229388237, + "learning_rate": 4.8853481410001225e-06, + "logits/chosen": -2.9496335983276367, + "logits/rejected": -2.7019166946411133, + "logps/chosen": -106.0539779663086, + "logps/rejected": -148.11280822753906, + "loss": 0.4723885953426361, + "rewards/accuracies": 0.84375, + "rewards/chosen": -2.460500717163086, + "rewards/margins": 1.022800087928772, + "rewards/rejected": -3.4833006858825684, + "step": 61 + }, + { + "epoch": 0.2, + "grad_norm": 0.9357581734657288, + "learning_rate": 4.881525289050823e-06, + "logits/chosen": -2.77065372467041, + "logits/rejected": -2.6192283630371094, + "logps/chosen": -130.91944885253906, + "logps/rejected": -149.3533935546875, + "loss": 0.5586632490158081, + "rewards/accuracies": 0.6875, + "rewards/chosen": -2.1316962242126465, + "rewards/margins": 0.6954216957092285, + "rewards/rejected": -2.827117681503296, + "step": 62 + }, + { + "epoch": 0.2032258064516129, + "grad_norm": 0.7778428196907043, + "learning_rate": 4.8776412907378845e-06, + "logits/chosen": -2.7207984924316406, + "logits/rejected": -2.508455276489258, + "logps/chosen": -100.13325500488281, + "logps/rejected": -154.1763916015625, + "loss": 0.4038059711456299, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.372382640838623, + "rewards/margins": 1.304573893547058, + "rewards/rejected": -3.6769564151763916, + "step": 63 + }, + { + "epoch": 0.2064516129032258, + "grad_norm": 0.9325763583183289, + "learning_rate": 4.873696245784106e-06, + "logits/chosen": -2.285740375518799, + "logits/rejected": -2.1688804626464844, + "logps/chosen": -133.5624542236328, + "logps/rejected": -158.0366668701172, + "loss": 0.5424321293830872, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.1653249263763428, + "rewards/margins": 0.7148800492286682, + "rewards/rejected": -2.880204677581787, + "step": 64 + }, + { + "epoch": 0.20967741935483872, + "grad_norm": 0.9311462044715881, + "learning_rate": 4.8696902554796824e-06, + "logits/chosen": -2.6557462215423584, + "logits/rejected": -2.5266714096069336, + "logps/chosen": -129.286376953125, + "logps/rejected": -164.18695068359375, + "loss": 0.5229653120040894, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.370563507080078, + "rewards/margins": 0.8608935475349426, + "rewards/rejected": -3.231456995010376, + "step": 65 + }, + { + "epoch": 0.2129032258064516, + "grad_norm": 0.9174665212631226, + "learning_rate": 4.865623422679593e-06, + "logits/chosen": -2.8143653869628906, + "logits/rejected": -2.586948871612549, + "logps/chosen": -108.68220520019531, + "logps/rejected": -140.1006622314453, + "loss": 0.5581181049346924, + "rewards/accuracies": 0.71875, + "rewards/chosen": -2.5576014518737793, + "rewards/margins": 0.7612634897232056, + "rewards/rejected": -3.3188650608062744, + "step": 66 + }, + { + "epoch": 0.2161290322580645, + "grad_norm": 1.003322958946228, + "learning_rate": 4.861495851800968e-06, + "logits/chosen": -2.7197985649108887, + "logits/rejected": -2.613180160522461, + "logps/chosen": -132.04977416992188, + "logps/rejected": -156.76223754882812, + "loss": 0.6440731287002563, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.7013721466064453, + "rewards/margins": 0.3256293535232544, + "rewards/rejected": -3.02700138092041, + "step": 67 + }, + { + "epoch": 0.21935483870967742, + "grad_norm": 0.7528925538063049, + "learning_rate": 4.857307648820409e-06, + "logits/chosen": -2.47288179397583, + "logits/rejected": -2.3103513717651367, + "logps/chosen": -133.77581787109375, + "logps/rejected": -167.21572875976562, + "loss": 0.4236561059951782, + "rewards/accuracies": 0.78125, + "rewards/chosen": -2.5838000774383545, + "rewards/margins": 1.0029029846191406, + "rewards/rejected": -3.586703062057495, + "step": 68 + }, + { + "epoch": 0.22258064516129034, + "grad_norm": 1.148950457572937, + "learning_rate": 4.853058921271259e-06, + "logits/chosen": -2.322709798812866, + "logits/rejected": -2.1581592559814453, + "logps/chosen": -134.0292205810547, + "logps/rejected": -167.06533813476562, + "loss": 0.4543103277683258, + "rewards/accuracies": 0.71875, + "rewards/chosen": -2.312931537628174, + "rewards/margins": 1.102271556854248, + "rewards/rejected": -3.415203094482422, + "step": 69 + }, + { + "epoch": 0.22580645161290322, + "grad_norm": 0.8065391778945923, + "learning_rate": 4.84874977824085e-06, + "logits/chosen": -2.314046859741211, + "logits/rejected": -2.165867805480957, + "logps/chosen": -131.90957641601562, + "logps/rejected": -164.817626953125, + "loss": 0.43934166431427, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.61452579498291, + "rewards/margins": 1.0483322143554688, + "rewards/rejected": -3.662858009338379, + "step": 70 + }, + { + "epoch": 0.22903225806451613, + "grad_norm": 0.70816570520401, + "learning_rate": 4.844380330367701e-06, + "logits/chosen": -2.426614284515381, + "logits/rejected": -2.2347359657287598, + "logps/chosen": -127.90621185302734, + "logps/rejected": -165.37213134765625, + "loss": 0.3680841326713562, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.334412097930908, + "rewards/margins": 1.211841344833374, + "rewards/rejected": -3.546253204345703, + "step": 71 + }, + { + "epoch": 0.23225806451612904, + "grad_norm": 0.7996051907539368, + "learning_rate": 4.839950689838674e-06, + "logits/chosen": -2.362088680267334, + "logits/rejected": -2.163127899169922, + "logps/chosen": -144.364501953125, + "logps/rejected": -182.89500427246094, + "loss": 0.43939828872680664, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.8185462951660156, + "rewards/margins": 0.9022763967514038, + "rewards/rejected": -3.72082257270813, + "step": 72 + }, + { + "epoch": 0.23548387096774193, + "grad_norm": 0.7642343044281006, + "learning_rate": 4.835460970386093e-06, + "logits/chosen": -2.3497159481048584, + "logits/rejected": -2.1166491508483887, + "logps/chosen": -134.3424835205078, + "logps/rejected": -175.50112915039062, + "loss": 0.4305233061313629, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.630145311355591, + "rewards/margins": 0.9792301654815674, + "rewards/rejected": -3.609375476837158, + "step": 73 + }, + { + "epoch": 0.23870967741935484, + "grad_norm": 0.9543027877807617, + "learning_rate": 4.83091128728483e-06, + "logits/chosen": -2.4201152324676514, + "logits/rejected": -2.2325682640075684, + "logps/chosen": -127.69469451904297, + "logps/rejected": -152.89657592773438, + "loss": 0.632413387298584, + "rewards/accuracies": 0.6875, + "rewards/chosen": -2.69545841217041, + "rewards/margins": 0.48617762327194214, + "rewards/rejected": -3.181636095046997, + "step": 74 + }, + { + "epoch": 0.24193548387096775, + "grad_norm": 0.6152648329734802, + "learning_rate": 4.826301757349337e-06, + "logits/chosen": -2.612950563430786, + "logits/rejected": -2.358858585357666, + "logps/chosen": -117.93904113769531, + "logps/rejected": -162.6607666015625, + "loss": 0.3176124095916748, + "rewards/accuracies": 0.90625, + "rewards/chosen": -2.620302438735962, + "rewards/margins": 1.325659990310669, + "rewards/rejected": -3.945962429046631, + "step": 75 + }, + { + "epoch": 0.24516129032258063, + "grad_norm": 0.771643340587616, + "learning_rate": 4.821632498930656e-06, + "logits/chosen": -2.359376907348633, + "logits/rejected": -2.176957607269287, + "logps/chosen": -137.79673767089844, + "logps/rejected": -167.8568115234375, + "loss": 0.4373764991760254, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.7868497371673584, + "rewards/margins": 1.0548901557922363, + "rewards/rejected": -3.8417396545410156, + "step": 76 + }, + { + "epoch": 0.24838709677419354, + "grad_norm": 0.7945637106895447, + "learning_rate": 4.816903631913372e-06, + "logits/chosen": -2.5555307865142822, + "logits/rejected": -2.3610169887542725, + "logps/chosen": -121.39008331298828, + "logps/rejected": -156.8350067138672, + "loss": 0.47777238488197327, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.8233864307403564, + "rewards/margins": 0.945284366607666, + "rewards/rejected": -3.7686707973480225, + "step": 77 + }, + { + "epoch": 0.25161290322580643, + "grad_norm": 0.7087196707725525, + "learning_rate": 4.812115277712539e-06, + "logits/chosen": -2.5300731658935547, + "logits/rejected": -2.3655776977539062, + "logps/chosen": -127.15848541259766, + "logps/rejected": -163.5643310546875, + "loss": 0.4068886637687683, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.928337574005127, + "rewards/margins": 1.12626314163208, + "rewards/rejected": -4.054600715637207, + "step": 78 + }, + { + "epoch": 0.25483870967741934, + "grad_norm": 0.6794445514678955, + "learning_rate": 4.807267559270563e-06, + "logits/chosen": -2.559274911880493, + "logits/rejected": -2.374232769012451, + "logps/chosen": -132.4824981689453, + "logps/rejected": -171.53933715820312, + "loss": 0.37371641397476196, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.011838436126709, + "rewards/margins": 1.4339759349822998, + "rewards/rejected": -4.44581413269043, + "step": 79 + }, + { + "epoch": 0.25806451612903225, + "grad_norm": 0.9782342314720154, + "learning_rate": 4.802360601054042e-06, + "logits/chosen": -2.2338345050811768, + "logits/rejected": -2.005814790725708, + "logps/chosen": -131.63963317871094, + "logps/rejected": -168.6005401611328, + "loss": 0.5639984607696533, + "rewards/accuracies": 0.6875, + "rewards/chosen": -3.223055839538574, + "rewards/margins": 0.7143847942352295, + "rewards/rejected": -3.9374403953552246, + "step": 80 + }, + { + "epoch": 0.26129032258064516, + "grad_norm": 0.7722119092941284, + "learning_rate": 4.797394529050577e-06, + "logits/chosen": -2.2159998416900635, + "logits/rejected": -1.9760271310806274, + "logps/chosen": -138.00804138183594, + "logps/rejected": -191.0758056640625, + "loss": 0.35425710678100586, + "rewards/accuracies": 0.78125, + "rewards/chosen": -3.038595676422119, + "rewards/margins": 1.5962517261505127, + "rewards/rejected": -4.634847640991211, + "step": 81 + }, + { + "epoch": 0.2645161290322581, + "grad_norm": 0.781527578830719, + "learning_rate": 4.792369470765528e-06, + "logits/chosen": -2.332066059112549, + "logits/rejected": -2.1308486461639404, + "logps/chosen": -134.24362182617188, + "logps/rejected": -175.5936279296875, + "loss": 0.410447895526886, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.196249485015869, + "rewards/margins": 1.2090076208114624, + "rewards/rejected": -4.405257225036621, + "step": 82 + }, + { + "epoch": 0.267741935483871, + "grad_norm": 1.4148427248001099, + "learning_rate": 4.787285555218748e-06, + "logits/chosen": -2.541987895965576, + "logits/rejected": -2.3397560119628906, + "logps/chosen": -125.69288635253906, + "logps/rejected": -169.29953002929688, + "loss": 0.3594983220100403, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.1516847610473633, + "rewards/margins": 1.4236831665039062, + "rewards/rejected": -4.5753679275512695, + "step": 83 + }, + { + "epoch": 0.2709677419354839, + "grad_norm": 0.8784657120704651, + "learning_rate": 4.782142912941268e-06, + "logits/chosen": -2.2671427726745605, + "logits/rejected": -2.1503725051879883, + "logps/chosen": -147.03195190429688, + "logps/rejected": -179.34947204589844, + "loss": 0.5158642530441284, + "rewards/accuracies": 0.78125, + "rewards/chosen": -3.4720330238342285, + "rewards/margins": 0.8386823534965515, + "rewards/rejected": -4.310715198516846, + "step": 84 + }, + { + "epoch": 0.27419354838709675, + "grad_norm": 0.6953436732292175, + "learning_rate": 4.776941675971941e-06, + "logits/chosen": -2.3191490173339844, + "logits/rejected": -2.090640068054199, + "logps/chosen": -114.88062286376953, + "logps/rejected": -165.6884765625, + "loss": 0.3098244369029999, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.7739365100860596, + "rewards/margins": 2.004124879837036, + "rewards/rejected": -4.7780609130859375, + "step": 85 + }, + { + "epoch": 0.27741935483870966, + "grad_norm": 1.1434965133666992, + "learning_rate": 4.771681977854062e-06, + "logits/chosen": -2.098564624786377, + "logits/rejected": -1.9888715744018555, + "logps/chosen": -151.57363891601562, + "logps/rejected": -181.7247314453125, + "loss": 0.5879724025726318, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.6309304237365723, + "rewards/margins": 0.8588354587554932, + "rewards/rejected": -4.489765644073486, + "step": 86 + }, + { + "epoch": 0.2806451612903226, + "grad_norm": 0.7763506174087524, + "learning_rate": 4.7663639536319304e-06, + "logits/chosen": -2.165168046951294, + "logits/rejected": -1.9731296300888062, + "logps/chosen": -139.3979034423828, + "logps/rejected": -178.37872314453125, + "loss": 0.45243924856185913, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.6948037147521973, + "rewards/margins": 1.2531442642211914, + "rewards/rejected": -4.947947978973389, + "step": 87 + }, + { + "epoch": 0.2838709677419355, + "grad_norm": 0.7684445381164551, + "learning_rate": 4.760987739847383e-06, + "logits/chosen": -2.128974199295044, + "logits/rejected": -1.9912292957305908, + "logps/chosen": -132.04293823242188, + "logps/rejected": -171.23690795898438, + "loss": 0.4750169515609741, + "rewards/accuracies": 0.84375, + "rewards/chosen": -3.4750771522521973, + "rewards/margins": 0.9534658193588257, + "rewards/rejected": -4.4285430908203125, + "step": 88 + }, + { + "epoch": 0.2870967741935484, + "grad_norm": 0.7377817630767822, + "learning_rate": 4.755553474536294e-06, + "logits/chosen": -2.0988073348999023, + "logits/rejected": -1.902956247329712, + "logps/chosen": -141.26329040527344, + "logps/rejected": -177.63734436035156, + "loss": 0.43820852041244507, + "rewards/accuracies": 0.84375, + "rewards/chosen": -3.559805393218994, + "rewards/margins": 1.0688958168029785, + "rewards/rejected": -4.628701210021973, + "step": 89 + }, + { + "epoch": 0.2903225806451613, + "grad_norm": 0.6428540945053101, + "learning_rate": 4.750061297225028e-06, + "logits/chosen": -2.3366332054138184, + "logits/rejected": -2.1300148963928223, + "logps/chosen": -128.06130981445312, + "logps/rejected": -175.9393768310547, + "loss": 0.32989248633384705, + "rewards/accuracies": 0.90625, + "rewards/chosen": -3.4514639377593994, + "rewards/margins": 1.4905400276184082, + "rewards/rejected": -4.9420037269592285, + "step": 90 + }, + { + "epoch": 0.29354838709677417, + "grad_norm": 0.7805766463279724, + "learning_rate": 4.744511348926855e-06, + "logits/chosen": -2.0861973762512207, + "logits/rejected": -1.9446799755096436, + "logps/chosen": -138.7248077392578, + "logps/rejected": -170.54974365234375, + "loss": 0.454301118850708, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.870506763458252, + "rewards/margins": 1.0038337707519531, + "rewards/rejected": -4.874340534210205, + "step": 91 + }, + { + "epoch": 0.2967741935483871, + "grad_norm": 0.8089374303817749, + "learning_rate": 4.7389037721383325e-06, + "logits/chosen": -2.1515049934387207, + "logits/rejected": -1.9613507986068726, + "logps/chosen": -138.1538848876953, + "logps/rejected": -171.8699951171875, + "loss": 0.42631715536117554, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.4975502490997314, + "rewards/margins": 1.3891708850860596, + "rewards/rejected": -4.886720657348633, + "step": 92 + }, + { + "epoch": 0.3, + "grad_norm": 0.8091368675231934, + "learning_rate": 4.733238710835648e-06, + "logits/chosen": -1.947023868560791, + "logits/rejected": -1.7635173797607422, + "logps/chosen": -159.03768920898438, + "logps/rejected": -197.48944091796875, + "loss": 0.47380709648132324, + "rewards/accuracies": 0.6875, + "rewards/chosen": -4.0391106605529785, + "rewards/margins": 1.2754480838775635, + "rewards/rejected": -5.314558982849121, + "step": 93 + }, + { + "epoch": 0.3032258064516129, + "grad_norm": 0.7619990110397339, + "learning_rate": 4.72751631047092e-06, + "logits/chosen": -2.460020065307617, + "logits/rejected": -2.312316417694092, + "logps/chosen": -139.6368408203125, + "logps/rejected": -178.7040557861328, + "loss": 0.48125335574150085, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.6689484119415283, + "rewards/margins": 1.3045544624328613, + "rewards/rejected": -4.973503112792969, + "step": 94 + }, + { + "epoch": 0.3064516129032258, + "grad_norm": 0.6219649910926819, + "learning_rate": 4.721736717968465e-06, + "logits/chosen": -1.954379916191101, + "logits/rejected": -1.7585666179656982, + "logps/chosen": -145.32232666015625, + "logps/rejected": -185.040771484375, + "loss": 0.3768487572669983, + "rewards/accuracies": 0.84375, + "rewards/chosen": -3.559706926345825, + "rewards/margins": 1.4724528789520264, + "rewards/rejected": -5.032159805297852, + "step": 95 + }, + { + "epoch": 0.3096774193548387, + "grad_norm": 0.7134078741073608, + "learning_rate": 4.715900081721021e-06, + "logits/chosen": -1.9498239755630493, + "logits/rejected": -1.7411441802978516, + "logps/chosen": -143.168701171875, + "logps/rejected": -181.01402282714844, + "loss": 0.4029703140258789, + "rewards/accuracies": 0.84375, + "rewards/chosen": -3.5624961853027344, + "rewards/margins": 1.211552619934082, + "rewards/rejected": -4.774048805236816, + "step": 96 + }, + { + "epoch": 0.31290322580645163, + "grad_norm": 0.7324331402778625, + "learning_rate": 4.7100065515859464e-06, + "logits/chosen": -2.3302841186523438, + "logits/rejected": -2.1917896270751953, + "logps/chosen": -123.150634765625, + "logps/rejected": -165.6932373046875, + "loss": 0.43078821897506714, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.5430026054382324, + "rewards/margins": 1.245470404624939, + "rewards/rejected": -4.788473129272461, + "step": 97 + }, + { + "epoch": 0.3161290322580645, + "grad_norm": 0.7748900651931763, + "learning_rate": 4.704056278881364e-06, + "logits/chosen": -1.8153760433197021, + "logits/rejected": -1.6069302558898926, + "logps/chosen": -138.3715362548828, + "logps/rejected": -180.33352661132812, + "loss": 0.4739260673522949, + "rewards/accuracies": 0.71875, + "rewards/chosen": -3.8387598991394043, + "rewards/margins": 1.237455129623413, + "rewards/rejected": -5.076214790344238, + "step": 98 + }, + { + "epoch": 0.3193548387096774, + "grad_norm": 0.7280204892158508, + "learning_rate": 4.698049416382277e-06, + "logits/chosen": -2.13297176361084, + "logits/rejected": -1.9608895778656006, + "logps/chosen": -140.18515014648438, + "logps/rejected": -181.5408935546875, + "loss": 0.5066293478012085, + "rewards/accuracies": 0.78125, + "rewards/chosen": -3.9341986179351807, + "rewards/margins": 1.4060182571411133, + "rewards/rejected": -5.340217113494873, + "step": 99 + }, + { + "epoch": 0.3225806451612903, + "grad_norm": 0.6968139410018921, + "learning_rate": 4.691986118316654e-06, + "logits/chosen": -2.0721585750579834, + "logits/rejected": -1.9022136926651, + "logps/chosen": -136.25216674804688, + "logps/rejected": -178.95773315429688, + "loss": 0.41828787326812744, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.9730870723724365, + "rewards/margins": 1.3703317642211914, + "rewards/rejected": -5.343419075012207, + "step": 100 + }, + { + "epoch": 0.3258064516129032, + "grad_norm": 0.838504433631897, + "learning_rate": 4.685866540361456e-06, + "logits/chosen": -1.8501439094543457, + "logits/rejected": -1.6102899312973022, + "logps/chosen": -149.552978515625, + "logps/rejected": -200.82159423828125, + "loss": 0.4767953157424927, + "rewards/accuracies": 0.71875, + "rewards/chosen": -4.187356948852539, + "rewards/margins": 1.5327056646347046, + "rewards/rejected": -5.720062255859375, + "step": 101 + }, + { + "epoch": 0.32903225806451614, + "grad_norm": 0.8937166929244995, + "learning_rate": 4.679690839638652e-06, + "logits/chosen": -1.9279000759124756, + "logits/rejected": -1.8128533363342285, + "logps/chosen": -136.27706909179688, + "logps/rejected": -172.40426635742188, + "loss": 0.5405164957046509, + "rewards/accuracies": 0.71875, + "rewards/chosen": -4.254796504974365, + "rewards/margins": 1.0396616458892822, + "rewards/rejected": -5.294458389282227, + "step": 102 + }, + { + "epoch": 0.33225806451612905, + "grad_norm": 0.9040670990943909, + "learning_rate": 4.673459174711178e-06, + "logits/chosen": -1.983347773551941, + "logits/rejected": -1.779608964920044, + "logps/chosen": -136.58360290527344, + "logps/rejected": -177.53189086914062, + "loss": 0.5051164627075195, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.310066223144531, + "rewards/margins": 1.0662548542022705, + "rewards/rejected": -5.376320838928223, + "step": 103 + }, + { + "epoch": 0.33548387096774196, + "grad_norm": 0.6081104874610901, + "learning_rate": 4.6671717055788675e-06, + "logits/chosen": -2.1749377250671387, + "logits/rejected": -1.8677318096160889, + "logps/chosen": -116.5163345336914, + "logps/rejected": -171.98924255371094, + "loss": 0.3056028187274933, + "rewards/accuracies": 0.84375, + "rewards/chosen": -3.4489426612854004, + "rewards/margins": 2.0040981769561768, + "rewards/rejected": -5.453040599822998, + "step": 104 + }, + { + "epoch": 0.3387096774193548, + "grad_norm": 0.7601423859596252, + "learning_rate": 4.660828593674344e-06, + "logits/chosen": -1.764312505722046, + "logits/rejected": -1.5848668813705444, + "logps/chosen": -151.72991943359375, + "logps/rejected": -196.42237854003906, + "loss": 0.39171379804611206, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.2623291015625, + "rewards/margins": 1.3185968399047852, + "rewards/rejected": -5.580925464630127, + "step": 105 + }, + { + "epoch": 0.3419354838709677, + "grad_norm": 0.7062326669692993, + "learning_rate": 4.654430001858874e-06, + "logits/chosen": -1.9848129749298096, + "logits/rejected": -1.7872685194015503, + "logps/chosen": -160.7518310546875, + "logps/rejected": -198.28277587890625, + "loss": 0.3898007273674011, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.177804946899414, + "rewards/margins": 1.501824140548706, + "rewards/rejected": -5.679628849029541, + "step": 106 + }, + { + "epoch": 0.34516129032258064, + "grad_norm": 0.7348251342773438, + "learning_rate": 4.64797609441819e-06, + "logits/chosen": -2.0225300788879395, + "logits/rejected": -1.909589409828186, + "logps/chosen": -161.80633544921875, + "logps/rejected": -194.06915283203125, + "loss": 0.4167836606502533, + "rewards/accuracies": 0.78125, + "rewards/chosen": -4.554932117462158, + "rewards/margins": 1.3303924798965454, + "rewards/rejected": -5.885324478149414, + "step": 107 + }, + { + "epoch": 0.34838709677419355, + "grad_norm": 0.6127936244010925, + "learning_rate": 4.641467037058264e-06, + "logits/chosen": -1.7549917697906494, + "logits/rejected": -1.5760438442230225, + "logps/chosen": -134.44053649902344, + "logps/rejected": -176.8062744140625, + "loss": 0.3423386216163635, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.8029491901397705, + "rewards/margins": 1.6363918781280518, + "rewards/rejected": -5.439341068267822, + "step": 108 + }, + { + "epoch": 0.35161290322580646, + "grad_norm": 0.778430163860321, + "learning_rate": 4.634902996901065e-06, + "logits/chosen": -2.3479363918304443, + "logits/rejected": -2.1825313568115234, + "logps/chosen": -135.63031005859375, + "logps/rejected": -177.00146484375, + "loss": 0.46787750720977783, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.918789863586426, + "rewards/margins": 1.161969780921936, + "rewards/rejected": -5.080759525299072, + "step": 109 + }, + { + "epoch": 0.3548387096774194, + "grad_norm": 0.9373202919960022, + "learning_rate": 4.628284142480256e-06, + "logits/chosen": -1.9162163734436035, + "logits/rejected": -1.7722245454788208, + "logps/chosen": -154.00108337402344, + "logps/rejected": -198.04161071777344, + "loss": 0.5773978233337402, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.933002471923828, + "rewards/margins": 1.0351779460906982, + "rewards/rejected": -5.9681806564331055, + "step": 110 + }, + { + "epoch": 0.3580645161290323, + "grad_norm": 0.8041221499443054, + "learning_rate": 4.621610643736878e-06, + "logits/chosen": -2.068186044692993, + "logits/rejected": -1.924422264099121, + "logps/chosen": -141.45101928710938, + "logps/rejected": -175.1551055908203, + "loss": 0.5035527944564819, + "rewards/accuracies": 0.71875, + "rewards/chosen": -4.4271955490112305, + "rewards/margins": 1.2815918922424316, + "rewards/rejected": -5.708787441253662, + "step": 111 + }, + { + "epoch": 0.36129032258064514, + "grad_norm": 0.8020619750022888, + "learning_rate": 4.614882672014975e-06, + "logits/chosen": -1.8857342004776, + "logits/rejected": -1.6965585947036743, + "logps/chosen": -145.23861694335938, + "logps/rejected": -185.3410186767578, + "loss": 0.4687632918357849, + "rewards/accuracies": 0.71875, + "rewards/chosen": -4.3011884689331055, + "rewards/margins": 1.2721554040908813, + "rewards/rejected": -5.5733442306518555, + "step": 112 + }, + { + "epoch": 0.36451612903225805, + "grad_norm": 0.9149249196052551, + "learning_rate": 4.608100400057206e-06, + "logits/chosen": -1.7518459558486938, + "logits/rejected": -1.629075050354004, + "logps/chosen": -138.32345581054688, + "logps/rejected": -159.2861785888672, + "loss": 0.5886192917823792, + "rewards/accuracies": 0.625, + "rewards/chosen": -4.447440147399902, + "rewards/margins": 0.8766449689865112, + "rewards/rejected": -5.324085235595703, + "step": 113 + }, + { + "epoch": 0.36774193548387096, + "grad_norm": 0.9651359915733337, + "learning_rate": 4.601264002000401e-06, + "logits/chosen": -1.7441390752792358, + "logits/rejected": -1.5445384979248047, + "logps/chosen": -157.6129150390625, + "logps/rejected": -211.08529663085938, + "loss": 0.26776331663131714, + "rewards/accuracies": 0.84375, + "rewards/chosen": -4.520727157592773, + "rewards/margins": 2.193711519241333, + "rewards/rejected": -6.714438438415527, + "step": 114 + }, + { + "epoch": 0.3709677419354839, + "grad_norm": 0.9474994540214539, + "learning_rate": 4.594373653371095e-06, + "logits/chosen": -2.02126407623291, + "logits/rejected": -1.8315093517303467, + "logps/chosen": -147.41787719726562, + "logps/rejected": -182.80154418945312, + "loss": 0.6504412293434143, + "rewards/accuracies": 0.6875, + "rewards/chosen": -4.341954708099365, + "rewards/margins": 0.9969633221626282, + "rewards/rejected": -5.3389177322387695, + "step": 115 + }, + { + "epoch": 0.3741935483870968, + "grad_norm": 0.7878331542015076, + "learning_rate": 4.587429531081019e-06, + "logits/chosen": -1.8967138528823853, + "logits/rejected": -1.7459156513214111, + "logps/chosen": -148.2082977294922, + "logps/rejected": -193.70852661132812, + "loss": 0.47428834438323975, + "rewards/accuracies": 0.78125, + "rewards/chosen": -4.678706645965576, + "rewards/margins": 1.3782602548599243, + "rewards/rejected": -6.056967258453369, + "step": 116 + }, + { + "epoch": 0.3774193548387097, + "grad_norm": 0.7913106679916382, + "learning_rate": 4.58043181342256e-06, + "logits/chosen": -2.071049213409424, + "logits/rejected": -1.8497998714447021, + "logps/chosen": -130.3113250732422, + "logps/rejected": -169.584228515625, + "loss": 0.3851902484893799, + "rewards/accuracies": 0.84375, + "rewards/chosen": -3.724860191345215, + "rewards/margins": 1.4488303661346436, + "rewards/rejected": -5.1736907958984375, + "step": 117 + }, + { + "epoch": 0.38064516129032255, + "grad_norm": 0.8633780479431152, + "learning_rate": 4.573380680064182e-06, + "logits/chosen": -2.0043578147888184, + "logits/rejected": -1.8761675357818604, + "logps/chosen": -154.30734252929688, + "logps/rejected": -186.54644775390625, + "loss": 0.4509797990322113, + "rewards/accuracies": 0.84375, + "rewards/chosen": -4.510204315185547, + "rewards/margins": 1.4796333312988281, + "rewards/rejected": -5.989838123321533, + "step": 118 + }, + { + "epoch": 0.38387096774193546, + "grad_norm": 0.6685623526573181, + "learning_rate": 4.56627631204581e-06, + "logits/chosen": -1.7765039205551147, + "logits/rejected": -1.6367878913879395, + "logps/chosen": -146.83633422851562, + "logps/rejected": -185.90267944335938, + "loss": 0.3379678726196289, + "rewards/accuracies": 0.84375, + "rewards/chosen": -4.602056980133057, + "rewards/margins": 1.587829828262329, + "rewards/rejected": -6.189886569976807, + "step": 119 + }, + { + "epoch": 0.3870967741935484, + "grad_norm": 0.7955985069274902, + "learning_rate": 4.559118891774188e-06, + "logits/chosen": -1.9045928716659546, + "logits/rejected": -1.7364416122436523, + "logps/chosen": -155.47647094726562, + "logps/rejected": -202.11959838867188, + "loss": 0.43171846866607666, + "rewards/accuracies": 0.78125, + "rewards/chosen": -4.851472854614258, + "rewards/margins": 1.498561143875122, + "rewards/rejected": -6.350033760070801, + "step": 120 + }, + { + "epoch": 0.3903225806451613, + "grad_norm": 0.7248550057411194, + "learning_rate": 4.551908603018191e-06, + "logits/chosen": -1.9593379497528076, + "logits/rejected": -1.8014620542526245, + "logps/chosen": -152.8414306640625, + "logps/rejected": -201.25314331054688, + "loss": 0.4184379279613495, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.653316497802734, + "rewards/margins": 1.7582554817199707, + "rewards/rejected": -6.411572456359863, + "step": 121 + }, + { + "epoch": 0.3935483870967742, + "grad_norm": 0.9849889874458313, + "learning_rate": 4.544645630904106e-06, + "logits/chosen": -1.7142870426177979, + "logits/rejected": -1.4873881340026855, + "logps/chosen": -148.92617797851562, + "logps/rejected": -194.00558471679688, + "loss": 0.6807998418807983, + "rewards/accuracies": 0.625, + "rewards/chosen": -4.889307975769043, + "rewards/margins": 1.3624918460845947, + "rewards/rejected": -6.251799583435059, + "step": 122 + }, + { + "epoch": 0.3967741935483871, + "grad_norm": 0.6775311827659607, + "learning_rate": 4.537330161910886e-06, + "logits/chosen": -1.7008912563323975, + "logits/rejected": -1.497498869895935, + "logps/chosen": -151.26678466796875, + "logps/rejected": -198.6688995361328, + "loss": 0.3310524523258209, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.602053642272949, + "rewards/margins": 1.850661277770996, + "rewards/rejected": -6.452714920043945, + "step": 123 + }, + { + "epoch": 0.4, + "grad_norm": 0.678304135799408, + "learning_rate": 4.529962383865349e-06, + "logits/chosen": -2.0466251373291016, + "logits/rejected": -1.8669424057006836, + "logps/chosen": -139.44723510742188, + "logps/rejected": -187.18295288085938, + "loss": 0.4072120189666748, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.538634300231934, + "rewards/margins": 1.4482630491256714, + "rewards/rejected": -5.986896991729736, + "step": 124 + }, + { + "epoch": 0.4032258064516129, + "grad_norm": 0.6273353099822998, + "learning_rate": 4.522542485937369e-06, + "logits/chosen": -1.9951345920562744, + "logits/rejected": -1.8110040426254272, + "logps/chosen": -153.32125854492188, + "logps/rejected": -200.26136779785156, + "loss": 0.37352219223976135, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.848642826080322, + "rewards/margins": 2.124814987182617, + "rewards/rejected": -6.973457336425781, + "step": 125 + }, + { + "epoch": 0.4064516129032258, + "grad_norm": 0.6553208827972412, + "learning_rate": 4.515070658635013e-06, + "logits/chosen": -2.2541942596435547, + "logits/rejected": -2.039757490158081, + "logps/chosen": -132.11251831054688, + "logps/rejected": -180.3010711669922, + "loss": 0.28102147579193115, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.45237398147583, + "rewards/margins": 2.0114006996154785, + "rewards/rejected": -6.463775157928467, + "step": 126 + }, + { + "epoch": 0.4096774193548387, + "grad_norm": 0.5834031701087952, + "learning_rate": 4.507547093799648e-06, + "logits/chosen": -2.0005455017089844, + "logits/rejected": -1.7890722751617432, + "logps/chosen": -145.47592163085938, + "logps/rejected": -187.8236083984375, + "loss": 0.2758556008338928, + "rewards/accuracies": 0.90625, + "rewards/chosen": -4.4232354164123535, + "rewards/margins": 1.7412219047546387, + "rewards/rejected": -6.164457321166992, + "step": 127 + }, + { + "epoch": 0.4129032258064516, + "grad_norm": 0.8683422803878784, + "learning_rate": 4.49997198460102e-06, + "logits/chosen": -1.6352663040161133, + "logits/rejected": -1.5131361484527588, + "logps/chosen": -159.2270965576172, + "logps/rejected": -194.1593780517578, + "loss": 0.5153539180755615, + "rewards/accuracies": 0.65625, + "rewards/chosen": -5.191349029541016, + "rewards/margins": 1.322933316230774, + "rewards/rejected": -6.514283180236816, + "step": 128 + }, + { + "epoch": 0.4161290322580645, + "grad_norm": 0.6821188926696777, + "learning_rate": 4.492345525532288e-06, + "logits/chosen": -2.017963171005249, + "logits/rejected": -1.8079328536987305, + "logps/chosen": -139.15896606445312, + "logps/rejected": -180.54229736328125, + "loss": 0.370669960975647, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.349949836730957, + "rewards/margins": 1.6598310470581055, + "rewards/rejected": -6.009780406951904, + "step": 129 + }, + { + "epoch": 0.41935483870967744, + "grad_norm": 0.8660934567451477, + "learning_rate": 4.484667912405038e-06, + "logits/chosen": -1.5309303998947144, + "logits/rejected": -1.3144739866256714, + "logps/chosen": -149.6410369873047, + "logps/rejected": -205.96067810058594, + "loss": 0.5232619047164917, + "rewards/accuracies": 0.65625, + "rewards/chosen": -5.3874006271362305, + "rewards/margins": 1.597520112991333, + "rewards/rejected": -6.984920501708984, + "step": 130 + }, + { + "epoch": 0.42258064516129035, + "grad_norm": 0.6375477313995361, + "learning_rate": 4.476939342344246e-06, + "logits/chosen": -1.796859860420227, + "logits/rejected": -1.5831903219223022, + "logps/chosen": -138.51058959960938, + "logps/rejected": -201.3404541015625, + "loss": 0.3488934636116028, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.85967493057251, + "rewards/margins": 2.1172547340393066, + "rewards/rejected": -6.976929664611816, + "step": 131 + }, + { + "epoch": 0.4258064516129032, + "grad_norm": 0.7806010246276855, + "learning_rate": 4.469160013783231e-06, + "logits/chosen": -1.8287787437438965, + "logits/rejected": -1.6582368612289429, + "logps/chosen": -161.08322143554688, + "logps/rejected": -204.06918334960938, + "loss": 0.4884554147720337, + "rewards/accuracies": 0.8125, + "rewards/chosen": -5.596766471862793, + "rewards/margins": 1.6360560655593872, + "rewards/rejected": -7.232822418212891, + "step": 132 + }, + { + "epoch": 0.4290322580645161, + "grad_norm": 0.7818947434425354, + "learning_rate": 4.461330126458544e-06, + "logits/chosen": -1.6161482334136963, + "logits/rejected": -1.4406477212905884, + "logps/chosen": -164.41973876953125, + "logps/rejected": -198.6041259765625, + "loss": 0.4584003686904907, + "rewards/accuracies": 0.78125, + "rewards/chosen": -5.313333511352539, + "rewards/margins": 1.2815909385681152, + "rewards/rejected": -6.594923973083496, + "step": 133 + }, + { + "epoch": 0.432258064516129, + "grad_norm": 0.806587815284729, + "learning_rate": 4.453449881404849e-06, + "logits/chosen": -1.5250306129455566, + "logits/rejected": -1.3801239728927612, + "logps/chosen": -167.21826171875, + "logps/rejected": -209.19235229492188, + "loss": 0.4259398579597473, + "rewards/accuracies": 0.8125, + "rewards/chosen": -5.2768168449401855, + "rewards/margins": 1.6459184885025024, + "rewards/rejected": -6.922735214233398, + "step": 134 + }, + { + "epoch": 0.43548387096774194, + "grad_norm": 0.7138063907623291, + "learning_rate": 4.445519480949761e-06, + "logits/chosen": -2.032522201538086, + "logits/rejected": -1.8461859226226807, + "logps/chosen": -139.901611328125, + "logps/rejected": -176.5264434814453, + "loss": 0.49521660804748535, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.6639556884765625, + "rewards/margins": 1.493511438369751, + "rewards/rejected": -6.157467842102051, + "step": 135 + }, + { + "epoch": 0.43870967741935485, + "grad_norm": 0.754875123500824, + "learning_rate": 4.437539128708647e-06, + "logits/chosen": -2.0726211071014404, + "logits/rejected": -1.909871220588684, + "logps/chosen": -146.85386657714844, + "logps/rejected": -191.2569580078125, + "loss": 0.406306654214859, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.798932075500488, + "rewards/margins": 1.6079185009002686, + "rewards/rejected": -6.4068498611450195, + "step": 136 + }, + { + "epoch": 0.44193548387096776, + "grad_norm": 0.8478382229804993, + "learning_rate": 4.429509029579405e-06, + "logits/chosen": -1.7719223499298096, + "logits/rejected": -1.5842394828796387, + "logps/chosen": -148.5624542236328, + "logps/rejected": -185.21328735351562, + "loss": 0.5379185676574707, + "rewards/accuracies": 0.71875, + "rewards/chosen": -5.318252086639404, + "rewards/margins": 1.0121225118637085, + "rewards/rejected": -6.330374240875244, + "step": 137 + }, + { + "epoch": 0.44516129032258067, + "grad_norm": 0.5856479406356812, + "learning_rate": 4.421429389737196e-06, + "logits/chosen": -2.040005683898926, + "logits/rejected": -1.7596739530563354, + "logps/chosen": -122.85853576660156, + "logps/rejected": -174.30482482910156, + "loss": 0.28845539689064026, + "rewards/accuracies": 0.90625, + "rewards/chosen": -4.208980560302734, + "rewards/margins": 2.153196096420288, + "rewards/rejected": -6.362176418304443, + "step": 138 + }, + { + "epoch": 0.4483870967741935, + "grad_norm": 0.8572343587875366, + "learning_rate": 4.413300416629154e-06, + "logits/chosen": -1.5686633586883545, + "logits/rejected": -1.4056203365325928, + "logps/chosen": -174.7129669189453, + "logps/rejected": -208.05381774902344, + "loss": 0.5018977522850037, + "rewards/accuracies": 0.78125, + "rewards/chosen": -5.936600685119629, + "rewards/margins": 1.217970848083496, + "rewards/rejected": -7.154571056365967, + "step": 139 + }, + { + "epoch": 0.45161290322580644, + "grad_norm": 0.96775883436203, + "learning_rate": 4.4051223189690585e-06, + "logits/chosen": -1.9180068969726562, + "logits/rejected": -1.7714178562164307, + "logps/chosen": -158.73330688476562, + "logps/rejected": -201.98733520507812, + "loss": 0.6449487209320068, + "rewards/accuracies": 0.71875, + "rewards/chosen": -5.293525218963623, + "rewards/margins": 1.344764232635498, + "rewards/rejected": -6.638289928436279, + "step": 140 + }, + { + "epoch": 0.45483870967741935, + "grad_norm": 0.7184005379676819, + "learning_rate": 4.396895306731978e-06, + "logits/chosen": -2.029144048690796, + "logits/rejected": -1.8254203796386719, + "logps/chosen": -159.30470275878906, + "logps/rejected": -202.5131072998047, + "loss": 0.3617313802242279, + "rewards/accuracies": 0.78125, + "rewards/chosen": -5.206952095031738, + "rewards/margins": 1.84663987159729, + "rewards/rejected": -7.053591728210449, + "step": 141 + }, + { + "epoch": 0.45806451612903226, + "grad_norm": 0.6119160056114197, + "learning_rate": 4.388619591148875e-06, + "logits/chosen": -1.702289342880249, + "logits/rejected": -1.4510695934295654, + "logps/chosen": -146.5979766845703, + "logps/rejected": -190.3888397216797, + "loss": 0.3313656151294708, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.872472286224365, + "rewards/margins": 2.124823808670044, + "rewards/rejected": -6.997296333312988, + "step": 142 + }, + { + "epoch": 0.4612903225806452, + "grad_norm": 0.8173593282699585, + "learning_rate": 4.380295384701186e-06, + "logits/chosen": -1.592297077178955, + "logits/rejected": -1.371172308921814, + "logps/chosen": -161.451416015625, + "logps/rejected": -208.75277709960938, + "loss": 0.48401570320129395, + "rewards/accuracies": 0.75, + "rewards/chosen": -5.649110317230225, + "rewards/margins": 2.1297523975372314, + "rewards/rejected": -7.778862476348877, + "step": 143 + }, + { + "epoch": 0.4645161290322581, + "grad_norm": 0.8249958753585815, + "learning_rate": 4.371922901115367e-06, + "logits/chosen": -1.7442785501480103, + "logits/rejected": -1.5723543167114258, + "logps/chosen": -154.55165100097656, + "logps/rejected": -187.0003204345703, + "loss": 0.504601776599884, + "rewards/accuracies": 0.75, + "rewards/chosen": -5.453955173492432, + "rewards/margins": 1.3852546215057373, + "rewards/rejected": -6.839210510253906, + "step": 144 + }, + { + "epoch": 0.46774193548387094, + "grad_norm": 0.6368381381034851, + "learning_rate": 4.3635023553574e-06, + "logits/chosen": -1.7754096984863281, + "logits/rejected": -1.607948899269104, + "logps/chosen": -134.15525817871094, + "logps/rejected": -196.18272399902344, + "loss": 0.25741511583328247, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.806275367736816, + "rewards/margins": 2.7785263061523438, + "rewards/rejected": -7.584801197052002, + "step": 145 + }, + { + "epoch": 0.47096774193548385, + "grad_norm": 0.615225076675415, + "learning_rate": 4.355033963627277e-06, + "logits/chosen": -1.9712847471237183, + "logits/rejected": -1.742748737335205, + "logps/chosen": -144.54286193847656, + "logps/rejected": -198.54017639160156, + "loss": 0.29680246114730835, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.868497371673584, + "rewards/margins": 2.3029892444610596, + "rewards/rejected": -7.1714863777160645, + "step": 146 + }, + { + "epoch": 0.47419354838709676, + "grad_norm": 0.7174091339111328, + "learning_rate": 4.346517943353454e-06, + "logits/chosen": -1.79178786277771, + "logits/rejected": -1.5321691036224365, + "logps/chosen": -157.57235717773438, + "logps/rejected": -212.04339599609375, + "loss": 0.3283613622188568, + "rewards/accuracies": 0.84375, + "rewards/chosen": -5.730096817016602, + "rewards/margins": 2.1431491374969482, + "rewards/rejected": -7.873246192932129, + "step": 147 + }, + { + "epoch": 0.4774193548387097, + "grad_norm": 0.9096499085426331, + "learning_rate": 4.337954513187261e-06, + "logits/chosen": -1.5230883359909058, + "logits/rejected": -1.295538067817688, + "logps/chosen": -171.4082794189453, + "logps/rejected": -230.9026641845703, + "loss": 0.4473147392272949, + "rewards/accuracies": 0.84375, + "rewards/chosen": -6.100051403045654, + "rewards/margins": 2.206608772277832, + "rewards/rejected": -8.306659698486328, + "step": 148 + }, + { + "epoch": 0.4806451612903226, + "grad_norm": 0.850471019744873, + "learning_rate": 4.3293438929972896e-06, + "logits/chosen": -1.798743486404419, + "logits/rejected": -1.6180821657180786, + "logps/chosen": -151.7516632080078, + "logps/rejected": -185.33834838867188, + "loss": 0.49031680822372437, + "rewards/accuracies": 0.78125, + "rewards/chosen": -5.337520122528076, + "rewards/margins": 1.3362945318222046, + "rewards/rejected": -6.67381477355957, + "step": 149 + }, + { + "epoch": 0.4838709677419355, + "grad_norm": 0.8199206590652466, + "learning_rate": 4.320686303863752e-06, + "logits/chosen": -1.822701334953308, + "logits/rejected": -1.6954931020736694, + "logps/chosen": -173.2248992919922, + "logps/rejected": -206.9976043701172, + "loss": 0.47184550762176514, + "rewards/accuracies": 0.75, + "rewards/chosen": -5.998141288757324, + "rewards/margins": 1.5172470808029175, + "rewards/rejected": -7.515388488769531, + "step": 150 + }, + { + "epoch": 0.4870967741935484, + "grad_norm": 0.6827844381332397, + "learning_rate": 4.3119819680728e-06, + "logits/chosen": -1.2705650329589844, + "logits/rejected": -1.084989070892334, + "logps/chosen": -177.86563110351562, + "logps/rejected": -215.4129638671875, + "loss": 0.3346361815929413, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.931788921356201, + "rewards/margins": 1.785906195640564, + "rewards/rejected": -7.7176947593688965, + "step": 151 + }, + { + "epoch": 0.49032258064516127, + "grad_norm": 0.8796378374099731, + "learning_rate": 4.303231109110821e-06, + "logits/chosen": -1.7787282466888428, + "logits/rejected": -1.5340662002563477, + "logps/chosen": -155.4757843017578, + "logps/rejected": -198.17584228515625, + "loss": 0.4421862065792084, + "rewards/accuracies": 0.75, + "rewards/chosen": -5.767148017883301, + "rewards/margins": 1.6090242862701416, + "rewards/rejected": -7.37617301940918, + "step": 152 + }, + { + "epoch": 0.4935483870967742, + "grad_norm": 0.8411730527877808, + "learning_rate": 4.294433951658698e-06, + "logits/chosen": -1.648699164390564, + "logits/rejected": -1.4484217166900635, + "logps/chosen": -162.52825927734375, + "logps/rejected": -196.4520263671875, + "loss": 0.4758791923522949, + "rewards/accuracies": 0.75, + "rewards/chosen": -5.455954551696777, + "rewards/margins": 1.4209301471710205, + "rewards/rejected": -6.876884460449219, + "step": 153 + }, + { + "epoch": 0.4967741935483871, + "grad_norm": 0.6339321732521057, + "learning_rate": 4.285590721586039e-06, + "logits/chosen": -1.8830703496932983, + "logits/rejected": -1.5869327783584595, + "logps/chosen": -139.29800415039062, + "logps/rejected": -196.91796875, + "loss": 0.22343483567237854, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.459966659545898, + "rewards/margins": 2.71433162689209, + "rewards/rejected": -7.174298286437988, + "step": 154 + }, + { + "epoch": 0.5, + "grad_norm": 0.6575607657432556, + "learning_rate": 4.276701645945384e-06, + "logits/chosen": -2.1467416286468506, + "logits/rejected": -1.892794132232666, + "logps/chosen": -154.65928649902344, + "logps/rejected": -213.82528686523438, + "loss": 0.2633991241455078, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.381623268127441, + "rewards/margins": 2.5349740982055664, + "rewards/rejected": -7.916597366333008, + "step": 155 + }, + { + "epoch": 0.5032258064516129, + "grad_norm": 0.8809133768081665, + "learning_rate": 4.267766952966369e-06, + "logits/chosen": -1.8720148801803589, + "logits/rejected": -1.607576847076416, + "logps/chosen": -149.6011962890625, + "logps/rejected": -193.8123321533203, + "loss": 0.4617472290992737, + "rewards/accuracies": 0.8125, + "rewards/chosen": -5.031999111175537, + "rewards/margins": 2.0407233238220215, + "rewards/rejected": -7.072722434997559, + "step": 156 + }, + { + "epoch": 0.5064516129032258, + "grad_norm": 0.660529613494873, + "learning_rate": 4.25878687204987e-06, + "logits/chosen": -2.024183511734009, + "logits/rejected": -1.8310062885284424, + "logps/chosen": -152.48403930664062, + "logps/rejected": -203.5169677734375, + "loss": 0.3005449175834656, + "rewards/accuracies": 0.84375, + "rewards/chosen": -4.750589370727539, + "rewards/margins": 2.2591700553894043, + "rewards/rejected": -7.009759902954102, + "step": 157 + }, + { + "epoch": 0.5096774193548387, + "grad_norm": 0.8802087306976318, + "learning_rate": 4.2497616337621115e-06, + "logits/chosen": -1.8386707305908203, + "logits/rejected": -1.586627721786499, + "logps/chosen": -154.0318603515625, + "logps/rejected": -205.48057556152344, + "loss": 0.4543035924434662, + "rewards/accuracies": 0.84375, + "rewards/chosen": -5.153034210205078, + "rewards/margins": 2.193976640701294, + "rewards/rejected": -7.347011566162109, + "step": 158 + }, + { + "epoch": 0.5129032258064516, + "grad_norm": 0.9531585574150085, + "learning_rate": 4.240691469828745e-06, + "logits/chosen": -2.1372509002685547, + "logits/rejected": -2.0333480834960938, + "logps/chosen": -169.7491912841797, + "logps/rejected": -210.4459228515625, + "loss": 0.5746184587478638, + "rewards/accuracies": 0.65625, + "rewards/chosen": -5.571075439453125, + "rewards/margins": 1.5784751176834106, + "rewards/rejected": -7.149550914764404, + "step": 159 + }, + { + "epoch": 0.5161290322580645, + "grad_norm": 0.7346033453941345, + "learning_rate": 4.231576613128902e-06, + "logits/chosen": -2.0919246673583984, + "logits/rejected": -1.8873045444488525, + "logps/chosen": -153.13253784179688, + "logps/rejected": -200.02569580078125, + "loss": 0.3784785270690918, + "rewards/accuracies": 0.84375, + "rewards/chosen": -5.044937610626221, + "rewards/margins": 2.0969479084014893, + "rewards/rejected": -7.141885280609131, + "step": 160 + }, + { + "epoch": 0.5193548387096775, + "grad_norm": 0.9784480929374695, + "learning_rate": 4.222417297689217e-06, + "logits/chosen": -1.9859681129455566, + "logits/rejected": -1.7799007892608643, + "logps/chosen": -150.41250610351562, + "logps/rejected": -184.49789428710938, + "loss": 0.5596226453781128, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.557621479034424, + "rewards/margins": 1.3568779230117798, + "rewards/rejected": -5.914499282836914, + "step": 161 + }, + { + "epoch": 0.5225806451612903, + "grad_norm": 0.87236088514328, + "learning_rate": 4.21321375867781e-06, + "logits/chosen": -1.7091286182403564, + "logits/rejected": -1.5344655513763428, + "logps/chosen": -153.80831909179688, + "logps/rejected": -193.2026824951172, + "loss": 0.48372751474380493, + "rewards/accuracies": 0.8125, + "rewards/chosen": -5.051312446594238, + "rewards/margins": 1.568933129310608, + "rewards/rejected": -6.620245456695557, + "step": 162 + }, + { + "epoch": 0.5258064516129032, + "grad_norm": 0.7265849709510803, + "learning_rate": 4.203966232398261e-06, + "logits/chosen": -2.2141776084899902, + "logits/rejected": -1.72068190574646, + "logps/chosen": -135.98233032226562, + "logps/rejected": -195.78211975097656, + "loss": 0.3879455327987671, + "rewards/accuracies": 0.84375, + "rewards/chosen": -4.7070722579956055, + "rewards/margins": 2.4663829803466797, + "rewards/rejected": -7.173455238342285, + "step": 163 + }, + { + "epoch": 0.5290322580645161, + "grad_norm": 0.7596819996833801, + "learning_rate": 4.194674956283529e-06, + "logits/chosen": -1.6569631099700928, + "logits/rejected": -1.389373540878296, + "logps/chosen": -152.79058837890625, + "logps/rejected": -198.65692138671875, + "loss": 0.39065128564834595, + "rewards/accuracies": 0.84375, + "rewards/chosen": -4.617537975311279, + "rewards/margins": 2.011427402496338, + "rewards/rejected": -6.628965377807617, + "step": 164 + }, + { + "epoch": 0.532258064516129, + "grad_norm": 0.7323077917098999, + "learning_rate": 4.185340168889869e-06, + "logits/chosen": -2.1654136180877686, + "logits/rejected": -1.9493908882141113, + "logps/chosen": -160.08444213867188, + "logps/rejected": -210.05970764160156, + "loss": 0.3324809670448303, + "rewards/accuracies": 0.75, + "rewards/chosen": -5.448432922363281, + "rewards/margins": 2.039762496948242, + "rewards/rejected": -7.48819637298584, + "step": 165 + }, + { + "epoch": 0.535483870967742, + "grad_norm": 0.5825879573822021, + "learning_rate": 4.175962109890697e-06, + "logits/chosen": -1.89616858959198, + "logits/rejected": -1.663448452949524, + "logps/chosen": -126.10738372802734, + "logps/rejected": -181.8113250732422, + "loss": 0.2588610351085663, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.298904895782471, + "rewards/margins": 2.3480236530303955, + "rewards/rejected": -6.646928787231445, + "step": 166 + }, + { + "epoch": 0.5387096774193548, + "grad_norm": 0.9141851663589478, + "learning_rate": 4.166541020070442e-06, + "logits/chosen": -1.9237167835235596, + "logits/rejected": -1.6572012901306152, + "logps/chosen": -141.69476318359375, + "logps/rejected": -187.3744659423828, + "loss": 0.4472041130065918, + "rewards/accuracies": 0.78125, + "rewards/chosen": -4.35133171081543, + "rewards/margins": 1.9088027477264404, + "rewards/rejected": -6.260134220123291, + "step": 167 + }, + { + "epoch": 0.5419354838709678, + "grad_norm": 1.011690378189087, + "learning_rate": 4.157077141318361e-06, + "logits/chosen": -2.032428503036499, + "logits/rejected": -1.7909128665924072, + "logps/chosen": -138.17669677734375, + "logps/rejected": -184.1501007080078, + "loss": 0.3964395225048065, + "rewards/accuracies": 0.84375, + "rewards/chosen": -4.0889787673950195, + "rewards/margins": 1.6615996360778809, + "rewards/rejected": -5.750578880310059, + "step": 168 + }, + { + "epoch": 0.5451612903225806, + "grad_norm": 0.6844170689582825, + "learning_rate": 4.14757071662233e-06, + "logits/chosen": -2.0451266765594482, + "logits/rejected": -1.8144954442977905, + "logps/chosen": -128.15585327148438, + "logps/rejected": -182.09280395507812, + "loss": 0.3033115863800049, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.129886627197266, + "rewards/margins": 2.4071736335754395, + "rewards/rejected": -6.537059783935547, + "step": 169 + }, + { + "epoch": 0.5483870967741935, + "grad_norm": 0.7212929725646973, + "learning_rate": 4.138021990062606e-06, + "logits/chosen": -1.9469044208526611, + "logits/rejected": -1.7142751216888428, + "logps/chosen": -150.60662841796875, + "logps/rejected": -205.27867126464844, + "loss": 0.30135634541511536, + "rewards/accuracies": 0.90625, + "rewards/chosen": -4.973240375518799, + "rewards/margins": 2.420989513397217, + "rewards/rejected": -7.394229412078857, + "step": 170 + }, + { + "epoch": 0.5516129032258065, + "grad_norm": 0.9071898460388184, + "learning_rate": 4.128431206805556e-06, + "logits/chosen": -1.8445392847061157, + "logits/rejected": -1.6512104272842407, + "logps/chosen": -167.3872528076172, + "logps/rejected": -207.1650390625, + "loss": 0.4723031520843506, + "rewards/accuracies": 0.75, + "rewards/chosen": -5.54469108581543, + "rewards/margins": 1.5533931255340576, + "rewards/rejected": -7.098084449768066, + "step": 171 + }, + { + "epoch": 0.5548387096774193, + "grad_norm": 0.7936817407608032, + "learning_rate": 4.118798613097367e-06, + "logits/chosen": -1.840900182723999, + "logits/rejected": -1.6130969524383545, + "logps/chosen": -162.12991333007812, + "logps/rejected": -213.62379455566406, + "loss": 0.3520919978618622, + "rewards/accuracies": 0.8125, + "rewards/chosen": -5.226555824279785, + "rewards/margins": 2.199876546859741, + "rewards/rejected": -7.426431655883789, + "step": 172 + }, + { + "epoch": 0.5580645161290323, + "grad_norm": 0.8109973073005676, + "learning_rate": 4.109124456257722e-06, + "logits/chosen": -1.7801843881607056, + "logits/rejected": -1.5753393173217773, + "logps/chosen": -164.92132568359375, + "logps/rejected": -209.00552368164062, + "loss": 0.37818557024002075, + "rewards/accuracies": 0.84375, + "rewards/chosen": -5.394079208374023, + "rewards/margins": 2.0151584148406982, + "rewards/rejected": -7.409237861633301, + "step": 173 + }, + { + "epoch": 0.5612903225806452, + "grad_norm": 0.8981033563613892, + "learning_rate": 4.099408984673448e-06, + "logits/chosen": -1.9823046922683716, + "logits/rejected": -1.7255879640579224, + "logps/chosen": -147.62057495117188, + "logps/rejected": -189.5701904296875, + "loss": 0.46110501885414124, + "rewards/accuracies": 0.78125, + "rewards/chosen": -5.077602386474609, + "rewards/margins": 1.8654961585998535, + "rewards/rejected": -6.943098545074463, + "step": 174 + }, + { + "epoch": 0.5645161290322581, + "grad_norm": 0.725642740726471, + "learning_rate": 4.089652447792141e-06, + "logits/chosen": -1.9010741710662842, + "logits/rejected": -1.595578908920288, + "logps/chosen": -152.0930938720703, + "logps/rejected": -208.27474975585938, + "loss": 0.31729233264923096, + "rewards/accuracies": 0.84375, + "rewards/chosen": -5.1363019943237305, + "rewards/margins": 2.5898420810699463, + "rewards/rejected": -7.7261433601379395, + "step": 175 + }, + { + "epoch": 0.567741935483871, + "grad_norm": 0.9278317093849182, + "learning_rate": 4.07985509611576e-06, + "logits/chosen": -1.8088712692260742, + "logits/rejected": -1.6028716564178467, + "logps/chosen": -159.88787841796875, + "logps/rejected": -204.56011962890625, + "loss": 0.5430783629417419, + "rewards/accuracies": 0.75, + "rewards/chosen": -6.032896041870117, + "rewards/margins": 1.7545185089111328, + "rewards/rejected": -7.78741455078125, + "step": 176 + }, + { + "epoch": 0.5709677419354838, + "grad_norm": 0.6002618074417114, + "learning_rate": 4.070017181194199e-06, + "logits/chosen": -2.0451059341430664, + "logits/rejected": -1.8140971660614014, + "logps/chosen": -144.2393341064453, + "logps/rejected": -188.33346557617188, + "loss": 0.26023587584495544, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.727838516235352, + "rewards/margins": 2.057196617126465, + "rewards/rejected": -6.785035133361816, + "step": 177 + }, + { + "epoch": 0.5741935483870968, + "grad_norm": 0.7452195882797241, + "learning_rate": 4.0601389556188205e-06, + "logits/chosen": -1.6749149560928345, + "logits/rejected": -1.5378282070159912, + "logps/chosen": -163.5107879638672, + "logps/rejected": -200.22984313964844, + "loss": 0.37158316373825073, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.478737831115723, + "rewards/margins": 1.5227606296539307, + "rewards/rejected": -7.001498222351074, + "step": 178 + }, + { + "epoch": 0.5774193548387097, + "grad_norm": 0.6364844441413879, + "learning_rate": 4.050220673015979e-06, + "logits/chosen": -2.103788137435913, + "logits/rejected": -1.8177173137664795, + "logps/chosen": -142.53990173339844, + "logps/rejected": -198.9427490234375, + "loss": 0.2621723413467407, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.8717546463012695, + "rewards/margins": 2.7382116317749023, + "rewards/rejected": -7.609966278076172, + "step": 179 + }, + { + "epoch": 0.5806451612903226, + "grad_norm": 1.1201379299163818, + "learning_rate": 4.040262588040503e-06, + "logits/chosen": -1.8769949674606323, + "logits/rejected": -1.7055038213729858, + "logps/chosen": -156.72299194335938, + "logps/rejected": -197.8905792236328, + "loss": 0.5500263571739197, + "rewards/accuracies": 0.8125, + "rewards/chosen": -5.502621650695801, + "rewards/margins": 1.5961947441101074, + "rewards/rejected": -7.09881591796875, + "step": 180 + }, + { + "epoch": 0.5838709677419355, + "grad_norm": 0.524496853351593, + "learning_rate": 4.030264956369158e-06, + "logits/chosen": -2.2530908584594727, + "logits/rejected": -2.0049355030059814, + "logps/chosen": -134.3117218017578, + "logps/rejected": -177.72940063476562, + "loss": 0.2530081868171692, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.458559989929199, + "rewards/margins": 2.3075766563415527, + "rewards/rejected": -6.766136169433594, + "step": 181 + }, + { + "epoch": 0.5870967741935483, + "grad_norm": 0.9195796251296997, + "learning_rate": 4.020228034694083e-06, + "logits/chosen": -1.649029016494751, + "logits/rejected": -1.4227406978607178, + "logps/chosen": -160.95321655273438, + "logps/rejected": -215.07814025878906, + "loss": 0.3454977869987488, + "rewards/accuracies": 0.78125, + "rewards/chosen": -5.202361106872559, + "rewards/margins": 2.460273027420044, + "rewards/rejected": -7.662634372711182, + "step": 182 + }, + { + "epoch": 0.5903225806451613, + "grad_norm": 0.5352334380149841, + "learning_rate": 4.010152080716202e-06, + "logits/chosen": -2.089765787124634, + "logits/rejected": -1.8100600242614746, + "logps/chosen": -138.4512176513672, + "logps/rejected": -194.85545349121094, + "loss": 0.20018336176872253, + "rewards/accuracies": 0.90625, + "rewards/chosen": -4.481593132019043, + "rewards/margins": 2.971552610397339, + "rewards/rejected": -7.453145980834961, + "step": 183 + }, + { + "epoch": 0.5935483870967742, + "grad_norm": 0.7269738912582397, + "learning_rate": 4.000037353138602e-06, + "logits/chosen": -1.8054555654525757, + "logits/rejected": -1.598811388015747, + "logps/chosen": -151.69924926757812, + "logps/rejected": -187.41851806640625, + "loss": 0.3388046622276306, + "rewards/accuracies": 0.78125, + "rewards/chosen": -5.017839431762695, + "rewards/margins": 1.4641319513320923, + "rewards/rejected": -6.481971740722656, + "step": 184 + }, + { + "epoch": 0.5967741935483871, + "grad_norm": 0.8876580595970154, + "learning_rate": 3.989884111659893e-06, + "logits/chosen": -1.8593127727508545, + "logits/rejected": -1.6329864263534546, + "logps/chosen": -148.35565185546875, + "logps/rejected": -205.98257446289062, + "loss": 0.40850383043289185, + "rewards/accuracies": 0.78125, + "rewards/chosen": -5.808213233947754, + "rewards/margins": 2.492269277572632, + "rewards/rejected": -8.300481796264648, + "step": 185 + }, + { + "epoch": 0.6, + "grad_norm": 0.7655280232429504, + "learning_rate": 3.979692616967543e-06, + "logits/chosen": -2.026470422744751, + "logits/rejected": -1.7303847074508667, + "logps/chosen": -139.6387481689453, + "logps/rejected": -198.58078002929688, + "loss": 0.3410119414329529, + "rewards/accuracies": 0.8125, + "rewards/chosen": -5.268441200256348, + "rewards/margins": 2.493990898132324, + "rewards/rejected": -7.762432098388672, + "step": 186 + }, + { + "epoch": 0.603225806451613, + "grad_norm": 0.8385968804359436, + "learning_rate": 3.969463130731183e-06, + "logits/chosen": -1.709697961807251, + "logits/rejected": -1.5222282409667969, + "logps/chosen": -189.8544158935547, + "logps/rejected": -231.8427734375, + "loss": 0.39192935824394226, + "rewards/accuracies": 0.84375, + "rewards/chosen": -6.106621265411377, + "rewards/margins": 2.2979493141174316, + "rewards/rejected": -8.404570579528809, + "step": 187 + }, + { + "epoch": 0.6064516129032258, + "grad_norm": 0.9917109608650208, + "learning_rate": 3.959195915595886e-06, + "logits/chosen": -1.9611066579818726, + "logits/rejected": -1.8462345600128174, + "logps/chosen": -149.73031616210938, + "logps/rejected": -192.25274658203125, + "loss": 0.5630193948745728, + "rewards/accuracies": 0.75, + "rewards/chosen": -5.245723724365234, + "rewards/margins": 1.6644206047058105, + "rewards/rejected": -6.910144329071045, + "step": 188 + }, + { + "epoch": 0.6096774193548387, + "grad_norm": 0.8959775567054749, + "learning_rate": 3.948891235175425e-06, + "logits/chosen": -1.646284580230713, + "logits/rejected": -1.379030466079712, + "logps/chosen": -160.4254913330078, + "logps/rejected": -211.20452880859375, + "loss": 0.4511195719242096, + "rewards/accuracies": 0.75, + "rewards/chosen": -5.744508266448975, + "rewards/margins": 2.126063346862793, + "rewards/rejected": -7.870571613311768, + "step": 189 + }, + { + "epoch": 0.6129032258064516, + "grad_norm": 0.5826210379600525, + "learning_rate": 3.938549354045508e-06, + "logits/chosen": -1.51802396774292, + "logits/rejected": -1.2956496477127075, + "logps/chosen": -157.11676025390625, + "logps/rejected": -213.88731384277344, + "loss": 0.22183524072170258, + "rewards/accuracies": 0.90625, + "rewards/chosen": -5.172170639038086, + "rewards/margins": 2.546001434326172, + "rewards/rejected": -7.718172073364258, + "step": 190 + }, + { + "epoch": 0.6161290322580645, + "grad_norm": 1.0212091207504272, + "learning_rate": 3.9281705377369814e-06, + "logits/chosen": -2.1612565517425537, + "logits/rejected": -1.9319891929626465, + "logps/chosen": -150.02903747558594, + "logps/rejected": -197.55706787109375, + "loss": 0.5443016290664673, + "rewards/accuracies": 0.71875, + "rewards/chosen": -4.873559951782227, + "rewards/margins": 2.2266910076141357, + "rewards/rejected": -7.1002516746521, + "step": 191 + }, + { + "epoch": 0.6193548387096774, + "grad_norm": 0.7122681140899658, + "learning_rate": 3.917755052729011e-06, + "logits/chosen": -2.106039524078369, + "logits/rejected": -1.750291347503662, + "logps/chosen": -144.48170471191406, + "logps/rejected": -200.36827087402344, + "loss": 0.24308393895626068, + "rewards/accuracies": 0.90625, + "rewards/chosen": -4.827795505523682, + "rewards/margins": 2.734774112701416, + "rewards/rejected": -7.562569618225098, + "step": 192 + }, + { + "epoch": 0.6225806451612903, + "grad_norm": 0.7401944994926453, + "learning_rate": 3.907303166442245e-06, + "logits/chosen": -1.975453495979309, + "logits/rejected": -1.845196008682251, + "logps/chosen": -182.4921417236328, + "logps/rejected": -214.56707763671875, + "loss": 0.33322787284851074, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.488518714904785, + "rewards/margins": 1.8690803050994873, + "rewards/rejected": -7.357598781585693, + "step": 193 + }, + { + "epoch": 0.6258064516129033, + "grad_norm": 0.8645218014717102, + "learning_rate": 3.896815147231943e-06, + "logits/chosen": -1.6347211599349976, + "logits/rejected": -1.384649634361267, + "logps/chosen": -169.53807067871094, + "logps/rejected": -217.01519775390625, + "loss": 0.40609508752822876, + "rewards/accuracies": 0.78125, + "rewards/chosen": -5.351966857910156, + "rewards/margins": 1.9269593954086304, + "rewards/rejected": -7.278926372528076, + "step": 194 + }, + { + "epoch": 0.6290322580645161, + "grad_norm": 0.9169072508811951, + "learning_rate": 3.8862912643810895e-06, + "logits/chosen": -1.6547520160675049, + "logits/rejected": -1.3922858238220215, + "logps/chosen": -169.7218017578125, + "logps/rejected": -233.12631225585938, + "loss": 0.47453373670578003, + "rewards/accuracies": 0.78125, + "rewards/chosen": -5.81014347076416, + "rewards/margins": 2.4779911041259766, + "rewards/rejected": -8.288135528564453, + "step": 195 + }, + { + "epoch": 0.632258064516129, + "grad_norm": 0.6964331865310669, + "learning_rate": 3.875731788093478e-06, + "logits/chosen": -1.6203590631484985, + "logits/rejected": -1.37406587600708, + "logps/chosen": -172.435302734375, + "logps/rejected": -223.4750213623047, + "loss": 0.28000563383102417, + "rewards/accuracies": 0.84375, + "rewards/chosen": -5.44480037689209, + "rewards/margins": 2.7131171226501465, + "rewards/rejected": -8.157917976379395, + "step": 196 + }, + { + "epoch": 0.635483870967742, + "grad_norm": 1.0018017292022705, + "learning_rate": 3.865136989486776e-06, + "logits/chosen": -2.125566244125366, + "logits/rejected": -1.9877275228500366, + "logps/chosen": -174.42189025878906, + "logps/rejected": -209.52536010742188, + "loss": 0.4606223404407501, + "rewards/accuracies": 0.78125, + "rewards/chosen": -5.8737664222717285, + "rewards/margins": 1.626832365989685, + "rewards/rejected": -7.500598907470703, + "step": 197 + }, + { + "epoch": 0.6387096774193548, + "grad_norm": 0.6335120797157288, + "learning_rate": 3.8545071405855574e-06, + "logits/chosen": -2.0536115169525146, + "logits/rejected": -1.8321561813354492, + "logps/chosen": -154.74710083007812, + "logps/rejected": -209.40707397460938, + "loss": 0.23459070920944214, + "rewards/accuracies": 0.90625, + "rewards/chosen": -5.506530284881592, + "rewards/margins": 2.8299872875213623, + "rewards/rejected": -8.336517333984375, + "step": 198 + }, + { + "epoch": 0.6419354838709678, + "grad_norm": 0.9899296164512634, + "learning_rate": 3.8438425143143265e-06, + "logits/chosen": -1.8322534561157227, + "logits/rejected": -1.6016404628753662, + "logps/chosen": -134.58560180664062, + "logps/rejected": -187.51687622070312, + "loss": 0.44235897064208984, + "rewards/accuracies": 0.78125, + "rewards/chosen": -4.430156707763672, + "rewards/margins": 2.332374095916748, + "rewards/rejected": -6.762531280517578, + "step": 199 + }, + { + "epoch": 0.6451612903225806, + "grad_norm": 0.7978684306144714, + "learning_rate": 3.833143384490506e-06, + "logits/chosen": -1.9407423734664917, + "logits/rejected": -1.8073129653930664, + "logps/chosen": -164.80926513671875, + "logps/rejected": -207.02963256835938, + "loss": 0.32675695419311523, + "rewards/accuracies": 0.84375, + "rewards/chosen": -5.683391094207764, + "rewards/margins": 2.388817071914673, + "rewards/rejected": -8.072208404541016, + "step": 200 + }, + { + "epoch": 0.6483870967741936, + "grad_norm": 1.105037808418274, + "learning_rate": 3.8224100258174066e-06, + "logits/chosen": -2.0903000831604004, + "logits/rejected": -1.861089825630188, + "logps/chosen": -146.48716735839844, + "logps/rejected": -193.67787170410156, + "loss": 0.6486673951148987, + "rewards/accuracies": 0.75, + "rewards/chosen": -5.065411567687988, + "rewards/margins": 1.8852479457855225, + "rewards/rejected": -6.95065975189209, + "step": 201 + }, + { + "epoch": 0.6516129032258065, + "grad_norm": 0.5039254426956177, + "learning_rate": 3.811642713877175e-06, + "logits/chosen": -1.7253732681274414, + "logits/rejected": -1.4285393953323364, + "logps/chosen": -148.22134399414062, + "logps/rejected": -213.10714721679688, + "loss": 0.17612317204475403, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.6895952224731445, + "rewards/margins": 3.1369845867156982, + "rewards/rejected": -7.826580047607422, + "step": 202 + }, + { + "epoch": 0.6548387096774193, + "grad_norm": 0.4975990056991577, + "learning_rate": 3.8008417251237208e-06, + "logits/chosen": -2.020594358444214, + "logits/rejected": -1.7548696994781494, + "logps/chosen": -153.95767211914062, + "logps/rejected": -216.39849853515625, + "loss": 0.1482263207435608, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.834309101104736, + "rewards/margins": 3.244680643081665, + "rewards/rejected": -8.078989028930664, + "step": 203 + }, + { + "epoch": 0.6580645161290323, + "grad_norm": 0.7523185610771179, + "learning_rate": 3.79000733687561e-06, + "logits/chosen": -2.1262402534484863, + "logits/rejected": -1.8249822854995728, + "logps/chosen": -138.27684020996094, + "logps/rejected": -194.3658447265625, + "loss": 0.3780391812324524, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.251176357269287, + "rewards/margins": 2.609586238861084, + "rewards/rejected": -6.860762596130371, + "step": 204 + }, + { + "epoch": 0.6612903225806451, + "grad_norm": 0.7087878584861755, + "learning_rate": 3.7791398273089562e-06, + "logits/chosen": -2.0547335147857666, + "logits/rejected": -1.7882875204086304, + "logps/chosen": -150.56541442871094, + "logps/rejected": -204.3115234375, + "loss": 0.28930363059043884, + "rewards/accuracies": 0.84375, + "rewards/chosen": -4.768935680389404, + "rewards/margins": 2.5629935264587402, + "rewards/rejected": -7.331929683685303, + "step": 205 + }, + { + "epoch": 0.6645161290322581, + "grad_norm": 0.6253401041030884, + "learning_rate": 3.7682394754502687e-06, + "logits/chosen": -2.046987295150757, + "logits/rejected": -1.855947732925415, + "logps/chosen": -160.48355102539062, + "logps/rejected": -204.4710693359375, + "loss": 0.25897252559661865, + "rewards/accuracies": 0.90625, + "rewards/chosen": -5.5119524002075195, + "rewards/margins": 1.971421480178833, + "rewards/rejected": -7.48337459564209, + "step": 206 + }, + { + "epoch": 0.667741935483871, + "grad_norm": 0.6977007985115051, + "learning_rate": 3.7573065611692948e-06, + "logits/chosen": -1.9548072814941406, + "logits/rejected": -1.6766889095306396, + "logps/chosen": -155.93496704101562, + "logps/rejected": -221.83482360839844, + "loss": 0.309578001499176, + "rewards/accuracies": 0.90625, + "rewards/chosen": -5.157824993133545, + "rewards/margins": 3.1803603172302246, + "rewards/rejected": -8.338184356689453, + "step": 207 + }, + { + "epoch": 0.6709677419354839, + "grad_norm": 0.9557754993438721, + "learning_rate": 3.7463413651718304e-06, + "logits/chosen": -1.9568748474121094, + "logits/rejected": -1.7348158359527588, + "logps/chosen": -164.6311798095703, + "logps/rejected": -209.5601348876953, + "loss": 0.5549408197402954, + "rewards/accuracies": 0.78125, + "rewards/chosen": -5.9888458251953125, + "rewards/margins": 2.138120651245117, + "rewards/rejected": -8.126965522766113, + "step": 208 + }, + { + "epoch": 0.6741935483870968, + "grad_norm": 0.8854509592056274, + "learning_rate": 3.7353441689925144e-06, + "logits/chosen": -2.291874885559082, + "logits/rejected": -2.0419085025787354, + "logps/chosen": -156.27235412597656, + "logps/rejected": -210.60018920898438, + "loss": 0.3768148720264435, + "rewards/accuracies": 0.84375, + "rewards/chosen": -4.670940399169922, + "rewards/margins": 3.0637500286102295, + "rewards/rejected": -7.7346906661987305, + "step": 209 + }, + { + "epoch": 0.6774193548387096, + "grad_norm": 0.9439530968666077, + "learning_rate": 3.7243152549875995e-06, + "logits/chosen": -2.140773057937622, + "logits/rejected": -1.8929061889648438, + "logps/chosen": -157.63751220703125, + "logps/rejected": -200.3406524658203, + "loss": 0.41107749938964844, + "rewards/accuracies": 0.78125, + "rewards/chosen": -5.264650821685791, + "rewards/margins": 2.0409224033355713, + "rewards/rejected": -7.305573463439941, + "step": 210 + }, + { + "epoch": 0.6806451612903226, + "grad_norm": 0.8497869968414307, + "learning_rate": 3.7132549063277033e-06, + "logits/chosen": -2.4181089401245117, + "logits/rejected": -2.282496452331543, + "logps/chosen": -135.24644470214844, + "logps/rejected": -171.06222534179688, + "loss": 0.4564102590084076, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.036481857299805, + "rewards/margins": 1.5632761716842651, + "rewards/rejected": -5.599758148193359, + "step": 211 + }, + { + "epoch": 0.6838709677419355, + "grad_norm": 0.6613027453422546, + "learning_rate": 3.7021634069905355e-06, + "logits/chosen": -1.9340821504592896, + "logits/rejected": -1.7125706672668457, + "logps/chosen": -153.77017211914062, + "logps/rejected": -206.60797119140625, + "loss": 0.27007001638412476, + "rewards/accuracies": 0.90625, + "rewards/chosen": -5.424838066101074, + "rewards/margins": 2.3895516395568848, + "rewards/rejected": -7.814389705657959, + "step": 212 + }, + { + "epoch": 0.6870967741935484, + "grad_norm": 1.148823618888855, + "learning_rate": 3.691041041753613e-06, + "logits/chosen": -2.016974449157715, + "logits/rejected": -1.7639240026474, + "logps/chosen": -152.53939819335938, + "logps/rejected": -205.86460876464844, + "loss": 0.5762124061584473, + "rewards/accuracies": 0.78125, + "rewards/chosen": -5.334196090698242, + "rewards/margins": 2.5293712615966797, + "rewards/rejected": -7.863567352294922, + "step": 213 + }, + { + "epoch": 0.6903225806451613, + "grad_norm": 0.9124322533607483, + "learning_rate": 3.6798880961869386e-06, + "logits/chosen": -2.0975124835968018, + "logits/rejected": -1.8551663160324097, + "logps/chosen": -165.72335815429688, + "logps/rejected": -212.5291748046875, + "loss": 0.43939220905303955, + "rewards/accuracies": 0.8125, + "rewards/chosen": -5.707298278808594, + "rewards/margins": 2.0160140991210938, + "rewards/rejected": -7.7233123779296875, + "step": 214 + }, + { + "epoch": 0.6935483870967742, + "grad_norm": 0.6490616798400879, + "learning_rate": 3.6687048566456783e-06, + "logits/chosen": -1.7563501596450806, + "logits/rejected": -1.5969226360321045, + "logps/chosen": -150.6394805908203, + "logps/rejected": -202.0006561279297, + "loss": 0.24775174260139465, + "rewards/accuracies": 0.90625, + "rewards/chosen": -4.777384281158447, + "rewards/margins": 2.58968448638916, + "rewards/rejected": -7.367069244384766, + "step": 215 + }, + { + "epoch": 0.6967741935483871, + "grad_norm": 0.7321914434432983, + "learning_rate": 3.657491610262802e-06, + "logits/chosen": -2.1910958290100098, + "logits/rejected": -1.9808027744293213, + "logps/chosen": -143.98077392578125, + "logps/rejected": -197.44918823242188, + "loss": 0.36529406905174255, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.628123760223389, + "rewards/margins": 2.3086495399475098, + "rewards/rejected": -6.936773777008057, + "step": 216 + }, + { + "epoch": 0.7, + "grad_norm": 0.6766554117202759, + "learning_rate": 3.646248644941716e-06, + "logits/chosen": -2.1157803535461426, + "logits/rejected": -1.8787240982055664, + "logps/chosen": -147.38084411621094, + "logps/rejected": -190.40176391601562, + "loss": 0.23784853518009186, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.631725788116455, + "rewards/margins": 2.343129873275757, + "rewards/rejected": -6.974855899810791, + "step": 217 + }, + { + "epoch": 0.7032258064516129, + "grad_norm": 0.6971132755279541, + "learning_rate": 3.634976249348867e-06, + "logits/chosen": -1.9888451099395752, + "logits/rejected": -1.666069507598877, + "logps/chosen": -141.85008239746094, + "logps/rejected": -206.53433227539062, + "loss": 0.2547075152397156, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.588260650634766, + "rewards/margins": 3.034471035003662, + "rewards/rejected": -7.622732162475586, + "step": 218 + }, + { + "epoch": 0.7064516129032258, + "grad_norm": 1.014033317565918, + "learning_rate": 3.6236747129063353e-06, + "logits/chosen": -1.693007230758667, + "logits/rejected": -1.5424453020095825, + "logps/chosen": -165.82598876953125, + "logps/rejected": -207.29913330078125, + "loss": 0.5212239623069763, + "rewards/accuracies": 0.71875, + "rewards/chosen": -5.077567100524902, + "rewards/margins": 1.8947848081588745, + "rewards/rejected": -6.972352027893066, + "step": 219 + }, + { + "epoch": 0.7096774193548387, + "grad_norm": 0.889532208442688, + "learning_rate": 3.6123443257843985e-06, + "logits/chosen": -1.9983710050582886, + "logits/rejected": -1.7099499702453613, + "logps/chosen": -149.2031707763672, + "logps/rejected": -204.5345916748047, + "loss": 0.3786216378211975, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.6922101974487305, + "rewards/margins": 2.7204079627990723, + "rewards/rejected": -7.412618637084961, + "step": 220 + }, + { + "epoch": 0.7129032258064516, + "grad_norm": 0.7448631525039673, + "learning_rate": 3.600985378894086e-06, + "logits/chosen": -1.9120534658432007, + "logits/rejected": -1.6604293584823608, + "logps/chosen": -145.59979248046875, + "logps/rejected": -193.89862060546875, + "loss": 0.3285577595233917, + "rewards/accuracies": 0.84375, + "rewards/chosen": -4.52049446105957, + "rewards/margins": 2.195286512374878, + "rewards/rejected": -6.715780258178711, + "step": 221 + }, + { + "epoch": 0.7161290322580646, + "grad_norm": 0.6082993149757385, + "learning_rate": 3.589598163879706e-06, + "logits/chosen": -2.3130810260772705, + "logits/rejected": -1.9831249713897705, + "logps/chosen": -130.06045532226562, + "logps/rejected": -193.2603759765625, + "loss": 0.2183692455291748, + "rewards/accuracies": 0.90625, + "rewards/chosen": -4.439502716064453, + "rewards/margins": 2.903191089630127, + "rewards/rejected": -7.342694282531738, + "step": 222 + }, + { + "epoch": 0.7193548387096774, + "grad_norm": 0.8937051892280579, + "learning_rate": 3.5781829731113605e-06, + "logits/chosen": -1.9590139389038086, + "logits/rejected": -1.7677487134933472, + "logps/chosen": -166.64971923828125, + "logps/rejected": -211.19219970703125, + "loss": 0.4538525342941284, + "rewards/accuracies": 0.84375, + "rewards/chosen": -5.182065010070801, + "rewards/margins": 1.9423487186431885, + "rewards/rejected": -7.124413967132568, + "step": 223 + }, + { + "epoch": 0.7225806451612903, + "grad_norm": 1.022118091583252, + "learning_rate": 3.566740099677436e-06, + "logits/chosen": -2.3347010612487793, + "logits/rejected": -2.102405548095703, + "logps/chosen": -137.7996368408203, + "logps/rejected": -190.09060668945312, + "loss": 0.5034906268119812, + "rewards/accuracies": 0.71875, + "rewards/chosen": -4.532764434814453, + "rewards/margins": 2.5061287879943848, + "rewards/rejected": -7.038893222808838, + "step": 224 + }, + { + "epoch": 0.7258064516129032, + "grad_norm": 0.7584444880485535, + "learning_rate": 3.55526983737708e-06, + "logits/chosen": -1.9208580255508423, + "logits/rejected": -1.7654166221618652, + "logps/chosen": -155.05662536621094, + "logps/rejected": -197.6698760986328, + "loss": 0.3522125482559204, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.8355560302734375, + "rewards/margins": 2.2215867042541504, + "rewards/rejected": -7.05714225769043, + "step": 225 + }, + { + "epoch": 0.7290322580645161, + "grad_norm": 0.9140284657478333, + "learning_rate": 3.5437724807126583e-06, + "logits/chosen": -2.352722644805908, + "logits/rejected": -2.1613006591796875, + "logps/chosen": -145.38134765625, + "logps/rejected": -192.044921875, + "loss": 0.43635356426239014, + "rewards/accuracies": 0.6875, + "rewards/chosen": -4.508688926696777, + "rewards/margins": 1.8607778549194336, + "rewards/rejected": -6.369466781616211, + "step": 226 + }, + { + "epoch": 0.7322580645161291, + "grad_norm": 0.7812667489051819, + "learning_rate": 3.5322483248821905e-06, + "logits/chosen": -1.846239447593689, + "logits/rejected": -1.5585601329803467, + "logps/chosen": -165.45079040527344, + "logps/rejected": -219.214111328125, + "loss": 0.27440738677978516, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.201958656311035, + "rewards/margins": 2.816474676132202, + "rewards/rejected": -8.018433570861816, + "step": 227 + }, + { + "epoch": 0.7354838709677419, + "grad_norm": 0.8155746459960938, + "learning_rate": 3.5206976657717744e-06, + "logits/chosen": -2.210322380065918, + "logits/rejected": -2.045366048812866, + "logps/chosen": -166.80979919433594, + "logps/rejected": -211.31402587890625, + "loss": 0.33929508924484253, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.07341194152832, + "rewards/margins": 2.169137954711914, + "rewards/rejected": -7.242549896240234, + "step": 228 + }, + { + "epoch": 0.7387096774193549, + "grad_norm": 0.8510491251945496, + "learning_rate": 3.509120799947987e-06, + "logits/chosen": -1.881466269493103, + "logits/rejected": -1.6687889099121094, + "logps/chosen": -164.90065002441406, + "logps/rejected": -216.87759399414062, + "loss": 0.34565114974975586, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.524971008300781, + "rewards/margins": 2.7519350051879883, + "rewards/rejected": -8.276906967163086, + "step": 229 + }, + { + "epoch": 0.7419354838709677, + "grad_norm": 0.8683707118034363, + "learning_rate": 3.4975180246502694e-06, + "logits/chosen": -2.1603784561157227, + "logits/rejected": -1.9462230205535889, + "logps/chosen": -150.9666748046875, + "logps/rejected": -194.62677001953125, + "loss": 0.35645627975463867, + "rewards/accuracies": 0.78125, + "rewards/chosen": -4.169332504272461, + "rewards/margins": 2.1779627799987793, + "rewards/rejected": -6.347295761108398, + "step": 230 + }, + { + "epoch": 0.7451612903225806, + "grad_norm": 0.8181908130645752, + "learning_rate": 3.4858896377832966e-06, + "logits/chosen": -2.579606056213379, + "logits/rejected": -2.3366498947143555, + "logps/chosen": -155.29588317871094, + "logps/rejected": -196.72500610351562, + "loss": 0.3560987114906311, + "rewards/accuracies": 0.90625, + "rewards/chosen": -5.2712554931640625, + "rewards/margins": 1.8674944639205933, + "rewards/rejected": -7.138750076293945, + "step": 231 + }, + { + "epoch": 0.7483870967741936, + "grad_norm": 0.8175391554832458, + "learning_rate": 3.474235937909329e-06, + "logits/chosen": -2.486018657684326, + "logits/rejected": -2.283841609954834, + "logps/chosen": -144.54238891601562, + "logps/rejected": -197.2718048095703, + "loss": 0.39171290397644043, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.539939880371094, + "rewards/margins": 2.590193033218384, + "rewards/rejected": -7.130133152008057, + "step": 232 + }, + { + "epoch": 0.7516129032258064, + "grad_norm": 0.6144670844078064, + "learning_rate": 3.4625572242405446e-06, + "logits/chosen": -2.331089496612549, + "logits/rejected": -2.0649709701538086, + "logps/chosen": -137.04869079589844, + "logps/rejected": -195.30335998535156, + "loss": 0.2142164707183838, + "rewards/accuracies": 0.90625, + "rewards/chosen": -4.864482402801514, + "rewards/margins": 2.7930479049682617, + "rewards/rejected": -7.657529830932617, + "step": 233 + }, + { + "epoch": 0.7548387096774194, + "grad_norm": 0.6806922554969788, + "learning_rate": 3.4508537966313605e-06, + "logits/chosen": -2.0631325244903564, + "logits/rejected": -1.8709652423858643, + "logps/chosen": -178.80401611328125, + "logps/rejected": -234.48159790039062, + "loss": 0.25653570890426636, + "rewards/accuracies": 0.875, + "rewards/chosen": -6.064234256744385, + "rewards/margins": 2.477729558944702, + "rewards/rejected": -8.541963577270508, + "step": 234 + }, + { + "epoch": 0.7580645161290323, + "grad_norm": 1.2282917499542236, + "learning_rate": 3.4391259555707258e-06, + "logits/chosen": -1.718582272529602, + "logits/rejected": -1.5619916915893555, + "logps/chosen": -162.3804168701172, + "logps/rejected": -208.4605712890625, + "loss": 0.5654730200767517, + "rewards/accuracies": 0.78125, + "rewards/chosen": -5.41401481628418, + "rewards/margins": 1.942406177520752, + "rewards/rejected": -7.35642147064209, + "step": 235 + }, + { + "epoch": 0.7612903225806451, + "grad_norm": 0.7021056413650513, + "learning_rate": 3.42737400217442e-06, + "logits/chosen": -2.1502299308776855, + "logits/rejected": -1.8625457286834717, + "logps/chosen": -130.69139099121094, + "logps/rejected": -189.042724609375, + "loss": 0.24078813195228577, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.40123987197876, + "rewards/margins": 2.540182590484619, + "rewards/rejected": -6.941422462463379, + "step": 236 + }, + { + "epoch": 0.7645161290322581, + "grad_norm": 0.4305199980735779, + "learning_rate": 3.4155982381773074e-06, + "logits/chosen": -2.451601028442383, + "logits/rejected": -2.086921215057373, + "logps/chosen": -116.90483093261719, + "logps/rejected": -181.51046752929688, + "loss": 0.14265665411949158, + "rewards/accuracies": 0.96875, + "rewards/chosen": -3.7053639888763428, + "rewards/margins": 3.288050651550293, + "rewards/rejected": -6.993414878845215, + "step": 237 + }, + { + "epoch": 0.7677419354838709, + "grad_norm": 0.7569338083267212, + "learning_rate": 3.4037989659255998e-06, + "logits/chosen": -2.1827783584594727, + "logits/rejected": -1.9417102336883545, + "logps/chosen": -155.0686492919922, + "logps/rejected": -205.53671264648438, + "loss": 0.30644071102142334, + "rewards/accuracies": 0.90625, + "rewards/chosen": -4.89786958694458, + "rewards/margins": 2.2091331481933594, + "rewards/rejected": -7.107002258300781, + "step": 238 + }, + { + "epoch": 0.7709677419354839, + "grad_norm": 0.6807054281234741, + "learning_rate": 3.3919764883690892e-06, + "logits/chosen": -2.1077489852905273, + "logits/rejected": -1.7725837230682373, + "logps/chosen": -137.52195739746094, + "logps/rejected": -195.14187622070312, + "loss": 0.2705943286418915, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.612639904022217, + "rewards/margins": 2.7255260944366455, + "rewards/rejected": -7.338165283203125, + "step": 239 + }, + { + "epoch": 0.7741935483870968, + "grad_norm": 0.7687151432037354, + "learning_rate": 3.3801311090533713e-06, + "logits/chosen": -2.2328834533691406, + "logits/rejected": -1.9268276691436768, + "logps/chosen": -155.4674835205078, + "logps/rejected": -219.96664428710938, + "loss": 0.2569769024848938, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.455958366394043, + "rewards/margins": 3.1837010383605957, + "rewards/rejected": -8.639659881591797, + "step": 240 + }, + { + "epoch": 0.7774193548387097, + "grad_norm": 0.9292513132095337, + "learning_rate": 3.3682631321120507e-06, + "logits/chosen": -1.8700426816940308, + "logits/rejected": -1.7026903629302979, + "logps/chosen": -166.23532104492188, + "logps/rejected": -216.99081420898438, + "loss": 0.39584052562713623, + "rewards/accuracies": 0.84375, + "rewards/chosen": -5.327211856842041, + "rewards/margins": 2.511094570159912, + "rewards/rejected": -7.838306427001953, + "step": 241 + }, + { + "epoch": 0.7806451612903226, + "grad_norm": 0.6035842299461365, + "learning_rate": 3.3563728622589335e-06, + "logits/chosen": -2.2338809967041016, + "logits/rejected": -1.9739981889724731, + "logps/chosen": -126.01628112792969, + "logps/rejected": -182.52577209472656, + "loss": 0.22547340393066406, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.3564653396606445, + "rewards/margins": 2.6233766078948975, + "rewards/rejected": -6.979841709136963, + "step": 242 + }, + { + "epoch": 0.7838709677419354, + "grad_norm": 0.8133144974708557, + "learning_rate": 3.3444606047802018e-06, + "logits/chosen": -2.296985149383545, + "logits/rejected": -1.9562444686889648, + "logps/chosen": -138.7552947998047, + "logps/rejected": -202.73809814453125, + "loss": 0.33858245611190796, + "rewards/accuracies": 0.84375, + "rewards/chosen": -4.2327985763549805, + "rewards/margins": 3.3319268226623535, + "rewards/rejected": -7.564725875854492, + "step": 243 + }, + { + "epoch": 0.7870967741935484, + "grad_norm": 0.626816987991333, + "learning_rate": 3.332526665526577e-06, + "logits/chosen": -1.8298629522323608, + "logits/rejected": -1.528571605682373, + "logps/chosen": -156.6051025390625, + "logps/rejected": -224.70257568359375, + "loss": 0.17712946236133575, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.212545394897461, + "rewards/margins": 3.80438494682312, + "rewards/rejected": -9.016929626464844, + "step": 244 + }, + { + "epoch": 0.7903225806451613, + "grad_norm": 1.5558580160140991, + "learning_rate": 3.320571350905466e-06, + "logits/chosen": -2.0828301906585693, + "logits/rejected": -1.801743745803833, + "logps/chosen": -158.08493041992188, + "logps/rejected": -204.83648681640625, + "loss": 0.5623299479484558, + "rewards/accuracies": 0.78125, + "rewards/chosen": -5.441231727600098, + "rewards/margins": 2.246556282043457, + "rewards/rejected": -7.6877875328063965, + "step": 245 + }, + { + "epoch": 0.7935483870967742, + "grad_norm": 0.6557853817939758, + "learning_rate": 3.3085949678730953e-06, + "logits/chosen": -2.0652894973754883, + "logits/rejected": -1.883588194847107, + "logps/chosen": -150.181640625, + "logps/rejected": -219.64108276367188, + "loss": 0.2832576036453247, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.247066497802734, + "rewards/margins": 3.2221381664276123, + "rewards/rejected": -8.469204902648926, + "step": 246 + }, + { + "epoch": 0.7967741935483871, + "grad_norm": 0.8382725715637207, + "learning_rate": 3.2965978239266292e-06, + "logits/chosen": -2.383835554122925, + "logits/rejected": -2.0919713973999023, + "logps/chosen": -162.2133331298828, + "logps/rejected": -214.61122131347656, + "loss": 0.3238731622695923, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.967742919921875, + "rewards/margins": 2.712991237640381, + "rewards/rejected": -7.680734157562256, + "step": 247 + }, + { + "epoch": 0.8, + "grad_norm": 1.1319209337234497, + "learning_rate": 3.2845802270962736e-06, + "logits/chosen": -2.1036744117736816, + "logits/rejected": -1.820197343826294, + "logps/chosen": -179.16018676757812, + "logps/rejected": -229.18484497070312, + "loss": 0.5245007276535034, + "rewards/accuracies": 0.78125, + "rewards/chosen": -6.1170454025268555, + "rewards/margins": 2.1524717807769775, + "rewards/rejected": -8.269516944885254, + "step": 248 + }, + { + "epoch": 0.8032258064516129, + "grad_norm": 0.8215903043746948, + "learning_rate": 3.272542485937369e-06, + "logits/chosen": -1.901473045349121, + "logits/rejected": -1.5790214538574219, + "logps/chosen": -166.28927612304688, + "logps/rejected": -216.44970703125, + "loss": 0.3197699785232544, + "rewards/accuracies": 0.8125, + "rewards/chosen": -5.5512495040893555, + "rewards/margins": 2.5551297664642334, + "rewards/rejected": -8.106379508972168, + "step": 249 + }, + { + "epoch": 0.8064516129032258, + "grad_norm": 0.9459141492843628, + "learning_rate": 3.2604849095224666e-06, + "logits/chosen": -1.822584629058838, + "logits/rejected": -1.5994616746902466, + "logps/chosen": -162.4025421142578, + "logps/rejected": -216.37953186035156, + "loss": 0.3979412913322449, + "rewards/accuracies": 0.75, + "rewards/chosen": -5.902081489562988, + "rewards/margins": 2.4182045459747314, + "rewards/rejected": -8.32028579711914, + "step": 250 + }, + { + "epoch": 0.8096774193548387, + "grad_norm": 1.1870235204696655, + "learning_rate": 3.248407807433396e-06, + "logits/chosen": -2.156320571899414, + "logits/rejected": -1.949864149093628, + "logps/chosen": -172.03271484375, + "logps/rejected": -209.26580810546875, + "loss": 0.5271865129470825, + "rewards/accuracies": 0.71875, + "rewards/chosen": -6.29229736328125, + "rewards/margins": 1.8417139053344727, + "rewards/rejected": -8.134011268615723, + "step": 251 + }, + { + "epoch": 0.8129032258064516, + "grad_norm": 0.948936402797699, + "learning_rate": 3.2363114897533095e-06, + "logits/chosen": -1.9766535758972168, + "logits/rejected": -1.7797175645828247, + "logps/chosen": -151.3767852783203, + "logps/rejected": -203.56295776367188, + "loss": 0.4569253921508789, + "rewards/accuracies": 0.84375, + "rewards/chosen": -5.442215919494629, + "rewards/margins": 2.54245924949646, + "rewards/rejected": -7.984674453735352, + "step": 252 + }, + { + "epoch": 0.8161290322580645, + "grad_norm": 0.7237331867218018, + "learning_rate": 3.2241962670587312e-06, + "logits/chosen": -2.1904726028442383, + "logits/rejected": -1.8860106468200684, + "logps/chosen": -157.61126708984375, + "logps/rejected": -198.8243865966797, + "loss": 0.2793291211128235, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.772243976593018, + "rewards/margins": 2.6312925815582275, + "rewards/rejected": -7.403536796569824, + "step": 253 + }, + { + "epoch": 0.8193548387096774, + "grad_norm": 0.7647410035133362, + "learning_rate": 3.212062450411572e-06, + "logits/chosen": -2.5955586433410645, + "logits/rejected": -2.3625893592834473, + "logps/chosen": -154.66592407226562, + "logps/rejected": -209.16885375976562, + "loss": 0.27114158868789673, + "rewards/accuracies": 0.84375, + "rewards/chosen": -5.201878070831299, + "rewards/margins": 2.639791488647461, + "rewards/rejected": -7.841669082641602, + "step": 254 + }, + { + "epoch": 0.8225806451612904, + "grad_norm": 1.015256404876709, + "learning_rate": 3.1999103513511528e-06, + "logits/chosen": -2.0532336235046387, + "logits/rejected": -1.8524454832077026, + "logps/chosen": -149.565673828125, + "logps/rejected": -197.91754150390625, + "loss": 0.4156733751296997, + "rewards/accuracies": 0.78125, + "rewards/chosen": -4.924131393432617, + "rewards/margins": 2.6219491958618164, + "rewards/rejected": -7.546079635620117, + "step": 255 + }, + { + "epoch": 0.8258064516129032, + "grad_norm": 0.6962264776229858, + "learning_rate": 3.1877402818861954e-06, + "logits/chosen": -2.1449079513549805, + "logits/rejected": -1.8325960636138916, + "logps/chosen": -154.07302856445312, + "logps/rejected": -211.74591064453125, + "loss": 0.2757159471511841, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.438961982727051, + "rewards/margins": 3.2228941917419434, + "rewards/rejected": -8.661855697631836, + "step": 256 + }, + { + "epoch": 0.8290322580645161, + "grad_norm": 0.7284089922904968, + "learning_rate": 3.175552554486822e-06, + "logits/chosen": -2.0035736560821533, + "logits/rejected": -1.7146947383880615, + "logps/chosen": -161.89108276367188, + "logps/rejected": -232.00650024414062, + "loss": 0.2483557164669037, + "rewards/accuracies": 0.875, + "rewards/chosen": -6.063234806060791, + "rewards/margins": 3.6449978351593018, + "rewards/rejected": -9.708232879638672, + "step": 257 + }, + { + "epoch": 0.832258064516129, + "grad_norm": 0.7539389729499817, + "learning_rate": 3.1633474820765243e-06, + "logits/chosen": -2.1420180797576904, + "logits/rejected": -1.845510482788086, + "logps/chosen": -166.37063598632812, + "logps/rejected": -222.24362182617188, + "loss": 0.2929441034793854, + "rewards/accuracies": 0.8125, + "rewards/chosen": -6.11561918258667, + "rewards/margins": 2.9755101203918457, + "rewards/rejected": -9.091129302978516, + "step": 258 + }, + { + "epoch": 0.8354838709677419, + "grad_norm": 0.6925541758537292, + "learning_rate": 3.1511253780241334e-06, + "logits/chosen": -2.2203574180603027, + "logits/rejected": -1.9398572444915771, + "logps/chosen": -157.34991455078125, + "logps/rejected": -214.30502319335938, + "loss": 0.27536725997924805, + "rewards/accuracies": 0.90625, + "rewards/chosen": -5.440666198730469, + "rewards/margins": 3.108762741088867, + "rewards/rejected": -8.549429893493652, + "step": 259 + }, + { + "epoch": 0.8387096774193549, + "grad_norm": 0.9221241474151611, + "learning_rate": 3.1388865561357727e-06, + "logits/chosen": -1.9219483137130737, + "logits/rejected": -1.714083194732666, + "logps/chosen": -173.53128051757812, + "logps/rejected": -221.80712890625, + "loss": 0.3261393904685974, + "rewards/accuracies": 0.78125, + "rewards/chosen": -6.202951431274414, + "rewards/margins": 2.770026445388794, + "rewards/rejected": -8.972977638244629, + "step": 260 + }, + { + "epoch": 0.8419354838709677, + "grad_norm": 0.7738852500915527, + "learning_rate": 3.1266313306468018e-06, + "logits/chosen": -2.082782745361328, + "logits/rejected": -1.8457026481628418, + "logps/chosen": -151.79415893554688, + "logps/rejected": -202.09934997558594, + "loss": 0.31562310457229614, + "rewards/accuracies": 0.84375, + "rewards/chosen": -4.857356548309326, + "rewards/margins": 3.0326480865478516, + "rewards/rejected": -7.8900041580200195, + "step": 261 + }, + { + "epoch": 0.8451612903225807, + "grad_norm": 0.6938282251358032, + "learning_rate": 3.1143600162137454e-06, + "logits/chosen": -1.8344019651412964, + "logits/rejected": -1.5771279335021973, + "logps/chosen": -166.30076599121094, + "logps/rejected": -221.72154235839844, + "loss": 0.2417217195034027, + "rewards/accuracies": 0.90625, + "rewards/chosen": -5.7909040451049805, + "rewards/margins": 2.8294739723205566, + "rewards/rejected": -8.620379447937012, + "step": 262 + }, + { + "epoch": 0.8483870967741935, + "grad_norm": 0.5476120710372925, + "learning_rate": 3.1020729279062194e-06, + "logits/chosen": -1.976428747177124, + "logits/rejected": -1.6645793914794922, + "logps/chosen": -165.8445587158203, + "logps/rejected": -224.68304443359375, + "loss": 0.13630928099155426, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.6656494140625, + "rewards/margins": 3.6471195220947266, + "rewards/rejected": -9.312768936157227, + "step": 263 + }, + { + "epoch": 0.8516129032258064, + "grad_norm": 0.678778886795044, + "learning_rate": 3.089770381198836e-06, + "logits/chosen": -2.016895294189453, + "logits/rejected": -1.7314876317977905, + "logps/chosen": -153.4365692138672, + "logps/rejected": -226.68487548828125, + "loss": 0.1903066337108612, + "rewards/accuracies": 0.90625, + "rewards/chosen": -5.526880741119385, + "rewards/margins": 3.849604606628418, + "rewards/rejected": -9.376485824584961, + "step": 264 + }, + { + "epoch": 0.8548387096774194, + "grad_norm": 0.9369924068450928, + "learning_rate": 3.077452691963109e-06, + "logits/chosen": -1.753978967666626, + "logits/rejected": -1.5365220308303833, + "logps/chosen": -162.50265502929688, + "logps/rejected": -208.20266723632812, + "loss": 0.35749346017837524, + "rewards/accuracies": 0.78125, + "rewards/chosen": -5.593669414520264, + "rewards/margins": 2.4858767986297607, + "rewards/rejected": -8.079545974731445, + "step": 265 + }, + { + "epoch": 0.8580645161290322, + "grad_norm": 0.8137779831886292, + "learning_rate": 3.0651201764593375e-06, + "logits/chosen": -2.141214370727539, + "logits/rejected": -1.7222539186477661, + "logps/chosen": -141.45132446289062, + "logps/rejected": -208.5587158203125, + "loss": 0.22137892246246338, + "rewards/accuracies": 0.90625, + "rewards/chosen": -5.138533592224121, + "rewards/margins": 3.4072115421295166, + "rewards/rejected": -8.545745849609375, + "step": 266 + }, + { + "epoch": 0.8612903225806452, + "grad_norm": 0.6460467576980591, + "learning_rate": 3.052773151328493e-06, + "logits/chosen": -2.018655776977539, + "logits/rejected": -1.7169384956359863, + "logps/chosen": -157.74383544921875, + "logps/rejected": -213.468505859375, + "loss": 0.24283014237880707, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.804964065551758, + "rewards/margins": 2.993710517883301, + "rewards/rejected": -8.798675537109375, + "step": 267 + }, + { + "epoch": 0.864516129032258, + "grad_norm": 0.7242834568023682, + "learning_rate": 3.0404119335840844e-06, + "logits/chosen": -1.9196281433105469, + "logits/rejected": -1.55117928981781, + "logps/chosen": -145.27304077148438, + "logps/rejected": -216.47206115722656, + "loss": 0.224162757396698, + "rewards/accuracies": 0.84375, + "rewards/chosen": -5.446750640869141, + "rewards/margins": 3.864659070968628, + "rewards/rejected": -9.311409950256348, + "step": 268 + }, + { + "epoch": 0.867741935483871, + "grad_norm": 0.836055338382721, + "learning_rate": 3.028036840604019e-06, + "logits/chosen": -2.284193277359009, + "logits/rejected": -2.016956329345703, + "logps/chosen": -143.85357666015625, + "logps/rejected": -188.52374267578125, + "loss": 0.2850709557533264, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.036369800567627, + "rewards/margins": 2.801107406616211, + "rewards/rejected": -7.837477684020996, + "step": 269 + }, + { + "epoch": 0.8709677419354839, + "grad_norm": 1.087480902671814, + "learning_rate": 3.0156481901224573e-06, + "logits/chosen": -1.7794990539550781, + "logits/rejected": -1.509232759475708, + "logps/chosen": -168.82406616210938, + "logps/rejected": -228.95497131347656, + "loss": 0.432708740234375, + "rewards/accuracies": 0.78125, + "rewards/chosen": -6.362540245056152, + "rewards/margins": 3.126552104949951, + "rewards/rejected": -9.489091873168945, + "step": 270 + }, + { + "epoch": 0.8741935483870967, + "grad_norm": 1.1399589776992798, + "learning_rate": 3.0032463002216504e-06, + "logits/chosen": -1.7899502515792847, + "logits/rejected": -1.5215126276016235, + "logps/chosen": -172.01519775390625, + "logps/rejected": -225.363037109375, + "loss": 0.5371558666229248, + "rewards/accuracies": 0.8125, + "rewards/chosen": -5.870519638061523, + "rewards/margins": 2.98347544670105, + "rewards/rejected": -8.853995323181152, + "step": 271 + }, + { + "epoch": 0.8774193548387097, + "grad_norm": 1.1370657682418823, + "learning_rate": 2.990831489323778e-06, + "logits/chosen": -1.944093942642212, + "logits/rejected": -1.6760796308517456, + "logps/chosen": -179.9832000732422, + "logps/rejected": -238.34197998046875, + "loss": 0.38257351517677307, + "rewards/accuracies": 0.8125, + "rewards/chosen": -6.727067470550537, + "rewards/margins": 2.7800581455230713, + "rewards/rejected": -9.507125854492188, + "step": 272 + }, + { + "epoch": 0.8806451612903226, + "grad_norm": 0.8905435800552368, + "learning_rate": 2.978404076182766e-06, + "logits/chosen": -1.9256776571273804, + "logits/rejected": -1.6671595573425293, + "logps/chosen": -148.33238220214844, + "logps/rejected": -205.4830322265625, + "loss": 0.38766035437583923, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.708691596984863, + "rewards/margins": 3.060739517211914, + "rewards/rejected": -7.769431114196777, + "step": 273 + }, + { + "epoch": 0.8838709677419355, + "grad_norm": 0.8603253364562988, + "learning_rate": 2.965964379876112e-06, + "logits/chosen": -1.9746428728103638, + "logits/rejected": -1.6785167455673218, + "logps/chosen": -156.52313232421875, + "logps/rejected": -218.1029815673828, + "loss": 0.3216756284236908, + "rewards/accuracies": 0.8125, + "rewards/chosen": -5.569048881530762, + "rewards/margins": 2.990276575088501, + "rewards/rejected": -8.559325218200684, + "step": 274 + }, + { + "epoch": 0.8870967741935484, + "grad_norm": 0.7117762565612793, + "learning_rate": 2.953512719796683e-06, + "logits/chosen": -1.9952205419540405, + "logits/rejected": -1.6901299953460693, + "logps/chosen": -147.76272583007812, + "logps/rejected": -213.8414306640625, + "loss": 0.3437959551811218, + "rewards/accuracies": 0.90625, + "rewards/chosen": -5.44565486907959, + "rewards/margins": 3.351426601409912, + "rewards/rejected": -8.79708194732666, + "step": 275 + }, + { + "epoch": 0.8903225806451613, + "grad_norm": 0.9500011205673218, + "learning_rate": 2.941049415644522e-06, + "logits/chosen": -1.6247742176055908, + "logits/rejected": -1.3834972381591797, + "logps/chosen": -164.38734436035156, + "logps/rejected": -213.3121337890625, + "loss": 0.3649950623512268, + "rewards/accuracies": 0.8125, + "rewards/chosen": -6.385304927825928, + "rewards/margins": 2.792433977127075, + "rewards/rejected": -9.177739143371582, + "step": 276 + }, + { + "epoch": 0.8935483870967742, + "grad_norm": 0.924250602722168, + "learning_rate": 2.9285747874186345e-06, + "logits/chosen": -1.9340094327926636, + "logits/rejected": -1.5965994596481323, + "logps/chosen": -150.30023193359375, + "logps/rejected": -220.32684326171875, + "loss": 0.3774546980857849, + "rewards/accuracies": 0.78125, + "rewards/chosen": -4.861666679382324, + "rewards/margins": 3.435051202774048, + "rewards/rejected": -8.296717643737793, + "step": 277 + }, + { + "epoch": 0.896774193548387, + "grad_norm": 0.6413925290107727, + "learning_rate": 2.916089155408778e-06, + "logits/chosen": -1.7598278522491455, + "logits/rejected": -1.4939292669296265, + "logps/chosen": -171.2310791015625, + "logps/rejected": -219.10232543945312, + "loss": 0.21341019868850708, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.737621307373047, + "rewards/margins": 2.784102201461792, + "rewards/rejected": -8.521722793579102, + "step": 278 + }, + { + "epoch": 0.9, + "grad_norm": 0.5707635283470154, + "learning_rate": 2.9035928401872336e-06, + "logits/chosen": -2.0271811485290527, + "logits/rejected": -1.7087454795837402, + "logps/chosen": -167.06146240234375, + "logps/rejected": -230.35206604003906, + "loss": 0.17337509989738464, + "rewards/accuracies": 0.90625, + "rewards/chosen": -5.494855880737305, + "rewards/margins": 3.8487350940704346, + "rewards/rejected": -9.34359073638916, + "step": 279 + }, + { + "epoch": 0.9032258064516129, + "grad_norm": 0.852750837802887, + "learning_rate": 2.8910861626005774e-06, + "logits/chosen": -1.9636651277542114, + "logits/rejected": -1.675203561782837, + "logps/chosen": -151.33860778808594, + "logps/rejected": -214.5986785888672, + "loss": 0.2913362979888916, + "rewards/accuracies": 0.8125, + "rewards/chosen": -5.54245138168335, + "rewards/margins": 3.5291976928710938, + "rewards/rejected": -9.071649551391602, + "step": 280 + }, + { + "epoch": 0.9064516129032258, + "grad_norm": 0.6375999450683594, + "learning_rate": 2.878569443761442e-06, + "logits/chosen": -1.943008303642273, + "logits/rejected": -1.580283761024475, + "logps/chosen": -144.96212768554688, + "logps/rejected": -210.97207641601562, + "loss": 0.13702642917633057, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.5446062088012695, + "rewards/margins": 3.7235865592956543, + "rewards/rejected": -9.268192291259766, + "step": 281 + }, + { + "epoch": 0.9096774193548387, + "grad_norm": 0.8081200122833252, + "learning_rate": 2.866043005040272e-06, + "logits/chosen": -1.9890791177749634, + "logits/rejected": -1.6013154983520508, + "logps/chosen": -170.92636108398438, + "logps/rejected": -234.09942626953125, + "loss": 0.26003971695899963, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.83311653137207, + "rewards/margins": 3.469595432281494, + "rewards/rejected": -9.302712440490723, + "step": 282 + }, + { + "epoch": 0.9129032258064517, + "grad_norm": 1.0241976976394653, + "learning_rate": 2.8535071680570735e-06, + "logits/chosen": -2.450965642929077, + "logits/rejected": -2.1390633583068848, + "logps/chosen": -145.23941040039062, + "logps/rejected": -197.3958740234375, + "loss": 0.5087136030197144, + "rewards/accuracies": 0.84375, + "rewards/chosen": -4.842174530029297, + "rewards/margins": 2.902985095977783, + "rewards/rejected": -7.745160102844238, + "step": 283 + }, + { + "epoch": 0.9161290322580645, + "grad_norm": 0.8598202466964722, + "learning_rate": 2.840962254673156e-06, + "logits/chosen": -2.0429110527038574, + "logits/rejected": -1.732966423034668, + "logps/chosen": -166.97962951660156, + "logps/rejected": -218.309326171875, + "loss": 0.3195725977420807, + "rewards/accuracies": 0.8125, + "rewards/chosen": -6.148334503173828, + "rewards/margins": 2.3881964683532715, + "rewards/rejected": -8.536530494689941, + "step": 284 + }, + { + "epoch": 0.9193548387096774, + "grad_norm": 0.9417896270751953, + "learning_rate": 2.8284085869828664e-06, + "logits/chosen": -2.085608720779419, + "logits/rejected": -1.8423289060592651, + "logps/chosen": -160.0970001220703, + "logps/rejected": -206.8173370361328, + "loss": 0.4341191351413727, + "rewards/accuracies": 0.84375, + "rewards/chosen": -5.784702301025391, + "rewards/margins": 2.376483917236328, + "rewards/rejected": -8.161186218261719, + "step": 285 + }, + { + "epoch": 0.9225806451612903, + "grad_norm": 1.097767949104309, + "learning_rate": 2.8158464873053236e-06, + "logits/chosen": -2.0979247093200684, + "logits/rejected": -1.7694401741027832, + "logps/chosen": -152.70974731445312, + "logps/rejected": -214.92532348632812, + "loss": 0.3746206760406494, + "rewards/accuracies": 0.8125, + "rewards/chosen": -5.5105414390563965, + "rewards/margins": 2.975477695465088, + "rewards/rejected": -8.486019134521484, + "step": 286 + }, + { + "epoch": 0.9258064516129032, + "grad_norm": 1.082982063293457, + "learning_rate": 2.8032762781761376e-06, + "logits/chosen": -2.130652666091919, + "logits/rejected": -1.8887903690338135, + "logps/chosen": -152.620849609375, + "logps/rejected": -197.34225463867188, + "loss": 0.46686190366744995, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.933390140533447, + "rewards/margins": 2.3892898559570312, + "rewards/rejected": -7.32267951965332, + "step": 287 + }, + { + "epoch": 0.9290322580645162, + "grad_norm": 0.8161581754684448, + "learning_rate": 2.790698282339133e-06, + "logits/chosen": -2.4120960235595703, + "logits/rejected": -2.0684590339660645, + "logps/chosen": -128.7693634033203, + "logps/rejected": -190.23130798339844, + "loss": 0.3193214535713196, + "rewards/accuracies": 0.84375, + "rewards/chosen": -4.401476860046387, + "rewards/margins": 3.0362207889556885, + "rewards/rejected": -7.437697410583496, + "step": 288 + }, + { + "epoch": 0.932258064516129, + "grad_norm": 0.9246862530708313, + "learning_rate": 2.7781128227380595e-06, + "logits/chosen": -2.4583992958068848, + "logits/rejected": -2.163020610809326, + "logps/chosen": -149.81240844726562, + "logps/rejected": -206.80284118652344, + "loss": 0.33852618932724, + "rewards/accuracies": 0.84375, + "rewards/chosen": -4.645122528076172, + "rewards/margins": 2.739440441131592, + "rewards/rejected": -7.384562969207764, + "step": 289 + }, + { + "epoch": 0.9354838709677419, + "grad_norm": 0.8552197217941284, + "learning_rate": 2.765520222508302e-06, + "logits/chosen": -2.0391581058502197, + "logits/rejected": -1.685469388961792, + "logps/chosen": -160.41790771484375, + "logps/rejected": -214.21324157714844, + "loss": 0.3561515808105469, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.971251487731934, + "rewards/margins": 2.890326738357544, + "rewards/rejected": -7.861577987670898, + "step": 290 + }, + { + "epoch": 0.9387096774193548, + "grad_norm": 1.0460398197174072, + "learning_rate": 2.752920804968581e-06, + "logits/chosen": -2.1437153816223145, + "logits/rejected": -2.0144546031951904, + "logps/chosen": -166.39102172851562, + "logps/rejected": -208.45326232910156, + "loss": 0.47864845395088196, + "rewards/accuracies": 0.84375, + "rewards/chosen": -5.143545627593994, + "rewards/margins": 1.8947696685791016, + "rewards/rejected": -7.038315773010254, + "step": 291 + }, + { + "epoch": 0.9419354838709677, + "grad_norm": 0.44466134905815125, + "learning_rate": 2.740314893612656e-06, + "logits/chosen": -2.0010452270507812, + "logits/rejected": -1.6200562715530396, + "logps/chosen": -137.75228881835938, + "logps/rejected": -212.52597045898438, + "loss": 0.13167953491210938, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.997614622116089, + "rewards/margins": 4.028761863708496, + "rewards/rejected": -8.026376724243164, + "step": 292 + }, + { + "epoch": 0.9451612903225807, + "grad_norm": 0.8740090131759644, + "learning_rate": 2.727702812101016e-06, + "logits/chosen": -2.171569347381592, + "logits/rejected": -1.9190174341201782, + "logps/chosen": -145.79901123046875, + "logps/rejected": -209.12521362304688, + "loss": 0.25788789987564087, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.480965614318848, + "rewards/margins": 2.9965264797210693, + "rewards/rejected": -7.47749137878418, + "step": 293 + }, + { + "epoch": 0.9483870967741935, + "grad_norm": 0.9442008137702942, + "learning_rate": 2.7150848842525707e-06, + "logits/chosen": -1.9686082601547241, + "logits/rejected": -1.769315242767334, + "logps/chosen": -161.62548828125, + "logps/rejected": -213.39918518066406, + "loss": 0.3881966173648834, + "rewards/accuracies": 0.8125, + "rewards/chosen": -5.6515092849731445, + "rewards/margins": 2.542401075363159, + "rewards/rejected": -8.193910598754883, + "step": 294 + }, + { + "epoch": 0.9516129032258065, + "grad_norm": 0.8742654323577881, + "learning_rate": 2.7024614340363365e-06, + "logits/chosen": -1.7042088508605957, + "logits/rejected": -1.550293207168579, + "logps/chosen": -176.0225830078125, + "logps/rejected": -210.4657745361328, + "loss": 0.423173725605011, + "rewards/accuracies": 0.84375, + "rewards/chosen": -5.02152156829834, + "rewards/margins": 2.2260966300964355, + "rewards/rejected": -7.247618198394775, + "step": 295 + }, + { + "epoch": 0.9548387096774194, + "grad_norm": 0.6472557783126831, + "learning_rate": 2.689832785563116e-06, + "logits/chosen": -1.7272298336029053, + "logits/rejected": -1.3803935050964355, + "logps/chosen": -164.98233032226562, + "logps/rejected": -229.7456817626953, + "loss": 0.20416554808616638, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.407524108886719, + "rewards/margins": 3.4527716636657715, + "rewards/rejected": -8.860294342041016, + "step": 296 + }, + { + "epoch": 0.9580645161290322, + "grad_norm": 0.6706839203834534, + "learning_rate": 2.6771992630771824e-06, + "logits/chosen": -1.798374056816101, + "logits/rejected": -1.4753650426864624, + "logps/chosen": -163.36074829101562, + "logps/rejected": -225.06646728515625, + "loss": 0.17277327179908752, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.153785705566406, + "rewards/margins": 3.462008237838745, + "rewards/rejected": -8.61579418182373, + "step": 297 + }, + { + "epoch": 0.9612903225806452, + "grad_norm": 1.067395806312561, + "learning_rate": 2.6645611909479484e-06, + "logits/chosen": -2.2678630352020264, + "logits/rejected": -2.0605649948120117, + "logps/chosen": -148.91651916503906, + "logps/rejected": -192.33926391601562, + "loss": 0.5109429955482483, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.3728766441345215, + "rewards/margins": 2.1208512783050537, + "rewards/rejected": -6.493727684020996, + "step": 298 + }, + { + "epoch": 0.964516129032258, + "grad_norm": 0.8057169914245605, + "learning_rate": 2.65191889366164e-06, + "logits/chosen": -2.063993453979492, + "logits/rejected": -1.9815953969955444, + "logps/chosen": -158.8735809326172, + "logps/rejected": -204.44784545898438, + "loss": 0.29160138964653015, + "rewards/accuracies": 0.84375, + "rewards/chosen": -4.964184761047363, + "rewards/margins": 2.093125820159912, + "rewards/rejected": -7.057311058044434, + "step": 299 + }, + { + "epoch": 0.967741935483871, + "grad_norm": 0.831497073173523, + "learning_rate": 2.6392726958129653e-06, + "logits/chosen": -2.101418972015381, + "logits/rejected": -1.8252373933792114, + "logps/chosen": -144.0537109375, + "logps/rejected": -195.8699951171875, + "loss": 0.3357257843017578, + "rewards/accuracies": 0.84375, + "rewards/chosen": -5.049233436584473, + "rewards/margins": 2.7640864849090576, + "rewards/rejected": -7.813320159912109, + "step": 300 + }, + { + "epoch": 0.9709677419354839, + "grad_norm": 0.7803052663803101, + "learning_rate": 2.626622922096782e-06, + "logits/chosen": -1.9163737297058105, + "logits/rejected": -1.6509616374969482, + "logps/chosen": -155.09027099609375, + "logps/rejected": -208.73690795898438, + "loss": 0.26208123564720154, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.911482810974121, + "rewards/margins": 2.844212532043457, + "rewards/rejected": -7.75569486618042, + "step": 301 + }, + { + "epoch": 0.9741935483870968, + "grad_norm": 0.8996358513832092, + "learning_rate": 2.613969897299757e-06, + "logits/chosen": -2.241206645965576, + "logits/rejected": -2.0861012935638428, + "logps/chosen": -149.37405395507812, + "logps/rejected": -192.15811157226562, + "loss": 0.31268051266670227, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.1501898765563965, + "rewards/margins": 2.2168045043945312, + "rewards/rejected": -7.366994380950928, + "step": 302 + }, + { + "epoch": 0.9774193548387097, + "grad_norm": 0.7502899169921875, + "learning_rate": 2.6013139462920328e-06, + "logits/chosen": -1.9167232513427734, + "logits/rejected": -1.5168886184692383, + "logps/chosen": -150.22955322265625, + "logps/rejected": -229.1195068359375, + "loss": 0.18935909867286682, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.952530860900879, + "rewards/margins": 4.4970598220825195, + "rewards/rejected": -9.449591636657715, + "step": 303 + }, + { + "epoch": 0.9806451612903225, + "grad_norm": 0.8504204154014587, + "learning_rate": 2.588655394018879e-06, + "logits/chosen": -2.1758933067321777, + "logits/rejected": -1.9743213653564453, + "logps/chosen": -145.38467407226562, + "logps/rejected": -210.38336181640625, + "loss": 0.2861388921737671, + "rewards/accuracies": 0.8125, + "rewards/chosen": -5.063721656799316, + "rewards/margins": 3.1153056621551514, + "rewards/rejected": -8.17902660369873, + "step": 304 + }, + { + "epoch": 0.9838709677419355, + "grad_norm": 0.7789464592933655, + "learning_rate": 2.5759945654923575e-06, + "logits/chosen": -2.010745048522949, + "logits/rejected": -1.8690520524978638, + "logps/chosen": -141.83558654785156, + "logps/rejected": -194.40895080566406, + "loss": 0.23465344309806824, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.525321006774902, + "rewards/margins": 2.7977147102355957, + "rewards/rejected": -7.32303524017334, + "step": 305 + }, + { + "epoch": 0.9870967741935484, + "grad_norm": 1.0136746168136597, + "learning_rate": 2.56333178578297e-06, + "logits/chosen": -2.1397182941436768, + "logits/rejected": -1.9402344226837158, + "logps/chosen": -172.6583709716797, + "logps/rejected": -228.28738403320312, + "loss": 0.44921568036079407, + "rewards/accuracies": 0.78125, + "rewards/chosen": -6.19842004776001, + "rewards/margins": 3.0715999603271484, + "rewards/rejected": -9.27001953125, + "step": 306 + }, + { + "epoch": 0.9903225806451613, + "grad_norm": 0.965210497379303, + "learning_rate": 2.5506673800113196e-06, + "logits/chosen": -2.106553554534912, + "logits/rejected": -1.8104825019836426, + "logps/chosen": -172.39144897460938, + "logps/rejected": -240.924072265625, + "loss": 0.24592578411102295, + "rewards/accuracies": 0.90625, + "rewards/chosen": -6.227846622467041, + "rewards/margins": 3.537505626678467, + "rewards/rejected": -9.765352249145508, + "step": 307 + }, + { + "epoch": 0.9935483870967742, + "grad_norm": 0.9008381962776184, + "learning_rate": 2.538001673339754e-06, + "logits/chosen": -1.8831700086593628, + "logits/rejected": -1.5683573484420776, + "logps/chosen": -174.13272094726562, + "logps/rejected": -222.25619506835938, + "loss": 0.3525466322898865, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.677732467651367, + "rewards/margins": 2.817638635635376, + "rewards/rejected": -8.495370864868164, + "step": 308 + }, + { + "epoch": 0.9967741935483871, + "grad_norm": 0.8979695439338684, + "learning_rate": 2.525334990964028e-06, + "logits/chosen": -2.152176856994629, + "logits/rejected": -1.8049001693725586, + "logps/chosen": -136.01487731933594, + "logps/rejected": -209.47459411621094, + "loss": 0.38963136076927185, + "rewards/accuracies": 0.78125, + "rewards/chosen": -4.782433032989502, + "rewards/margins": 3.5359387397766113, + "rewards/rejected": -8.318371772766113, + "step": 309 + }, + { + "epoch": 1.0, + "grad_norm": 0.8429840803146362, + "learning_rate": 2.5126676581049413e-06, + "logits/chosen": -2.113762378692627, + "logits/rejected": -1.818450927734375, + "logps/chosen": -167.97877502441406, + "logps/rejected": -227.44183349609375, + "loss": 0.30903160572052, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.818127155303955, + "rewards/margins": 3.091285467147827, + "rewards/rejected": -8.909412384033203, + "step": 310 + }, + { + "epoch": 1.0032258064516129, + "grad_norm": 0.6775534152984619, + "learning_rate": 2.5e-06, + "logits/chosen": -2.1721529960632324, + "logits/rejected": -1.9033340215682983, + "logps/chosen": -163.31370544433594, + "logps/rejected": -222.24139404296875, + "loss": 0.21583493053913116, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.506072044372559, + "rewards/margins": 3.459946632385254, + "rewards/rejected": -8.966018676757812, + "step": 311 + }, + { + "epoch": 1.0064516129032257, + "grad_norm": 0.580848753452301, + "learning_rate": 2.487332341895059e-06, + "logits/chosen": -2.0694403648376465, + "logits/rejected": -1.747127652168274, + "logps/chosen": -149.02215576171875, + "logps/rejected": -212.60113525390625, + "loss": 0.16063669323921204, + "rewards/accuracies": 0.90625, + "rewards/chosen": -4.965436935424805, + "rewards/margins": 3.742509365081787, + "rewards/rejected": -8.70794677734375, + "step": 312 + }, + { + "epoch": 1.0096774193548388, + "grad_norm": 0.38149571418762207, + "learning_rate": 2.4746650090359733e-06, + "logits/chosen": -2.2192482948303223, + "logits/rejected": -1.9255717992782593, + "logps/chosen": -151.22067260742188, + "logps/rejected": -207.1800079345703, + "loss": 0.1110997125506401, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.916757583618164, + "rewards/margins": 3.4106850624084473, + "rewards/rejected": -8.327442169189453, + "step": 313 + }, + { + "epoch": 1.0129032258064516, + "grad_norm": 0.5131282210350037, + "learning_rate": 2.461998326660246e-06, + "logits/chosen": -2.0201852321624756, + "logits/rejected": -1.7313728332519531, + "logps/chosen": -157.92855834960938, + "logps/rejected": -210.2177734375, + "loss": 0.15024253726005554, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.242098331451416, + "rewards/margins": 3.1161749362945557, + "rewards/rejected": -8.358272552490234, + "step": 314 + }, + { + "epoch": 1.0161290322580645, + "grad_norm": 0.6810634136199951, + "learning_rate": 2.4493326199886813e-06, + "logits/chosen": -2.2641396522521973, + "logits/rejected": -2.006556272506714, + "logps/chosen": -158.7135009765625, + "logps/rejected": -215.43856811523438, + "loss": 0.19132620096206665, + "rewards/accuracies": 0.90625, + "rewards/chosen": -4.935375213623047, + "rewards/margins": 3.58085298538208, + "rewards/rejected": -8.516228675842285, + "step": 315 + }, + { + "epoch": 1.0193548387096774, + "grad_norm": 0.5285111665725708, + "learning_rate": 2.436668214217031e-06, + "logits/chosen": -2.0756735801696777, + "logits/rejected": -1.842437744140625, + "logps/chosen": -178.60182189941406, + "logps/rejected": -226.6915283203125, + "loss": 0.1702248901128769, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.96095085144043, + "rewards/margins": 3.078915596008301, + "rewards/rejected": -9.03986644744873, + "step": 316 + }, + { + "epoch": 1.0225806451612902, + "grad_norm": 0.5416440367698669, + "learning_rate": 2.424005434507644e-06, + "logits/chosen": -1.8712788820266724, + "logits/rejected": -1.5404462814331055, + "logps/chosen": -181.17999267578125, + "logps/rejected": -250.18528747558594, + "loss": 0.1307283192873001, + "rewards/accuracies": 0.96875, + "rewards/chosen": -6.399822235107422, + "rewards/margins": 4.233570098876953, + "rewards/rejected": -10.633392333984375, + "step": 317 + }, + { + "epoch": 1.0258064516129033, + "grad_norm": 0.4123290777206421, + "learning_rate": 2.4113446059811217e-06, + "logits/chosen": -1.9893611669540405, + "logits/rejected": -1.720315933227539, + "logps/chosen": -172.31666564941406, + "logps/rejected": -229.56271362304688, + "loss": 0.11024826765060425, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.526625633239746, + "rewards/margins": 3.5636584758758545, + "rewards/rejected": -9.09028434753418, + "step": 318 + }, + { + "epoch": 1.0290322580645161, + "grad_norm": 0.7053832411766052, + "learning_rate": 2.3986860537079676e-06, + "logits/chosen": -1.8367996215820312, + "logits/rejected": -1.4809139966964722, + "logps/chosen": -155.18275451660156, + "logps/rejected": -212.97434997558594, + "loss": 0.21443486213684082, + "rewards/accuracies": 0.90625, + "rewards/chosen": -5.035017967224121, + "rewards/margins": 3.13305926322937, + "rewards/rejected": -8.16807746887207, + "step": 319 + }, + { + "epoch": 1.032258064516129, + "grad_norm": 0.2714214622974396, + "learning_rate": 2.3860301027002432e-06, + "logits/chosen": -1.9262335300445557, + "logits/rejected": -1.5826025009155273, + "logps/chosen": -141.29434204101562, + "logps/rejected": -219.14004516601562, + "loss": 0.07531583309173584, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.152288436889648, + "rewards/margins": 4.746103286743164, + "rewards/rejected": -9.898390769958496, + "step": 320 + }, + { + "epoch": 1.0354838709677419, + "grad_norm": 0.4167848527431488, + "learning_rate": 2.3733770779032185e-06, + "logits/chosen": -2.2220044136047363, + "logits/rejected": -1.7983975410461426, + "logps/chosen": -133.76486206054688, + "logps/rejected": -216.0334014892578, + "loss": 0.12337338924407959, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.694788932800293, + "rewards/margins": 4.698427677154541, + "rewards/rejected": -9.393217086791992, + "step": 321 + }, + { + "epoch": 1.038709677419355, + "grad_norm": 0.424111545085907, + "learning_rate": 2.3607273041870355e-06, + "logits/chosen": -2.181683301925659, + "logits/rejected": -1.7694246768951416, + "logps/chosen": -133.67471313476562, + "logps/rejected": -207.6949920654297, + "loss": 0.08804497122764587, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.045483112335205, + "rewards/margins": 4.212948799133301, + "rewards/rejected": -8.258432388305664, + "step": 322 + }, + { + "epoch": 1.0419354838709678, + "grad_norm": 0.6895016431808472, + "learning_rate": 2.3480811063383603e-06, + "logits/chosen": -2.051720142364502, + "logits/rejected": -1.7328163385391235, + "logps/chosen": -158.16627502441406, + "logps/rejected": -217.3858642578125, + "loss": 0.19755461812019348, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.892249584197998, + "rewards/margins": 3.8145363330841064, + "rewards/rejected": -8.706785202026367, + "step": 323 + }, + { + "epoch": 1.0451612903225806, + "grad_norm": 0.4730503559112549, + "learning_rate": 2.335438809052052e-06, + "logits/chosen": -2.212907075881958, + "logits/rejected": -1.7950420379638672, + "logps/chosen": -147.1448211669922, + "logps/rejected": -224.41746520996094, + "loss": 0.12352735549211502, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.53288459777832, + "rewards/margins": 4.16781759262085, + "rewards/rejected": -9.700702667236328, + "step": 324 + }, + { + "epoch": 1.0483870967741935, + "grad_norm": 0.37026146054267883, + "learning_rate": 2.322800736922818e-06, + "logits/chosen": -1.8258745670318604, + "logits/rejected": -1.4101828336715698, + "logps/chosen": -165.53468322753906, + "logps/rejected": -251.0711669921875, + "loss": 0.07076412439346313, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.959969520568848, + "rewards/margins": 5.046208381652832, + "rewards/rejected": -11.006175994873047, + "step": 325 + }, + { + "epoch": 1.0516129032258064, + "grad_norm": 0.3133352994918823, + "learning_rate": 2.310167214436885e-06, + "logits/chosen": -2.0526835918426514, + "logits/rejected": -1.789970874786377, + "logps/chosen": -149.29171752929688, + "logps/rejected": -222.2518310546875, + "loss": 0.08292093873023987, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.683323383331299, + "rewards/margins": 4.279122829437256, + "rewards/rejected": -8.962446212768555, + "step": 326 + }, + { + "epoch": 1.0548387096774194, + "grad_norm": 0.5130003690719604, + "learning_rate": 2.297538565963665e-06, + "logits/chosen": -2.2903833389282227, + "logits/rejected": -1.977923035621643, + "logps/chosen": -140.78390502929688, + "logps/rejected": -208.8832550048828, + "loss": 0.18747130036354065, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.893636703491211, + "rewards/margins": 4.177481651306152, + "rewards/rejected": -9.071118354797363, + "step": 327 + }, + { + "epoch": 1.0580645161290323, + "grad_norm": 0.4587818682193756, + "learning_rate": 2.2849151157474297e-06, + "logits/chosen": -1.6147241592407227, + "logits/rejected": -1.2173912525177002, + "logps/chosen": -166.45401000976562, + "logps/rejected": -244.99241638183594, + "loss": 0.11498372256755829, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.213277816772461, + "rewards/margins": 4.60243034362793, + "rewards/rejected": -10.81570816040039, + "step": 328 + }, + { + "epoch": 1.0612903225806452, + "grad_norm": 0.41809093952178955, + "learning_rate": 2.2722971878989843e-06, + "logits/chosen": -1.6840178966522217, + "logits/rejected": -1.3177802562713623, + "logps/chosen": -172.50921630859375, + "logps/rejected": -257.9876403808594, + "loss": 0.07585393637418747, + "rewards/accuracies": 0.96875, + "rewards/chosen": -6.2325544357299805, + "rewards/margins": 4.915229320526123, + "rewards/rejected": -11.147783279418945, + "step": 329 + }, + { + "epoch": 1.064516129032258, + "grad_norm": 0.4529079496860504, + "learning_rate": 2.259685106387345e-06, + "logits/chosen": -2.2479963302612305, + "logits/rejected": -1.8993940353393555, + "logps/chosen": -142.16958618164062, + "logps/rejected": -214.33688354492188, + "loss": 0.14075274765491486, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.91881799697876, + "rewards/margins": 4.307929992675781, + "rewards/rejected": -9.2267484664917, + "step": 330 + }, + { + "epoch": 1.067741935483871, + "grad_norm": 0.8104569911956787, + "learning_rate": 2.24707919503142e-06, + "logits/chosen": -1.9945495128631592, + "logits/rejected": -1.7296099662780762, + "logps/chosen": -170.532958984375, + "logps/rejected": -235.51565551757812, + "loss": 0.2429746687412262, + "rewards/accuracies": 0.90625, + "rewards/chosen": -6.278332710266113, + "rewards/margins": 3.6663362979888916, + "rewards/rejected": -9.944669723510742, + "step": 331 + }, + { + "epoch": 1.070967741935484, + "grad_norm": 0.6494706869125366, + "learning_rate": 2.2344797774917e-06, + "logits/chosen": -2.075692653656006, + "logits/rejected": -1.7635282278060913, + "logps/chosen": -141.7442169189453, + "logps/rejected": -219.43185424804688, + "loss": 0.1584240049123764, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.1548686027526855, + "rewards/margins": 4.707845687866211, + "rewards/rejected": -9.862714767456055, + "step": 332 + }, + { + "epoch": 1.0741935483870968, + "grad_norm": 0.5460749268531799, + "learning_rate": 2.2218871772619414e-06, + "logits/chosen": -1.5849077701568604, + "logits/rejected": -1.2364206314086914, + "logps/chosen": -164.15150451660156, + "logps/rejected": -232.79945373535156, + "loss": 0.13620774447917938, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.4682841300964355, + "rewards/margins": 4.182202339172363, + "rewards/rejected": -9.65048599243164, + "step": 333 + }, + { + "epoch": 1.0774193548387097, + "grad_norm": 0.9638095498085022, + "learning_rate": 2.2093017176608678e-06, + "logits/chosen": -2.240260601043701, + "logits/rejected": -1.9081519842147827, + "logps/chosen": -140.4049835205078, + "logps/rejected": -199.9825439453125, + "loss": 0.280463844537735, + "rewards/accuracies": 0.90625, + "rewards/chosen": -4.912430286407471, + "rewards/margins": 3.654731035232544, + "rewards/rejected": -8.567161560058594, + "step": 334 + }, + { + "epoch": 1.0806451612903225, + "grad_norm": 0.6865034699440002, + "learning_rate": 2.196723721823863e-06, + "logits/chosen": -1.5741136074066162, + "logits/rejected": -1.3333443403244019, + "logps/chosen": -160.82054138183594, + "logps/rejected": -229.42044067382812, + "loss": 0.2099701464176178, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.163061141967773, + "rewards/margins": 3.6050832271575928, + "rewards/rejected": -9.768144607543945, + "step": 335 + }, + { + "epoch": 1.0838709677419356, + "grad_norm": 0.5247408747673035, + "learning_rate": 2.1841535126946777e-06, + "logits/chosen": -1.8184685707092285, + "logits/rejected": -1.509084939956665, + "logps/chosen": -175.3823699951172, + "logps/rejected": -234.17550659179688, + "loss": 0.13480132818222046, + "rewards/accuracies": 0.96875, + "rewards/chosen": -6.176336288452148, + "rewards/margins": 3.4384005069732666, + "rewards/rejected": -9.614737510681152, + "step": 336 + }, + { + "epoch": 1.0870967741935484, + "grad_norm": 0.3721334934234619, + "learning_rate": 2.171591413017134e-06, + "logits/chosen": -2.2993922233581543, + "logits/rejected": -1.9334840774536133, + "logps/chosen": -140.510986328125, + "logps/rejected": -212.41583251953125, + "loss": 0.10910043120384216, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.957540988922119, + "rewards/margins": 4.744095325469971, + "rewards/rejected": -9.701635360717773, + "step": 337 + }, + { + "epoch": 1.0903225806451613, + "grad_norm": 0.5746122002601624, + "learning_rate": 2.159037745326844e-06, + "logits/chosen": -1.9673866033554077, + "logits/rejected": -1.6878376007080078, + "logps/chosen": -161.87252807617188, + "logps/rejected": -217.6410369873047, + "loss": 0.1833765208721161, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.09132194519043, + "rewards/margins": 3.5239312648773193, + "rewards/rejected": -8.615253448486328, + "step": 338 + }, + { + "epoch": 1.0935483870967742, + "grad_norm": 0.6791886687278748, + "learning_rate": 2.146492831942927e-06, + "logits/chosen": -1.4476041793823242, + "logits/rejected": -1.112692952156067, + "logps/chosen": -196.64627075195312, + "logps/rejected": -260.59063720703125, + "loss": 0.15957781672477722, + "rewards/accuracies": 0.9375, + "rewards/chosen": -7.228837013244629, + "rewards/margins": 3.7981371879577637, + "rewards/rejected": -11.026973724365234, + "step": 339 + }, + { + "epoch": 1.096774193548387, + "grad_norm": 0.3706646263599396, + "learning_rate": 2.1339569949597284e-06, + "logits/chosen": -2.0635392665863037, + "logits/rejected": -1.683074951171875, + "logps/chosen": -140.18783569335938, + "logps/rejected": -207.48167419433594, + "loss": 0.08880328387022018, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.723042964935303, + "rewards/margins": 4.2141523361206055, + "rewards/rejected": -8.93719482421875, + "step": 340 + }, + { + "epoch": 1.1, + "grad_norm": 0.3307356834411621, + "learning_rate": 2.1214305562385592e-06, + "logits/chosen": -2.034179210662842, + "logits/rejected": -1.678894281387329, + "logps/chosen": -139.79940795898438, + "logps/rejected": -222.440185546875, + "loss": 0.07107400894165039, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.712695598602295, + "rewards/margins": 4.9850287437438965, + "rewards/rejected": -9.697725296020508, + "step": 341 + }, + { + "epoch": 1.103225806451613, + "grad_norm": 0.65967857837677, + "learning_rate": 2.1089138373994226e-06, + "logits/chosen": -2.1784443855285645, + "logits/rejected": -1.9255266189575195, + "logps/chosen": -152.837646484375, + "logps/rejected": -213.63534545898438, + "loss": 0.22975148260593414, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.133075714111328, + "rewards/margins": 3.4656684398651123, + "rewards/rejected": -8.59874439239502, + "step": 342 + }, + { + "epoch": 1.1064516129032258, + "grad_norm": 0.35155734419822693, + "learning_rate": 2.096407159812767e-06, + "logits/chosen": -2.32820463180542, + "logits/rejected": -1.971595287322998, + "logps/chosen": -145.63475036621094, + "logps/rejected": -218.73211669921875, + "loss": 0.07563674449920654, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.143950462341309, + "rewards/margins": 4.366302490234375, + "rewards/rejected": -9.510252952575684, + "step": 343 + }, + { + "epoch": 1.1096774193548387, + "grad_norm": 0.734062910079956, + "learning_rate": 2.0839108445912222e-06, + "logits/chosen": -1.8277361392974854, + "logits/rejected": -1.6029260158538818, + "logps/chosen": -171.10125732421875, + "logps/rejected": -232.6971435546875, + "loss": 0.20384469628334045, + "rewards/accuracies": 0.90625, + "rewards/chosen": -5.091307640075684, + "rewards/margins": 3.67594313621521, + "rewards/rejected": -8.767250061035156, + "step": 344 + }, + { + "epoch": 1.1129032258064515, + "grad_norm": 0.5717563033103943, + "learning_rate": 2.0714252125813667e-06, + "logits/chosen": -1.7721185684204102, + "logits/rejected": -1.4680637121200562, + "logps/chosen": -173.13760375976562, + "logps/rejected": -239.3628387451172, + "loss": 0.1405162811279297, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.596169471740723, + "rewards/margins": 3.965106725692749, + "rewards/rejected": -10.561275482177734, + "step": 345 + }, + { + "epoch": 1.1161290322580646, + "grad_norm": 0.6655075550079346, + "learning_rate": 2.05895058435548e-06, + "logits/chosen": -1.9385762214660645, + "logits/rejected": -1.6046488285064697, + "logps/chosen": -165.9267578125, + "logps/rejected": -237.217041015625, + "loss": 0.1554121971130371, + "rewards/accuracies": 0.96875, + "rewards/chosen": -6.313894271850586, + "rewards/margins": 4.2116498947143555, + "rewards/rejected": -10.525544166564941, + "step": 346 + }, + { + "epoch": 1.1193548387096774, + "grad_norm": 0.5084536075592041, + "learning_rate": 2.0464872802033174e-06, + "logits/chosen": -2.077972412109375, + "logits/rejected": -1.7521743774414062, + "logps/chosen": -163.38247680664062, + "logps/rejected": -234.90512084960938, + "loss": 0.1205044537782669, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.103501796722412, + "rewards/margins": 4.484037399291992, + "rewards/rejected": -10.587538719177246, + "step": 347 + }, + { + "epoch": 1.1225806451612903, + "grad_norm": 0.2510414123535156, + "learning_rate": 2.0340356201238883e-06, + "logits/chosen": -2.283794403076172, + "logits/rejected": -1.8476622104644775, + "logps/chosen": -145.0723876953125, + "logps/rejected": -223.35067749023438, + "loss": 0.05795178934931755, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.06467866897583, + "rewards/margins": 4.865185737609863, + "rewards/rejected": -9.929864883422852, + "step": 348 + }, + { + "epoch": 1.1258064516129032, + "grad_norm": 0.8760903477668762, + "learning_rate": 2.0215959238172343e-06, + "logits/chosen": -2.1980106830596924, + "logits/rejected": -1.9146432876586914, + "logps/chosen": -155.1454620361328, + "logps/rejected": -213.25482177734375, + "loss": 0.22108781337738037, + "rewards/accuracies": 0.90625, + "rewards/chosen": -6.415738105773926, + "rewards/margins": 3.4907078742980957, + "rewards/rejected": -9.906445503234863, + "step": 349 + }, + { + "epoch": 1.129032258064516, + "grad_norm": 0.640398383140564, + "learning_rate": 2.0091685106762233e-06, + "logits/chosen": -1.9763402938842773, + "logits/rejected": -1.772810459136963, + "logps/chosen": -174.51522827148438, + "logps/rejected": -224.62802124023438, + "loss": 0.20412787795066833, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.7324604988098145, + "rewards/margins": 3.1668648719787598, + "rewards/rejected": -9.899325370788574, + "step": 350 + }, + { + "epoch": 1.132258064516129, + "grad_norm": 0.34046998620033264, + "learning_rate": 1.9967536997783495e-06, + "logits/chosen": -2.074873924255371, + "logits/rejected": -1.7466737031936646, + "logps/chosen": -143.79574584960938, + "logps/rejected": -227.416259765625, + "loss": 0.0682026743888855, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.0686492919921875, + "rewards/margins": 4.900749206542969, + "rewards/rejected": -9.969398498535156, + "step": 351 + }, + { + "epoch": 1.135483870967742, + "grad_norm": 0.8615036010742188, + "learning_rate": 1.984351809877544e-06, + "logits/chosen": -1.5838978290557861, + "logits/rejected": -1.3324167728424072, + "logps/chosen": -170.6785888671875, + "logps/rejected": -229.92874145507812, + "loss": 0.19454488158226013, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.850698471069336, + "rewards/margins": 3.9882798194885254, + "rewards/rejected": -9.838977813720703, + "step": 352 + }, + { + "epoch": 1.1387096774193548, + "grad_norm": 0.40390846133232117, + "learning_rate": 1.9719631593959818e-06, + "logits/chosen": -2.2553353309631348, + "logits/rejected": -1.9675357341766357, + "logps/chosen": -144.84364318847656, + "logps/rejected": -208.33547973632812, + "loss": 0.09588846564292908, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.0572829246521, + "rewards/margins": 4.1294379234313965, + "rewards/rejected": -9.18671989440918, + "step": 353 + }, + { + "epoch": 1.1419354838709677, + "grad_norm": 0.7221540808677673, + "learning_rate": 1.959588066415917e-06, + "logits/chosen": -1.7495440244674683, + "logits/rejected": -1.3849916458129883, + "logps/chosen": -162.43798828125, + "logps/rejected": -229.17469787597656, + "loss": 0.1190003752708435, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.087733268737793, + "rewards/margins": 4.036655426025391, + "rewards/rejected": -9.124388694763184, + "step": 354 + }, + { + "epoch": 1.1451612903225807, + "grad_norm": 0.6402134895324707, + "learning_rate": 1.947226848671508e-06, + "logits/chosen": -2.255587339401245, + "logits/rejected": -1.9237840175628662, + "logps/chosen": -150.15924072265625, + "logps/rejected": -218.59298706054688, + "loss": 0.18148204684257507, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.075443267822266, + "rewards/margins": 3.923079252243042, + "rewards/rejected": -8.998522758483887, + "step": 355 + }, + { + "epoch": 1.1483870967741936, + "grad_norm": 0.6949121952056885, + "learning_rate": 1.934879823540663e-06, + "logits/chosen": -2.109438896179199, + "logits/rejected": -1.7586997747421265, + "logps/chosen": -160.052734375, + "logps/rejected": -225.48373413085938, + "loss": 0.1760554015636444, + "rewards/accuracies": 0.90625, + "rewards/chosen": -5.738818645477295, + "rewards/margins": 4.271374702453613, + "rewards/rejected": -10.01019287109375, + "step": 356 + }, + { + "epoch": 1.1516129032258065, + "grad_norm": 0.5934706330299377, + "learning_rate": 1.9225473080368916e-06, + "logits/chosen": -2.27186918258667, + "logits/rejected": -1.974424123764038, + "logps/chosen": -158.73870849609375, + "logps/rejected": -232.62786865234375, + "loss": 0.1641635000705719, + "rewards/accuracies": 0.90625, + "rewards/chosen": -5.279523849487305, + "rewards/margins": 4.108285903930664, + "rewards/rejected": -9.387809753417969, + "step": 357 + }, + { + "epoch": 1.1548387096774193, + "grad_norm": 0.6596993207931519, + "learning_rate": 1.9102296188011644e-06, + "logits/chosen": -2.232776165008545, + "logits/rejected": -1.857932209968567, + "logps/chosen": -143.13937377929688, + "logps/rejected": -226.6849365234375, + "loss": 0.14514769613742828, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.097674369812012, + "rewards/margins": 5.24989128112793, + "rewards/rejected": -10.347566604614258, + "step": 358 + }, + { + "epoch": 1.1580645161290322, + "grad_norm": 0.6565665006637573, + "learning_rate": 1.8979270720937814e-06, + "logits/chosen": -1.938962459564209, + "logits/rejected": -1.64393150806427, + "logps/chosen": -158.95477294921875, + "logps/rejected": -219.6165008544922, + "loss": 0.16598346829414368, + "rewards/accuracies": 0.90625, + "rewards/chosen": -5.99850606918335, + "rewards/margins": 3.776855707168579, + "rewards/rejected": -9.775362014770508, + "step": 359 + }, + { + "epoch": 1.1612903225806452, + "grad_norm": 0.5938535928726196, + "learning_rate": 1.8856399837862552e-06, + "logits/chosen": -2.0286355018615723, + "logits/rejected": -1.68312668800354, + "logps/chosen": -168.68099975585938, + "logps/rejected": -229.54739379882812, + "loss": 0.11951416730880737, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.778509140014648, + "rewards/margins": 3.883187770843506, + "rewards/rejected": -9.661696434020996, + "step": 360 + }, + { + "epoch": 1.164516129032258, + "grad_norm": 0.3561262786388397, + "learning_rate": 1.8733686693531986e-06, + "logits/chosen": -2.127842903137207, + "logits/rejected": -1.7695016860961914, + "logps/chosen": -144.55734252929688, + "logps/rejected": -229.44363403320312, + "loss": 0.07146456092596054, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.509417533874512, + "rewards/margins": 4.857423305511475, + "rewards/rejected": -10.366840362548828, + "step": 361 + }, + { + "epoch": 1.167741935483871, + "grad_norm": 0.8494337797164917, + "learning_rate": 1.8611134438642277e-06, + "logits/chosen": -1.7025121450424194, + "logits/rejected": -1.4264708757400513, + "logps/chosen": -169.04269409179688, + "logps/rejected": -221.38189697265625, + "loss": 0.2456420212984085, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.7377424240112305, + "rewards/margins": 3.4128007888793945, + "rewards/rejected": -9.150543212890625, + "step": 362 + }, + { + "epoch": 1.1709677419354838, + "grad_norm": 0.6328747272491455, + "learning_rate": 1.8488746219758674e-06, + "logits/chosen": -2.2445268630981445, + "logits/rejected": -1.9536311626434326, + "logps/chosen": -175.17105102539062, + "logps/rejected": -228.45208740234375, + "loss": 0.17112946510314941, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.968300819396973, + "rewards/margins": 3.3772494792938232, + "rewards/rejected": -9.345550537109375, + "step": 363 + }, + { + "epoch": 1.1741935483870969, + "grad_norm": 0.6168525815010071, + "learning_rate": 1.836652517923477e-06, + "logits/chosen": -2.116086959838867, + "logits/rejected": -1.8520147800445557, + "logps/chosen": -138.18699645996094, + "logps/rejected": -202.0654754638672, + "loss": 0.16664910316467285, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.481403827667236, + "rewards/margins": 3.620983600616455, + "rewards/rejected": -8.102387428283691, + "step": 364 + }, + { + "epoch": 1.1774193548387097, + "grad_norm": 0.721930205821991, + "learning_rate": 1.824447445513179e-06, + "logits/chosen": -1.7406378984451294, + "logits/rejected": -1.482226014137268, + "logps/chosen": -179.15814208984375, + "logps/rejected": -250.98904418945312, + "loss": 0.1676761507987976, + "rewards/accuracies": 0.90625, + "rewards/chosen": -6.1544904708862305, + "rewards/margins": 4.405542373657227, + "rewards/rejected": -10.56003189086914, + "step": 365 + }, + { + "epoch": 1.1806451612903226, + "grad_norm": 0.6200963854789734, + "learning_rate": 1.812259718113805e-06, + "logits/chosen": -2.065372943878174, + "logits/rejected": -1.7523372173309326, + "logps/chosen": -162.01683044433594, + "logps/rejected": -223.36329650878906, + "loss": 0.15973971784114838, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.798542499542236, + "rewards/margins": 3.4644269943237305, + "rewards/rejected": -9.262969017028809, + "step": 366 + }, + { + "epoch": 1.1838709677419355, + "grad_norm": 0.38376542925834656, + "learning_rate": 1.8000896486488478e-06, + "logits/chosen": -2.161773443222046, + "logits/rejected": -1.9010320901870728, + "logps/chosen": -153.3382110595703, + "logps/rejected": -215.8674774169922, + "loss": 0.09716545045375824, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.17132568359375, + "rewards/margins": 4.096748352050781, + "rewards/rejected": -9.268074035644531, + "step": 367 + }, + { + "epoch": 1.1870967741935483, + "grad_norm": 0.7107370495796204, + "learning_rate": 1.7879375495884285e-06, + "logits/chosen": -2.1105496883392334, + "logits/rejected": -1.827610731124878, + "logps/chosen": -152.8114471435547, + "logps/rejected": -214.80062866210938, + "loss": 0.20002783834934235, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.211606979370117, + "rewards/margins": 3.4694228172302246, + "rewards/rejected": -8.681029319763184, + "step": 368 + }, + { + "epoch": 1.1903225806451614, + "grad_norm": 0.7678977847099304, + "learning_rate": 1.77580373294127e-06, + "logits/chosen": -2.063249111175537, + "logits/rejected": -1.708777666091919, + "logps/chosen": -158.5166778564453, + "logps/rejected": -214.08209228515625, + "loss": 0.2671286463737488, + "rewards/accuracies": 0.90625, + "rewards/chosen": -5.262528419494629, + "rewards/margins": 3.365039348602295, + "rewards/rejected": -8.627568244934082, + "step": 369 + }, + { + "epoch": 1.1935483870967742, + "grad_norm": 0.3605065643787384, + "learning_rate": 1.7636885102466907e-06, + "logits/chosen": -2.5887768268585205, + "logits/rejected": -2.2808783054351807, + "logps/chosen": -130.79820251464844, + "logps/rejected": -197.219482421875, + "loss": 0.1083921268582344, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.052940845489502, + "rewards/margins": 4.416059494018555, + "rewards/rejected": -8.468999862670898, + "step": 370 + }, + { + "epoch": 1.196774193548387, + "grad_norm": 0.7315830588340759, + "learning_rate": 1.7515921925666053e-06, + "logits/chosen": -2.358499765396118, + "logits/rejected": -2.0349740982055664, + "logps/chosen": -155.2823486328125, + "logps/rejected": -230.02227783203125, + "loss": 0.14842908084392548, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.6453447341918945, + "rewards/margins": 4.513706684112549, + "rewards/rejected": -10.159051895141602, + "step": 371 + }, + { + "epoch": 1.2, + "grad_norm": 0.5185221433639526, + "learning_rate": 1.7395150904775338e-06, + "logits/chosen": -2.276341438293457, + "logits/rejected": -1.9259616136550903, + "logps/chosen": -131.30264282226562, + "logps/rejected": -202.33523559570312, + "loss": 0.15730315446853638, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.405592441558838, + "rewards/margins": 4.261539459228516, + "rewards/rejected": -8.667131423950195, + "step": 372 + }, + { + "epoch": 1.2032258064516128, + "grad_norm": 0.7151278853416443, + "learning_rate": 1.7274575140626318e-06, + "logits/chosen": -2.0519351959228516, + "logits/rejected": -1.5820385217666626, + "logps/chosen": -155.93443298339844, + "logps/rejected": -236.36830139160156, + "loss": 0.16425088047981262, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.275310516357422, + "rewards/margins": 4.754761695861816, + "rewards/rejected": -10.030072212219238, + "step": 373 + }, + { + "epoch": 1.206451612903226, + "grad_norm": 0.660406231880188, + "learning_rate": 1.7154197729037275e-06, + "logits/chosen": -2.5248794555664062, + "logits/rejected": -2.073789596557617, + "logps/chosen": -144.71421813964844, + "logps/rejected": -224.99496459960938, + "loss": 0.18090461194515228, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.965693473815918, + "rewards/margins": 5.039268970489502, + "rewards/rejected": -10.004961967468262, + "step": 374 + }, + { + "epoch": 1.2096774193548387, + "grad_norm": 0.7491795420646667, + "learning_rate": 1.7034021760733712e-06, + "logits/chosen": -2.0052719116210938, + "logits/rejected": -1.725099802017212, + "logps/chosen": -165.0349578857422, + "logps/rejected": -229.08590698242188, + "loss": 0.2221168726682663, + "rewards/accuracies": 0.90625, + "rewards/chosen": -5.368182182312012, + "rewards/margins": 3.6441545486450195, + "rewards/rejected": -9.012336730957031, + "step": 375 + }, + { + "epoch": 1.2129032258064516, + "grad_norm": 0.8390048146247864, + "learning_rate": 1.6914050321269049e-06, + "logits/chosen": -2.0131216049194336, + "logits/rejected": -1.734924077987671, + "logps/chosen": -157.00872802734375, + "logps/rejected": -220.60801696777344, + "loss": 0.21280023455619812, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.416422367095947, + "rewards/margins": 3.55110502243042, + "rewards/rejected": -8.967527389526367, + "step": 376 + }, + { + "epoch": 1.2161290322580645, + "grad_norm": 0.6836002469062805, + "learning_rate": 1.6794286490945344e-06, + "logits/chosen": -2.0585014820098877, + "logits/rejected": -1.724048137664795, + "logps/chosen": -152.38882446289062, + "logps/rejected": -214.6417236328125, + "loss": 0.17855605483055115, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.036785125732422, + "rewards/margins": 3.633176565170288, + "rewards/rejected": -8.669961929321289, + "step": 377 + }, + { + "epoch": 1.2193548387096773, + "grad_norm": 0.46255868673324585, + "learning_rate": 1.667473334473424e-06, + "logits/chosen": -1.8597807884216309, + "logits/rejected": -1.426900863647461, + "logps/chosen": -182.52023315429688, + "logps/rejected": -284.5487365722656, + "loss": 0.07632362842559814, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.2685160636901855, + "rewards/margins": 6.5974955558776855, + "rewards/rejected": -12.866011619567871, + "step": 378 + }, + { + "epoch": 1.2225806451612904, + "grad_norm": 0.43195584416389465, + "learning_rate": 1.655539395219799e-06, + "logits/chosen": -2.1914172172546387, + "logits/rejected": -1.8688502311706543, + "logps/chosen": -156.40907287597656, + "logps/rejected": -236.13815307617188, + "loss": 0.0872289389371872, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.669783592224121, + "rewards/margins": 4.961936950683594, + "rewards/rejected": -10.631721496582031, + "step": 379 + }, + { + "epoch": 1.2258064516129032, + "grad_norm": 0.7333582639694214, + "learning_rate": 1.6436271377410667e-06, + "logits/chosen": -2.359461545944214, + "logits/rejected": -2.069000005722046, + "logps/chosen": -133.92742919921875, + "logps/rejected": -201.80978393554688, + "loss": 0.20327237248420715, + "rewards/accuracies": 0.90625, + "rewards/chosen": -4.1502580642700195, + "rewards/margins": 3.974135637283325, + "rewards/rejected": -8.124393463134766, + "step": 380 + }, + { + "epoch": 1.229032258064516, + "grad_norm": 0.42488953471183777, + "learning_rate": 1.6317368678879497e-06, + "logits/chosen": -2.406670093536377, + "logits/rejected": -2.07307505607605, + "logps/chosen": -133.93527221679688, + "logps/rejected": -200.25289916992188, + "loss": 0.12039235234260559, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.170150279998779, + "rewards/margins": 3.830730438232422, + "rewards/rejected": -8.000880241394043, + "step": 381 + }, + { + "epoch": 1.232258064516129, + "grad_norm": 0.4017215669155121, + "learning_rate": 1.6198688909466292e-06, + "logits/chosen": -2.832380533218384, + "logits/rejected": -2.3897719383239746, + "logps/chosen": -130.95968627929688, + "logps/rejected": -199.89097595214844, + "loss": 0.10956410318613052, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.363296985626221, + "rewards/margins": 4.319589614868164, + "rewards/rejected": -8.682886123657227, + "step": 382 + }, + { + "epoch": 1.2354838709677418, + "grad_norm": 0.6206420063972473, + "learning_rate": 1.6080235116309114e-06, + "logits/chosen": -2.4406814575195312, + "logits/rejected": -2.160674810409546, + "logps/chosen": -142.58071899414062, + "logps/rejected": -200.98794555664062, + "loss": 0.1694997251033783, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.247890472412109, + "rewards/margins": 3.134093761444092, + "rewards/rejected": -8.38198471069336, + "step": 383 + }, + { + "epoch": 1.238709677419355, + "grad_norm": 0.2462724894285202, + "learning_rate": 1.5962010340744006e-06, + "logits/chosen": -2.235247850418091, + "logits/rejected": -1.847717046737671, + "logps/chosen": -146.453125, + "logps/rejected": -225.43215942382812, + "loss": 0.03917820006608963, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.979830265045166, + "rewards/margins": 4.634283065795898, + "rewards/rejected": -9.614113807678223, + "step": 384 + }, + { + "epoch": 1.2419354838709677, + "grad_norm": 0.5250591039657593, + "learning_rate": 1.5844017618226934e-06, + "logits/chosen": -2.19504451751709, + "logits/rejected": -1.866036057472229, + "logps/chosen": -151.13436889648438, + "logps/rejected": -220.41619873046875, + "loss": 0.13191558420658112, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.161625862121582, + "rewards/margins": 4.320361137390137, + "rewards/rejected": -9.481987953186035, + "step": 385 + }, + { + "epoch": 1.2451612903225806, + "grad_norm": 0.448537677526474, + "learning_rate": 1.572625997825581e-06, + "logits/chosen": -2.195721387863159, + "logits/rejected": -1.8460835218429565, + "logps/chosen": -153.8927001953125, + "logps/rejected": -227.37582397460938, + "loss": 0.11286298930644989, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.872514247894287, + "rewards/margins": 4.24212646484375, + "rewards/rejected": -9.114640235900879, + "step": 386 + }, + { + "epoch": 1.2483870967741935, + "grad_norm": 0.6990474462509155, + "learning_rate": 1.5608740444292746e-06, + "logits/chosen": -2.3243963718414307, + "logits/rejected": -2.0378341674804688, + "logps/chosen": -149.29092407226562, + "logps/rejected": -207.21595764160156, + "loss": 0.18665999174118042, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.657018184661865, + "rewards/margins": 3.210756301879883, + "rewards/rejected": -7.867774486541748, + "step": 387 + }, + { + "epoch": 1.2516129032258063, + "grad_norm": 0.8297081589698792, + "learning_rate": 1.5491462033686411e-06, + "logits/chosen": -2.18031907081604, + "logits/rejected": -1.826414942741394, + "logps/chosen": -158.47360229492188, + "logps/rejected": -234.47384643554688, + "loss": 0.16636508703231812, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.654425621032715, + "rewards/margins": 4.482308387756348, + "rewards/rejected": -10.136734008789062, + "step": 388 + }, + { + "epoch": 1.2548387096774194, + "grad_norm": 1.3903393745422363, + "learning_rate": 1.5374427757594552e-06, + "logits/chosen": -2.297201156616211, + "logits/rejected": -2.06970477104187, + "logps/chosen": -178.5050048828125, + "logps/rejected": -219.87823486328125, + "loss": 0.39618611335754395, + "rewards/accuracies": 0.90625, + "rewards/chosen": -6.283227920532227, + "rewards/margins": 2.801511764526367, + "rewards/rejected": -9.084739685058594, + "step": 389 + }, + { + "epoch": 1.2580645161290323, + "grad_norm": 0.4080447256565094, + "learning_rate": 1.525764062090671e-06, + "logits/chosen": -1.8646701574325562, + "logits/rejected": -1.555259108543396, + "logps/chosen": -169.2515869140625, + "logps/rejected": -238.75172424316406, + "loss": 0.08469586074352264, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.7809553146362305, + "rewards/margins": 3.9082202911376953, + "rewards/rejected": -9.689175605773926, + "step": 390 + }, + { + "epoch": 1.261290322580645, + "grad_norm": 0.8623878359794617, + "learning_rate": 1.5141103622167042e-06, + "logits/chosen": -2.4216866493225098, + "logits/rejected": -2.0609347820281982, + "logps/chosen": -148.01690673828125, + "logps/rejected": -201.63174438476562, + "loss": 0.242265522480011, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.647133827209473, + "rewards/margins": 3.4030661582946777, + "rewards/rejected": -8.050199508666992, + "step": 391 + }, + { + "epoch": 1.2645161290322582, + "grad_norm": 0.4071905314922333, + "learning_rate": 1.5024819753497316e-06, + "logits/chosen": -2.114326238632202, + "logits/rejected": -1.8489298820495605, + "logps/chosen": -154.75469970703125, + "logps/rejected": -223.68875122070312, + "loss": 0.10187827050685883, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.854604721069336, + "rewards/margins": 4.386443138122559, + "rewards/rejected": -9.241047859191895, + "step": 392 + }, + { + "epoch": 1.267741935483871, + "grad_norm": 0.5191037058830261, + "learning_rate": 1.490879200052014e-06, + "logits/chosen": -2.666534662246704, + "logits/rejected": -2.3881566524505615, + "logps/chosen": -153.2815399169922, + "logps/rejected": -220.57666015625, + "loss": 0.13561299443244934, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.065132141113281, + "rewards/margins": 3.8070566654205322, + "rewards/rejected": -9.872188568115234, + "step": 393 + }, + { + "epoch": 1.270967741935484, + "grad_norm": 0.5940090417861938, + "learning_rate": 1.4793023342282258e-06, + "logits/chosen": -2.3885128498077393, + "logits/rejected": -2.043498992919922, + "logps/chosen": -142.4330596923828, + "logps/rejected": -230.61424255371094, + "loss": 0.1435917317867279, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.768260955810547, + "rewards/margins": 4.874237537384033, + "rewards/rejected": -9.642498970031738, + "step": 394 + }, + { + "epoch": 1.2741935483870968, + "grad_norm": 0.5542668700218201, + "learning_rate": 1.46775167511781e-06, + "logits/chosen": -2.55564284324646, + "logits/rejected": -1.9429116249084473, + "logps/chosen": -122.74253845214844, + "logps/rejected": -221.68040466308594, + "loss": 0.14152149856090546, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.333782196044922, + "rewards/margins": 6.17290735244751, + "rewards/rejected": -10.506689071655273, + "step": 395 + }, + { + "epoch": 1.2774193548387096, + "grad_norm": 0.598746657371521, + "learning_rate": 1.456227519287343e-06, + "logits/chosen": -2.619776725769043, + "logits/rejected": -2.364832878112793, + "logps/chosen": -162.55870056152344, + "logps/rejected": -218.89852905273438, + "loss": 0.14930739998817444, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.192800521850586, + "rewards/margins": 3.228623867034912, + "rewards/rejected": -8.421424865722656, + "step": 396 + }, + { + "epoch": 1.2806451612903227, + "grad_norm": 0.8427586555480957, + "learning_rate": 1.4447301626229204e-06, + "logits/chosen": -2.049372434616089, + "logits/rejected": -1.662196159362793, + "logps/chosen": -153.92330932617188, + "logps/rejected": -217.40673828125, + "loss": 0.1911974847316742, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.926740646362305, + "rewards/margins": 3.7954201698303223, + "rewards/rejected": -8.722161293029785, + "step": 397 + }, + { + "epoch": 1.2838709677419355, + "grad_norm": 0.5858323574066162, + "learning_rate": 1.4332599003225638e-06, + "logits/chosen": -1.8799211978912354, + "logits/rejected": -1.451097846031189, + "logps/chosen": -149.65142822265625, + "logps/rejected": -241.16909790039062, + "loss": 0.11262927204370499, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.039501667022705, + "rewards/margins": 5.284974575042725, + "rewards/rejected": -10.32447624206543, + "step": 398 + }, + { + "epoch": 1.2870967741935484, + "grad_norm": 0.4971863329410553, + "learning_rate": 1.42181702688864e-06, + "logits/chosen": -2.286614179611206, + "logits/rejected": -1.8966314792633057, + "logps/chosen": -153.2661590576172, + "logps/rejected": -220.55250549316406, + "loss": 0.1035381481051445, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.003832817077637, + "rewards/margins": 4.208715438842773, + "rewards/rejected": -9.21254825592041, + "step": 399 + }, + { + "epoch": 1.2903225806451613, + "grad_norm": 0.3167995810508728, + "learning_rate": 1.4104018361202947e-06, + "logits/chosen": -2.4894747734069824, + "logits/rejected": -2.0620884895324707, + "logps/chosen": -146.4293975830078, + "logps/rejected": -245.97506713867188, + "loss": 0.075407974421978, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.003347396850586, + "rewards/margins": 5.8961663246154785, + "rewards/rejected": -10.899513244628906, + "step": 400 + }, + { + "epoch": 1.293548387096774, + "grad_norm": 1.0359872579574585, + "learning_rate": 1.3990146211059141e-06, + "logits/chosen": -2.4616971015930176, + "logits/rejected": -2.154454231262207, + "logps/chosen": -159.11233520507812, + "logps/rejected": -222.264892578125, + "loss": 0.26626330614089966, + "rewards/accuracies": 0.84375, + "rewards/chosen": -5.6828179359436035, + "rewards/margins": 4.052268981933594, + "rewards/rejected": -9.735086441040039, + "step": 401 + }, + { + "epoch": 1.2967741935483872, + "grad_norm": 1.037102460861206, + "learning_rate": 1.3876556742156019e-06, + "logits/chosen": -2.124492883682251, + "logits/rejected": -1.8085477352142334, + "logps/chosen": -170.0220489501953, + "logps/rejected": -233.81515502929688, + "loss": 0.25808218121528625, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.954598426818848, + "rewards/margins": 3.8529767990112305, + "rewards/rejected": -9.807575225830078, + "step": 402 + }, + { + "epoch": 1.3, + "grad_norm": 1.154233455657959, + "learning_rate": 1.376325287093665e-06, + "logits/chosen": -2.3976852893829346, + "logits/rejected": -2.059356212615967, + "logps/chosen": -174.79681396484375, + "logps/rejected": -234.56651306152344, + "loss": 0.3070518374443054, + "rewards/accuracies": 0.90625, + "rewards/chosen": -5.778817176818848, + "rewards/margins": 3.863279342651367, + "rewards/rejected": -9.642096519470215, + "step": 403 + }, + { + "epoch": 1.303225806451613, + "grad_norm": 0.7991430163383484, + "learning_rate": 1.3650237506511333e-06, + "logits/chosen": -2.269876003265381, + "logits/rejected": -1.9741662740707397, + "logps/chosen": -154.88619995117188, + "logps/rejected": -223.62730407714844, + "loss": 0.21480777859687805, + "rewards/accuracies": 0.90625, + "rewards/chosen": -4.390778541564941, + "rewards/margins": 4.292700290679932, + "rewards/rejected": -8.683478355407715, + "step": 404 + }, + { + "epoch": 1.3064516129032258, + "grad_norm": 0.5895324349403381, + "learning_rate": 1.3537513550582853e-06, + "logits/chosen": -2.2187764644622803, + "logits/rejected": -1.8166173696517944, + "logps/chosen": -148.30221557617188, + "logps/rejected": -213.80755615234375, + "loss": 0.13769668340682983, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.786943435668945, + "rewards/margins": 4.149323463439941, + "rewards/rejected": -8.93626594543457, + "step": 405 + }, + { + "epoch": 1.3096774193548386, + "grad_norm": 1.0179868936538696, + "learning_rate": 1.3425083897371983e-06, + "logits/chosen": -2.0878705978393555, + "logits/rejected": -1.8105435371398926, + "logps/chosen": -149.2059326171875, + "logps/rejected": -210.22061157226562, + "loss": 0.25650089979171753, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.216985702514648, + "rewards/margins": 3.597675323486328, + "rewards/rejected": -8.814661026000977, + "step": 406 + }, + { + "epoch": 1.3129032258064517, + "grad_norm": 0.7849494218826294, + "learning_rate": 1.3312951433543225e-06, + "logits/chosen": -2.366304874420166, + "logits/rejected": -1.9977753162384033, + "logps/chosen": -147.30950927734375, + "logps/rejected": -221.3260498046875, + "loss": 0.1955363154411316, + "rewards/accuracies": 0.90625, + "rewards/chosen": -4.945062160491943, + "rewards/margins": 4.327850818634033, + "rewards/rejected": -9.27291202545166, + "step": 407 + }, + { + "epoch": 1.3161290322580645, + "grad_norm": 0.4468243420124054, + "learning_rate": 1.3201119038130616e-06, + "logits/chosen": -2.472445487976074, + "logits/rejected": -1.9949541091918945, + "logps/chosen": -135.80923461914062, + "logps/rejected": -214.80462646484375, + "loss": 0.08190619945526123, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.554195404052734, + "rewards/margins": 4.7136735916137695, + "rewards/rejected": -9.267868041992188, + "step": 408 + }, + { + "epoch": 1.3193548387096774, + "grad_norm": 0.4178885817527771, + "learning_rate": 1.308958958246388e-06, + "logits/chosen": -2.448024272918701, + "logits/rejected": -2.0670201778411865, + "logps/chosen": -140.41001892089844, + "logps/rejected": -214.8358917236328, + "loss": 0.10611550509929657, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.203770160675049, + "rewards/margins": 4.342594146728516, + "rewards/rejected": -9.546364784240723, + "step": 409 + }, + { + "epoch": 1.3225806451612903, + "grad_norm": 0.6719059944152832, + "learning_rate": 1.2978365930094645e-06, + "logits/chosen": -2.3373985290527344, + "logits/rejected": -2.0481557846069336, + "logps/chosen": -158.54962158203125, + "logps/rejected": -216.07199096679688, + "loss": 0.21998706459999084, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.216798782348633, + "rewards/margins": 3.790421962738037, + "rewards/rejected": -9.007221221923828, + "step": 410 + }, + { + "epoch": 1.3258064516129031, + "grad_norm": 0.5259553790092468, + "learning_rate": 1.286745093672298e-06, + "logits/chosen": -1.8555116653442383, + "logits/rejected": -1.5601491928100586, + "logps/chosen": -173.27001953125, + "logps/rejected": -232.70184326171875, + "loss": 0.11464345455169678, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.655244827270508, + "rewards/margins": 3.616917371749878, + "rewards/rejected": -9.272162437438965, + "step": 411 + }, + { + "epoch": 1.3290322580645162, + "grad_norm": 1.0269285440444946, + "learning_rate": 1.2756847450124005e-06, + "logits/chosen": -2.258531093597412, + "logits/rejected": -1.9175965785980225, + "logps/chosen": -150.10646057128906, + "logps/rejected": -217.81976318359375, + "loss": 0.2900359034538269, + "rewards/accuracies": 0.90625, + "rewards/chosen": -4.862858772277832, + "rewards/margins": 3.9900403022766113, + "rewards/rejected": -8.852899551391602, + "step": 412 + }, + { + "epoch": 1.332258064516129, + "grad_norm": 0.6336823105812073, + "learning_rate": 1.264655831007486e-06, + "logits/chosen": -2.4869401454925537, + "logits/rejected": -2.178582191467285, + "logps/chosen": -146.45761108398438, + "logps/rejected": -209.02230834960938, + "loss": 0.13850894570350647, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.8014020919799805, + "rewards/margins": 3.8616671562194824, + "rewards/rejected": -8.663068771362305, + "step": 413 + }, + { + "epoch": 1.335483870967742, + "grad_norm": 0.7770991921424866, + "learning_rate": 1.2536586348281694e-06, + "logits/chosen": -2.089435338973999, + "logits/rejected": -1.866056203842163, + "logps/chosen": -178.45458984375, + "logps/rejected": -243.360107421875, + "loss": 0.2114502340555191, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.556073188781738, + "rewards/margins": 3.712599515914917, + "rewards/rejected": -10.268672943115234, + "step": 414 + }, + { + "epoch": 1.3387096774193548, + "grad_norm": 0.5494731068611145, + "learning_rate": 1.2426934388307059e-06, + "logits/chosen": -2.485060214996338, + "logits/rejected": -2.085818290710449, + "logps/chosen": -141.69973754882812, + "logps/rejected": -208.56036376953125, + "loss": 0.16660693287849426, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.942885875701904, + "rewards/margins": 3.965348958969116, + "rewards/rejected": -8.908235549926758, + "step": 415 + }, + { + "epoch": 1.3419354838709676, + "grad_norm": 0.3686054050922394, + "learning_rate": 1.2317605245497324e-06, + "logits/chosen": -2.1087584495544434, + "logits/rejected": -1.6341150999069214, + "logps/chosen": -152.7848663330078, + "logps/rejected": -238.4124755859375, + "loss": 0.07035782933235168, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.851728439331055, + "rewards/margins": 5.564634799957275, + "rewards/rejected": -10.416362762451172, + "step": 416 + }, + { + "epoch": 1.3451612903225807, + "grad_norm": 0.5881558656692505, + "learning_rate": 1.2208601726910446e-06, + "logits/chosen": -2.7871689796447754, + "logits/rejected": -2.365675449371338, + "logps/chosen": -123.9314956665039, + "logps/rejected": -190.04649353027344, + "loss": 0.2168859839439392, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.762516498565674, + "rewards/margins": 4.015682697296143, + "rewards/rejected": -7.778199672698975, + "step": 417 + }, + { + "epoch": 1.3483870967741935, + "grad_norm": 0.9616515636444092, + "learning_rate": 1.209992663124391e-06, + "logits/chosen": -2.182853937149048, + "logits/rejected": -1.8745319843292236, + "logps/chosen": -161.31716918945312, + "logps/rejected": -216.4304962158203, + "loss": 0.21536722779273987, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.1654744148254395, + "rewards/margins": 3.8585994243621826, + "rewards/rejected": -9.02407455444336, + "step": 418 + }, + { + "epoch": 1.3516129032258064, + "grad_norm": 0.4572608470916748, + "learning_rate": 1.19915827487628e-06, + "logits/chosen": -2.0564944744110107, + "logits/rejected": -1.74418306350708, + "logps/chosen": -140.11302185058594, + "logps/rejected": -209.9811553955078, + "loss": 0.1145668774843216, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.56220817565918, + "rewards/margins": 4.219762325286865, + "rewards/rejected": -8.781970977783203, + "step": 419 + }, + { + "epoch": 1.3548387096774195, + "grad_norm": 0.8865814208984375, + "learning_rate": 1.1883572861228255e-06, + "logits/chosen": -2.37459659576416, + "logits/rejected": -2.109685182571411, + "logps/chosen": -145.40548706054688, + "logps/rejected": -212.8759765625, + "loss": 0.24796389043331146, + "rewards/accuracies": 0.90625, + "rewards/chosen": -4.966710090637207, + "rewards/margins": 3.646275281906128, + "rewards/rejected": -8.612985610961914, + "step": 420 + }, + { + "epoch": 1.3580645161290323, + "grad_norm": 0.6821106672286987, + "learning_rate": 1.1775899741825947e-06, + "logits/chosen": -2.328369617462158, + "logits/rejected": -2.013854503631592, + "logps/chosen": -141.02191162109375, + "logps/rejected": -205.2490234375, + "loss": 0.20715095102787018, + "rewards/accuracies": 0.90625, + "rewards/chosen": -4.649036407470703, + "rewards/margins": 4.007871627807617, + "rewards/rejected": -8.65690803527832, + "step": 421 + }, + { + "epoch": 1.3612903225806452, + "grad_norm": 0.5044422745704651, + "learning_rate": 1.1668566155094948e-06, + "logits/chosen": -2.54693865776062, + "logits/rejected": -1.9353411197662354, + "logps/chosen": -131.88148498535156, + "logps/rejected": -202.45828247070312, + "loss": 0.0967191830277443, + "rewards/accuracies": 0.96875, + "rewards/chosen": -3.5916037559509277, + "rewards/margins": 4.169280529022217, + "rewards/rejected": -7.7608842849731445, + "step": 422 + }, + { + "epoch": 1.364516129032258, + "grad_norm": 0.8407202959060669, + "learning_rate": 1.1561574856856737e-06, + "logits/chosen": -2.190084934234619, + "logits/rejected": -1.87167489528656, + "logps/chosen": -153.39193725585938, + "logps/rejected": -216.16046142578125, + "loss": 0.21430709958076477, + "rewards/accuracies": 0.90625, + "rewards/chosen": -4.597943305969238, + "rewards/margins": 3.650974988937378, + "rewards/rejected": -8.248918533325195, + "step": 423 + }, + { + "epoch": 1.367741935483871, + "grad_norm": 0.866381049156189, + "learning_rate": 1.1454928594144432e-06, + "logits/chosen": -2.4623706340789795, + "logits/rejected": -2.085210084915161, + "logps/chosen": -142.26776123046875, + "logps/rejected": -206.5341796875, + "loss": 0.2235676646232605, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.408275604248047, + "rewards/margins": 3.4640631675720215, + "rewards/rejected": -7.87233829498291, + "step": 424 + }, + { + "epoch": 1.370967741935484, + "grad_norm": 0.630285382270813, + "learning_rate": 1.1348630105132253e-06, + "logits/chosen": -2.5449862480163574, + "logits/rejected": -2.1377205848693848, + "logps/chosen": -150.03143310546875, + "logps/rejected": -216.3634490966797, + "loss": 0.1601426601409912, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.416558265686035, + "rewards/margins": 4.073892593383789, + "rewards/rejected": -8.490450859069824, + "step": 425 + }, + { + "epoch": 1.3741935483870968, + "grad_norm": 0.4551900029182434, + "learning_rate": 1.1242682119065217e-06, + "logits/chosen": -2.3819632530212402, + "logits/rejected": -2.057983875274658, + "logps/chosen": -144.26976013183594, + "logps/rejected": -227.2911376953125, + "loss": 0.10965423285961151, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.817654132843018, + "rewards/margins": 4.581267356872559, + "rewards/rejected": -9.398920059204102, + "step": 426 + }, + { + "epoch": 1.3774193548387097, + "grad_norm": 0.7748322486877441, + "learning_rate": 1.1137087356189105e-06, + "logits/chosen": -2.4593541622161865, + "logits/rejected": -2.1655220985412598, + "logps/chosen": -147.06605529785156, + "logps/rejected": -211.48818969726562, + "loss": 0.20653697848320007, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.7266435623168945, + "rewards/margins": 3.2662453651428223, + "rewards/rejected": -7.992889404296875, + "step": 427 + }, + { + "epoch": 1.3806451612903226, + "grad_norm": 0.5784196257591248, + "learning_rate": 1.1031848527680577e-06, + "logits/chosen": -2.141270875930786, + "logits/rejected": -1.8732938766479492, + "logps/chosen": -167.12008666992188, + "logps/rejected": -239.15773010253906, + "loss": 0.1471468210220337, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.631338119506836, + "rewards/margins": 4.0043134689331055, + "rewards/rejected": -9.635651588439941, + "step": 428 + }, + { + "epoch": 1.3838709677419354, + "grad_norm": 0.8778082728385925, + "learning_rate": 1.0926968335577566e-06, + "logits/chosen": -2.401782989501953, + "logits/rejected": -2.091179609298706, + "logps/chosen": -135.68911743164062, + "logps/rejected": -196.52935791015625, + "loss": 0.24231696128845215, + "rewards/accuracies": 0.90625, + "rewards/chosen": -3.9860541820526123, + "rewards/margins": 3.5753464698791504, + "rewards/rejected": -7.561400413513184, + "step": 429 + }, + { + "epoch": 1.3870967741935485, + "grad_norm": 1.0340511798858643, + "learning_rate": 1.0822449472709907e-06, + "logits/chosen": -2.586111545562744, + "logits/rejected": -2.177614688873291, + "logps/chosen": -146.74525451660156, + "logps/rejected": -209.59957885742188, + "loss": 0.22286778688430786, + "rewards/accuracies": 0.90625, + "rewards/chosen": -4.652778148651123, + "rewards/margins": 3.4029431343078613, + "rewards/rejected": -8.055720329284668, + "step": 430 + }, + { + "epoch": 1.3903225806451613, + "grad_norm": 0.5904428362846375, + "learning_rate": 1.0718294622630188e-06, + "logits/chosen": -2.47729229927063, + "logits/rejected": -2.0854480266571045, + "logps/chosen": -152.6089630126953, + "logps/rejected": -223.96041870117188, + "loss": 0.11524567008018494, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.076026916503906, + "rewards/margins": 3.904932975769043, + "rewards/rejected": -8.98095989227295, + "step": 431 + }, + { + "epoch": 1.3935483870967742, + "grad_norm": 0.539168119430542, + "learning_rate": 1.0614506459544921e-06, + "logits/chosen": -2.2595126628875732, + "logits/rejected": -1.925595998764038, + "logps/chosen": -162.54318237304688, + "logps/rejected": -223.33636474609375, + "loss": 0.10885189473628998, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.511690139770508, + "rewards/margins": 3.9950408935546875, + "rewards/rejected": -9.506731033325195, + "step": 432 + }, + { + "epoch": 1.396774193548387, + "grad_norm": 0.5645772814750671, + "learning_rate": 1.0511087648245758e-06, + "logits/chosen": -2.578536033630371, + "logits/rejected": -2.2838306427001953, + "logps/chosen": -132.77256774902344, + "logps/rejected": -199.54373168945312, + "loss": 0.12419108301401138, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.038809776306152, + "rewards/margins": 3.914381980895996, + "rewards/rejected": -7.953191757202148, + "step": 433 + }, + { + "epoch": 1.4, + "grad_norm": 0.5010483860969543, + "learning_rate": 1.0408040844041157e-06, + "logits/chosen": -1.9756072759628296, + "logits/rejected": -1.714409351348877, + "logps/chosen": -170.2490692138672, + "logps/rejected": -227.61837768554688, + "loss": 0.13830138742923737, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.366008758544922, + "rewards/margins": 3.824510097503662, + "rewards/rejected": -9.190519332885742, + "step": 434 + }, + { + "epoch": 1.403225806451613, + "grad_norm": 0.5239137411117554, + "learning_rate": 1.0305368692688175e-06, + "logits/chosen": -2.5020527839660645, + "logits/rejected": -2.03378963470459, + "logps/chosen": -144.1491241455078, + "logps/rejected": -233.28515625, + "loss": 0.0852818563580513, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.3349151611328125, + "rewards/margins": 5.156275272369385, + "rewards/rejected": -10.491190910339355, + "step": 435 + }, + { + "epoch": 1.4064516129032258, + "grad_norm": 0.7590228915214539, + "learning_rate": 1.0203073830324566e-06, + "logits/chosen": -2.05299973487854, + "logits/rejected": -1.7430031299591064, + "logps/chosen": -156.0850830078125, + "logps/rejected": -216.1945037841797, + "loss": 0.246930330991745, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.060250759124756, + "rewards/margins": 3.4490652084350586, + "rewards/rejected": -8.509315490722656, + "step": 436 + }, + { + "epoch": 1.4096774193548387, + "grad_norm": 0.4939168691635132, + "learning_rate": 1.0101158883401078e-06, + "logits/chosen": -2.5271737575531006, + "logits/rejected": -2.085583209991455, + "logps/chosen": -136.91073608398438, + "logps/rejected": -218.0567626953125, + "loss": 0.10788646340370178, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.469447612762451, + "rewards/margins": 4.4227495193481445, + "rewards/rejected": -8.892196655273438, + "step": 437 + }, + { + "epoch": 1.4129032258064516, + "grad_norm": 0.5392411351203918, + "learning_rate": 9.999626468613997e-07, + "logits/chosen": -2.2997260093688965, + "logits/rejected": -1.9779574871063232, + "logps/chosen": -175.44947814941406, + "logps/rejected": -253.67218017578125, + "loss": 0.11333902180194855, + "rewards/accuracies": 0.96875, + "rewards/chosen": -6.0964202880859375, + "rewards/margins": 4.6486334800720215, + "rewards/rejected": -10.745054244995117, + "step": 438 + }, + { + "epoch": 1.4161290322580644, + "grad_norm": 1.1507418155670166, + "learning_rate": 9.898479192837985e-07, + "logits/chosen": -2.0347580909729004, + "logits/rejected": -1.662597417831421, + "logps/chosen": -159.89138793945312, + "logps/rejected": -218.62335205078125, + "loss": 0.298279732465744, + "rewards/accuracies": 0.84375, + "rewards/chosen": -4.849677085876465, + "rewards/margins": 3.3393747806549072, + "rewards/rejected": -8.189051628112793, + "step": 439 + }, + { + "epoch": 1.4193548387096775, + "grad_norm": 0.6101773381233215, + "learning_rate": 9.797719653059176e-07, + "logits/chosen": -2.025186538696289, + "logits/rejected": -1.7302336692810059, + "logps/chosen": -167.60308837890625, + "logps/rejected": -238.1725616455078, + "loss": 0.12761417031288147, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.349034309387207, + "rewards/margins": 4.279293537139893, + "rewards/rejected": -9.628328323364258, + "step": 440 + }, + { + "epoch": 1.4225806451612903, + "grad_norm": 0.3005771338939667, + "learning_rate": 9.697350436308428e-07, + "logits/chosen": -2.4265613555908203, + "logits/rejected": -2.0480599403381348, + "logps/chosen": -156.69139099121094, + "logps/rejected": -237.74905395507812, + "loss": 0.05653847008943558, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.761828422546387, + "rewards/margins": 4.787032127380371, + "rewards/rejected": -9.548859596252441, + "step": 441 + }, + { + "epoch": 1.4258064516129032, + "grad_norm": 0.8191932439804077, + "learning_rate": 9.597374119594981e-07, + "logits/chosen": -2.304115056991577, + "logits/rejected": -2.032526969909668, + "logps/chosen": -156.66375732421875, + "logps/rejected": -208.8516082763672, + "loss": 0.20551738142967224, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.349687576293945, + "rewards/margins": 3.4436111450195312, + "rewards/rejected": -8.793298721313477, + "step": 442 + }, + { + "epoch": 1.429032258064516, + "grad_norm": 0.8888070583343506, + "learning_rate": 9.497793269840211e-07, + "logits/chosen": -2.4185256958007812, + "logits/rejected": -2.081573009490967, + "logps/chosen": -145.60757446289062, + "logps/rejected": -212.35897827148438, + "loss": 0.2327883243560791, + "rewards/accuracies": 0.90625, + "rewards/chosen": -5.171741485595703, + "rewards/margins": 3.747623920440674, + "rewards/rejected": -8.919364929199219, + "step": 443 + }, + { + "epoch": 1.432258064516129, + "grad_norm": 0.4799133837223053, + "learning_rate": 9.398610443811798e-07, + "logits/chosen": -1.9818705320358276, + "logits/rejected": -1.634839653968811, + "logps/chosen": -174.2476348876953, + "logps/rejected": -235.02545166015625, + "loss": 0.09316743910312653, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.465350151062012, + "rewards/margins": 3.838430166244507, + "rewards/rejected": -9.303780555725098, + "step": 444 + }, + { + "epoch": 1.435483870967742, + "grad_norm": 0.9450538754463196, + "learning_rate": 9.299828188058013e-07, + "logits/chosen": -2.247403144836426, + "logits/rejected": -1.9464889764785767, + "logps/chosen": -170.24978637695312, + "logps/rejected": -219.93753051757812, + "loss": 0.2306218445301056, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.57850456237793, + "rewards/margins": 2.9838309288024902, + "rewards/rejected": -8.562335968017578, + "step": 445 + }, + { + "epoch": 1.4387096774193548, + "grad_norm": 0.5895780920982361, + "learning_rate": 9.201449038842403e-07, + "logits/chosen": -2.0624327659606934, + "logits/rejected": -1.702972650527954, + "logps/chosen": -159.513671875, + "logps/rejected": -227.80357360839844, + "loss": 0.14580538868904114, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.874773979187012, + "rewards/margins": 3.9577934741973877, + "rewards/rejected": -8.83256721496582, + "step": 446 + }, + { + "epoch": 1.4419354838709677, + "grad_norm": 0.8507987856864929, + "learning_rate": 9.103475522078606e-07, + "logits/chosen": -2.2239818572998047, + "logits/rejected": -1.928896188735962, + "logps/chosen": -169.17330932617188, + "logps/rejected": -226.74343872070312, + "loss": 0.20541514456272125, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.307161808013916, + "rewards/margins": 3.5340442657470703, + "rewards/rejected": -8.841205596923828, + "step": 447 + }, + { + "epoch": 1.4451612903225808, + "grad_norm": 0.6203137636184692, + "learning_rate": 9.005910153265531e-07, + "logits/chosen": -2.4608101844787598, + "logits/rejected": -2.152571439743042, + "logps/chosen": -141.15274047851562, + "logps/rejected": -203.83888244628906, + "loss": 0.1468571275472641, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.174438953399658, + "rewards/margins": 3.7283501625061035, + "rewards/rejected": -7.9027886390686035, + "step": 448 + }, + { + "epoch": 1.4483870967741934, + "grad_norm": 0.6260938048362732, + "learning_rate": 8.908755437422792e-07, + "logits/chosen": -2.321335792541504, + "logits/rejected": -2.009199380874634, + "logps/chosen": -140.9883270263672, + "logps/rejected": -197.89700317382812, + "loss": 0.15317237377166748, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.808759927749634, + "rewards/margins": 3.3746590614318848, + "rewards/rejected": -7.183419704437256, + "step": 449 + }, + { + "epoch": 1.4516129032258065, + "grad_norm": 0.4619046151638031, + "learning_rate": 8.812013869026334e-07, + "logits/chosen": -2.4269673824310303, + "logits/rejected": -2.013381004333496, + "logps/chosen": -139.54246520996094, + "logps/rejected": -216.44473266601562, + "loss": 0.1075877994298935, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.083252906799316, + "rewards/margins": 4.468310832977295, + "rewards/rejected": -9.55156421661377, + "step": 450 + }, + { + "epoch": 1.4548387096774194, + "grad_norm": 0.47495365142822266, + "learning_rate": 8.71568793194445e-07, + "logits/chosen": -2.4188807010650635, + "logits/rejected": -2.1440956592559814, + "logps/chosen": -141.9351806640625, + "logps/rejected": -200.09054565429688, + "loss": 0.12590685486793518, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.05722713470459, + "rewards/margins": 3.3638811111450195, + "rewards/rejected": -8.42110824584961, + "step": 451 + }, + { + "epoch": 1.4580645161290322, + "grad_norm": 0.8909400701522827, + "learning_rate": 8.619780099373946e-07, + "logits/chosen": -2.4395079612731934, + "logits/rejected": -2.122635841369629, + "logps/chosen": -136.7159423828125, + "logps/rejected": -203.58587646484375, + "loss": 0.23426640033721924, + "rewards/accuracies": 0.90625, + "rewards/chosen": -4.406562805175781, + "rewards/margins": 3.873863935470581, + "rewards/rejected": -8.280426979064941, + "step": 452 + }, + { + "epoch": 1.4612903225806453, + "grad_norm": 0.47088634967803955, + "learning_rate": 8.524292833776707e-07, + "logits/chosen": -2.372608184814453, + "logits/rejected": -1.9174907207489014, + "logps/chosen": -133.89572143554688, + "logps/rejected": -198.1195831298828, + "loss": 0.10052140057086945, + "rewards/accuracies": 0.96875, + "rewards/chosen": -3.832404851913452, + "rewards/margins": 3.9090652465820312, + "rewards/rejected": -7.741469860076904, + "step": 453 + }, + { + "epoch": 1.4645161290322581, + "grad_norm": 0.4684765338897705, + "learning_rate": 8.429228586816405e-07, + "logits/chosen": -2.636364459991455, + "logits/rejected": -2.2065482139587402, + "logps/chosen": -149.46168518066406, + "logps/rejected": -220.55462646484375, + "loss": 0.10019417107105255, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.581504821777344, + "rewards/margins": 4.545250415802002, + "rewards/rejected": -9.126755714416504, + "step": 454 + }, + { + "epoch": 1.467741935483871, + "grad_norm": 0.6270551085472107, + "learning_rate": 8.334589799295592e-07, + "logits/chosen": -2.204833745956421, + "logits/rejected": -1.9377959966659546, + "logps/chosen": -151.548583984375, + "logps/rejected": -210.1044921875, + "loss": 0.15584611892700195, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.5387187004089355, + "rewards/margins": 3.5987823009490967, + "rewards/rejected": -8.13750171661377, + "step": 455 + }, + { + "epoch": 1.4709677419354839, + "grad_norm": 0.3135511577129364, + "learning_rate": 8.240378901093035e-07, + "logits/chosen": -2.537592649459839, + "logits/rejected": -2.218113422393799, + "logps/chosen": -144.51199340820312, + "logps/rejected": -225.44337463378906, + "loss": 0.06254112720489502, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.4782867431640625, + "rewards/margins": 4.516358375549316, + "rewards/rejected": -8.994644165039062, + "step": 456 + }, + { + "epoch": 1.4741935483870967, + "grad_norm": 1.0306941270828247, + "learning_rate": 8.146598311101317e-07, + "logits/chosen": -2.2063868045806885, + "logits/rejected": -1.911825180053711, + "logps/chosen": -159.69532775878906, + "logps/rejected": -212.94476318359375, + "loss": 0.23534297943115234, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.0022382736206055, + "rewards/margins": 3.1327128410339355, + "rewards/rejected": -9.134952545166016, + "step": 457 + }, + { + "epoch": 1.4774193548387098, + "grad_norm": 0.7191207408905029, + "learning_rate": 8.053250437164716e-07, + "logits/chosen": -2.118504762649536, + "logits/rejected": -1.8495814800262451, + "logps/chosen": -155.6160430908203, + "logps/rejected": -211.62818908691406, + "loss": 0.1392982304096222, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.2305908203125, + "rewards/margins": 3.6847500801086426, + "rewards/rejected": -7.915340900421143, + "step": 458 + }, + { + "epoch": 1.4806451612903226, + "grad_norm": 0.6096194386482239, + "learning_rate": 7.960337676017396e-07, + "logits/chosen": -2.7578587532043457, + "logits/rejected": -2.4358770847320557, + "logps/chosen": -143.50048828125, + "logps/rejected": -202.1869659423828, + "loss": 0.15191882848739624, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.54219913482666, + "rewards/margins": 3.695138454437256, + "rewards/rejected": -8.237337112426758, + "step": 459 + }, + { + "epoch": 1.4838709677419355, + "grad_norm": 0.5208762288093567, + "learning_rate": 7.867862413221894e-07, + "logits/chosen": -2.467456579208374, + "logits/rejected": -2.0829808712005615, + "logps/chosen": -131.95315551757812, + "logps/rejected": -202.734375, + "loss": 0.11966469138860703, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.343000888824463, + "rewards/margins": 4.022573471069336, + "rewards/rejected": -8.36557388305664, + "step": 460 + }, + { + "epoch": 1.4870967741935484, + "grad_norm": 0.4789765179157257, + "learning_rate": 7.775827023107835e-07, + "logits/chosen": -2.528873920440674, + "logits/rejected": -2.153320789337158, + "logps/chosen": -142.16531372070312, + "logps/rejected": -210.26077270507812, + "loss": 0.09036726504564285, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.478372573852539, + "rewards/margins": 4.291580677032471, + "rewards/rejected": -8.769953727722168, + "step": 461 + }, + { + "epoch": 1.4903225806451612, + "grad_norm": 0.7295042276382446, + "learning_rate": 7.684233868710983e-07, + "logits/chosen": -2.4204530715942383, + "logits/rejected": -2.1346089839935303, + "logps/chosen": -166.9981689453125, + "logps/rejected": -230.27064514160156, + "loss": 0.17296504974365234, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.720881462097168, + "rewards/margins": 3.7996249198913574, + "rewards/rejected": -9.520506858825684, + "step": 462 + }, + { + "epoch": 1.4935483870967743, + "grad_norm": 0.7209341526031494, + "learning_rate": 7.593085301712566e-07, + "logits/chosen": -2.1364479064941406, + "logits/rejected": -1.8027024269104004, + "logps/chosen": -177.41847229003906, + "logps/rejected": -242.67233276367188, + "loss": 0.18686425685882568, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.450223922729492, + "rewards/margins": 3.922041416168213, + "rewards/rejected": -9.372265815734863, + "step": 463 + }, + { + "epoch": 1.4967741935483871, + "grad_norm": 0.6717926263809204, + "learning_rate": 7.502383662378895e-07, + "logits/chosen": -2.4188666343688965, + "logits/rejected": -2.0470993518829346, + "logps/chosen": -131.65296936035156, + "logps/rejected": -193.15231323242188, + "loss": 0.21131563186645508, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.352531909942627, + "rewards/margins": 3.5780229568481445, + "rewards/rejected": -7.930554389953613, + "step": 464 + }, + { + "epoch": 1.5, + "grad_norm": 0.6198790073394775, + "learning_rate": 7.412131279501297e-07, + "logits/chosen": -2.3598873615264893, + "logits/rejected": -2.030845880508423, + "logps/chosen": -155.17327880859375, + "logps/rejected": -218.849365234375, + "loss": 0.17705798149108887, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.077648162841797, + "rewards/margins": 3.896883726119995, + "rewards/rejected": -8.974532127380371, + "step": 465 + }, + { + "epoch": 1.5032258064516129, + "grad_norm": 0.7314626574516296, + "learning_rate": 7.322330470336314e-07, + "logits/chosen": -2.189079761505127, + "logits/rejected": -1.7819743156433105, + "logps/chosen": -150.0357666015625, + "logps/rejected": -217.533935546875, + "loss": 0.12745235860347748, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.761589050292969, + "rewards/margins": 4.053315162658691, + "rewards/rejected": -8.81490421295166, + "step": 466 + }, + { + "epoch": 1.5064516129032257, + "grad_norm": 0.736470639705658, + "learning_rate": 7.232983540546173e-07, + "logits/chosen": -2.2979016304016113, + "logits/rejected": -1.9683010578155518, + "logps/chosen": -146.45066833496094, + "logps/rejected": -209.13137817382812, + "loss": 0.19569313526153564, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.635673999786377, + "rewards/margins": 3.676464557647705, + "rewards/rejected": -8.312139511108398, + "step": 467 + }, + { + "epoch": 1.5096774193548388, + "grad_norm": 0.3781033754348755, + "learning_rate": 7.144092784139619e-07, + "logits/chosen": -2.039719581604004, + "logits/rejected": -1.7212536334991455, + "logps/chosen": -171.434326171875, + "logps/rejected": -237.9990234375, + "loss": 0.10456052422523499, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.9777727127075195, + "rewards/margins": 4.449230670928955, + "rewards/rejected": -9.427002906799316, + "step": 468 + }, + { + "epoch": 1.5129032258064516, + "grad_norm": 0.7195813059806824, + "learning_rate": 7.05566048341303e-07, + "logits/chosen": -1.9777944087982178, + "logits/rejected": -1.6352850198745728, + "logps/chosen": -180.8108673095703, + "logps/rejected": -257.6719665527344, + "loss": 0.1284591257572174, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.535017013549805, + "rewards/margins": 4.279155731201172, + "rewards/rejected": -10.814172744750977, + "step": 469 + }, + { + "epoch": 1.5161290322580645, + "grad_norm": 1.0195460319519043, + "learning_rate": 6.967688908891793e-07, + "logits/chosen": -2.204637050628662, + "logits/rejected": -1.9138691425323486, + "logps/chosen": -160.77059936523438, + "logps/rejected": -235.45228576660156, + "loss": 0.2252466380596161, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.683173179626465, + "rewards/margins": 4.687705993652344, + "rewards/rejected": -10.370879173278809, + "step": 470 + }, + { + "epoch": 1.5193548387096776, + "grad_norm": 0.4896174371242523, + "learning_rate": 6.880180319272006e-07, + "logits/chosen": -2.68131160736084, + "logits/rejected": -2.368668794631958, + "logps/chosen": -145.79098510742188, + "logps/rejected": -197.18484497070312, + "loss": 0.1361011266708374, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.403603553771973, + "rewards/margins": 3.4635801315307617, + "rewards/rejected": -7.867183685302734, + "step": 471 + }, + { + "epoch": 1.5225806451612902, + "grad_norm": 0.6289461255073547, + "learning_rate": 6.79313696136249e-07, + "logits/chosen": -2.1100573539733887, + "logits/rejected": -1.6402158737182617, + "logps/chosen": -154.7132568359375, + "logps/rejected": -222.39056396484375, + "loss": 0.15632954239845276, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.803866863250732, + "rewards/margins": 4.1356987953186035, + "rewards/rejected": -8.939565658569336, + "step": 472 + }, + { + "epoch": 1.5258064516129033, + "grad_norm": 0.34602445363998413, + "learning_rate": 6.706561070027109e-07, + "logits/chosen": -2.1718649864196777, + "logits/rejected": -1.7549580335617065, + "logps/chosen": -162.49391174316406, + "logps/rejected": -237.9075927734375, + "loss": 0.07619393616914749, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.069583892822266, + "rewards/margins": 4.553929805755615, + "rewards/rejected": -9.623514175415039, + "step": 473 + }, + { + "epoch": 1.5290322580645161, + "grad_norm": 0.9247276782989502, + "learning_rate": 6.620454868127396e-07, + "logits/chosen": -1.8749210834503174, + "logits/rejected": -1.5326533317565918, + "logps/chosen": -163.49847412109375, + "logps/rejected": -231.64306640625, + "loss": 0.2411191463470459, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.642821788787842, + "rewards/margins": 4.313276290893555, + "rewards/rejected": -9.956097602844238, + "step": 474 + }, + { + "epoch": 1.532258064516129, + "grad_norm": 0.5158165693283081, + "learning_rate": 6.534820566465464e-07, + "logits/chosen": -2.297236204147339, + "logits/rejected": -1.91456139087677, + "logps/chosen": -144.25331115722656, + "logps/rejected": -201.99111938476562, + "loss": 0.15576979517936707, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.625210762023926, + "rewards/margins": 3.590076446533203, + "rewards/rejected": -8.215287208557129, + "step": 475 + }, + { + "epoch": 1.535483870967742, + "grad_norm": 0.6987940669059753, + "learning_rate": 6.449660363727236e-07, + "logits/chosen": -2.297675132751465, + "logits/rejected": -1.7836105823516846, + "logps/chosen": -151.13223266601562, + "logps/rejected": -221.970947265625, + "loss": 0.1767854392528534, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.299595355987549, + "rewards/margins": 4.221803188323975, + "rewards/rejected": -9.521398544311523, + "step": 476 + }, + { + "epoch": 1.5387096774193547, + "grad_norm": 0.601422905921936, + "learning_rate": 6.36497644642601e-07, + "logits/chosen": -2.286315441131592, + "logits/rejected": -1.9132990837097168, + "logps/chosen": -149.023193359375, + "logps/rejected": -229.86410522460938, + "loss": 0.10134474188089371, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.733626365661621, + "rewards/margins": 4.846975803375244, + "rewards/rejected": -10.580602645874023, + "step": 477 + }, + { + "epoch": 1.5419354838709678, + "grad_norm": 0.6406868696212769, + "learning_rate": 6.28077098884633e-07, + "logits/chosen": -2.4604923725128174, + "logits/rejected": -2.084846019744873, + "logps/chosen": -139.864013671875, + "logps/rejected": -211.58987426757812, + "loss": 0.13258515298366547, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.099147319793701, + "rewards/margins": 4.616920471191406, + "rewards/rejected": -8.716068267822266, + "step": 478 + }, + { + "epoch": 1.5451612903225806, + "grad_norm": 1.0441559553146362, + "learning_rate": 6.197046152988137e-07, + "logits/chosen": -2.1741297245025635, + "logits/rejected": -1.8643018007278442, + "logps/chosen": -168.7382049560547, + "logps/rejected": -230.0182647705078, + "loss": 0.2577561140060425, + "rewards/accuracies": 0.90625, + "rewards/chosen": -6.1876912117004395, + "rewards/margins": 3.6677284240722656, + "rewards/rejected": -9.855419158935547, + "step": 479 + }, + { + "epoch": 1.5483870967741935, + "grad_norm": 0.9108280539512634, + "learning_rate": 6.113804088511261e-07, + "logits/chosen": -1.954637885093689, + "logits/rejected": -1.6181361675262451, + "logps/chosen": -175.4678955078125, + "logps/rejected": -237.62701416015625, + "loss": 0.16138975322246552, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.869152545928955, + "rewards/margins": 3.9080886840820312, + "rewards/rejected": -9.777240753173828, + "step": 480 + }, + { + "epoch": 1.5516129032258066, + "grad_norm": 0.24027889966964722, + "learning_rate": 6.031046932680229e-07, + "logits/chosen": -2.4597034454345703, + "logits/rejected": -2.1957783699035645, + "logps/chosen": -157.1087646484375, + "logps/rejected": -233.65301513671875, + "loss": 0.05748886615037918, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.280271053314209, + "rewards/margins": 4.910508155822754, + "rewards/rejected": -10.190779685974121, + "step": 481 + }, + { + "epoch": 1.5548387096774192, + "grad_norm": 0.9712018966674805, + "learning_rate": 5.948776810309423e-07, + "logits/chosen": -2.126974582672119, + "logits/rejected": -1.7661800384521484, + "logps/chosen": -151.3673858642578, + "logps/rejected": -218.9346466064453, + "loss": 0.24991604685783386, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.019974708557129, + "rewards/margins": 3.8181777000427246, + "rewards/rejected": -8.838151931762695, + "step": 482 + }, + { + "epoch": 1.5580645161290323, + "grad_norm": 0.7719769477844238, + "learning_rate": 5.866995833708464e-07, + "logits/chosen": -1.708822250366211, + "logits/rejected": -1.3647013902664185, + "logps/chosen": -164.8237762451172, + "logps/rejected": -234.05825805664062, + "loss": 0.18122202157974243, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.682433605194092, + "rewards/margins": 4.063777923583984, + "rewards/rejected": -9.746211051940918, + "step": 483 + }, + { + "epoch": 1.5612903225806452, + "grad_norm": 0.3257688283920288, + "learning_rate": 5.785706102628044e-07, + "logits/chosen": -2.233219861984253, + "logits/rejected": -1.8412196636199951, + "logps/chosen": -152.9199981689453, + "logps/rejected": -216.41781616210938, + "loss": 0.07986757159233093, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.557250499725342, + "rewards/margins": 4.080105781555176, + "rewards/rejected": -8.63735580444336, + "step": 484 + }, + { + "epoch": 1.564516129032258, + "grad_norm": 0.6999625563621521, + "learning_rate": 5.704909704205949e-07, + "logits/chosen": -1.9356735944747925, + "logits/rejected": -1.7792093753814697, + "logps/chosen": -166.04051208496094, + "logps/rejected": -219.11294555664062, + "loss": 0.18605303764343262, + "rewards/accuracies": 0.90625, + "rewards/chosen": -5.293807029724121, + "rewards/margins": 3.0410327911376953, + "rewards/rejected": -8.334840774536133, + "step": 485 + }, + { + "epoch": 1.567741935483871, + "grad_norm": 0.8240074515342712, + "learning_rate": 5.624608712913531e-07, + "logits/chosen": -2.172473192214966, + "logits/rejected": -1.7739810943603516, + "logps/chosen": -136.5169219970703, + "logps/rejected": -208.26185607910156, + "loss": 0.16257520020008087, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.544417858123779, + "rewards/margins": 4.220807075500488, + "rewards/rejected": -8.76522445678711, + "step": 486 + }, + { + "epoch": 1.5709677419354837, + "grad_norm": 0.6017993688583374, + "learning_rate": 5.544805190502406e-07, + "logits/chosen": -2.2712013721466064, + "logits/rejected": -1.9342961311340332, + "logps/chosen": -151.09034729003906, + "logps/rejected": -212.1373291015625, + "loss": 0.15379485487937927, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.01048469543457, + "rewards/margins": 3.8621017932891846, + "rewards/rejected": -8.872586250305176, + "step": 487 + }, + { + "epoch": 1.5741935483870968, + "grad_norm": 0.5773161053657532, + "learning_rate": 5.465501185951519e-07, + "logits/chosen": -2.4476537704467773, + "logits/rejected": -2.045992851257324, + "logps/chosen": -151.75369262695312, + "logps/rejected": -227.71702575683594, + "loss": 0.1062774658203125, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.999873161315918, + "rewards/margins": 4.466883182525635, + "rewards/rejected": -9.466756820678711, + "step": 488 + }, + { + "epoch": 1.5774193548387097, + "grad_norm": 0.4516680836677551, + "learning_rate": 5.386698735414572e-07, + "logits/chosen": -2.406808853149414, + "logits/rejected": -1.889483094215393, + "logps/chosen": -124.93113708496094, + "logps/rejected": -212.51937866210938, + "loss": 0.0906469076871872, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.496485710144043, + "rewards/margins": 5.360855579376221, + "rewards/rejected": -9.857341766357422, + "step": 489 + }, + { + "epoch": 1.5806451612903225, + "grad_norm": 0.29949402809143066, + "learning_rate": 5.308399862167693e-07, + "logits/chosen": -2.1711933612823486, + "logits/rejected": -1.7267756462097168, + "logps/chosen": -146.49330139160156, + "logps/rejected": -239.75428771972656, + "loss": 0.07038956135511398, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.902659893035889, + "rewards/margins": 5.8861589431762695, + "rewards/rejected": -10.788818359375, + "step": 490 + }, + { + "epoch": 1.5838709677419356, + "grad_norm": 0.6430067420005798, + "learning_rate": 5.23060657655754e-07, + "logits/chosen": -2.275541067123413, + "logits/rejected": -1.8227019309997559, + "logps/chosen": -160.78375244140625, + "logps/rejected": -235.040771484375, + "loss": 0.13946972787380219, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.349231719970703, + "rewards/margins": 4.761424541473389, + "rewards/rejected": -10.110654830932617, + "step": 491 + }, + { + "epoch": 1.5870967741935482, + "grad_norm": 0.560444176197052, + "learning_rate": 5.153320875949632e-07, + "logits/chosen": -2.355436086654663, + "logits/rejected": -2.036803960800171, + "logps/chosen": -154.39837646484375, + "logps/rejected": -220.92160034179688, + "loss": 0.1334306001663208, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.16274356842041, + "rewards/margins": 4.023294925689697, + "rewards/rejected": -9.186038970947266, + "step": 492 + }, + { + "epoch": 1.5903225806451613, + "grad_norm": 0.5349669456481934, + "learning_rate": 5.076544744677128e-07, + "logits/chosen": -2.483330726623535, + "logits/rejected": -2.1197609901428223, + "logps/chosen": -156.27420043945312, + "logps/rejected": -217.24514770507812, + "loss": 0.108750119805336, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.527863502502441, + "rewards/margins": 3.8662824630737305, + "rewards/rejected": -8.394145965576172, + "step": 493 + }, + { + "epoch": 1.5935483870967742, + "grad_norm": 0.2775120437145233, + "learning_rate": 5.000280153989806e-07, + "logits/chosen": -2.387772560119629, + "logits/rejected": -1.9994821548461914, + "logps/chosen": -151.18211364746094, + "logps/rejected": -227.11680603027344, + "loss": 0.04961947351694107, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.598075866699219, + "rewards/margins": 4.705333709716797, + "rewards/rejected": -9.303409576416016, + "step": 494 + }, + { + "epoch": 1.596774193548387, + "grad_norm": 0.6997309923171997, + "learning_rate": 4.924529062003522e-07, + "logits/chosen": -2.093170166015625, + "logits/rejected": -1.5877161026000977, + "logps/chosen": -138.06466674804688, + "logps/rejected": -211.8524932861328, + "loss": 0.1541440337896347, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.241321563720703, + "rewards/margins": 4.32204008102417, + "rewards/rejected": -8.563361167907715, + "step": 495 + }, + { + "epoch": 1.6, + "grad_norm": 0.7098890542984009, + "learning_rate": 4.84929341364988e-07, + "logits/chosen": -1.9759151935577393, + "logits/rejected": -1.6069056987762451, + "logps/chosen": -157.34674072265625, + "logps/rejected": -219.7688446044922, + "loss": 0.1809576451778412, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.922688007354736, + "rewards/margins": 3.9587292671203613, + "rewards/rejected": -8.881418228149414, + "step": 496 + }, + { + "epoch": 1.603225806451613, + "grad_norm": 0.5172654986381531, + "learning_rate": 4.774575140626317e-07, + "logits/chosen": -2.007028102874756, + "logits/rejected": -1.6205710172653198, + "logps/chosen": -166.93746948242188, + "logps/rejected": -236.51893615722656, + "loss": 0.09278646111488342, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.094544887542725, + "rewards/margins": 4.20934534072876, + "rewards/rejected": -9.303890228271484, + "step": 497 + }, + { + "epoch": 1.6064516129032258, + "grad_norm": 0.3379948139190674, + "learning_rate": 4.7003761613465174e-07, + "logits/chosen": -2.038321018218994, + "logits/rejected": -1.653014898300171, + "logps/chosen": -166.12400817871094, + "logps/rejected": -238.2560577392578, + "loss": 0.05834685266017914, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.0331950187683105, + "rewards/margins": 4.559978008270264, + "rewards/rejected": -9.593173027038574, + "step": 498 + }, + { + "epoch": 1.6096774193548387, + "grad_norm": 1.2031432390213013, + "learning_rate": 4.626698380891154e-07, + "logits/chosen": -2.3232650756835938, + "logits/rejected": -2.098174810409546, + "logps/chosen": -154.74142456054688, + "logps/rejected": -210.04086303710938, + "loss": 0.26611971855163574, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.1587114334106445, + "rewards/margins": 3.5554258823394775, + "rewards/rejected": -8.71413803100586, + "step": 499 + }, + { + "epoch": 1.6129032258064515, + "grad_norm": 0.7430384755134583, + "learning_rate": 4.553543690958939e-07, + "logits/chosen": -2.6903367042541504, + "logits/rejected": -2.353358507156372, + "logps/chosen": -147.15798950195312, + "logps/rejected": -199.20314025878906, + "loss": 0.24731270968914032, + "rewards/accuracies": 0.90625, + "rewards/chosen": -5.127081394195557, + "rewards/margins": 2.9009103775024414, + "rewards/rejected": -8.027992248535156, + "step": 500 + }, + { + "epoch": 1.6161290322580646, + "grad_norm": 0.505378782749176, + "learning_rate": 4.480913969818099e-07, + "logits/chosen": -2.323559284210205, + "logits/rejected": -1.9315087795257568, + "logps/chosen": -161.15017700195312, + "logps/rejected": -225.9407196044922, + "loss": 0.15063539147377014, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.602543354034424, + "rewards/margins": 3.9944305419921875, + "rewards/rejected": -9.59697437286377, + "step": 501 + }, + { + "epoch": 1.6193548387096774, + "grad_norm": 0.5478779077529907, + "learning_rate": 4.408811082258116e-07, + "logits/chosen": -2.405697822570801, + "logits/rejected": -2.060330629348755, + "logps/chosen": -149.43878173828125, + "logps/rejected": -203.4425506591797, + "loss": 0.11198721826076508, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.570002555847168, + "rewards/margins": 3.5500755310058594, + "rewards/rejected": -8.120078086853027, + "step": 502 + }, + { + "epoch": 1.6225806451612903, + "grad_norm": 0.8584948182106018, + "learning_rate": 4.3372368795419006e-07, + "logits/chosen": -2.1318094730377197, + "logits/rejected": -1.7252811193466187, + "logps/chosen": -156.61129760742188, + "logps/rejected": -231.13629150390625, + "loss": 0.18643182516098022, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.35899543762207, + "rewards/margins": 4.680524826049805, + "rewards/rejected": -10.039520263671875, + "step": 503 + }, + { + "epoch": 1.6258064516129034, + "grad_norm": 0.8634819388389587, + "learning_rate": 4.2661931993581874e-07, + "logits/chosen": -2.2070565223693848, + "logits/rejected": -1.828540563583374, + "logps/chosen": -158.256591796875, + "logps/rejected": -222.89712524414062, + "loss": 0.18253442645072937, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.994098663330078, + "rewards/margins": 3.824960470199585, + "rewards/rejected": -8.819059371948242, + "step": 504 + }, + { + "epoch": 1.629032258064516, + "grad_norm": 0.39533722400665283, + "learning_rate": 4.1956818657744065e-07, + "logits/chosen": -2.7217161655426025, + "logits/rejected": -2.2938411235809326, + "logps/chosen": -132.1091766357422, + "logps/rejected": -214.93460083007812, + "loss": 0.10780246555805206, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.512347221374512, + "rewards/margins": 5.154045104980469, + "rewards/rejected": -9.66639232635498, + "step": 505 + }, + { + "epoch": 1.632258064516129, + "grad_norm": 0.4706231653690338, + "learning_rate": 4.125704689189819e-07, + "logits/chosen": -2.0221657752990723, + "logits/rejected": -1.658431887626648, + "logps/chosen": -164.7430419921875, + "logps/rejected": -224.69056701660156, + "loss": 0.09151042252779007, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.65877628326416, + "rewards/margins": 4.031785011291504, + "rewards/rejected": -8.690561294555664, + "step": 506 + }, + { + "epoch": 1.635483870967742, + "grad_norm": 0.5980255603790283, + "learning_rate": 4.056263466289059e-07, + "logits/chosen": -2.4284605979919434, + "logits/rejected": -2.004915475845337, + "logps/chosen": -146.16546630859375, + "logps/rejected": -226.4752197265625, + "loss": 0.10536962002515793, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.9078874588012695, + "rewards/margins": 4.797103404998779, + "rewards/rejected": -9.70499038696289, + "step": 507 + }, + { + "epoch": 1.6387096774193548, + "grad_norm": 0.905908465385437, + "learning_rate": 3.987359979996e-07, + "logits/chosen": -1.9300092458724976, + "logits/rejected": -1.616002082824707, + "logps/chosen": -175.74253845214844, + "logps/rejected": -242.51318359375, + "loss": 0.1710321605205536, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.3357696533203125, + "rewards/margins": 4.0363850593566895, + "rewards/rejected": -10.37215518951416, + "step": 508 + }, + { + "epoch": 1.6419354838709679, + "grad_norm": 0.792195737361908, + "learning_rate": 3.9189959994279493e-07, + "logits/chosen": -2.461611032485962, + "logits/rejected": -2.0225419998168945, + "logps/chosen": -144.42999267578125, + "logps/rejected": -218.36630249023438, + "loss": 0.19733068346977234, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.6903486251831055, + "rewards/margins": 4.069197177886963, + "rewards/rejected": -8.759546279907227, + "step": 509 + }, + { + "epoch": 1.6451612903225805, + "grad_norm": 0.5894666910171509, + "learning_rate": 3.851173279850251e-07, + "logits/chosen": -2.2512221336364746, + "logits/rejected": -1.8765403032302856, + "logps/chosen": -161.9225616455078, + "logps/rejected": -218.21316528320312, + "loss": 0.13903504610061646, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.284883975982666, + "rewards/margins": 3.688009738922119, + "rewards/rejected": -8.972892761230469, + "step": 510 + }, + { + "epoch": 1.6483870967741936, + "grad_norm": 0.6753301024436951, + "learning_rate": 3.7838935626312246e-07, + "logits/chosen": -2.39656400680542, + "logits/rejected": -2.112570285797119, + "logps/chosen": -132.52127075195312, + "logps/rejected": -195.42550659179688, + "loss": 0.17207223176956177, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.37971830368042, + "rewards/margins": 3.5059244632720947, + "rewards/rejected": -7.885643005371094, + "step": 511 + }, + { + "epoch": 1.6516129032258065, + "grad_norm": 0.26837578415870667, + "learning_rate": 3.717158575197441e-07, + "logits/chosen": -2.2023987770080566, + "logits/rejected": -1.7415173053741455, + "logps/chosen": -152.79617309570312, + "logps/rejected": -234.56285095214844, + "loss": 0.0666898787021637, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.663599967956543, + "rewards/margins": 4.916285514831543, + "rewards/rejected": -10.579885482788086, + "step": 512 + }, + { + "epoch": 1.6548387096774193, + "grad_norm": 0.5426744818687439, + "learning_rate": 3.650970030989362e-07, + "logits/chosen": -2.204011917114258, + "logits/rejected": -1.8495509624481201, + "logps/chosen": -128.29156494140625, + "logps/rejected": -205.69387817382812, + "loss": 0.14435040950775146, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.655958652496338, + "rewards/margins": 4.450262546539307, + "rewards/rejected": -9.106222152709961, + "step": 513 + }, + { + "epoch": 1.6580645161290324, + "grad_norm": 0.30497443675994873, + "learning_rate": 3.5853296294173665e-07, + "logits/chosen": -2.109001398086548, + "logits/rejected": -1.6162521839141846, + "logps/chosen": -156.780517578125, + "logps/rejected": -229.7040252685547, + "loss": 0.05599897354841232, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.427706718444824, + "rewards/margins": 4.8288421630859375, + "rewards/rejected": -9.256548881530762, + "step": 514 + }, + { + "epoch": 1.661290322580645, + "grad_norm": 0.4224660396575928, + "learning_rate": 3.5202390558181145e-07, + "logits/chosen": -2.2887954711914062, + "logits/rejected": -1.880237340927124, + "logps/chosen": -160.7728271484375, + "logps/rejected": -238.85765075683594, + "loss": 0.0738740861415863, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.382646083831787, + "rewards/margins": 4.997879505157471, + "rewards/rejected": -10.380525588989258, + "step": 515 + }, + { + "epoch": 1.664516129032258, + "grad_norm": 0.4132245182991028, + "learning_rate": 3.455699981411259e-07, + "logits/chosen": -2.2288949489593506, + "logits/rejected": -1.6625796556472778, + "logps/chosen": -136.56069946289062, + "logps/rejected": -225.6313934326172, + "loss": 0.0787404254078865, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.833522796630859, + "rewards/margins": 5.683566093444824, + "rewards/rejected": -10.517088890075684, + "step": 516 + }, + { + "epoch": 1.667741935483871, + "grad_norm": 0.5380642414093018, + "learning_rate": 3.3917140632565627e-07, + "logits/chosen": -2.1946632862091064, + "logits/rejected": -1.7250945568084717, + "logps/chosen": -149.41163635253906, + "logps/rejected": -227.01788330078125, + "loss": 0.10217361897230148, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.115983009338379, + "rewards/margins": 4.956330299377441, + "rewards/rejected": -10.07231330871582, + "step": 517 + }, + { + "epoch": 1.6709677419354838, + "grad_norm": 0.6858349442481995, + "learning_rate": 3.3282829442113277e-07, + "logits/chosen": -2.756497383117676, + "logits/rejected": -2.385478973388672, + "logps/chosen": -132.74374389648438, + "logps/rejected": -202.3121795654297, + "loss": 0.17126917839050293, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.5010786056518555, + "rewards/margins": 4.135538578033447, + "rewards/rejected": -8.636617660522461, + "step": 518 + }, + { + "epoch": 1.6741935483870969, + "grad_norm": 0.3403320908546448, + "learning_rate": 3.265408252888222e-07, + "logits/chosen": -2.14156436920166, + "logits/rejected": -1.7484135627746582, + "logps/chosen": -150.6089630126953, + "logps/rejected": -227.8260955810547, + "loss": 0.06928884983062744, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.599906921386719, + "rewards/margins": 5.042827129364014, + "rewards/rejected": -9.642733573913574, + "step": 519 + }, + { + "epoch": 1.6774193548387095, + "grad_norm": 0.5155129432678223, + "learning_rate": 3.2030916036134866e-07, + "logits/chosen": -2.3275980949401855, + "logits/rejected": -2.018320322036743, + "logps/chosen": -136.49685668945312, + "logps/rejected": -214.154541015625, + "loss": 0.12636500597000122, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.527489185333252, + "rewards/margins": 4.5036139488220215, + "rewards/rejected": -9.031103134155273, + "step": 520 + }, + { + "epoch": 1.6806451612903226, + "grad_norm": 0.555239200592041, + "learning_rate": 3.141334596385448e-07, + "logits/chosen": -2.2510530948638916, + "logits/rejected": -1.8884137868881226, + "logps/chosen": -149.95242309570312, + "logps/rejected": -211.98965454101562, + "loss": 0.11958973854780197, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.119822978973389, + "rewards/margins": 3.77842378616333, + "rewards/rejected": -8.898246765136719, + "step": 521 + }, + { + "epoch": 1.6838709677419355, + "grad_norm": 0.24120715260505676, + "learning_rate": 3.0801388168334723e-07, + "logits/chosen": -1.8827189207077026, + "logits/rejected": -1.5083305835723877, + "logps/chosen": -169.45654296875, + "logps/rejected": -259.73150634765625, + "loss": 0.04009680077433586, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.678956031799316, + "rewards/margins": 5.2564616203308105, + "rewards/rejected": -11.935418128967285, + "step": 522 + }, + { + "epoch": 1.6870967741935483, + "grad_norm": 0.6411151885986328, + "learning_rate": 3.019505836177228e-07, + "logits/chosen": -2.701399326324463, + "logits/rejected": -2.2651937007904053, + "logps/chosen": -143.605712890625, + "logps/rejected": -207.84645080566406, + "loss": 0.16435886919498444, + "rewards/accuracies": 0.90625, + "rewards/chosen": -4.970480442047119, + "rewards/margins": 4.199647903442383, + "rewards/rejected": -9.17012882232666, + "step": 523 + }, + { + "epoch": 1.6903225806451614, + "grad_norm": 0.31048446893692017, + "learning_rate": 2.9594372111863696e-07, + "logits/chosen": -2.088878631591797, + "logits/rejected": -1.5833330154418945, + "logps/chosen": -148.25637817382812, + "logps/rejected": -240.0180206298828, + "loss": 0.047217682003974915, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.118111610412598, + "rewards/margins": 5.628313064575195, + "rewards/rejected": -10.74642562866211, + "step": 524 + }, + { + "epoch": 1.6935483870967742, + "grad_norm": 0.6864602565765381, + "learning_rate": 2.8999344841405377e-07, + "logits/chosen": -2.192661762237549, + "logits/rejected": -1.765640139579773, + "logps/chosen": -153.21095275878906, + "logps/rejected": -230.85977172851562, + "loss": 0.16634179651737213, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.749712944030762, + "rewards/margins": 4.466973304748535, + "rewards/rejected": -9.216686248779297, + "step": 525 + }, + { + "epoch": 1.696774193548387, + "grad_norm": 0.5602472424507141, + "learning_rate": 2.840999182789797e-07, + "logits/chosen": -2.3613505363464355, + "logits/rejected": -1.963671088218689, + "logps/chosen": -158.62982177734375, + "logps/rejected": -232.7010498046875, + "loss": 0.1083802878856659, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.514021873474121, + "rewards/margins": 4.288410663604736, + "rewards/rejected": -9.802433013916016, + "step": 526 + }, + { + "epoch": 1.7, + "grad_norm": 0.8596751093864441, + "learning_rate": 2.782632820315362e-07, + "logits/chosen": -2.513413190841675, + "logits/rejected": -2.0594162940979004, + "logps/chosen": -135.89236450195312, + "logps/rejected": -203.92601013183594, + "loss": 0.184890478849411, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.378267765045166, + "rewards/margins": 4.425320625305176, + "rewards/rejected": -8.803587913513184, + "step": 527 + }, + { + "epoch": 1.7032258064516128, + "grad_norm": 0.8277157545089722, + "learning_rate": 2.7248368952908055e-07, + "logits/chosen": -2.415259599685669, + "logits/rejected": -2.0948903560638428, + "logps/chosen": -143.08612060546875, + "logps/rejected": -203.56727600097656, + "loss": 0.16695170104503632, + "rewards/accuracies": 0.90625, + "rewards/chosen": -4.928262710571289, + "rewards/margins": 3.7459940910339355, + "rewards/rejected": -8.674257278442383, + "step": 528 + }, + { + "epoch": 1.706451612903226, + "grad_norm": 0.4720885753631592, + "learning_rate": 2.6676128916435256e-07, + "logits/chosen": -2.2250518798828125, + "logits/rejected": -1.8019331693649292, + "logps/chosen": -154.83782958984375, + "logps/rejected": -236.45587158203125, + "loss": 0.08195596933364868, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.60328483581543, + "rewards/margins": 4.934556007385254, + "rewards/rejected": -10.537841796875, + "step": 529 + }, + { + "epoch": 1.7096774193548387, + "grad_norm": 0.6525325179100037, + "learning_rate": 2.61096227861668e-07, + "logits/chosen": -1.870182752609253, + "logits/rejected": -1.4910308122634888, + "logps/chosen": -170.4964599609375, + "logps/rejected": -226.6752471923828, + "loss": 0.15303362905979156, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.455026626586914, + "rewards/margins": 3.813904047012329, + "rewards/rejected": -9.268930435180664, + "step": 530 + }, + { + "epoch": 1.7129032258064516, + "grad_norm": 0.5887526273727417, + "learning_rate": 2.5548865107314606e-07, + "logits/chosen": -2.2285823822021484, + "logits/rejected": -1.960855484008789, + "logps/chosen": -161.62039184570312, + "logps/rejected": -224.82574462890625, + "loss": 0.13227765262126923, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.081503391265869, + "rewards/margins": 3.7586007118225098, + "rewards/rejected": -8.840105056762695, + "step": 531 + }, + { + "epoch": 1.7161290322580647, + "grad_norm": 0.906437873840332, + "learning_rate": 2.499387027749725e-07, + "logits/chosen": -2.0604171752929688, + "logits/rejected": -1.7882590293884277, + "logps/chosen": -158.12396240234375, + "logps/rejected": -211.99673461914062, + "loss": 0.252509206533432, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.8061137199401855, + "rewards/margins": 3.07973575592041, + "rewards/rejected": -7.885849952697754, + "step": 532 + }, + { + "epoch": 1.7193548387096773, + "grad_norm": 0.9765410423278809, + "learning_rate": 2.4444652546370627e-07, + "logits/chosen": -2.2366061210632324, + "logits/rejected": -1.9035241603851318, + "logps/chosen": -156.17935180664062, + "logps/rejected": -217.71038818359375, + "loss": 0.2236594706773758, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.191309928894043, + "rewards/margins": 3.9683432579040527, + "rewards/rejected": -9.159652709960938, + "step": 533 + }, + { + "epoch": 1.7225806451612904, + "grad_norm": 0.42072594165802, + "learning_rate": 2.3901226015261793e-07, + "logits/chosen": -2.519008159637451, + "logits/rejected": -2.1948413848876953, + "logps/chosen": -139.654296875, + "logps/rejected": -209.64926147460938, + "loss": 0.09390527009963989, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.155945301055908, + "rewards/margins": 4.245931148529053, + "rewards/rejected": -8.401876449584961, + "step": 534 + }, + { + "epoch": 1.7258064516129032, + "grad_norm": 0.7160782814025879, + "learning_rate": 2.3363604636807065e-07, + "logits/chosen": -2.3279876708984375, + "logits/rejected": -1.9171054363250732, + "logps/chosen": -151.30838012695312, + "logps/rejected": -216.0570831298828, + "loss": 0.1738387942314148, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.881179332733154, + "rewards/margins": 3.874018669128418, + "rewards/rejected": -8.755197525024414, + "step": 535 + }, + { + "epoch": 1.729032258064516, + "grad_norm": 0.8418323993682861, + "learning_rate": 2.2831802214593774e-07, + "logits/chosen": -2.156343460083008, + "logits/rejected": -1.78789222240448, + "logps/chosen": -152.94851684570312, + "logps/rejected": -220.4888916015625, + "loss": 0.18803098797798157, + "rewards/accuracies": 0.90625, + "rewards/chosen": -5.578521728515625, + "rewards/margins": 3.9799790382385254, + "rewards/rejected": -9.558500289916992, + "step": 536 + }, + { + "epoch": 1.7322580645161292, + "grad_norm": 0.4620969593524933, + "learning_rate": 2.23058324028059e-07, + "logits/chosen": -2.0751829147338867, + "logits/rejected": -1.639204978942871, + "logps/chosen": -158.0532684326172, + "logps/rejected": -241.23397827148438, + "loss": 0.09422552585601807, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.242846965789795, + "rewards/margins": 5.0318121910095215, + "rewards/rejected": -10.274659156799316, + "step": 537 + }, + { + "epoch": 1.7354838709677418, + "grad_norm": 0.6936494708061218, + "learning_rate": 2.178570870587335e-07, + "logits/chosen": -2.4572184085845947, + "logits/rejected": -2.1089305877685547, + "logps/chosen": -157.9110565185547, + "logps/rejected": -213.38551330566406, + "loss": 0.16482675075531006, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.1806135177612305, + "rewards/margins": 3.4487881660461426, + "rewards/rejected": -8.629401206970215, + "step": 538 + }, + { + "epoch": 1.738709677419355, + "grad_norm": 0.8675318360328674, + "learning_rate": 2.1271444478125237e-07, + "logits/chosen": -2.329207181930542, + "logits/rejected": -2.0246644020080566, + "logps/chosen": -138.73007202148438, + "logps/rejected": -216.8204345703125, + "loss": 0.1617818921804428, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.572966575622559, + "rewards/margins": 5.178282737731934, + "rewards/rejected": -9.751248359680176, + "step": 539 + }, + { + "epoch": 1.7419354838709677, + "grad_norm": 0.7694476842880249, + "learning_rate": 2.0763052923447214e-07, + "logits/chosen": -2.207973003387451, + "logits/rejected": -1.862140417098999, + "logps/chosen": -136.8199920654297, + "logps/rejected": -210.47402954101562, + "loss": 0.1648537665605545, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.768884181976318, + "rewards/margins": 4.165104866027832, + "rewards/rejected": -8.933990478515625, + "step": 540 + }, + { + "epoch": 1.7451612903225806, + "grad_norm": 0.8003881573677063, + "learning_rate": 2.026054709494235e-07, + "logits/chosen": -2.4957833290100098, + "logits/rejected": -2.2322189807891846, + "logps/chosen": -150.86410522460938, + "logps/rejected": -222.50686645507812, + "loss": 0.17683330178260803, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.864772319793701, + "rewards/margins": 4.093851089477539, + "rewards/rejected": -8.958623886108398, + "step": 541 + }, + { + "epoch": 1.7483870967741937, + "grad_norm": 0.6695418357849121, + "learning_rate": 1.9763939894595795e-07, + "logits/chosen": -1.9199615716934204, + "logits/rejected": -1.6785365343093872, + "logps/chosen": -173.23507690429688, + "logps/rejected": -242.41043090820312, + "loss": 0.1188519224524498, + "rewards/accuracies": 0.96875, + "rewards/chosen": -6.151148796081543, + "rewards/margins": 4.460818290710449, + "rewards/rejected": -10.611968040466309, + "step": 542 + }, + { + "epoch": 1.7516129032258063, + "grad_norm": 0.45999041199684143, + "learning_rate": 1.9273244072943814e-07, + "logits/chosen": -2.499255895614624, + "logits/rejected": -2.0790059566497803, + "logps/chosen": -143.73182678222656, + "logps/rejected": -218.94778442382812, + "loss": 0.07506464421749115, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.273706436157227, + "rewards/margins": 4.5024261474609375, + "rewards/rejected": -9.776132583618164, + "step": 543 + }, + { + "epoch": 1.7548387096774194, + "grad_norm": 0.36987289786338806, + "learning_rate": 1.8788472228746135e-07, + "logits/chosen": -2.343918800354004, + "logits/rejected": -1.8907790184020996, + "logps/chosen": -159.0539093017578, + "logps/rejected": -232.19241333007812, + "loss": 0.08276303112506866, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.670258522033691, + "rewards/margins": 4.740736961364746, + "rewards/rejected": -10.410995483398438, + "step": 544 + }, + { + "epoch": 1.7580645161290323, + "grad_norm": 0.6856768727302551, + "learning_rate": 1.830963680866285e-07, + "logits/chosen": -2.5793392658233643, + "logits/rejected": -2.156630516052246, + "logps/chosen": -127.28002166748047, + "logps/rejected": -204.24288940429688, + "loss": 0.16320952773094177, + "rewards/accuracies": 0.90625, + "rewards/chosen": -4.079771995544434, + "rewards/margins": 4.579602241516113, + "rewards/rejected": -8.659374237060547, + "step": 545 + }, + { + "epoch": 1.761290322580645, + "grad_norm": 0.8824412226676941, + "learning_rate": 1.7836750106934475e-07, + "logits/chosen": -2.4374589920043945, + "logits/rejected": -2.1196401119232178, + "logps/chosen": -159.99725341796875, + "logps/rejected": -218.28028869628906, + "loss": 0.15264299511909485, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.216504096984863, + "rewards/margins": 3.364132881164551, + "rewards/rejected": -8.580636978149414, + "step": 546 + }, + { + "epoch": 1.7645161290322582, + "grad_norm": 0.49898338317871094, + "learning_rate": 1.7369824265066366e-07, + "logits/chosen": -2.0640761852264404, + "logits/rejected": -1.6560142040252686, + "logps/chosen": -155.5439453125, + "logps/rejected": -215.54800415039062, + "loss": 0.12484726309776306, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.699676990509033, + "rewards/margins": 3.61027455329895, + "rewards/rejected": -8.309951782226562, + "step": 547 + }, + { + "epoch": 1.7677419354838708, + "grad_norm": 1.0586748123168945, + "learning_rate": 1.6908871271517136e-07, + "logits/chosen": -1.8716487884521484, + "logits/rejected": -1.5878236293792725, + "logps/chosen": -177.3343963623047, + "logps/rejected": -235.95562744140625, + "loss": 0.28872251510620117, + "rewards/accuracies": 0.875, + "rewards/chosen": -6.277359962463379, + "rewards/margins": 4.477985858917236, + "rewards/rejected": -10.755345344543457, + "step": 548 + }, + { + "epoch": 1.770967741935484, + "grad_norm": 0.6965610980987549, + "learning_rate": 1.645390296139074e-07, + "logits/chosen": -1.5745385885238647, + "logits/rejected": -1.214267373085022, + "logps/chosen": -175.0247344970703, + "logps/rejected": -248.30154418945312, + "loss": 0.14276829361915588, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.623499870300293, + "rewards/margins": 4.304838180541992, + "rewards/rejected": -10.928339004516602, + "step": 549 + }, + { + "epoch": 1.7741935483870968, + "grad_norm": 0.4304961860179901, + "learning_rate": 1.600493101613268e-07, + "logits/chosen": -2.1939003467559814, + "logits/rejected": -1.7763912677764893, + "logps/chosen": -164.8372344970703, + "logps/rejected": -233.04421997070312, + "loss": 0.08470362424850464, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.929935455322266, + "rewards/margins": 4.351900577545166, + "rewards/rejected": -10.281835556030273, + "step": 550 + }, + { + "epoch": 1.7774193548387096, + "grad_norm": 0.6119385957717896, + "learning_rate": 1.5561966963229925e-07, + "logits/chosen": -2.0900025367736816, + "logits/rejected": -1.6825987100601196, + "logps/chosen": -151.122802734375, + "logps/rejected": -216.6434326171875, + "loss": 0.1607978641986847, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.659897327423096, + "rewards/margins": 3.7245254516601562, + "rewards/rejected": -8.384422302246094, + "step": 551 + }, + { + "epoch": 1.7806451612903227, + "grad_norm": 0.5334950685501099, + "learning_rate": 1.5125022175914993e-07, + "logits/chosen": -2.012514352798462, + "logits/rejected": -1.6403321027755737, + "logps/chosen": -164.17938232421875, + "logps/rejected": -229.32122802734375, + "loss": 0.11097535490989685, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.964521408081055, + "rewards/margins": 4.160644054412842, + "rewards/rejected": -9.125164985656738, + "step": 552 + }, + { + "epoch": 1.7838709677419353, + "grad_norm": 0.8820840716362, + "learning_rate": 1.4694107872874174e-07, + "logits/chosen": -1.9263832569122314, + "logits/rejected": -1.649777889251709, + "logps/chosen": -168.28428649902344, + "logps/rejected": -234.61843872070312, + "loss": 0.18569695949554443, + "rewards/accuracies": 0.90625, + "rewards/chosen": -5.886368751525879, + "rewards/margins": 4.445061683654785, + "rewards/rejected": -10.331430435180664, + "step": 553 + }, + { + "epoch": 1.7870967741935484, + "grad_norm": 0.47653189301490784, + "learning_rate": 1.4269235117959218e-07, + "logits/chosen": -2.272602081298828, + "logits/rejected": -2.015300750732422, + "logps/chosen": -148.469970703125, + "logps/rejected": -214.323974609375, + "loss": 0.11035692691802979, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.494589328765869, + "rewards/margins": 3.9238247871398926, + "rewards/rejected": -8.418414115905762, + "step": 554 + }, + { + "epoch": 1.7903225806451613, + "grad_norm": 0.48935648798942566, + "learning_rate": 1.3850414819903235e-07, + "logits/chosen": -2.5061533451080322, + "logits/rejected": -2.0984086990356445, + "logps/chosen": -127.44873046875, + "logps/rejected": -191.14419555664062, + "loss": 0.11538471281528473, + "rewards/accuracies": 0.96875, + "rewards/chosen": -3.8038997650146484, + "rewards/margins": 4.107954978942871, + "rewards/rejected": -7.9118547439575195, + "step": 555 + }, + { + "epoch": 1.793548387096774, + "grad_norm": 0.5510221719741821, + "learning_rate": 1.3437657732040783e-07, + "logits/chosen": -2.023258924484253, + "logits/rejected": -1.7574663162231445, + "logps/chosen": -152.7783203125, + "logps/rejected": -229.17276000976562, + "loss": 0.09697185456752777, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.315288543701172, + "rewards/margins": 4.344141006469727, + "rewards/rejected": -9.659428596496582, + "step": 556 + }, + { + "epoch": 1.7967741935483872, + "grad_norm": 1.3715373277664185, + "learning_rate": 1.3030974452031832e-07, + "logits/chosen": -2.4522571563720703, + "logits/rejected": -2.184933662414551, + "logps/chosen": -160.92742919921875, + "logps/rejected": -226.46864318847656, + "loss": 0.19726327061653137, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.374527931213379, + "rewards/margins": 4.175776958465576, + "rewards/rejected": -9.550305366516113, + "step": 557 + }, + { + "epoch": 1.8, + "grad_norm": 1.3102973699569702, + "learning_rate": 1.2630375421589374e-07, + "logits/chosen": -2.0639586448669434, + "logits/rejected": -1.6796153783798218, + "logps/chosen": -159.63888549804688, + "logps/rejected": -221.36654663085938, + "loss": 0.3832627832889557, + "rewards/accuracies": 0.78125, + "rewards/chosen": -5.068192958831787, + "rewards/margins": 3.830645799636841, + "rewards/rejected": -8.898838996887207, + "step": 558 + }, + { + "epoch": 1.803225806451613, + "grad_norm": 0.4269888699054718, + "learning_rate": 1.223587092621162e-07, + "logits/chosen": -2.1528987884521484, + "logits/rejected": -1.7997894287109375, + "logps/chosen": -169.46499633789062, + "logps/rejected": -241.46487426757812, + "loss": 0.09280017018318176, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.810900688171387, + "rewards/margins": 4.045185089111328, + "rewards/rejected": -9.856085777282715, + "step": 559 + }, + { + "epoch": 1.8064516129032258, + "grad_norm": 0.4622645974159241, + "learning_rate": 1.1847471094917711e-07, + "logits/chosen": -2.6282200813293457, + "logits/rejected": -2.194746732711792, + "logps/chosen": -116.16856384277344, + "logps/rejected": -188.41812133789062, + "loss": 0.11614560335874557, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.8644115924835205, + "rewards/margins": 4.392536640167236, + "rewards/rejected": -8.256948471069336, + "step": 560 + }, + { + "epoch": 1.8096774193548386, + "grad_norm": 0.8068135976791382, + "learning_rate": 1.1465185899987797e-07, + "logits/chosen": -2.1790473461151123, + "logits/rejected": -1.7810087203979492, + "logps/chosen": -153.4788818359375, + "logps/rejected": -218.73696899414062, + "loss": 0.22984439134597778, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.740139484405518, + "rewards/margins": 3.968252658843994, + "rewards/rejected": -8.708392143249512, + "step": 561 + }, + { + "epoch": 1.8129032258064517, + "grad_norm": 0.5755476355552673, + "learning_rate": 1.1089025156706928e-07, + "logits/chosen": -2.1778645515441895, + "logits/rejected": -1.817617654800415, + "logps/chosen": -144.53890991210938, + "logps/rejected": -222.02716064453125, + "loss": 0.10252237319946289, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.7554545402526855, + "rewards/margins": 4.461153984069824, + "rewards/rejected": -9.216609001159668, + "step": 562 + }, + { + "epoch": 1.8161290322580645, + "grad_norm": 0.7859362363815308, + "learning_rate": 1.0718998523113006e-07, + "logits/chosen": -2.283513307571411, + "logits/rejected": -1.9074876308441162, + "logps/chosen": -162.03208923339844, + "logps/rejected": -224.34286499023438, + "loss": 0.16266335546970367, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.444174766540527, + "rewards/margins": 4.131298542022705, + "rewards/rejected": -9.57547378540039, + "step": 563 + }, + { + "epoch": 1.8193548387096774, + "grad_norm": 0.5560816526412964, + "learning_rate": 1.0355115499748936e-07, + "logits/chosen": -2.54559588432312, + "logits/rejected": -2.2199432849884033, + "logps/chosen": -146.66653442382812, + "logps/rejected": -212.36798095703125, + "loss": 0.11472531408071518, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.957542419433594, + "rewards/margins": 4.0164031982421875, + "rewards/rejected": -8.973945617675781, + "step": 564 + }, + { + "epoch": 1.8225806451612905, + "grad_norm": 0.7271311283111572, + "learning_rate": 9.997385429418555e-08, + "logits/chosen": -2.290648937225342, + "logits/rejected": -2.024205207824707, + "logps/chosen": -164.578857421875, + "logps/rejected": -206.93765258789062, + "loss": 0.20374655723571777, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.871495246887207, + "rewards/margins": 2.7768454551696777, + "rewards/rejected": -8.648341178894043, + "step": 565 + }, + { + "epoch": 1.8258064516129031, + "grad_norm": 0.42589902877807617, + "learning_rate": 9.645817496946902e-08, + "logits/chosen": -2.3634767532348633, + "logits/rejected": -1.9045495986938477, + "logps/chosen": -151.49630737304688, + "logps/rejected": -217.68666076660156, + "loss": 0.08909858763217926, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.860687255859375, + "rewards/margins": 4.105985641479492, + "rewards/rejected": -8.966672897338867, + "step": 566 + }, + { + "epoch": 1.8290322580645162, + "grad_norm": 0.8605185151100159, + "learning_rate": 9.300420728944132e-08, + "logits/chosen": -2.215791702270508, + "logits/rejected": -1.8884910345077515, + "logps/chosen": -168.70660400390625, + "logps/rejected": -231.85137939453125, + "loss": 0.2555576264858246, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.966517925262451, + "rewards/margins": 3.8281943798065186, + "rewards/rejected": -8.79471206665039, + "step": 567 + }, + { + "epoch": 1.832258064516129, + "grad_norm": 0.7926627397537231, + "learning_rate": 8.961203993574197e-08, + "logits/chosen": -2.421079635620117, + "logits/rejected": -2.14444637298584, + "logps/chosen": -165.5140380859375, + "logps/rejected": -218.92263793945312, + "loss": 0.18454545736312866, + "rewards/accuracies": 0.90625, + "rewards/chosen": -5.383303642272949, + "rewards/margins": 3.148193359375, + "rewards/rejected": -8.53149700164795, + "step": 568 + }, + { + "epoch": 1.835483870967742, + "grad_norm": 0.4896734654903412, + "learning_rate": 8.628176000326615e-08, + "logits/chosen": -2.4777512550354004, + "logits/rejected": -2.0716423988342285, + "logps/chosen": -161.4052734375, + "logps/rejected": -238.86143493652344, + "loss": 0.10428795218467712, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.8029398918151855, + "rewards/margins": 4.6206560134887695, + "rewards/rejected": -10.423595428466797, + "step": 569 + }, + { + "epoch": 1.838709677419355, + "grad_norm": 0.552992045879364, + "learning_rate": 8.301345299793374e-08, + "logits/chosen": -2.6197102069854736, + "logits/rejected": -2.2024412155151367, + "logps/chosen": -136.18997192382812, + "logps/rejected": -202.0336151123047, + "loss": 0.17170380055904388, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.436511516571045, + "rewards/margins": 4.078784942626953, + "rewards/rejected": -8.515296936035156, + "step": 570 + }, + { + "epoch": 1.8419354838709676, + "grad_norm": 0.8881261944770813, + "learning_rate": 7.980720283448957e-08, + "logits/chosen": -1.9090025424957275, + "logits/rejected": -1.6317459344863892, + "logps/chosen": -166.48141479492188, + "logps/rejected": -233.93344116210938, + "loss": 0.21583732962608337, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.640261650085449, + "rewards/margins": 4.06472635269165, + "rewards/rejected": -9.704988479614258, + "step": 571 + }, + { + "epoch": 1.8451612903225807, + "grad_norm": 0.5913529396057129, + "learning_rate": 7.66630918343511e-08, + "logits/chosen": -2.508117198944092, + "logits/rejected": -2.0965323448181152, + "logps/chosen": -141.76434326171875, + "logps/rejected": -216.82269287109375, + "loss": 0.12084457278251648, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.573372840881348, + "rewards/margins": 4.82564115524292, + "rewards/rejected": -9.399014472961426, + "step": 572 + }, + { + "epoch": 1.8483870967741935, + "grad_norm": 0.5654717087745667, + "learning_rate": 7.358120072349594e-08, + "logits/chosen": -2.3709940910339355, + "logits/rejected": -2.038872718811035, + "logps/chosen": -153.72909545898438, + "logps/rejected": -225.5278778076172, + "loss": 0.13232877850532532, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.121915340423584, + "rewards/margins": 4.319830894470215, + "rewards/rejected": -9.44174575805664, + "step": 573 + }, + { + "epoch": 1.8516129032258064, + "grad_norm": 0.8132126331329346, + "learning_rate": 7.056160863038564e-08, + "logits/chosen": -2.2554118633270264, + "logits/rejected": -1.993767261505127, + "logps/chosen": -158.86770629882812, + "logps/rejected": -216.81568908691406, + "loss": 0.1489611566066742, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.455775737762451, + "rewards/margins": 3.815854072570801, + "rewards/rejected": -9.27163028717041, + "step": 574 + }, + { + "epoch": 1.8548387096774195, + "grad_norm": 0.6348745822906494, + "learning_rate": 6.760439308393763e-08, + "logits/chosen": -2.1734232902526855, + "logits/rejected": -1.7729003429412842, + "logps/chosen": -137.09434509277344, + "logps/rejected": -200.48419189453125, + "loss": 0.16260844469070435, + "rewards/accuracies": 0.90625, + "rewards/chosen": -4.978243350982666, + "rewards/margins": 4.0154242515563965, + "rewards/rejected": -8.993667602539062, + "step": 575 + }, + { + "epoch": 1.8580645161290321, + "grad_norm": 0.5179709196090698, + "learning_rate": 6.470963001153268e-08, + "logits/chosen": -2.1407740116119385, + "logits/rejected": -1.700477123260498, + "logps/chosen": -152.31427001953125, + "logps/rejected": -230.4638671875, + "loss": 0.10425004363059998, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.169086933135986, + "rewards/margins": 4.689014434814453, + "rewards/rejected": -9.858100891113281, + "step": 576 + }, + { + "epoch": 1.8612903225806452, + "grad_norm": 0.8017939329147339, + "learning_rate": 6.187739373706508e-08, + "logits/chosen": -2.4910433292388916, + "logits/rejected": -2.189812660217285, + "logps/chosen": -161.86709594726562, + "logps/rejected": -220.63807678222656, + "loss": 0.20899598300457, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.962458610534668, + "rewards/margins": 3.5529963970184326, + "rewards/rejected": -8.515454292297363, + "step": 577 + }, + { + "epoch": 1.864516129032258, + "grad_norm": 0.8252779841423035, + "learning_rate": 5.9107756979036035e-08, + "logits/chosen": -1.9336351156234741, + "logits/rejected": -1.6209434270858765, + "logps/chosen": -185.7202606201172, + "logps/rejected": -260.7825927734375, + "loss": 0.13948240876197815, + "rewards/accuracies": 0.9375, + "rewards/chosen": -7.039210319519043, + "rewards/margins": 4.54232120513916, + "rewards/rejected": -11.581531524658203, + "step": 578 + }, + { + "epoch": 1.867741935483871, + "grad_norm": 0.4776524603366852, + "learning_rate": 5.6400790848686325e-08, + "logits/chosen": -2.4437499046325684, + "logits/rejected": -2.0726723670959473, + "logps/chosen": -157.41360473632812, + "logps/rejected": -224.64364624023438, + "loss": 0.12375371158123016, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.997191905975342, + "rewards/margins": 4.007498264312744, + "rewards/rejected": -9.004690170288086, + "step": 579 + }, + { + "epoch": 1.870967741935484, + "grad_norm": 0.9288763999938965, + "learning_rate": 5.3756564848168325e-08, + "logits/chosen": -2.0364584922790527, + "logits/rejected": -1.7277495861053467, + "logps/chosen": -169.73043823242188, + "logps/rejected": -241.12673950195312, + "loss": 0.17077437043190002, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.451491832733154, + "rewards/margins": 4.377066135406494, + "rewards/rejected": -9.828557968139648, + "step": 580 + }, + { + "epoch": 1.8741935483870966, + "grad_norm": 0.4808553159236908, + "learning_rate": 5.117514686876379e-08, + "logits/chosen": -2.3133416175842285, + "logits/rejected": -1.9516518115997314, + "logps/chosen": -145.88031005859375, + "logps/rejected": -223.6663818359375, + "loss": 0.0891776904463768, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.209961414337158, + "rewards/margins": 4.986030578613281, + "rewards/rejected": -10.195992469787598, + "step": 581 + }, + { + "epoch": 1.8774193548387097, + "grad_norm": 0.8164253234863281, + "learning_rate": 4.8656603189140564e-08, + "logits/chosen": -2.275332450866699, + "logits/rejected": -1.8718898296356201, + "logps/chosen": -138.21881103515625, + "logps/rejected": -209.18975830078125, + "loss": 0.18211618065834045, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.704702377319336, + "rewards/margins": 4.319535732269287, + "rewards/rejected": -9.024238586425781, + "step": 582 + }, + { + "epoch": 1.8806451612903226, + "grad_norm": 0.5213284492492676, + "learning_rate": 4.620099847364973e-08, + "logits/chosen": -2.350086212158203, + "logits/rejected": -1.9384825229644775, + "logps/chosen": -149.28497314453125, + "logps/rejected": -218.31735229492188, + "loss": 0.0979819968342781, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.209005355834961, + "rewards/margins": 3.8068172931671143, + "rewards/rejected": -9.015822410583496, + "step": 583 + }, + { + "epoch": 1.8838709677419354, + "grad_norm": 0.5111794471740723, + "learning_rate": 4.380839577066587e-08, + "logits/chosen": -2.0842952728271484, + "logits/rejected": -1.742661952972412, + "logps/chosen": -152.95252990722656, + "logps/rejected": -217.0407257080078, + "loss": 0.10743395984172821, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.22687292098999, + "rewards/margins": 3.8219170570373535, + "rewards/rejected": -9.048789978027344, + "step": 584 + }, + { + "epoch": 1.8870967741935485, + "grad_norm": 0.6543387174606323, + "learning_rate": 4.147885651096861e-08, + "logits/chosen": -2.292243480682373, + "logits/rejected": -1.9942082166671753, + "logps/chosen": -148.08607482910156, + "logps/rejected": -208.68533325195312, + "loss": 0.14559054374694824, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.75385856628418, + "rewards/margins": 3.6896753311157227, + "rewards/rejected": -8.443533897399902, + "step": 585 + }, + { + "epoch": 1.8903225806451613, + "grad_norm": 0.5183025002479553, + "learning_rate": 3.9212440506164465e-08, + "logits/chosen": -2.3551645278930664, + "logits/rejected": -1.9850412607192993, + "logps/chosen": -156.36065673828125, + "logps/rejected": -223.7450714111328, + "loss": 0.104829341173172, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.985340595245361, + "rewards/margins": 3.9502832889556885, + "rewards/rejected": -8.935624122619629, + "step": 586 + }, + { + "epoch": 1.8935483870967742, + "grad_norm": 0.5007240176200867, + "learning_rate": 3.7009205947153294e-08, + "logits/chosen": -2.016148090362549, + "logits/rejected": -1.6376087665557861, + "logps/chosen": -167.86480712890625, + "logps/rejected": -239.8389129638672, + "loss": 0.07380706071853638, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.1963348388671875, + "rewards/margins": 4.371429920196533, + "rewards/rejected": -9.567764282226562, + "step": 587 + }, + { + "epoch": 1.896774193548387, + "grad_norm": 0.6565898060798645, + "learning_rate": 3.486920940263094e-08, + "logits/chosen": -2.4202284812927246, + "logits/rejected": -2.0095739364624023, + "logps/chosen": -150.2890167236328, + "logps/rejected": -225.1298828125, + "loss": 0.12143023312091827, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.002315044403076, + "rewards/margins": 4.677301406860352, + "rewards/rejected": -9.679616928100586, + "step": 588 + }, + { + "epoch": 1.9, + "grad_norm": 0.7280709147453308, + "learning_rate": 3.279250581763982e-08, + "logits/chosen": -2.5148582458496094, + "logits/rejected": -2.265883207321167, + "logps/chosen": -165.6021728515625, + "logps/rejected": -224.98973083496094, + "loss": 0.19175627827644348, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.013904571533203, + "rewards/margins": 3.702694892883301, + "rewards/rejected": -8.716598510742188, + "step": 589 + }, + { + "epoch": 1.903225806451613, + "grad_norm": 0.8753358721733093, + "learning_rate": 3.077914851215585e-08, + "logits/chosen": -2.1750917434692383, + "logits/rejected": -1.677757740020752, + "logps/chosen": -157.30673217773438, + "logps/rejected": -259.028076171875, + "loss": 0.18018171191215515, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.870169639587402, + "rewards/margins": 5.68065881729126, + "rewards/rejected": -10.550827980041504, + "step": 590 + }, + { + "epoch": 1.9064516129032258, + "grad_norm": 0.6306760311126709, + "learning_rate": 2.8829189179721552e-08, + "logits/chosen": -2.1184749603271484, + "logits/rejected": -1.7403565645217896, + "logps/chosen": -165.62957763671875, + "logps/rejected": -245.212158203125, + "loss": 0.10819903016090393, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.537095069885254, + "rewards/margins": 5.161804676055908, + "rewards/rejected": -10.69890022277832, + "step": 591 + }, + { + "epoch": 1.9096774193548387, + "grad_norm": 0.6319319009780884, + "learning_rate": 2.6942677886117597e-08, + "logits/chosen": -2.0213987827301025, + "logits/rejected": -1.6281509399414062, + "logps/chosen": -151.06640625, + "logps/rejected": -223.082275390625, + "loss": 0.14870238304138184, + "rewards/accuracies": 0.90625, + "rewards/chosen": -4.421160697937012, + "rewards/margins": 4.47739315032959, + "rewards/rejected": -8.898553848266602, + "step": 592 + }, + { + "epoch": 1.9129032258064518, + "grad_norm": 0.5766472816467285, + "learning_rate": 2.5119663068077227e-08, + "logits/chosen": -2.718550443649292, + "logits/rejected": -2.379741668701172, + "logps/chosen": -138.78890991210938, + "logps/rejected": -220.3733673095703, + "loss": 0.12025254219770432, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.328763961791992, + "rewards/margins": 5.262520790100098, + "rewards/rejected": -9.59128475189209, + "step": 593 + }, + { + "epoch": 1.9161290322580644, + "grad_norm": 0.865578830242157, + "learning_rate": 2.3360191532043053e-08, + "logits/chosen": -2.043041229248047, + "logits/rejected": -1.6594171524047852, + "logps/chosen": -142.37728881835938, + "logps/rejected": -221.1278076171875, + "loss": 0.18678346276283264, + "rewards/accuracies": 0.90625, + "rewards/chosen": -5.289588928222656, + "rewards/margins": 4.628904342651367, + "rewards/rejected": -9.91849422454834, + "step": 594 + }, + { + "epoch": 1.9193548387096775, + "grad_norm": 0.5385801196098328, + "learning_rate": 2.1664308452965798e-08, + "logits/chosen": -2.386172294616699, + "logits/rejected": -2.0510740280151367, + "logps/chosen": -153.6859130859375, + "logps/rejected": -218.6775360107422, + "loss": 0.12221887707710266, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.658649444580078, + "rewards/margins": 4.1918463706970215, + "rewards/rejected": -9.850496292114258, + "step": 595 + }, + { + "epoch": 1.9225806451612903, + "grad_norm": 0.31810611486434937, + "learning_rate": 2.0032057373142453e-08, + "logits/chosen": -2.2256276607513428, + "logits/rejected": -1.9017874002456665, + "logps/chosen": -160.43661499023438, + "logps/rejected": -223.5597381591797, + "loss": 0.060307130217552185, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.350681781768799, + "rewards/margins": 4.087976455688477, + "rewards/rejected": -9.438658714294434, + "step": 596 + }, + { + "epoch": 1.9258064516129032, + "grad_norm": 0.6564087867736816, + "learning_rate": 1.8463480201101337e-08, + "logits/chosen": -2.4924027919769287, + "logits/rejected": -2.1421899795532227, + "logps/chosen": -145.89862060546875, + "logps/rejected": -214.23382568359375, + "loss": 0.15689480304718018, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.8156819343566895, + "rewards/margins": 4.231740951538086, + "rewards/rejected": -9.047422409057617, + "step": 597 + }, + { + "epoch": 1.9290322580645163, + "grad_norm": 0.7558223605155945, + "learning_rate": 1.6958617210522942e-08, + "logits/chosen": -2.3059439659118652, + "logits/rejected": -1.9703770875930786, + "logps/chosen": -179.74252319335938, + "logps/rejected": -242.72195434570312, + "loss": 0.15578067302703857, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.681976318359375, + "rewards/margins": 4.2997260093688965, + "rewards/rejected": -9.98170280456543, + "step": 598 + }, + { + "epoch": 1.932258064516129, + "grad_norm": 0.6187397241592407, + "learning_rate": 1.5517507039208568e-08, + "logits/chosen": -1.8563120365142822, + "logits/rejected": -1.6304610967636108, + "logps/chosen": -172.39527893066406, + "logps/rejected": -239.09451293945312, + "loss": 0.12176580727100372, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.339761734008789, + "rewards/margins": 3.85723614692688, + "rewards/rejected": -9.19699764251709, + "step": 599 + }, + { + "epoch": 1.935483870967742, + "grad_norm": 0.5930323600769043, + "learning_rate": 1.4140186688086365e-08, + "logits/chosen": -2.1559739112854004, + "logits/rejected": -1.8187296390533447, + "logps/chosen": -153.97042846679688, + "logps/rejected": -229.720947265625, + "loss": 0.10695869475603104, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.2712507247924805, + "rewards/margins": 4.516932487487793, + "rewards/rejected": -9.788183212280273, + "step": 600 + }, + { + "epoch": 1.9387096774193548, + "grad_norm": 0.41866037249565125, + "learning_rate": 1.2826691520262114e-08, + "logits/chosen": -2.31526517868042, + "logits/rejected": -2.0141377449035645, + "logps/chosen": -157.0247802734375, + "logps/rejected": -221.9925079345703, + "loss": 0.11068223416805267, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.260191917419434, + "rewards/margins": 3.905627489089966, + "rewards/rejected": -9.16581916809082, + "step": 601 + }, + { + "epoch": 1.9419354838709677, + "grad_norm": 1.0719863176345825, + "learning_rate": 1.1577055260111603e-08, + "logits/chosen": -2.173069715499878, + "logits/rejected": -1.8304716348648071, + "logps/chosen": -166.02662658691406, + "logps/rejected": -226.69769287109375, + "loss": 0.28377968072891235, + "rewards/accuracies": 0.84375, + "rewards/chosen": -5.536633014678955, + "rewards/margins": 3.4802064895629883, + "rewards/rejected": -9.016839981079102, + "step": 602 + }, + { + "epoch": 1.9451612903225808, + "grad_norm": 0.8208643794059753, + "learning_rate": 1.0391309992413834e-08, + "logits/chosen": -2.191689968109131, + "logits/rejected": -1.8869071006774902, + "logps/chosen": -150.7781219482422, + "logps/rejected": -217.19967651367188, + "loss": 0.20699000358581543, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.127926826477051, + "rewards/margins": 3.8796911239624023, + "rewards/rejected": -9.007617950439453, + "step": 603 + }, + { + "epoch": 1.9483870967741934, + "grad_norm": 0.913461446762085, + "learning_rate": 9.269486161528063e-09, + "logits/chosen": -2.360179901123047, + "logits/rejected": -1.9940139055252075, + "logps/chosen": -161.97451782226562, + "logps/rejected": -243.32638549804688, + "loss": 0.1867237687110901, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.740466594696045, + "rewards/margins": 4.959558486938477, + "rewards/rejected": -10.700023651123047, + "step": 604 + }, + { + "epoch": 1.9516129032258065, + "grad_norm": 0.4413685202598572, + "learning_rate": 8.211612570611927e-09, + "logits/chosen": -2.2911806106567383, + "logits/rejected": -1.9480736255645752, + "logps/chosen": -145.26788330078125, + "logps/rejected": -212.7537384033203, + "loss": 0.099663145840168, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.860638618469238, + "rewards/margins": 3.9052321910858154, + "rewards/rejected": -8.765871047973633, + "step": 605 + }, + { + "epoch": 1.9548387096774194, + "grad_norm": 0.7804948091506958, + "learning_rate": 7.217716380881479e-09, + "logits/chosen": -2.3659746646881104, + "logits/rejected": -1.9991470575332642, + "logps/chosen": -147.31109619140625, + "logps/rejected": -211.473876953125, + "loss": 0.21457377076148987, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.22733211517334, + "rewards/margins": 4.1717915534973145, + "rewards/rejected": -8.399125099182129, + "step": 606 + }, + { + "epoch": 1.9580645161290322, + "grad_norm": 0.29649555683135986, + "learning_rate": 6.287823110914248e-09, + "logits/chosen": -1.943648099899292, + "logits/rejected": -1.5592997074127197, + "logps/chosen": -172.8476104736328, + "logps/rejected": -252.05905151367188, + "loss": 0.0389375314116478, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.832505226135254, + "rewards/margins": 5.257543563842773, + "rewards/rejected": -11.090048789978027, + "step": 607 + }, + { + "epoch": 1.9612903225806453, + "grad_norm": 0.5518164038658142, + "learning_rate": 5.4219566359939305e-09, + "logits/chosen": -2.3233723640441895, + "logits/rejected": -2.0299534797668457, + "logps/chosen": -174.52838134765625, + "logps/rejected": -244.19309997558594, + "loss": 0.12132851779460907, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.807446479797363, + "rewards/margins": 4.241092681884766, + "rewards/rejected": -10.048539161682129, + "step": 608 + }, + { + "epoch": 1.964516129032258, + "grad_norm": 0.5421614646911621, + "learning_rate": 4.620139187497818e-09, + "logits/chosen": -2.4302639961242676, + "logits/rejected": -2.0234873294830322, + "logps/chosen": -138.5733642578125, + "logps/rejected": -205.07244873046875, + "loss": 0.09998336434364319, + "rewards/accuracies": 0.96875, + "rewards/chosen": -3.7667317390441895, + "rewards/margins": 4.176835060119629, + "rewards/rejected": -7.94356632232666, + "step": 609 + }, + { + "epoch": 1.967741935483871, + "grad_norm": 1.0250368118286133, + "learning_rate": 3.882391352324766e-09, + "logits/chosen": -2.216160297393799, + "logits/rejected": -1.7999509572982788, + "logps/chosen": -148.7297821044922, + "logps/rejected": -213.1514892578125, + "loss": 0.2962940037250519, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.382504940032959, + "rewards/margins": 4.019059181213379, + "rewards/rejected": -9.40156364440918, + "step": 610 + }, + { + "epoch": 1.9709677419354839, + "grad_norm": 0.4548644423484802, + "learning_rate": 3.208732072368104e-09, + "logits/chosen": -2.128699779510498, + "logits/rejected": -1.7592086791992188, + "logps/chosen": -170.3048858642578, + "logps/rejected": -246.21893310546875, + "loss": 0.06920068711042404, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.777827262878418, + "rewards/margins": 4.816149711608887, + "rewards/rejected": -10.593976974487305, + "step": 611 + }, + { + "epoch": 1.9741935483870967, + "grad_norm": 0.6790652871131897, + "learning_rate": 2.5991786440282553e-09, + "logits/chosen": -2.0420777797698975, + "logits/rejected": -1.7261956930160522, + "logps/chosen": -162.41305541992188, + "logps/rejected": -225.78868103027344, + "loss": 0.11259308457374573, + "rewards/accuracies": 0.96875, + "rewards/chosen": -5.203654766082764, + "rewards/margins": 3.9687769412994385, + "rewards/rejected": -9.172431945800781, + "step": 612 + }, + { + "epoch": 1.9774193548387098, + "grad_norm": 0.4410402476787567, + "learning_rate": 2.0537467177692007e-09, + "logits/chosen": -2.582838296890259, + "logits/rejected": -2.1445116996765137, + "logps/chosen": -147.5789794921875, + "logps/rejected": -225.4355010986328, + "loss": 0.08542226999998093, + "rewards/accuracies": 0.96875, + "rewards/chosen": -4.422325134277344, + "rewards/margins": 5.232996940612793, + "rewards/rejected": -9.655322074890137, + "step": 613 + }, + { + "epoch": 1.9806451612903224, + "grad_norm": 0.8383966684341431, + "learning_rate": 1.5724502977162993e-09, + "logits/chosen": -2.198012590408325, + "logits/rejected": -1.8719542026519775, + "logps/chosen": -147.04000854492188, + "logps/rejected": -218.75167846679688, + "loss": 0.15390494465827942, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.836050033569336, + "rewards/margins": 4.455644607543945, + "rewards/rejected": -9.291694641113281, + "step": 614 + }, + { + "epoch": 1.9838709677419355, + "grad_norm": 0.7288622260093689, + "learning_rate": 1.1553017412971323e-09, + "logits/chosen": -2.1771492958068848, + "logits/rejected": -1.8729420900344849, + "logps/chosen": -152.79205322265625, + "logps/rejected": -225.03131103515625, + "loss": 0.16490189731121063, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.371936321258545, + "rewards/margins": 4.25457239151001, + "rewards/rejected": -9.626508712768555, + "step": 615 + }, + { + "epoch": 1.9870967741935484, + "grad_norm": 0.8851467370986938, + "learning_rate": 8.023117589237017e-10, + "logits/chosen": -2.1717655658721924, + "logits/rejected": -1.710573434829712, + "logps/chosen": -156.8731689453125, + "logps/rejected": -235.35928344726562, + "loss": 0.2400968074798584, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.178656101226807, + "rewards/margins": 4.434657573699951, + "rewards/rejected": -9.613313674926758, + "step": 616 + }, + { + "epoch": 1.9903225806451612, + "grad_norm": 1.0094678401947021, + "learning_rate": 5.13489413717927e-10, + "logits/chosen": -2.0871119499206543, + "logits/rejected": -1.812303900718689, + "logps/chosen": -159.29959106445312, + "logps/rejected": -223.72409057617188, + "loss": 0.2709569036960602, + "rewards/accuracies": 0.84375, + "rewards/chosen": -5.2238240242004395, + "rewards/margins": 3.8848769664764404, + "rewards/rejected": -9.1087007522583, + "step": 617 + }, + { + "epoch": 1.9935483870967743, + "grad_norm": 0.7299178242683411, + "learning_rate": 2.8884212127849867e-10, + "logits/chosen": -2.3765387535095215, + "logits/rejected": -1.9941563606262207, + "logps/chosen": -150.97113037109375, + "logps/rejected": -219.23605346679688, + "loss": 0.20615136623382568, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.328438758850098, + "rewards/margins": 3.9626264572143555, + "rewards/rejected": -9.291065216064453, + "step": 618 + }, + { + "epoch": 1.9967741935483871, + "grad_norm": 0.5003857612609863, + "learning_rate": 1.2837564949103043e-10, + "logits/chosen": -2.0507559776306152, + "logits/rejected": -1.7051759958267212, + "logps/chosen": -150.3610382080078, + "logps/rejected": -212.7919464111328, + "loss": 0.11825229227542877, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.341998100280762, + "rewards/margins": 3.884814500808716, + "rewards/rejected": -8.226812362670898, + "step": 619 + }, + { + "epoch": 2.0, + "grad_norm": 0.8774574995040894, + "learning_rate": 3.2094118379288885e-11, + "logits/chosen": -2.027413845062256, + "logits/rejected": -1.7117795944213867, + "logps/chosen": -164.06918334960938, + "logps/rejected": -234.24671936035156, + "loss": 0.17131692171096802, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.142941474914551, + "rewards/margins": 4.366891860961914, + "rewards/rejected": -9.509833335876465, + "step": 620 + }, + { + "epoch": 2.0, + "step": 620, + "total_flos": 1.1218583388589916e+18, + "train_loss": 0.2873070158064365, + "train_runtime": 4919.0164, + "train_samples_per_second": 4.033, + "train_steps_per_second": 0.126 + } + ], + "logging_steps": 1, + "max_steps": 620, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 5000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 1.1218583388589916e+18, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +}