diff --git "a/trainer_state.json" "b/trainer_state.json"
deleted file mode 100644--- "a/trainer_state.json"
+++ /dev/null
@@ -1,9343 +0,0 @@
-{
-  "best_global_step": null,
-  "best_metric": null,
-  "best_model_checkpoint": null,
-  "epoch": 2.0,
-  "eval_steps": 500,
-  "global_step": 620,
-  "is_hyper_param_search": false,
-  "is_local_process_zero": true,
-  "is_world_process_zero": true,
-  "log_history": [
-    {
-      "epoch": 0.0032258064516129032,
-      "grad_norm": 1.0198665857315063,
-      "learning_rate": 5e-06,
-      "logits/chosen": -3.4055747985839844,
-      "logits/rejected": -3.2500038146972656,
-      "logps/chosen": -85.92189025878906,
-      "logps/rejected": -115.17630004882812,
-      "loss": 0.6931471824645996,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": 0.0,
-      "rewards/margins": 0.0,
-      "rewards/rejected": 0.0,
-      "step": 1
-    },
-    {
-      "epoch": 0.0064516129032258064,
-      "grad_norm": 1.12330162525177,
-      "learning_rate": 4.999967905881621e-06,
-      "logits/chosen": -3.139765739440918,
-      "logits/rejected": -2.9487788677215576,
-      "logps/chosen": -107.27665710449219,
-      "logps/rejected": -138.18399047851562,
-      "loss": 0.7021108269691467,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": -0.01106848195195198,
-      "rewards/margins": -0.01585940644145012,
-      "rewards/rejected": 0.004790925420820713,
-      "step": 2
-    },
-    {
-      "epoch": 0.00967741935483871,
-      "grad_norm": 1.035811424255371,
-      "learning_rate": 4.99987162435051e-06,
-      "logits/chosen": -3.148529052734375,
-      "logits/rejected": -3.0062148571014404,
-      "logps/chosen": -103.82777404785156,
-      "logps/rejected": -128.03005981445312,
-      "loss": 0.668667197227478,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.03935158997774124,
-      "rewards/margins": 0.05168798193335533,
-      "rewards/rejected": -0.012336388230323792,
-      "step": 3
-    },
-    {
-      "epoch": 0.012903225806451613,
-      "grad_norm": 1.1256532669067383,
-      "learning_rate": 4.999711157878722e-06,
-      "logits/chosen": -2.938189744949341,
-      "logits/rejected": -2.779592514038086,
-      "logps/chosen": -100.74897003173828,
-      "logps/rejected": -134.83859252929688,
-      "loss": 0.6965321898460388,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.0025862313341349363,
-      "rewards/margins": -0.00128110870718956,
-      "rewards/rejected": 0.0038673398084938526,
-      "step": 4
-    },
-    {
-      "epoch": 0.016129032258064516,
-      "grad_norm": 1.0809041261672974,
-      "learning_rate": 4.999486510586282e-06,
-      "logits/chosen": -2.927344799041748,
-      "logits/rejected": -2.8265066146850586,
-      "logps/chosen": -109.85308837890625,
-      "logps/rejected": -142.212646484375,
-      "loss": 0.648101806640625,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.011704119853675365,
-      "rewards/margins": 0.09639576077461243,
-      "rewards/rejected": -0.08469165116548538,
-      "step": 5
-    },
-    {
-      "epoch": 0.01935483870967742,
-      "grad_norm": 1.0711078643798828,
-      "learning_rate": 4.999197688241076e-06,
-      "logits/chosen": -3.085522413253784,
-      "logits/rejected": -2.979081153869629,
-      "logps/chosen": -111.31352233886719,
-      "logps/rejected": -131.05545043945312,
-      "loss": 0.6814134120941162,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.013973352499306202,
-      "rewards/margins": 0.031338244676589966,
-      "rewards/rejected": -0.01736488938331604,
-      "step": 6
-    },
-    {
-      "epoch": 0.02258064516129032,
-      "grad_norm": 1.1095834970474243,
-      "learning_rate": 4.998844698258704e-06,
-      "logits/chosen": -2.7590060234069824,
-      "logits/rejected": -2.5014760494232178,
-      "logps/chosen": -102.21183776855469,
-      "logps/rejected": -141.93539428710938,
-      "loss": 0.6776243448257446,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -0.013474291190505028,
-      "rewards/margins": 0.0413118414580822,
-      "rewards/rejected": -0.054786134511232376,
-      "step": 7
-    },
-    {
-      "epoch": 0.025806451612903226,
-      "grad_norm": 0.9900579452514648,
-      "learning_rate": 4.998427549702284e-06,
-      "logits/chosen": -3.2602813243865967,
-      "logits/rejected": -3.1628756523132324,
-      "logps/chosen": -93.34729766845703,
-      "logps/rejected": -111.89982604980469,
-      "loss": 0.673986554145813,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.03970911726355553,
-      "rewards/margins": 0.04514574632048607,
-      "rewards/rejected": -0.08485487103462219,
-      "step": 8
-    },
-    {
-      "epoch": 0.02903225806451613,
-      "grad_norm": 1.0532326698303223,
-      "learning_rate": 4.9979462532822305e-06,
-      "logits/chosen": -3.270956516265869,
-      "logits/rejected": -3.086606502532959,
-      "logps/chosen": -101.70204162597656,
-      "logps/rejected": -145.21560668945312,
-      "loss": 0.6543633937835693,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.07679509371519089,
-      "rewards/margins": 0.0888080894947052,
-      "rewards/rejected": -0.16560319066047668,
-      "step": 9
-    },
-    {
-      "epoch": 0.03225806451612903,
-      "grad_norm": 1.2684348821640015,
-      "learning_rate": 4.9974008213559725e-06,
-      "logits/chosen": -3.132490396499634,
-      "logits/rejected": -3.018911838531494,
-      "logps/chosen": -116.5405502319336,
-      "logps/rejected": -137.92234802246094,
-      "loss": 0.6893508434295654,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": -0.13492733240127563,
-      "rewards/margins": 0.019388491287827492,
-      "rewards/rejected": -0.15431582927703857,
-      "step": 10
-    },
-    {
-      "epoch": 0.035483870967741936,
-      "grad_norm": 1.0866929292678833,
-      "learning_rate": 4.996791267927632e-06,
-      "logits/chosen": -2.9289159774780273,
-      "logits/rejected": -2.7686662673950195,
-      "logps/chosen": -116.60456085205078,
-      "logps/rejected": -143.89141845703125,
-      "loss": 0.6853089332580566,
-      "rewards/accuracies": 0.53125,
-      "rewards/chosen": -0.15715695917606354,
-      "rewards/margins": 0.03740931302309036,
-      "rewards/rejected": -0.1945662796497345,
-      "step": 11
-    },
-    {
-      "epoch": 0.03870967741935484,
-      "grad_norm": 1.0992016792297363,
-      "learning_rate": 4.996117608647676e-06,
-      "logits/chosen": -2.9300031661987305,
-      "logits/rejected": -2.8163886070251465,
-      "logps/chosen": -125.79206085205078,
-      "logps/rejected": -144.17791748046875,
-      "loss": 0.6614134907722473,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.1761740744113922,
-      "rewards/margins": 0.08016557991504669,
-      "rewards/rejected": -0.2563396394252777,
-      "step": 12
-    },
-    {
-      "epoch": 0.041935483870967745,
-      "grad_norm": 1.0385782718658447,
-      "learning_rate": 4.995379860812503e-06,
-      "logits/chosen": -3.0234384536743164,
-      "logits/rejected": -2.918104648590088,
-      "logps/chosen": -88.80581665039062,
-      "logps/rejected": -118.65660095214844,
-      "loss": 0.6563053131103516,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.11144424974918365,
-      "rewards/margins": 0.09995441138744354,
-      "rewards/rejected": -0.2113986611366272,
-      "step": 13
-    },
-    {
-      "epoch": 0.04516129032258064,
-      "grad_norm": 1.0566327571868896,
-      "learning_rate": 4.994578043364007e-06,
-      "logits/chosen": -3.111154556274414,
-      "logits/rejected": -2.9898476600646973,
-      "logps/chosen": -108.92998504638672,
-      "logps/rejected": -138.91552734375,
-      "loss": 0.6367413997650146,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.17835265398025513,
-      "rewards/margins": 0.1417873203754425,
-      "rewards/rejected": -0.32013997435569763,
-      "step": 14
-    },
-    {
-      "epoch": 0.04838709677419355,
-      "grad_norm": 0.9933570623397827,
-      "learning_rate": 4.993712176889086e-06,
-      "logits/chosen": -3.0580408573150635,
-      "logits/rejected": -2.8851232528686523,
-      "logps/chosen": -98.05117797851562,
-      "logps/rejected": -131.82333374023438,
-      "loss": 0.6436130404472351,
-      "rewards/accuracies": 0.53125,
-      "rewards/chosen": -0.2019367814064026,
-      "rewards/margins": 0.1264682561159134,
-      "rewards/rejected": -0.3284050226211548,
-      "step": 15
-    },
-    {
-      "epoch": 0.05161290322580645,
-      "grad_norm": 0.9798956513404846,
-      "learning_rate": 4.9927822836191185e-06,
-      "logits/chosen": -3.128474235534668,
-      "logits/rejected": -2.990847110748291,
-      "logps/chosen": -111.88795471191406,
-      "logps/rejected": -138.34701538085938,
-      "loss": 0.6226807832717896,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.21347865462303162,
-      "rewards/margins": 0.1850709319114685,
-      "rewards/rejected": -0.39854955673217773,
-      "step": 16
-    },
-    {
-      "epoch": 0.054838709677419356,
-      "grad_norm": 1.058609962463379,
-      "learning_rate": 4.991788387429388e-06,
-      "logits/chosen": -2.850104331970215,
-      "logits/rejected": -2.7300033569335938,
-      "logps/chosen": -113.05485534667969,
-      "logps/rejected": -142.72119140625,
-      "loss": 0.6618618369102478,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.37465834617614746,
-      "rewards/margins": 0.10391812026500702,
-      "rewards/rejected": -0.4785764813423157,
-      "step": 17
-    },
-    {
-      "epoch": 0.05806451612903226,
-      "grad_norm": 1.1318104267120361,
-      "learning_rate": 4.990730513838472e-06,
-      "logits/chosen": -3.082528591156006,
-      "logits/rejected": -2.8970086574554443,
-      "logps/chosen": -97.33485412597656,
-      "logps/rejected": -129.76806640625,
-      "loss": 0.605815589427948,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.34302327036857605,
-      "rewards/margins": 0.24319365620613098,
-      "rewards/rejected": -0.586216926574707,
-      "step": 18
-    },
-    {
-      "epoch": 0.06129032258064516,
-      "grad_norm": 0.9978364706039429,
-      "learning_rate": 4.9896086900075865e-06,
-      "logits/chosen": -2.8832178115844727,
-      "logits/rejected": -2.745048761367798,
-      "logps/chosen": -112.83102416992188,
-      "logps/rejected": -140.89207458496094,
-      "loss": 0.6403385400772095,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.4615880846977234,
-      "rewards/margins": 0.1403249055147171,
-      "rewards/rejected": -0.6019130349159241,
-      "step": 19
-    },
-    {
-      "epoch": 0.06451612903225806,
-      "grad_norm": 1.0464746952056885,
-      "learning_rate": 4.988422944739889e-06,
-      "logits/chosen": -2.8241257667541504,
-      "logits/rejected": -2.7562663555145264,
-      "logps/chosen": -109.80278015136719,
-      "logps/rejected": -139.4794921875,
-      "loss": 0.6105231046676636,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.40774962306022644,
-      "rewards/margins": 0.19766247272491455,
-      "rewards/rejected": -0.6054120659828186,
-      "step": 20
-    },
-    {
-      "epoch": 0.06774193548387097,
-      "grad_norm": 1.0746631622314453,
-      "learning_rate": 4.987173308479738e-06,
-      "logits/chosen": -3.0489606857299805,
-      "logits/rejected": -2.9206061363220215,
-      "logps/chosen": -107.85366821289062,
-      "logps/rejected": -138.60345458984375,
-      "loss": 0.6688461899757385,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.4516957104206085,
-      "rewards/margins": 0.10383725166320801,
-      "rewards/rejected": -0.5555329322814941,
-      "step": 21
-    },
-    {
-      "epoch": 0.07096774193548387,
-      "grad_norm": 1.0326505899429321,
-      "learning_rate": 4.985859813311914e-06,
-      "logits/chosen": -3.1783857345581055,
-      "logits/rejected": -3.0726547241210938,
-      "logps/chosen": -115.14910888671875,
-      "logps/rejected": -140.83590698242188,
-      "loss": 0.6399143934249878,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.5430137515068054,
-      "rewards/margins": 0.1574849635362625,
-      "rewards/rejected": -0.7004987001419067,
-      "step": 22
-    },
-    {
-      "epoch": 0.07419354838709677,
-      "grad_norm": 1.038557767868042,
-      "learning_rate": 4.984482492960792e-06,
-      "logits/chosen": -2.906726360321045,
-      "logits/rejected": -2.7238097190856934,
-      "logps/chosen": -105.02792358398438,
-      "logps/rejected": -135.632568359375,
-      "loss": 0.633417546749115,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.7342426180839539,
-      "rewards/margins": 0.15591225028038025,
-      "rewards/rejected": -0.8901548385620117,
-      "step": 23
-    },
-    {
-      "epoch": 0.07741935483870968,
-      "grad_norm": 0.9582953453063965,
-      "learning_rate": 4.983041382789478e-06,
-      "logits/chosen": -3.3154075145721436,
-      "logits/rejected": -3.2139768600463867,
-      "logps/chosen": -92.43376159667969,
-      "logps/rejected": -118.47042083740234,
-      "loss": 0.6151463985443115,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.6794711947441101,
-      "rewards/margins": 0.23817875981330872,
-      "rewards/rejected": -0.9176499843597412,
-      "step": 24
-    },
-    {
-      "epoch": 0.08064516129032258,
-      "grad_norm": 1.001360535621643,
-      "learning_rate": 4.981536519798899e-06,
-      "logits/chosen": -2.97209095954895,
-      "logits/rejected": -2.7885656356811523,
-      "logps/chosen": -111.38399505615234,
-      "logps/rejected": -144.8062744140625,
-      "loss": 0.6042543649673462,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.7312175035476685,
-      "rewards/margins": 0.26750829815864563,
-      "rewards/rejected": -0.9987257122993469,
-      "step": 25
-    },
-    {
-      "epoch": 0.08387096774193549,
-      "grad_norm": 1.0800671577453613,
-      "learning_rate": 4.9799679426268575e-06,
-      "logits/chosen": -2.918426990509033,
-      "logits/rejected": -2.8081374168395996,
-      "logps/chosen": -121.90701293945312,
-      "logps/rejected": -149.9063720703125,
-      "loss": 0.6208748817443848,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.9017692804336548,
-      "rewards/margins": 0.21920597553253174,
-      "rewards/rejected": -1.1209752559661865,
-      "step": 26
-    },
-    {
-      "epoch": 0.08709677419354839,
-      "grad_norm": 1.1123878955841064,
-      "learning_rate": 4.978335691547035e-06,
-      "logits/chosen": -3.031327247619629,
-      "logits/rejected": -2.892502784729004,
-      "logps/chosen": -109.16461181640625,
-      "logps/rejected": -138.34585571289062,
-      "loss": 0.6401875019073486,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.7755411267280579,
-      "rewards/margins": 0.2256021499633789,
-      "rewards/rejected": -1.001143217086792,
-      "step": 27
-    },
-    {
-      "epoch": 0.09032258064516129,
-      "grad_norm": 1.013746738433838,
-      "learning_rate": 4.976639808467957e-06,
-      "logits/chosen": -2.7579987049102783,
-      "logits/rejected": -2.5802083015441895,
-      "logps/chosen": -113.30269622802734,
-      "logps/rejected": -141.89920043945312,
-      "loss": 0.6180602312088013,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.8010274171829224,
-      "rewards/margins": 0.23785457015037537,
-      "rewards/rejected": -1.0388820171356201,
-      "step": 28
-    },
-    {
-      "epoch": 0.0935483870967742,
-      "grad_norm": 1.4393163919448853,
-      "learning_rate": 4.974880336931923e-06,
-      "logits/chosen": -2.799999952316284,
-      "logits/rejected": -2.6729190349578857,
-      "logps/chosen": -136.82716369628906,
-      "logps/rejected": -157.08663940429688,
-      "loss": 0.7176676988601685,
-      "rewards/accuracies": 0.46875,
-      "rewards/chosen": -1.0027340650558472,
-      "rewards/margins": 0.07778176665306091,
-      "rewards/rejected": -1.0805158615112305,
-      "step": 29
-    },
-    {
-      "epoch": 0.0967741935483871,
-      "grad_norm": 1.0653221607208252,
-      "learning_rate": 4.973057322113883e-06,
-      "logits/chosen": -3.0367493629455566,
-      "logits/rejected": -2.8530149459838867,
-      "logps/chosen": -102.79076385498047,
-      "logps/rejected": -136.4600830078125,
-      "loss": 0.6130280494689941,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.9152212142944336,
-      "rewards/margins": 0.359441876411438,
-      "rewards/rejected": -1.2746630907058716,
-      "step": 30
-    },
-    {
-      "epoch": 0.1,
-      "grad_norm": 0.9103254675865173,
-      "learning_rate": 4.971170810820279e-06,
-      "logits/chosen": -2.9878039360046387,
-      "logits/rejected": -2.897019863128662,
-      "logps/chosen": -113.10630798339844,
-      "logps/rejected": -142.175537109375,
-      "loss": 0.5397008657455444,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.9760595560073853,
-      "rewards/margins": 0.45220765471458435,
-      "rewards/rejected": -1.428267240524292,
-      "step": 31
-    },
-    {
-      "epoch": 0.1032258064516129,
-      "grad_norm": 1.028403639793396,
-      "learning_rate": 4.9692208514878445e-06,
-      "logits/chosen": -3.098134994506836,
-      "logits/rejected": -3.0521764755249023,
-      "logps/chosen": -111.02950286865234,
-      "logps/rejected": -137.7850341796875,
-      "loss": 0.6879711747169495,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -1.1832095384597778,
-      "rewards/margins": 0.1095898449420929,
-      "rewards/rejected": -1.2927993535995483,
-      "step": 32
-    },
-    {
-      "epoch": 0.1064516129032258,
-      "grad_norm": 0.9742422103881836,
-      "learning_rate": 4.967207494182361e-06,
-      "logits/chosen": -2.9038455486297607,
-      "logits/rejected": -2.777085542678833,
-      "logps/chosen": -114.38245391845703,
-      "logps/rejected": -147.81060791015625,
-      "loss": 0.5629678964614868,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.9888625144958496,
-      "rewards/margins": 0.44450241327285767,
-      "rewards/rejected": -1.4333648681640625,
-      "step": 33
-    },
-    {
-      "epoch": 0.10967741935483871,
-      "grad_norm": 0.9404062628746033,
-      "learning_rate": 4.965130790597369e-06,
-      "logits/chosen": -3.026500701904297,
-      "logits/rejected": -2.8468196392059326,
-      "logps/chosen": -110.82350158691406,
-      "logps/rejected": -139.73025512695312,
-      "loss": 0.5878911018371582,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.8643066883087158,
-      "rewards/margins": 0.43531858921051025,
-      "rewards/rejected": -1.2996253967285156,
-      "step": 34
-    },
-    {
-      "epoch": 0.11290322580645161,
-      "grad_norm": 0.9421868920326233,
-      "learning_rate": 4.962990794052847e-06,
-      "logits/chosen": -2.985588550567627,
-      "logits/rejected": -2.7967123985290527,
-      "logps/chosen": -114.51589965820312,
-      "logps/rejected": -140.2834930419922,
-      "loss": 0.5792871713638306,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.0735702514648438,
-      "rewards/margins": 0.41116511821746826,
-      "rewards/rejected": -1.484735369682312,
-      "step": 35
-    },
-    {
-      "epoch": 0.11612903225806452,
-      "grad_norm": 0.9833240509033203,
-      "learning_rate": 4.960787559493836e-06,
-      "logits/chosen": -2.819369316101074,
-      "logits/rejected": -2.6581039428710938,
-      "logps/chosen": -126.71798706054688,
-      "logps/rejected": -154.09063720703125,
-      "loss": 0.5701221227645874,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.0069059133529663,
-      "rewards/margins": 0.3965708017349243,
-      "rewards/rejected": -1.4034767150878906,
-      "step": 36
-    },
-    {
-      "epoch": 0.11935483870967742,
-      "grad_norm": 0.9587810039520264,
-      "learning_rate": 4.958521143489032e-06,
-      "logits/chosen": -2.9700632095336914,
-      "logits/rejected": -2.7929463386535645,
-      "logps/chosen": -117.53889465332031,
-      "logps/rejected": -141.08412170410156,
-      "loss": 0.5579148530960083,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.2454460859298706,
-      "rewards/margins": 0.46080076694488525,
-      "rewards/rejected": -1.7062468528747559,
-      "step": 37
-    },
-    {
-      "epoch": 0.12258064516129032,
-      "grad_norm": 0.952377200126648,
-      "learning_rate": 4.956191604229335e-06,
-      "logits/chosen": -2.7670180797576904,
-      "logits/rejected": -2.653087615966797,
-      "logps/chosen": -122.13555908203125,
-      "logps/rejected": -154.33908081054688,
-      "loss": 0.5420794486999512,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.417367935180664,
-      "rewards/margins": 0.5341382026672363,
-      "rewards/rejected": -1.9515061378479004,
-      "step": 38
-    },
-    {
-      "epoch": 0.12580645161290321,
-      "grad_norm": 0.9858971834182739,
-      "learning_rate": 4.9537990015263505e-06,
-      "logits/chosen": -2.846860647201538,
-      "logits/rejected": -2.6801350116729736,
-      "logps/chosen": -109.1124267578125,
-      "logps/rejected": -141.81492614746094,
-      "loss": 0.5824028253555298,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -1.316831350326538,
-      "rewards/margins": 0.42448168992996216,
-      "rewards/rejected": -1.7413129806518555,
-      "step": 39
-    },
-    {
-      "epoch": 0.12903225806451613,
-      "grad_norm": 1.1429812908172607,
-      "learning_rate": 4.95134339681086e-06,
-      "logits/chosen": -3.008500337600708,
-      "logits/rejected": -2.8374381065368652,
-      "logps/chosen": -111.08134460449219,
-      "logps/rejected": -140.59872436523438,
-      "loss": 0.6458199620246887,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -1.5981895923614502,
-      "rewards/margins": 0.2978651523590088,
-      "rewards/rejected": -1.896054744720459,
-      "step": 40
-    },
-    {
-      "epoch": 0.13225806451612904,
-      "grad_norm": 0.8448069095611572,
-      "learning_rate": 4.948824853131237e-06,
-      "logits/chosen": -2.907655715942383,
-      "logits/rejected": -2.7536003589630127,
-      "logps/chosen": -113.13584899902344,
-      "logps/rejected": -144.23391723632812,
-      "loss": 0.4713582396507263,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -1.285538673400879,
-      "rewards/margins": 0.6696875095367432,
-      "rewards/rejected": -1.955226182937622,
-      "step": 41
-    },
-    {
-      "epoch": 0.13548387096774195,
-      "grad_norm": 1.1781965494155884,
-      "learning_rate": 4.946243435151833e-06,
-      "logits/chosen": -2.7036216259002686,
-      "logits/rejected": -2.5935046672821045,
-      "logps/chosen": -133.15252685546875,
-      "logps/rejected": -154.00750732421875,
-      "loss": 0.7647944688796997,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -1.6891489028930664,
-      "rewards/margins": 0.04440240561962128,
-      "rewards/rejected": -1.733551263809204,
-      "step": 42
-    },
-    {
-      "epoch": 0.13870967741935483,
-      "grad_norm": 0.9805737733840942,
-      "learning_rate": 4.943599209151314e-06,
-      "logits/chosen": -2.7226176261901855,
-      "logits/rejected": -2.517169237136841,
-      "logps/chosen": -125.3690185546875,
-      "logps/rejected": -162.36477661132812,
-      "loss": 0.5878539085388184,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -1.4429672956466675,
-      "rewards/margins": 0.4025247097015381,
-      "rewards/rejected": -1.8454921245574951,
-      "step": 43
-    },
-    {
-      "epoch": 0.14193548387096774,
-      "grad_norm": 0.8965495228767395,
-      "learning_rate": 4.940892243020964e-06,
-      "logits/chosen": -2.879836082458496,
-      "logits/rejected": -2.6803863048553467,
-      "logps/chosen": -111.92822265625,
-      "logps/rejected": -144.03843688964844,
-      "loss": 0.5402083396911621,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.189293384552002,
-      "rewards/margins": 0.5704534649848938,
-      "rewards/rejected": -1.759746789932251,
-      "step": 44
-    },
-    {
-      "epoch": 0.14516129032258066,
-      "grad_norm": 1.0602669715881348,
-      "learning_rate": 4.938122606262935e-06,
-      "logits/chosen": -2.8707027435302734,
-      "logits/rejected": -2.67893123626709,
-      "logps/chosen": -125.02201080322266,
-      "logps/rejected": -153.2523956298828,
-      "loss": 0.6121110916137695,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -1.6958320140838623,
-      "rewards/margins": 0.445503294467926,
-      "rewards/rejected": -2.1413354873657227,
-      "step": 45
-    },
-    {
-      "epoch": 0.14838709677419354,
-      "grad_norm": 0.927438497543335,
-      "learning_rate": 4.935290369988468e-06,
-      "logits/chosen": -2.7243499755859375,
-      "logits/rejected": -2.563385486602783,
-      "logps/chosen": -132.1870880126953,
-      "logps/rejected": -171.1417236328125,
-      "loss": 0.4953867793083191,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.4100818634033203,
-      "rewards/margins": 0.6493280529975891,
-      "rewards/rejected": -2.0594100952148438,
-      "step": 46
-    },
-    {
-      "epoch": 0.15161290322580645,
-      "grad_norm": 0.8959330916404724,
-      "learning_rate": 4.932395606916063e-06,
-      "logits/chosen": -2.4819681644439697,
-      "logits/rejected": -2.2971127033233643,
-      "logps/chosen": -122.25642395019531,
-      "logps/rejected": -159.7163543701172,
-      "loss": 0.4567532241344452,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -1.6163623332977295,
-      "rewards/margins": 0.7664045691490173,
-      "rewards/rejected": -2.3827669620513916,
-      "step": 47
-    },
-    {
-      "epoch": 0.15483870967741936,
-      "grad_norm": 0.854138970375061,
-      "learning_rate": 4.9294383913696145e-06,
-      "logits/chosen": -2.6553072929382324,
-      "logits/rejected": -2.431748628616333,
-      "logps/chosen": -122.9141616821289,
-      "logps/rejected": -171.38320922851562,
-      "loss": 0.4372290372848511,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.455812692642212,
-      "rewards/margins": 0.9929529428482056,
-      "rewards/rejected": -2.448765754699707,
-      "step": 48
-    },
-    {
-      "epoch": 0.15806451612903225,
-      "grad_norm": 0.7815476655960083,
-      "learning_rate": 4.926418799276505e-06,
-      "logits/chosen": -2.765216827392578,
-      "logits/rejected": -2.5981082916259766,
-      "logps/chosen": -111.628173828125,
-      "logps/rejected": -138.0706024169922,
-      "loss": 0.479827880859375,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -1.5760267972946167,
-      "rewards/margins": 0.758277177810669,
-      "rewards/rejected": -2.334303855895996,
-      "step": 49
-    },
-    {
-      "epoch": 0.16129032258064516,
-      "grad_norm": 1.0033049583435059,
-      "learning_rate": 4.923336908165649e-06,
-      "logits/chosen": -2.8565001487731934,
-      "logits/rejected": -2.618370294570923,
-      "logps/chosen": -108.14005279541016,
-      "logps/rejected": -148.12445068359375,
-      "loss": 0.42941758036613464,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -1.5442157983779907,
-      "rewards/margins": 0.8375117182731628,
-      "rewards/rejected": -2.381727695465088,
-      "step": 50
-    },
-    {
-      "epoch": 0.16451612903225807,
-      "grad_norm": 0.904968798160553,
-      "learning_rate": 4.920192797165511e-06,
-      "logits/chosen": -2.844186782836914,
-      "logits/rejected": -2.6398918628692627,
-      "logps/chosen": -125.80914306640625,
-      "logps/rejected": -163.47320556640625,
-      "loss": 0.471732497215271,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.6446901559829712,
-      "rewards/margins": 0.8008061647415161,
-      "rewards/rejected": -2.4454965591430664,
-      "step": 51
-    },
-    {
-      "epoch": 0.16774193548387098,
-      "grad_norm": 0.9294782280921936,
-      "learning_rate": 4.916986547002067e-06,
-      "logits/chosen": -2.8216707706451416,
-      "logits/rejected": -2.7185850143432617,
-      "logps/chosen": -124.8201904296875,
-      "logps/rejected": -155.616455078125,
-      "loss": 0.4811817705631256,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -2.0028302669525146,
-      "rewards/margins": 0.6963652968406677,
-      "rewards/rejected": -2.699195623397827,
-      "step": 52
-    },
-    {
-      "epoch": 0.17096774193548386,
-      "grad_norm": 0.8786033391952515,
-      "learning_rate": 4.913718239996734e-06,
-      "logits/chosen": -2.832566261291504,
-      "logits/rejected": -2.6430118083953857,
-      "logps/chosen": -122.54075622558594,
-      "logps/rejected": -158.80088806152344,
-      "loss": 0.5157533884048462,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -1.8400042057037354,
-      "rewards/margins": 0.6210693717002869,
-      "rewards/rejected": -2.461073398590088,
-      "step": 53
-    },
-    {
-      "epoch": 0.17419354838709677,
-      "grad_norm": 0.8750291466712952,
-      "learning_rate": 4.910387960064259e-06,
-      "logits/chosen": -2.6718342304229736,
-      "logits/rejected": -2.5271155834198,
-      "logps/chosen": -131.72975158691406,
-      "logps/rejected": -167.03482055664062,
-      "loss": 0.4826596975326538,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -2.0635039806365967,
-      "rewards/margins": 0.7045244574546814,
-      "rewards/rejected": -2.768028736114502,
-      "step": 54
-    },
-    {
-      "epoch": 0.1774193548387097,
-      "grad_norm": 0.8463932275772095,
-      "learning_rate": 4.906995792710559e-06,
-      "logits/chosen": -2.793056011199951,
-      "logits/rejected": -2.6334168910980225,
-      "logps/chosen": -108.91849517822266,
-      "logps/rejected": -152.1488800048828,
-      "loss": 0.4304446578025818,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -1.7403039932250977,
-      "rewards/margins": 1.1434907913208008,
-      "rewards/rejected": -2.8837947845458984,
-      "step": 55
-    },
-    {
-      "epoch": 0.18064516129032257,
-      "grad_norm": 0.8992181420326233,
-      "learning_rate": 4.9035418250305314e-06,
-      "logits/chosen": -2.832119941711426,
-      "logits/rejected": -2.666569709777832,
-      "logps/chosen": -107.66205596923828,
-      "logps/rejected": -141.9404754638672,
-      "loss": 0.5210002660751343,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -2.036715030670166,
-      "rewards/margins": 0.9046896696090698,
-      "rewards/rejected": -2.9414048194885254,
-      "step": 56
-    },
-    {
-      "epoch": 0.18387096774193548,
-      "grad_norm": 1.4827739000320435,
-      "learning_rate": 4.900026145705815e-06,
-      "logits/chosen": -2.7140512466430664,
-      "logits/rejected": -2.4837052822113037,
-      "logps/chosen": -121.64815521240234,
-      "logps/rejected": -159.62066650390625,
-      "loss": 0.4381827414035797,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.80775785446167,
-      "rewards/margins": 1.076915979385376,
-      "rewards/rejected": -2.884673833847046,
-      "step": 57
-    },
-    {
-      "epoch": 0.1870967741935484,
-      "grad_norm": 1.0054007768630981,
-      "learning_rate": 4.896448845002511e-06,
-      "logits/chosen": -2.706526517868042,
-      "logits/rejected": -2.487896680831909,
-      "logps/chosen": -122.99580383300781,
-      "logps/rejected": -156.40049743652344,
-      "loss": 0.5580032467842102,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -2.2970235347747803,
-      "rewards/margins": 0.7499445080757141,
-      "rewards/rejected": -3.0469677448272705,
-      "step": 58
-    },
-    {
-      "epoch": 0.19032258064516128,
-      "grad_norm": 1.0605281591415405,
-      "learning_rate": 4.8928100147688705e-06,
-      "logits/chosen": -2.9178082942962646,
-      "logits/rejected": -2.6102988719940186,
-      "logps/chosen": -114.86691284179688,
-      "logps/rejected": -154.53656005859375,
-      "loss": 0.6092868447303772,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -2.3286643028259277,
-      "rewards/margins": 0.5103977918624878,
-      "rewards/rejected": -2.839062213897705,
-      "step": 59
-    },
-    {
-      "epoch": 0.1935483870967742,
-      "grad_norm": 0.7592489123344421,
-      "learning_rate": 4.889109748432932e-06,
-      "logits/chosen": -2.9385297298431396,
-      "logits/rejected": -2.6411538124084473,
-      "logps/chosen": -115.66314697265625,
-      "logps/rejected": -149.95323181152344,
-      "loss": 0.4233652651309967,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -1.9713090658187866,
-      "rewards/margins": 0.8833580613136292,
-      "rewards/rejected": -2.8546671867370605,
-      "step": 60
-    },
-    {
-      "epoch": 0.1967741935483871,
-      "grad_norm": 0.7794229388237,
-      "learning_rate": 4.8853481410001225e-06,
-      "logits/chosen": -2.9496335983276367,
-      "logits/rejected": -2.7019166946411133,
-      "logps/chosen": -106.0539779663086,
-      "logps/rejected": -148.11280822753906,
-      "loss": 0.4723885953426361,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -2.460500717163086,
-      "rewards/margins": 1.022800087928772,
-      "rewards/rejected": -3.4833006858825684,
-      "step": 61
-    },
-    {
-      "epoch": 0.2,
-      "grad_norm": 0.9357581734657288,
-      "learning_rate": 4.881525289050823e-06,
-      "logits/chosen": -2.77065372467041,
-      "logits/rejected": -2.6192283630371094,
-      "logps/chosen": -130.91944885253906,
-      "logps/rejected": -149.3533935546875,
-      "loss": 0.5586632490158081,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -2.1316962242126465,
-      "rewards/margins": 0.6954216957092285,
-      "rewards/rejected": -2.827117681503296,
-      "step": 62
-    },
-    {
-      "epoch": 0.2032258064516129,
-      "grad_norm": 0.7778428196907043,
-      "learning_rate": 4.8776412907378845e-06,
-      "logits/chosen": -2.7207984924316406,
-      "logits/rejected": -2.508455276489258,
-      "logps/chosen": -100.13325500488281,
-      "logps/rejected": -154.1763916015625,
-      "loss": 0.4038059711456299,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -2.372382640838623,
-      "rewards/margins": 1.304573893547058,
-      "rewards/rejected": -3.6769564151763916,
-      "step": 63
-    },
-    {
-      "epoch": 0.2064516129032258,
-      "grad_norm": 0.9325763583183289,
-      "learning_rate": 4.873696245784106e-06,
-      "logits/chosen": -2.285740375518799,
-      "logits/rejected": -2.1688804626464844,
-      "logps/chosen": -133.5624542236328,
-      "logps/rejected": -158.0366668701172,
-      "loss": 0.5424321293830872,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -2.1653249263763428,
-      "rewards/margins": 0.7148800492286682,
-      "rewards/rejected": -2.880204677581787,
-      "step": 64
-    },
-    {
-      "epoch": 0.20967741935483872,
-      "grad_norm": 0.9311462044715881,
-      "learning_rate": 4.8696902554796824e-06,
-      "logits/chosen": -2.6557462215423584,
-      "logits/rejected": -2.5266714096069336,
-      "logps/chosen": -129.286376953125,
-      "logps/rejected": -164.18695068359375,
-      "loss": 0.5229653120040894,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -2.370563507080078,
-      "rewards/margins": 0.8608935475349426,
-      "rewards/rejected": -3.231456995010376,
-      "step": 65
-    },
-    {
-      "epoch": 0.2129032258064516,
-      "grad_norm": 0.9174665212631226,
-      "learning_rate": 4.865623422679593e-06,
-      "logits/chosen": -2.8143653869628906,
-      "logits/rejected": -2.586948871612549,
-      "logps/chosen": -108.68220520019531,
-      "logps/rejected": -140.1006622314453,
-      "loss": 0.5581181049346924,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -2.5576014518737793,
-      "rewards/margins": 0.7612634897232056,
-      "rewards/rejected": -3.3188650608062744,
-      "step": 66
-    },
-    {
-      "epoch": 0.2161290322580645,
-      "grad_norm": 1.003322958946228,
-      "learning_rate": 4.861495851800968e-06,
-      "logits/chosen": -2.7197985649108887,
-      "logits/rejected": -2.613180160522461,
-      "logps/chosen": -132.04977416992188,
-      "logps/rejected": -156.76223754882812,
-      "loss": 0.6440731287002563,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -2.7013721466064453,
-      "rewards/margins": 0.3256293535232544,
-      "rewards/rejected": -3.02700138092041,
-      "step": 67
-    },
-    {
-      "epoch": 0.21935483870967742,
-      "grad_norm": 0.7528925538063049,
-      "learning_rate": 4.857307648820409e-06,
-      "logits/chosen": -2.47288179397583,
-      "logits/rejected": -2.3103513717651367,
-      "logps/chosen": -133.77581787109375,
-      "logps/rejected": -167.21572875976562,
-      "loss": 0.4236561059951782,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -2.5838000774383545,
-      "rewards/margins": 1.0029029846191406,
-      "rewards/rejected": -3.586703062057495,
-      "step": 68
-    },
-    {
-      "epoch": 0.22258064516129034,
-      "grad_norm": 1.148950457572937,
-      "learning_rate": 4.853058921271259e-06,
-      "logits/chosen": -2.322709798812866,
-      "logits/rejected": -2.1581592559814453,
-      "logps/chosen": -134.0292205810547,
-      "logps/rejected": -167.06533813476562,
-      "loss": 0.4543103277683258,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -2.312931537628174,
-      "rewards/margins": 1.102271556854248,
-      "rewards/rejected": -3.415203094482422,
-      "step": 69
-    },
-    {
-      "epoch": 0.22580645161290322,
-      "grad_norm": 0.8065391778945923,
-      "learning_rate": 4.84874977824085e-06,
-      "logits/chosen": -2.314046859741211,
-      "logits/rejected": -2.165867805480957,
-      "logps/chosen": -131.90957641601562,
-      "logps/rejected": -164.817626953125,
-      "loss": 0.43934166431427,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -2.61452579498291,
-      "rewards/margins": 1.0483322143554688,
-      "rewards/rejected": -3.662858009338379,
-      "step": 70
-    },
-    {
-      "epoch": 0.22903225806451613,
-      "grad_norm": 0.70816570520401,
-      "learning_rate": 4.844380330367701e-06,
-      "logits/chosen": -2.426614284515381,
-      "logits/rejected": -2.2347359657287598,
-      "logps/chosen": -127.90621185302734,
-      "logps/rejected": -165.37213134765625,
-      "loss": 0.3680841326713562,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -2.334412097930908,
-      "rewards/margins": 1.211841344833374,
-      "rewards/rejected": -3.546253204345703,
-      "step": 71
-    },
-    {
-      "epoch": 0.23225806451612904,
-      "grad_norm": 0.7996051907539368,
-      "learning_rate": 4.839950689838674e-06,
-      "logits/chosen": -2.362088680267334,
-      "logits/rejected": -2.163127899169922,
-      "logps/chosen": -144.364501953125,
-      "logps/rejected": -182.89500427246094,
-      "loss": 0.43939828872680664,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -2.8185462951660156,
-      "rewards/margins": 0.9022763967514038,
-      "rewards/rejected": -3.72082257270813,
-      "step": 72
-    },
-    {
-      "epoch": 0.23548387096774193,
-      "grad_norm": 0.7642343044281006,
-      "learning_rate": 4.835460970386093e-06,
-      "logits/chosen": -2.3497159481048584,
-      "logits/rejected": -2.1166491508483887,
-      "logps/chosen": -134.3424835205078,
-      "logps/rejected": -175.50112915039062,
-      "loss": 0.4305233061313629,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -2.630145311355591,
-      "rewards/margins": 0.9792301654815674,
-      "rewards/rejected": -3.609375476837158,
-      "step": 73
-    },
-    {
-      "epoch": 0.23870967741935484,
-      "grad_norm": 0.9543027877807617,
-      "learning_rate": 4.83091128728483e-06,
-      "logits/chosen": -2.4201152324676514,
-      "logits/rejected": -2.2325682640075684,
-      "logps/chosen": -127.69469451904297,
-      "logps/rejected": -152.89657592773438,
-      "loss": 0.632413387298584,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -2.69545841217041,
-      "rewards/margins": 0.48617762327194214,
-      "rewards/rejected": -3.181636095046997,
-      "step": 74
-    },
-    {
-      "epoch": 0.24193548387096775,
-      "grad_norm": 0.6152648329734802,
-      "learning_rate": 4.826301757349337e-06,
-      "logits/chosen": -2.612950563430786,
-      "logits/rejected": -2.358858585357666,
-      "logps/chosen": -117.93904113769531,
-      "logps/rejected": -162.6607666015625,
-      "loss": 0.3176124095916748,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -2.620302438735962,
-      "rewards/margins": 1.325659990310669,
-      "rewards/rejected": -3.945962429046631,
-      "step": 75
-    },
-    {
-      "epoch": 0.24516129032258063,
-      "grad_norm": 0.771643340587616,
-      "learning_rate": 4.821632498930656e-06,
-      "logits/chosen": -2.359376907348633,
-      "logits/rejected": -2.176957607269287,
-      "logps/chosen": -137.79673767089844,
-      "logps/rejected": -167.8568115234375,
-      "loss": 0.4373764991760254,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -2.7868497371673584,
-      "rewards/margins": 1.0548901557922363,
-      "rewards/rejected": -3.8417396545410156,
-      "step": 76
-    },
-    {
-      "epoch": 0.24838709677419354,
-      "grad_norm": 0.7945637106895447,
-      "learning_rate": 4.816903631913372e-06,
-      "logits/chosen": -2.5555307865142822,
-      "logits/rejected": -2.3610169887542725,
-      "logps/chosen": -121.39008331298828,
-      "logps/rejected": -156.8350067138672,
-      "loss": 0.47777238488197327,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -2.8233864307403564,
-      "rewards/margins": 0.945284366607666,
-      "rewards/rejected": -3.7686707973480225,
-      "step": 77
-    },
-    {
-      "epoch": 0.25161290322580643,
-      "grad_norm": 0.7087196707725525,
-      "learning_rate": 4.812115277712539e-06,
-      "logits/chosen": -2.5300731658935547,
-      "logits/rejected": -2.3655776977539062,
-      "logps/chosen": -127.15848541259766,
-      "logps/rejected": -163.5643310546875,
-      "loss": 0.4068886637687683,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -2.928337574005127,
-      "rewards/margins": 1.12626314163208,
-      "rewards/rejected": -4.054600715637207,
-      "step": 78
-    },
-    {
-      "epoch": 0.25483870967741934,
-      "grad_norm": 0.6794445514678955,
-      "learning_rate": 4.807267559270563e-06,
-      "logits/chosen": -2.559274911880493,
-      "logits/rejected": -2.374232769012451,
-      "logps/chosen": -132.4824981689453,
-      "logps/rejected": -171.53933715820312,
-      "loss": 0.37371641397476196,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -3.011838436126709,
-      "rewards/margins": 1.4339759349822998,
-      "rewards/rejected": -4.44581413269043,
-      "step": 79
-    },
-    {
-      "epoch": 0.25806451612903225,
-      "grad_norm": 0.9782342314720154,
-      "learning_rate": 4.802360601054042e-06,
-      "logits/chosen": -2.2338345050811768,
-      "logits/rejected": -2.005814790725708,
-      "logps/chosen": -131.63963317871094,
-      "logps/rejected": -168.6005401611328,
-      "loss": 0.5639984607696533,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -3.223055839538574,
-      "rewards/margins": 0.7143847942352295,
-      "rewards/rejected": -3.9374403953552246,
-      "step": 80
-    },
-    {
-      "epoch": 0.26129032258064516,
-      "grad_norm": 0.7722119092941284,
-      "learning_rate": 4.797394529050577e-06,
-      "logits/chosen": -2.2159998416900635,
-      "logits/rejected": -1.9760271310806274,
-      "logps/chosen": -138.00804138183594,
-      "logps/rejected": -191.0758056640625,
-      "loss": 0.35425710678100586,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -3.038595676422119,
-      "rewards/margins": 1.5962517261505127,
-      "rewards/rejected": -4.634847640991211,
-      "step": 81
-    },
-    {
-      "epoch": 0.2645161290322581,
-      "grad_norm": 0.781527578830719,
-      "learning_rate": 4.792369470765528e-06,
-      "logits/chosen": -2.332066059112549,
-      "logits/rejected": -2.1308486461639404,
-      "logps/chosen": -134.24362182617188,
-      "logps/rejected": -175.5936279296875,
-      "loss": 0.410447895526886,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -3.196249485015869,
-      "rewards/margins": 1.2090076208114624,
-      "rewards/rejected": -4.405257225036621,
-      "step": 82
-    },
-    {
-      "epoch": 0.267741935483871,
-      "grad_norm": 1.4148427248001099,
-      "learning_rate": 4.787285555218748e-06,
-      "logits/chosen": -2.541987895965576,
-      "logits/rejected": -2.3397560119628906,
-      "logps/chosen": -125.69288635253906,
-      "logps/rejected": -169.29953002929688,
-      "loss": 0.3594983220100403,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -3.1516847610473633,
-      "rewards/margins": 1.4236831665039062,
-      "rewards/rejected": -4.5753679275512695,
-      "step": 83
-    },
-    {
-      "epoch": 0.2709677419354839,
-      "grad_norm": 0.8784657120704651,
-      "learning_rate": 4.782142912941268e-06,
-      "logits/chosen": -2.2671427726745605,
-      "logits/rejected": -2.1503725051879883,
-      "logps/chosen": -147.03195190429688,
-      "logps/rejected": -179.34947204589844,
-      "loss": 0.5158642530441284,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -3.4720330238342285,
-      "rewards/margins": 0.8386823534965515,
-      "rewards/rejected": -4.310715198516846,
-      "step": 84
-    },
-    {
-      "epoch": 0.27419354838709675,
-      "grad_norm": 0.6953436732292175,
-      "learning_rate": 4.776941675971941e-06,
-      "logits/chosen": -2.3191490173339844,
-      "logits/rejected": -2.090640068054199,
-      "logps/chosen": -114.88062286376953,
-      "logps/rejected": -165.6884765625,
-      "loss": 0.3098244369029999,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -2.7739365100860596,
-      "rewards/margins": 2.004124879837036,
-      "rewards/rejected": -4.7780609130859375,
-      "step": 85
-    },
-    {
-      "epoch": 0.27741935483870966,
-      "grad_norm": 1.1434965133666992,
-      "learning_rate": 4.771681977854062e-06,
-      "logits/chosen": -2.098564624786377,
-      "logits/rejected": -1.9888715744018555,
-      "logps/chosen": -151.57363891601562,
-      "logps/rejected": -181.7247314453125,
-      "loss": 0.5879724025726318,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -3.6309304237365723,
-      "rewards/margins": 0.8588354587554932,
-      "rewards/rejected": -4.489765644073486,
-      "step": 86
-    },
-    {
-      "epoch": 0.2806451612903226,
-      "grad_norm": 0.7763506174087524,
-      "learning_rate": 4.7663639536319304e-06,
-      "logits/chosen": -2.165168046951294,
-      "logits/rejected": -1.9731296300888062,
-      "logps/chosen": -139.3979034423828,
-      "logps/rejected": -178.37872314453125,
-      "loss": 0.45243924856185913,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -3.6948037147521973,
-      "rewards/margins": 1.2531442642211914,
-      "rewards/rejected": -4.947947978973389,
-      "step": 87
-    },
-    {
-      "epoch": 0.2838709677419355,
-      "grad_norm": 0.7684445381164551,
-      "learning_rate": 4.760987739847383e-06,
-      "logits/chosen": -2.128974199295044,
-      "logits/rejected": -1.9912292957305908,
-      "logps/chosen": -132.04293823242188,
-      "logps/rejected": -171.23690795898438,
-      "loss": 0.4750169515609741,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -3.4750771522521973,
-      "rewards/margins": 0.9534658193588257,
-      "rewards/rejected": -4.4285430908203125,
-      "step": 88
-    },
-    {
-      "epoch": 0.2870967741935484,
-      "grad_norm": 0.7377817630767822,
-      "learning_rate": 4.755553474536294e-06,
-      "logits/chosen": -2.0988073348999023,
-      "logits/rejected": -1.902956247329712,
-      "logps/chosen": -141.26329040527344,
-      "logps/rejected": -177.63734436035156,
-      "loss": 0.43820852041244507,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -3.559805393218994,
-      "rewards/margins": 1.0688958168029785,
-      "rewards/rejected": -4.628701210021973,
-      "step": 89
-    },
-    {
-      "epoch": 0.2903225806451613,
-      "grad_norm": 0.6428540945053101,
-      "learning_rate": 4.750061297225028e-06,
-      "logits/chosen": -2.3366332054138184,
-      "logits/rejected": -2.1300148963928223,
-      "logps/chosen": -128.06130981445312,
-      "logps/rejected": -175.9393768310547,
-      "loss": 0.32989248633384705,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -3.4514639377593994,
-      "rewards/margins": 1.4905400276184082,
-      "rewards/rejected": -4.9420037269592285,
-      "step": 90
-    },
-    {
-      "epoch": 0.29354838709677417,
-      "grad_norm": 0.7805766463279724,
-      "learning_rate": 4.744511348926855e-06,
-      "logits/chosen": -2.0861973762512207,
-      "logits/rejected": -1.9446799755096436,
-      "logps/chosen": -138.7248077392578,
-      "logps/rejected": -170.54974365234375,
-      "loss": 0.454301118850708,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -3.870506763458252,
-      "rewards/margins": 1.0038337707519531,
-      "rewards/rejected": -4.874340534210205,
-      "step": 91
-    },
-    {
-      "epoch": 0.2967741935483871,
-      "grad_norm": 0.8089374303817749,
-      "learning_rate": 4.7389037721383325e-06,
-      "logits/chosen": -2.1515049934387207,
-      "logits/rejected": -1.9613507986068726,
-      "logps/chosen": -138.1538848876953,
-      "logps/rejected": -171.8699951171875,
-      "loss": 0.42631715536117554,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -3.4975502490997314,
-      "rewards/margins": 1.3891708850860596,
-      "rewards/rejected": -4.886720657348633,
-      "step": 92
-    },
-    {
-      "epoch": 0.3,
-      "grad_norm": 0.8091368675231934,
-      "learning_rate": 4.733238710835648e-06,
-      "logits/chosen": -1.947023868560791,
-      "logits/rejected": -1.7635173797607422,
-      "logps/chosen": -159.03768920898438,
-      "logps/rejected": -197.48944091796875,
-      "loss": 0.47380709648132324,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -4.0391106605529785,
-      "rewards/margins": 1.2754480838775635,
-      "rewards/rejected": -5.314558982849121,
-      "step": 93
-    },
-    {
-      "epoch": 0.3032258064516129,
-      "grad_norm": 0.7619990110397339,
-      "learning_rate": 4.72751631047092e-06,
-      "logits/chosen": -2.460020065307617,
-      "logits/rejected": -2.312316417694092,
-      "logps/chosen": -139.6368408203125,
-      "logps/rejected": -178.7040557861328,
-      "loss": 0.48125335574150085,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -3.6689484119415283,
-      "rewards/margins": 1.3045544624328613,
-      "rewards/rejected": -4.973503112792969,
-      "step": 94
-    },
-    {
-      "epoch": 0.3064516129032258,
-      "grad_norm": 0.6219649910926819,
-      "learning_rate": 4.721736717968465e-06,
-      "logits/chosen": -1.954379916191101,
-      "logits/rejected": -1.7585666179656982,
-      "logps/chosen": -145.32232666015625,
-      "logps/rejected": -185.040771484375,
-      "loss": 0.3768487572669983,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -3.559706926345825,
-      "rewards/margins": 1.4724528789520264,
-      "rewards/rejected": -5.032159805297852,
-      "step": 95
-    },
-    {
-      "epoch": 0.3096774193548387,
-      "grad_norm": 0.7134078741073608,
-      "learning_rate": 4.715900081721021e-06,
-      "logits/chosen": -1.9498239755630493,
-      "logits/rejected": -1.7411441802978516,
-      "logps/chosen": -143.168701171875,
-      "logps/rejected": -181.01402282714844,
-      "loss": 0.4029703140258789,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -3.5624961853027344,
-      "rewards/margins": 1.211552619934082,
-      "rewards/rejected": -4.774048805236816,
-      "step": 96
-    },
-    {
-      "epoch": 0.31290322580645163,
-      "grad_norm": 0.7324331402778625,
-      "learning_rate": 4.7100065515859464e-06,
-      "logits/chosen": -2.3302841186523438,
-      "logits/rejected": -2.1917896270751953,
-      "logps/chosen": -123.150634765625,
-      "logps/rejected": -165.6932373046875,
-      "loss": 0.43078821897506714,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -3.5430026054382324,
-      "rewards/margins": 1.245470404624939,
-      "rewards/rejected": -4.788473129272461,
-      "step": 97
-    },
-    {
-      "epoch": 0.3161290322580645,
-      "grad_norm": 0.7748900651931763,
-      "learning_rate": 4.704056278881364e-06,
-      "logits/chosen": -1.8153760433197021,
-      "logits/rejected": -1.6069302558898926,
-      "logps/chosen": -138.3715362548828,
-      "logps/rejected": -180.33352661132812,
-      "loss": 0.4739260673522949,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -3.8387598991394043,
-      "rewards/margins": 1.237455129623413,
-      "rewards/rejected": -5.076214790344238,
-      "step": 98
-    },
-    {
-      "epoch": 0.3193548387096774,
-      "grad_norm": 0.7280204892158508,
-      "learning_rate": 4.698049416382277e-06,
-      "logits/chosen": -2.13297176361084,
-      "logits/rejected": -1.9608895778656006,
-      "logps/chosen": -140.18515014648438,
-      "logps/rejected": -181.5408935546875,
-      "loss": 0.5066293478012085,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -3.9341986179351807,
-      "rewards/margins": 1.4060182571411133,
-      "rewards/rejected": -5.340217113494873,
-      "step": 99
-    },
-    {
-      "epoch": 0.3225806451612903,
-      "grad_norm": 0.6968139410018921,
-      "learning_rate": 4.691986118316654e-06,
-      "logits/chosen": -2.0721585750579834,
-      "logits/rejected": -1.9022136926651,
-      "logps/chosen": -136.25216674804688,
-      "logps/rejected": -178.95773315429688,
-      "loss": 0.41828787326812744,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -3.9730870723724365,
-      "rewards/margins": 1.3703317642211914,
-      "rewards/rejected": -5.343419075012207,
-      "step": 100
-    },
-    {
-      "epoch": 0.3258064516129032,
-      "grad_norm": 0.838504433631897,
-      "learning_rate": 4.685866540361456e-06,
-      "logits/chosen": -1.8501439094543457,
-      "logits/rejected": -1.6102899312973022,
-      "logps/chosen": -149.552978515625,
-      "logps/rejected": -200.82159423828125,
-      "loss": 0.4767953157424927,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -4.187356948852539,
-      "rewards/margins": 1.5327056646347046,
-      "rewards/rejected": -5.720062255859375,
-      "step": 101
-    },
-    {
-      "epoch": 0.32903225806451614,
-      "grad_norm": 0.8937166929244995,
-      "learning_rate": 4.679690839638652e-06,
-      "logits/chosen": -1.9279000759124756,
-      "logits/rejected": -1.8128533363342285,
-      "logps/chosen": -136.27706909179688,
-      "logps/rejected": -172.40426635742188,
-      "loss": 0.5405164957046509,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -4.254796504974365,
-      "rewards/margins": 1.0396616458892822,
-      "rewards/rejected": -5.294458389282227,
-      "step": 102
-    },
-    {
-      "epoch": 0.33225806451612905,
-      "grad_norm": 0.9040670990943909,
-      "learning_rate": 4.673459174711178e-06,
-      "logits/chosen": -1.983347773551941,
-      "logits/rejected": -1.779608964920044,
-      "logps/chosen": -136.58360290527344,
-      "logps/rejected": -177.53189086914062,
-      "loss": 0.5051164627075195,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -4.310066223144531,
-      "rewards/margins": 1.0662548542022705,
-      "rewards/rejected": -5.376320838928223,
-      "step": 103
-    },
-    {
-      "epoch": 0.33548387096774196,
-      "grad_norm": 0.6081104874610901,
-      "learning_rate": 4.6671717055788675e-06,
-      "logits/chosen": -2.1749377250671387,
-      "logits/rejected": -1.8677318096160889,
-      "logps/chosen": -116.5163345336914,
-      "logps/rejected": -171.98924255371094,
-      "loss": 0.3056028187274933,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -3.4489426612854004,
-      "rewards/margins": 2.0040981769561768,
-      "rewards/rejected": -5.453040599822998,
-      "step": 104
-    },
-    {
-      "epoch": 0.3387096774193548,
-      "grad_norm": 0.7601423859596252,
-      "learning_rate": 4.660828593674344e-06,
-      "logits/chosen": -1.764312505722046,
-      "logits/rejected": -1.5848668813705444,
-      "logps/chosen": -151.72991943359375,
-      "logps/rejected": -196.42237854003906,
-      "loss": 0.39171379804611206,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -4.2623291015625,
-      "rewards/margins": 1.3185968399047852,
-      "rewards/rejected": -5.580925464630127,
-      "step": 105
-    },
-    {
-      "epoch": 0.3419354838709677,
-      "grad_norm": 0.7062326669692993,
-      "learning_rate": 4.654430001858874e-06,
-      "logits/chosen": -1.9848129749298096,
-      "logits/rejected": -1.7872685194015503,
-      "logps/chosen": -160.7518310546875,
-      "logps/rejected": -198.28277587890625,
-      "loss": 0.3898007273674011,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -4.177804946899414,
-      "rewards/margins": 1.501824140548706,
-      "rewards/rejected": -5.679628849029541,
-      "step": 106
-    },
-    {
-      "epoch": 0.34516129032258064,
-      "grad_norm": 0.7348251342773438,
-      "learning_rate": 4.64797609441819e-06,
-      "logits/chosen": -2.0225300788879395,
-      "logits/rejected": -1.909589409828186,
-      "logps/chosen": -161.80633544921875,
-      "logps/rejected": -194.06915283203125,
-      "loss": 0.4167836606502533,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -4.554932117462158,
-      "rewards/margins": 1.3303924798965454,
-      "rewards/rejected": -5.885324478149414,
-      "step": 107
-    },
-    {
-      "epoch": 0.34838709677419355,
-      "grad_norm": 0.6127936244010925,
-      "learning_rate": 4.641467037058264e-06,
-      "logits/chosen": -1.7549917697906494,
-      "logits/rejected": -1.5760438442230225,
-      "logps/chosen": -134.44053649902344,
-      "logps/rejected": -176.8062744140625,
-      "loss": 0.3423386216163635,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -3.8029491901397705,
-      "rewards/margins": 1.6363918781280518,
-      "rewards/rejected": -5.439341068267822,
-      "step": 108
-    },
-    {
-      "epoch": 0.35161290322580646,
-      "grad_norm": 0.778430163860321,
-      "learning_rate": 4.634902996901065e-06,
-      "logits/chosen": -2.3479363918304443,
-      "logits/rejected": -2.1825313568115234,
-      "logps/chosen": -135.63031005859375,
-      "logps/rejected": -177.00146484375,
-      "loss": 0.46787750720977783,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -3.918789863586426,
-      "rewards/margins": 1.161969780921936,
-      "rewards/rejected": -5.080759525299072,
-      "step": 109
-    },
-    {
-      "epoch": 0.3548387096774194,
-      "grad_norm": 0.9373202919960022,
-      "learning_rate": 4.628284142480256e-06,
-      "logits/chosen": -1.9162163734436035,
-      "logits/rejected": -1.7722245454788208,
-      "logps/chosen": -154.00108337402344,
-      "logps/rejected": -198.04161071777344,
-      "loss": 0.5773978233337402,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -4.933002471923828,
-      "rewards/margins": 1.0351779460906982,
-      "rewards/rejected": -5.9681806564331055,
-      "step": 110
-    },
-    {
-      "epoch": 0.3580645161290323,
-      "grad_norm": 0.8041221499443054,
-      "learning_rate": 4.621610643736878e-06,
-      "logits/chosen": -2.068186044692993,
-      "logits/rejected": -1.924422264099121,
-      "logps/chosen": -141.45101928710938,
-      "logps/rejected": -175.1551055908203,
-      "loss": 0.5035527944564819,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -4.4271955490112305,
-      "rewards/margins": 1.2815918922424316,
-      "rewards/rejected": -5.708787441253662,
-      "step": 111
-    },
-    {
-      "epoch": 0.36129032258064514,
-      "grad_norm": 0.8020619750022888,
-      "learning_rate": 4.614882672014975e-06,
-      "logits/chosen": -1.8857342004776,
-      "logits/rejected": -1.6965585947036743,
-      "logps/chosen": -145.23861694335938,
-      "logps/rejected": -185.3410186767578,
-      "loss": 0.4687632918357849,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -4.3011884689331055,
-      "rewards/margins": 1.2721554040908813,
-      "rewards/rejected": -5.5733442306518555,
-      "step": 112
-    },
-    {
-      "epoch": 0.36451612903225805,
-      "grad_norm": 0.9149249196052551,
-      "learning_rate": 4.608100400057206e-06,
-      "logits/chosen": -1.7518459558486938,
-      "logits/rejected": -1.629075050354004,
-      "logps/chosen": -138.32345581054688,
-      "logps/rejected": -159.2861785888672,
-      "loss": 0.5886192917823792,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -4.447440147399902,
-      "rewards/margins": 0.8766449689865112,
-      "rewards/rejected": -5.324085235595703,
-      "step": 113
-    },
-    {
-      "epoch": 0.36774193548387096,
-      "grad_norm": 0.9651359915733337,
-      "learning_rate": 4.601264002000401e-06,
-      "logits/chosen": -1.7441390752792358,
-      "logits/rejected": -1.5445384979248047,
-      "logps/chosen": -157.6129150390625,
-      "logps/rejected": -211.08529663085938,
-      "loss": 0.26776331663131714,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -4.520727157592773,
-      "rewards/margins": 2.193711519241333,
-      "rewards/rejected": -6.714438438415527,
-      "step": 114
-    },
-    {
-      "epoch": 0.3709677419354839,
-      "grad_norm": 0.9474994540214539,
-      "learning_rate": 4.594373653371095e-06,
-      "logits/chosen": -2.02126407623291,
-      "logits/rejected": -1.8315093517303467,
-      "logps/chosen": -147.41787719726562,
-      "logps/rejected": -182.80154418945312,
-      "loss": 0.6504412293434143,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -4.341954708099365,
-      "rewards/margins": 0.9969633221626282,
-      "rewards/rejected": -5.3389177322387695,
-      "step": 115
-    },
-    {
-      "epoch": 0.3741935483870968,
-      "grad_norm": 0.7878331542015076,
-      "learning_rate": 4.587429531081019e-06,
-      "logits/chosen": -1.8967138528823853,
-      "logits/rejected": -1.7459156513214111,
-      "logps/chosen": -148.2082977294922,
-      "logps/rejected": -193.70852661132812,
-      "loss": 0.47428834438323975,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -4.678706645965576,
-      "rewards/margins": 1.3782602548599243,
-      "rewards/rejected": -6.056967258453369,
-      "step": 116
-    },
-    {
-      "epoch": 0.3774193548387097,
-      "grad_norm": 0.7913106679916382,
-      "learning_rate": 4.58043181342256e-06,
-      "logits/chosen": -2.071049213409424,
-      "logits/rejected": -1.8497998714447021,
-      "logps/chosen": -130.3113250732422,
-      "logps/rejected": -169.584228515625,
-      "loss": 0.3851902484893799,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -3.724860191345215,
-      "rewards/margins": 1.4488303661346436,
-      "rewards/rejected": -5.1736907958984375,
-      "step": 117
-    },
-    {
-      "epoch": 0.38064516129032255,
-      "grad_norm": 0.8633780479431152,
-      "learning_rate": 4.573380680064182e-06,
-      "logits/chosen": -2.0043578147888184,
-      "logits/rejected": -1.8761675357818604,
-      "logps/chosen": -154.30734252929688,
-      "logps/rejected": -186.54644775390625,
-      "loss": 0.4509797990322113,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -4.510204315185547,
-      "rewards/margins": 1.4796333312988281,
-      "rewards/rejected": -5.989838123321533,
-      "step": 118
-    },
-    {
-      "epoch": 0.38387096774193546,
-      "grad_norm": 0.6685623526573181,
-      "learning_rate": 4.56627631204581e-06,
-      "logits/chosen": -1.7765039205551147,
-      "logits/rejected": -1.6367878913879395,
-      "logps/chosen": -146.83633422851562,
-      "logps/rejected": -185.90267944335938,
-      "loss": 0.3379678726196289,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -4.602056980133057,
-      "rewards/margins": 1.587829828262329,
-      "rewards/rejected": -6.189886569976807,
-      "step": 119
-    },
-    {
-      "epoch": 0.3870967741935484,
-      "grad_norm": 0.7955985069274902,
-      "learning_rate": 4.559118891774188e-06,
-      "logits/chosen": -1.9045928716659546,
-      "logits/rejected": -1.7364416122436523,
-      "logps/chosen": -155.47647094726562,
-      "logps/rejected": -202.11959838867188,
-      "loss": 0.43171846866607666,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -4.851472854614258,
-      "rewards/margins": 1.498561143875122,
-      "rewards/rejected": -6.350033760070801,
-      "step": 120
-    },
-    {
-      "epoch": 0.3903225806451613,
-      "grad_norm": 0.7248550057411194,
-      "learning_rate": 4.551908603018191e-06,
-      "logits/chosen": -1.9593379497528076,
-      "logits/rejected": -1.8014620542526245,
-      "logps/chosen": -152.8414306640625,
-      "logps/rejected": -201.25314331054688,
-      "loss": 0.4184379279613495,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -4.653316497802734,
-      "rewards/margins": 1.7582554817199707,
-      "rewards/rejected": -6.411572456359863,
-      "step": 121
-    },
-    {
-      "epoch": 0.3935483870967742,
-      "grad_norm": 0.9849889874458313,
-      "learning_rate": 4.544645630904106e-06,
-      "logits/chosen": -1.7142870426177979,
-      "logits/rejected": -1.4873881340026855,
-      "logps/chosen": -148.92617797851562,
-      "logps/rejected": -194.00558471679688,
-      "loss": 0.6807998418807983,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -4.889307975769043,
-      "rewards/margins": 1.3624918460845947,
-      "rewards/rejected": -6.251799583435059,
-      "step": 122
-    },
-    {
-      "epoch": 0.3967741935483871,
-      "grad_norm": 0.6775311827659607,
-      "learning_rate": 4.537330161910886e-06,
-      "logits/chosen": -1.7008912563323975,
-      "logits/rejected": -1.497498869895935,
-      "logps/chosen": -151.26678466796875,
-      "logps/rejected": -198.6688995361328,
-      "loss": 0.3310524523258209,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -4.602053642272949,
-      "rewards/margins": 1.850661277770996,
-      "rewards/rejected": -6.452714920043945,
-      "step": 123
-    },
-    {
-      "epoch": 0.4,
-      "grad_norm": 0.678304135799408,
-      "learning_rate": 4.529962383865349e-06,
-      "logits/chosen": -2.0466251373291016,
-      "logits/rejected": -1.8669424057006836,
-      "logps/chosen": -139.44723510742188,
-      "logps/rejected": -187.18295288085938,
-      "loss": 0.4072120189666748,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -4.538634300231934,
-      "rewards/margins": 1.4482630491256714,
-      "rewards/rejected": -5.986896991729736,
-      "step": 124
-    },
-    {
-      "epoch": 0.4032258064516129,
-      "grad_norm": 0.6273353099822998,
-      "learning_rate": 4.522542485937369e-06,
-      "logits/chosen": -1.9951345920562744,
-      "logits/rejected": -1.8110040426254272,
-      "logps/chosen": -153.32125854492188,
-      "logps/rejected": -200.26136779785156,
-      "loss": 0.37352219223976135,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.848642826080322,
-      "rewards/margins": 2.124814987182617,
-      "rewards/rejected": -6.973457336425781,
-      "step": 125
-    },
-    {
-      "epoch": 0.4064516129032258,
-      "grad_norm": 0.6553208827972412,
-      "learning_rate": 4.515070658635013e-06,
-      "logits/chosen": -2.2541942596435547,
-      "logits/rejected": -2.039757490158081,
-      "logps/chosen": -132.11251831054688,
-      "logps/rejected": -180.3010711669922,
-      "loss": 0.28102147579193115,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.45237398147583,
-      "rewards/margins": 2.0114006996154785,
-      "rewards/rejected": -6.463775157928467,
-      "step": 126
-    },
-    {
-      "epoch": 0.4096774193548387,
-      "grad_norm": 0.5834031701087952,
-      "learning_rate": 4.507547093799648e-06,
-      "logits/chosen": -2.0005455017089844,
-      "logits/rejected": -1.7890722751617432,
-      "logps/chosen": -145.47592163085938,
-      "logps/rejected": -187.8236083984375,
-      "loss": 0.2758556008338928,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -4.4232354164123535,
-      "rewards/margins": 1.7412219047546387,
-      "rewards/rejected": -6.164457321166992,
-      "step": 127
-    },
-    {
-      "epoch": 0.4129032258064516,
-      "grad_norm": 0.8683422803878784,
-      "learning_rate": 4.49997198460102e-06,
-      "logits/chosen": -1.6352663040161133,
-      "logits/rejected": -1.5131361484527588,
-      "logps/chosen": -159.2270965576172,
-      "logps/rejected": -194.1593780517578,
-      "loss": 0.5153539180755615,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -5.191349029541016,
-      "rewards/margins": 1.322933316230774,
-      "rewards/rejected": -6.514283180236816,
-      "step": 128
-    },
-    {
-      "epoch": 0.4161290322580645,
-      "grad_norm": 0.6821188926696777,
-      "learning_rate": 4.492345525532288e-06,
-      "logits/chosen": -2.017963171005249,
-      "logits/rejected": -1.8079328536987305,
-      "logps/chosen": -139.15896606445312,
-      "logps/rejected": -180.54229736328125,
-      "loss": 0.370669960975647,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -4.349949836730957,
-      "rewards/margins": 1.6598310470581055,
-      "rewards/rejected": -6.009780406951904,
-      "step": 129
-    },
-    {
-      "epoch": 0.41935483870967744,
-      "grad_norm": 0.8660934567451477,
-      "learning_rate": 4.484667912405038e-06,
-      "logits/chosen": -1.5309303998947144,
-      "logits/rejected": -1.3144739866256714,
-      "logps/chosen": -149.6410369873047,
-      "logps/rejected": -205.96067810058594,
-      "loss": 0.5232619047164917,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -5.3874006271362305,
-      "rewards/margins": 1.597520112991333,
-      "rewards/rejected": -6.984920501708984,
-      "step": 130
-    },
-    {
-      "epoch": 0.42258064516129035,
-      "grad_norm": 0.6375477313995361,
-      "learning_rate": 4.476939342344246e-06,
-      "logits/chosen": -1.796859860420227,
-      "logits/rejected": -1.5831903219223022,
-      "logps/chosen": -138.51058959960938,
-      "logps/rejected": -201.3404541015625,
-      "loss": 0.3488934636116028,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -4.85967493057251,
-      "rewards/margins": 2.1172547340393066,
-      "rewards/rejected": -6.976929664611816,
-      "step": 131
-    },
-    {
-      "epoch": 0.4258064516129032,
-      "grad_norm": 0.7806010246276855,
-      "learning_rate": 4.469160013783231e-06,
-      "logits/chosen": -1.8287787437438965,
-      "logits/rejected": -1.6582368612289429,
-      "logps/chosen": -161.08322143554688,
-      "logps/rejected": -204.06918334960938,
-      "loss": 0.4884554147720337,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -5.596766471862793,
-      "rewards/margins": 1.6360560655593872,
-      "rewards/rejected": -7.232822418212891,
-      "step": 132
-    },
-    {
-      "epoch": 0.4290322580645161,
-      "grad_norm": 0.7818947434425354,
-      "learning_rate": 4.461330126458544e-06,
-      "logits/chosen": -1.6161482334136963,
-      "logits/rejected": -1.4406477212905884,
-      "logps/chosen": -164.41973876953125,
-      "logps/rejected": -198.6041259765625,
-      "loss": 0.4584003686904907,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -5.313333511352539,
-      "rewards/margins": 1.2815909385681152,
-      "rewards/rejected": -6.594923973083496,
-      "step": 133
-    },
-    {
-      "epoch": 0.432258064516129,
-      "grad_norm": 0.806587815284729,
-      "learning_rate": 4.453449881404849e-06,
-      "logits/chosen": -1.5250306129455566,
-      "logits/rejected": -1.3801239728927612,
-      "logps/chosen": -167.21826171875,
-      "logps/rejected": -209.19235229492188,
-      "loss": 0.4259398579597473,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -5.2768168449401855,
-      "rewards/margins": 1.6459184885025024,
-      "rewards/rejected": -6.922735214233398,
-      "step": 134
-    },
-    {
-      "epoch": 0.43548387096774194,
-      "grad_norm": 0.7138063907623291,
-      "learning_rate": 4.445519480949761e-06,
-      "logits/chosen": -2.032522201538086,
-      "logits/rejected": -1.8461859226226807,
-      "logps/chosen": -139.901611328125,
-      "logps/rejected": -176.5264434814453,
-      "loss": 0.49521660804748535,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -4.6639556884765625,
-      "rewards/margins": 1.493511438369751,
-      "rewards/rejected": -6.157467842102051,
-      "step": 135
-    },
-    {
-      "epoch": 0.43870967741935485,
-      "grad_norm": 0.754875123500824,
-      "learning_rate": 4.437539128708647e-06,
-      "logits/chosen": -2.0726211071014404,
-      "logits/rejected": -1.909871220588684,
-      "logps/chosen": -146.85386657714844,
-      "logps/rejected": -191.2569580078125,
-      "loss": 0.406306654214859,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -4.798932075500488,
-      "rewards/margins": 1.6079185009002686,
-      "rewards/rejected": -6.4068498611450195,
-      "step": 136
-    },
-    {
-      "epoch": 0.44193548387096776,
-      "grad_norm": 0.8478382229804993,
-      "learning_rate": 4.429509029579405e-06,
-      "logits/chosen": -1.7719223499298096,
-      "logits/rejected": -1.5842394828796387,
-      "logps/chosen": -148.5624542236328,
-      "logps/rejected": -185.21328735351562,
-      "loss": 0.5379185676574707,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -5.318252086639404,
-      "rewards/margins": 1.0121225118637085,
-      "rewards/rejected": -6.330374240875244,
-      "step": 137
-    },
-    {
-      "epoch": 0.44516129032258067,
-      "grad_norm": 0.5856479406356812,
-      "learning_rate": 4.421429389737196e-06,
-      "logits/chosen": -2.040005683898926,
-      "logits/rejected": -1.7596739530563354,
-      "logps/chosen": -122.85853576660156,
-      "logps/rejected": -174.30482482910156,
-      "loss": 0.28845539689064026,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -4.208980560302734,
-      "rewards/margins": 2.153196096420288,
-      "rewards/rejected": -6.362176418304443,
-      "step": 138
-    },
-    {
-      "epoch": 0.4483870967741935,
-      "grad_norm": 0.8572343587875366,
-      "learning_rate": 4.413300416629154e-06,
-      "logits/chosen": -1.5686633586883545,
-      "logits/rejected": -1.4056203365325928,
-      "logps/chosen": -174.7129669189453,
-      "logps/rejected": -208.05381774902344,
-      "loss": 0.5018977522850037,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -5.936600685119629,
-      "rewards/margins": 1.217970848083496,
-      "rewards/rejected": -7.154571056365967,
-      "step": 139
-    },
-    {
-      "epoch": 0.45161290322580644,
-      "grad_norm": 0.96775883436203,
-      "learning_rate": 4.4051223189690585e-06,
-      "logits/chosen": -1.9180068969726562,
-      "logits/rejected": -1.7714178562164307,
-      "logps/chosen": -158.73330688476562,
-      "logps/rejected": -201.98733520507812,
-      "loss": 0.6449487209320068,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -5.293525218963623,
-      "rewards/margins": 1.344764232635498,
-      "rewards/rejected": -6.638289928436279,
-      "step": 140
-    },
-    {
-      "epoch": 0.45483870967741935,
-      "grad_norm": 0.7184005379676819,
-      "learning_rate": 4.396895306731978e-06,
-      "logits/chosen": -2.029144048690796,
-      "logits/rejected": -1.8254203796386719,
-      "logps/chosen": -159.30470275878906,
-      "logps/rejected": -202.5131072998047,
-      "loss": 0.3617313802242279,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -5.206952095031738,
-      "rewards/margins": 1.84663987159729,
-      "rewards/rejected": -7.053591728210449,
-      "step": 141
-    },
-    {
-      "epoch": 0.45806451612903226,
-      "grad_norm": 0.6119160056114197,
-      "learning_rate": 4.388619591148875e-06,
-      "logits/chosen": -1.702289342880249,
-      "logits/rejected": -1.4510695934295654,
-      "logps/chosen": -146.5979766845703,
-      "logps/rejected": -190.3888397216797,
-      "loss": 0.3313656151294708,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -4.872472286224365,
-      "rewards/margins": 2.124823808670044,
-      "rewards/rejected": -6.997296333312988,
-      "step": 142
-    },
-    {
-      "epoch": 0.4612903225806452,
-      "grad_norm": 0.8173593282699585,
-      "learning_rate": 4.380295384701186e-06,
-      "logits/chosen": -1.592297077178955,
-      "logits/rejected": -1.371172308921814,
-      "logps/chosen": -161.451416015625,
-      "logps/rejected": -208.75277709960938,
-      "loss": 0.48401570320129395,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -5.649110317230225,
-      "rewards/margins": 2.1297523975372314,
-      "rewards/rejected": -7.778862476348877,
-      "step": 143
-    },
-    {
-      "epoch": 0.4645161290322581,
-      "grad_norm": 0.8249958753585815,
-      "learning_rate": 4.371922901115367e-06,
-      "logits/chosen": -1.7442785501480103,
-      "logits/rejected": -1.5723543167114258,
-      "logps/chosen": -154.55165100097656,
-      "logps/rejected": -187.0003204345703,
-      "loss": 0.504601776599884,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -5.453955173492432,
-      "rewards/margins": 1.3852546215057373,
-      "rewards/rejected": -6.839210510253906,
-      "step": 144
-    },
-    {
-      "epoch": 0.46774193548387094,
-      "grad_norm": 0.6368381381034851,
-      "learning_rate": 4.3635023553574e-06,
-      "logits/chosen": -1.7754096984863281,
-      "logits/rejected": -1.607948899269104,
-      "logps/chosen": -134.15525817871094,
-      "logps/rejected": -196.18272399902344,
-      "loss": 0.25741511583328247,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.806275367736816,
-      "rewards/margins": 2.7785263061523438,
-      "rewards/rejected": -7.584801197052002,
-      "step": 145
-    },
-    {
-      "epoch": 0.47096774193548385,
-      "grad_norm": 0.615225076675415,
-      "learning_rate": 4.355033963627277e-06,
-      "logits/chosen": -1.9712847471237183,
-      "logits/rejected": -1.742748737335205,
-      "logps/chosen": -144.54286193847656,
-      "logps/rejected": -198.54017639160156,
-      "loss": 0.29680246114730835,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.868497371673584,
-      "rewards/margins": 2.3029892444610596,
-      "rewards/rejected": -7.1714863777160645,
-      "step": 146
-    },
-    {
-      "epoch": 0.47419354838709676,
-      "grad_norm": 0.7174091339111328,
-      "learning_rate": 4.346517943353454e-06,
-      "logits/chosen": -1.79178786277771,
-      "logits/rejected": -1.5321691036224365,
-      "logps/chosen": -157.57235717773438,
-      "logps/rejected": -212.04339599609375,
-      "loss": 0.3283613622188568,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -5.730096817016602,
-      "rewards/margins": 2.1431491374969482,
-      "rewards/rejected": -7.873246192932129,
-      "step": 147
-    },
-    {
-      "epoch": 0.4774193548387097,
-      "grad_norm": 0.9096499085426331,
-      "learning_rate": 4.337954513187261e-06,
-      "logits/chosen": -1.5230883359909058,
-      "logits/rejected": -1.295538067817688,
-      "logps/chosen": -171.4082794189453,
-      "logps/rejected": -230.9026641845703,
-      "loss": 0.4473147392272949,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -6.100051403045654,
-      "rewards/margins": 2.206608772277832,
-      "rewards/rejected": -8.306659698486328,
-      "step": 148
-    },
-    {
-      "epoch": 0.4806451612903226,
-      "grad_norm": 0.850471019744873,
-      "learning_rate": 4.3293438929972896e-06,
-      "logits/chosen": -1.798743486404419,
-      "logits/rejected": -1.6180821657180786,
-      "logps/chosen": -151.7516632080078,
-      "logps/rejected": -185.33834838867188,
-      "loss": 0.49031680822372437,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -5.337520122528076,
-      "rewards/margins": 1.3362945318222046,
-      "rewards/rejected": -6.67381477355957,
-      "step": 149
-    },
-    {
-      "epoch": 0.4838709677419355,
-      "grad_norm": 0.8199206590652466,
-      "learning_rate": 4.320686303863752e-06,
-      "logits/chosen": -1.822701334953308,
-      "logits/rejected": -1.6954931020736694,
-      "logps/chosen": -173.2248992919922,
-      "logps/rejected": -206.9976043701172,
-      "loss": 0.47184550762176514,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -5.998141288757324,
-      "rewards/margins": 1.5172470808029175,
-      "rewards/rejected": -7.515388488769531,
-      "step": 150
-    },
-    {
-      "epoch": 0.4870967741935484,
-      "grad_norm": 0.6827844381332397,
-      "learning_rate": 4.3119819680728e-06,
-      "logits/chosen": -1.2705650329589844,
-      "logits/rejected": -1.084989070892334,
-      "logps/chosen": -177.86563110351562,
-      "logps/rejected": -215.4129638671875,
-      "loss": 0.3346361815929413,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.931788921356201,
-      "rewards/margins": 1.785906195640564,
-      "rewards/rejected": -7.7176947593688965,
-      "step": 151
-    },
-    {
-      "epoch": 0.49032258064516127,
-      "grad_norm": 0.8796378374099731,
-      "learning_rate": 4.303231109110821e-06,
-      "logits/chosen": -1.7787282466888428,
-      "logits/rejected": -1.5340662002563477,
-      "logps/chosen": -155.4757843017578,
-      "logps/rejected": -198.17584228515625,
-      "loss": 0.4421862065792084,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -5.767148017883301,
-      "rewards/margins": 1.6090242862701416,
-      "rewards/rejected": -7.37617301940918,
-      "step": 152
-    },
-    {
-      "epoch": 0.4935483870967742,
-      "grad_norm": 0.8411730527877808,
-      "learning_rate": 4.294433951658698e-06,
-      "logits/chosen": -1.648699164390564,
-      "logits/rejected": -1.4484217166900635,
-      "logps/chosen": -162.52825927734375,
-      "logps/rejected": -196.4520263671875,
-      "loss": 0.4758791923522949,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -5.455954551696777,
-      "rewards/margins": 1.4209301471710205,
-      "rewards/rejected": -6.876884460449219,
-      "step": 153
-    },
-    {
-      "epoch": 0.4967741935483871,
-      "grad_norm": 0.6339321732521057,
-      "learning_rate": 4.285590721586039e-06,
-      "logits/chosen": -1.8830703496932983,
-      "logits/rejected": -1.5869327783584595,
-      "logps/chosen": -139.29800415039062,
-      "logps/rejected": -196.91796875,
-      "loss": 0.22343483567237854,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.459966659545898,
-      "rewards/margins": 2.71433162689209,
-      "rewards/rejected": -7.174298286437988,
-      "step": 154
-    },
-    {
-      "epoch": 0.5,
-      "grad_norm": 0.6575607657432556,
-      "learning_rate": 4.276701645945384e-06,
-      "logits/chosen": -2.1467416286468506,
-      "logits/rejected": -1.892794132232666,
-      "logps/chosen": -154.65928649902344,
-      "logps/rejected": -213.82528686523438,
-      "loss": 0.2633991241455078,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.381623268127441,
-      "rewards/margins": 2.5349740982055664,
-      "rewards/rejected": -7.916597366333008,
-      "step": 155
-    },
-    {
-      "epoch": 0.5032258064516129,
-      "grad_norm": 0.8809133768081665,
-      "learning_rate": 4.267766952966369e-06,
-      "logits/chosen": -1.8720148801803589,
-      "logits/rejected": -1.607576847076416,
-      "logps/chosen": -149.6011962890625,
-      "logps/rejected": -193.8123321533203,
-      "loss": 0.4617472290992737,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -5.031999111175537,
-      "rewards/margins": 2.0407233238220215,
-      "rewards/rejected": -7.072722434997559,
-      "step": 156
-    },
-    {
-      "epoch": 0.5064516129032258,
-      "grad_norm": 0.660529613494873,
-      "learning_rate": 4.25878687204987e-06,
-      "logits/chosen": -2.024183511734009,
-      "logits/rejected": -1.8310062885284424,
-      "logps/chosen": -152.48403930664062,
-      "logps/rejected": -203.5169677734375,
-      "loss": 0.3005449175834656,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -4.750589370727539,
-      "rewards/margins": 2.2591700553894043,
-      "rewards/rejected": -7.009759902954102,
-      "step": 157
-    },
-    {
-      "epoch": 0.5096774193548387,
-      "grad_norm": 0.8802087306976318,
-      "learning_rate": 4.2497616337621115e-06,
-      "logits/chosen": -1.8386707305908203,
-      "logits/rejected": -1.586627721786499,
-      "logps/chosen": -154.0318603515625,
-      "logps/rejected": -205.48057556152344,
-      "loss": 0.4543035924434662,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -5.153034210205078,
-      "rewards/margins": 2.193976640701294,
-      "rewards/rejected": -7.347011566162109,
-      "step": 158
-    },
-    {
-      "epoch": 0.5129032258064516,
-      "grad_norm": 0.9531585574150085,
-      "learning_rate": 4.240691469828745e-06,
-      "logits/chosen": -2.1372509002685547,
-      "logits/rejected": -2.0333480834960938,
-      "logps/chosen": -169.7491912841797,
-      "logps/rejected": -210.4459228515625,
-      "loss": 0.5746184587478638,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -5.571075439453125,
-      "rewards/margins": 1.5784751176834106,
-      "rewards/rejected": -7.149550914764404,
-      "step": 159
-    },
-    {
-      "epoch": 0.5161290322580645,
-      "grad_norm": 0.7346033453941345,
-      "learning_rate": 4.231576613128902e-06,
-      "logits/chosen": -2.0919246673583984,
-      "logits/rejected": -1.8873045444488525,
-      "logps/chosen": -153.13253784179688,
-      "logps/rejected": -200.02569580078125,
-      "loss": 0.3784785270690918,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -5.044937610626221,
-      "rewards/margins": 2.0969479084014893,
-      "rewards/rejected": -7.141885280609131,
-      "step": 160
-    },
-    {
-      "epoch": 0.5193548387096775,
-      "grad_norm": 0.9784480929374695,
-      "learning_rate": 4.222417297689217e-06,
-      "logits/chosen": -1.9859681129455566,
-      "logits/rejected": -1.7799007892608643,
-      "logps/chosen": -150.41250610351562,
-      "logps/rejected": -184.49789428710938,
-      "loss": 0.5596226453781128,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -4.557621479034424,
-      "rewards/margins": 1.3568779230117798,
-      "rewards/rejected": -5.914499282836914,
-      "step": 161
-    },
-    {
-      "epoch": 0.5225806451612903,
-      "grad_norm": 0.87236088514328,
-      "learning_rate": 4.21321375867781e-06,
-      "logits/chosen": -1.7091286182403564,
-      "logits/rejected": -1.5344655513763428,
-      "logps/chosen": -153.80831909179688,
-      "logps/rejected": -193.2026824951172,
-      "loss": 0.48372751474380493,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -5.051312446594238,
-      "rewards/margins": 1.568933129310608,
-      "rewards/rejected": -6.620245456695557,
-      "step": 162
-    },
-    {
-      "epoch": 0.5258064516129032,
-      "grad_norm": 0.7265849709510803,
-      "learning_rate": 4.203966232398261e-06,
-      "logits/chosen": -2.2141776084899902,
-      "logits/rejected": -1.72068190574646,
-      "logps/chosen": -135.98233032226562,
-      "logps/rejected": -195.78211975097656,
-      "loss": 0.3879455327987671,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -4.7070722579956055,
-      "rewards/margins": 2.4663829803466797,
-      "rewards/rejected": -7.173455238342285,
-      "step": 163
-    },
-    {
-      "epoch": 0.5290322580645161,
-      "grad_norm": 0.7596819996833801,
-      "learning_rate": 4.194674956283529e-06,
-      "logits/chosen": -1.6569631099700928,
-      "logits/rejected": -1.389373540878296,
-      "logps/chosen": -152.79058837890625,
-      "logps/rejected": -198.65692138671875,
-      "loss": 0.39065128564834595,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -4.617537975311279,
-      "rewards/margins": 2.011427402496338,
-      "rewards/rejected": -6.628965377807617,
-      "step": 164
-    },
-    {
-      "epoch": 0.532258064516129,
-      "grad_norm": 0.7323077917098999,
-      "learning_rate": 4.185340168889869e-06,
-      "logits/chosen": -2.1654136180877686,
-      "logits/rejected": -1.9493908882141113,
-      "logps/chosen": -160.08444213867188,
-      "logps/rejected": -210.05970764160156,
-      "loss": 0.3324809670448303,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -5.448432922363281,
-      "rewards/margins": 2.039762496948242,
-      "rewards/rejected": -7.48819637298584,
-      "step": 165
-    },
-    {
-      "epoch": 0.535483870967742,
-      "grad_norm": 0.5825879573822021,
-      "learning_rate": 4.175962109890697e-06,
-      "logits/chosen": -1.89616858959198,
-      "logits/rejected": -1.663448452949524,
-      "logps/chosen": -126.10738372802734,
-      "logps/rejected": -181.8113250732422,
-      "loss": 0.2588610351085663,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.298904895782471,
-      "rewards/margins": 2.3480236530303955,
-      "rewards/rejected": -6.646928787231445,
-      "step": 166
-    },
-    {
-      "epoch": 0.5387096774193548,
-      "grad_norm": 0.9141851663589478,
-      "learning_rate": 4.166541020070442e-06,
-      "logits/chosen": -1.9237167835235596,
-      "logits/rejected": -1.6572012901306152,
-      "logps/chosen": -141.69476318359375,
-      "logps/rejected": -187.3744659423828,
-      "loss": 0.4472041130065918,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -4.35133171081543,
-      "rewards/margins": 1.9088027477264404,
-      "rewards/rejected": -6.260134220123291,
-      "step": 167
-    },
-    {
-      "epoch": 0.5419354838709678,
-      "grad_norm": 1.011690378189087,
-      "learning_rate": 4.157077141318361e-06,
-      "logits/chosen": -2.032428503036499,
-      "logits/rejected": -1.7909128665924072,
-      "logps/chosen": -138.17669677734375,
-      "logps/rejected": -184.1501007080078,
-      "loss": 0.3964395225048065,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -4.0889787673950195,
-      "rewards/margins": 1.6615996360778809,
-      "rewards/rejected": -5.750578880310059,
-      "step": 168
-    },
-    {
-      "epoch": 0.5451612903225806,
-      "grad_norm": 0.6844170689582825,
-      "learning_rate": 4.14757071662233e-06,
-      "logits/chosen": -2.0451266765594482,
-      "logits/rejected": -1.8144954442977905,
-      "logps/chosen": -128.15585327148438,
-      "logps/rejected": -182.09280395507812,
-      "loss": 0.3033115863800049,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -4.129886627197266,
-      "rewards/margins": 2.4071736335754395,
-      "rewards/rejected": -6.537059783935547,
-      "step": 169
-    },
-    {
-      "epoch": 0.5483870967741935,
-      "grad_norm": 0.7212929725646973,
-      "learning_rate": 4.138021990062606e-06,
-      "logits/chosen": -1.9469044208526611,
-      "logits/rejected": -1.7142751216888428,
-      "logps/chosen": -150.60662841796875,
-      "logps/rejected": -205.27867126464844,
-      "loss": 0.30135634541511536,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -4.973240375518799,
-      "rewards/margins": 2.420989513397217,
-      "rewards/rejected": -7.394229412078857,
-      "step": 170
-    },
-    {
-      "epoch": 0.5516129032258065,
-      "grad_norm": 0.9071898460388184,
-      "learning_rate": 4.128431206805556e-06,
-      "logits/chosen": -1.8445392847061157,
-      "logits/rejected": -1.6512104272842407,
-      "logps/chosen": -167.3872528076172,
-      "logps/rejected": -207.1650390625,
-      "loss": 0.4723031520843506,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -5.54469108581543,
-      "rewards/margins": 1.5533931255340576,
-      "rewards/rejected": -7.098084449768066,
-      "step": 171
-    },
-    {
-      "epoch": 0.5548387096774193,
-      "grad_norm": 0.7936817407608032,
-      "learning_rate": 4.118798613097367e-06,
-      "logits/chosen": -1.840900182723999,
-      "logits/rejected": -1.6130969524383545,
-      "logps/chosen": -162.12991333007812,
-      "logps/rejected": -213.62379455566406,
-      "loss": 0.3520919978618622,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -5.226555824279785,
-      "rewards/margins": 2.199876546859741,
-      "rewards/rejected": -7.426431655883789,
-      "step": 172
-    },
-    {
-      "epoch": 0.5580645161290323,
-      "grad_norm": 0.8109973073005676,
-      "learning_rate": 4.109124456257722e-06,
-      "logits/chosen": -1.7801843881607056,
-      "logits/rejected": -1.5753393173217773,
-      "logps/chosen": -164.92132568359375,
-      "logps/rejected": -209.00552368164062,
-      "loss": 0.37818557024002075,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -5.394079208374023,
-      "rewards/margins": 2.0151584148406982,
-      "rewards/rejected": -7.409237861633301,
-      "step": 173
-    },
-    {
-      "epoch": 0.5612903225806452,
-      "grad_norm": 0.8981033563613892,
-      "learning_rate": 4.099408984673448e-06,
-      "logits/chosen": -1.9823046922683716,
-      "logits/rejected": -1.7255879640579224,
-      "logps/chosen": -147.62057495117188,
-      "logps/rejected": -189.5701904296875,
-      "loss": 0.46110501885414124,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -5.077602386474609,
-      "rewards/margins": 1.8654961585998535,
-      "rewards/rejected": -6.943098545074463,
-      "step": 174
-    },
-    {
-      "epoch": 0.5645161290322581,
-      "grad_norm": 0.725642740726471,
-      "learning_rate": 4.089652447792141e-06,
-      "logits/chosen": -1.9010741710662842,
-      "logits/rejected": -1.595578908920288,
-      "logps/chosen": -152.0930938720703,
-      "logps/rejected": -208.27474975585938,
-      "loss": 0.31729233264923096,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -5.1363019943237305,
-      "rewards/margins": 2.5898420810699463,
-      "rewards/rejected": -7.7261433601379395,
-      "step": 175
-    },
-    {
-      "epoch": 0.567741935483871,
-      "grad_norm": 0.9278317093849182,
-      "learning_rate": 4.07985509611576e-06,
-      "logits/chosen": -1.8088712692260742,
-      "logits/rejected": -1.6028716564178467,
-      "logps/chosen": -159.88787841796875,
-      "logps/rejected": -204.56011962890625,
-      "loss": 0.5430783629417419,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -6.032896041870117,
-      "rewards/margins": 1.7545185089111328,
-      "rewards/rejected": -7.78741455078125,
-      "step": 176
-    },
-    {
-      "epoch": 0.5709677419354838,
-      "grad_norm": 0.6002618074417114,
-      "learning_rate": 4.070017181194199e-06,
-      "logits/chosen": -2.0451059341430664,
-      "logits/rejected": -1.8140971660614014,
-      "logps/chosen": -144.2393341064453,
-      "logps/rejected": -188.33346557617188,
-      "loss": 0.26023587584495544,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.727838516235352,
-      "rewards/margins": 2.057196617126465,
-      "rewards/rejected": -6.785035133361816,
-      "step": 177
-    },
-    {
-      "epoch": 0.5741935483870968,
-      "grad_norm": 0.7452195882797241,
-      "learning_rate": 4.0601389556188205e-06,
-      "logits/chosen": -1.6749149560928345,
-      "logits/rejected": -1.5378282070159912,
-      "logps/chosen": -163.5107879638672,
-      "logps/rejected": -200.22984313964844,
-      "loss": 0.37158316373825073,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.478737831115723,
-      "rewards/margins": 1.5227606296539307,
-      "rewards/rejected": -7.001498222351074,
-      "step": 178
-    },
-    {
-      "epoch": 0.5774193548387097,
-      "grad_norm": 0.6364844441413879,
-      "learning_rate": 4.050220673015979e-06,
-      "logits/chosen": -2.103788137435913,
-      "logits/rejected": -1.8177173137664795,
-      "logps/chosen": -142.53990173339844,
-      "logps/rejected": -198.9427490234375,
-      "loss": 0.2621723413467407,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.8717546463012695,
-      "rewards/margins": 2.7382116317749023,
-      "rewards/rejected": -7.609966278076172,
-      "step": 179
-    },
-    {
-      "epoch": 0.5806451612903226,
-      "grad_norm": 1.1201379299163818,
-      "learning_rate": 4.040262588040503e-06,
-      "logits/chosen": -1.8769949674606323,
-      "logits/rejected": -1.7055038213729858,
-      "logps/chosen": -156.72299194335938,
-      "logps/rejected": -197.8905792236328,
-      "loss": 0.5500263571739197,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -5.502621650695801,
-      "rewards/margins": 1.5961947441101074,
-      "rewards/rejected": -7.09881591796875,
-      "step": 180
-    },
-    {
-      "epoch": 0.5838709677419355,
-      "grad_norm": 0.524496853351593,
-      "learning_rate": 4.030264956369158e-06,
-      "logits/chosen": -2.2530908584594727,
-      "logits/rejected": -2.0049355030059814,
-      "logps/chosen": -134.3117218017578,
-      "logps/rejected": -177.72940063476562,
-      "loss": 0.2530081868171692,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.458559989929199,
-      "rewards/margins": 2.3075766563415527,
-      "rewards/rejected": -6.766136169433594,
-      "step": 181
-    },
-    {
-      "epoch": 0.5870967741935483,
-      "grad_norm": 0.9195796251296997,
-      "learning_rate": 4.020228034694083e-06,
-      "logits/chosen": -1.649029016494751,
-      "logits/rejected": -1.4227406978607178,
-      "logps/chosen": -160.95321655273438,
-      "logps/rejected": -215.07814025878906,
-      "loss": 0.3454977869987488,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -5.202361106872559,
-      "rewards/margins": 2.460273027420044,
-      "rewards/rejected": -7.662634372711182,
-      "step": 182
-    },
-    {
-      "epoch": 0.5903225806451613,
-      "grad_norm": 0.5352334380149841,
-      "learning_rate": 4.010152080716202e-06,
-      "logits/chosen": -2.089765787124634,
-      "logits/rejected": -1.8100600242614746,
-      "logps/chosen": -138.4512176513672,
-      "logps/rejected": -194.85545349121094,
-      "loss": 0.20018336176872253,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -4.481593132019043,
-      "rewards/margins": 2.971552610397339,
-      "rewards/rejected": -7.453145980834961,
-      "step": 183
-    },
-    {
-      "epoch": 0.5935483870967742,
-      "grad_norm": 0.7269738912582397,
-      "learning_rate": 4.000037353138602e-06,
-      "logits/chosen": -1.8054555654525757,
-      "logits/rejected": -1.598811388015747,
-      "logps/chosen": -151.69924926757812,
-      "logps/rejected": -187.41851806640625,
-      "loss": 0.3388046622276306,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -5.017839431762695,
-      "rewards/margins": 1.4641319513320923,
-      "rewards/rejected": -6.481971740722656,
-      "step": 184
-    },
-    {
-      "epoch": 0.5967741935483871,
-      "grad_norm": 0.8876580595970154,
-      "learning_rate": 3.989884111659893e-06,
-      "logits/chosen": -1.8593127727508545,
-      "logits/rejected": -1.6329864263534546,
-      "logps/chosen": -148.35565185546875,
-      "logps/rejected": -205.98257446289062,
-      "loss": 0.40850383043289185,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -5.808213233947754,
-      "rewards/margins": 2.492269277572632,
-      "rewards/rejected": -8.300481796264648,
-      "step": 185
-    },
-    {
-      "epoch": 0.6,
-      "grad_norm": 0.7655280232429504,
-      "learning_rate": 3.979692616967543e-06,
-      "logits/chosen": -2.026470422744751,
-      "logits/rejected": -1.7303847074508667,
-      "logps/chosen": -139.6387481689453,
-      "logps/rejected": -198.58078002929688,
-      "loss": 0.3410119414329529,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -5.268441200256348,
-      "rewards/margins": 2.493990898132324,
-      "rewards/rejected": -7.762432098388672,
-      "step": 186
-    },
-    {
-      "epoch": 0.603225806451613,
-      "grad_norm": 0.8385968804359436,
-      "learning_rate": 3.969463130731183e-06,
-      "logits/chosen": -1.709697961807251,
-      "logits/rejected": -1.5222282409667969,
-      "logps/chosen": -189.8544158935547,
-      "logps/rejected": -231.8427734375,
-      "loss": 0.39192935824394226,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -6.106621265411377,
-      "rewards/margins": 2.2979493141174316,
-      "rewards/rejected": -8.404570579528809,
-      "step": 187
-    },
-    {
-      "epoch": 0.6064516129032258,
-      "grad_norm": 0.9917109608650208,
-      "learning_rate": 3.959195915595886e-06,
-      "logits/chosen": -1.9611066579818726,
-      "logits/rejected": -1.8462345600128174,
-      "logps/chosen": -149.73031616210938,
-      "logps/rejected": -192.25274658203125,
-      "loss": 0.5630193948745728,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -5.245723724365234,
-      "rewards/margins": 1.6644206047058105,
-      "rewards/rejected": -6.910144329071045,
-      "step": 188
-    },
-    {
-      "epoch": 0.6096774193548387,
-      "grad_norm": 0.8959775567054749,
-      "learning_rate": 3.948891235175425e-06,
-      "logits/chosen": -1.646284580230713,
-      "logits/rejected": -1.379030466079712,
-      "logps/chosen": -160.4254913330078,
-      "logps/rejected": -211.20452880859375,
-      "loss": 0.4511195719242096,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -5.744508266448975,
-      "rewards/margins": 2.126063346862793,
-      "rewards/rejected": -7.870571613311768,
-      "step": 189
-    },
-    {
-      "epoch": 0.6129032258064516,
-      "grad_norm": 0.5826210379600525,
-      "learning_rate": 3.938549354045508e-06,
-      "logits/chosen": -1.51802396774292,
-      "logits/rejected": -1.2956496477127075,
-      "logps/chosen": -157.11676025390625,
-      "logps/rejected": -213.88731384277344,
-      "loss": 0.22183524072170258,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -5.172170639038086,
-      "rewards/margins": 2.546001434326172,
-      "rewards/rejected": -7.718172073364258,
-      "step": 190
-    },
-    {
-      "epoch": 0.6161290322580645,
-      "grad_norm": 1.0212091207504272,
-      "learning_rate": 3.9281705377369814e-06,
-      "logits/chosen": -2.1612565517425537,
-      "logits/rejected": -1.9319891929626465,
-      "logps/chosen": -150.02903747558594,
-      "logps/rejected": -197.55706787109375,
-      "loss": 0.5443016290664673,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -4.873559951782227,
-      "rewards/margins": 2.2266910076141357,
-      "rewards/rejected": -7.1002516746521,
-      "step": 191
-    },
-    {
-      "epoch": 0.6193548387096774,
-      "grad_norm": 0.7122681140899658,
-      "learning_rate": 3.917755052729011e-06,
-      "logits/chosen": -2.106039524078369,
-      "logits/rejected": -1.750291347503662,
-      "logps/chosen": -144.48170471191406,
-      "logps/rejected": -200.36827087402344,
-      "loss": 0.24308393895626068,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -4.827795505523682,
-      "rewards/margins": 2.734774112701416,
-      "rewards/rejected": -7.562569618225098,
-      "step": 192
-    },
-    {
-      "epoch": 0.6225806451612903,
-      "grad_norm": 0.7401944994926453,
-      "learning_rate": 3.907303166442245e-06,
-      "logits/chosen": -1.975453495979309,
-      "logits/rejected": -1.845196008682251,
-      "logps/chosen": -182.4921417236328,
-      "logps/rejected": -214.56707763671875,
-      "loss": 0.33322787284851074,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.488518714904785,
-      "rewards/margins": 1.8690803050994873,
-      "rewards/rejected": -7.357598781585693,
-      "step": 193
-    },
-    {
-      "epoch": 0.6258064516129033,
-      "grad_norm": 0.8645218014717102,
-      "learning_rate": 3.896815147231943e-06,
-      "logits/chosen": -1.6347211599349976,
-      "logits/rejected": -1.384649634361267,
-      "logps/chosen": -169.53807067871094,
-      "logps/rejected": -217.01519775390625,
-      "loss": 0.40609508752822876,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -5.351966857910156,
-      "rewards/margins": 1.9269593954086304,
-      "rewards/rejected": -7.278926372528076,
-      "step": 194
-    },
-    {
-      "epoch": 0.6290322580645161,
-      "grad_norm": 0.9169072508811951,
-      "learning_rate": 3.8862912643810895e-06,
-      "logits/chosen": -1.6547520160675049,
-      "logits/rejected": -1.3922858238220215,
-      "logps/chosen": -169.7218017578125,
-      "logps/rejected": -233.12631225585938,
-      "loss": 0.47453373670578003,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -5.81014347076416,
-      "rewards/margins": 2.4779911041259766,
-      "rewards/rejected": -8.288135528564453,
-      "step": 195
-    },
-    {
-      "epoch": 0.632258064516129,
-      "grad_norm": 0.6964331865310669,
-      "learning_rate": 3.875731788093478e-06,
-      "logits/chosen": -1.6203590631484985,
-      "logits/rejected": -1.37406587600708,
-      "logps/chosen": -172.435302734375,
-      "logps/rejected": -223.4750213623047,
-      "loss": 0.28000563383102417,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -5.44480037689209,
-      "rewards/margins": 2.7131171226501465,
-      "rewards/rejected": -8.157917976379395,
-      "step": 196
-    },
-    {
-      "epoch": 0.635483870967742,
-      "grad_norm": 1.0018017292022705,
-      "learning_rate": 3.865136989486776e-06,
-      "logits/chosen": -2.125566244125366,
-      "logits/rejected": -1.9877275228500366,
-      "logps/chosen": -174.42189025878906,
-      "logps/rejected": -209.52536010742188,
-      "loss": 0.4606223404407501,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -5.8737664222717285,
-      "rewards/margins": 1.626832365989685,
-      "rewards/rejected": -7.500598907470703,
-      "step": 197
-    },
-    {
-      "epoch": 0.6387096774193548,
-      "grad_norm": 0.6335120797157288,
-      "learning_rate": 3.8545071405855574e-06,
-      "logits/chosen": -2.0536115169525146,
-      "logits/rejected": -1.8321561813354492,
-      "logps/chosen": -154.74710083007812,
-      "logps/rejected": -209.40707397460938,
-      "loss": 0.23459070920944214,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -5.506530284881592,
-      "rewards/margins": 2.8299872875213623,
-      "rewards/rejected": -8.336517333984375,
-      "step": 198
-    },
-    {
-      "epoch": 0.6419354838709678,
-      "grad_norm": 0.9899296164512634,
-      "learning_rate": 3.8438425143143265e-06,
-      "logits/chosen": -1.8322534561157227,
-      "logits/rejected": -1.6016404628753662,
-      "logps/chosen": -134.58560180664062,
-      "logps/rejected": -187.51687622070312,
-      "loss": 0.44235897064208984,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -4.430156707763672,
-      "rewards/margins": 2.332374095916748,
-      "rewards/rejected": -6.762531280517578,
-      "step": 199
-    },
-    {
-      "epoch": 0.6451612903225806,
-      "grad_norm": 0.7978684306144714,
-      "learning_rate": 3.833143384490506e-06,
-      "logits/chosen": -1.9407423734664917,
-      "logits/rejected": -1.8073129653930664,
-      "logps/chosen": -164.80926513671875,
-      "logps/rejected": -207.02963256835938,
-      "loss": 0.32675695419311523,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -5.683391094207764,
-      "rewards/margins": 2.388817071914673,
-      "rewards/rejected": -8.072208404541016,
-      "step": 200
-    },
-    {
-      "epoch": 0.6483870967741936,
-      "grad_norm": 1.105037808418274,
-      "learning_rate": 3.8224100258174066e-06,
-      "logits/chosen": -2.0903000831604004,
-      "logits/rejected": -1.861089825630188,
-      "logps/chosen": -146.48716735839844,
-      "logps/rejected": -193.67787170410156,
-      "loss": 0.6486673951148987,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -5.065411567687988,
-      "rewards/margins": 1.8852479457855225,
-      "rewards/rejected": -6.95065975189209,
-      "step": 201
-    },
-    {
-      "epoch": 0.6516129032258065,
-      "grad_norm": 0.5039254426956177,
-      "learning_rate": 3.811642713877175e-06,
-      "logits/chosen": -1.7253732681274414,
-      "logits/rejected": -1.4285393953323364,
-      "logps/chosen": -148.22134399414062,
-      "logps/rejected": -213.10714721679688,
-      "loss": 0.17612317204475403,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.6895952224731445,
-      "rewards/margins": 3.1369845867156982,
-      "rewards/rejected": -7.826580047607422,
-      "step": 202
-    },
-    {
-      "epoch": 0.6548387096774193,
-      "grad_norm": 0.4975990056991577,
-      "learning_rate": 3.8008417251237208e-06,
-      "logits/chosen": -2.020594358444214,
-      "logits/rejected": -1.7548696994781494,
-      "logps/chosen": -153.95767211914062,
-      "logps/rejected": -216.39849853515625,
-      "loss": 0.1482263207435608,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.834309101104736,
-      "rewards/margins": 3.244680643081665,
-      "rewards/rejected": -8.078989028930664,
-      "step": 203
-    },
-    {
-      "epoch": 0.6580645161290323,
-      "grad_norm": 0.7523185610771179,
-      "learning_rate": 3.79000733687561e-06,
-      "logits/chosen": -2.1262402534484863,
-      "logits/rejected": -1.8249822854995728,
-      "logps/chosen": -138.27684020996094,
-      "logps/rejected": -194.3658447265625,
-      "loss": 0.3780391812324524,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -4.251176357269287,
-      "rewards/margins": 2.609586238861084,
-      "rewards/rejected": -6.860762596130371,
-      "step": 204
-    },
-    {
-      "epoch": 0.6612903225806451,
-      "grad_norm": 0.7087878584861755,
-      "learning_rate": 3.7791398273089562e-06,
-      "logits/chosen": -2.0547335147857666,
-      "logits/rejected": -1.7882875204086304,
-      "logps/chosen": -150.56541442871094,
-      "logps/rejected": -204.3115234375,
-      "loss": 0.28930363059043884,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -4.768935680389404,
-      "rewards/margins": 2.5629935264587402,
-      "rewards/rejected": -7.331929683685303,
-      "step": 205
-    },
-    {
-      "epoch": 0.6645161290322581,
-      "grad_norm": 0.6253401041030884,
-      "learning_rate": 3.7682394754502687e-06,
-      "logits/chosen": -2.046987295150757,
-      "logits/rejected": -1.855947732925415,
-      "logps/chosen": -160.48355102539062,
-      "logps/rejected": -204.4710693359375,
-      "loss": 0.25897252559661865,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -5.5119524002075195,
-      "rewards/margins": 1.971421480178833,
-      "rewards/rejected": -7.48337459564209,
-      "step": 206
-    },
-    {
-      "epoch": 0.667741935483871,
-      "grad_norm": 0.6977007985115051,
-      "learning_rate": 3.7573065611692948e-06,
-      "logits/chosen": -1.9548072814941406,
-      "logits/rejected": -1.6766889095306396,
-      "logps/chosen": -155.93496704101562,
-      "logps/rejected": -221.83482360839844,
-      "loss": 0.309578001499176,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -5.157824993133545,
-      "rewards/margins": 3.1803603172302246,
-      "rewards/rejected": -8.338184356689453,
-      "step": 207
-    },
-    {
-      "epoch": 0.6709677419354839,
-      "grad_norm": 0.9557754993438721,
-      "learning_rate": 3.7463413651718304e-06,
-      "logits/chosen": -1.9568748474121094,
-      "logits/rejected": -1.7348158359527588,
-      "logps/chosen": -164.6311798095703,
-      "logps/rejected": -209.5601348876953,
-      "loss": 0.5549408197402954,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -5.9888458251953125,
-      "rewards/margins": 2.138120651245117,
-      "rewards/rejected": -8.126965522766113,
-      "step": 208
-    },
-    {
-      "epoch": 0.6741935483870968,
-      "grad_norm": 0.8854509592056274,
-      "learning_rate": 3.7353441689925144e-06,
-      "logits/chosen": -2.291874885559082,
-      "logits/rejected": -2.0419085025787354,
-      "logps/chosen": -156.27235412597656,
-      "logps/rejected": -210.60018920898438,
-      "loss": 0.3768148720264435,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -4.670940399169922,
-      "rewards/margins": 3.0637500286102295,
-      "rewards/rejected": -7.7346906661987305,
-      "step": 209
-    },
-    {
-      "epoch": 0.6774193548387096,
-      "grad_norm": 0.9439530968666077,
-      "learning_rate": 3.7243152549875995e-06,
-      "logits/chosen": -2.140773057937622,
-      "logits/rejected": -1.8929061889648438,
-      "logps/chosen": -157.63751220703125,
-      "logps/rejected": -200.3406524658203,
-      "loss": 0.41107749938964844,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -5.264650821685791,
-      "rewards/margins": 2.0409224033355713,
-      "rewards/rejected": -7.305573463439941,
-      "step": 210
-    },
-    {
-      "epoch": 0.6806451612903226,
-      "grad_norm": 0.8497869968414307,
-      "learning_rate": 3.7132549063277033e-06,
-      "logits/chosen": -2.4181089401245117,
-      "logits/rejected": -2.282496452331543,
-      "logps/chosen": -135.24644470214844,
-      "logps/rejected": -171.06222534179688,
-      "loss": 0.4564102590084076,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.036481857299805,
-      "rewards/margins": 1.5632761716842651,
-      "rewards/rejected": -5.599758148193359,
-      "step": 211
-    },
-    {
-      "epoch": 0.6838709677419355,
-      "grad_norm": 0.6613027453422546,
-      "learning_rate": 3.7021634069905355e-06,
-      "logits/chosen": -1.9340821504592896,
-      "logits/rejected": -1.7125706672668457,
-      "logps/chosen": -153.77017211914062,
-      "logps/rejected": -206.60797119140625,
-      "loss": 0.27007001638412476,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -5.424838066101074,
-      "rewards/margins": 2.3895516395568848,
-      "rewards/rejected": -7.814389705657959,
-      "step": 212
-    },
-    {
-      "epoch": 0.6870967741935484,
-      "grad_norm": 1.148823618888855,
-      "learning_rate": 3.691041041753613e-06,
-      "logits/chosen": -2.016974449157715,
-      "logits/rejected": -1.7639240026474,
-      "logps/chosen": -152.53939819335938,
-      "logps/rejected": -205.86460876464844,
-      "loss": 0.5762124061584473,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -5.334196090698242,
-      "rewards/margins": 2.5293712615966797,
-      "rewards/rejected": -7.863567352294922,
-      "step": 213
-    },
-    {
-      "epoch": 0.6903225806451613,
-      "grad_norm": 0.9124322533607483,
-      "learning_rate": 3.6798880961869386e-06,
-      "logits/chosen": -2.0975124835968018,
-      "logits/rejected": -1.8551663160324097,
-      "logps/chosen": -165.72335815429688,
-      "logps/rejected": -212.5291748046875,
-      "loss": 0.43939220905303955,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -5.707298278808594,
-      "rewards/margins": 2.0160140991210938,
-      "rewards/rejected": -7.7233123779296875,
-      "step": 214
-    },
-    {
-      "epoch": 0.6935483870967742,
-      "grad_norm": 0.6490616798400879,
-      "learning_rate": 3.6687048566456783e-06,
-      "logits/chosen": -1.7563501596450806,
-      "logits/rejected": -1.5969226360321045,
-      "logps/chosen": -150.6394805908203,
-      "logps/rejected": -202.0006561279297,
-      "loss": 0.24775174260139465,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -4.777384281158447,
-      "rewards/margins": 2.58968448638916,
-      "rewards/rejected": -7.367069244384766,
-      "step": 215
-    },
-    {
-      "epoch": 0.6967741935483871,
-      "grad_norm": 0.7321914434432983,
-      "learning_rate": 3.657491610262802e-06,
-      "logits/chosen": -2.1910958290100098,
-      "logits/rejected": -1.9808027744293213,
-      "logps/chosen": -143.98077392578125,
-      "logps/rejected": -197.44918823242188,
-      "loss": 0.36529406905174255,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.628123760223389,
-      "rewards/margins": 2.3086495399475098,
-      "rewards/rejected": -6.936773777008057,
-      "step": 216
-    },
-    {
-      "epoch": 0.7,
-      "grad_norm": 0.6766554117202759,
-      "learning_rate": 3.646248644941716e-06,
-      "logits/chosen": -2.1157803535461426,
-      "logits/rejected": -1.8787240982055664,
-      "logps/chosen": -147.38084411621094,
-      "logps/rejected": -190.40176391601562,
-      "loss": 0.23784853518009186,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.631725788116455,
-      "rewards/margins": 2.343129873275757,
-      "rewards/rejected": -6.974855899810791,
-      "step": 217
-    },
-    {
-      "epoch": 0.7032258064516129,
-      "grad_norm": 0.6971132755279541,
-      "learning_rate": 3.634976249348867e-06,
-      "logits/chosen": -1.9888451099395752,
-      "logits/rejected": -1.666069507598877,
-      "logps/chosen": -141.85008239746094,
-      "logps/rejected": -206.53433227539062,
-      "loss": 0.2547075152397156,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.588260650634766,
-      "rewards/margins": 3.034471035003662,
-      "rewards/rejected": -7.622732162475586,
-      "step": 218
-    },
-    {
-      "epoch": 0.7064516129032258,
-      "grad_norm": 1.014033317565918,
-      "learning_rate": 3.6236747129063353e-06,
-      "logits/chosen": -1.693007230758667,
-      "logits/rejected": -1.5424453020095825,
-      "logps/chosen": -165.82598876953125,
-      "logps/rejected": -207.29913330078125,
-      "loss": 0.5212239623069763,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -5.077567100524902,
-      "rewards/margins": 1.8947848081588745,
-      "rewards/rejected": -6.972352027893066,
-      "step": 219
-    },
-    {
-      "epoch": 0.7096774193548387,
-      "grad_norm": 0.889532208442688,
-      "learning_rate": 3.6123443257843985e-06,
-      "logits/chosen": -1.9983710050582886,
-      "logits/rejected": -1.7099499702453613,
-      "logps/chosen": -149.2031707763672,
-      "logps/rejected": -204.5345916748047,
-      "loss": 0.3786216378211975,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.6922101974487305,
-      "rewards/margins": 2.7204079627990723,
-      "rewards/rejected": -7.412618637084961,
-      "step": 220
-    },
-    {
-      "epoch": 0.7129032258064516,
-      "grad_norm": 0.7448631525039673,
-      "learning_rate": 3.600985378894086e-06,
-      "logits/chosen": -1.9120534658432007,
-      "logits/rejected": -1.6604293584823608,
-      "logps/chosen": -145.59979248046875,
-      "logps/rejected": -193.89862060546875,
-      "loss": 0.3285577595233917,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -4.52049446105957,
-      "rewards/margins": 2.195286512374878,
-      "rewards/rejected": -6.715780258178711,
-      "step": 221
-    },
-    {
-      "epoch": 0.7161290322580646,
-      "grad_norm": 0.6082993149757385,
-      "learning_rate": 3.589598163879706e-06,
-      "logits/chosen": -2.3130810260772705,
-      "logits/rejected": -1.9831249713897705,
-      "logps/chosen": -130.06045532226562,
-      "logps/rejected": -193.2603759765625,
-      "loss": 0.2183692455291748,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -4.439502716064453,
-      "rewards/margins": 2.903191089630127,
-      "rewards/rejected": -7.342694282531738,
-      "step": 222
-    },
-    {
-      "epoch": 0.7193548387096774,
-      "grad_norm": 0.8937051892280579,
-      "learning_rate": 3.5781829731113605e-06,
-      "logits/chosen": -1.9590139389038086,
-      "logits/rejected": -1.7677487134933472,
-      "logps/chosen": -166.64971923828125,
-      "logps/rejected": -211.19219970703125,
-      "loss": 0.4538525342941284,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -5.182065010070801,
-      "rewards/margins": 1.9423487186431885,
-      "rewards/rejected": -7.124413967132568,
-      "step": 223
-    },
-    {
-      "epoch": 0.7225806451612903,
-      "grad_norm": 1.022118091583252,
-      "learning_rate": 3.566740099677436e-06,
-      "logits/chosen": -2.3347010612487793,
-      "logits/rejected": -2.102405548095703,
-      "logps/chosen": -137.7996368408203,
-      "logps/rejected": -190.09060668945312,
-      "loss": 0.5034906268119812,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -4.532764434814453,
-      "rewards/margins": 2.5061287879943848,
-      "rewards/rejected": -7.038893222808838,
-      "step": 224
-    },
-    {
-      "epoch": 0.7258064516129032,
-      "grad_norm": 0.7584444880485535,
-      "learning_rate": 3.55526983737708e-06,
-      "logits/chosen": -1.9208580255508423,
-      "logits/rejected": -1.7654166221618652,
-      "logps/chosen": -155.05662536621094,
-      "logps/rejected": -197.6698760986328,
-      "loss": 0.3522125482559204,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -4.8355560302734375,
-      "rewards/margins": 2.2215867042541504,
-      "rewards/rejected": -7.05714225769043,
-      "step": 225
-    },
-    {
-      "epoch": 0.7290322580645161,
-      "grad_norm": 0.9140284657478333,
-      "learning_rate": 3.5437724807126583e-06,
-      "logits/chosen": -2.352722644805908,
-      "logits/rejected": -2.1613006591796875,
-      "logps/chosen": -145.38134765625,
-      "logps/rejected": -192.044921875,
-      "loss": 0.43635356426239014,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -4.508688926696777,
-      "rewards/margins": 1.8607778549194336,
-      "rewards/rejected": -6.369466781616211,
-      "step": 226
-    },
-    {
-      "epoch": 0.7322580645161291,
-      "grad_norm": 0.7812667489051819,
-      "learning_rate": 3.5322483248821905e-06,
-      "logits/chosen": -1.846239447593689,
-      "logits/rejected": -1.5585601329803467,
-      "logps/chosen": -165.45079040527344,
-      "logps/rejected": -219.214111328125,
-      "loss": 0.27440738677978516,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.201958656311035,
-      "rewards/margins": 2.816474676132202,
-      "rewards/rejected": -8.018433570861816,
-      "step": 227
-    },
-    {
-      "epoch": 0.7354838709677419,
-      "grad_norm": 0.8155746459960938,
-      "learning_rate": 3.5206976657717744e-06,
-      "logits/chosen": -2.210322380065918,
-      "logits/rejected": -2.045366048812866,
-      "logps/chosen": -166.80979919433594,
-      "logps/rejected": -211.31402587890625,
-      "loss": 0.33929508924484253,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.07341194152832,
-      "rewards/margins": 2.169137954711914,
-      "rewards/rejected": -7.242549896240234,
-      "step": 228
-    },
-    {
-      "epoch": 0.7387096774193549,
-      "grad_norm": 0.8510491251945496,
-      "learning_rate": 3.509120799947987e-06,
-      "logits/chosen": -1.881466269493103,
-      "logits/rejected": -1.6687889099121094,
-      "logps/chosen": -164.90065002441406,
-      "logps/rejected": -216.87759399414062,
-      "loss": 0.34565114974975586,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.524971008300781,
-      "rewards/margins": 2.7519350051879883,
-      "rewards/rejected": -8.276906967163086,
-      "step": 229
-    },
-    {
-      "epoch": 0.7419354838709677,
-      "grad_norm": 0.8683707118034363,
-      "learning_rate": 3.4975180246502694e-06,
-      "logits/chosen": -2.1603784561157227,
-      "logits/rejected": -1.9462230205535889,
-      "logps/chosen": -150.9666748046875,
-      "logps/rejected": -194.62677001953125,
-      "loss": 0.35645627975463867,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -4.169332504272461,
-      "rewards/margins": 2.1779627799987793,
-      "rewards/rejected": -6.347295761108398,
-      "step": 230
-    },
-    {
-      "epoch": 0.7451612903225806,
-      "grad_norm": 0.8181908130645752,
-      "learning_rate": 3.4858896377832966e-06,
-      "logits/chosen": -2.579606056213379,
-      "logits/rejected": -2.3366498947143555,
-      "logps/chosen": -155.29588317871094,
-      "logps/rejected": -196.72500610351562,
-      "loss": 0.3560987114906311,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -5.2712554931640625,
-      "rewards/margins": 1.8674944639205933,
-      "rewards/rejected": -7.138750076293945,
-      "step": 231
-    },
-    {
-      "epoch": 0.7483870967741936,
-      "grad_norm": 0.8175391554832458,
-      "learning_rate": 3.474235937909329e-06,
-      "logits/chosen": -2.486018657684326,
-      "logits/rejected": -2.283841609954834,
-      "logps/chosen": -144.54238891601562,
-      "logps/rejected": -197.2718048095703,
-      "loss": 0.39171290397644043,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.539939880371094,
-      "rewards/margins": 2.590193033218384,
-      "rewards/rejected": -7.130133152008057,
-      "step": 232
-    },
-    {
-      "epoch": 0.7516129032258064,
-      "grad_norm": 0.6144670844078064,
-      "learning_rate": 3.4625572242405446e-06,
-      "logits/chosen": -2.331089496612549,
-      "logits/rejected": -2.0649709701538086,
-      "logps/chosen": -137.04869079589844,
-      "logps/rejected": -195.30335998535156,
-      "loss": 0.2142164707183838,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -4.864482402801514,
-      "rewards/margins": 2.7930479049682617,
-      "rewards/rejected": -7.657529830932617,
-      "step": 233
-    },
-    {
-      "epoch": 0.7548387096774194,
-      "grad_norm": 0.6806922554969788,
-      "learning_rate": 3.4508537966313605e-06,
-      "logits/chosen": -2.0631325244903564,
-      "logits/rejected": -1.8709652423858643,
-      "logps/chosen": -178.80401611328125,
-      "logps/rejected": -234.48159790039062,
-      "loss": 0.25653570890426636,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.064234256744385,
-      "rewards/margins": 2.477729558944702,
-      "rewards/rejected": -8.541963577270508,
-      "step": 234
-    },
-    {
-      "epoch": 0.7580645161290323,
-      "grad_norm": 1.2282917499542236,
-      "learning_rate": 3.4391259555707258e-06,
-      "logits/chosen": -1.718582272529602,
-      "logits/rejected": -1.5619916915893555,
-      "logps/chosen": -162.3804168701172,
-      "logps/rejected": -208.4605712890625,
-      "loss": 0.5654730200767517,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -5.41401481628418,
-      "rewards/margins": 1.942406177520752,
-      "rewards/rejected": -7.35642147064209,
-      "step": 235
-    },
-    {
-      "epoch": 0.7612903225806451,
-      "grad_norm": 0.7021056413650513,
-      "learning_rate": 3.42737400217442e-06,
-      "logits/chosen": -2.1502299308776855,
-      "logits/rejected": -1.8625457286834717,
-      "logps/chosen": -130.69139099121094,
-      "logps/rejected": -189.042724609375,
-      "loss": 0.24078813195228577,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.40123987197876,
-      "rewards/margins": 2.540182590484619,
-      "rewards/rejected": -6.941422462463379,
-      "step": 236
-    },
-    {
-      "epoch": 0.7645161290322581,
-      "grad_norm": 0.4305199980735779,
-      "learning_rate": 3.4155982381773074e-06,
-      "logits/chosen": -2.451601028442383,
-      "logits/rejected": -2.086921215057373,
-      "logps/chosen": -116.90483093261719,
-      "logps/rejected": -181.51046752929688,
-      "loss": 0.14265665411949158,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -3.7053639888763428,
-      "rewards/margins": 3.288050651550293,
-      "rewards/rejected": -6.993414878845215,
-      "step": 237
-    },
-    {
-      "epoch": 0.7677419354838709,
-      "grad_norm": 0.7569338083267212,
-      "learning_rate": 3.4037989659255998e-06,
-      "logits/chosen": -2.1827783584594727,
-      "logits/rejected": -1.9417102336883545,
-      "logps/chosen": -155.0686492919922,
-      "logps/rejected": -205.53671264648438,
-      "loss": 0.30644071102142334,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -4.89786958694458,
-      "rewards/margins": 2.2091331481933594,
-      "rewards/rejected": -7.107002258300781,
-      "step": 238
-    },
-    {
-      "epoch": 0.7709677419354839,
-      "grad_norm": 0.6807054281234741,
-      "learning_rate": 3.3919764883690892e-06,
-      "logits/chosen": -2.1077489852905273,
-      "logits/rejected": -1.7725837230682373,
-      "logps/chosen": -137.52195739746094,
-      "logps/rejected": -195.14187622070312,
-      "loss": 0.2705943286418915,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.612639904022217,
-      "rewards/margins": 2.7255260944366455,
-      "rewards/rejected": -7.338165283203125,
-      "step": 239
-    },
-    {
-      "epoch": 0.7741935483870968,
-      "grad_norm": 0.7687151432037354,
-      "learning_rate": 3.3801311090533713e-06,
-      "logits/chosen": -2.2328834533691406,
-      "logits/rejected": -1.9268276691436768,
-      "logps/chosen": -155.4674835205078,
-      "logps/rejected": -219.96664428710938,
-      "loss": 0.2569769024848938,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.455958366394043,
-      "rewards/margins": 3.1837010383605957,
-      "rewards/rejected": -8.639659881591797,
-      "step": 240
-    },
-    {
-      "epoch": 0.7774193548387097,
-      "grad_norm": 0.9292513132095337,
-      "learning_rate": 3.3682631321120507e-06,
-      "logits/chosen": -1.8700426816940308,
-      "logits/rejected": -1.7026903629302979,
-      "logps/chosen": -166.23532104492188,
-      "logps/rejected": -216.99081420898438,
-      "loss": 0.39584052562713623,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -5.327211856842041,
-      "rewards/margins": 2.511094570159912,
-      "rewards/rejected": -7.838306427001953,
-      "step": 241
-    },
-    {
-      "epoch": 0.7806451612903226,
-      "grad_norm": 0.6035842299461365,
-      "learning_rate": 3.3563728622589335e-06,
-      "logits/chosen": -2.2338809967041016,
-      "logits/rejected": -1.9739981889724731,
-      "logps/chosen": -126.01628112792969,
-      "logps/rejected": -182.52577209472656,
-      "loss": 0.22547340393066406,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.3564653396606445,
-      "rewards/margins": 2.6233766078948975,
-      "rewards/rejected": -6.979841709136963,
-      "step": 242
-    },
-    {
-      "epoch": 0.7838709677419354,
-      "grad_norm": 0.8133144974708557,
-      "learning_rate": 3.3444606047802018e-06,
-      "logits/chosen": -2.296985149383545,
-      "logits/rejected": -1.9562444686889648,
-      "logps/chosen": -138.7552947998047,
-      "logps/rejected": -202.73809814453125,
-      "loss": 0.33858245611190796,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -4.2327985763549805,
-      "rewards/margins": 3.3319268226623535,
-      "rewards/rejected": -7.564725875854492,
-      "step": 243
-    },
-    {
-      "epoch": 0.7870967741935484,
-      "grad_norm": 0.626816987991333,
-      "learning_rate": 3.332526665526577e-06,
-      "logits/chosen": -1.8298629522323608,
-      "logits/rejected": -1.528571605682373,
-      "logps/chosen": -156.6051025390625,
-      "logps/rejected": -224.70257568359375,
-      "loss": 0.17712946236133575,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.212545394897461,
-      "rewards/margins": 3.80438494682312,
-      "rewards/rejected": -9.016929626464844,
-      "step": 244
-    },
-    {
-      "epoch": 0.7903225806451613,
-      "grad_norm": 1.5558580160140991,
-      "learning_rate": 3.320571350905466e-06,
-      "logits/chosen": -2.0828301906585693,
-      "logits/rejected": -1.801743745803833,
-      "logps/chosen": -158.08493041992188,
-      "logps/rejected": -204.83648681640625,
-      "loss": 0.5623299479484558,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -5.441231727600098,
-      "rewards/margins": 2.246556282043457,
-      "rewards/rejected": -7.6877875328063965,
-      "step": 245
-    },
-    {
-      "epoch": 0.7935483870967742,
-      "grad_norm": 0.6557853817939758,
-      "learning_rate": 3.3085949678730953e-06,
-      "logits/chosen": -2.0652894973754883,
-      "logits/rejected": -1.883588194847107,
-      "logps/chosen": -150.181640625,
-      "logps/rejected": -219.64108276367188,
-      "loss": 0.2832576036453247,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.247066497802734,
-      "rewards/margins": 3.2221381664276123,
-      "rewards/rejected": -8.469204902648926,
-      "step": 246
-    },
-    {
-      "epoch": 0.7967741935483871,
-      "grad_norm": 0.8382725715637207,
-      "learning_rate": 3.2965978239266292e-06,
-      "logits/chosen": -2.383835554122925,
-      "logits/rejected": -2.0919713973999023,
-      "logps/chosen": -162.2133331298828,
-      "logps/rejected": -214.61122131347656,
-      "loss": 0.3238731622695923,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.967742919921875,
-      "rewards/margins": 2.712991237640381,
-      "rewards/rejected": -7.680734157562256,
-      "step": 247
-    },
-    {
-      "epoch": 0.8,
-      "grad_norm": 1.1319209337234497,
-      "learning_rate": 3.2845802270962736e-06,
-      "logits/chosen": -2.1036744117736816,
-      "logits/rejected": -1.820197343826294,
-      "logps/chosen": -179.16018676757812,
-      "logps/rejected": -229.18484497070312,
-      "loss": 0.5245007276535034,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -6.1170454025268555,
-      "rewards/margins": 2.1524717807769775,
-      "rewards/rejected": -8.269516944885254,
-      "step": 248
-    },
-    {
-      "epoch": 0.8032258064516129,
-      "grad_norm": 0.8215903043746948,
-      "learning_rate": 3.272542485937369e-06,
-      "logits/chosen": -1.901473045349121,
-      "logits/rejected": -1.5790214538574219,
-      "logps/chosen": -166.28927612304688,
-      "logps/rejected": -216.44970703125,
-      "loss": 0.3197699785232544,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -5.5512495040893555,
-      "rewards/margins": 2.5551297664642334,
-      "rewards/rejected": -8.106379508972168,
-      "step": 249
-    },
-    {
-      "epoch": 0.8064516129032258,
-      "grad_norm": 0.9459141492843628,
-      "learning_rate": 3.2604849095224666e-06,
-      "logits/chosen": -1.822584629058838,
-      "logits/rejected": -1.5994616746902466,
-      "logps/chosen": -162.4025421142578,
-      "logps/rejected": -216.37953186035156,
-      "loss": 0.3979412913322449,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -5.902081489562988,
-      "rewards/margins": 2.4182045459747314,
-      "rewards/rejected": -8.32028579711914,
-      "step": 250
-    },
-    {
-      "epoch": 0.8096774193548387,
-      "grad_norm": 1.1870235204696655,
-      "learning_rate": 3.248407807433396e-06,
-      "logits/chosen": -2.156320571899414,
-      "logits/rejected": -1.949864149093628,
-      "logps/chosen": -172.03271484375,
-      "logps/rejected": -209.26580810546875,
-      "loss": 0.5271865129470825,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -6.29229736328125,
-      "rewards/margins": 1.8417139053344727,
-      "rewards/rejected": -8.134011268615723,
-      "step": 251
-    },
-    {
-      "epoch": 0.8129032258064516,
-      "grad_norm": 0.948936402797699,
-      "learning_rate": 3.2363114897533095e-06,
-      "logits/chosen": -1.9766535758972168,
-      "logits/rejected": -1.7797175645828247,
-      "logps/chosen": -151.3767852783203,
-      "logps/rejected": -203.56295776367188,
-      "loss": 0.4569253921508789,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -5.442215919494629,
-      "rewards/margins": 2.54245924949646,
-      "rewards/rejected": -7.984674453735352,
-      "step": 252
-    },
-    {
-      "epoch": 0.8161290322580645,
-      "grad_norm": 0.7237331867218018,
-      "learning_rate": 3.2241962670587312e-06,
-      "logits/chosen": -2.1904726028442383,
-      "logits/rejected": -1.8860106468200684,
-      "logps/chosen": -157.61126708984375,
-      "logps/rejected": -198.8243865966797,
-      "loss": 0.2793291211128235,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.772243976593018,
-      "rewards/margins": 2.6312925815582275,
-      "rewards/rejected": -7.403536796569824,
-      "step": 253
-    },
-    {
-      "epoch": 0.8193548387096774,
-      "grad_norm": 0.7647410035133362,
-      "learning_rate": 3.212062450411572e-06,
-      "logits/chosen": -2.5955586433410645,
-      "logits/rejected": -2.3625893592834473,
-      "logps/chosen": -154.66592407226562,
-      "logps/rejected": -209.16885375976562,
-      "loss": 0.27114158868789673,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -5.201878070831299,
-      "rewards/margins": 2.639791488647461,
-      "rewards/rejected": -7.841669082641602,
-      "step": 254
-    },
-    {
-      "epoch": 0.8225806451612904,
-      "grad_norm": 1.015256404876709,
-      "learning_rate": 3.1999103513511528e-06,
-      "logits/chosen": -2.0532336235046387,
-      "logits/rejected": -1.8524454832077026,
-      "logps/chosen": -149.565673828125,
-      "logps/rejected": -197.91754150390625,
-      "loss": 0.4156733751296997,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -4.924131393432617,
-      "rewards/margins": 2.6219491958618164,
-      "rewards/rejected": -7.546079635620117,
-      "step": 255
-    },
-    {
-      "epoch": 0.8258064516129032,
-      "grad_norm": 0.6962264776229858,
-      "learning_rate": 3.1877402818861954e-06,
-      "logits/chosen": -2.1449079513549805,
-      "logits/rejected": -1.8325960636138916,
-      "logps/chosen": -154.07302856445312,
-      "logps/rejected": -211.74591064453125,
-      "loss": 0.2757159471511841,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.438961982727051,
-      "rewards/margins": 3.2228941917419434,
-      "rewards/rejected": -8.661855697631836,
-      "step": 256
-    },
-    {
-      "epoch": 0.8290322580645161,
-      "grad_norm": 0.7284089922904968,
-      "learning_rate": 3.175552554486822e-06,
-      "logits/chosen": -2.0035736560821533,
-      "logits/rejected": -1.7146947383880615,
-      "logps/chosen": -161.89108276367188,
-      "logps/rejected": -232.00650024414062,
-      "loss": 0.2483557164669037,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.063234806060791,
-      "rewards/margins": 3.6449978351593018,
-      "rewards/rejected": -9.708232879638672,
-      "step": 257
-    },
-    {
-      "epoch": 0.832258064516129,
-      "grad_norm": 0.7539389729499817,
-      "learning_rate": 3.1633474820765243e-06,
-      "logits/chosen": -2.1420180797576904,
-      "logits/rejected": -1.845510482788086,
-      "logps/chosen": -166.37063598632812,
-      "logps/rejected": -222.24362182617188,
-      "loss": 0.2929441034793854,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.11561918258667,
-      "rewards/margins": 2.9755101203918457,
-      "rewards/rejected": -9.091129302978516,
-      "step": 258
-    },
-    {
-      "epoch": 0.8354838709677419,
-      "grad_norm": 0.6925541758537292,
-      "learning_rate": 3.1511253780241334e-06,
-      "logits/chosen": -2.2203574180603027,
-      "logits/rejected": -1.9398572444915771,
-      "logps/chosen": -157.34991455078125,
-      "logps/rejected": -214.30502319335938,
-      "loss": 0.27536725997924805,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -5.440666198730469,
-      "rewards/margins": 3.108762741088867,
-      "rewards/rejected": -8.549429893493652,
-      "step": 259
-    },
-    {
-      "epoch": 0.8387096774193549,
-      "grad_norm": 0.9221241474151611,
-      "learning_rate": 3.1388865561357727e-06,
-      "logits/chosen": -1.9219483137130737,
-      "logits/rejected": -1.714083194732666,
-      "logps/chosen": -173.53128051757812,
-      "logps/rejected": -221.80712890625,
-      "loss": 0.3261393904685974,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -6.202951431274414,
-      "rewards/margins": 2.770026445388794,
-      "rewards/rejected": -8.972977638244629,
-      "step": 260
-    },
-    {
-      "epoch": 0.8419354838709677,
-      "grad_norm": 0.7738852500915527,
-      "learning_rate": 3.1266313306468018e-06,
-      "logits/chosen": -2.082782745361328,
-      "logits/rejected": -1.8457026481628418,
-      "logps/chosen": -151.79415893554688,
-      "logps/rejected": -202.09934997558594,
-      "loss": 0.31562310457229614,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -4.857356548309326,
-      "rewards/margins": 3.0326480865478516,
-      "rewards/rejected": -7.8900041580200195,
-      "step": 261
-    },
-    {
-      "epoch": 0.8451612903225807,
-      "grad_norm": 0.6938282251358032,
-      "learning_rate": 3.1143600162137454e-06,
-      "logits/chosen": -1.8344019651412964,
-      "logits/rejected": -1.5771279335021973,
-      "logps/chosen": -166.30076599121094,
-      "logps/rejected": -221.72154235839844,
-      "loss": 0.2417217195034027,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -5.7909040451049805,
-      "rewards/margins": 2.8294739723205566,
-      "rewards/rejected": -8.620379447937012,
-      "step": 262
-    },
-    {
-      "epoch": 0.8483870967741935,
-      "grad_norm": 0.5476120710372925,
-      "learning_rate": 3.1020729279062194e-06,
-      "logits/chosen": -1.976428747177124,
-      "logits/rejected": -1.6645793914794922,
-      "logps/chosen": -165.8445587158203,
-      "logps/rejected": -224.68304443359375,
-      "loss": 0.13630928099155426,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.6656494140625,
-      "rewards/margins": 3.6471195220947266,
-      "rewards/rejected": -9.312768936157227,
-      "step": 263
-    },
-    {
-      "epoch": 0.8516129032258064,
-      "grad_norm": 0.678778886795044,
-      "learning_rate": 3.089770381198836e-06,
-      "logits/chosen": -2.016895294189453,
-      "logits/rejected": -1.7314876317977905,
-      "logps/chosen": -153.4365692138672,
-      "logps/rejected": -226.68487548828125,
-      "loss": 0.1903066337108612,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -5.526880741119385,
-      "rewards/margins": 3.849604606628418,
-      "rewards/rejected": -9.376485824584961,
-      "step": 264
-    },
-    {
-      "epoch": 0.8548387096774194,
-      "grad_norm": 0.9369924068450928,
-      "learning_rate": 3.077452691963109e-06,
-      "logits/chosen": -1.753978967666626,
-      "logits/rejected": -1.5365220308303833,
-      "logps/chosen": -162.50265502929688,
-      "logps/rejected": -208.20266723632812,
-      "loss": 0.35749346017837524,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -5.593669414520264,
-      "rewards/margins": 2.4858767986297607,
-      "rewards/rejected": -8.079545974731445,
-      "step": 265
-    },
-    {
-      "epoch": 0.8580645161290322,
-      "grad_norm": 0.8137779831886292,
-      "learning_rate": 3.0651201764593375e-06,
-      "logits/chosen": -2.141214370727539,
-      "logits/rejected": -1.7222539186477661,
-      "logps/chosen": -141.45132446289062,
-      "logps/rejected": -208.5587158203125,
-      "loss": 0.22137892246246338,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -5.138533592224121,
-      "rewards/margins": 3.4072115421295166,
-      "rewards/rejected": -8.545745849609375,
-      "step": 266
-    },
-    {
-      "epoch": 0.8612903225806452,
-      "grad_norm": 0.6460467576980591,
-      "learning_rate": 3.052773151328493e-06,
-      "logits/chosen": -2.018655776977539,
-      "logits/rejected": -1.7169384956359863,
-      "logps/chosen": -157.74383544921875,
-      "logps/rejected": -213.468505859375,
-      "loss": 0.24283014237880707,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.804964065551758,
-      "rewards/margins": 2.993710517883301,
-      "rewards/rejected": -8.798675537109375,
-      "step": 267
-    },
-    {
-      "epoch": 0.864516129032258,
-      "grad_norm": 0.7242834568023682,
-      "learning_rate": 3.0404119335840844e-06,
-      "logits/chosen": -1.9196281433105469,
-      "logits/rejected": -1.55117928981781,
-      "logps/chosen": -145.27304077148438,
-      "logps/rejected": -216.47206115722656,
-      "loss": 0.224162757396698,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -5.446750640869141,
-      "rewards/margins": 3.864659070968628,
-      "rewards/rejected": -9.311409950256348,
-      "step": 268
-    },
-    {
-      "epoch": 0.867741935483871,
-      "grad_norm": 0.836055338382721,
-      "learning_rate": 3.028036840604019e-06,
-      "logits/chosen": -2.284193277359009,
-      "logits/rejected": -2.016956329345703,
-      "logps/chosen": -143.85357666015625,
-      "logps/rejected": -188.52374267578125,
-      "loss": 0.2850709557533264,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.036369800567627,
-      "rewards/margins": 2.801107406616211,
-      "rewards/rejected": -7.837477684020996,
-      "step": 269
-    },
-    {
-      "epoch": 0.8709677419354839,
-      "grad_norm": 1.087480902671814,
-      "learning_rate": 3.0156481901224573e-06,
-      "logits/chosen": -1.7794990539550781,
-      "logits/rejected": -1.509232759475708,
-      "logps/chosen": -168.82406616210938,
-      "logps/rejected": -228.95497131347656,
-      "loss": 0.432708740234375,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -6.362540245056152,
-      "rewards/margins": 3.126552104949951,
-      "rewards/rejected": -9.489091873168945,
-      "step": 270
-    },
-    {
-      "epoch": 0.8741935483870967,
-      "grad_norm": 1.1399589776992798,
-      "learning_rate": 3.0032463002216504e-06,
-      "logits/chosen": -1.7899502515792847,
-      "logits/rejected": -1.5215126276016235,
-      "logps/chosen": -172.01519775390625,
-      "logps/rejected": -225.363037109375,
-      "loss": 0.5371558666229248,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -5.870519638061523,
-      "rewards/margins": 2.98347544670105,
-      "rewards/rejected": -8.853995323181152,
-      "step": 271
-    },
-    {
-      "epoch": 0.8774193548387097,
-      "grad_norm": 1.1370657682418823,
-      "learning_rate": 2.990831489323778e-06,
-      "logits/chosen": -1.944093942642212,
-      "logits/rejected": -1.6760796308517456,
-      "logps/chosen": -179.9832000732422,
-      "logps/rejected": -238.34197998046875,
-      "loss": 0.38257351517677307,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.727067470550537,
-      "rewards/margins": 2.7800581455230713,
-      "rewards/rejected": -9.507125854492188,
-      "step": 272
-    },
-    {
-      "epoch": 0.8806451612903226,
-      "grad_norm": 0.8905435800552368,
-      "learning_rate": 2.978404076182766e-06,
-      "logits/chosen": -1.9256776571273804,
-      "logits/rejected": -1.6671595573425293,
-      "logps/chosen": -148.33238220214844,
-      "logps/rejected": -205.4830322265625,
-      "loss": 0.38766035437583923,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.708691596984863,
-      "rewards/margins": 3.060739517211914,
-      "rewards/rejected": -7.769431114196777,
-      "step": 273
-    },
-    {
-      "epoch": 0.8838709677419355,
-      "grad_norm": 0.8603253364562988,
-      "learning_rate": 2.965964379876112e-06,
-      "logits/chosen": -1.9746428728103638,
-      "logits/rejected": -1.6785167455673218,
-      "logps/chosen": -156.52313232421875,
-      "logps/rejected": -218.1029815673828,
-      "loss": 0.3216756284236908,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -5.569048881530762,
-      "rewards/margins": 2.990276575088501,
-      "rewards/rejected": -8.559325218200684,
-      "step": 274
-    },
-    {
-      "epoch": 0.8870967741935484,
-      "grad_norm": 0.7117762565612793,
-      "learning_rate": 2.953512719796683e-06,
-      "logits/chosen": -1.9952205419540405,
-      "logits/rejected": -1.6901299953460693,
-      "logps/chosen": -147.76272583007812,
-      "logps/rejected": -213.8414306640625,
-      "loss": 0.3437959551811218,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -5.44565486907959,
-      "rewards/margins": 3.351426601409912,
-      "rewards/rejected": -8.79708194732666,
-      "step": 275
-    },
-    {
-      "epoch": 0.8903225806451613,
-      "grad_norm": 0.9500011205673218,
-      "learning_rate": 2.941049415644522e-06,
-      "logits/chosen": -1.6247742176055908,
-      "logits/rejected": -1.3834972381591797,
-      "logps/chosen": -164.38734436035156,
-      "logps/rejected": -213.3121337890625,
-      "loss": 0.3649950623512268,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.385304927825928,
-      "rewards/margins": 2.792433977127075,
-      "rewards/rejected": -9.177739143371582,
-      "step": 276
-    },
-    {
-      "epoch": 0.8935483870967742,
-      "grad_norm": 0.924250602722168,
-      "learning_rate": 2.9285747874186345e-06,
-      "logits/chosen": -1.9340094327926636,
-      "logits/rejected": -1.5965994596481323,
-      "logps/chosen": -150.30023193359375,
-      "logps/rejected": -220.32684326171875,
-      "loss": 0.3774546980857849,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -4.861666679382324,
-      "rewards/margins": 3.435051202774048,
-      "rewards/rejected": -8.296717643737793,
-      "step": 277
-    },
-    {
-      "epoch": 0.896774193548387,
-      "grad_norm": 0.6413925290107727,
-      "learning_rate": 2.916089155408778e-06,
-      "logits/chosen": -1.7598278522491455,
-      "logits/rejected": -1.4939292669296265,
-      "logps/chosen": -171.2310791015625,
-      "logps/rejected": -219.10232543945312,
-      "loss": 0.21341019868850708,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.737621307373047,
-      "rewards/margins": 2.784102201461792,
-      "rewards/rejected": -8.521722793579102,
-      "step": 278
-    },
-    {
-      "epoch": 0.9,
-      "grad_norm": 0.5707635283470154,
-      "learning_rate": 2.9035928401872336e-06,
-      "logits/chosen": -2.0271811485290527,
-      "logits/rejected": -1.7087454795837402,
-      "logps/chosen": -167.06146240234375,
-      "logps/rejected": -230.35206604003906,
-      "loss": 0.17337509989738464,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -5.494855880737305,
-      "rewards/margins": 3.8487350940704346,
-      "rewards/rejected": -9.34359073638916,
-      "step": 279
-    },
-    {
-      "epoch": 0.9032258064516129,
-      "grad_norm": 0.852750837802887,
-      "learning_rate": 2.8910861626005774e-06,
-      "logits/chosen": -1.9636651277542114,
-      "logits/rejected": -1.675203561782837,
-      "logps/chosen": -151.33860778808594,
-      "logps/rejected": -214.5986785888672,
-      "loss": 0.2913362979888916,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -5.54245138168335,
-      "rewards/margins": 3.5291976928710938,
-      "rewards/rejected": -9.071649551391602,
-      "step": 280
-    },
-    {
-      "epoch": 0.9064516129032258,
-      "grad_norm": 0.6375999450683594,
-      "learning_rate": 2.878569443761442e-06,
-      "logits/chosen": -1.943008303642273,
-      "logits/rejected": -1.580283761024475,
-      "logps/chosen": -144.96212768554688,
-      "logps/rejected": -210.97207641601562,
-      "loss": 0.13702642917633057,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.5446062088012695,
-      "rewards/margins": 3.7235865592956543,
-      "rewards/rejected": -9.268192291259766,
-      "step": 281
-    },
-    {
-      "epoch": 0.9096774193548387,
-      "grad_norm": 0.8081200122833252,
-      "learning_rate": 2.866043005040272e-06,
-      "logits/chosen": -1.9890791177749634,
-      "logits/rejected": -1.6013154983520508,
-      "logps/chosen": -170.92636108398438,
-      "logps/rejected": -234.09942626953125,
-      "loss": 0.26003971695899963,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.83311653137207,
-      "rewards/margins": 3.469595432281494,
-      "rewards/rejected": -9.302712440490723,
-      "step": 282
-    },
-    {
-      "epoch": 0.9129032258064517,
-      "grad_norm": 1.0241976976394653,
-      "learning_rate": 2.8535071680570735e-06,
-      "logits/chosen": -2.450965642929077,
-      "logits/rejected": -2.1390633583068848,
-      "logps/chosen": -145.23941040039062,
-      "logps/rejected": -197.3958740234375,
-      "loss": 0.5087136030197144,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -4.842174530029297,
-      "rewards/margins": 2.902985095977783,
-      "rewards/rejected": -7.745160102844238,
-      "step": 283
-    },
-    {
-      "epoch": 0.9161290322580645,
-      "grad_norm": 0.8598202466964722,
-      "learning_rate": 2.840962254673156e-06,
-      "logits/chosen": -2.0429110527038574,
-      "logits/rejected": -1.732966423034668,
-      "logps/chosen": -166.97962951660156,
-      "logps/rejected": -218.309326171875,
-      "loss": 0.3195725977420807,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.148334503173828,
-      "rewards/margins": 2.3881964683532715,
-      "rewards/rejected": -8.536530494689941,
-      "step": 284
-    },
-    {
-      "epoch": 0.9193548387096774,
-      "grad_norm": 0.9417896270751953,
-      "learning_rate": 2.8284085869828664e-06,
-      "logits/chosen": -2.085608720779419,
-      "logits/rejected": -1.8423289060592651,
-      "logps/chosen": -160.0970001220703,
-      "logps/rejected": -206.8173370361328,
-      "loss": 0.4341191351413727,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -5.784702301025391,
-      "rewards/margins": 2.376483917236328,
-      "rewards/rejected": -8.161186218261719,
-      "step": 285
-    },
-    {
-      "epoch": 0.9225806451612903,
-      "grad_norm": 1.097767949104309,
-      "learning_rate": 2.8158464873053236e-06,
-      "logits/chosen": -2.0979247093200684,
-      "logits/rejected": -1.7694401741027832,
-      "logps/chosen": -152.70974731445312,
-      "logps/rejected": -214.92532348632812,
-      "loss": 0.3746206760406494,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -5.5105414390563965,
-      "rewards/margins": 2.975477695465088,
-      "rewards/rejected": -8.486019134521484,
-      "step": 286
-    },
-    {
-      "epoch": 0.9258064516129032,
-      "grad_norm": 1.082982063293457,
-      "learning_rate": 2.8032762781761376e-06,
-      "logits/chosen": -2.130652666091919,
-      "logits/rejected": -1.8887903690338135,
-      "logps/chosen": -152.620849609375,
-      "logps/rejected": -197.34225463867188,
-      "loss": 0.46686190366744995,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -4.933390140533447,
-      "rewards/margins": 2.3892898559570312,
-      "rewards/rejected": -7.32267951965332,
-      "step": 287
-    },
-    {
-      "epoch": 0.9290322580645162,
-      "grad_norm": 0.8161581754684448,
-      "learning_rate": 2.790698282339133e-06,
-      "logits/chosen": -2.4120960235595703,
-      "logits/rejected": -2.0684590339660645,
-      "logps/chosen": -128.7693634033203,
-      "logps/rejected": -190.23130798339844,
-      "loss": 0.3193214535713196,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -4.401476860046387,
-      "rewards/margins": 3.0362207889556885,
-      "rewards/rejected": -7.437697410583496,
-      "step": 288
-    },
-    {
-      "epoch": 0.932258064516129,
-      "grad_norm": 0.9246862530708313,
-      "learning_rate": 2.7781128227380595e-06,
-      "logits/chosen": -2.4583992958068848,
-      "logits/rejected": -2.163020610809326,
-      "logps/chosen": -149.81240844726562,
-      "logps/rejected": -206.80284118652344,
-      "loss": 0.33852618932724,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -4.645122528076172,
-      "rewards/margins": 2.739440441131592,
-      "rewards/rejected": -7.384562969207764,
-      "step": 289
-    },
-    {
-      "epoch": 0.9354838709677419,
-      "grad_norm": 0.8552197217941284,
-      "learning_rate": 2.765520222508302e-06,
-      "logits/chosen": -2.0391581058502197,
-      "logits/rejected": -1.685469388961792,
-      "logps/chosen": -160.41790771484375,
-      "logps/rejected": -214.21324157714844,
-      "loss": 0.3561515808105469,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.971251487731934,
-      "rewards/margins": 2.890326738357544,
-      "rewards/rejected": -7.861577987670898,
-      "step": 290
-    },
-    {
-      "epoch": 0.9387096774193548,
-      "grad_norm": 1.0460398197174072,
-      "learning_rate": 2.752920804968581e-06,
-      "logits/chosen": -2.1437153816223145,
-      "logits/rejected": -2.0144546031951904,
-      "logps/chosen": -166.39102172851562,
-      "logps/rejected": -208.45326232910156,
-      "loss": 0.47864845395088196,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -5.143545627593994,
-      "rewards/margins": 1.8947696685791016,
-      "rewards/rejected": -7.038315773010254,
-      "step": 291
-    },
-    {
-      "epoch": 0.9419354838709677,
-      "grad_norm": 0.44466134905815125,
-      "learning_rate": 2.740314893612656e-06,
-      "logits/chosen": -2.0010452270507812,
-      "logits/rejected": -1.6200562715530396,
-      "logps/chosen": -137.75228881835938,
-      "logps/rejected": -212.52597045898438,
-      "loss": 0.13167953491210938,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -3.997614622116089,
-      "rewards/margins": 4.028761863708496,
-      "rewards/rejected": -8.026376724243164,
-      "step": 292
-    },
-    {
-      "epoch": 0.9451612903225807,
-      "grad_norm": 0.8740090131759644,
-      "learning_rate": 2.727702812101016e-06,
-      "logits/chosen": -2.171569347381592,
-      "logits/rejected": -1.9190174341201782,
-      "logps/chosen": -145.79901123046875,
-      "logps/rejected": -209.12521362304688,
-      "loss": 0.25788789987564087,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.480965614318848,
-      "rewards/margins": 2.9965264797210693,
-      "rewards/rejected": -7.47749137878418,
-      "step": 293
-    },
-    {
-      "epoch": 0.9483870967741935,
-      "grad_norm": 0.9442008137702942,
-      "learning_rate": 2.7150848842525707e-06,
-      "logits/chosen": -1.9686082601547241,
-      "logits/rejected": -1.769315242767334,
-      "logps/chosen": -161.62548828125,
-      "logps/rejected": -213.39918518066406,
-      "loss": 0.3881966173648834,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -5.6515092849731445,
-      "rewards/margins": 2.542401075363159,
-      "rewards/rejected": -8.193910598754883,
-      "step": 294
-    },
-    {
-      "epoch": 0.9516129032258065,
-      "grad_norm": 0.8742654323577881,
-      "learning_rate": 2.7024614340363365e-06,
-      "logits/chosen": -1.7042088508605957,
-      "logits/rejected": -1.550293207168579,
-      "logps/chosen": -176.0225830078125,
-      "logps/rejected": -210.4657745361328,
-      "loss": 0.423173725605011,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -5.02152156829834,
-      "rewards/margins": 2.2260966300964355,
-      "rewards/rejected": -7.247618198394775,
-      "step": 295
-    },
-    {
-      "epoch": 0.9548387096774194,
-      "grad_norm": 0.6472557783126831,
-      "learning_rate": 2.689832785563116e-06,
-      "logits/chosen": -1.7272298336029053,
-      "logits/rejected": -1.3803935050964355,
-      "logps/chosen": -164.98233032226562,
-      "logps/rejected": -229.7456817626953,
-      "loss": 0.20416554808616638,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.407524108886719,
-      "rewards/margins": 3.4527716636657715,
-      "rewards/rejected": -8.860294342041016,
-      "step": 296
-    },
-    {
-      "epoch": 0.9580645161290322,
-      "grad_norm": 0.6706839203834534,
-      "learning_rate": 2.6771992630771824e-06,
-      "logits/chosen": -1.798374056816101,
-      "logits/rejected": -1.4753650426864624,
-      "logps/chosen": -163.36074829101562,
-      "logps/rejected": -225.06646728515625,
-      "loss": 0.17277327179908752,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.153785705566406,
-      "rewards/margins": 3.462008237838745,
-      "rewards/rejected": -8.61579418182373,
-      "step": 297
-    },
-    {
-      "epoch": 0.9612903225806452,
-      "grad_norm": 1.067395806312561,
-      "learning_rate": 2.6645611909479484e-06,
-      "logits/chosen": -2.2678630352020264,
-      "logits/rejected": -2.0605649948120117,
-      "logps/chosen": -148.91651916503906,
-      "logps/rejected": -192.33926391601562,
-      "loss": 0.5109429955482483,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -4.3728766441345215,
-      "rewards/margins": 2.1208512783050537,
-      "rewards/rejected": -6.493727684020996,
-      "step": 298
-    },
-    {
-      "epoch": 0.964516129032258,
-      "grad_norm": 0.8057169914245605,
-      "learning_rate": 2.65191889366164e-06,
-      "logits/chosen": -2.063993453979492,
-      "logits/rejected": -1.9815953969955444,
-      "logps/chosen": -158.8735809326172,
-      "logps/rejected": -204.44784545898438,
-      "loss": 0.29160138964653015,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -4.964184761047363,
-      "rewards/margins": 2.093125820159912,
-      "rewards/rejected": -7.057311058044434,
-      "step": 299
-    },
-    {
-      "epoch": 0.967741935483871,
-      "grad_norm": 0.831497073173523,
-      "learning_rate": 2.6392726958129653e-06,
-      "logits/chosen": -2.101418972015381,
-      "logits/rejected": -1.8252373933792114,
-      "logps/chosen": -144.0537109375,
-      "logps/rejected": -195.8699951171875,
-      "loss": 0.3357257843017578,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -5.049233436584473,
-      "rewards/margins": 2.7640864849090576,
-      "rewards/rejected": -7.813320159912109,
-      "step": 300
-    },
-    {
-      "epoch": 0.9709677419354839,
-      "grad_norm": 0.7803052663803101,
-      "learning_rate": 2.626622922096782e-06,
-      "logits/chosen": -1.9163737297058105,
-      "logits/rejected": -1.6509616374969482,
-      "logps/chosen": -155.09027099609375,
-      "logps/rejected": -208.73690795898438,
-      "loss": 0.26208123564720154,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.911482810974121,
-      "rewards/margins": 2.844212532043457,
-      "rewards/rejected": -7.75569486618042,
-      "step": 301
-    },
-    {
-      "epoch": 0.9741935483870968,
-      "grad_norm": 0.8996358513832092,
-      "learning_rate": 2.613969897299757e-06,
-      "logits/chosen": -2.241206645965576,
-      "logits/rejected": -2.0861012935638428,
-      "logps/chosen": -149.37405395507812,
-      "logps/rejected": -192.15811157226562,
-      "loss": 0.31268051266670227,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.1501898765563965,
-      "rewards/margins": 2.2168045043945312,
-      "rewards/rejected": -7.366994380950928,
-      "step": 302
-    },
-    {
-      "epoch": 0.9774193548387097,
-      "grad_norm": 0.7502899169921875,
-      "learning_rate": 2.6013139462920328e-06,
-      "logits/chosen": -1.9167232513427734,
-      "logits/rejected": -1.5168886184692383,
-      "logps/chosen": -150.22955322265625,
-      "logps/rejected": -229.1195068359375,
-      "loss": 0.18935909867286682,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.952530860900879,
-      "rewards/margins": 4.4970598220825195,
-      "rewards/rejected": -9.449591636657715,
-      "step": 303
-    },
-    {
-      "epoch": 0.9806451612903225,
-      "grad_norm": 0.8504204154014587,
-      "learning_rate": 2.588655394018879e-06,
-      "logits/chosen": -2.1758933067321777,
-      "logits/rejected": -1.9743213653564453,
-      "logps/chosen": -145.38467407226562,
-      "logps/rejected": -210.38336181640625,
-      "loss": 0.2861388921737671,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -5.063721656799316,
-      "rewards/margins": 3.1153056621551514,
-      "rewards/rejected": -8.17902660369873,
-      "step": 304
-    },
-    {
-      "epoch": 0.9838709677419355,
-      "grad_norm": 0.7789464592933655,
-      "learning_rate": 2.5759945654923575e-06,
-      "logits/chosen": -2.010745048522949,
-      "logits/rejected": -1.8690520524978638,
-      "logps/chosen": -141.83558654785156,
-      "logps/rejected": -194.40895080566406,
-      "loss": 0.23465344309806824,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.525321006774902,
-      "rewards/margins": 2.7977147102355957,
-      "rewards/rejected": -7.32303524017334,
-      "step": 305
-    },
-    {
-      "epoch": 0.9870967741935484,
-      "grad_norm": 1.0136746168136597,
-      "learning_rate": 2.56333178578297e-06,
-      "logits/chosen": -2.1397182941436768,
-      "logits/rejected": -1.9402344226837158,
-      "logps/chosen": -172.6583709716797,
-      "logps/rejected": -228.28738403320312,
-      "loss": 0.44921568036079407,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -6.19842004776001,
-      "rewards/margins": 3.0715999603271484,
-      "rewards/rejected": -9.27001953125,
-      "step": 306
-    },
-    {
-      "epoch": 0.9903225806451613,
-      "grad_norm": 0.965210497379303,
-      "learning_rate": 2.5506673800113196e-06,
-      "logits/chosen": -2.106553554534912,
-      "logits/rejected": -1.8104825019836426,
-      "logps/chosen": -172.39144897460938,
-      "logps/rejected": -240.924072265625,
-      "loss": 0.24592578411102295,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -6.227846622467041,
-      "rewards/margins": 3.537505626678467,
-      "rewards/rejected": -9.765352249145508,
-      "step": 307
-    },
-    {
-      "epoch": 0.9935483870967742,
-      "grad_norm": 0.9008381962776184,
-      "learning_rate": 2.538001673339754e-06,
-      "logits/chosen": -1.8831700086593628,
-      "logits/rejected": -1.5683573484420776,
-      "logps/chosen": -174.13272094726562,
-      "logps/rejected": -222.25619506835938,
-      "loss": 0.3525466322898865,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.677732467651367,
-      "rewards/margins": 2.817638635635376,
-      "rewards/rejected": -8.495370864868164,
-      "step": 308
-    },
-    {
-      "epoch": 0.9967741935483871,
-      "grad_norm": 0.8979695439338684,
-      "learning_rate": 2.525334990964028e-06,
-      "logits/chosen": -2.152176856994629,
-      "logits/rejected": -1.8049001693725586,
-      "logps/chosen": -136.01487731933594,
-      "logps/rejected": -209.47459411621094,
-      "loss": 0.38963136076927185,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -4.782433032989502,
-      "rewards/margins": 3.5359387397766113,
-      "rewards/rejected": -8.318371772766113,
-      "step": 309
-    },
-    {
-      "epoch": 1.0,
-      "grad_norm": 0.8429840803146362,
-      "learning_rate": 2.5126676581049413e-06,
-      "logits/chosen": -2.113762378692627,
-      "logits/rejected": -1.818450927734375,
-      "logps/chosen": -167.97877502441406,
-      "logps/rejected": -227.44183349609375,
-      "loss": 0.30903160572052,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.818127155303955,
-      "rewards/margins": 3.091285467147827,
-      "rewards/rejected": -8.909412384033203,
-      "step": 310
-    },
-    {
-      "epoch": 1.0032258064516129,
-      "grad_norm": 0.6775534152984619,
-      "learning_rate": 2.5e-06,
-      "logits/chosen": -2.1721529960632324,
-      "logits/rejected": -1.9033340215682983,
-      "logps/chosen": -163.31370544433594,
-      "logps/rejected": -222.24139404296875,
-      "loss": 0.21583493053913116,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.506072044372559,
-      "rewards/margins": 3.459946632385254,
-      "rewards/rejected": -8.966018676757812,
-      "step": 311
-    },
-    {
-      "epoch": 1.0064516129032257,
-      "grad_norm": 0.580848753452301,
-      "learning_rate": 2.487332341895059e-06,
-      "logits/chosen": -2.0694403648376465,
-      "logits/rejected": -1.747127652168274,
-      "logps/chosen": -149.02215576171875,
-      "logps/rejected": -212.60113525390625,
-      "loss": 0.16063669323921204,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -4.965436935424805,
-      "rewards/margins": 3.742509365081787,
-      "rewards/rejected": -8.70794677734375,
-      "step": 312
-    },
-    {
-      "epoch": 1.0096774193548388,
-      "grad_norm": 0.38149571418762207,
-      "learning_rate": 2.4746650090359733e-06,
-      "logits/chosen": -2.2192482948303223,
-      "logits/rejected": -1.9255717992782593,
-      "logps/chosen": -151.22067260742188,
-      "logps/rejected": -207.1800079345703,
-      "loss": 0.1110997125506401,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -4.916757583618164,
-      "rewards/margins": 3.4106850624084473,
-      "rewards/rejected": -8.327442169189453,
-      "step": 313
-    },
-    {
-      "epoch": 1.0129032258064516,
-      "grad_norm": 0.5131282210350037,
-      "learning_rate": 2.461998326660246e-06,
-      "logits/chosen": -2.0201852321624756,
-      "logits/rejected": -1.7313728332519531,
-      "logps/chosen": -157.92855834960938,
-      "logps/rejected": -210.2177734375,
-      "loss": 0.15024253726005554,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.242098331451416,
-      "rewards/margins": 3.1161749362945557,
-      "rewards/rejected": -8.358272552490234,
-      "step": 314
-    },
-    {
-      "epoch": 1.0161290322580645,
-      "grad_norm": 0.6810634136199951,
-      "learning_rate": 2.4493326199886813e-06,
-      "logits/chosen": -2.2641396522521973,
-      "logits/rejected": -2.006556272506714,
-      "logps/chosen": -158.7135009765625,
-      "logps/rejected": -215.43856811523438,
-      "loss": 0.19132620096206665,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -4.935375213623047,
-      "rewards/margins": 3.58085298538208,
-      "rewards/rejected": -8.516228675842285,
-      "step": 315
-    },
-    {
-      "epoch": 1.0193548387096774,
-      "grad_norm": 0.5285111665725708,
-      "learning_rate": 2.436668214217031e-06,
-      "logits/chosen": -2.0756735801696777,
-      "logits/rejected": -1.842437744140625,
-      "logps/chosen": -178.60182189941406,
-      "logps/rejected": -226.6915283203125,
-      "loss": 0.1702248901128769,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.96095085144043,
-      "rewards/margins": 3.078915596008301,
-      "rewards/rejected": -9.03986644744873,
-      "step": 316
-    },
-    {
-      "epoch": 1.0225806451612902,
-      "grad_norm": 0.5416440367698669,
-      "learning_rate": 2.424005434507644e-06,
-      "logits/chosen": -1.8712788820266724,
-      "logits/rejected": -1.5404462814331055,
-      "logps/chosen": -181.17999267578125,
-      "logps/rejected": -250.18528747558594,
-      "loss": 0.1307283192873001,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -6.399822235107422,
-      "rewards/margins": 4.233570098876953,
-      "rewards/rejected": -10.633392333984375,
-      "step": 317
-    },
-    {
-      "epoch": 1.0258064516129033,
-      "grad_norm": 0.4123290777206421,
-      "learning_rate": 2.4113446059811217e-06,
-      "logits/chosen": -1.9893611669540405,
-      "logits/rejected": -1.720315933227539,
-      "logps/chosen": -172.31666564941406,
-      "logps/rejected": -229.56271362304688,
-      "loss": 0.11024826765060425,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.526625633239746,
-      "rewards/margins": 3.5636584758758545,
-      "rewards/rejected": -9.09028434753418,
-      "step": 318
-    },
-    {
-      "epoch": 1.0290322580645161,
-      "grad_norm": 0.7053832411766052,
-      "learning_rate": 2.3986860537079676e-06,
-      "logits/chosen": -1.8367996215820312,
-      "logits/rejected": -1.4809139966964722,
-      "logps/chosen": -155.18275451660156,
-      "logps/rejected": -212.97434997558594,
-      "loss": 0.21443486213684082,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -5.035017967224121,
-      "rewards/margins": 3.13305926322937,
-      "rewards/rejected": -8.16807746887207,
-      "step": 319
-    },
-    {
-      "epoch": 1.032258064516129,
-      "grad_norm": 0.2714214622974396,
-      "learning_rate": 2.3860301027002432e-06,
-      "logits/chosen": -1.9262335300445557,
-      "logits/rejected": -1.5826025009155273,
-      "logps/chosen": -141.29434204101562,
-      "logps/rejected": -219.14004516601562,
-      "loss": 0.07531583309173584,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.152288436889648,
-      "rewards/margins": 4.746103286743164,
-      "rewards/rejected": -9.898390769958496,
-      "step": 320
-    },
-    {
-      "epoch": 1.0354838709677419,
-      "grad_norm": 0.4167848527431488,
-      "learning_rate": 2.3733770779032185e-06,
-      "logits/chosen": -2.2220044136047363,
-      "logits/rejected": -1.7983975410461426,
-      "logps/chosen": -133.76486206054688,
-      "logps/rejected": -216.0334014892578,
-      "loss": 0.12337338924407959,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -4.694788932800293,
-      "rewards/margins": 4.698427677154541,
-      "rewards/rejected": -9.393217086791992,
-      "step": 321
-    },
-    {
-      "epoch": 1.038709677419355,
-      "grad_norm": 0.424111545085907,
-      "learning_rate": 2.3607273041870355e-06,
-      "logits/chosen": -2.181683301925659,
-      "logits/rejected": -1.7694246768951416,
-      "logps/chosen": -133.67471313476562,
-      "logps/rejected": -207.6949920654297,
-      "loss": 0.08804497122764587,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -4.045483112335205,
-      "rewards/margins": 4.212948799133301,
-      "rewards/rejected": -8.258432388305664,
-      "step": 322
-    },
-    {
-      "epoch": 1.0419354838709678,
-      "grad_norm": 0.6895016431808472,
-      "learning_rate": 2.3480811063383603e-06,
-      "logits/chosen": -2.051720142364502,
-      "logits/rejected": -1.7328163385391235,
-      "logps/chosen": -158.16627502441406,
-      "logps/rejected": -217.3858642578125,
-      "loss": 0.19755461812019348,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.892249584197998,
-      "rewards/margins": 3.8145363330841064,
-      "rewards/rejected": -8.706785202026367,
-      "step": 323
-    },
-    {
-      "epoch": 1.0451612903225806,
-      "grad_norm": 0.4730503559112549,
-      "learning_rate": 2.335438809052052e-06,
-      "logits/chosen": -2.212907075881958,
-      "logits/rejected": -1.7950420379638672,
-      "logps/chosen": -147.1448211669922,
-      "logps/rejected": -224.41746520996094,
-      "loss": 0.12352735549211502,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.53288459777832,
-      "rewards/margins": 4.16781759262085,
-      "rewards/rejected": -9.700702667236328,
-      "step": 324
-    },
-    {
-      "epoch": 1.0483870967741935,
-      "grad_norm": 0.37026146054267883,
-      "learning_rate": 2.322800736922818e-06,
-      "logits/chosen": -1.8258745670318604,
-      "logits/rejected": -1.4101828336715698,
-      "logps/chosen": -165.53468322753906,
-      "logps/rejected": -251.0711669921875,
-      "loss": 0.07076412439346313,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.959969520568848,
-      "rewards/margins": 5.046208381652832,
-      "rewards/rejected": -11.006175994873047,
-      "step": 325
-    },
-    {
-      "epoch": 1.0516129032258064,
-      "grad_norm": 0.3133352994918823,
-      "learning_rate": 2.310167214436885e-06,
-      "logits/chosen": -2.0526835918426514,
-      "logits/rejected": -1.789970874786377,
-      "logps/chosen": -149.29171752929688,
-      "logps/rejected": -222.2518310546875,
-      "loss": 0.08292093873023987,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -4.683323383331299,
-      "rewards/margins": 4.279122829437256,
-      "rewards/rejected": -8.962446212768555,
-      "step": 326
-    },
-    {
-      "epoch": 1.0548387096774194,
-      "grad_norm": 0.5130003690719604,
-      "learning_rate": 2.297538565963665e-06,
-      "logits/chosen": -2.2903833389282227,
-      "logits/rejected": -1.977923035621643,
-      "logps/chosen": -140.78390502929688,
-      "logps/rejected": -208.8832550048828,
-      "loss": 0.18747130036354065,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.893636703491211,
-      "rewards/margins": 4.177481651306152,
-      "rewards/rejected": -9.071118354797363,
-      "step": 327
-    },
-    {
-      "epoch": 1.0580645161290323,
-      "grad_norm": 0.4587818682193756,
-      "learning_rate": 2.2849151157474297e-06,
-      "logits/chosen": -1.6147241592407227,
-      "logits/rejected": -1.2173912525177002,
-      "logps/chosen": -166.45401000976562,
-      "logps/rejected": -244.99241638183594,
-      "loss": 0.11498372256755829,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -6.213277816772461,
-      "rewards/margins": 4.60243034362793,
-      "rewards/rejected": -10.81570816040039,
-      "step": 328
-    },
-    {
-      "epoch": 1.0612903225806452,
-      "grad_norm": 0.41809093952178955,
-      "learning_rate": 2.2722971878989843e-06,
-      "logits/chosen": -1.6840178966522217,
-      "logits/rejected": -1.3177802562713623,
-      "logps/chosen": -172.50921630859375,
-      "logps/rejected": -257.9876403808594,
-      "loss": 0.07585393637418747,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -6.2325544357299805,
-      "rewards/margins": 4.915229320526123,
-      "rewards/rejected": -11.147783279418945,
-      "step": 329
-    },
-    {
-      "epoch": 1.064516129032258,
-      "grad_norm": 0.4529079496860504,
-      "learning_rate": 2.259685106387345e-06,
-      "logits/chosen": -2.2479963302612305,
-      "logits/rejected": -1.8993940353393555,
-      "logps/chosen": -142.16958618164062,
-      "logps/rejected": -214.33688354492188,
-      "loss": 0.14075274765491486,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -4.91881799697876,
-      "rewards/margins": 4.307929992675781,
-      "rewards/rejected": -9.2267484664917,
-      "step": 330
-    },
-    {
-      "epoch": 1.067741935483871,
-      "grad_norm": 0.8104569911956787,
-      "learning_rate": 2.24707919503142e-06,
-      "logits/chosen": -1.9945495128631592,
-      "logits/rejected": -1.7296099662780762,
-      "logps/chosen": -170.532958984375,
-      "logps/rejected": -235.51565551757812,
-      "loss": 0.2429746687412262,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -6.278332710266113,
-      "rewards/margins": 3.6663362979888916,
-      "rewards/rejected": -9.944669723510742,
-      "step": 331
-    },
-    {
-      "epoch": 1.070967741935484,
-      "grad_norm": 0.6494706869125366,
-      "learning_rate": 2.2344797774917e-06,
-      "logits/chosen": -2.075692653656006,
-      "logits/rejected": -1.7635282278060913,
-      "logps/chosen": -141.7442169189453,
-      "logps/rejected": -219.43185424804688,
-      "loss": 0.1584240049123764,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.1548686027526855,
-      "rewards/margins": 4.707845687866211,
-      "rewards/rejected": -9.862714767456055,
-      "step": 332
-    },
-    {
-      "epoch": 1.0741935483870968,
-      "grad_norm": 0.5460749268531799,
-      "learning_rate": 2.2218871772619414e-06,
-      "logits/chosen": -1.5849077701568604,
-      "logits/rejected": -1.2364206314086914,
-      "logps/chosen": -164.15150451660156,
-      "logps/rejected": -232.79945373535156,
-      "loss": 0.13620774447917938,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.4682841300964355,
-      "rewards/margins": 4.182202339172363,
-      "rewards/rejected": -9.65048599243164,
-      "step": 333
-    },
-    {
-      "epoch": 1.0774193548387097,
-      "grad_norm": 0.9638095498085022,
-      "learning_rate": 2.2093017176608678e-06,
-      "logits/chosen": -2.240260601043701,
-      "logits/rejected": -1.9081519842147827,
-      "logps/chosen": -140.4049835205078,
-      "logps/rejected": -199.9825439453125,
-      "loss": 0.280463844537735,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -4.912430286407471,
-      "rewards/margins": 3.654731035232544,
-      "rewards/rejected": -8.567161560058594,
-      "step": 334
-    },
-    {
-      "epoch": 1.0806451612903225,
-      "grad_norm": 0.6865034699440002,
-      "learning_rate": 2.196723721823863e-06,
-      "logits/chosen": -1.5741136074066162,
-      "logits/rejected": -1.3333443403244019,
-      "logps/chosen": -160.82054138183594,
-      "logps/rejected": -229.42044067382812,
-      "loss": 0.2099701464176178,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -6.163061141967773,
-      "rewards/margins": 3.6050832271575928,
-      "rewards/rejected": -9.768144607543945,
-      "step": 335
-    },
-    {
-      "epoch": 1.0838709677419356,
-      "grad_norm": 0.5247408747673035,
-      "learning_rate": 2.1841535126946777e-06,
-      "logits/chosen": -1.8184685707092285,
-      "logits/rejected": -1.509084939956665,
-      "logps/chosen": -175.3823699951172,
-      "logps/rejected": -234.17550659179688,
-      "loss": 0.13480132818222046,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -6.176336288452148,
-      "rewards/margins": 3.4384005069732666,
-      "rewards/rejected": -9.614737510681152,
-      "step": 336
-    },
-    {
-      "epoch": 1.0870967741935484,
-      "grad_norm": 0.3721334934234619,
-      "learning_rate": 2.171591413017134e-06,
-      "logits/chosen": -2.2993922233581543,
-      "logits/rejected": -1.9334840774536133,
-      "logps/chosen": -140.510986328125,
-      "logps/rejected": -212.41583251953125,
-      "loss": 0.10910043120384216,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.957540988922119,
-      "rewards/margins": 4.744095325469971,
-      "rewards/rejected": -9.701635360717773,
-      "step": 337
-    },
-    {
-      "epoch": 1.0903225806451613,
-      "grad_norm": 0.5746122002601624,
-      "learning_rate": 2.159037745326844e-06,
-      "logits/chosen": -1.9673866033554077,
-      "logits/rejected": -1.6878376007080078,
-      "logps/chosen": -161.87252807617188,
-      "logps/rejected": -217.6410369873047,
-      "loss": 0.1833765208721161,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.09132194519043,
-      "rewards/margins": 3.5239312648773193,
-      "rewards/rejected": -8.615253448486328,
-      "step": 338
-    },
-    {
-      "epoch": 1.0935483870967742,
-      "grad_norm": 0.6791886687278748,
-      "learning_rate": 2.146492831942927e-06,
-      "logits/chosen": -1.4476041793823242,
-      "logits/rejected": -1.112692952156067,
-      "logps/chosen": -196.64627075195312,
-      "logps/rejected": -260.59063720703125,
-      "loss": 0.15957781672477722,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -7.228837013244629,
-      "rewards/margins": 3.7981371879577637,
-      "rewards/rejected": -11.026973724365234,
-      "step": 339
-    },
-    {
-      "epoch": 1.096774193548387,
-      "grad_norm": 0.3706646263599396,
-      "learning_rate": 2.1339569949597284e-06,
-      "logits/chosen": -2.0635392665863037,
-      "logits/rejected": -1.683074951171875,
-      "logps/chosen": -140.18783569335938,
-      "logps/rejected": -207.48167419433594,
-      "loss": 0.08880328387022018,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -4.723042964935303,
-      "rewards/margins": 4.2141523361206055,
-      "rewards/rejected": -8.93719482421875,
-      "step": 340
-    },
-    {
-      "epoch": 1.1,
-      "grad_norm": 0.3307356834411621,
-      "learning_rate": 2.1214305562385592e-06,
-      "logits/chosen": -2.034179210662842,
-      "logits/rejected": -1.678894281387329,
-      "logps/chosen": -139.79940795898438,
-      "logps/rejected": -222.440185546875,
-      "loss": 0.07107400894165039,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -4.712695598602295,
-      "rewards/margins": 4.9850287437438965,
-      "rewards/rejected": -9.697725296020508,
-      "step": 341
-    },
-    {
-      "epoch": 1.103225806451613,
-      "grad_norm": 0.65967857837677,
-      "learning_rate": 2.1089138373994226e-06,
-      "logits/chosen": -2.1784443855285645,
-      "logits/rejected": -1.9255266189575195,
-      "logps/chosen": -152.837646484375,
-      "logps/rejected": -213.63534545898438,
-      "loss": 0.22975148260593414,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.133075714111328,
-      "rewards/margins": 3.4656684398651123,
-      "rewards/rejected": -8.59874439239502,
-      "step": 342
-    },
-    {
-      "epoch": 1.1064516129032258,
-      "grad_norm": 0.35155734419822693,
-      "learning_rate": 2.096407159812767e-06,
-      "logits/chosen": -2.32820463180542,
-      "logits/rejected": -1.971595287322998,
-      "logps/chosen": -145.63475036621094,
-      "logps/rejected": -218.73211669921875,
-      "loss": 0.07563674449920654,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.143950462341309,
-      "rewards/margins": 4.366302490234375,
-      "rewards/rejected": -9.510252952575684,
-      "step": 343
-    },
-    {
-      "epoch": 1.1096774193548387,
-      "grad_norm": 0.734062910079956,
-      "learning_rate": 2.0839108445912222e-06,
-      "logits/chosen": -1.8277361392974854,
-      "logits/rejected": -1.6029260158538818,
-      "logps/chosen": -171.10125732421875,
-      "logps/rejected": -232.6971435546875,
-      "loss": 0.20384469628334045,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -5.091307640075684,
-      "rewards/margins": 3.67594313621521,
-      "rewards/rejected": -8.767250061035156,
-      "step": 344
-    },
-    {
-      "epoch": 1.1129032258064515,
-      "grad_norm": 0.5717563033103943,
-      "learning_rate": 2.0714252125813667e-06,
-      "logits/chosen": -1.7721185684204102,
-      "logits/rejected": -1.4680637121200562,
-      "logps/chosen": -173.13760375976562,
-      "logps/rejected": -239.3628387451172,
-      "loss": 0.1405162811279297,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -6.596169471740723,
-      "rewards/margins": 3.965106725692749,
-      "rewards/rejected": -10.561275482177734,
-      "step": 345
-    },
-    {
-      "epoch": 1.1161290322580646,
-      "grad_norm": 0.6655075550079346,
-      "learning_rate": 2.05895058435548e-06,
-      "logits/chosen": -1.9385762214660645,
-      "logits/rejected": -1.6046488285064697,
-      "logps/chosen": -165.9267578125,
-      "logps/rejected": -237.217041015625,
-      "loss": 0.1554121971130371,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -6.313894271850586,
-      "rewards/margins": 4.2116498947143555,
-      "rewards/rejected": -10.525544166564941,
-      "step": 346
-    },
-    {
-      "epoch": 1.1193548387096774,
-      "grad_norm": 0.5084536075592041,
-      "learning_rate": 2.0464872802033174e-06,
-      "logits/chosen": -2.077972412109375,
-      "logits/rejected": -1.7521743774414062,
-      "logps/chosen": -163.38247680664062,
-      "logps/rejected": -234.90512084960938,
-      "loss": 0.1205044537782669,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -6.103501796722412,
-      "rewards/margins": 4.484037399291992,
-      "rewards/rejected": -10.587538719177246,
-      "step": 347
-    },
-    {
-      "epoch": 1.1225806451612903,
-      "grad_norm": 0.2510414123535156,
-      "learning_rate": 2.0340356201238883e-06,
-      "logits/chosen": -2.283794403076172,
-      "logits/rejected": -1.8476622104644775,
-      "logps/chosen": -145.0723876953125,
-      "logps/rejected": -223.35067749023438,
-      "loss": 0.05795178934931755,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.06467866897583,
-      "rewards/margins": 4.865185737609863,
-      "rewards/rejected": -9.929864883422852,
-      "step": 348
-    },
-    {
-      "epoch": 1.1258064516129032,
-      "grad_norm": 0.8760903477668762,
-      "learning_rate": 2.0215959238172343e-06,
-      "logits/chosen": -2.1980106830596924,
-      "logits/rejected": -1.9146432876586914,
-      "logps/chosen": -155.1454620361328,
-      "logps/rejected": -213.25482177734375,
-      "loss": 0.22108781337738037,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -6.415738105773926,
-      "rewards/margins": 3.4907078742980957,
-      "rewards/rejected": -9.906445503234863,
-      "step": 349
-    },
-    {
-      "epoch": 1.129032258064516,
-      "grad_norm": 0.640398383140564,
-      "learning_rate": 2.0091685106762233e-06,
-      "logits/chosen": -1.9763402938842773,
-      "logits/rejected": -1.772810459136963,
-      "logps/chosen": -174.51522827148438,
-      "logps/rejected": -224.62802124023438,
-      "loss": 0.20412787795066833,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -6.7324604988098145,
-      "rewards/margins": 3.1668648719787598,
-      "rewards/rejected": -9.899325370788574,
-      "step": 350
-    },
-    {
-      "epoch": 1.132258064516129,
-      "grad_norm": 0.34046998620033264,
-      "learning_rate": 1.9967536997783495e-06,
-      "logits/chosen": -2.074873924255371,
-      "logits/rejected": -1.7466737031936646,
-      "logps/chosen": -143.79574584960938,
-      "logps/rejected": -227.416259765625,
-      "loss": 0.0682026743888855,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.0686492919921875,
-      "rewards/margins": 4.900749206542969,
-      "rewards/rejected": -9.969398498535156,
-      "step": 351
-    },
-    {
-      "epoch": 1.135483870967742,
-      "grad_norm": 0.8615036010742188,
-      "learning_rate": 1.984351809877544e-06,
-      "logits/chosen": -1.5838978290557861,
-      "logits/rejected": -1.3324167728424072,
-      "logps/chosen": -170.6785888671875,
-      "logps/rejected": -229.92874145507812,
-      "loss": 0.19454488158226013,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.850698471069336,
-      "rewards/margins": 3.9882798194885254,
-      "rewards/rejected": -9.838977813720703,
-      "step": 352
-    },
-    {
-      "epoch": 1.1387096774193548,
-      "grad_norm": 0.40390846133232117,
-      "learning_rate": 1.9719631593959818e-06,
-      "logits/chosen": -2.2553353309631348,
-      "logits/rejected": -1.9675357341766357,
-      "logps/chosen": -144.84364318847656,
-      "logps/rejected": -208.33547973632812,
-      "loss": 0.09588846564292908,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.0572829246521,
-      "rewards/margins": 4.1294379234313965,
-      "rewards/rejected": -9.18671989440918,
-      "step": 353
-    },
-    {
-      "epoch": 1.1419354838709677,
-      "grad_norm": 0.7221540808677673,
-      "learning_rate": 1.959588066415917e-06,
-      "logits/chosen": -1.7495440244674683,
-      "logits/rejected": -1.3849916458129883,
-      "logps/chosen": -162.43798828125,
-      "logps/rejected": -229.17469787597656,
-      "loss": 0.1190003752708435,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.087733268737793,
-      "rewards/margins": 4.036655426025391,
-      "rewards/rejected": -9.124388694763184,
-      "step": 354
-    },
-    {
-      "epoch": 1.1451612903225807,
-      "grad_norm": 0.6402134895324707,
-      "learning_rate": 1.947226848671508e-06,
-      "logits/chosen": -2.255587339401245,
-      "logits/rejected": -1.9237840175628662,
-      "logps/chosen": -150.15924072265625,
-      "logps/rejected": -218.59298706054688,
-      "loss": 0.18148204684257507,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.075443267822266,
-      "rewards/margins": 3.923079252243042,
-      "rewards/rejected": -8.998522758483887,
-      "step": 355
-    },
-    {
-      "epoch": 1.1483870967741936,
-      "grad_norm": 0.6949121952056885,
-      "learning_rate": 1.934879823540663e-06,
-      "logits/chosen": -2.109438896179199,
-      "logits/rejected": -1.7586997747421265,
-      "logps/chosen": -160.052734375,
-      "logps/rejected": -225.48373413085938,
-      "loss": 0.1760554015636444,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -5.738818645477295,
-      "rewards/margins": 4.271374702453613,
-      "rewards/rejected": -10.01019287109375,
-      "step": 356
-    },
-    {
-      "epoch": 1.1516129032258065,
-      "grad_norm": 0.5934706330299377,
-      "learning_rate": 1.9225473080368916e-06,
-      "logits/chosen": -2.27186918258667,
-      "logits/rejected": -1.974424123764038,
-      "logps/chosen": -158.73870849609375,
-      "logps/rejected": -232.62786865234375,
-      "loss": 0.1641635000705719,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -5.279523849487305,
-      "rewards/margins": 4.108285903930664,
-      "rewards/rejected": -9.387809753417969,
-      "step": 357
-    },
-    {
-      "epoch": 1.1548387096774193,
-      "grad_norm": 0.6596993207931519,
-      "learning_rate": 1.9102296188011644e-06,
-      "logits/chosen": -2.232776165008545,
-      "logits/rejected": -1.857932209968567,
-      "logps/chosen": -143.13937377929688,
-      "logps/rejected": -226.6849365234375,
-      "loss": 0.14514769613742828,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.097674369812012,
-      "rewards/margins": 5.24989128112793,
-      "rewards/rejected": -10.347566604614258,
-      "step": 358
-    },
-    {
-      "epoch": 1.1580645161290322,
-      "grad_norm": 0.6565665006637573,
-      "learning_rate": 1.8979270720937814e-06,
-      "logits/chosen": -1.938962459564209,
-      "logits/rejected": -1.64393150806427,
-      "logps/chosen": -158.95477294921875,
-      "logps/rejected": -219.6165008544922,
-      "loss": 0.16598346829414368,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -5.99850606918335,
-      "rewards/margins": 3.776855707168579,
-      "rewards/rejected": -9.775362014770508,
-      "step": 359
-    },
-    {
-      "epoch": 1.1612903225806452,
-      "grad_norm": 0.5938535928726196,
-      "learning_rate": 1.8856399837862552e-06,
-      "logits/chosen": -2.0286355018615723,
-      "logits/rejected": -1.68312668800354,
-      "logps/chosen": -168.68099975585938,
-      "logps/rejected": -229.54739379882812,
-      "loss": 0.11951416730880737,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.778509140014648,
-      "rewards/margins": 3.883187770843506,
-      "rewards/rejected": -9.661696434020996,
-      "step": 360
-    },
-    {
-      "epoch": 1.164516129032258,
-      "grad_norm": 0.3561262786388397,
-      "learning_rate": 1.8733686693531986e-06,
-      "logits/chosen": -2.127842903137207,
-      "logits/rejected": -1.7695016860961914,
-      "logps/chosen": -144.55734252929688,
-      "logps/rejected": -229.44363403320312,
-      "loss": 0.07146456092596054,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.509417533874512,
-      "rewards/margins": 4.857423305511475,
-      "rewards/rejected": -10.366840362548828,
-      "step": 361
-    },
-    {
-      "epoch": 1.167741935483871,
-      "grad_norm": 0.8494337797164917,
-      "learning_rate": 1.8611134438642277e-06,
-      "logits/chosen": -1.7025121450424194,
-      "logits/rejected": -1.4264708757400513,
-      "logps/chosen": -169.04269409179688,
-      "logps/rejected": -221.38189697265625,
-      "loss": 0.2456420212984085,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.7377424240112305,
-      "rewards/margins": 3.4128007888793945,
-      "rewards/rejected": -9.150543212890625,
-      "step": 362
-    },
-    {
-      "epoch": 1.1709677419354838,
-      "grad_norm": 0.6328747272491455,
-      "learning_rate": 1.8488746219758674e-06,
-      "logits/chosen": -2.2445268630981445,
-      "logits/rejected": -1.9536311626434326,
-      "logps/chosen": -175.17105102539062,
-      "logps/rejected": -228.45208740234375,
-      "loss": 0.17112946510314941,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.968300819396973,
-      "rewards/margins": 3.3772494792938232,
-      "rewards/rejected": -9.345550537109375,
-      "step": 363
-    },
-    {
-      "epoch": 1.1741935483870969,
-      "grad_norm": 0.6168525815010071,
-      "learning_rate": 1.836652517923477e-06,
-      "logits/chosen": -2.116086959838867,
-      "logits/rejected": -1.8520147800445557,
-      "logps/chosen": -138.18699645996094,
-      "logps/rejected": -202.0654754638672,
-      "loss": 0.16664910316467285,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.481403827667236,
-      "rewards/margins": 3.620983600616455,
-      "rewards/rejected": -8.102387428283691,
-      "step": 364
-    },
-    {
-      "epoch": 1.1774193548387097,
-      "grad_norm": 0.721930205821991,
-      "learning_rate": 1.824447445513179e-06,
-      "logits/chosen": -1.7406378984451294,
-      "logits/rejected": -1.482226014137268,
-      "logps/chosen": -179.15814208984375,
-      "logps/rejected": -250.98904418945312,
-      "loss": 0.1676761507987976,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -6.1544904708862305,
-      "rewards/margins": 4.405542373657227,
-      "rewards/rejected": -10.56003189086914,
-      "step": 365
-    },
-    {
-      "epoch": 1.1806451612903226,
-      "grad_norm": 0.6200963854789734,
-      "learning_rate": 1.812259718113805e-06,
-      "logits/chosen": -2.065372943878174,
-      "logits/rejected": -1.7523372173309326,
-      "logps/chosen": -162.01683044433594,
-      "logps/rejected": -223.36329650878906,
-      "loss": 0.15973971784114838,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.798542499542236,
-      "rewards/margins": 3.4644269943237305,
-      "rewards/rejected": -9.262969017028809,
-      "step": 366
-    },
-    {
-      "epoch": 1.1838709677419355,
-      "grad_norm": 0.38376542925834656,
-      "learning_rate": 1.8000896486488478e-06,
-      "logits/chosen": -2.161773443222046,
-      "logits/rejected": -1.9010320901870728,
-      "logps/chosen": -153.3382110595703,
-      "logps/rejected": -215.8674774169922,
-      "loss": 0.09716545045375824,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.17132568359375,
-      "rewards/margins": 4.096748352050781,
-      "rewards/rejected": -9.268074035644531,
-      "step": 367
-    },
-    {
-      "epoch": 1.1870967741935483,
-      "grad_norm": 0.7107370495796204,
-      "learning_rate": 1.7879375495884285e-06,
-      "logits/chosen": -2.1105496883392334,
-      "logits/rejected": -1.827610731124878,
-      "logps/chosen": -152.8114471435547,
-      "logps/rejected": -214.80062866210938,
-      "loss": 0.20002783834934235,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.211606979370117,
-      "rewards/margins": 3.4694228172302246,
-      "rewards/rejected": -8.681029319763184,
-      "step": 368
-    },
-    {
-      "epoch": 1.1903225806451614,
-      "grad_norm": 0.7678977847099304,
-      "learning_rate": 1.77580373294127e-06,
-      "logits/chosen": -2.063249111175537,
-      "logits/rejected": -1.708777666091919,
-      "logps/chosen": -158.5166778564453,
-      "logps/rejected": -214.08209228515625,
-      "loss": 0.2671286463737488,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -5.262528419494629,
-      "rewards/margins": 3.365039348602295,
-      "rewards/rejected": -8.627568244934082,
-      "step": 369
-    },
-    {
-      "epoch": 1.1935483870967742,
-      "grad_norm": 0.3605065643787384,
-      "learning_rate": 1.7636885102466907e-06,
-      "logits/chosen": -2.5887768268585205,
-      "logits/rejected": -2.2808783054351807,
-      "logps/chosen": -130.79820251464844,
-      "logps/rejected": -197.219482421875,
-      "loss": 0.1083921268582344,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.052940845489502,
-      "rewards/margins": 4.416059494018555,
-      "rewards/rejected": -8.468999862670898,
-      "step": 370
-    },
-    {
-      "epoch": 1.196774193548387,
-      "grad_norm": 0.7315830588340759,
-      "learning_rate": 1.7515921925666053e-06,
-      "logits/chosen": -2.358499765396118,
-      "logits/rejected": -2.0349740982055664,
-      "logps/chosen": -155.2823486328125,
-      "logps/rejected": -230.02227783203125,
-      "loss": 0.14842908084392548,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.6453447341918945,
-      "rewards/margins": 4.513706684112549,
-      "rewards/rejected": -10.159051895141602,
-      "step": 371
-    },
-    {
-      "epoch": 1.2,
-      "grad_norm": 0.5185221433639526,
-      "learning_rate": 1.7395150904775338e-06,
-      "logits/chosen": -2.276341438293457,
-      "logits/rejected": -1.9259616136550903,
-      "logps/chosen": -131.30264282226562,
-      "logps/rejected": -202.33523559570312,
-      "loss": 0.15730315446853638,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.405592441558838,
-      "rewards/margins": 4.261539459228516,
-      "rewards/rejected": -8.667131423950195,
-      "step": 372
-    },
-    {
-      "epoch": 1.2032258064516128,
-      "grad_norm": 0.7151278853416443,
-      "learning_rate": 1.7274575140626318e-06,
-      "logits/chosen": -2.0519351959228516,
-      "logits/rejected": -1.5820385217666626,
-      "logps/chosen": -155.93443298339844,
-      "logps/rejected": -236.36830139160156,
-      "loss": 0.16425088047981262,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.275310516357422,
-      "rewards/margins": 4.754761695861816,
-      "rewards/rejected": -10.030072212219238,
-      "step": 373
-    },
-    {
-      "epoch": 1.206451612903226,
-      "grad_norm": 0.660406231880188,
-      "learning_rate": 1.7154197729037275e-06,
-      "logits/chosen": -2.5248794555664062,
-      "logits/rejected": -2.073789596557617,
-      "logps/chosen": -144.71421813964844,
-      "logps/rejected": -224.99496459960938,
-      "loss": 0.18090461194515228,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.965693473815918,
-      "rewards/margins": 5.039268970489502,
-      "rewards/rejected": -10.004961967468262,
-      "step": 374
-    },
-    {
-      "epoch": 1.2096774193548387,
-      "grad_norm": 0.7491795420646667,
-      "learning_rate": 1.7034021760733712e-06,
-      "logits/chosen": -2.0052719116210938,
-      "logits/rejected": -1.725099802017212,
-      "logps/chosen": -165.0349578857422,
-      "logps/rejected": -229.08590698242188,
-      "loss": 0.2221168726682663,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -5.368182182312012,
-      "rewards/margins": 3.6441545486450195,
-      "rewards/rejected": -9.012336730957031,
-      "step": 375
-    },
-    {
-      "epoch": 1.2129032258064516,
-      "grad_norm": 0.8390048146247864,
-      "learning_rate": 1.6914050321269049e-06,
-      "logits/chosen": -2.0131216049194336,
-      "logits/rejected": -1.734924077987671,
-      "logps/chosen": -157.00872802734375,
-      "logps/rejected": -220.60801696777344,
-      "loss": 0.21280023455619812,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.416422367095947,
-      "rewards/margins": 3.55110502243042,
-      "rewards/rejected": -8.967527389526367,
-      "step": 376
-    },
-    {
-      "epoch": 1.2161290322580645,
-      "grad_norm": 0.6836002469062805,
-      "learning_rate": 1.6794286490945344e-06,
-      "logits/chosen": -2.0585014820098877,
-      "logits/rejected": -1.724048137664795,
-      "logps/chosen": -152.38882446289062,
-      "logps/rejected": -214.6417236328125,
-      "loss": 0.17855605483055115,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.036785125732422,
-      "rewards/margins": 3.633176565170288,
-      "rewards/rejected": -8.669961929321289,
-      "step": 377
-    },
-    {
-      "epoch": 1.2193548387096773,
-      "grad_norm": 0.46255868673324585,
-      "learning_rate": 1.667473334473424e-06,
-      "logits/chosen": -1.8597807884216309,
-      "logits/rejected": -1.426900863647461,
-      "logps/chosen": -182.52023315429688,
-      "logps/rejected": -284.5487365722656,
-      "loss": 0.07632362842559814,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -6.2685160636901855,
-      "rewards/margins": 6.5974955558776855,
-      "rewards/rejected": -12.866011619567871,
-      "step": 378
-    },
-    {
-      "epoch": 1.2225806451612904,
-      "grad_norm": 0.43195584416389465,
-      "learning_rate": 1.655539395219799e-06,
-      "logits/chosen": -2.1914172172546387,
-      "logits/rejected": -1.8688502311706543,
-      "logps/chosen": -156.40907287597656,
-      "logps/rejected": -236.13815307617188,
-      "loss": 0.0872289389371872,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.669783592224121,
-      "rewards/margins": 4.961936950683594,
-      "rewards/rejected": -10.631721496582031,
-      "step": 379
-    },
-    {
-      "epoch": 1.2258064516129032,
-      "grad_norm": 0.7333582639694214,
-      "learning_rate": 1.6436271377410667e-06,
-      "logits/chosen": -2.359461545944214,
-      "logits/rejected": -2.069000005722046,
-      "logps/chosen": -133.92742919921875,
-      "logps/rejected": -201.80978393554688,
-      "loss": 0.20327237248420715,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -4.1502580642700195,
-      "rewards/margins": 3.974135637283325,
-      "rewards/rejected": -8.124393463134766,
-      "step": 380
-    },
-    {
-      "epoch": 1.229032258064516,
-      "grad_norm": 0.42488953471183777,
-      "learning_rate": 1.6317368678879497e-06,
-      "logits/chosen": -2.406670093536377,
-      "logits/rejected": -2.07307505607605,
-      "logps/chosen": -133.93527221679688,
-      "logps/rejected": -200.25289916992188,
-      "loss": 0.12039235234260559,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.170150279998779,
-      "rewards/margins": 3.830730438232422,
-      "rewards/rejected": -8.000880241394043,
-      "step": 381
-    },
-    {
-      "epoch": 1.232258064516129,
-      "grad_norm": 0.4017215669155121,
-      "learning_rate": 1.6198688909466292e-06,
-      "logits/chosen": -2.832380533218384,
-      "logits/rejected": -2.3897719383239746,
-      "logps/chosen": -130.95968627929688,
-      "logps/rejected": -199.89097595214844,
-      "loss": 0.10956410318613052,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.363296985626221,
-      "rewards/margins": 4.319589614868164,
-      "rewards/rejected": -8.682886123657227,
-      "step": 382
-    },
-    {
-      "epoch": 1.2354838709677418,
-      "grad_norm": 0.6206420063972473,
-      "learning_rate": 1.6080235116309114e-06,
-      "logits/chosen": -2.4406814575195312,
-      "logits/rejected": -2.160674810409546,
-      "logps/chosen": -142.58071899414062,
-      "logps/rejected": -200.98794555664062,
-      "loss": 0.1694997251033783,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.247890472412109,
-      "rewards/margins": 3.134093761444092,
-      "rewards/rejected": -8.38198471069336,
-      "step": 383
-    },
-    {
-      "epoch": 1.238709677419355,
-      "grad_norm": 0.2462724894285202,
-      "learning_rate": 1.5962010340744006e-06,
-      "logits/chosen": -2.235247850418091,
-      "logits/rejected": -1.847717046737671,
-      "logps/chosen": -146.453125,
-      "logps/rejected": -225.43215942382812,
-      "loss": 0.03917820006608963,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -4.979830265045166,
-      "rewards/margins": 4.634283065795898,
-      "rewards/rejected": -9.614113807678223,
-      "step": 384
-    },
-    {
-      "epoch": 1.2419354838709677,
-      "grad_norm": 0.5250591039657593,
-      "learning_rate": 1.5844017618226934e-06,
-      "logits/chosen": -2.19504451751709,
-      "logits/rejected": -1.866036057472229,
-      "logps/chosen": -151.13436889648438,
-      "logps/rejected": -220.41619873046875,
-      "loss": 0.13191558420658112,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.161625862121582,
-      "rewards/margins": 4.320361137390137,
-      "rewards/rejected": -9.481987953186035,
-      "step": 385
-    },
-    {
-      "epoch": 1.2451612903225806,
-      "grad_norm": 0.448537677526474,
-      "learning_rate": 1.572625997825581e-06,
-      "logits/chosen": -2.195721387863159,
-      "logits/rejected": -1.8460835218429565,
-      "logps/chosen": -153.8927001953125,
-      "logps/rejected": -227.37582397460938,
-      "loss": 0.11286298930644989,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.872514247894287,
-      "rewards/margins": 4.24212646484375,
-      "rewards/rejected": -9.114640235900879,
-      "step": 386
-    },
-    {
-      "epoch": 1.2483870967741935,
-      "grad_norm": 0.6990474462509155,
-      "learning_rate": 1.5608740444292746e-06,
-      "logits/chosen": -2.3243963718414307,
-      "logits/rejected": -2.0378341674804688,
-      "logps/chosen": -149.29092407226562,
-      "logps/rejected": -207.21595764160156,
-      "loss": 0.18665999174118042,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.657018184661865,
-      "rewards/margins": 3.210756301879883,
-      "rewards/rejected": -7.867774486541748,
-      "step": 387
-    },
-    {
-      "epoch": 1.2516129032258063,
-      "grad_norm": 0.8297081589698792,
-      "learning_rate": 1.5491462033686411e-06,
-      "logits/chosen": -2.18031907081604,
-      "logits/rejected": -1.826414942741394,
-      "logps/chosen": -158.47360229492188,
-      "logps/rejected": -234.47384643554688,
-      "loss": 0.16636508703231812,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.654425621032715,
-      "rewards/margins": 4.482308387756348,
-      "rewards/rejected": -10.136734008789062,
-      "step": 388
-    },
-    {
-      "epoch": 1.2548387096774194,
-      "grad_norm": 1.3903393745422363,
-      "learning_rate": 1.5374427757594552e-06,
-      "logits/chosen": -2.297201156616211,
-      "logits/rejected": -2.06970477104187,
-      "logps/chosen": -178.5050048828125,
-      "logps/rejected": -219.87823486328125,
-      "loss": 0.39618611335754395,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -6.283227920532227,
-      "rewards/margins": 2.801511764526367,
-      "rewards/rejected": -9.084739685058594,
-      "step": 389
-    },
-    {
-      "epoch": 1.2580645161290323,
-      "grad_norm": 0.4080447256565094,
-      "learning_rate": 1.525764062090671e-06,
-      "logits/chosen": -1.8646701574325562,
-      "logits/rejected": -1.555259108543396,
-      "logps/chosen": -169.2515869140625,
-      "logps/rejected": -238.75172424316406,
-      "loss": 0.08469586074352264,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.7809553146362305,
-      "rewards/margins": 3.9082202911376953,
-      "rewards/rejected": -9.689175605773926,
-      "step": 390
-    },
-    {
-      "epoch": 1.261290322580645,
-      "grad_norm": 0.8623878359794617,
-      "learning_rate": 1.5141103622167042e-06,
-      "logits/chosen": -2.4216866493225098,
-      "logits/rejected": -2.0609347820281982,
-      "logps/chosen": -148.01690673828125,
-      "logps/rejected": -201.63174438476562,
-      "loss": 0.242265522480011,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.647133827209473,
-      "rewards/margins": 3.4030661582946777,
-      "rewards/rejected": -8.050199508666992,
-      "step": 391
-    },
-    {
-      "epoch": 1.2645161290322582,
-      "grad_norm": 0.4071905314922333,
-      "learning_rate": 1.5024819753497316e-06,
-      "logits/chosen": -2.114326238632202,
-      "logits/rejected": -1.8489298820495605,
-      "logps/chosen": -154.75469970703125,
-      "logps/rejected": -223.68875122070312,
-      "loss": 0.10187827050685883,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.854604721069336,
-      "rewards/margins": 4.386443138122559,
-      "rewards/rejected": -9.241047859191895,
-      "step": 392
-    },
-    {
-      "epoch": 1.267741935483871,
-      "grad_norm": 0.5191037058830261,
-      "learning_rate": 1.490879200052014e-06,
-      "logits/chosen": -2.666534662246704,
-      "logits/rejected": -2.3881566524505615,
-      "logps/chosen": -153.2815399169922,
-      "logps/rejected": -220.57666015625,
-      "loss": 0.13561299443244934,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -6.065132141113281,
-      "rewards/margins": 3.8070566654205322,
-      "rewards/rejected": -9.872188568115234,
-      "step": 393
-    },
-    {
-      "epoch": 1.270967741935484,
-      "grad_norm": 0.5940090417861938,
-      "learning_rate": 1.4793023342282258e-06,
-      "logits/chosen": -2.3885128498077393,
-      "logits/rejected": -2.043498992919922,
-      "logps/chosen": -142.4330596923828,
-      "logps/rejected": -230.61424255371094,
-      "loss": 0.1435917317867279,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.768260955810547,
-      "rewards/margins": 4.874237537384033,
-      "rewards/rejected": -9.642498970031738,
-      "step": 394
-    },
-    {
-      "epoch": 1.2741935483870968,
-      "grad_norm": 0.5542668700218201,
-      "learning_rate": 1.46775167511781e-06,
-      "logits/chosen": -2.55564284324646,
-      "logits/rejected": -1.9429116249084473,
-      "logps/chosen": -122.74253845214844,
-      "logps/rejected": -221.68040466308594,
-      "loss": 0.14152149856090546,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.333782196044922,
-      "rewards/margins": 6.17290735244751,
-      "rewards/rejected": -10.506689071655273,
-      "step": 395
-    },
-    {
-      "epoch": 1.2774193548387096,
-      "grad_norm": 0.598746657371521,
-      "learning_rate": 1.456227519287343e-06,
-      "logits/chosen": -2.619776725769043,
-      "logits/rejected": -2.364832878112793,
-      "logps/chosen": -162.55870056152344,
-      "logps/rejected": -218.89852905273438,
-      "loss": 0.14930739998817444,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.192800521850586,
-      "rewards/margins": 3.228623867034912,
-      "rewards/rejected": -8.421424865722656,
-      "step": 396
-    },
-    {
-      "epoch": 1.2806451612903227,
-      "grad_norm": 0.8427586555480957,
-      "learning_rate": 1.4447301626229204e-06,
-      "logits/chosen": -2.049372434616089,
-      "logits/rejected": -1.662196159362793,
-      "logps/chosen": -153.92330932617188,
-      "logps/rejected": -217.40673828125,
-      "loss": 0.1911974847316742,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.926740646362305,
-      "rewards/margins": 3.7954201698303223,
-      "rewards/rejected": -8.722161293029785,
-      "step": 397
-    },
-    {
-      "epoch": 1.2838709677419355,
-      "grad_norm": 0.5858323574066162,
-      "learning_rate": 1.4332599003225638e-06,
-      "logits/chosen": -1.8799211978912354,
-      "logits/rejected": -1.451097846031189,
-      "logps/chosen": -149.65142822265625,
-      "logps/rejected": -241.16909790039062,
-      "loss": 0.11262927204370499,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.039501667022705,
-      "rewards/margins": 5.284974575042725,
-      "rewards/rejected": -10.32447624206543,
-      "step": 398
-    },
-    {
-      "epoch": 1.2870967741935484,
-      "grad_norm": 0.4971863329410553,
-      "learning_rate": 1.42181702688864e-06,
-      "logits/chosen": -2.286614179611206,
-      "logits/rejected": -1.8966314792633057,
-      "logps/chosen": -153.2661590576172,
-      "logps/rejected": -220.55250549316406,
-      "loss": 0.1035381481051445,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.003832817077637,
-      "rewards/margins": 4.208715438842773,
-      "rewards/rejected": -9.21254825592041,
-      "step": 399
-    },
-    {
-      "epoch": 1.2903225806451613,
-      "grad_norm": 0.3167995810508728,
-      "learning_rate": 1.4104018361202947e-06,
-      "logits/chosen": -2.4894747734069824,
-      "logits/rejected": -2.0620884895324707,
-      "logps/chosen": -146.4293975830078,
-      "logps/rejected": -245.97506713867188,
-      "loss": 0.075407974421978,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.003347396850586,
-      "rewards/margins": 5.8961663246154785,
-      "rewards/rejected": -10.899513244628906,
-      "step": 400
-    },
-    {
-      "epoch": 1.293548387096774,
-      "grad_norm": 1.0359872579574585,
-      "learning_rate": 1.3990146211059141e-06,
-      "logits/chosen": -2.4616971015930176,
-      "logits/rejected": -2.154454231262207,
-      "logps/chosen": -159.11233520507812,
-      "logps/rejected": -222.264892578125,
-      "loss": 0.26626330614089966,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -5.6828179359436035,
-      "rewards/margins": 4.052268981933594,
-      "rewards/rejected": -9.735086441040039,
-      "step": 401
-    },
-    {
-      "epoch": 1.2967741935483872,
-      "grad_norm": 1.037102460861206,
-      "learning_rate": 1.3876556742156019e-06,
-      "logits/chosen": -2.124492883682251,
-      "logits/rejected": -1.8085477352142334,
-      "logps/chosen": -170.0220489501953,
-      "logps/rejected": -233.81515502929688,
-      "loss": 0.25808218121528625,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.954598426818848,
-      "rewards/margins": 3.8529767990112305,
-      "rewards/rejected": -9.807575225830078,
-      "step": 402
-    },
-    {
-      "epoch": 1.3,
-      "grad_norm": 1.154233455657959,
-      "learning_rate": 1.376325287093665e-06,
-      "logits/chosen": -2.3976852893829346,
-      "logits/rejected": -2.059356212615967,
-      "logps/chosen": -174.79681396484375,
-      "logps/rejected": -234.56651306152344,
-      "loss": 0.3070518374443054,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -5.778817176818848,
-      "rewards/margins": 3.863279342651367,
-      "rewards/rejected": -9.642096519470215,
-      "step": 403
-    },
-    {
-      "epoch": 1.303225806451613,
-      "grad_norm": 0.7991430163383484,
-      "learning_rate": 1.3650237506511333e-06,
-      "logits/chosen": -2.269876003265381,
-      "logits/rejected": -1.9741662740707397,
-      "logps/chosen": -154.88619995117188,
-      "logps/rejected": -223.62730407714844,
-      "loss": 0.21480777859687805,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -4.390778541564941,
-      "rewards/margins": 4.292700290679932,
-      "rewards/rejected": -8.683478355407715,
-      "step": 404
-    },
-    {
-      "epoch": 1.3064516129032258,
-      "grad_norm": 0.5895324349403381,
-      "learning_rate": 1.3537513550582853e-06,
-      "logits/chosen": -2.2187764644622803,
-      "logits/rejected": -1.8166173696517944,
-      "logps/chosen": -148.30221557617188,
-      "logps/rejected": -213.80755615234375,
-      "loss": 0.13769668340682983,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.786943435668945,
-      "rewards/margins": 4.149323463439941,
-      "rewards/rejected": -8.93626594543457,
-      "step": 405
-    },
-    {
-      "epoch": 1.3096774193548386,
-      "grad_norm": 1.0179868936538696,
-      "learning_rate": 1.3425083897371983e-06,
-      "logits/chosen": -2.0878705978393555,
-      "logits/rejected": -1.8105435371398926,
-      "logps/chosen": -149.2059326171875,
-      "logps/rejected": -210.22061157226562,
-      "loss": 0.25650089979171753,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.216985702514648,
-      "rewards/margins": 3.597675323486328,
-      "rewards/rejected": -8.814661026000977,
-      "step": 406
-    },
-    {
-      "epoch": 1.3129032258064517,
-      "grad_norm": 0.7849494218826294,
-      "learning_rate": 1.3312951433543225e-06,
-      "logits/chosen": -2.366304874420166,
-      "logits/rejected": -1.9977753162384033,
-      "logps/chosen": -147.30950927734375,
-      "logps/rejected": -221.3260498046875,
-      "loss": 0.1955363154411316,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -4.945062160491943,
-      "rewards/margins": 4.327850818634033,
-      "rewards/rejected": -9.27291202545166,
-      "step": 407
-    },
-    {
-      "epoch": 1.3161290322580645,
-      "grad_norm": 0.4468243420124054,
-      "learning_rate": 1.3201119038130616e-06,
-      "logits/chosen": -2.472445487976074,
-      "logits/rejected": -1.9949541091918945,
-      "logps/chosen": -135.80923461914062,
-      "logps/rejected": -214.80462646484375,
-      "loss": 0.08190619945526123,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.554195404052734,
-      "rewards/margins": 4.7136735916137695,
-      "rewards/rejected": -9.267868041992188,
-      "step": 408
-    },
-    {
-      "epoch": 1.3193548387096774,
-      "grad_norm": 0.4178885817527771,
-      "learning_rate": 1.308958958246388e-06,
-      "logits/chosen": -2.448024272918701,
-      "logits/rejected": -2.0670201778411865,
-      "logps/chosen": -140.41001892089844,
-      "logps/rejected": -214.8358917236328,
-      "loss": 0.10611550509929657,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.203770160675049,
-      "rewards/margins": 4.342594146728516,
-      "rewards/rejected": -9.546364784240723,
-      "step": 409
-    },
-    {
-      "epoch": 1.3225806451612903,
-      "grad_norm": 0.6719059944152832,
-      "learning_rate": 1.2978365930094645e-06,
-      "logits/chosen": -2.3373985290527344,
-      "logits/rejected": -2.0481557846069336,
-      "logps/chosen": -158.54962158203125,
-      "logps/rejected": -216.07199096679688,
-      "loss": 0.21998706459999084,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.216798782348633,
-      "rewards/margins": 3.790421962738037,
-      "rewards/rejected": -9.007221221923828,
-      "step": 410
-    },
-    {
-      "epoch": 1.3258064516129031,
-      "grad_norm": 0.5259553790092468,
-      "learning_rate": 1.286745093672298e-06,
-      "logits/chosen": -1.8555116653442383,
-      "logits/rejected": -1.5601491928100586,
-      "logps/chosen": -173.27001953125,
-      "logps/rejected": -232.70184326171875,
-      "loss": 0.11464345455169678,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.655244827270508,
-      "rewards/margins": 3.616917371749878,
-      "rewards/rejected": -9.272162437438965,
-      "step": 411
-    },
-    {
-      "epoch": 1.3290322580645162,
-      "grad_norm": 1.0269285440444946,
-      "learning_rate": 1.2756847450124005e-06,
-      "logits/chosen": -2.258531093597412,
-      "logits/rejected": -1.9175965785980225,
-      "logps/chosen": -150.10646057128906,
-      "logps/rejected": -217.81976318359375,
-      "loss": 0.2900359034538269,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -4.862858772277832,
-      "rewards/margins": 3.9900403022766113,
-      "rewards/rejected": -8.852899551391602,
-      "step": 412
-    },
-    {
-      "epoch": 1.332258064516129,
-      "grad_norm": 0.6336823105812073,
-      "learning_rate": 1.264655831007486e-06,
-      "logits/chosen": -2.4869401454925537,
-      "logits/rejected": -2.178582191467285,
-      "logps/chosen": -146.45761108398438,
-      "logps/rejected": -209.02230834960938,
-      "loss": 0.13850894570350647,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.8014020919799805,
-      "rewards/margins": 3.8616671562194824,
-      "rewards/rejected": -8.663068771362305,
-      "step": 413
-    },
-    {
-      "epoch": 1.335483870967742,
-      "grad_norm": 0.7770991921424866,
-      "learning_rate": 1.2536586348281694e-06,
-      "logits/chosen": -2.089435338973999,
-      "logits/rejected": -1.866056203842163,
-      "logps/chosen": -178.45458984375,
-      "logps/rejected": -243.360107421875,
-      "loss": 0.2114502340555191,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -6.556073188781738,
-      "rewards/margins": 3.712599515914917,
-      "rewards/rejected": -10.268672943115234,
-      "step": 414
-    },
-    {
-      "epoch": 1.3387096774193548,
-      "grad_norm": 0.5494731068611145,
-      "learning_rate": 1.2426934388307059e-06,
-      "logits/chosen": -2.485060214996338,
-      "logits/rejected": -2.085818290710449,
-      "logps/chosen": -141.69973754882812,
-      "logps/rejected": -208.56036376953125,
-      "loss": 0.16660693287849426,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.942885875701904,
-      "rewards/margins": 3.965348958969116,
-      "rewards/rejected": -8.908235549926758,
-      "step": 415
-    },
-    {
-      "epoch": 1.3419354838709676,
-      "grad_norm": 0.3686054050922394,
-      "learning_rate": 1.2317605245497324e-06,
-      "logits/chosen": -2.1087584495544434,
-      "logits/rejected": -1.6341150999069214,
-      "logps/chosen": -152.7848663330078,
-      "logps/rejected": -238.4124755859375,
-      "loss": 0.07035782933235168,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -4.851728439331055,
-      "rewards/margins": 5.564634799957275,
-      "rewards/rejected": -10.416362762451172,
-      "step": 416
-    },
-    {
-      "epoch": 1.3451612903225807,
-      "grad_norm": 0.5881558656692505,
-      "learning_rate": 1.2208601726910446e-06,
-      "logits/chosen": -2.7871689796447754,
-      "logits/rejected": -2.365675449371338,
-      "logps/chosen": -123.9314956665039,
-      "logps/rejected": -190.04649353027344,
-      "loss": 0.2168859839439392,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -3.762516498565674,
-      "rewards/margins": 4.015682697296143,
-      "rewards/rejected": -7.778199672698975,
-      "step": 417
-    },
-    {
-      "epoch": 1.3483870967741935,
-      "grad_norm": 0.9616515636444092,
-      "learning_rate": 1.209992663124391e-06,
-      "logits/chosen": -2.182853937149048,
-      "logits/rejected": -1.8745319843292236,
-      "logps/chosen": -161.31716918945312,
-      "logps/rejected": -216.4304962158203,
-      "loss": 0.21536722779273987,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.1654744148254395,
-      "rewards/margins": 3.8585994243621826,
-      "rewards/rejected": -9.02407455444336,
-      "step": 418
-    },
-    {
-      "epoch": 1.3516129032258064,
-      "grad_norm": 0.4572608470916748,
-      "learning_rate": 1.19915827487628e-06,
-      "logits/chosen": -2.0564944744110107,
-      "logits/rejected": -1.74418306350708,
-      "logps/chosen": -140.11302185058594,
-      "logps/rejected": -209.9811553955078,
-      "loss": 0.1145668774843216,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -4.56220817565918,
-      "rewards/margins": 4.219762325286865,
-      "rewards/rejected": -8.781970977783203,
-      "step": 419
-    },
-    {
-      "epoch": 1.3548387096774195,
-      "grad_norm": 0.8865814208984375,
-      "learning_rate": 1.1883572861228255e-06,
-      "logits/chosen": -2.37459659576416,
-      "logits/rejected": -2.109685182571411,
-      "logps/chosen": -145.40548706054688,
-      "logps/rejected": -212.8759765625,
-      "loss": 0.24796389043331146,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -4.966710090637207,
-      "rewards/margins": 3.646275281906128,
-      "rewards/rejected": -8.612985610961914,
-      "step": 420
-    },
-    {
-      "epoch": 1.3580645161290323,
-      "grad_norm": 0.6821106672286987,
-      "learning_rate": 1.1775899741825947e-06,
-      "logits/chosen": -2.328369617462158,
-      "logits/rejected": -2.013854503631592,
-      "logps/chosen": -141.02191162109375,
-      "logps/rejected": -205.2490234375,
-      "loss": 0.20715095102787018,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -4.649036407470703,
-      "rewards/margins": 4.007871627807617,
-      "rewards/rejected": -8.65690803527832,
-      "step": 421
-    },
-    {
-      "epoch": 1.3612903225806452,
-      "grad_norm": 0.5044422745704651,
-      "learning_rate": 1.1668566155094948e-06,
-      "logits/chosen": -2.54693865776062,
-      "logits/rejected": -1.9353411197662354,
-      "logps/chosen": -131.88148498535156,
-      "logps/rejected": -202.45828247070312,
-      "loss": 0.0967191830277443,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -3.5916037559509277,
-      "rewards/margins": 4.169280529022217,
-      "rewards/rejected": -7.7608842849731445,
-      "step": 422
-    },
-    {
-      "epoch": 1.364516129032258,
-      "grad_norm": 0.8407202959060669,
-      "learning_rate": 1.1561574856856737e-06,
-      "logits/chosen": -2.190084934234619,
-      "logits/rejected": -1.87167489528656,
-      "logps/chosen": -153.39193725585938,
-      "logps/rejected": -216.16046142578125,
-      "loss": 0.21430709958076477,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -4.597943305969238,
-      "rewards/margins": 3.650974988937378,
-      "rewards/rejected": -8.248918533325195,
-      "step": 423
-    },
-    {
-      "epoch": 1.367741935483871,
-      "grad_norm": 0.866381049156189,
-      "learning_rate": 1.1454928594144432e-06,
-      "logits/chosen": -2.4623706340789795,
-      "logits/rejected": -2.085210084915161,
-      "logps/chosen": -142.26776123046875,
-      "logps/rejected": -206.5341796875,
-      "loss": 0.2235676646232605,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.408275604248047,
-      "rewards/margins": 3.4640631675720215,
-      "rewards/rejected": -7.87233829498291,
-      "step": 424
-    },
-    {
-      "epoch": 1.370967741935484,
-      "grad_norm": 0.630285382270813,
-      "learning_rate": 1.1348630105132253e-06,
-      "logits/chosen": -2.5449862480163574,
-      "logits/rejected": -2.1377205848693848,
-      "logps/chosen": -150.03143310546875,
-      "logps/rejected": -216.3634490966797,
-      "loss": 0.1601426601409912,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.416558265686035,
-      "rewards/margins": 4.073892593383789,
-      "rewards/rejected": -8.490450859069824,
-      "step": 425
-    },
-    {
-      "epoch": 1.3741935483870968,
-      "grad_norm": 0.4551900029182434,
-      "learning_rate": 1.1242682119065217e-06,
-      "logits/chosen": -2.3819632530212402,
-      "logits/rejected": -2.057983875274658,
-      "logps/chosen": -144.26976013183594,
-      "logps/rejected": -227.2911376953125,
-      "loss": 0.10965423285961151,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.817654132843018,
-      "rewards/margins": 4.581267356872559,
-      "rewards/rejected": -9.398920059204102,
-      "step": 426
-    },
-    {
-      "epoch": 1.3774193548387097,
-      "grad_norm": 0.7748322486877441,
-      "learning_rate": 1.1137087356189105e-06,
-      "logits/chosen": -2.4593541622161865,
-      "logits/rejected": -2.1655220985412598,
-      "logps/chosen": -147.06605529785156,
-      "logps/rejected": -211.48818969726562,
-      "loss": 0.20653697848320007,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.7266435623168945,
-      "rewards/margins": 3.2662453651428223,
-      "rewards/rejected": -7.992889404296875,
-      "step": 427
-    },
-    {
-      "epoch": 1.3806451612903226,
-      "grad_norm": 0.5784196257591248,
-      "learning_rate": 1.1031848527680577e-06,
-      "logits/chosen": -2.141270875930786,
-      "logits/rejected": -1.8732938766479492,
-      "logps/chosen": -167.12008666992188,
-      "logps/rejected": -239.15773010253906,
-      "loss": 0.1471468210220337,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.631338119506836,
-      "rewards/margins": 4.0043134689331055,
-      "rewards/rejected": -9.635651588439941,
-      "step": 428
-    },
-    {
-      "epoch": 1.3838709677419354,
-      "grad_norm": 0.8778082728385925,
-      "learning_rate": 1.0926968335577566e-06,
-      "logits/chosen": -2.401782989501953,
-      "logits/rejected": -2.091179609298706,
-      "logps/chosen": -135.68911743164062,
-      "logps/rejected": -196.52935791015625,
-      "loss": 0.24231696128845215,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -3.9860541820526123,
-      "rewards/margins": 3.5753464698791504,
-      "rewards/rejected": -7.561400413513184,
-      "step": 429
-    },
-    {
-      "epoch": 1.3870967741935485,
-      "grad_norm": 1.0340511798858643,
-      "learning_rate": 1.0822449472709907e-06,
-      "logits/chosen": -2.586111545562744,
-      "logits/rejected": -2.177614688873291,
-      "logps/chosen": -146.74525451660156,
-      "logps/rejected": -209.59957885742188,
-      "loss": 0.22286778688430786,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -4.652778148651123,
-      "rewards/margins": 3.4029431343078613,
-      "rewards/rejected": -8.055720329284668,
-      "step": 430
-    },
-    {
-      "epoch": 1.3903225806451613,
-      "grad_norm": 0.5904428362846375,
-      "learning_rate": 1.0718294622630188e-06,
-      "logits/chosen": -2.47729229927063,
-      "logits/rejected": -2.0854480266571045,
-      "logps/chosen": -152.6089630126953,
-      "logps/rejected": -223.96041870117188,
-      "loss": 0.11524567008018494,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.076026916503906,
-      "rewards/margins": 3.904932975769043,
-      "rewards/rejected": -8.98095989227295,
-      "step": 431
-    },
-    {
-      "epoch": 1.3935483870967742,
-      "grad_norm": 0.539168119430542,
-      "learning_rate": 1.0614506459544921e-06,
-      "logits/chosen": -2.2595126628875732,
-      "logits/rejected": -1.925595998764038,
-      "logps/chosen": -162.54318237304688,
-      "logps/rejected": -223.33636474609375,
-      "loss": 0.10885189473628998,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.511690139770508,
-      "rewards/margins": 3.9950408935546875,
-      "rewards/rejected": -9.506731033325195,
-      "step": 432
-    },
-    {
-      "epoch": 1.396774193548387,
-      "grad_norm": 0.5645772814750671,
-      "learning_rate": 1.0511087648245758e-06,
-      "logits/chosen": -2.578536033630371,
-      "logits/rejected": -2.2838306427001953,
-      "logps/chosen": -132.77256774902344,
-      "logps/rejected": -199.54373168945312,
-      "loss": 0.12419108301401138,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.038809776306152,
-      "rewards/margins": 3.914381980895996,
-      "rewards/rejected": -7.953191757202148,
-      "step": 433
-    },
-    {
-      "epoch": 1.4,
-      "grad_norm": 0.5010483860969543,
-      "learning_rate": 1.0408040844041157e-06,
-      "logits/chosen": -1.9756072759628296,
-      "logits/rejected": -1.714409351348877,
-      "logps/chosen": -170.2490692138672,
-      "logps/rejected": -227.61837768554688,
-      "loss": 0.13830138742923737,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.366008758544922,
-      "rewards/margins": 3.824510097503662,
-      "rewards/rejected": -9.190519332885742,
-      "step": 434
-    },
-    {
-      "epoch": 1.403225806451613,
-      "grad_norm": 0.5239137411117554,
-      "learning_rate": 1.0305368692688175e-06,
-      "logits/chosen": -2.5020527839660645,
-      "logits/rejected": -2.03378963470459,
-      "logps/chosen": -144.1491241455078,
-      "logps/rejected": -233.28515625,
-      "loss": 0.0852818563580513,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.3349151611328125,
-      "rewards/margins": 5.156275272369385,
-      "rewards/rejected": -10.491190910339355,
-      "step": 435
-    },
-    {
-      "epoch": 1.4064516129032258,
-      "grad_norm": 0.7590228915214539,
-      "learning_rate": 1.0203073830324566e-06,
-      "logits/chosen": -2.05299973487854,
-      "logits/rejected": -1.7430031299591064,
-      "logps/chosen": -156.0850830078125,
-      "logps/rejected": -216.1945037841797,
-      "loss": 0.246930330991745,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.060250759124756,
-      "rewards/margins": 3.4490652084350586,
-      "rewards/rejected": -8.509315490722656,
-      "step": 436
-    },
-    {
-      "epoch": 1.4096774193548387,
-      "grad_norm": 0.4939168691635132,
-      "learning_rate": 1.0101158883401078e-06,
-      "logits/chosen": -2.5271737575531006,
-      "logits/rejected": -2.085583209991455,
-      "logps/chosen": -136.91073608398438,
-      "logps/rejected": -218.0567626953125,
-      "loss": 0.10788646340370178,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.469447612762451,
-      "rewards/margins": 4.4227495193481445,
-      "rewards/rejected": -8.892196655273438,
-      "step": 437
-    },
-    {
-      "epoch": 1.4129032258064516,
-      "grad_norm": 0.5392411351203918,
-      "learning_rate": 9.999626468613997e-07,
-      "logits/chosen": -2.2997260093688965,
-      "logits/rejected": -1.9779574871063232,
-      "logps/chosen": -175.44947814941406,
-      "logps/rejected": -253.67218017578125,
-      "loss": 0.11333902180194855,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -6.0964202880859375,
-      "rewards/margins": 4.6486334800720215,
-      "rewards/rejected": -10.745054244995117,
-      "step": 438
-    },
-    {
-      "epoch": 1.4161290322580644,
-      "grad_norm": 1.1507418155670166,
-      "learning_rate": 9.898479192837985e-07,
-      "logits/chosen": -2.0347580909729004,
-      "logits/rejected": -1.662597417831421,
-      "logps/chosen": -159.89138793945312,
-      "logps/rejected": -218.62335205078125,
-      "loss": 0.298279732465744,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -4.849677085876465,
-      "rewards/margins": 3.3393747806549072,
-      "rewards/rejected": -8.189051628112793,
-      "step": 439
-    },
-    {
-      "epoch": 1.4193548387096775,
-      "grad_norm": 0.6101773381233215,
-      "learning_rate": 9.797719653059176e-07,
-      "logits/chosen": -2.025186538696289,
-      "logits/rejected": -1.7302336692810059,
-      "logps/chosen": -167.60308837890625,
-      "logps/rejected": -238.1725616455078,
-      "loss": 0.12761417031288147,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.349034309387207,
-      "rewards/margins": 4.279293537139893,
-      "rewards/rejected": -9.628328323364258,
-      "step": 440
-    },
-    {
-      "epoch": 1.4225806451612903,
-      "grad_norm": 0.3005771338939667,
-      "learning_rate": 9.697350436308428e-07,
-      "logits/chosen": -2.4265613555908203,
-      "logits/rejected": -2.0480599403381348,
-      "logps/chosen": -156.69139099121094,
-      "logps/rejected": -237.74905395507812,
-      "loss": 0.05653847008943558,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -4.761828422546387,
-      "rewards/margins": 4.787032127380371,
-      "rewards/rejected": -9.548859596252441,
-      "step": 441
-    },
-    {
-      "epoch": 1.4258064516129032,
-      "grad_norm": 0.8191932439804077,
-      "learning_rate": 9.597374119594981e-07,
-      "logits/chosen": -2.304115056991577,
-      "logits/rejected": -2.032526969909668,
-      "logps/chosen": -156.66375732421875,
-      "logps/rejected": -208.8516082763672,
-      "loss": 0.20551738142967224,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.349687576293945,
-      "rewards/margins": 3.4436111450195312,
-      "rewards/rejected": -8.793298721313477,
-      "step": 442
-    },
-    {
-      "epoch": 1.429032258064516,
-      "grad_norm": 0.8888070583343506,
-      "learning_rate": 9.497793269840211e-07,
-      "logits/chosen": -2.4185256958007812,
-      "logits/rejected": -2.081573009490967,
-      "logps/chosen": -145.60757446289062,
-      "logps/rejected": -212.35897827148438,
-      "loss": 0.2327883243560791,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -5.171741485595703,
-      "rewards/margins": 3.747623920440674,
-      "rewards/rejected": -8.919364929199219,
-      "step": 443
-    },
-    {
-      "epoch": 1.432258064516129,
-      "grad_norm": 0.4799133837223053,
-      "learning_rate": 9.398610443811798e-07,
-      "logits/chosen": -1.9818705320358276,
-      "logits/rejected": -1.634839653968811,
-      "logps/chosen": -174.2476348876953,
-      "logps/rejected": -235.02545166015625,
-      "loss": 0.09316743910312653,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.465350151062012,
-      "rewards/margins": 3.838430166244507,
-      "rewards/rejected": -9.303780555725098,
-      "step": 444
-    },
-    {
-      "epoch": 1.435483870967742,
-      "grad_norm": 0.9450538754463196,
-      "learning_rate": 9.299828188058013e-07,
-      "logits/chosen": -2.247403144836426,
-      "logits/rejected": -1.9464889764785767,
-      "logps/chosen": -170.24978637695312,
-      "logps/rejected": -219.93753051757812,
-      "loss": 0.2306218445301056,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.57850456237793,
-      "rewards/margins": 2.9838309288024902,
-      "rewards/rejected": -8.562335968017578,
-      "step": 445
-    },
-    {
-      "epoch": 1.4387096774193548,
-      "grad_norm": 0.5895780920982361,
-      "learning_rate": 9.201449038842403e-07,
-      "logits/chosen": -2.0624327659606934,
-      "logits/rejected": -1.702972650527954,
-      "logps/chosen": -159.513671875,
-      "logps/rejected": -227.80357360839844,
-      "loss": 0.14580538868904114,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.874773979187012,
-      "rewards/margins": 3.9577934741973877,
-      "rewards/rejected": -8.83256721496582,
-      "step": 446
-    },
-    {
-      "epoch": 1.4419354838709677,
-      "grad_norm": 0.8507987856864929,
-      "learning_rate": 9.103475522078606e-07,
-      "logits/chosen": -2.2239818572998047,
-      "logits/rejected": -1.928896188735962,
-      "logps/chosen": -169.17330932617188,
-      "logps/rejected": -226.74343872070312,
-      "loss": 0.20541514456272125,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.307161808013916,
-      "rewards/margins": 3.5340442657470703,
-      "rewards/rejected": -8.841205596923828,
-      "step": 447
-    },
-    {
-      "epoch": 1.4451612903225808,
-      "grad_norm": 0.6203137636184692,
-      "learning_rate": 9.005910153265531e-07,
-      "logits/chosen": -2.4608101844787598,
-      "logits/rejected": -2.152571439743042,
-      "logps/chosen": -141.15274047851562,
-      "logps/rejected": -203.83888244628906,
-      "loss": 0.1468571275472641,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.174438953399658,
-      "rewards/margins": 3.7283501625061035,
-      "rewards/rejected": -7.9027886390686035,
-      "step": 448
-    },
-    {
-      "epoch": 1.4483870967741934,
-      "grad_norm": 0.6260938048362732,
-      "learning_rate": 8.908755437422792e-07,
-      "logits/chosen": -2.321335792541504,
-      "logits/rejected": -2.009199380874634,
-      "logps/chosen": -140.9883270263672,
-      "logps/rejected": -197.89700317382812,
-      "loss": 0.15317237377166748,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -3.808759927749634,
-      "rewards/margins": 3.3746590614318848,
-      "rewards/rejected": -7.183419704437256,
-      "step": 449
-    },
-    {
-      "epoch": 1.4516129032258065,
-      "grad_norm": 0.4619046151638031,
-      "learning_rate": 8.812013869026334e-07,
-      "logits/chosen": -2.4269673824310303,
-      "logits/rejected": -2.013381004333496,
-      "logps/chosen": -139.54246520996094,
-      "logps/rejected": -216.44473266601562,
-      "loss": 0.1075877994298935,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.083252906799316,
-      "rewards/margins": 4.468310832977295,
-      "rewards/rejected": -9.55156421661377,
-      "step": 450
-    },
-    {
-      "epoch": 1.4548387096774194,
-      "grad_norm": 0.47495365142822266,
-      "learning_rate": 8.71568793194445e-07,
-      "logits/chosen": -2.4188807010650635,
-      "logits/rejected": -2.1440956592559814,
-      "logps/chosen": -141.9351806640625,
-      "logps/rejected": -200.09054565429688,
-      "loss": 0.12590685486793518,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.05722713470459,
-      "rewards/margins": 3.3638811111450195,
-      "rewards/rejected": -8.42110824584961,
-      "step": 451
-    },
-    {
-      "epoch": 1.4580645161290322,
-      "grad_norm": 0.8909400701522827,
-      "learning_rate": 8.619780099373946e-07,
-      "logits/chosen": -2.4395079612731934,
-      "logits/rejected": -2.122635841369629,
-      "logps/chosen": -136.7159423828125,
-      "logps/rejected": -203.58587646484375,
-      "loss": 0.23426640033721924,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -4.406562805175781,
-      "rewards/margins": 3.873863935470581,
-      "rewards/rejected": -8.280426979064941,
-      "step": 452
-    },
-    {
-      "epoch": 1.4612903225806453,
-      "grad_norm": 0.47088634967803955,
-      "learning_rate": 8.524292833776707e-07,
-      "logits/chosen": -2.372608184814453,
-      "logits/rejected": -1.9174907207489014,
-      "logps/chosen": -133.89572143554688,
-      "logps/rejected": -198.1195831298828,
-      "loss": 0.10052140057086945,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -3.832404851913452,
-      "rewards/margins": 3.9090652465820312,
-      "rewards/rejected": -7.741469860076904,
-      "step": 453
-    },
-    {
-      "epoch": 1.4645161290322581,
-      "grad_norm": 0.4684765338897705,
-      "learning_rate": 8.429228586816405e-07,
-      "logits/chosen": -2.636364459991455,
-      "logits/rejected": -2.2065482139587402,
-      "logps/chosen": -149.46168518066406,
-      "logps/rejected": -220.55462646484375,
-      "loss": 0.10019417107105255,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -4.581504821777344,
-      "rewards/margins": 4.545250415802002,
-      "rewards/rejected": -9.126755714416504,
-      "step": 454
-    },
-    {
-      "epoch": 1.467741935483871,
-      "grad_norm": 0.6270551085472107,
-      "learning_rate": 8.334589799295592e-07,
-      "logits/chosen": -2.204833745956421,
-      "logits/rejected": -1.9377959966659546,
-      "logps/chosen": -151.548583984375,
-      "logps/rejected": -210.1044921875,
-      "loss": 0.15584611892700195,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.5387187004089355,
-      "rewards/margins": 3.5987823009490967,
-      "rewards/rejected": -8.13750171661377,
-      "step": 455
-    },
-    {
-      "epoch": 1.4709677419354839,
-      "grad_norm": 0.3135511577129364,
-      "learning_rate": 8.240378901093035e-07,
-      "logits/chosen": -2.537592649459839,
-      "logits/rejected": -2.218113422393799,
-      "logps/chosen": -144.51199340820312,
-      "logps/rejected": -225.44337463378906,
-      "loss": 0.06254112720489502,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -4.4782867431640625,
-      "rewards/margins": 4.516358375549316,
-      "rewards/rejected": -8.994644165039062,
-      "step": 456
-    },
-    {
-      "epoch": 1.4741935483870967,
-      "grad_norm": 1.0306941270828247,
-      "learning_rate": 8.146598311101317e-07,
-      "logits/chosen": -2.2063868045806885,
-      "logits/rejected": -1.911825180053711,
-      "logps/chosen": -159.69532775878906,
-      "logps/rejected": -212.94476318359375,
-      "loss": 0.23534297943115234,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -6.0022382736206055,
-      "rewards/margins": 3.1327128410339355,
-      "rewards/rejected": -9.134952545166016,
-      "step": 457
-    },
-    {
-      "epoch": 1.4774193548387098,
-      "grad_norm": 0.7191207408905029,
-      "learning_rate": 8.053250437164716e-07,
-      "logits/chosen": -2.118504762649536,
-      "logits/rejected": -1.8495814800262451,
-      "logps/chosen": -155.6160430908203,
-      "logps/rejected": -211.62818908691406,
-      "loss": 0.1392982304096222,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.2305908203125,
-      "rewards/margins": 3.6847500801086426,
-      "rewards/rejected": -7.915340900421143,
-      "step": 458
-    },
-    {
-      "epoch": 1.4806451612903226,
-      "grad_norm": 0.6096194386482239,
-      "learning_rate": 7.960337676017396e-07,
-      "logits/chosen": -2.7578587532043457,
-      "logits/rejected": -2.4358770847320557,
-      "logps/chosen": -143.50048828125,
-      "logps/rejected": -202.1869659423828,
-      "loss": 0.15191882848739624,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.54219913482666,
-      "rewards/margins": 3.695138454437256,
-      "rewards/rejected": -8.237337112426758,
-      "step": 459
-    },
-    {
-      "epoch": 1.4838709677419355,
-      "grad_norm": 0.5208762288093567,
-      "learning_rate": 7.867862413221894e-07,
-      "logits/chosen": -2.467456579208374,
-      "logits/rejected": -2.0829808712005615,
-      "logps/chosen": -131.95315551757812,
-      "logps/rejected": -202.734375,
-      "loss": 0.11966469138860703,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.343000888824463,
-      "rewards/margins": 4.022573471069336,
-      "rewards/rejected": -8.36557388305664,
-      "step": 460
-    },
-    {
-      "epoch": 1.4870967741935484,
-      "grad_norm": 0.4789765179157257,
-      "learning_rate": 7.775827023107835e-07,
-      "logits/chosen": -2.528873920440674,
-      "logits/rejected": -2.153320789337158,
-      "logps/chosen": -142.16531372070312,
-      "logps/rejected": -210.26077270507812,
-      "loss": 0.09036726504564285,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -4.478372573852539,
-      "rewards/margins": 4.291580677032471,
-      "rewards/rejected": -8.769953727722168,
-      "step": 461
-    },
-    {
-      "epoch": 1.4903225806451612,
-      "grad_norm": 0.7295042276382446,
-      "learning_rate": 7.684233868710983e-07,
-      "logits/chosen": -2.4204530715942383,
-      "logits/rejected": -2.1346089839935303,
-      "logps/chosen": -166.9981689453125,
-      "logps/rejected": -230.27064514160156,
-      "loss": 0.17296504974365234,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.720881462097168,
-      "rewards/margins": 3.7996249198913574,
-      "rewards/rejected": -9.520506858825684,
-      "step": 462
-    },
-    {
-      "epoch": 1.4935483870967743,
-      "grad_norm": 0.7209341526031494,
-      "learning_rate": 7.593085301712566e-07,
-      "logits/chosen": -2.1364479064941406,
-      "logits/rejected": -1.8027024269104004,
-      "logps/chosen": -177.41847229003906,
-      "logps/rejected": -242.67233276367188,
-      "loss": 0.18686425685882568,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.450223922729492,
-      "rewards/margins": 3.922041416168213,
-      "rewards/rejected": -9.372265815734863,
-      "step": 463
-    },
-    {
-      "epoch": 1.4967741935483871,
-      "grad_norm": 0.6717926263809204,
-      "learning_rate": 7.502383662378895e-07,
-      "logits/chosen": -2.4188666343688965,
-      "logits/rejected": -2.0470993518829346,
-      "logps/chosen": -131.65296936035156,
-      "logps/rejected": -193.15231323242188,
-      "loss": 0.21131563186645508,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.352531909942627,
-      "rewards/margins": 3.5780229568481445,
-      "rewards/rejected": -7.930554389953613,
-      "step": 464
-    },
-    {
-      "epoch": 1.5,
-      "grad_norm": 0.6198790073394775,
-      "learning_rate": 7.412131279501297e-07,
-      "logits/chosen": -2.3598873615264893,
-      "logits/rejected": -2.030845880508423,
-      "logps/chosen": -155.17327880859375,
-      "logps/rejected": -218.849365234375,
-      "loss": 0.17705798149108887,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.077648162841797,
-      "rewards/margins": 3.896883726119995,
-      "rewards/rejected": -8.974532127380371,
-      "step": 465
-    },
-    {
-      "epoch": 1.5032258064516129,
-      "grad_norm": 0.7314626574516296,
-      "learning_rate": 7.322330470336314e-07,
-      "logits/chosen": -2.189079761505127,
-      "logits/rejected": -1.7819743156433105,
-      "logps/chosen": -150.0357666015625,
-      "logps/rejected": -217.533935546875,
-      "loss": 0.12745235860347748,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.761589050292969,
-      "rewards/margins": 4.053315162658691,
-      "rewards/rejected": -8.81490421295166,
-      "step": 466
-    },
-    {
-      "epoch": 1.5064516129032257,
-      "grad_norm": 0.736470639705658,
-      "learning_rate": 7.232983540546173e-07,
-      "logits/chosen": -2.2979016304016113,
-      "logits/rejected": -1.9683010578155518,
-      "logps/chosen": -146.45066833496094,
-      "logps/rejected": -209.13137817382812,
-      "loss": 0.19569313526153564,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.635673999786377,
-      "rewards/margins": 3.676464557647705,
-      "rewards/rejected": -8.312139511108398,
-      "step": 467
-    },
-    {
-      "epoch": 1.5096774193548388,
-      "grad_norm": 0.3781033754348755,
-      "learning_rate": 7.144092784139619e-07,
-      "logits/chosen": -2.039719581604004,
-      "logits/rejected": -1.7212536334991455,
-      "logps/chosen": -171.434326171875,
-      "logps/rejected": -237.9990234375,
-      "loss": 0.10456052422523499,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.9777727127075195,
-      "rewards/margins": 4.449230670928955,
-      "rewards/rejected": -9.427002906799316,
-      "step": 468
-    },
-    {
-      "epoch": 1.5129032258064516,
-      "grad_norm": 0.7195813059806824,
-      "learning_rate": 7.05566048341303e-07,
-      "logits/chosen": -1.9777944087982178,
-      "logits/rejected": -1.6352850198745728,
-      "logps/chosen": -180.8108673095703,
-      "logps/rejected": -257.6719665527344,
-      "loss": 0.1284591257572174,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -6.535017013549805,
-      "rewards/margins": 4.279155731201172,
-      "rewards/rejected": -10.814172744750977,
-      "step": 469
-    },
-    {
-      "epoch": 1.5161290322580645,
-      "grad_norm": 1.0195460319519043,
-      "learning_rate": 6.967688908891793e-07,
-      "logits/chosen": -2.204637050628662,
-      "logits/rejected": -1.9138691425323486,
-      "logps/chosen": -160.77059936523438,
-      "logps/rejected": -235.45228576660156,
-      "loss": 0.2252466380596161,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.683173179626465,
-      "rewards/margins": 4.687705993652344,
-      "rewards/rejected": -10.370879173278809,
-      "step": 470
-    },
-    {
-      "epoch": 1.5193548387096776,
-      "grad_norm": 0.4896174371242523,
-      "learning_rate": 6.880180319272006e-07,
-      "logits/chosen": -2.68131160736084,
-      "logits/rejected": -2.368668794631958,
-      "logps/chosen": -145.79098510742188,
-      "logps/rejected": -197.18484497070312,
-      "loss": 0.1361011266708374,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.403603553771973,
-      "rewards/margins": 3.4635801315307617,
-      "rewards/rejected": -7.867183685302734,
-      "step": 471
-    },
-    {
-      "epoch": 1.5225806451612902,
-      "grad_norm": 0.6289461255073547,
-      "learning_rate": 6.79313696136249e-07,
-      "logits/chosen": -2.1100573539733887,
-      "logits/rejected": -1.6402158737182617,
-      "logps/chosen": -154.7132568359375,
-      "logps/rejected": -222.39056396484375,
-      "loss": 0.15632954239845276,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.803866863250732,
-      "rewards/margins": 4.1356987953186035,
-      "rewards/rejected": -8.939565658569336,
-      "step": 472
-    },
-    {
-      "epoch": 1.5258064516129033,
-      "grad_norm": 0.34602445363998413,
-      "learning_rate": 6.706561070027109e-07,
-      "logits/chosen": -2.1718649864196777,
-      "logits/rejected": -1.7549580335617065,
-      "logps/chosen": -162.49391174316406,
-      "logps/rejected": -237.9075927734375,
-      "loss": 0.07619393616914749,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.069583892822266,
-      "rewards/margins": 4.553929805755615,
-      "rewards/rejected": -9.623514175415039,
-      "step": 473
-    },
-    {
-      "epoch": 1.5290322580645161,
-      "grad_norm": 0.9247276782989502,
-      "learning_rate": 6.620454868127396e-07,
-      "logits/chosen": -1.8749210834503174,
-      "logits/rejected": -1.5326533317565918,
-      "logps/chosen": -163.49847412109375,
-      "logps/rejected": -231.64306640625,
-      "loss": 0.2411191463470459,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.642821788787842,
-      "rewards/margins": 4.313276290893555,
-      "rewards/rejected": -9.956097602844238,
-      "step": 474
-    },
-    {
-      "epoch": 1.532258064516129,
-      "grad_norm": 0.5158165693283081,
-      "learning_rate": 6.534820566465464e-07,
-      "logits/chosen": -2.297236204147339,
-      "logits/rejected": -1.91456139087677,
-      "logps/chosen": -144.25331115722656,
-      "logps/rejected": -201.99111938476562,
-      "loss": 0.15576979517936707,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.625210762023926,
-      "rewards/margins": 3.590076446533203,
-      "rewards/rejected": -8.215287208557129,
-      "step": 475
-    },
-    {
-      "epoch": 1.535483870967742,
-      "grad_norm": 0.6987940669059753,
-      "learning_rate": 6.449660363727236e-07,
-      "logits/chosen": -2.297675132751465,
-      "logits/rejected": -1.7836105823516846,
-      "logps/chosen": -151.13223266601562,
-      "logps/rejected": -221.970947265625,
-      "loss": 0.1767854392528534,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.299595355987549,
-      "rewards/margins": 4.221803188323975,
-      "rewards/rejected": -9.521398544311523,
-      "step": 476
-    },
-    {
-      "epoch": 1.5387096774193547,
-      "grad_norm": 0.601422905921936,
-      "learning_rate": 6.36497644642601e-07,
-      "logits/chosen": -2.286315441131592,
-      "logits/rejected": -1.9132990837097168,
-      "logps/chosen": -149.023193359375,
-      "logps/rejected": -229.86410522460938,
-      "loss": 0.10134474188089371,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.733626365661621,
-      "rewards/margins": 4.846975803375244,
-      "rewards/rejected": -10.580602645874023,
-      "step": 477
-    },
-    {
-      "epoch": 1.5419354838709678,
-      "grad_norm": 0.6406868696212769,
-      "learning_rate": 6.28077098884633e-07,
-      "logits/chosen": -2.4604923725128174,
-      "logits/rejected": -2.084846019744873,
-      "logps/chosen": -139.864013671875,
-      "logps/rejected": -211.58987426757812,
-      "loss": 0.13258515298366547,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.099147319793701,
-      "rewards/margins": 4.616920471191406,
-      "rewards/rejected": -8.716068267822266,
-      "step": 478
-    },
-    {
-      "epoch": 1.5451612903225806,
-      "grad_norm": 1.0441559553146362,
-      "learning_rate": 6.197046152988137e-07,
-      "logits/chosen": -2.1741297245025635,
-      "logits/rejected": -1.8643018007278442,
-      "logps/chosen": -168.7382049560547,
-      "logps/rejected": -230.0182647705078,
-      "loss": 0.2577561140060425,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -6.1876912117004395,
-      "rewards/margins": 3.6677284240722656,
-      "rewards/rejected": -9.855419158935547,
-      "step": 479
-    },
-    {
-      "epoch": 1.5483870967741935,
-      "grad_norm": 0.9108280539512634,
-      "learning_rate": 6.113804088511261e-07,
-      "logits/chosen": -1.954637885093689,
-      "logits/rejected": -1.6181361675262451,
-      "logps/chosen": -175.4678955078125,
-      "logps/rejected": -237.62701416015625,
-      "loss": 0.16138975322246552,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.869152545928955,
-      "rewards/margins": 3.9080886840820312,
-      "rewards/rejected": -9.777240753173828,
-      "step": 480
-    },
-    {
-      "epoch": 1.5516129032258066,
-      "grad_norm": 0.24027889966964722,
-      "learning_rate": 6.031046932680229e-07,
-      "logits/chosen": -2.4597034454345703,
-      "logits/rejected": -2.1957783699035645,
-      "logps/chosen": -157.1087646484375,
-      "logps/rejected": -233.65301513671875,
-      "loss": 0.05748886615037918,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.280271053314209,
-      "rewards/margins": 4.910508155822754,
-      "rewards/rejected": -10.190779685974121,
-      "step": 481
-    },
-    {
-      "epoch": 1.5548387096774192,
-      "grad_norm": 0.9712018966674805,
-      "learning_rate": 5.948776810309423e-07,
-      "logits/chosen": -2.126974582672119,
-      "logits/rejected": -1.7661800384521484,
-      "logps/chosen": -151.3673858642578,
-      "logps/rejected": -218.9346466064453,
-      "loss": 0.24991604685783386,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.019974708557129,
-      "rewards/margins": 3.8181777000427246,
-      "rewards/rejected": -8.838151931762695,
-      "step": 482
-    },
-    {
-      "epoch": 1.5580645161290323,
-      "grad_norm": 0.7719769477844238,
-      "learning_rate": 5.866995833708464e-07,
-      "logits/chosen": -1.708822250366211,
-      "logits/rejected": -1.3647013902664185,
-      "logps/chosen": -164.8237762451172,
-      "logps/rejected": -234.05825805664062,
-      "loss": 0.18122202157974243,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.682433605194092,
-      "rewards/margins": 4.063777923583984,
-      "rewards/rejected": -9.746211051940918,
-      "step": 483
-    },
-    {
-      "epoch": 1.5612903225806452,
-      "grad_norm": 0.3257688283920288,
-      "learning_rate": 5.785706102628044e-07,
-      "logits/chosen": -2.233219861984253,
-      "logits/rejected": -1.8412196636199951,
-      "logps/chosen": -152.9199981689453,
-      "logps/rejected": -216.41781616210938,
-      "loss": 0.07986757159233093,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -4.557250499725342,
-      "rewards/margins": 4.080105781555176,
-      "rewards/rejected": -8.63735580444336,
-      "step": 484
-    },
-    {
-      "epoch": 1.564516129032258,
-      "grad_norm": 0.6999625563621521,
-      "learning_rate": 5.704909704205949e-07,
-      "logits/chosen": -1.9356735944747925,
-      "logits/rejected": -1.7792093753814697,
-      "logps/chosen": -166.04051208496094,
-      "logps/rejected": -219.11294555664062,
-      "loss": 0.18605303764343262,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -5.293807029724121,
-      "rewards/margins": 3.0410327911376953,
-      "rewards/rejected": -8.334840774536133,
-      "step": 485
-    },
-    {
-      "epoch": 1.567741935483871,
-      "grad_norm": 0.8240074515342712,
-      "learning_rate": 5.624608712913531e-07,
-      "logits/chosen": -2.172473192214966,
-      "logits/rejected": -1.7739810943603516,
-      "logps/chosen": -136.5169219970703,
-      "logps/rejected": -208.26185607910156,
-      "loss": 0.16257520020008087,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.544417858123779,
-      "rewards/margins": 4.220807075500488,
-      "rewards/rejected": -8.76522445678711,
-      "step": 486
-    },
-    {
-      "epoch": 1.5709677419354837,
-      "grad_norm": 0.6017993688583374,
-      "learning_rate": 5.544805190502406e-07,
-      "logits/chosen": -2.2712013721466064,
-      "logits/rejected": -1.9342961311340332,
-      "logps/chosen": -151.09034729003906,
-      "logps/rejected": -212.1373291015625,
-      "loss": 0.15379485487937927,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.01048469543457,
-      "rewards/margins": 3.8621017932891846,
-      "rewards/rejected": -8.872586250305176,
-      "step": 487
-    },
-    {
-      "epoch": 1.5741935483870968,
-      "grad_norm": 0.5773161053657532,
-      "learning_rate": 5.465501185951519e-07,
-      "logits/chosen": -2.4476537704467773,
-      "logits/rejected": -2.045992851257324,
-      "logps/chosen": -151.75369262695312,
-      "logps/rejected": -227.71702575683594,
-      "loss": 0.1062774658203125,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.999873161315918,
-      "rewards/margins": 4.466883182525635,
-      "rewards/rejected": -9.466756820678711,
-      "step": 488
-    },
-    {
-      "epoch": 1.5774193548387097,
-      "grad_norm": 0.4516680836677551,
-      "learning_rate": 5.386698735414572e-07,
-      "logits/chosen": -2.406808853149414,
-      "logits/rejected": -1.889483094215393,
-      "logps/chosen": -124.93113708496094,
-      "logps/rejected": -212.51937866210938,
-      "loss": 0.0906469076871872,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.496485710144043,
-      "rewards/margins": 5.360855579376221,
-      "rewards/rejected": -9.857341766357422,
-      "step": 489
-    },
-    {
-      "epoch": 1.5806451612903225,
-      "grad_norm": 0.29949402809143066,
-      "learning_rate": 5.308399862167693e-07,
-      "logits/chosen": -2.1711933612823486,
-      "logits/rejected": -1.7267756462097168,
-      "logps/chosen": -146.49330139160156,
-      "logps/rejected": -239.75428771972656,
-      "loss": 0.07038956135511398,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -4.902659893035889,
-      "rewards/margins": 5.8861589431762695,
-      "rewards/rejected": -10.788818359375,
-      "step": 490
-    },
-    {
-      "epoch": 1.5838709677419356,
-      "grad_norm": 0.6430067420005798,
-      "learning_rate": 5.23060657655754e-07,
-      "logits/chosen": -2.275541067123413,
-      "logits/rejected": -1.8227019309997559,
-      "logps/chosen": -160.78375244140625,
-      "logps/rejected": -235.040771484375,
-      "loss": 0.13946972787380219,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.349231719970703,
-      "rewards/margins": 4.761424541473389,
-      "rewards/rejected": -10.110654830932617,
-      "step": 491
-    },
-    {
-      "epoch": 1.5870967741935482,
-      "grad_norm": 0.560444176197052,
-      "learning_rate": 5.153320875949632e-07,
-      "logits/chosen": -2.355436086654663,
-      "logits/rejected": -2.036803960800171,
-      "logps/chosen": -154.39837646484375,
-      "logps/rejected": -220.92160034179688,
-      "loss": 0.1334306001663208,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.16274356842041,
-      "rewards/margins": 4.023294925689697,
-      "rewards/rejected": -9.186038970947266,
-      "step": 492
-    },
-    {
-      "epoch": 1.5903225806451613,
-      "grad_norm": 0.5349669456481934,
-      "learning_rate": 5.076544744677128e-07,
-      "logits/chosen": -2.483330726623535,
-      "logits/rejected": -2.1197609901428223,
-      "logps/chosen": -156.27420043945312,
-      "logps/rejected": -217.24514770507812,
-      "loss": 0.108750119805336,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.527863502502441,
-      "rewards/margins": 3.8662824630737305,
-      "rewards/rejected": -8.394145965576172,
-      "step": 493
-    },
-    {
-      "epoch": 1.5935483870967742,
-      "grad_norm": 0.2775120437145233,
-      "learning_rate": 5.000280153989806e-07,
-      "logits/chosen": -2.387772560119629,
-      "logits/rejected": -1.9994821548461914,
-      "logps/chosen": -151.18211364746094,
-      "logps/rejected": -227.11680603027344,
-      "loss": 0.04961947351694107,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -4.598075866699219,
-      "rewards/margins": 4.705333709716797,
-      "rewards/rejected": -9.303409576416016,
-      "step": 494
-    },
-    {
-      "epoch": 1.596774193548387,
-      "grad_norm": 0.6997309923171997,
-      "learning_rate": 4.924529062003522e-07,
-      "logits/chosen": -2.093170166015625,
-      "logits/rejected": -1.5877161026000977,
-      "logps/chosen": -138.06466674804688,
-      "logps/rejected": -211.8524932861328,
-      "loss": 0.1541440337896347,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.241321563720703,
-      "rewards/margins": 4.32204008102417,
-      "rewards/rejected": -8.563361167907715,
-      "step": 495
-    },
-    {
-      "epoch": 1.6,
-      "grad_norm": 0.7098890542984009,
-      "learning_rate": 4.84929341364988e-07,
-      "logits/chosen": -1.9759151935577393,
-      "logits/rejected": -1.6069056987762451,
-      "logps/chosen": -157.34674072265625,
-      "logps/rejected": -219.7688446044922,
-      "loss": 0.1809576451778412,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.922688007354736,
-      "rewards/margins": 3.9587292671203613,
-      "rewards/rejected": -8.881418228149414,
-      "step": 496
-    },
-    {
-      "epoch": 1.603225806451613,
-      "grad_norm": 0.5172654986381531,
-      "learning_rate": 4.774575140626317e-07,
-      "logits/chosen": -2.007028102874756,
-      "logits/rejected": -1.6205710172653198,
-      "logps/chosen": -166.93746948242188,
-      "logps/rejected": -236.51893615722656,
-      "loss": 0.09278646111488342,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.094544887542725,
-      "rewards/margins": 4.20934534072876,
-      "rewards/rejected": -9.303890228271484,
-      "step": 497
-    },
-    {
-      "epoch": 1.6064516129032258,
-      "grad_norm": 0.3379948139190674,
-      "learning_rate": 4.7003761613465174e-07,
-      "logits/chosen": -2.038321018218994,
-      "logits/rejected": -1.653014898300171,
-      "logps/chosen": -166.12400817871094,
-      "logps/rejected": -238.2560577392578,
-      "loss": 0.05834685266017914,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.0331950187683105,
-      "rewards/margins": 4.559978008270264,
-      "rewards/rejected": -9.593173027038574,
-      "step": 498
-    },
-    {
-      "epoch": 1.6096774193548387,
-      "grad_norm": 1.2031432390213013,
-      "learning_rate": 4.626698380891154e-07,
-      "logits/chosen": -2.3232650756835938,
-      "logits/rejected": -2.098174810409546,
-      "logps/chosen": -154.74142456054688,
-      "logps/rejected": -210.04086303710938,
-      "loss": 0.26611971855163574,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.1587114334106445,
-      "rewards/margins": 3.5554258823394775,
-      "rewards/rejected": -8.71413803100586,
-      "step": 499
-    },
-    {
-      "epoch": 1.6129032258064515,
-      "grad_norm": 0.7430384755134583,
-      "learning_rate": 4.553543690958939e-07,
-      "logits/chosen": -2.6903367042541504,
-      "logits/rejected": -2.353358507156372,
-      "logps/chosen": -147.15798950195312,
-      "logps/rejected": -199.20314025878906,
-      "loss": 0.24731270968914032,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -5.127081394195557,
-      "rewards/margins": 2.9009103775024414,
-      "rewards/rejected": -8.027992248535156,
-      "step": 500
-    },
-    {
-      "epoch": 1.6161290322580646,
-      "grad_norm": 0.505378782749176,
-      "learning_rate": 4.480913969818099e-07,
-      "logits/chosen": -2.323559284210205,
-      "logits/rejected": -1.9315087795257568,
-      "logps/chosen": -161.15017700195312,
-      "logps/rejected": -225.9407196044922,
-      "loss": 0.15063539147377014,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.602543354034424,
-      "rewards/margins": 3.9944305419921875,
-      "rewards/rejected": -9.59697437286377,
-      "step": 501
-    },
-    {
-      "epoch": 1.6193548387096774,
-      "grad_norm": 0.5478779077529907,
-      "learning_rate": 4.408811082258116e-07,
-      "logits/chosen": -2.405697822570801,
-      "logits/rejected": -2.060330629348755,
-      "logps/chosen": -149.43878173828125,
-      "logps/rejected": -203.4425506591797,
-      "loss": 0.11198721826076508,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -4.570002555847168,
-      "rewards/margins": 3.5500755310058594,
-      "rewards/rejected": -8.120078086853027,
-      "step": 502
-    },
-    {
-      "epoch": 1.6225806451612903,
-      "grad_norm": 0.8584948182106018,
-      "learning_rate": 4.3372368795419006e-07,
-      "logits/chosen": -2.1318094730377197,
-      "logits/rejected": -1.7252811193466187,
-      "logps/chosen": -156.61129760742188,
-      "logps/rejected": -231.13629150390625,
-      "loss": 0.18643182516098022,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.35899543762207,
-      "rewards/margins": 4.680524826049805,
-      "rewards/rejected": -10.039520263671875,
-      "step": 503
-    },
-    {
-      "epoch": 1.6258064516129034,
-      "grad_norm": 0.8634819388389587,
-      "learning_rate": 4.2661931993581874e-07,
-      "logits/chosen": -2.2070565223693848,
-      "logits/rejected": -1.828540563583374,
-      "logps/chosen": -158.256591796875,
-      "logps/rejected": -222.89712524414062,
-      "loss": 0.18253442645072937,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.994098663330078,
-      "rewards/margins": 3.824960470199585,
-      "rewards/rejected": -8.819059371948242,
-      "step": 504
-    },
-    {
-      "epoch": 1.629032258064516,
-      "grad_norm": 0.39533722400665283,
-      "learning_rate": 4.1956818657744065e-07,
-      "logits/chosen": -2.7217161655426025,
-      "logits/rejected": -2.2938411235809326,
-      "logps/chosen": -132.1091766357422,
-      "logps/rejected": -214.93460083007812,
-      "loss": 0.10780246555805206,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -4.512347221374512,
-      "rewards/margins": 5.154045104980469,
-      "rewards/rejected": -9.66639232635498,
-      "step": 505
-    },
-    {
-      "epoch": 1.632258064516129,
-      "grad_norm": 0.4706231653690338,
-      "learning_rate": 4.125704689189819e-07,
-      "logits/chosen": -2.0221657752990723,
-      "logits/rejected": -1.658431887626648,
-      "logps/chosen": -164.7430419921875,
-      "logps/rejected": -224.69056701660156,
-      "loss": 0.09151042252779007,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.65877628326416,
-      "rewards/margins": 4.031785011291504,
-      "rewards/rejected": -8.690561294555664,
-      "step": 506
-    },
-    {
-      "epoch": 1.635483870967742,
-      "grad_norm": 0.5980255603790283,
-      "learning_rate": 4.056263466289059e-07,
-      "logits/chosen": -2.4284605979919434,
-      "logits/rejected": -2.004915475845337,
-      "logps/chosen": -146.16546630859375,
-      "logps/rejected": -226.4752197265625,
-      "loss": 0.10536962002515793,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.9078874588012695,
-      "rewards/margins": 4.797103404998779,
-      "rewards/rejected": -9.70499038696289,
-      "step": 507
-    },
-    {
-      "epoch": 1.6387096774193548,
-      "grad_norm": 0.905908465385437,
-      "learning_rate": 3.987359979996e-07,
-      "logits/chosen": -1.9300092458724976,
-      "logits/rejected": -1.616002082824707,
-      "logps/chosen": -175.74253845214844,
-      "logps/rejected": -242.51318359375,
-      "loss": 0.1710321605205536,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -6.3357696533203125,
-      "rewards/margins": 4.0363850593566895,
-      "rewards/rejected": -10.37215518951416,
-      "step": 508
-    },
-    {
-      "epoch": 1.6419354838709679,
-      "grad_norm": 0.792195737361908,
-      "learning_rate": 3.9189959994279493e-07,
-      "logits/chosen": -2.461611032485962,
-      "logits/rejected": -2.0225419998168945,
-      "logps/chosen": -144.42999267578125,
-      "logps/rejected": -218.36630249023438,
-      "loss": 0.19733068346977234,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.6903486251831055,
-      "rewards/margins": 4.069197177886963,
-      "rewards/rejected": -8.759546279907227,
-      "step": 509
-    },
-    {
-      "epoch": 1.6451612903225805,
-      "grad_norm": 0.5894666910171509,
-      "learning_rate": 3.851173279850251e-07,
-      "logits/chosen": -2.2512221336364746,
-      "logits/rejected": -1.8765403032302856,
-      "logps/chosen": -161.9225616455078,
-      "logps/rejected": -218.21316528320312,
-      "loss": 0.13903504610061646,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.284883975982666,
-      "rewards/margins": 3.688009738922119,
-      "rewards/rejected": -8.972892761230469,
-      "step": 510
-    },
-    {
-      "epoch": 1.6483870967741936,
-      "grad_norm": 0.6753301024436951,
-      "learning_rate": 3.7838935626312246e-07,
-      "logits/chosen": -2.39656400680542,
-      "logits/rejected": -2.112570285797119,
-      "logps/chosen": -132.52127075195312,
-      "logps/rejected": -195.42550659179688,
-      "loss": 0.17207223176956177,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.37971830368042,
-      "rewards/margins": 3.5059244632720947,
-      "rewards/rejected": -7.885643005371094,
-      "step": 511
-    },
-    {
-      "epoch": 1.6516129032258065,
-      "grad_norm": 0.26837578415870667,
-      "learning_rate": 3.717158575197441e-07,
-      "logits/chosen": -2.2023987770080566,
-      "logits/rejected": -1.7415173053741455,
-      "logps/chosen": -152.79617309570312,
-      "logps/rejected": -234.56285095214844,
-      "loss": 0.0666898787021637,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.663599967956543,
-      "rewards/margins": 4.916285514831543,
-      "rewards/rejected": -10.579885482788086,
-      "step": 512
-    },
-    {
-      "epoch": 1.6548387096774193,
-      "grad_norm": 0.5426744818687439,
-      "learning_rate": 3.650970030989362e-07,
-      "logits/chosen": -2.204011917114258,
-      "logits/rejected": -1.8495509624481201,
-      "logps/chosen": -128.29156494140625,
-      "logps/rejected": -205.69387817382812,
-      "loss": 0.14435040950775146,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.655958652496338,
-      "rewards/margins": 4.450262546539307,
-      "rewards/rejected": -9.106222152709961,
-      "step": 513
-    },
-    {
-      "epoch": 1.6580645161290324,
-      "grad_norm": 0.30497443675994873,
-      "learning_rate": 3.5853296294173665e-07,
-      "logits/chosen": -2.109001398086548,
-      "logits/rejected": -1.6162521839141846,
-      "logps/chosen": -156.780517578125,
-      "logps/rejected": -229.7040252685547,
-      "loss": 0.05599897354841232,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -4.427706718444824,
-      "rewards/margins": 4.8288421630859375,
-      "rewards/rejected": -9.256548881530762,
-      "step": 514
-    },
-    {
-      "epoch": 1.661290322580645,
-      "grad_norm": 0.4224660396575928,
-      "learning_rate": 3.5202390558181145e-07,
-      "logits/chosen": -2.2887954711914062,
-      "logits/rejected": -1.880237340927124,
-      "logps/chosen": -160.7728271484375,
-      "logps/rejected": -238.85765075683594,
-      "loss": 0.0738740861415863,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.382646083831787,
-      "rewards/margins": 4.997879505157471,
-      "rewards/rejected": -10.380525588989258,
-      "step": 515
-    },
-    {
-      "epoch": 1.664516129032258,
-      "grad_norm": 0.4132245182991028,
-      "learning_rate": 3.455699981411259e-07,
-      "logits/chosen": -2.2288949489593506,
-      "logits/rejected": -1.6625796556472778,
-      "logps/chosen": -136.56069946289062,
-      "logps/rejected": -225.6313934326172,
-      "loss": 0.0787404254078865,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.833522796630859,
-      "rewards/margins": 5.683566093444824,
-      "rewards/rejected": -10.517088890075684,
-      "step": 516
-    },
-    {
-      "epoch": 1.667741935483871,
-      "grad_norm": 0.5380642414093018,
-      "learning_rate": 3.3917140632565627e-07,
-      "logits/chosen": -2.1946632862091064,
-      "logits/rejected": -1.7250945568084717,
-      "logps/chosen": -149.41163635253906,
-      "logps/rejected": -227.01788330078125,
-      "loss": 0.10217361897230148,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.115983009338379,
-      "rewards/margins": 4.956330299377441,
-      "rewards/rejected": -10.07231330871582,
-      "step": 517
-    },
-    {
-      "epoch": 1.6709677419354838,
-      "grad_norm": 0.6858349442481995,
-      "learning_rate": 3.3282829442113277e-07,
-      "logits/chosen": -2.756497383117676,
-      "logits/rejected": -2.385478973388672,
-      "logps/chosen": -132.74374389648438,
-      "logps/rejected": -202.3121795654297,
-      "loss": 0.17126917839050293,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.5010786056518555,
-      "rewards/margins": 4.135538578033447,
-      "rewards/rejected": -8.636617660522461,
-      "step": 518
-    },
-    {
-      "epoch": 1.6741935483870969,
-      "grad_norm": 0.3403320908546448,
-      "learning_rate": 3.265408252888222e-07,
-      "logits/chosen": -2.14156436920166,
-      "logits/rejected": -1.7484135627746582,
-      "logps/chosen": -150.6089630126953,
-      "logps/rejected": -227.8260955810547,
-      "loss": 0.06928884983062744,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.599906921386719,
-      "rewards/margins": 5.042827129364014,
-      "rewards/rejected": -9.642733573913574,
-      "step": 519
-    },
-    {
-      "epoch": 1.6774193548387095,
-      "grad_norm": 0.5155129432678223,
-      "learning_rate": 3.2030916036134866e-07,
-      "logits/chosen": -2.3275980949401855,
-      "logits/rejected": -2.018320322036743,
-      "logps/chosen": -136.49685668945312,
-      "logps/rejected": -214.154541015625,
-      "loss": 0.12636500597000122,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.527489185333252,
-      "rewards/margins": 4.5036139488220215,
-      "rewards/rejected": -9.031103134155273,
-      "step": 520
-    },
-    {
-      "epoch": 1.6806451612903226,
-      "grad_norm": 0.555239200592041,
-      "learning_rate": 3.141334596385448e-07,
-      "logits/chosen": -2.2510530948638916,
-      "logits/rejected": -1.8884137868881226,
-      "logps/chosen": -149.95242309570312,
-      "logps/rejected": -211.98965454101562,
-      "loss": 0.11958973854780197,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.119822978973389,
-      "rewards/margins": 3.77842378616333,
-      "rewards/rejected": -8.898246765136719,
-      "step": 521
-    },
-    {
-      "epoch": 1.6838709677419355,
-      "grad_norm": 0.24120715260505676,
-      "learning_rate": 3.0801388168334723e-07,
-      "logits/chosen": -1.8827189207077026,
-      "logits/rejected": -1.5083305835723877,
-      "logps/chosen": -169.45654296875,
-      "logps/rejected": -259.73150634765625,
-      "loss": 0.04009680077433586,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -6.678956031799316,
-      "rewards/margins": 5.2564616203308105,
-      "rewards/rejected": -11.935418128967285,
-      "step": 522
-    },
-    {
-      "epoch": 1.6870967741935483,
-      "grad_norm": 0.6411151885986328,
-      "learning_rate": 3.019505836177228e-07,
-      "logits/chosen": -2.701399326324463,
-      "logits/rejected": -2.2651937007904053,
-      "logps/chosen": -143.605712890625,
-      "logps/rejected": -207.84645080566406,
-      "loss": 0.16435886919498444,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -4.970480442047119,
-      "rewards/margins": 4.199647903442383,
-      "rewards/rejected": -9.17012882232666,
-      "step": 523
-    },
-    {
-      "epoch": 1.6903225806451614,
-      "grad_norm": 0.31048446893692017,
-      "learning_rate": 2.9594372111863696e-07,
-      "logits/chosen": -2.088878631591797,
-      "logits/rejected": -1.5833330154418945,
-      "logps/chosen": -148.25637817382812,
-      "logps/rejected": -240.0180206298828,
-      "loss": 0.047217682003974915,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.118111610412598,
-      "rewards/margins": 5.628313064575195,
-      "rewards/rejected": -10.74642562866211,
-      "step": 524
-    },
-    {
-      "epoch": 1.6935483870967742,
-      "grad_norm": 0.6864602565765381,
-      "learning_rate": 2.8999344841405377e-07,
-      "logits/chosen": -2.192661762237549,
-      "logits/rejected": -1.765640139579773,
-      "logps/chosen": -153.21095275878906,
-      "logps/rejected": -230.85977172851562,
-      "loss": 0.16634179651737213,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.749712944030762,
-      "rewards/margins": 4.466973304748535,
-      "rewards/rejected": -9.216686248779297,
-      "step": 525
-    },
-    {
-      "epoch": 1.696774193548387,
-      "grad_norm": 0.5602472424507141,
-      "learning_rate": 2.840999182789797e-07,
-      "logits/chosen": -2.3613505363464355,
-      "logits/rejected": -1.963671088218689,
-      "logps/chosen": -158.62982177734375,
-      "logps/rejected": -232.7010498046875,
-      "loss": 0.1083802878856659,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.514021873474121,
-      "rewards/margins": 4.288410663604736,
-      "rewards/rejected": -9.802433013916016,
-      "step": 526
-    },
-    {
-      "epoch": 1.7,
-      "grad_norm": 0.8596751093864441,
-      "learning_rate": 2.782632820315362e-07,
-      "logits/chosen": -2.513413190841675,
-      "logits/rejected": -2.0594162940979004,
-      "logps/chosen": -135.89236450195312,
-      "logps/rejected": -203.92601013183594,
-      "loss": 0.184890478849411,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.378267765045166,
-      "rewards/margins": 4.425320625305176,
-      "rewards/rejected": -8.803587913513184,
-      "step": 527
-    },
-    {
-      "epoch": 1.7032258064516128,
-      "grad_norm": 0.8277157545089722,
-      "learning_rate": 2.7248368952908055e-07,
-      "logits/chosen": -2.415259599685669,
-      "logits/rejected": -2.0948903560638428,
-      "logps/chosen": -143.08612060546875,
-      "logps/rejected": -203.56727600097656,
-      "loss": 0.16695170104503632,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -4.928262710571289,
-      "rewards/margins": 3.7459940910339355,
-      "rewards/rejected": -8.674257278442383,
-      "step": 528
-    },
-    {
-      "epoch": 1.706451612903226,
-      "grad_norm": 0.4720885753631592,
-      "learning_rate": 2.6676128916435256e-07,
-      "logits/chosen": -2.2250518798828125,
-      "logits/rejected": -1.8019331693649292,
-      "logps/chosen": -154.83782958984375,
-      "logps/rejected": -236.45587158203125,
-      "loss": 0.08195596933364868,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.60328483581543,
-      "rewards/margins": 4.934556007385254,
-      "rewards/rejected": -10.537841796875,
-      "step": 529
-    },
-    {
-      "epoch": 1.7096774193548387,
-      "grad_norm": 0.6525325179100037,
-      "learning_rate": 2.61096227861668e-07,
-      "logits/chosen": -1.870182752609253,
-      "logits/rejected": -1.4910308122634888,
-      "logps/chosen": -170.4964599609375,
-      "logps/rejected": -226.6752471923828,
-      "loss": 0.15303362905979156,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.455026626586914,
-      "rewards/margins": 3.813904047012329,
-      "rewards/rejected": -9.268930435180664,
-      "step": 530
-    },
-    {
-      "epoch": 1.7129032258064516,
-      "grad_norm": 0.5887526273727417,
-      "learning_rate": 2.5548865107314606e-07,
-      "logits/chosen": -2.2285823822021484,
-      "logits/rejected": -1.960855484008789,
-      "logps/chosen": -161.62039184570312,
-      "logps/rejected": -224.82574462890625,
-      "loss": 0.13227765262126923,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.081503391265869,
-      "rewards/margins": 3.7586007118225098,
-      "rewards/rejected": -8.840105056762695,
-      "step": 531
-    },
-    {
-      "epoch": 1.7161290322580647,
-      "grad_norm": 0.906437873840332,
-      "learning_rate": 2.499387027749725e-07,
-      "logits/chosen": -2.0604171752929688,
-      "logits/rejected": -1.7882590293884277,
-      "logps/chosen": -158.12396240234375,
-      "logps/rejected": -211.99673461914062,
-      "loss": 0.252509206533432,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.8061137199401855,
-      "rewards/margins": 3.07973575592041,
-      "rewards/rejected": -7.885849952697754,
-      "step": 532
-    },
-    {
-      "epoch": 1.7193548387096773,
-      "grad_norm": 0.9765410423278809,
-      "learning_rate": 2.4444652546370627e-07,
-      "logits/chosen": -2.2366061210632324,
-      "logits/rejected": -1.9035241603851318,
-      "logps/chosen": -156.17935180664062,
-      "logps/rejected": -217.71038818359375,
-      "loss": 0.2236594706773758,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.191309928894043,
-      "rewards/margins": 3.9683432579040527,
-      "rewards/rejected": -9.159652709960938,
-      "step": 533
-    },
-    {
-      "epoch": 1.7225806451612904,
-      "grad_norm": 0.42072594165802,
-      "learning_rate": 2.3901226015261793e-07,
-      "logits/chosen": -2.519008159637451,
-      "logits/rejected": -2.1948413848876953,
-      "logps/chosen": -139.654296875,
-      "logps/rejected": -209.64926147460938,
-      "loss": 0.09390527009963989,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.155945301055908,
-      "rewards/margins": 4.245931148529053,
-      "rewards/rejected": -8.401876449584961,
-      "step": 534
-    },
-    {
-      "epoch": 1.7258064516129032,
-      "grad_norm": 0.7160782814025879,
-      "learning_rate": 2.3363604636807065e-07,
-      "logits/chosen": -2.3279876708984375,
-      "logits/rejected": -1.9171054363250732,
-      "logps/chosen": -151.30838012695312,
-      "logps/rejected": -216.0570831298828,
-      "loss": 0.1738387942314148,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.881179332733154,
-      "rewards/margins": 3.874018669128418,
-      "rewards/rejected": -8.755197525024414,
-      "step": 535
-    },
-    {
-      "epoch": 1.729032258064516,
-      "grad_norm": 0.8418323993682861,
-      "learning_rate": 2.2831802214593774e-07,
-      "logits/chosen": -2.156343460083008,
-      "logits/rejected": -1.78789222240448,
-      "logps/chosen": -152.94851684570312,
-      "logps/rejected": -220.4888916015625,
-      "loss": 0.18803098797798157,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -5.578521728515625,
-      "rewards/margins": 3.9799790382385254,
-      "rewards/rejected": -9.558500289916992,
-      "step": 536
-    },
-    {
-      "epoch": 1.7322580645161292,
-      "grad_norm": 0.4620969593524933,
-      "learning_rate": 2.23058324028059e-07,
-      "logits/chosen": -2.0751829147338867,
-      "logits/rejected": -1.639204978942871,
-      "logps/chosen": -158.0532684326172,
-      "logps/rejected": -241.23397827148438,
-      "loss": 0.09422552585601807,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.242846965789795,
-      "rewards/margins": 5.0318121910095215,
-      "rewards/rejected": -10.274659156799316,
-      "step": 537
-    },
-    {
-      "epoch": 1.7354838709677418,
-      "grad_norm": 0.6936494708061218,
-      "learning_rate": 2.178570870587335e-07,
-      "logits/chosen": -2.4572184085845947,
-      "logits/rejected": -2.1089305877685547,
-      "logps/chosen": -157.9110565185547,
-      "logps/rejected": -213.38551330566406,
-      "loss": 0.16482675075531006,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.1806135177612305,
-      "rewards/margins": 3.4487881660461426,
-      "rewards/rejected": -8.629401206970215,
-      "step": 538
-    },
-    {
-      "epoch": 1.738709677419355,
-      "grad_norm": 0.8675318360328674,
-      "learning_rate": 2.1271444478125237e-07,
-      "logits/chosen": -2.329207181930542,
-      "logits/rejected": -2.0246644020080566,
-      "logps/chosen": -138.73007202148438,
-      "logps/rejected": -216.8204345703125,
-      "loss": 0.1617818921804428,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.572966575622559,
-      "rewards/margins": 5.178282737731934,
-      "rewards/rejected": -9.751248359680176,
-      "step": 539
-    },
-    {
-      "epoch": 1.7419354838709677,
-      "grad_norm": 0.7694476842880249,
-      "learning_rate": 2.0763052923447214e-07,
-      "logits/chosen": -2.207973003387451,
-      "logits/rejected": -1.862140417098999,
-      "logps/chosen": -136.8199920654297,
-      "logps/rejected": -210.47402954101562,
-      "loss": 0.1648537665605545,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.768884181976318,
-      "rewards/margins": 4.165104866027832,
-      "rewards/rejected": -8.933990478515625,
-      "step": 540
-    },
-    {
-      "epoch": 1.7451612903225806,
-      "grad_norm": 0.8003881573677063,
-      "learning_rate": 2.026054709494235e-07,
-      "logits/chosen": -2.4957833290100098,
-      "logits/rejected": -2.2322189807891846,
-      "logps/chosen": -150.86410522460938,
-      "logps/rejected": -222.50686645507812,
-      "loss": 0.17683330178260803,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.864772319793701,
-      "rewards/margins": 4.093851089477539,
-      "rewards/rejected": -8.958623886108398,
-      "step": 541
-    },
-    {
-      "epoch": 1.7483870967741937,
-      "grad_norm": 0.6695418357849121,
-      "learning_rate": 1.9763939894595795e-07,
-      "logits/chosen": -1.9199615716934204,
-      "logits/rejected": -1.6785365343093872,
-      "logps/chosen": -173.23507690429688,
-      "logps/rejected": -242.41043090820312,
-      "loss": 0.1188519224524498,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -6.151148796081543,
-      "rewards/margins": 4.460818290710449,
-      "rewards/rejected": -10.611968040466309,
-      "step": 542
-    },
-    {
-      "epoch": 1.7516129032258063,
-      "grad_norm": 0.45999041199684143,
-      "learning_rate": 1.9273244072943814e-07,
-      "logits/chosen": -2.499255895614624,
-      "logits/rejected": -2.0790059566497803,
-      "logps/chosen": -143.73182678222656,
-      "logps/rejected": -218.94778442382812,
-      "loss": 0.07506464421749115,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.273706436157227,
-      "rewards/margins": 4.5024261474609375,
-      "rewards/rejected": -9.776132583618164,
-      "step": 543
-    },
-    {
-      "epoch": 1.7548387096774194,
-      "grad_norm": 0.36987289786338806,
-      "learning_rate": 1.8788472228746135e-07,
-      "logits/chosen": -2.343918800354004,
-      "logits/rejected": -1.8907790184020996,
-      "logps/chosen": -159.0539093017578,
-      "logps/rejected": -232.19241333007812,
-      "loss": 0.08276303112506866,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.670258522033691,
-      "rewards/margins": 4.740736961364746,
-      "rewards/rejected": -10.410995483398438,
-      "step": 544
-    },
-    {
-      "epoch": 1.7580645161290323,
-      "grad_norm": 0.6856768727302551,
-      "learning_rate": 1.830963680866285e-07,
-      "logits/chosen": -2.5793392658233643,
-      "logits/rejected": -2.156630516052246,
-      "logps/chosen": -127.28002166748047,
-      "logps/rejected": -204.24288940429688,
-      "loss": 0.16320952773094177,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -4.079771995544434,
-      "rewards/margins": 4.579602241516113,
-      "rewards/rejected": -8.659374237060547,
-      "step": 545
-    },
-    {
-      "epoch": 1.761290322580645,
-      "grad_norm": 0.8824412226676941,
-      "learning_rate": 1.7836750106934475e-07,
-      "logits/chosen": -2.4374589920043945,
-      "logits/rejected": -2.1196401119232178,
-      "logps/chosen": -159.99725341796875,
-      "logps/rejected": -218.28028869628906,
-      "loss": 0.15264299511909485,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.216504096984863,
-      "rewards/margins": 3.364132881164551,
-      "rewards/rejected": -8.580636978149414,
-      "step": 546
-    },
-    {
-      "epoch": 1.7645161290322582,
-      "grad_norm": 0.49898338317871094,
-      "learning_rate": 1.7369824265066366e-07,
-      "logits/chosen": -2.0640761852264404,
-      "logits/rejected": -1.6560142040252686,
-      "logps/chosen": -155.5439453125,
-      "logps/rejected": -215.54800415039062,
-      "loss": 0.12484726309776306,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -4.699676990509033,
-      "rewards/margins": 3.61027455329895,
-      "rewards/rejected": -8.309951782226562,
-      "step": 547
-    },
-    {
-      "epoch": 1.7677419354838708,
-      "grad_norm": 1.0586748123168945,
-      "learning_rate": 1.6908871271517136e-07,
-      "logits/chosen": -1.8716487884521484,
-      "logits/rejected": -1.5878236293792725,
-      "logps/chosen": -177.3343963623047,
-      "logps/rejected": -235.95562744140625,
-      "loss": 0.28872251510620117,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.277359962463379,
-      "rewards/margins": 4.477985858917236,
-      "rewards/rejected": -10.755345344543457,
-      "step": 548
-    },
-    {
-      "epoch": 1.770967741935484,
-      "grad_norm": 0.6965610980987549,
-      "learning_rate": 1.645390296139074e-07,
-      "logits/chosen": -1.5745385885238647,
-      "logits/rejected": -1.214267373085022,
-      "logps/chosen": -175.0247344970703,
-      "logps/rejected": -248.30154418945312,
-      "loss": 0.14276829361915588,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -6.623499870300293,
-      "rewards/margins": 4.304838180541992,
-      "rewards/rejected": -10.928339004516602,
-      "step": 549
-    },
-    {
-      "epoch": 1.7741935483870968,
-      "grad_norm": 0.4304961860179901,
-      "learning_rate": 1.600493101613268e-07,
-      "logits/chosen": -2.1939003467559814,
-      "logits/rejected": -1.7763912677764893,
-      "logps/chosen": -164.8372344970703,
-      "logps/rejected": -233.04421997070312,
-      "loss": 0.08470362424850464,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.929935455322266,
-      "rewards/margins": 4.351900577545166,
-      "rewards/rejected": -10.281835556030273,
-      "step": 550
-    },
-    {
-      "epoch": 1.7774193548387096,
-      "grad_norm": 0.6119385957717896,
-      "learning_rate": 1.5561966963229925e-07,
-      "logits/chosen": -2.0900025367736816,
-      "logits/rejected": -1.6825987100601196,
-      "logps/chosen": -151.122802734375,
-      "logps/rejected": -216.6434326171875,
-      "loss": 0.1607978641986847,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.659897327423096,
-      "rewards/margins": 3.7245254516601562,
-      "rewards/rejected": -8.384422302246094,
-      "step": 551
-    },
-    {
-      "epoch": 1.7806451612903227,
-      "grad_norm": 0.5334950685501099,
-      "learning_rate": 1.5125022175914993e-07,
-      "logits/chosen": -2.012514352798462,
-      "logits/rejected": -1.6403321027755737,
-      "logps/chosen": -164.17938232421875,
-      "logps/rejected": -229.32122802734375,
-      "loss": 0.11097535490989685,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.964521408081055,
-      "rewards/margins": 4.160644054412842,
-      "rewards/rejected": -9.125164985656738,
-      "step": 552
-    },
-    {
-      "epoch": 1.7838709677419353,
-      "grad_norm": 0.8820840716362,
-      "learning_rate": 1.4694107872874174e-07,
-      "logits/chosen": -1.9263832569122314,
-      "logits/rejected": -1.649777889251709,
-      "logps/chosen": -168.28428649902344,
-      "logps/rejected": -234.61843872070312,
-      "loss": 0.18569695949554443,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -5.886368751525879,
-      "rewards/margins": 4.445061683654785,
-      "rewards/rejected": -10.331430435180664,
-      "step": 553
-    },
-    {
-      "epoch": 1.7870967741935484,
-      "grad_norm": 0.47653189301490784,
-      "learning_rate": 1.4269235117959218e-07,
-      "logits/chosen": -2.272602081298828,
-      "logits/rejected": -2.015300750732422,
-      "logps/chosen": -148.469970703125,
-      "logps/rejected": -214.323974609375,
-      "loss": 0.11035692691802979,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.494589328765869,
-      "rewards/margins": 3.9238247871398926,
-      "rewards/rejected": -8.418414115905762,
-      "step": 554
-    },
-    {
-      "epoch": 1.7903225806451613,
-      "grad_norm": 0.48935648798942566,
-      "learning_rate": 1.3850414819903235e-07,
-      "logits/chosen": -2.5061533451080322,
-      "logits/rejected": -2.0984086990356445,
-      "logps/chosen": -127.44873046875,
-      "logps/rejected": -191.14419555664062,
-      "loss": 0.11538471281528473,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -3.8038997650146484,
-      "rewards/margins": 4.107954978942871,
-      "rewards/rejected": -7.9118547439575195,
-      "step": 555
-    },
-    {
-      "epoch": 1.793548387096774,
-      "grad_norm": 0.5510221719741821,
-      "learning_rate": 1.3437657732040783e-07,
-      "logits/chosen": -2.023258924484253,
-      "logits/rejected": -1.7574663162231445,
-      "logps/chosen": -152.7783203125,
-      "logps/rejected": -229.17276000976562,
-      "loss": 0.09697185456752777,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.315288543701172,
-      "rewards/margins": 4.344141006469727,
-      "rewards/rejected": -9.659428596496582,
-      "step": 556
-    },
-    {
-      "epoch": 1.7967741935483872,
-      "grad_norm": 1.3715373277664185,
-      "learning_rate": 1.3030974452031832e-07,
-      "logits/chosen": -2.4522571563720703,
-      "logits/rejected": -2.184933662414551,
-      "logps/chosen": -160.92742919921875,
-      "logps/rejected": -226.46864318847656,
-      "loss": 0.19726327061653137,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.374527931213379,
-      "rewards/margins": 4.175776958465576,
-      "rewards/rejected": -9.550305366516113,
-      "step": 557
-    },
-    {
-      "epoch": 1.8,
-      "grad_norm": 1.3102973699569702,
-      "learning_rate": 1.2630375421589374e-07,
-      "logits/chosen": -2.0639586448669434,
-      "logits/rejected": -1.6796153783798218,
-      "logps/chosen": -159.63888549804688,
-      "logps/rejected": -221.36654663085938,
-      "loss": 0.3832627832889557,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -5.068192958831787,
-      "rewards/margins": 3.830645799636841,
-      "rewards/rejected": -8.898838996887207,
-      "step": 558
-    },
-    {
-      "epoch": 1.803225806451613,
-      "grad_norm": 0.4269888699054718,
-      "learning_rate": 1.223587092621162e-07,
-      "logits/chosen": -2.1528987884521484,
-      "logits/rejected": -1.7997894287109375,
-      "logps/chosen": -169.46499633789062,
-      "logps/rejected": -241.46487426757812,
-      "loss": 0.09280017018318176,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.810900688171387,
-      "rewards/margins": 4.045185089111328,
-      "rewards/rejected": -9.856085777282715,
-      "step": 559
-    },
-    {
-      "epoch": 1.8064516129032258,
-      "grad_norm": 0.4622645974159241,
-      "learning_rate": 1.1847471094917711e-07,
-      "logits/chosen": -2.6282200813293457,
-      "logits/rejected": -2.194746732711792,
-      "logps/chosen": -116.16856384277344,
-      "logps/rejected": -188.41812133789062,
-      "loss": 0.11614560335874557,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -3.8644115924835205,
-      "rewards/margins": 4.392536640167236,
-      "rewards/rejected": -8.256948471069336,
-      "step": 560
-    },
-    {
-      "epoch": 1.8096774193548386,
-      "grad_norm": 0.8068135976791382,
-      "learning_rate": 1.1465185899987797e-07,
-      "logits/chosen": -2.1790473461151123,
-      "logits/rejected": -1.7810087203979492,
-      "logps/chosen": -153.4788818359375,
-      "logps/rejected": -218.73696899414062,
-      "loss": 0.22984439134597778,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.740139484405518,
-      "rewards/margins": 3.968252658843994,
-      "rewards/rejected": -8.708392143249512,
-      "step": 561
-    },
-    {
-      "epoch": 1.8129032258064517,
-      "grad_norm": 0.5755476355552673,
-      "learning_rate": 1.1089025156706928e-07,
-      "logits/chosen": -2.1778645515441895,
-      "logits/rejected": -1.817617654800415,
-      "logps/chosen": -144.53890991210938,
-      "logps/rejected": -222.02716064453125,
-      "loss": 0.10252237319946289,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.7554545402526855,
-      "rewards/margins": 4.461153984069824,
-      "rewards/rejected": -9.216609001159668,
-      "step": 562
-    },
-    {
-      "epoch": 1.8161290322580645,
-      "grad_norm": 0.7859362363815308,
-      "learning_rate": 1.0718998523113006e-07,
-      "logits/chosen": -2.283513307571411,
-      "logits/rejected": -1.9074876308441162,
-      "logps/chosen": -162.03208923339844,
-      "logps/rejected": -224.34286499023438,
-      "loss": 0.16266335546970367,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.444174766540527,
-      "rewards/margins": 4.131298542022705,
-      "rewards/rejected": -9.57547378540039,
-      "step": 563
-    },
-    {
-      "epoch": 1.8193548387096774,
-      "grad_norm": 0.5560816526412964,
-      "learning_rate": 1.0355115499748936e-07,
-      "logits/chosen": -2.54559588432312,
-      "logits/rejected": -2.2199432849884033,
-      "logps/chosen": -146.66653442382812,
-      "logps/rejected": -212.36798095703125,
-      "loss": 0.11472531408071518,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.957542419433594,
-      "rewards/margins": 4.0164031982421875,
-      "rewards/rejected": -8.973945617675781,
-      "step": 564
-    },
-    {
-      "epoch": 1.8225806451612905,
-      "grad_norm": 0.7271311283111572,
-      "learning_rate": 9.997385429418555e-08,
-      "logits/chosen": -2.290648937225342,
-      "logits/rejected": -2.024205207824707,
-      "logps/chosen": -164.578857421875,
-      "logps/rejected": -206.93765258789062,
-      "loss": 0.20374655723571777,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.871495246887207,
-      "rewards/margins": 2.7768454551696777,
-      "rewards/rejected": -8.648341178894043,
-      "step": 565
-    },
-    {
-      "epoch": 1.8258064516129031,
-      "grad_norm": 0.42589902877807617,
-      "learning_rate": 9.645817496946902e-08,
-      "logits/chosen": -2.3634767532348633,
-      "logits/rejected": -1.9045495986938477,
-      "logps/chosen": -151.49630737304688,
-      "logps/rejected": -217.68666076660156,
-      "loss": 0.08909858763217926,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -4.860687255859375,
-      "rewards/margins": 4.105985641479492,
-      "rewards/rejected": -8.966672897338867,
-      "step": 566
-    },
-    {
-      "epoch": 1.8290322580645162,
-      "grad_norm": 0.8605185151100159,
-      "learning_rate": 9.300420728944132e-08,
-      "logits/chosen": -2.215791702270508,
-      "logits/rejected": -1.8884910345077515,
-      "logps/chosen": -168.70660400390625,
-      "logps/rejected": -231.85137939453125,
-      "loss": 0.2555576264858246,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.966517925262451,
-      "rewards/margins": 3.8281943798065186,
-      "rewards/rejected": -8.79471206665039,
-      "step": 567
-    },
-    {
-      "epoch": 1.832258064516129,
-      "grad_norm": 0.7926627397537231,
-      "learning_rate": 8.961203993574197e-08,
-      "logits/chosen": -2.421079635620117,
-      "logits/rejected": -2.14444637298584,
-      "logps/chosen": -165.5140380859375,
-      "logps/rejected": -218.92263793945312,
-      "loss": 0.18454545736312866,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -5.383303642272949,
-      "rewards/margins": 3.148193359375,
-      "rewards/rejected": -8.53149700164795,
-      "step": 568
-    },
-    {
-      "epoch": 1.835483870967742,
-      "grad_norm": 0.4896734654903412,
-      "learning_rate": 8.628176000326615e-08,
-      "logits/chosen": -2.4777512550354004,
-      "logits/rejected": -2.0716423988342285,
-      "logps/chosen": -161.4052734375,
-      "logps/rejected": -238.86143493652344,
-      "loss": 0.10428795218467712,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.8029398918151855,
-      "rewards/margins": 4.6206560134887695,
-      "rewards/rejected": -10.423595428466797,
-      "step": 569
-    },
-    {
-      "epoch": 1.838709677419355,
-      "grad_norm": 0.552992045879364,
-      "learning_rate": 8.301345299793374e-08,
-      "logits/chosen": -2.6197102069854736,
-      "logits/rejected": -2.2024412155151367,
-      "logps/chosen": -136.18997192382812,
-      "logps/rejected": -202.0336151123047,
-      "loss": 0.17170380055904388,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.436511516571045,
-      "rewards/margins": 4.078784942626953,
-      "rewards/rejected": -8.515296936035156,
-      "step": 570
-    },
-    {
-      "epoch": 1.8419354838709676,
-      "grad_norm": 0.8881261944770813,
-      "learning_rate": 7.980720283448957e-08,
-      "logits/chosen": -1.9090025424957275,
-      "logits/rejected": -1.6317459344863892,
-      "logps/chosen": -166.48141479492188,
-      "logps/rejected": -233.93344116210938,
-      "loss": 0.21583732962608337,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.640261650085449,
-      "rewards/margins": 4.06472635269165,
-      "rewards/rejected": -9.704988479614258,
-      "step": 571
-    },
-    {
-      "epoch": 1.8451612903225807,
-      "grad_norm": 0.5913529396057129,
-      "learning_rate": 7.66630918343511e-08,
-      "logits/chosen": -2.508117198944092,
-      "logits/rejected": -2.0965323448181152,
-      "logps/chosen": -141.76434326171875,
-      "logps/rejected": -216.82269287109375,
-      "loss": 0.12084457278251648,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.573372840881348,
-      "rewards/margins": 4.82564115524292,
-      "rewards/rejected": -9.399014472961426,
-      "step": 572
-    },
-    {
-      "epoch": 1.8483870967741935,
-      "grad_norm": 0.5654717087745667,
-      "learning_rate": 7.358120072349594e-08,
-      "logits/chosen": -2.3709940910339355,
-      "logits/rejected": -2.038872718811035,
-      "logps/chosen": -153.72909545898438,
-      "logps/rejected": -225.5278778076172,
-      "loss": 0.13232877850532532,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.121915340423584,
-      "rewards/margins": 4.319830894470215,
-      "rewards/rejected": -9.44174575805664,
-      "step": 573
-    },
-    {
-      "epoch": 1.8516129032258064,
-      "grad_norm": 0.8132126331329346,
-      "learning_rate": 7.056160863038564e-08,
-      "logits/chosen": -2.2554118633270264,
-      "logits/rejected": -1.993767261505127,
-      "logps/chosen": -158.86770629882812,
-      "logps/rejected": -216.81568908691406,
-      "loss": 0.1489611566066742,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.455775737762451,
-      "rewards/margins": 3.815854072570801,
-      "rewards/rejected": -9.27163028717041,
-      "step": 574
-    },
-    {
-      "epoch": 1.8548387096774195,
-      "grad_norm": 0.6348745822906494,
-      "learning_rate": 6.760439308393763e-08,
-      "logits/chosen": -2.1734232902526855,
-      "logits/rejected": -1.7729003429412842,
-      "logps/chosen": -137.09434509277344,
-      "logps/rejected": -200.48419189453125,
-      "loss": 0.16260844469070435,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -4.978243350982666,
-      "rewards/margins": 4.0154242515563965,
-      "rewards/rejected": -8.993667602539062,
-      "step": 575
-    },
-    {
-      "epoch": 1.8580645161290321,
-      "grad_norm": 0.5179709196090698,
-      "learning_rate": 6.470963001153268e-08,
-      "logits/chosen": -2.1407740116119385,
-      "logits/rejected": -1.700477123260498,
-      "logps/chosen": -152.31427001953125,
-      "logps/rejected": -230.4638671875,
-      "loss": 0.10425004363059998,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.169086933135986,
-      "rewards/margins": 4.689014434814453,
-      "rewards/rejected": -9.858100891113281,
-      "step": 576
-    },
-    {
-      "epoch": 1.8612903225806452,
-      "grad_norm": 0.8017939329147339,
-      "learning_rate": 6.187739373706508e-08,
-      "logits/chosen": -2.4910433292388916,
-      "logits/rejected": -2.189812660217285,
-      "logps/chosen": -161.86709594726562,
-      "logps/rejected": -220.63807678222656,
-      "loss": 0.20899598300457,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.962458610534668,
-      "rewards/margins": 3.5529963970184326,
-      "rewards/rejected": -8.515454292297363,
-      "step": 577
-    },
-    {
-      "epoch": 1.864516129032258,
-      "grad_norm": 0.8252779841423035,
-      "learning_rate": 5.9107756979036035e-08,
-      "logits/chosen": -1.9336351156234741,
-      "logits/rejected": -1.6209434270858765,
-      "logps/chosen": -185.7202606201172,
-      "logps/rejected": -260.7825927734375,
-      "loss": 0.13948240876197815,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -7.039210319519043,
-      "rewards/margins": 4.54232120513916,
-      "rewards/rejected": -11.581531524658203,
-      "step": 578
-    },
-    {
-      "epoch": 1.867741935483871,
-      "grad_norm": 0.4776524603366852,
-      "learning_rate": 5.6400790848686325e-08,
-      "logits/chosen": -2.4437499046325684,
-      "logits/rejected": -2.0726723670959473,
-      "logps/chosen": -157.41360473632812,
-      "logps/rejected": -224.64364624023438,
-      "loss": 0.12375371158123016,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.997191905975342,
-      "rewards/margins": 4.007498264312744,
-      "rewards/rejected": -9.004690170288086,
-      "step": 579
-    },
-    {
-      "epoch": 1.870967741935484,
-      "grad_norm": 0.9288763999938965,
-      "learning_rate": 5.3756564848168325e-08,
-      "logits/chosen": -2.0364584922790527,
-      "logits/rejected": -1.7277495861053467,
-      "logps/chosen": -169.73043823242188,
-      "logps/rejected": -241.12673950195312,
-      "loss": 0.17077437043190002,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.451491832733154,
-      "rewards/margins": 4.377066135406494,
-      "rewards/rejected": -9.828557968139648,
-      "step": 580
-    },
-    {
-      "epoch": 1.8741935483870966,
-      "grad_norm": 0.4808553159236908,
-      "learning_rate": 5.117514686876379e-08,
-      "logits/chosen": -2.3133416175842285,
-      "logits/rejected": -1.9516518115997314,
-      "logps/chosen": -145.88031005859375,
-      "logps/rejected": -223.6663818359375,
-      "loss": 0.0891776904463768,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.209961414337158,
-      "rewards/margins": 4.986030578613281,
-      "rewards/rejected": -10.195992469787598,
-      "step": 581
-    },
-    {
-      "epoch": 1.8774193548387097,
-      "grad_norm": 0.8164253234863281,
-      "learning_rate": 4.8656603189140564e-08,
-      "logits/chosen": -2.275332450866699,
-      "logits/rejected": -1.8718898296356201,
-      "logps/chosen": -138.21881103515625,
-      "logps/rejected": -209.18975830078125,
-      "loss": 0.18211618065834045,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.704702377319336,
-      "rewards/margins": 4.319535732269287,
-      "rewards/rejected": -9.024238586425781,
-      "step": 582
-    },
-    {
-      "epoch": 1.8806451612903226,
-      "grad_norm": 0.5213284492492676,
-      "learning_rate": 4.620099847364973e-08,
-      "logits/chosen": -2.350086212158203,
-      "logits/rejected": -1.9384825229644775,
-      "logps/chosen": -149.28497314453125,
-      "logps/rejected": -218.31735229492188,
-      "loss": 0.0979819968342781,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.209005355834961,
-      "rewards/margins": 3.8068172931671143,
-      "rewards/rejected": -9.015822410583496,
-      "step": 583
-    },
-    {
-      "epoch": 1.8838709677419354,
-      "grad_norm": 0.5111794471740723,
-      "learning_rate": 4.380839577066587e-08,
-      "logits/chosen": -2.0842952728271484,
-      "logits/rejected": -1.742661952972412,
-      "logps/chosen": -152.95252990722656,
-      "logps/rejected": -217.0407257080078,
-      "loss": 0.10743395984172821,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.22687292098999,
-      "rewards/margins": 3.8219170570373535,
-      "rewards/rejected": -9.048789978027344,
-      "step": 584
-    },
-    {
-      "epoch": 1.8870967741935485,
-      "grad_norm": 0.6543387174606323,
-      "learning_rate": 4.147885651096861e-08,
-      "logits/chosen": -2.292243480682373,
-      "logits/rejected": -1.9942082166671753,
-      "logps/chosen": -148.08607482910156,
-      "logps/rejected": -208.68533325195312,
-      "loss": 0.14559054374694824,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.75385856628418,
-      "rewards/margins": 3.6896753311157227,
-      "rewards/rejected": -8.443533897399902,
-      "step": 585
-    },
-    {
-      "epoch": 1.8903225806451613,
-      "grad_norm": 0.5183025002479553,
-      "learning_rate": 3.9212440506164465e-08,
-      "logits/chosen": -2.3551645278930664,
-      "logits/rejected": -1.9850412607192993,
-      "logps/chosen": -156.36065673828125,
-      "logps/rejected": -223.7450714111328,
-      "loss": 0.104829341173172,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.985340595245361,
-      "rewards/margins": 3.9502832889556885,
-      "rewards/rejected": -8.935624122619629,
-      "step": 586
-    },
-    {
-      "epoch": 1.8935483870967742,
-      "grad_norm": 0.5007240176200867,
-      "learning_rate": 3.7009205947153294e-08,
-      "logits/chosen": -2.016148090362549,
-      "logits/rejected": -1.6376087665557861,
-      "logps/chosen": -167.86480712890625,
-      "logps/rejected": -239.8389129638672,
-      "loss": 0.07380706071853638,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.1963348388671875,
-      "rewards/margins": 4.371429920196533,
-      "rewards/rejected": -9.567764282226562,
-      "step": 587
-    },
-    {
-      "epoch": 1.896774193548387,
-      "grad_norm": 0.6565898060798645,
-      "learning_rate": 3.486920940263094e-08,
-      "logits/chosen": -2.4202284812927246,
-      "logits/rejected": -2.0095739364624023,
-      "logps/chosen": -150.2890167236328,
-      "logps/rejected": -225.1298828125,
-      "loss": 0.12143023312091827,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.002315044403076,
-      "rewards/margins": 4.677301406860352,
-      "rewards/rejected": -9.679616928100586,
-      "step": 588
-    },
-    {
-      "epoch": 1.9,
-      "grad_norm": 0.7280709147453308,
-      "learning_rate": 3.279250581763982e-08,
-      "logits/chosen": -2.5148582458496094,
-      "logits/rejected": -2.265883207321167,
-      "logps/chosen": -165.6021728515625,
-      "logps/rejected": -224.98973083496094,
-      "loss": 0.19175627827644348,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.013904571533203,
-      "rewards/margins": 3.702694892883301,
-      "rewards/rejected": -8.716598510742188,
-      "step": 589
-    },
-    {
-      "epoch": 1.903225806451613,
-      "grad_norm": 0.8753358721733093,
-      "learning_rate": 3.077914851215585e-08,
-      "logits/chosen": -2.1750917434692383,
-      "logits/rejected": -1.677757740020752,
-      "logps/chosen": -157.30673217773438,
-      "logps/rejected": -259.028076171875,
-      "loss": 0.18018171191215515,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.870169639587402,
-      "rewards/margins": 5.68065881729126,
-      "rewards/rejected": -10.550827980041504,
-      "step": 590
-    },
-    {
-      "epoch": 1.9064516129032258,
-      "grad_norm": 0.6306760311126709,
-      "learning_rate": 2.8829189179721552e-08,
-      "logits/chosen": -2.1184749603271484,
-      "logits/rejected": -1.7403565645217896,
-      "logps/chosen": -165.62957763671875,
-      "logps/rejected": -245.212158203125,
-      "loss": 0.10819903016090393,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.537095069885254,
-      "rewards/margins": 5.161804676055908,
-      "rewards/rejected": -10.69890022277832,
-      "step": 591
-    },
-    {
-      "epoch": 1.9096774193548387,
-      "grad_norm": 0.6319319009780884,
-      "learning_rate": 2.6942677886117597e-08,
-      "logits/chosen": -2.0213987827301025,
-      "logits/rejected": -1.6281509399414062,
-      "logps/chosen": -151.06640625,
-      "logps/rejected": -223.082275390625,
-      "loss": 0.14870238304138184,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -4.421160697937012,
-      "rewards/margins": 4.47739315032959,
-      "rewards/rejected": -8.898553848266602,
-      "step": 592
-    },
-    {
-      "epoch": 1.9129032258064518,
-      "grad_norm": 0.5766472816467285,
-      "learning_rate": 2.5119663068077227e-08,
-      "logits/chosen": -2.718550443649292,
-      "logits/rejected": -2.379741668701172,
-      "logps/chosen": -138.78890991210938,
-      "logps/rejected": -220.3733673095703,
-      "loss": 0.12025254219770432,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.328763961791992,
-      "rewards/margins": 5.262520790100098,
-      "rewards/rejected": -9.59128475189209,
-      "step": 593
-    },
-    {
-      "epoch": 1.9161290322580644,
-      "grad_norm": 0.865578830242157,
-      "learning_rate": 2.3360191532043053e-08,
-      "logits/chosen": -2.043041229248047,
-      "logits/rejected": -1.6594171524047852,
-      "logps/chosen": -142.37728881835938,
-      "logps/rejected": -221.1278076171875,
-      "loss": 0.18678346276283264,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -5.289588928222656,
-      "rewards/margins": 4.628904342651367,
-      "rewards/rejected": -9.91849422454834,
-      "step": 594
-    },
-    {
-      "epoch": 1.9193548387096775,
-      "grad_norm": 0.5385801196098328,
-      "learning_rate": 2.1664308452965798e-08,
-      "logits/chosen": -2.386172294616699,
-      "logits/rejected": -2.0510740280151367,
-      "logps/chosen": -153.6859130859375,
-      "logps/rejected": -218.6775360107422,
-      "loss": 0.12221887707710266,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.658649444580078,
-      "rewards/margins": 4.1918463706970215,
-      "rewards/rejected": -9.850496292114258,
-      "step": 595
-    },
-    {
-      "epoch": 1.9225806451612903,
-      "grad_norm": 0.31810611486434937,
-      "learning_rate": 2.0032057373142453e-08,
-      "logits/chosen": -2.2256276607513428,
-      "logits/rejected": -1.9017874002456665,
-      "logps/chosen": -160.43661499023438,
-      "logps/rejected": -223.5597381591797,
-      "loss": 0.060307130217552185,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.350681781768799,
-      "rewards/margins": 4.087976455688477,
-      "rewards/rejected": -9.438658714294434,
-      "step": 596
-    },
-    {
-      "epoch": 1.9258064516129032,
-      "grad_norm": 0.6564087867736816,
-      "learning_rate": 1.8463480201101337e-08,
-      "logits/chosen": -2.4924027919769287,
-      "logits/rejected": -2.1421899795532227,
-      "logps/chosen": -145.89862060546875,
-      "logps/rejected": -214.23382568359375,
-      "loss": 0.15689480304718018,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.8156819343566895,
-      "rewards/margins": 4.231740951538086,
-      "rewards/rejected": -9.047422409057617,
-      "step": 597
-    },
-    {
-      "epoch": 1.9290322580645163,
-      "grad_norm": 0.7558223605155945,
-      "learning_rate": 1.6958617210522942e-08,
-      "logits/chosen": -2.3059439659118652,
-      "logits/rejected": -1.9703770875930786,
-      "logps/chosen": -179.74252319335938,
-      "logps/rejected": -242.72195434570312,
-      "loss": 0.15578067302703857,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.681976318359375,
-      "rewards/margins": 4.2997260093688965,
-      "rewards/rejected": -9.98170280456543,
-      "step": 598
-    },
-    {
-      "epoch": 1.932258064516129,
-      "grad_norm": 0.6187397241592407,
-      "learning_rate": 1.5517507039208568e-08,
-      "logits/chosen": -1.8563120365142822,
-      "logits/rejected": -1.6304610967636108,
-      "logps/chosen": -172.39527893066406,
-      "logps/rejected": -239.09451293945312,
-      "loss": 0.12176580727100372,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.339761734008789,
-      "rewards/margins": 3.85723614692688,
-      "rewards/rejected": -9.19699764251709,
-      "step": 599
-    },
-    {
-      "epoch": 1.935483870967742,
-      "grad_norm": 0.5930323600769043,
-      "learning_rate": 1.4140186688086365e-08,
-      "logits/chosen": -2.1559739112854004,
-      "logits/rejected": -1.8187296390533447,
-      "logps/chosen": -153.97042846679688,
-      "logps/rejected": -229.720947265625,
-      "loss": 0.10695869475603104,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.2712507247924805,
-      "rewards/margins": 4.516932487487793,
-      "rewards/rejected": -9.788183212280273,
-      "step": 600
-    },
-    {
-      "epoch": 1.9387096774193548,
-      "grad_norm": 0.41866037249565125,
-      "learning_rate": 1.2826691520262114e-08,
-      "logits/chosen": -2.31526517868042,
-      "logits/rejected": -2.0141377449035645,
-      "logps/chosen": -157.0247802734375,
-      "logps/rejected": -221.9925079345703,
-      "loss": 0.11068223416805267,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.260191917419434,
-      "rewards/margins": 3.905627489089966,
-      "rewards/rejected": -9.16581916809082,
-      "step": 601
-    },
-    {
-      "epoch": 1.9419354838709677,
-      "grad_norm": 1.0719863176345825,
-      "learning_rate": 1.1577055260111603e-08,
-      "logits/chosen": -2.173069715499878,
-      "logits/rejected": -1.8304716348648071,
-      "logps/chosen": -166.02662658691406,
-      "logps/rejected": -226.69769287109375,
-      "loss": 0.28377968072891235,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -5.536633014678955,
-      "rewards/margins": 3.4802064895629883,
-      "rewards/rejected": -9.016839981079102,
-      "step": 602
-    },
-    {
-      "epoch": 1.9451612903225808,
-      "grad_norm": 0.8208643794059753,
-      "learning_rate": 1.0391309992413834e-08,
-      "logits/chosen": -2.191689968109131,
-      "logits/rejected": -1.8869071006774902,
-      "logps/chosen": -150.7781219482422,
-      "logps/rejected": -217.19967651367188,
-      "loss": 0.20699000358581543,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.127926826477051,
-      "rewards/margins": 3.8796911239624023,
-      "rewards/rejected": -9.007617950439453,
-      "step": 603
-    },
-    {
-      "epoch": 1.9483870967741934,
-      "grad_norm": 0.913461446762085,
-      "learning_rate": 9.269486161528063e-09,
-      "logits/chosen": -2.360179901123047,
-      "logits/rejected": -1.9940139055252075,
-      "logps/chosen": -161.97451782226562,
-      "logps/rejected": -243.32638549804688,
-      "loss": 0.1867237687110901,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.740466594696045,
-      "rewards/margins": 4.959558486938477,
-      "rewards/rejected": -10.700023651123047,
-      "step": 604
-    },
-    {
-      "epoch": 1.9516129032258065,
-      "grad_norm": 0.4413685202598572,
-      "learning_rate": 8.211612570611927e-09,
-      "logits/chosen": -2.2911806106567383,
-      "logits/rejected": -1.9480736255645752,
-      "logps/chosen": -145.26788330078125,
-      "logps/rejected": -212.7537384033203,
-      "loss": 0.099663145840168,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.860638618469238,
-      "rewards/margins": 3.9052321910858154,
-      "rewards/rejected": -8.765871047973633,
-      "step": 605
-    },
-    {
-      "epoch": 1.9548387096774194,
-      "grad_norm": 0.7804948091506958,
-      "learning_rate": 7.217716380881479e-09,
-      "logits/chosen": -2.3659746646881104,
-      "logits/rejected": -1.9991470575332642,
-      "logps/chosen": -147.31109619140625,
-      "logps/rejected": -211.473876953125,
-      "loss": 0.21457377076148987,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.22733211517334,
-      "rewards/margins": 4.1717915534973145,
-      "rewards/rejected": -8.399125099182129,
-      "step": 606
-    },
-    {
-      "epoch": 1.9580645161290322,
-      "grad_norm": 0.29649555683135986,
-      "learning_rate": 6.287823110914248e-09,
-      "logits/chosen": -1.943648099899292,
-      "logits/rejected": -1.5592997074127197,
-      "logps/chosen": -172.8476104736328,
-      "logps/rejected": -252.05905151367188,
-      "loss": 0.0389375314116478,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -5.832505226135254,
-      "rewards/margins": 5.257543563842773,
-      "rewards/rejected": -11.090048789978027,
-      "step": 607
-    },
-    {
-      "epoch": 1.9612903225806453,
-      "grad_norm": 0.5518164038658142,
-      "learning_rate": 5.4219566359939305e-09,
-      "logits/chosen": -2.3233723640441895,
-      "logits/rejected": -2.0299534797668457,
-      "logps/chosen": -174.52838134765625,
-      "logps/rejected": -244.19309997558594,
-      "loss": 0.12132851779460907,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.807446479797363,
-      "rewards/margins": 4.241092681884766,
-      "rewards/rejected": -10.048539161682129,
-      "step": 608
-    },
-    {
-      "epoch": 1.964516129032258,
-      "grad_norm": 0.5421614646911621,
-      "learning_rate": 4.620139187497818e-09,
-      "logits/chosen": -2.4302639961242676,
-      "logits/rejected": -2.0234873294830322,
-      "logps/chosen": -138.5733642578125,
-      "logps/rejected": -205.07244873046875,
-      "loss": 0.09998336434364319,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -3.7667317390441895,
-      "rewards/margins": 4.176835060119629,
-      "rewards/rejected": -7.94356632232666,
-      "step": 609
-    },
-    {
-      "epoch": 1.967741935483871,
-      "grad_norm": 1.0250368118286133,
-      "learning_rate": 3.882391352324766e-09,
-      "logits/chosen": -2.216160297393799,
-      "logits/rejected": -1.7999509572982788,
-      "logps/chosen": -148.7297821044922,
-      "logps/rejected": -213.1514892578125,
-      "loss": 0.2962940037250519,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.382504940032959,
-      "rewards/margins": 4.019059181213379,
-      "rewards/rejected": -9.40156364440918,
-      "step": 610
-    },
-    {
-      "epoch": 1.9709677419354839,
-      "grad_norm": 0.4548644423484802,
-      "learning_rate": 3.208732072368104e-09,
-      "logits/chosen": -2.128699779510498,
-      "logits/rejected": -1.7592086791992188,
-      "logps/chosen": -170.3048858642578,
-      "logps/rejected": -246.21893310546875,
-      "loss": 0.06920068711042404,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.777827262878418,
-      "rewards/margins": 4.816149711608887,
-      "rewards/rejected": -10.593976974487305,
-      "step": 611
-    },
-    {
-      "epoch": 1.9741935483870967,
-      "grad_norm": 0.6790652871131897,
-      "learning_rate": 2.5991786440282553e-09,
-      "logits/chosen": -2.0420777797698975,
-      "logits/rejected": -1.7261956930160522,
-      "logps/chosen": -162.41305541992188,
-      "logps/rejected": -225.78868103027344,
-      "loss": 0.11259308457374573,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.203654766082764,
-      "rewards/margins": 3.9687769412994385,
-      "rewards/rejected": -9.172431945800781,
-      "step": 612
-    },
-    {
-      "epoch": 1.9774193548387098,
-      "grad_norm": 0.4410402476787567,
-      "learning_rate": 2.0537467177692007e-09,
-      "logits/chosen": -2.582838296890259,
-      "logits/rejected": -2.1445116996765137,
-      "logps/chosen": -147.5789794921875,
-      "logps/rejected": -225.4355010986328,
-      "loss": 0.08542226999998093,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -4.422325134277344,
-      "rewards/margins": 5.232996940612793,
-      "rewards/rejected": -9.655322074890137,
-      "step": 613
-    },
-    {
-      "epoch": 1.9806451612903224,
-      "grad_norm": 0.8383966684341431,
-      "learning_rate": 1.5724502977162993e-09,
-      "logits/chosen": -2.198012590408325,
-      "logits/rejected": -1.8719542026519775,
-      "logps/chosen": -147.04000854492188,
-      "logps/rejected": -218.75167846679688,
-      "loss": 0.15390494465827942,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.836050033569336,
-      "rewards/margins": 4.455644607543945,
-      "rewards/rejected": -9.291694641113281,
-      "step": 614
-    },
-    {
-      "epoch": 1.9838709677419355,
-      "grad_norm": 0.7288622260093689,
-      "learning_rate": 1.1553017412971323e-09,
-      "logits/chosen": -2.1771492958068848,
-      "logits/rejected": -1.8729420900344849,
-      "logps/chosen": -152.79205322265625,
-      "logps/rejected": -225.03131103515625,
-      "loss": 0.16490189731121063,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.371936321258545,
-      "rewards/margins": 4.25457239151001,
-      "rewards/rejected": -9.626508712768555,
-      "step": 615
-    },
-    {
-      "epoch": 1.9870967741935484,
-      "grad_norm": 0.8851467370986938,
-      "learning_rate": 8.023117589237017e-10,
-      "logits/chosen": -2.1717655658721924,
-      "logits/rejected": -1.710573434829712,
-      "logps/chosen": -156.8731689453125,
-      "logps/rejected": -235.35928344726562,
-      "loss": 0.2400968074798584,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.178656101226807,
-      "rewards/margins": 4.434657573699951,
-      "rewards/rejected": -9.613313674926758,
-      "step": 616
-    },
-    {
-      "epoch": 1.9903225806451612,
-      "grad_norm": 1.0094678401947021,
-      "learning_rate": 5.13489413717927e-10,
-      "logits/chosen": -2.0871119499206543,
-      "logits/rejected": -1.812303900718689,
-      "logps/chosen": -159.29959106445312,
-      "logps/rejected": -223.72409057617188,
-      "loss": 0.2709569036960602,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -5.2238240242004395,
-      "rewards/margins": 3.8848769664764404,
-      "rewards/rejected": -9.1087007522583,
-      "step": 617
-    },
-    {
-      "epoch": 1.9935483870967743,
-      "grad_norm": 0.7299178242683411,
-      "learning_rate": 2.8884212127849867e-10,
-      "logits/chosen": -2.3765387535095215,
-      "logits/rejected": -1.9941563606262207,
-      "logps/chosen": -150.97113037109375,
-      "logps/rejected": -219.23605346679688,
-      "loss": 0.20615136623382568,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.328438758850098,
-      "rewards/margins": 3.9626264572143555,
-      "rewards/rejected": -9.291065216064453,
-      "step": 618
-    },
-    {
-      "epoch": 1.9967741935483871,
-      "grad_norm": 0.5003857612609863,
-      "learning_rate": 1.2837564949103043e-10,
-      "logits/chosen": -2.0507559776306152,
-      "logits/rejected": -1.7051759958267212,
-      "logps/chosen": -150.3610382080078,
-      "logps/rejected": -212.7919464111328,
-      "loss": 0.11825229227542877,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -4.341998100280762,
-      "rewards/margins": 3.884814500808716,
-      "rewards/rejected": -8.226812362670898,
-      "step": 619
-    },
-    {
-      "epoch": 2.0,
-      "grad_norm": 0.8774574995040894,
-      "learning_rate": 3.2094118379288885e-11,
-      "logits/chosen": -2.027413845062256,
-      "logits/rejected": -1.7117795944213867,
-      "logps/chosen": -164.06918334960938,
-      "logps/rejected": -234.24671936035156,
-      "loss": 0.17131692171096802,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.142941474914551,
-      "rewards/margins": 4.366891860961914,
-      "rewards/rejected": -9.509833335876465,
-      "step": 620
-    },
-    {
-      "epoch": 2.0,
-      "step": 620,
-      "total_flos": 1.1218583388589916e+18,
-      "train_loss": 0.2873070158064365,
-      "train_runtime": 4919.0164,
-      "train_samples_per_second": 4.033,
-      "train_steps_per_second": 0.126
-    }
-  ],
-  "logging_steps": 1,
-  "max_steps": 620,
-  "num_input_tokens_seen": 0,
-  "num_train_epochs": 2,
-  "save_steps": 5000,
-  "stateful_callbacks": {
-    "TrainerControl": {
-      "args": {
-        "should_epoch_stop": false,
-        "should_evaluate": false,
-        "should_log": false,
-        "should_save": true,
-        "should_training_stop": true
-      },
-      "attributes": {}
-    }
-  },
-  "total_flos": 1.1218583388589916e+18,
-  "train_batch_size": 1,
-  "trial_name": null,
-  "trial_params": null
-}