{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 8.530250481695568,
  "eval_steps": 500,
  "global_step": 16600,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio": 0.0,
      "completion_length": 27.6790625,
      "epoch": 0.10276172125883108,
      "grad_norm": 5.0809830592989,
      "kl": 2.854698256850243,
      "learning_rate": 9.897225077081192e-07,
      "loss": 0.1145,
      "reward": 0.7617031238693744,
      "reward_std": 0.08005153443926247,
      "rewards/iqa_reward_score": 0.7617031238693744,
      "step": 200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.8296875,
      "epoch": 0.20552344251766216,
      "grad_norm": 9.623104019454459,
      "kl": 0.835672607421875,
      "learning_rate": 9.794450154162384e-07,
      "loss": 0.0334,
      "reward": 0.8317257825657726,
      "reward_std": 0.05232097678628634,
      "rewards/iqa_reward_score": 0.8317257825657726,
      "step": 400
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.5725,
      "epoch": 0.25690430314707774,
      "grad_norm": 13.777697334252704,
      "kl": 1.6955078125,
      "learning_rate": 9.74306269270298e-07,
      "loss": 0.0678,
      "reward": 0.8289125004410743,
      "reward_std": 0.054873663182370365,
      "rewards/iqa_reward_score": 0.8289125004410743,
      "step": 500
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 25.366875,
      "epoch": 0.30828516377649323,
      "grad_norm": 4.847558824602628,
      "kl": 0.994892578125,
      "learning_rate": 9.691675231243576e-07,
      "loss": 0.0398,
      "reward": 0.8202585917711258,
      "reward_std": 0.07689082582248374,
      "rewards/iqa_reward_score": 0.8202585917711258,
      "step": 600
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.56625,
      "epoch": 0.4110468850353243,
      "grad_norm": 4.395243940209246,
      "kl": 0.655250244140625,
      "learning_rate": 9.58890030832477e-07,
      "loss": 0.0262,
      "reward": 0.8299445299804211,
      "reward_std": 0.06683978972781915,
      "rewards/iqa_reward_score": 0.8299445299804211,
      "step": 800
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.8171875,
      "epoch": 0.5138086062941555,
      "grad_norm": 16.05160688969383,
      "kl": 0.8154248046875,
      "learning_rate": 9.486125385405961e-07,
      "loss": 0.0326,
      "reward": 0.8257433583214879,
      "reward_std": 0.06838053988220054,
      "rewards/iqa_reward_score": 0.8257433583214879,
      "step": 1000
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.175,
      "epoch": 0.6165703275529865,
      "grad_norm": 27.21633422585466,
      "kl": 0.713743896484375,
      "learning_rate": 9.383350462487153e-07,
      "loss": 0.0286,
      "reward": 0.8305933582037688,
      "reward_std": 0.06543343772966181,
      "rewards/iqa_reward_score": 0.8305933582037688,
      "step": 1200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 27.2303125,
      "epoch": 0.7193320488118176,
      "grad_norm": 3.4545070827612996,
      "kl": 0.751297607421875,
      "learning_rate": 9.280575539568345e-07,
      "loss": 0.03,
      "reward": 0.8306282886117696,
      "reward_std": 0.06594495058219764,
      "rewards/iqa_reward_score": 0.8306282886117696,
      "step": 1400
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.125625,
      "epoch": 0.7707129094412332,
      "grad_norm": 7.5528485456603915,
      "kl": 0.73730712890625,
      "learning_rate": 9.229188078108942e-07,
      "loss": 0.0295,
      "reward": 0.8229234362393618,
      "reward_std": 0.055542698715289586,
      "rewards/iqa_reward_score": 0.8229234362393618,
      "step": 1500
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.258125,
      "epoch": 0.8220937700706487,
      "grad_norm": 23.45674070812895,
      "kl": 0.69072021484375,
      "learning_rate": 9.177800616649537e-07,
      "loss": 0.0276,
      "reward": 0.832437502220273,
      "reward_std": 0.06442531289503677,
      "rewards/iqa_reward_score": 0.832437502220273,
      "step": 1600
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 27.23,
      "epoch": 0.9248554913294798,
      "grad_norm": 12.52502343446259,
      "kl": 0.839024658203125,
      "learning_rate": 9.075025693730729e-07,
      "loss": 0.0336,
      "reward": 0.8224937494471669,
      "reward_std": 0.08065707363217371,
      "rewards/iqa_reward_score": 0.8224937494471669,
      "step": 1800
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 27.955625,
      "epoch": 1.0277456647398844,
      "grad_norm": 5.6579978426164885,
      "kl": 0.96297119140625,
      "learning_rate": 8.972250770811921e-07,
      "loss": 0.0385,
      "reward": 0.832288283109665,
      "reward_std": 0.06895677681895904,
      "rewards/iqa_reward_score": 0.832288283109665,
      "step": 2000
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.1115625,
      "epoch": 1.1305073859987154,
      "grad_norm": 12.39659796004838,
      "kl": 0.695694580078125,
      "learning_rate": 8.869475847893114e-07,
      "loss": 0.0278,
      "reward": 0.8560353901609778,
      "reward_std": 0.06492814173863735,
      "rewards/iqa_reward_score": 0.8560353901609778,
      "step": 2200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.026875,
      "epoch": 1.2332691072575466,
      "grad_norm": 7.918243884203464,
      "kl": 0.668778076171875,
      "learning_rate": 8.766700924974306e-07,
      "loss": 0.0268,
      "reward": 0.8575455713272094,
      "reward_std": 0.05481337786943186,
      "rewards/iqa_reward_score": 0.8575455713272094,
      "step": 2400
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.2875,
      "epoch": 1.284649967886962,
      "grad_norm": 8.951939513541785,
      "kl": 0.63311767578125,
      "learning_rate": 8.715313463514901e-07,
      "loss": 0.0253,
      "reward": 0.8540877342224121,
      "reward_std": 0.05867271815717686,
      "rewards/iqa_reward_score": 0.8540877342224121,
      "step": 2500
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.24,
      "epoch": 1.3360308285163778,
      "grad_norm": 61.12484483215961,
      "kl": 0.57617919921875,
      "learning_rate": 8.663926002055498e-07,
      "loss": 0.023,
      "reward": 0.8668073464185,
      "reward_std": 0.04293165822920855,
      "rewards/iqa_reward_score": 0.8668073464185,
      "step": 2600
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.4415625,
      "epoch": 1.4387925497752088,
      "grad_norm": 9.595515127565994,
      "kl": 0.912845458984375,
      "learning_rate": 8.56115107913669e-07,
      "loss": 0.0365,
      "reward": 1.8409671924263238,
      "reward_std": 0.10609373373445123,
      "rewards/format_reward": 0.98,
      "rewards/iqa_reward_score": 0.8609671889990568,
      "step": 2800
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.39375,
      "epoch": 1.5415542710340397,
      "grad_norm": 14.47016329543208,
      "kl": 0.811949462890625,
      "learning_rate": 8.458376156217882e-07,
      "loss": 0.0325,
      "reward": 1.8501656317710877,
      "reward_std": 0.08619433440471766,
      "rewards/format_reward": 0.9828125,
      "rewards/iqa_reward_score": 0.8673531262949109,
      "step": 3000
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.2296875,
      "epoch": 1.644315992292871,
      "grad_norm": 38.07779098696265,
      "kl": 0.782005615234375,
      "learning_rate": 8.355601233299074e-07,
      "loss": 0.0313,
      "reward": 1.8367290687561035,
      "reward_std": 0.10177788778877585,
      "rewards/format_reward": 0.980625,
      "rewards/iqa_reward_score": 0.8561040629819036,
      "step": 3200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.8190625,
      "epoch": 1.747077713551702,
      "grad_norm": 7.776135628712392,
      "kl": 0.9329833984375,
      "learning_rate": 8.252826310380266e-07,
      "loss": 0.0373,
      "reward": 1.84249222189188,
      "reward_std": 0.09717632832071103,
      "rewards/format_reward": 0.98125,
      "rewards/iqa_reward_score": 0.8612422209605575,
      "step": 3400
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.059375,
      "epoch": 1.7984585741811174,
      "grad_norm": 14.07929192854843,
      "kl": 0.868226318359375,
      "learning_rate": 8.201438848920863e-07,
      "loss": 0.0347,
      "reward": 1.837091253399849,
      "reward_std": 0.08139224100508727,
      "rewards/format_reward": 0.98875,
      "rewards/iqa_reward_score": 0.8483412505686283,
      "step": 3500
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 27.99375,
      "epoch": 1.8498394348105331,
      "grad_norm": 11.735278559157656,
      "kl": 0.7970263671875,
      "learning_rate": 8.150051387461459e-07,
      "loss": 0.0319,
      "reward": 1.849564844816923,
      "reward_std": 0.07906727260153275,
      "rewards/format_reward": 0.9875,
      "rewards/iqa_reward_score": 0.8620648480206728,
      "step": 3600
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.0225,
      "epoch": 1.9526011560693641,
      "grad_norm": 7.583634891952906,
      "kl": 0.7811505126953125,
      "learning_rate": 8.047276464542652e-07,
      "loss": 0.0312,
      "reward": 1.8461687500029802,
      "reward_std": 0.08467546943458729,
      "rewards/format_reward": 0.9859375,
      "rewards/iqa_reward_score": 0.8602312506362796,
      "step": 3800
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 27.99625,
      "epoch": 2.0554913294797688,
      "grad_norm": 14.625364812925723,
      "kl": 0.786697998046875,
      "learning_rate": 7.944501541623844e-07,
      "loss": 0.0315,
      "reward": 1.8452205635607242,
      "reward_std": 0.09883114330208627,
      "rewards/format_reward": 0.980625,
      "rewards/iqa_reward_score": 0.8645955645292998,
      "step": 4000
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 27.9753125,
      "epoch": 2.1582530507385997,
      "grad_norm": 33.54392125627147,
      "kl": 0.760584716796875,
      "learning_rate": 7.841726618705036e-07,
      "loss": 0.0304,
      "reward": 1.8701025630533694,
      "reward_std": 0.08290246042713989,
      "rewards/format_reward": 0.985,
      "rewards/iqa_reward_score": 0.8851025623455644,
      "step": 4200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 27.9215625,
      "epoch": 2.2610147719974307,
      "grad_norm": 47.05045527355845,
      "kl": 0.8736767578125,
      "learning_rate": 7.738951695786228e-07,
      "loss": 0.0349,
      "reward": 1.8553532859683037,
      "reward_std": 0.10405993272172054,
      "rewards/format_reward": 0.980625,
      "rewards/iqa_reward_score": 0.8747282821312546,
      "step": 4400
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.350625,
      "epoch": 2.3123956326268464,
      "grad_norm": 17.410628447499427,
      "kl": 0.89654296875,
      "learning_rate": 7.687564234326824e-07,
      "loss": 0.0359,
      "reward": 1.8479799877107144,
      "reward_std": 0.12358368890389101,
      "rewards/format_reward": 0.976875,
      "rewards/iqa_reward_score": 0.871104982867837,
      "step": 4500
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.386875,
      "epoch": 2.363776493256262,
      "grad_norm": 8.447575577812339,
      "kl": 0.8929931640625,
      "learning_rate": 7.63617677286742e-07,
      "loss": 0.0357,
      "reward": 1.8610911390185356,
      "reward_std": 0.08579985730873886,
      "rewards/format_reward": 0.985,
      "rewards/iqa_reward_score": 0.8760911403596401,
      "step": 4600
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.0375,
      "epoch": 2.466538214515093,
      "grad_norm": 143.05962527749097,
      "kl": 0.93417236328125,
      "learning_rate": 7.533401849948612e-07,
      "loss": 0.0374,
      "reward": 1.8311420308053494,
      "reward_std": 0.11677662628731923,
      "rewards/format_reward": 0.9784375,
      "rewards/iqa_reward_score": 0.8527045329660177,
      "step": 4800
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.01125,
      "epoch": 2.569299935773924,
      "grad_norm": 7.1542045396705,
      "kl": 0.94165771484375,
      "learning_rate": 7.430626927029804e-07,
      "loss": 0.0377,
      "reward": 1.8192646995186805,
      "reward_std": 0.14661665820371128,
      "rewards/format_reward": 0.9696875,
      "rewards/iqa_reward_score": 0.8495772035047412,
      "step": 5000
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.18375,
      "epoch": 2.6720616570327556,
      "grad_norm": 6.8796582113899865,
      "kl": 0.9063818359375,
      "learning_rate": 7.327852004110997e-07,
      "loss": 0.0363,
      "reward": 1.8397980490326882,
      "reward_std": 0.1039076264774485,
      "rewards/format_reward": 0.97875,
      "rewards/iqa_reward_score": 0.8610480467230082,
      "step": 5200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 27.974375,
      "epoch": 2.7748233782915865,
      "grad_norm": 21.679690712318255,
      "kl": 1.140872802734375,
      "learning_rate": 7.225077081192189e-07,
      "loss": 0.0456,
      "reward": 1.843569800555706,
      "reward_std": 0.10958875504176831,
      "rewards/format_reward": 0.97875,
      "rewards/iqa_reward_score": 0.8648197993636131,
      "step": 5400
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 27.874375,
      "epoch": 2.826204238921002,
      "grad_norm": 15.562815242952489,
      "kl": 0.83869384765625,
      "learning_rate": 7.173689619732784e-07,
      "loss": 0.0335,
      "reward": 1.8646468751132488,
      "reward_std": 0.07459007016877876,
      "rewards/format_reward": 0.989375,
      "rewards/iqa_reward_score": 0.8752718791365623,
      "step": 5500
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 27.930625,
      "epoch": 2.8775850995504175,
      "grad_norm": 6.830195802437781,
      "kl": 0.790654296875,
      "learning_rate": 7.122302158273381e-07,
      "loss": 0.0316,
      "reward": 1.8246960963308811,
      "reward_std": 0.14376337262161543,
      "rewards/format_reward": 0.970625,
      "rewards/iqa_reward_score": 0.8540710960328579,
      "step": 5600
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.023125,
      "epoch": 2.9803468208092485,
      "grad_norm": 14.937325884677351,
      "kl": 0.72754638671875,
      "learning_rate": 7.019527235354573e-07,
      "loss": 0.0291,
      "reward": 1.8573167976737022,
      "reward_std": 0.07879514415035374,
      "rewards/format_reward": 0.986875,
      "rewards/iqa_reward_score": 0.8704417966306209,
      "step": 5800
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 27.8778125,
      "epoch": 3.083236994219653,
      "grad_norm": 20.657599431765522,
      "kl": 0.769075927734375,
      "learning_rate": 6.916752312435765e-07,
      "loss": 0.0308,
      "reward": 1.8757967218756675,
      "reward_std": 0.07059297368963598,
      "rewards/format_reward": 0.989375,
      "rewards/iqa_reward_score": 0.8864217212796212,
      "step": 6000
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.2796875,
      "epoch": 3.185998715478484,
      "grad_norm": 78.23537527732513,
      "kl": 0.89517333984375,
      "learning_rate": 6.813977389516957e-07,
      "loss": 0.0358,
      "reward": 1.8553570313751697,
      "reward_std": 0.09863820670958376,
      "rewards/format_reward": 0.9796875,
      "rewards/iqa_reward_score": 0.8756695335730911,
      "step": 6200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 27.8103125,
      "epoch": 3.2887604367373156,
      "grad_norm": 17.956803144421166,
      "kl": 0.8780908203125,
      "learning_rate": 6.711202466598149e-07,
      "loss": 0.0351,
      "reward": 1.8629054698348044,
      "reward_std": 0.0895115670125233,
      "rewards/format_reward": 0.9828125,
      "rewards/iqa_reward_score": 0.880092968903482,
      "step": 6400
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.025625,
      "epoch": 3.340141297366731,
      "grad_norm": 14.577257269858018,
      "kl": 0.918896484375,
      "learning_rate": 6.659815005138745e-07,
      "loss": 0.0368,
      "reward": 1.8509218773245812,
      "reward_std": 0.128468633540906,
      "rewards/format_reward": 0.97375,
      "rewards/iqa_reward_score": 0.8771718776226044,
      "step": 6500
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 27.506875,
      "epoch": 3.3915221579961465,
      "grad_norm": 11.431877019480062,
      "kl": 0.9472998046875,
      "learning_rate": 6.608427543679342e-07,
      "loss": 0.0379,
      "reward": 1.857479686588049,
      "reward_std": 0.1099585933983326,
      "rewards/format_reward": 0.975625,
      "rewards/iqa_reward_score": 0.8818546874821186,
      "step": 6600
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 27.7065625,
      "epoch": 3.4942838792549775,
      "grad_norm": 6.9566554996708,
      "kl": 0.9450537109375,
      "learning_rate": 6.505652620760534e-07,
      "loss": 0.0378,
      "reward": 1.870063279643655,
      "reward_std": 0.08640069031709573,
      "rewards/format_reward": 0.9828125,
      "rewards/iqa_reward_score": 0.8872507839277387,
      "step": 6800
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.3125,
      "epoch": 3.5970456005138085,
      "grad_norm": 9.466798266539996,
      "kl": 1.00124755859375,
      "learning_rate": 6.402877697841727e-07,
      "loss": 0.0401,
      "reward": 1.8952510181069373,
      "reward_std": 0.032708497148996686,
      "rewards/format_reward": 1.0,
      "rewards/iqa_reward_score": 0.8952510149776935,
      "step": 7000
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.36875,
      "epoch": 3.6998073217726395,
      "grad_norm": 2557.560579743778,
      "kl": 1.13185791015625,
      "learning_rate": 6.300102774922919e-07,
      "loss": 0.0453,
      "reward": 1.892985941618681,
      "reward_std": 0.034503267100517404,
      "rewards/format_reward": 0.999375,
      "rewards/iqa_reward_score": 0.8936109407246113,
      "step": 7200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.926875,
      "epoch": 3.802569043031471,
      "grad_norm": 10.085679643415842,
      "kl": 1.2185986328125,
      "learning_rate": 6.197327852004111e-07,
      "loss": 0.0487,
      "reward": 1.8877808587253093,
      "reward_std": 0.04178271594835678,
      "rewards/format_reward": 0.9965625,
      "rewards/iqa_reward_score": 0.891218360774219,
      "step": 7400
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 30.4325,
      "epoch": 3.853949903660886,
      "grad_norm": 4.810497524915146,
      "kl": 1.437158203125,
      "learning_rate": 6.145940390544707e-07,
      "loss": 0.0575,
      "reward": 1.8972664028406143,
      "reward_std": 0.04059907855524216,
      "rewards/format_reward": 0.995625,
      "rewards/iqa_reward_score": 0.9016414076089859,
      "step": 7500
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 29.053125,
      "epoch": 3.905330764290302,
      "grad_norm": 25.880547489302362,
      "kl": 1.35017578125,
      "learning_rate": 6.094552929085303e-07,
      "loss": 0.054,
      "reward": 1.8683781233429908,
      "reward_std": 0.07233420302451123,
      "rewards/format_reward": 0.98625,
      "rewards/iqa_reward_score": 0.8821281273663044,
      "step": 7600
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.375,
      "epoch": 4.008220937700706,
      "grad_norm": 66.69984916070555,
      "kl": 1.363798828125,
      "learning_rate": 5.991778006166495e-07,
      "loss": 0.0545,
      "reward": 1.83771797016263,
      "reward_std": 0.13687979618785903,
      "rewards/format_reward": 0.97,
      "rewards/iqa_reward_score": 0.8677179708704352,
      "step": 7800
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.53,
      "epoch": 4.1109826589595375,
      "grad_norm": 165.49295535810293,
      "kl": 1.10419921875,
      "learning_rate": 5.889003083247688e-07,
      "loss": 0.0442,
      "reward": 1.8460748683661223,
      "reward_std": 0.1447469698698842,
      "rewards/format_reward": 0.9675,
      "rewards/iqa_reward_score": 0.8785748684033752,
      "step": 8000
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 29.9115625,
      "epoch": 4.213744380218369,
      "grad_norm": 8.447151915362078,
      "kl": 1.172177734375,
      "learning_rate": 5.78622816032888e-07,
      "loss": 0.0469,
      "reward": 1.8300597659498452,
      "reward_std": 0.17003738911065738,
      "rewards/format_reward": 0.95875,
      "rewards/iqa_reward_score": 0.8713097679056228,
      "step": 8200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 29.536875,
      "epoch": 4.3165061014771995,
      "grad_norm": 139.19766910371885,
      "kl": 1.204990234375,
      "learning_rate": 5.683453237410072e-07,
      "loss": 0.0482,
      "reward": 1.8459081883728503,
      "reward_std": 0.1452132201299537,
      "rewards/format_reward": 0.966875,
      "rewards/iqa_reward_score": 0.8790331880003214,
      "step": 8400
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 27.16125,
      "epoch": 4.367886962106615,
      "grad_norm": 67.10745002633337,
      "kl": 1.1150390625,
      "learning_rate": 5.632065775950667e-07,
      "loss": 0.0446,
      "reward": 1.8599339020252228,
      "reward_std": 0.11604845726978966,
      "rewards/format_reward": 0.9775,
      "rewards/iqa_reward_score": 0.882433907315135,
      "step": 8500
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 27.22,
      "epoch": 4.419267822736031,
      "grad_norm": 7.122275917364418,
      "kl": 1.02694091796875,
      "learning_rate": 5.580678314491264e-07,
      "loss": 0.0411,
      "reward": 1.8712499970197678,
      "reward_std": 0.11151634595706128,
      "rewards/format_reward": 0.98,
      "rewards/iqa_reward_score": 0.8912500011175871,
      "step": 8600
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.8384375,
      "epoch": 4.5220295439948615,
      "grad_norm": 36.76062498057171,
      "kl": 1.05457763671875,
      "learning_rate": 5.477903391572456e-07,
      "loss": 0.0422,
      "reward": 1.8598616697639228,
      "reward_std": 0.1202874470622919,
      "rewards/format_reward": 0.9740625,
      "rewards/iqa_reward_score": 0.8857991739735007,
      "step": 8800
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 29.40375,
      "epoch": 4.624791265253693,
      "grad_norm": 11.169409676243108,
      "kl": 1.180712890625,
      "learning_rate": 5.375128468653648e-07,
      "loss": 0.0472,
      "reward": 1.8395798132568597,
      "reward_std": 0.1512143662823655,
      "rewards/format_reward": 0.96625,
      "rewards/iqa_reward_score": 0.8733298133686185,
      "step": 9000
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 29.4003125,
      "epoch": 4.727552986512524,
      "grad_norm": 7.381446678871813,
      "kl": 1.0400537109375,
      "learning_rate": 5.27235354573484e-07,
      "loss": 0.0416,
      "reward": 1.8565457689017058,
      "reward_std": 0.13243299395922803,
      "rewards/format_reward": 0.973125,
      "rewards/iqa_reward_score": 0.8834207731485367,
      "step": 9200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 28.5815625,
      "epoch": 4.830314707771355,
      "grad_norm": 13.559802380662985,
      "kl": 1.03513427734375,
      "learning_rate": 5.169578622816033e-07,
      "loss": 0.0414,
      "reward": 1.864812575764954,
      "reward_std": 0.12085459077716224,
      "rewards/format_reward": 0.9753125,
      "rewards/iqa_reward_score": 0.8895000796020031,
      "step": 9400
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 13.53,
      "epoch": 4.881695568400771,
      "grad_norm": 42.48053012611243,
      "kl": 1.8257421875,
      "learning_rate": 5.118191161356628e-07,
      "loss": 0.073,
      "reward": 1.8785359343886376,
      "reward_std": 0.08481779177935095,
      "rewards/format_reward": 0.98875,
      "rewards/iqa_reward_score": 0.8897859378904104,
      "step": 9500
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 18.813125,
      "epoch": 4.933076429030186,
      "grad_norm": 47.10713474426497,
      "kl": 2.1516796875,
      "learning_rate": 5.066803699897225e-07,
      "loss": 0.0861,
      "reward": 1.8353367167711259,
      "reward_std": 0.18400538112502546,
      "rewards/format_reward": 0.956875,
      "rewards/iqa_reward_score": 0.8784617200121283,
      "step": 9600
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 17.98125,
      "epoch": 5.035966602440591,
      "grad_norm": 29.850445718834017,
      "kl": 2.100009765625,
      "learning_rate": 4.964028776978417e-07,
      "loss": 0.084,
      "reward": 1.8705265574902297,
      "reward_std": 0.10621097847208148,
      "rewards/format_reward": 0.9790625,
      "rewards/iqa_reward_score": 0.8914640637487173,
      "step": 9800
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 20.6721875,
      "epoch": 5.138728323699422,
      "grad_norm": 24.790938831821737,
      "kl": 2.140517578125,
      "learning_rate": 4.861253854059609e-07,
      "loss": 0.0856,
      "reward": 1.8829230420291423,
      "reward_std": 0.10707416723642382,
      "rewards/format_reward": 0.9803125,
      "rewards/iqa_reward_score": 0.9026105474308133,
      "step": 10000
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 19.6746875,
      "epoch": 5.241490044958253,
      "grad_norm": 102.41178221797948,
      "kl": 2.09017578125,
      "learning_rate": 4.7584789311408014e-07,
      "loss": 0.0836,
      "reward": 1.8718234330415726,
      "reward_std": 0.12345175031165127,
      "rewards/format_reward": 0.97375,
      "rewards/iqa_reward_score": 0.8980734394118189,
      "step": 10200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 20.186875,
      "epoch": 5.344251766217084,
      "grad_norm": 10.598097581329563,
      "kl": 1.971005859375,
      "learning_rate": 4.6557040082219935e-07,
      "loss": 0.0788,
      "reward": 1.8742554614692926,
      "reward_std": 0.10413653948286082,
      "rewards/format_reward": 0.9796875,
      "rewards/iqa_reward_score": 0.8945679685845971,
      "step": 10400
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 19.570625,
      "epoch": 5.3956326268465,
      "grad_norm": 15.65196245891987,
      "kl": 1.966328125,
      "learning_rate": 4.60431654676259e-07,
      "loss": 0.0786,
      "reward": 1.873755464553833,
      "reward_std": 0.1017152764700586,
      "rewards/format_reward": 0.98125,
      "rewards/iqa_reward_score": 0.8925054696947337,
      "step": 10500
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 16.39375,
      "epoch": 5.447013487475915,
      "grad_norm": 17.05777069418545,
      "kl": 2.1209765625,
      "learning_rate": 4.552929085303186e-07,
      "loss": 0.0848,
      "reward": 1.8873523423075675,
      "reward_std": 0.07782931060733972,
      "rewards/format_reward": 0.988125,
      "rewards/iqa_reward_score": 0.8992273437976838,
      "step": 10600
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 19.811875,
      "epoch": 5.549775208734746,
      "grad_norm": 31.92408805805376,
      "kl": 2.054736328125,
      "learning_rate": 4.450154162384378e-07,
      "loss": 0.0822,
      "reward": 1.870059761852026,
      "reward_std": 0.10109247839223827,
      "rewards/format_reward": 0.9815625,
      "rewards/iqa_reward_score": 0.8884972659498453,
      "step": 10800
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 21.2728125,
      "epoch": 5.652536929993578,
      "grad_norm": 50.24699385323065,
      "kl": 1.977314453125,
      "learning_rate": 4.34737923946557e-07,
      "loss": 0.0791,
      "reward": 1.880501558482647,
      "reward_std": 0.09021925082866801,
      "rewards/format_reward": 0.984375,
      "rewards/iqa_reward_score": 0.8961265632137656,
      "step": 11000
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 18.17,
      "epoch": 5.755298651252408,
      "grad_norm": 16.003754754439743,
      "kl": 2.043388671875,
      "learning_rate": 4.2446043165467627e-07,
      "loss": 0.0817,
      "reward": 1.879718798995018,
      "reward_std": 0.09361302960853209,
      "rewards/format_reward": 0.9821875,
      "rewards/iqa_reward_score": 0.8975313076004386,
      "step": 11200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 19.6184375,
      "epoch": 5.85806037251124,
      "grad_norm": 42.24292206080392,
      "kl": 1.938994140625,
      "learning_rate": 4.141829393627955e-07,
      "loss": 0.0776,
      "reward": 1.8871759340167045,
      "reward_std": 0.08852186787073152,
      "rewards/format_reward": 0.9846875,
      "rewards/iqa_reward_score": 0.902488438487053,
      "step": 11400
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 20.74625,
      "epoch": 5.909441233140655,
      "grad_norm": 147.2735619120744,
      "kl": 1.90677734375,
      "learning_rate": 4.090441932168551e-07,
      "loss": 0.0763,
      "reward": 1.9003723841905593,
      "reward_std": 0.06933337734662928,
      "rewards/format_reward": 0.98875,
      "rewards/iqa_reward_score": 0.9116223914176226,
      "step": 11500
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 20.711875,
      "epoch": 5.960822093770071,
      "grad_norm": 69.36856845472299,
      "kl": 1.9169140625,
      "learning_rate": 4.039054470709147e-07,
      "loss": 0.0767,
      "reward": 1.8851546820998193,
      "reward_std": 0.09332547818601597,
      "rewards/format_reward": 0.983125,
      "rewards/iqa_reward_score": 0.9020296874642372,
      "step": 11600
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 20.3178125,
      "epoch": 6.063712267180475,
      "grad_norm": 20.48968236042837,
      "kl": 1.988623046875,
      "learning_rate": 3.936279547790339e-07,
      "loss": 0.0796,
      "reward": 1.8889730402827263,
      "reward_std": 0.08705994199059205,
      "rewards/format_reward": 0.9859375,
      "rewards/iqa_reward_score": 0.9030355482548476,
      "step": 11800
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 20.341875,
      "epoch": 6.166473988439306,
      "grad_norm": 17.947997271086912,
      "kl": 1.970908203125,
      "learning_rate": 3.8335046248715314e-07,
      "loss": 0.0788,
      "reward": 1.8987746007740498,
      "reward_std": 0.07162293007655535,
      "rewards/format_reward": 0.9884375,
      "rewards/iqa_reward_score": 0.9103371092304587,
      "step": 12000
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 20.67625,
      "epoch": 6.269235709698138,
      "grad_norm": 222.44736959018252,
      "kl": 1.915966796875,
      "learning_rate": 3.7307297019527235e-07,
      "loss": 0.0766,
      "reward": 1.8926163981109858,
      "reward_std": 0.07730939569650218,
      "rewards/format_reward": 0.9865625,
      "rewards/iqa_reward_score": 0.9060539072006941,
      "step": 12200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 20.5784375,
      "epoch": 6.371997430956968,
      "grad_norm": 13.356785918610642,
      "kl": 1.961318359375,
      "learning_rate": 3.6279547790339155e-07,
      "loss": 0.0784,
      "reward": 1.8869195252656936,
      "reward_std": 0.09981291456278996,
      "rewards/format_reward": 0.979375,
      "rewards/iqa_reward_score": 0.9075445308163762,
      "step": 12400
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 22.4525,
      "epoch": 6.423378291586384,
      "grad_norm": 30.8756155693914,
      "kl": 1.8610546875,
      "learning_rate": 3.5765673175745115e-07,
      "loss": 0.0744,
      "reward": 1.9010210880637168,
      "reward_std": 0.07242356748203747,
      "rewards/format_reward": 0.98875,
      "rewards/iqa_reward_score": 0.9122710923105478,
      "step": 12500
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 24.38,
      "epoch": 6.4747591522158,
      "grad_norm": 7.829345980386679,
      "kl": 1.74701171875,
      "learning_rate": 3.5251798561151076e-07,
      "loss": 0.0699,
      "reward": 1.8803515616059303,
      "reward_std": 0.10600787395669613,
      "rewards/format_reward": 0.979375,
      "rewards/iqa_reward_score": 0.9009765643626452,
      "step": 12600
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 23.6565625,
      "epoch": 6.577520873474631,
      "grad_norm": 20.29606750367699,
      "kl": 1.82380859375,
      "learning_rate": 3.4224049331963e-07,
      "loss": 0.0729,
      "reward": 1.8765503884851933,
      "reward_std": 0.10835037114477018,
      "rewards/format_reward": 0.9771875,
      "rewards/iqa_reward_score": 0.8993628909811378,
      "step": 12800
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 21.75625,
      "epoch": 6.680282594733462,
      "grad_norm": 13.068159937640354,
      "kl": 1.868056640625,
      "learning_rate": 3.319630010277492e-07,
      "loss": 0.0747,
      "reward": 1.8851296819746495,
      "reward_std": 0.09881990624518949,
      "rewards/format_reward": 0.9815625,
      "rewards/iqa_reward_score": 0.9035671878978611,
      "step": 13000
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 18.6534375,
      "epoch": 6.783044315992293,
      "grad_norm": 43.271540826002,
      "kl": 1.896494140625,
      "learning_rate": 3.216855087358684e-07,
      "loss": 0.0759,
      "reward": 1.8839531208574771,
      "reward_std": 0.10083580034945044,
      "rewards/format_reward": 0.980625,
      "rewards/iqa_reward_score": 0.9033281239122153,
      "step": 13200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 17.59375,
      "epoch": 6.885806037251124,
      "grad_norm": 51.02501405917888,
      "kl": 1.973603515625,
      "learning_rate": 3.114080164439877e-07,
      "loss": 0.0789,
      "reward": 1.8915468671917914,
      "reward_std": 0.08674899358520634,
      "rewards/format_reward": 0.9840625,
      "rewards/iqa_reward_score": 0.9074843747541308,
      "step": 13400
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 17.875,
      "epoch": 6.937186897880539,
      "grad_norm": 101.7576095129992,
      "kl": 1.999921875,
      "learning_rate": 3.062692702980473e-07,
      "loss": 0.08,
      "reward": 1.8977671816945076,
      "reward_std": 0.07746992219588719,
      "rewards/format_reward": 0.9875,
      "rewards/iqa_reward_score": 0.9102671888470649,
      "step": 13500
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 19.418125,
      "epoch": 6.988567758509955,
      "grad_norm": 31.809990191429367,
      "kl": 1.88990234375,
      "learning_rate": 3.011305241521069e-07,
      "loss": 0.0756,
      "reward": 1.8850296795368195,
      "reward_std": 0.08940189514425584,
      "rewards/format_reward": 0.983125,
      "rewards/iqa_reward_score": 0.9019046877324581,
      "step": 13600
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 20.1825,
      "epoch": 7.09145793192036,
      "grad_norm": 13.227429266121407,
      "kl": 1.850634765625,
      "learning_rate": 2.908530318602261e-07,
      "loss": 0.074,
      "reward": 1.8925906175374985,
      "reward_std": 0.09375914162839763,
      "rewards/format_reward": 0.9828125,
      "rewards/iqa_reward_score": 0.9097781255841255,
      "step": 13800
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 18.1321875,
      "epoch": 7.194219653179191,
      "grad_norm": 10.599748254211335,
      "kl": 1.89337890625,
      "learning_rate": 2.805755395683453e-07,
      "loss": 0.0757,
      "reward": 1.899733590334654,
      "reward_std": 0.08313908010721206,
      "rewards/format_reward": 0.984375,
      "rewards/iqa_reward_score": 0.9153585939109325,
      "step": 14000
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 19.83375,
      "epoch": 7.296981374438022,
      "grad_norm": 103.30184204894014,
      "kl": 1.8382421875,
      "learning_rate": 2.7029804727646455e-07,
      "loss": 0.0735,
      "reward": 1.8822011659294366,
      "reward_std": 0.1061160612545791,
      "rewards/format_reward": 0.9790625,
      "rewards/iqa_reward_score": 0.9031386712566019,
      "step": 14200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 18.7328125,
      "epoch": 7.399743095696853,
      "grad_norm": 21.624795896150882,
      "kl": 1.8940625,
      "learning_rate": 2.6002055498458376e-07,
      "loss": 0.0758,
      "reward": 1.8954588214308024,
      "reward_std": 0.08989560850925045,
      "rewards/format_reward": 0.985,
      "rewards/iqa_reward_score": 0.9104588272795081,
      "step": 14400
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 18.505625,
      "epoch": 7.451123956326269,
      "grad_norm": 35.276748712299856,
      "kl": 1.89419921875,
      "learning_rate": 2.5488180883864336e-07,
      "loss": 0.0758,
      "reward": 1.884902337193489,
      "reward_std": 0.10493175498064375,
      "rewards/format_reward": 0.979375,
      "rewards/iqa_reward_score": 0.905527343377471,
      "step": 14500
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 16.82625,
      "epoch": 7.502504816955684,
      "grad_norm": 84.53198174061639,
      "kl": 2.06251953125,
      "learning_rate": 2.4974306269270296e-07,
      "loss": 0.0825,
      "reward": 1.8909136363863945,
      "reward_std": 0.10835257644532248,
      "rewards/format_reward": 0.978125,
      "rewards/iqa_reward_score": 0.9127886415272951,
      "step": 14600
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 17.5196875,
      "epoch": 7.605266538214515,
      "grad_norm": 10.589764450261264,
      "kl": 2.156630859375,
      "learning_rate": 2.3946557040082217e-07,
      "loss": 0.0863,
      "reward": 1.8696386666595937,
      "reward_std": 0.12423644962807884,
      "rewards/format_reward": 0.9715625,
      "rewards/iqa_reward_score": 0.8980761723965407,
      "step": 14800
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 16.7325,
      "epoch": 7.7080282594733465,
      "grad_norm": 10.913131914126758,
      "kl": 2.070146484375,
      "learning_rate": 2.2918807810894143e-07,
      "loss": 0.0828,
      "reward": 1.8887575732171535,
      "reward_std": 0.11135890247678618,
      "rewards/format_reward": 0.9775,
      "rewards/iqa_reward_score": 0.9112575780972838,
      "step": 15000
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 15.9571875,
      "epoch": 7.810789980732177,
      "grad_norm": 45.002154023589725,
      "kl": 2.196806640625,
      "learning_rate": 2.1891058581706063e-07,
      "loss": 0.0879,
      "reward": 1.892361634373665,
      "reward_std": 0.1102386135366396,
      "rewards/format_reward": 0.9771875,
      "rewards/iqa_reward_score": 0.9151741417497397,
      "step": 15200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 16.425625,
      "epoch": 7.913551701991008,
      "grad_norm": 15.653934750546377,
      "kl": 2.142041015625,
      "learning_rate": 2.0863309352517986e-07,
      "loss": 0.0857,
      "reward": 1.8734804604202508,
      "reward_std": 0.12232352063700092,
      "rewards/format_reward": 0.97375,
      "rewards/iqa_reward_score": 0.8997304691933096,
      "step": 15400
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 17.148125,
      "epoch": 7.964932562620424,
      "grad_norm": 97.67216959654685,
      "kl": 2.11296875,
      "learning_rate": 2.0349434737923946e-07,
      "loss": 0.0845,
      "reward": 1.873739056289196,
      "reward_std": 0.12005866145074834,
      "rewards/format_reward": 0.976875,
      "rewards/iqa_reward_score": 0.8968640621006488,
      "step": 15500
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 17.506875,
      "epoch": 8.016441875401412,
      "grad_norm": 105.09965514480388,
      "kl": 2.1259375,
      "learning_rate": 1.9835560123329907e-07,
      "loss": 0.085,
      "reward": 1.876248899102211,
      "reward_std": 0.11951107540109661,
      "rewards/format_reward": 0.976875,
      "rewards/iqa_reward_score": 0.8993739089369774,
      "step": 15600
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 17.133125,
      "epoch": 8.119203596660244,
      "grad_norm": 6.128097746188641,
      "kl": 2.223037109375,
      "learning_rate": 1.880781089414183e-07,
      "loss": 0.0889,
      "reward": 1.880941744968295,
      "reward_std": 0.12711579945113044,
      "rewards/format_reward": 0.9734375,
      "rewards/iqa_reward_score": 0.9075042496249079,
      "step": 15800
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 17.825,
      "epoch": 8.221965317919075,
      "grad_norm": 21.00097459983442,
      "kl": 2.175458984375,
      "learning_rate": 1.778006166495375e-07,
      "loss": 0.087,
      "reward": 1.8709558783471585,
      "reward_std": 0.14294033588666935,
      "rewards/format_reward": 0.9684375,
      "rewards/iqa_reward_score": 0.9025183825194836,
      "step": 16000
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 17.31625,
      "epoch": 8.324727039177906,
      "grad_norm": 32.175418316210305,
      "kl": 2.24296875,
      "learning_rate": 1.6752312435765673e-07,
      "loss": 0.0897,
      "reward": 1.8707203871756792,
      "reward_std": 0.14462180348142284,
      "rewards/format_reward": 0.9675,
      "rewards/iqa_reward_score": 0.9032203914225101,
      "step": 16200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 17.0921875,
      "epoch": 8.427488760436738,
      "grad_norm": 19.585104477991443,
      "kl": 2.394765625,
      "learning_rate": 1.5724563206577597e-07,
      "loss": 0.0958,
      "reward": 1.865453900322318,
      "reward_std": 0.16036579100909876,
      "rewards/format_reward": 0.9640625,
      "rewards/iqa_reward_score": 0.9013914061337709,
      "step": 16400
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 18.23625,
      "epoch": 8.478869621066153,
      "grad_norm": 52.50729568021014,
      "kl": 2.45123046875,
      "learning_rate": 1.5210688591983557e-07,
      "loss": 0.098,
      "reward": 1.8535247252881526,
      "reward_std": 0.1715104495617561,
      "rewards/format_reward": 0.96125,
      "rewards/iqa_reward_score": 0.892274733260274,
      "step": 16500
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 17.33125,
      "epoch": 8.530250481695568,
      "grad_norm": 16.036354362208947,
      "kl": 2.4987109375,
      "learning_rate": 1.4696813977389517e-07,
      "loss": 0.1,
      "reward": 1.8759004624187947,
      "reward_std": 0.12627161706855985,
      "rewards/format_reward": 0.97375,
      "rewards/iqa_reward_score": 0.9021504689753056,
      "step": 16600
    }
  ],
  "logging_steps": 1.0,
  "max_steps": 19460,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 10,
  "save_steps": 200,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}