{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.006940627099539697,
  "eval_steps": 1000,
  "global_step": 2500,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio": 0.0,
      "completion_length": 240.56250762939453,
      "epoch": 2.776250839815879e-06,
      "grad_norm": 0.06479538977146149,
      "kl": 0.0,
      "learning_rate": 8.310249307479225e-09,
      "loss": -0.0006,
      "reward": 0.16875001788139343,
      "reward_std": 0.19138706848025322,
      "rewards/countdown_reward_func": 0.16875001788139343,
      "step": 1,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0,
      "epoch": 5.552501679631758e-06,
      "grad_norm": 0.055109903216362,
      "kl": 0.0,
      "learning_rate": 1.662049861495845e-08,
      "loss": -0.0006,
      "step": 2
    },
    {
      "clip_ratio": 0.0,
      "epoch": 8.328752519447637e-06,
      "grad_norm": 0.0616462379693985,
      "kl": 0.0006676262128166854,
      "learning_rate": 2.4930747922437675e-08,
      "loss": -0.0006,
      "step": 3
    },
    {
      "clip_ratio": 0.0,
      "epoch": 1.1105003359263517e-05,
      "grad_norm": 0.061320219188928604,
      "kl": 0.0007026523817330599,
      "learning_rate": 3.32409972299169e-08,
      "loss": -0.0005,
      "step": 4
    },
    {
      "clip_ratio": 0.0002470490289852023,
      "epoch": 1.3881254199079395e-05,
      "grad_norm": 0.059856366366147995,
      "kl": 0.0007550643058493733,
      "learning_rate": 4.155124653739612e-08,
      "loss": -0.0006,
      "step": 5
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 1.6657505038895273e-05,
      "grad_norm": 0.09312308579683304,
      "kl": 0.0007920752104837447,
      "learning_rate": 4.986149584487535e-08,
      "loss": -0.0009,
      "step": 6
    },
    {
      "clip_ratio": 0.0,
      "epoch": 1.9433755878711153e-05,
      "grad_norm": 0.0637163445353508,
      "kl": 0.0006995745643507689,
      "learning_rate": 5.8171745152354567e-08,
      "loss": -0.0007,
      "step": 7
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 2.2210006718527033e-05,
      "grad_norm": 0.055853065103292465,
      "kl": 0.0008612782985437661,
      "learning_rate": 6.64819944598338e-08,
      "loss": -0.0002,
      "step": 8
    },
    {
      "clip_ratio": 0.0,
      "epoch": 2.498625755834291e-05,
      "grad_norm": 0.060049429535865784,
      "kl": 0.00074524967931211,
      "learning_rate": 7.479224376731302e-08,
      "loss": -0.0007,
      "step": 9
    },
    {
      "clip_ratio": 0.0,
      "epoch": 2.776250839815879e-05,
      "grad_norm": 0.059901271015405655,
      "kl": 0.0008222198521252722,
      "learning_rate": 8.310249307479224e-08,
      "loss": -0.0004,
      "step": 10
    },
    {
      "clip_ratio": 0.00016566881095059216,
      "epoch": 3.0538759237974666e-05,
      "grad_norm": 0.05985812842845917,
      "kl": 0.0007750319491606206,
      "learning_rate": 9.141274238227148e-08,
      "loss": -0.0006,
      "step": 11
    },
    {
      "clip_ratio": 0.0,
      "epoch": 3.3315010077790546e-05,
      "grad_norm": 0.09739804267883301,
      "kl": 0.0008287512173410505,
      "learning_rate": 9.97229916897507e-08,
      "loss": -0.0006,
      "step": 12
    },
    {
      "clip_ratio": 8.928571332944557e-05,
      "completion_length": 224.6666717529297,
      "epoch": 3.6091260917606426e-05,
      "grad_norm": 0.19317923486232758,
      "kl": 0.0007415811996906996,
      "learning_rate": 1.0803324099722992e-07,
      "loss": 0.0222,
      "reward": 0.2645833343267441,
      "reward_std": 0.2765769511461258,
      "rewards/countdown_reward_func": 0.2645833268761635,
      "step": 13,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 8.827683632262051e-05,
      "epoch": 3.8867511757422306e-05,
      "grad_norm": 0.07646733522415161,
      "kl": 0.0008568160701543093,
      "learning_rate": 1.1634349030470913e-07,
      "loss": 0.0217,
      "step": 14
    },
    {
      "clip_ratio": 0.00019336564582772553,
      "epoch": 4.1643762597238186e-05,
      "grad_norm": 0.10118044912815094,
      "kl": 0.0007392842962872237,
      "learning_rate": 1.2465373961218836e-07,
      "loss": 0.0214,
      "step": 15
    },
    {
      "clip_ratio": 0.0004612546181306243,
      "epoch": 4.4420013437054066e-05,
      "grad_norm": 0.09648283571004868,
      "kl": 0.0007288659107871354,
      "learning_rate": 1.329639889196676e-07,
      "loss": 0.0215,
      "step": 16
    },
    {
      "clip_ratio": 0.0,
      "epoch": 4.7196264276869946e-05,
      "grad_norm": 0.10432377457618713,
      "kl": 0.0007946545956656337,
      "learning_rate": 1.4127423822714683e-07,
      "loss": 0.021,
      "step": 17
    },
    {
      "clip_ratio": 0.0,
      "epoch": 4.997251511668582e-05,
      "grad_norm": 0.0799316018819809,
      "kl": 0.0007485100359190255,
      "learning_rate": 1.4958448753462604e-07,
      "loss": 0.0217,
      "step": 18
    },
    {
      "clip_ratio": 8.480325777782127e-05,
      "epoch": 5.27487659565017e-05,
      "grad_norm": 0.20232859253883362,
      "kl": 0.0009814091317821294,
      "learning_rate": 1.5789473684210525e-07,
      "loss": 0.022,
      "step": 19
    },
    {
      "clip_ratio": 0.00027768261497840285,
      "epoch": 5.552501679631758e-05,
      "grad_norm": 0.07837554812431335,
      "kl": 0.000779987225541845,
      "learning_rate": 1.6620498614958448e-07,
      "loss": 0.0217,
      "step": 20
    },
    {
      "clip_ratio": 0.00010407993249827996,
      "epoch": 5.830126763613346e-05,
      "grad_norm": 0.09943106770515442,
      "kl": 0.0007467044633813202,
      "learning_rate": 1.7451523545706372e-07,
      "loss": 0.0214,
      "step": 21
    },
    {
      "clip_ratio": 0.0005535055533982813,
      "epoch": 6.107751847594933e-05,
      "grad_norm": 0.09467501193284988,
      "kl": 0.0007356623827945441,
      "learning_rate": 1.8282548476454296e-07,
      "loss": 0.0217,
      "step": 22
    },
    {
      "clip_ratio": 8.979885024018586e-05,
      "epoch": 6.385376931576521e-05,
      "grad_norm": 0.1044834554195404,
      "kl": 0.0007783641340211034,
      "learning_rate": 1.9113573407202217e-07,
      "loss": 0.0216,
      "step": 23
    },
    {
      "clip_ratio": 0.0,
      "epoch": 6.663002015558109e-05,
      "grad_norm": 0.08091197907924652,
      "kl": 0.0007477364561054856,
      "learning_rate": 1.994459833795014e-07,
      "loss": 0.0214,
      "step": 24
    },
    {
      "clip_ratio": 0.000327225134242326,
      "completion_length": 223.89583587646484,
      "epoch": 6.940627099539697e-05,
      "grad_norm": 0.07894178479909897,
      "kl": 0.0009490847878623754,
      "learning_rate": 2.0775623268698064e-07,
      "loss": 0.0032,
      "reward": 0.24583334475755692,
      "reward_std": 0.2639523148536682,
      "rewards/countdown_reward_func": 0.24583334475755692,
      "step": 25,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.00018698579515330493,
      "epoch": 7.218252183521285e-05,
      "grad_norm": 0.07660482078790665,
      "kl": 0.0009208305855281651,
      "learning_rate": 2.1606648199445985e-07,
      "loss": 0.0032,
      "step": 26
    },
    {
      "clip_ratio": 0.0,
      "epoch": 7.495877267502873e-05,
      "grad_norm": 0.10856402665376663,
      "kl": 0.0008324629161506891,
      "learning_rate": 2.2437673130193906e-07,
      "loss": 0.0029,
      "step": 27
    },
    {
      "clip_ratio": 0.0,
      "epoch": 7.773502351484461e-05,
      "grad_norm": 0.10259876400232315,
      "kl": 0.0007138713845051825,
      "learning_rate": 2.3268698060941827e-07,
      "loss": 0.0028,
      "step": 28
    },
    {
      "clip_ratio": 0.0005700875190086663,
      "epoch": 8.051127435466049e-05,
      "grad_norm": 0.07817202061414719,
      "kl": 0.0009188149124383926,
      "learning_rate": 2.409972299168975e-07,
      "loss": 0.003,
      "step": 29
    },
    {
      "clip_ratio": 8.18062835605815e-05,
      "epoch": 8.328752519447637e-05,
      "grad_norm": 0.09154564142227173,
      "kl": 0.000773191568441689,
      "learning_rate": 2.493074792243767e-07,
      "loss": 0.0026,
      "step": 30
    },
    {
      "clip_ratio": 0.00010195758659392595,
      "epoch": 8.606377603429225e-05,
      "grad_norm": 0.09764699637889862,
      "kl": 0.0008510929765179753,
      "learning_rate": 2.57617728531856e-07,
      "loss": 0.0032,
      "step": 31
    },
    {
      "clip_ratio": 0.0,
      "epoch": 8.884002687410813e-05,
      "grad_norm": 0.07986056804656982,
      "kl": 0.0009318325319327414,
      "learning_rate": 2.659279778393352e-07,
      "loss": 0.0025,
      "step": 32
    },
    {
      "clip_ratio": 0.00019616441568359733,
      "epoch": 9.161627771392401e-05,
      "grad_norm": 0.10944823920726776,
      "kl": 0.0008964207954704762,
      "learning_rate": 2.742382271468144e-07,
      "loss": 0.0024,
      "step": 33
    },
    {
      "clip_ratio": 0.0,
      "epoch": 9.439252855373989e-05,
      "grad_norm": 0.08562328666448593,
      "kl": 0.000787771336035803,
      "learning_rate": 2.8254847645429366e-07,
      "loss": 0.0027,
      "step": 34
    },
    {
      "clip_ratio": 0.00016318648704327643,
      "epoch": 9.716877939355577e-05,
      "grad_norm": 0.06373903900384903,
      "kl": 0.0009429152996744961,
      "learning_rate": 2.9085872576177287e-07,
      "loss": 0.0028,
      "step": 35
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 9.994503023337164e-05,
      "grad_norm": 0.0951249971985817,
      "kl": 0.0008729175024200231,
      "learning_rate": 2.991689750692521e-07,
      "loss": 0.0029,
      "step": 36
    },
    {
      "clip_ratio": 0.00010088780982187018,
      "completion_length": 234.20833587646484,
      "epoch": 0.00010272128107318752,
      "grad_norm": 0.0882072001695633,
      "kl": 0.001087184005882591,
      "learning_rate": 3.0747922437673134e-07,
      "loss": -0.0108,
      "reward": 0.24791668355464935,
      "reward_std": 0.2163795679807663,
      "rewards/countdown_reward_func": 0.24791668355464935,
      "step": 37,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0001054975319130034,
      "grad_norm": 0.08608131855726242,
      "kl": 0.0007613546913489699,
      "learning_rate": 3.157894736842105e-07,
      "loss": -0.0112,
      "step": 38
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00010827378275281928,
      "grad_norm": 0.081035315990448,
      "kl": 0.0008077043457888067,
      "learning_rate": 3.2409972299168976e-07,
      "loss": -0.0113,
      "step": 39
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00011105003359263516,
      "grad_norm": 0.07537035644054413,
      "kl": 0.0007854337454773486,
      "learning_rate": 3.3240997229916897e-07,
      "loss": -0.0118,
      "step": 40
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.00011382628443245104,
      "grad_norm": 0.08525540679693222,
      "kl": 0.0007534388569183648,
      "learning_rate": 3.407202216066482e-07,
      "loss": -0.0112,
      "step": 41
    },
    {
      "clip_ratio": 0.0003339054746902548,
      "epoch": 0.00011660253527226692,
      "grad_norm": 0.1856469362974167,
      "kl": 0.000957256241235882,
      "learning_rate": 3.4903047091412744e-07,
      "loss": -0.0105,
      "step": 42
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0001193787861120828,
      "grad_norm": 0.06687930226325989,
      "kl": 0.0008277066808659583,
      "learning_rate": 3.5734072022160665e-07,
      "loss": -0.011,
      "step": 43
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00012215503695189867,
      "grad_norm": 0.07720962911844254,
      "kl": 0.0007938558992464095,
      "learning_rate": 3.656509695290859e-07,
      "loss": -0.0115,
      "step": 44
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00012493128779171456,
      "grad_norm": 0.08875006437301636,
      "kl": 0.0007698170084040612,
      "learning_rate": 3.739612188365651e-07,
      "loss": -0.011,
      "step": 45
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00012770753863153043,
      "grad_norm": 0.07545354217290878,
      "kl": 0.0008280337788164616,
      "learning_rate": 3.8227146814404433e-07,
      "loss": -0.0114,
      "step": 46
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.00013048378947134632,
      "grad_norm": 0.07932056486606598,
      "kl": 0.000836798019008711,
      "learning_rate": 3.905817174515236e-07,
      "loss": -0.0109,
      "step": 47
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.00013326004031116219,
      "grad_norm": 0.11963900178670883,
      "kl": 0.0009241281659342349,
      "learning_rate": 3.988919667590028e-07,
      "loss": -0.011,
      "step": 48
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 218.25000762939453,
      "epoch": 0.00013603629115097808,
      "grad_norm": 0.06625409424304962,
      "kl": 0.0008119716076180339,
      "learning_rate": 4.07202216066482e-07,
      "loss": 0.0113,
      "reward": 0.24166668951511383,
      "reward_std": 0.18230264633893967,
      "rewards/countdown_reward_func": 0.24166668206453323,
      "step": 49,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00013881254199079395,
      "grad_norm": 0.07639951258897781,
      "kl": 0.0009117976296693087,
      "learning_rate": 4.155124653739613e-07,
      "loss": 0.0117,
      "step": 50
    },
    {
      "clip_ratio": 0.00010530749568715692,
      "epoch": 0.00014158879283060984,
      "grad_norm": 0.0806988924741745,
      "kl": 0.0008828463032841682,
      "learning_rate": 4.238227146814405e-07,
      "loss": 0.0111,
      "step": 51
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0001443650436704257,
      "grad_norm": 0.07537931948900223,
      "kl": 0.0007655246299691498,
      "learning_rate": 4.321329639889197e-07,
      "loss": 0.0111,
      "step": 52
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0001471412945102416,
      "grad_norm": 0.06568142026662827,
      "kl": 0.0007329773507080972,
      "learning_rate": 4.4044321329639896e-07,
      "loss": 0.0112,
      "step": 53
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00014991754535005747,
      "grad_norm": 0.08050818741321564,
      "kl": 0.0007917624025139958,
      "learning_rate": 4.487534626038781e-07,
      "loss": 0.0115,
      "step": 54
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00015269379618987336,
      "grad_norm": 0.07148700952529907,
      "kl": 0.0008034448546823114,
      "learning_rate": 4.570637119113573e-07,
      "loss": 0.0113,
      "step": 55
    },
    {
      "clip_ratio": 0.00021934154210612178,
      "epoch": 0.00015547004702968923,
      "grad_norm": 0.07921700179576874,
      "kl": 0.0008946515154093504,
      "learning_rate": 4.6537396121883653e-07,
      "loss": 0.0113,
      "step": 56
    },
    {
      "clip_ratio": 8.692628762219101e-05,
      "epoch": 0.0001582462978695051,
      "grad_norm": 0.07876580208539963,
      "kl": 0.0009213399316649884,
      "learning_rate": 4.736842105263158e-07,
      "loss": 0.011,
      "step": 57
    },
    {
      "clip_ratio": 0.0002706017985474318,
      "epoch": 0.00016102254870932099,
      "grad_norm": 0.06707654148340225,
      "kl": 0.0008947286114562303,
      "learning_rate": 4.81994459833795e-07,
      "loss": 0.0109,
      "step": 58
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00016379879954913685,
      "grad_norm": 0.08683553338050842,
      "kl": 0.0007533867028541863,
      "learning_rate": 4.903047091412742e-07,
      "loss": 0.0113,
      "step": 59
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00016657505038895275,
      "grad_norm": 0.07259709388017654,
      "kl": 0.0008512145723216236,
      "learning_rate": 4.986149584487534e-07,
      "loss": 0.0115,
      "step": 60
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 242.2291717529297,
      "epoch": 0.0001693513012287686,
      "grad_norm": 0.10226056724786758,
      "kl": 0.0007961629016790539,
      "learning_rate": 5.069252077562327e-07,
      "loss": 0.0107,
      "reward": 0.3229166716337204,
      "reward_std": 0.3417354077100754,
      "rewards/countdown_reward_func": 0.3229166716337204,
      "step": 61,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0001721275520685845,
      "grad_norm": 0.1283038705587387,
      "kl": 0.0008402638195548207,
      "learning_rate": 5.15235457063712e-07,
      "loss": 0.0097,
      "step": 62
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.00017490380290840037,
      "grad_norm": 0.10426519066095352,
      "kl": 0.0008127306937240064,
      "learning_rate": 5.235457063711912e-07,
      "loss": 0.0104,
      "step": 63
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00017768005374821627,
      "grad_norm": 0.09159515053033829,
      "kl": 0.0008558765111956745,
      "learning_rate": 5.318559556786704e-07,
      "loss": 0.0104,
      "step": 64
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00018045630458803213,
      "grad_norm": 0.09254452586174011,
      "kl": 0.00104894393007271,
      "learning_rate": 5.401662049861496e-07,
      "loss": 0.0107,
      "step": 65
    },
    {
      "clip_ratio": 0.00045153952669352293,
      "epoch": 0.00018323255542784803,
      "grad_norm": 0.10837826132774353,
      "kl": 0.0008054885256569833,
      "learning_rate": 5.484764542936288e-07,
      "loss": 0.0104,
      "step": 66
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0001860088062676639,
      "grad_norm": 0.09977608174085617,
      "kl": 0.0007916014001239091,
      "learning_rate": 5.567867036011081e-07,
      "loss": 0.011,
      "step": 67
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00018878505710747979,
      "grad_norm": 0.1362018585205078,
      "kl": 0.0008146155159920454,
      "learning_rate": 5.650969529085873e-07,
      "loss": 0.0102,
      "step": 68
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00019156130794729565,
      "grad_norm": 0.09612994641065598,
      "kl": 0.0008106507884804159,
      "learning_rate": 5.734072022160665e-07,
      "loss": 0.0099,
      "step": 69
    },
    {
      "clip_ratio": 0.00017850531003205106,
      "epoch": 0.00019433755878711155,
      "grad_norm": 0.10862355679273605,
      "kl": 0.0008707395172677934,
      "learning_rate": 5.817174515235457e-07,
      "loss": 0.0095,
      "step": 70
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0001971138096269274,
      "grad_norm": 0.09866558760404587,
      "kl": 0.0008735440496820956,
      "learning_rate": 5.900277008310249e-07,
      "loss": 0.0106,
      "step": 71
    },
    {
      "clip_ratio": 9.110787505051121e-05,
      "epoch": 0.00019989006046674328,
      "grad_norm": 0.09790490567684174,
      "kl": 0.0008786998223513365,
      "learning_rate": 5.983379501385042e-07,
      "loss": 0.0105,
      "step": 72
    },
    {
      "clip_ratio": 0.0001945525291375816,
      "completion_length": 232.2291717529297,
      "epoch": 0.00020266631130655917,
      "grad_norm": 0.09406734257936478,
      "kl": 0.0010395684512332082,
      "learning_rate": 6.066481994459835e-07,
      "loss": 0.0024,
      "reward": 0.23750000447034836,
      "reward_std": 0.2697988599538803,
      "rewards/countdown_reward_func": 0.23750000447034836,
      "step": 73,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.00020544256214637504,
      "grad_norm": 0.11244494467973709,
      "kl": 0.0007681146962568164,
      "learning_rate": 6.149584487534627e-07,
      "loss": 0.0034,
      "step": 74
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00020821881298619093,
      "grad_norm": 0.15458862483501434,
      "kl": 0.0007610543980263174,
      "learning_rate": 6.232686980609418e-07,
      "loss": 0.0025,
      "step": 75
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0002109950638260068,
      "grad_norm": 0.07957284897565842,
      "kl": 0.0009243077947758138,
      "learning_rate": 6.31578947368421e-07,
      "loss": 0.0031,
      "step": 76
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0002137713146658227,
      "grad_norm": 0.1434016078710556,
      "kl": 0.0009022459562402219,
      "learning_rate": 6.398891966759003e-07,
      "loss": 0.0033,
      "step": 77
    },
    {
      "clip_ratio": 0.0002997264382429421,
      "epoch": 0.00021654756550563856,
      "grad_norm": 0.07511159032583237,
      "kl": 0.0009360458934679627,
      "learning_rate": 6.481994459833795e-07,
      "loss": 0.0025,
      "step": 78
    },
    {
      "clip_ratio": 0.0002821488888002932,
      "epoch": 0.00021932381634545445,
      "grad_norm": 0.10179822146892548,
      "kl": 0.0009871571965049952,
      "learning_rate": 6.565096952908587e-07,
      "loss": 0.0025,
      "step": 79
    },
    {
      "clip_ratio": 9.72762645687908e-05,
      "epoch": 0.00022210006718527032,
      "grad_norm": 0.10691147297620773,
      "kl": 0.0007517710037063807,
      "learning_rate": 6.648199445983379e-07,
      "loss": 0.0032,
      "step": 80
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0002248763180250862,
      "grad_norm": 0.1533641219139099,
      "kl": 0.0008048239687923342,
      "learning_rate": 6.731301939058171e-07,
      "loss": 0.0022,
      "step": 81
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00022765256886490208,
      "grad_norm": 0.08312182873487473,
      "kl": 0.0009179475018754601,
      "learning_rate": 6.814404432132964e-07,
      "loss": 0.0032,
      "step": 82
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.00023042881970471797,
      "grad_norm": 0.11414949595928192,
      "kl": 0.0008502569980919361,
      "learning_rate": 6.897506925207757e-07,
      "loss": 0.0024,
      "step": 83
    },
    {
      "clip_ratio": 0.0001669251942075789,
      "epoch": 0.00023320507054453384,
      "grad_norm": 0.07714688032865524,
      "kl": 0.0008958573453128338,
      "learning_rate": 6.980609418282549e-07,
      "loss": 0.0027,
      "step": 84
    },
    {
      "clip_ratio": 0.0002005309797823429,
      "completion_length": 219.5416717529297,
      "epoch": 0.00023598132138434973,
      "grad_norm": 0.08387522399425507,
      "kl": 0.0009274522599298507,
      "learning_rate": 7.063711911357341e-07,
      "loss": 0.0284,
      "reward": 0.1666666716337204,
      "reward_std": 0.16661179438233376,
      "rewards/countdown_reward_func": 0.1666666641831398,
      "step": 85,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 9.077705180970952e-05,
      "epoch": 0.0002387575722241656,
      "grad_norm": 0.07613258063793182,
      "kl": 0.0010448671237099916,
      "learning_rate": 7.146814404432133e-07,
      "loss": 0.028,
      "step": 86
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00024153382306398146,
      "grad_norm": 0.11073552072048187,
      "kl": 0.000871124560944736,
      "learning_rate": 7.229916897506925e-07,
      "loss": 0.0284,
      "step": 87
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00024431007390379733,
      "grad_norm": 0.08452224731445312,
      "kl": 0.0010483066726010293,
      "learning_rate": 7.313019390581718e-07,
      "loss": 0.0288,
      "step": 88
    },
    {
      "clip_ratio": 0.00011130899656563997,
      "epoch": 0.0002470863247436132,
      "grad_norm": 0.0861300677061081,
      "kl": 0.0009267764107789844,
      "learning_rate": 7.39612188365651e-07,
      "loss": 0.028,
      "step": 89
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0002498625755834291,
      "grad_norm": 0.07036253809928894,
      "kl": 0.0008911852783057839,
      "learning_rate": 7.479224376731302e-07,
      "loss": 0.028,
      "step": 90
    },
    {
      "clip_ratio": 0.0003118399763479829,
      "epoch": 0.000252638826423245,
      "grad_norm": 0.09484658390283585,
      "kl": 0.0009728774311952293,
      "learning_rate": 7.562326869806093e-07,
      "loss": 0.0283,
      "step": 91
    },
    {
      "clip_ratio": 0.00011130899656563997,
      "epoch": 0.00025541507726306085,
      "grad_norm": 0.07554644346237183,
      "kl": 0.0008746770035941154,
      "learning_rate": 7.645429362880887e-07,
      "loss": 0.0278,
      "step": 92
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00025819132810287674,
      "grad_norm": 0.1146901547908783,
      "kl": 0.0008961272542364895,
      "learning_rate": 7.728531855955679e-07,
      "loss": 0.0286,
      "step": 93
    },
    {
      "clip_ratio": 8.191350207198411e-05,
      "epoch": 0.00026096757894269264,
      "grad_norm": 0.08794571459293365,
      "kl": 0.0010003510979004204,
      "learning_rate": 7.811634349030472e-07,
      "loss": 0.0281,
      "step": 94
    },
    {
      "clip_ratio": 0.00023231577506521717,
      "epoch": 0.00026374382978250853,
      "grad_norm": 0.08675868809223175,
      "kl": 0.0009201938519254327,
      "learning_rate": 7.894736842105263e-07,
      "loss": 0.028,
      "step": 95
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00026652008062232437,
      "grad_norm": 0.07415983825922012,
      "kl": 0.0008623613393865526,
      "learning_rate": 7.977839335180056e-07,
      "loss": 0.028,
      "step": 96
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 240.6041717529297,
      "epoch": 0.00026929633146214026,
      "grad_norm": 0.05498122051358223,
      "kl": 0.000935161137022078,
      "learning_rate": 8.060941828254847e-07,
      "loss": 0.0019,
      "reward": 0.18958335369825363,
      "reward_std": 0.20226776599884033,
      "rewards/countdown_reward_func": 0.18958335369825363,
      "step": 97,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 8.741259080125019e-05,
      "epoch": 0.00027207258230195616,
      "grad_norm": 0.0805509015917778,
      "kl": 0.0009494524856563658,
      "learning_rate": 8.14404432132964e-07,
      "loss": 0.0017,
      "step": 98
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00027484883314177205,
      "grad_norm": 0.06182171031832695,
      "kl": 0.0010764948674477637,
      "learning_rate": 8.227146814404432e-07,
      "loss": 0.0018,
      "step": 99
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0002776250839815879,
      "grad_norm": 0.05677403509616852,
      "kl": 0.000932041322812438,
      "learning_rate": 8.310249307479226e-07,
      "loss": 0.0022,
      "step": 100
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0002804013348214038,
      "grad_norm": 0.06866513192653656,
      "kl": 0.0009856745309662074,
      "learning_rate": 8.393351800554017e-07,
      "loss": 0.0018,
      "step": 101
    },
    {
      "clip_ratio": 0.00016465802764287218,
      "epoch": 0.0002831775856612197,
      "grad_norm": 0.0790778174996376,
      "kl": 0.0009655553149059415,
      "learning_rate": 8.47645429362881e-07,
      "loss": 0.0018,
      "step": 102
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0002859538365010355,
      "grad_norm": 0.06253324449062347,
      "kl": 0.0010523943637963384,
      "learning_rate": 8.559556786703601e-07,
      "loss": 0.0017,
      "step": 103
    },
    {
      "clip_ratio": 0.00016276042151730508,
      "epoch": 0.0002887300873408514,
      "grad_norm": 0.08336114138364792,
      "kl": 0.0009701263043098152,
      "learning_rate": 8.642659279778394e-07,
      "loss": 0.002,
      "step": 104
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0002915063381806673,
      "grad_norm": 0.05647846683859825,
      "kl": 0.0010485479142516851,
      "learning_rate": 8.725761772853186e-07,
      "loss": 0.0017,
      "step": 105
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0002942825890204832,
      "grad_norm": 0.07278945297002792,
      "kl": 0.0010502847435418516,
      "learning_rate": 8.808864265927979e-07,
      "loss": 0.0019,
      "step": 106
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00029705883986029904,
      "grad_norm": 0.06493644416332245,
      "kl": 0.0009983716008719057,
      "learning_rate": 8.89196675900277e-07,
      "loss": 0.0018,
      "step": 107
    },
    {
      "clip_ratio": 0.00026072279433719814,
      "epoch": 0.00029983509070011493,
      "grad_norm": 0.07456071674823761,
      "kl": 0.0010671942727640271,
      "learning_rate": 8.975069252077562e-07,
      "loss": 0.0021,
      "step": 108
    },
    {
      "clip_ratio": 0.00034843204775825143,
      "completion_length": 220.12500762939453,
      "epoch": 0.0003026113415399308,
      "grad_norm": 0.09195102006196976,
      "kl": 0.001304664183408022,
      "learning_rate": 9.058171745152355e-07,
      "loss": -0.0043,
      "reward": 0.260416679084301,
      "reward_std": 0.29160892963409424,
      "rewards/countdown_reward_func": 0.2604166641831398,
      "step": 109,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 0.00044802867341786623,
      "epoch": 0.0003053875923797467,
      "grad_norm": 0.11659281700849533,
      "kl": 0.001220056030433625,
      "learning_rate": 9.141274238227146e-07,
      "loss": -0.0037,
      "step": 110
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00030816384321956256,
      "grad_norm": 0.11090529710054398,
      "kl": 0.0011154624517075717,
      "learning_rate": 9.22437673130194e-07,
      "loss": -0.0036,
      "step": 111
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00031094009405937845,
      "grad_norm": 0.11202968657016754,
      "kl": 0.0010931476717814803,
      "learning_rate": 9.307479224376731e-07,
      "loss": -0.0036,
      "step": 112
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00031371634489919434,
      "grad_norm": 0.11670778691768646,
      "kl": 0.001031600550049916,
      "learning_rate": 9.390581717451524e-07,
      "loss": -0.0045,
      "step": 113
    },
    {
      "clip_ratio": 8.710801193956286e-05,
      "epoch": 0.0003164925957390102,
      "grad_norm": 0.10784825682640076,
      "kl": 0.0010588545119389892,
      "learning_rate": 9.473684210526316e-07,
      "loss": -0.0044,
      "step": 114
    },
    {
      "clip_ratio": 0.00035092979669570923,
      "epoch": 0.0003192688465788261,
      "grad_norm": 0.12178794294595718,
      "kl": 0.0014504955615848303,
      "learning_rate": 9.55678670360111e-07,
      "loss": -0.0033,
      "step": 115
    },
    {
      "clip_ratio": 0.00010288065823260695,
      "epoch": 0.00032204509741864197,
      "grad_norm": 0.092310331761837,
      "kl": 0.0012508891522884369,
      "learning_rate": 9.6398891966759e-07,
      "loss": -0.0034,
      "step": 116
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00032482134825845786,
      "grad_norm": 0.10470046103000641,
      "kl": 0.0011784560047090054,
      "learning_rate": 9.722991689750693e-07,
      "loss": -0.0037,
      "step": 117
    },
    {
      "clip_ratio": 8.710801193956286e-05,
      "epoch": 0.0003275975990982737,
      "grad_norm": 0.10416633635759354,
      "kl": 0.0012174599687568843,
      "learning_rate": 9.806094182825484e-07,
      "loss": -0.0036,
      "step": 118
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0003303738499380896,
      "grad_norm": 0.12231718003749847,
      "kl": 0.0010898615000769496,
      "learning_rate": 9.889196675900277e-07,
      "loss": -0.0035,
      "step": 119
    },
    {
      "clip_ratio": 0.00019340052676852793,
      "epoch": 0.0003331501007779055,
      "grad_norm": 0.11489807814359665,
      "kl": 0.001152284734416753,
      "learning_rate": 9.972299168975068e-07,
      "loss": -0.0045,
      "step": 120
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 221.6041717529297,
      "epoch": 0.0003359263516177214,
      "grad_norm": 0.05373651906847954,
      "kl": 0.0010495753376744688,
      "learning_rate": 1.0055401662049862e-06,
      "loss": -0.014,
      "reward": 0.19166668504476547,
      "reward_std": 0.15476077795028687,
      "rewards/countdown_reward_func": 0.19166666641831398,
      "step": 121,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0003387026024575372,
      "grad_norm": 0.07888471335172653,
      "kl": 0.0014498817035928369,
      "learning_rate": 1.0138504155124655e-06,
      "loss": -0.0136,
      "step": 122
    },
    {
      "clip_ratio": 9.491267701378092e-05,
      "epoch": 0.0003414788532973531,
      "grad_norm": 0.11604306101799011,
      "kl": 0.0012312422040849924,
      "learning_rate": 1.0221606648199446e-06,
      "loss": -0.0136,
      "step": 123
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.000344255104137169,
      "grad_norm": 0.059598926454782486,
      "kl": 0.001186943962238729,
      "learning_rate": 1.030470914127424e-06,
      "loss": -0.0142,
      "step": 124
    },
    {
      "clip_ratio": 9.307520667789504e-05,
      "epoch": 0.0003470313549769849,
      "grad_norm": 0.08821563422679901,
      "kl": 0.0014322291244752705,
      "learning_rate": 1.0387811634349032e-06,
      "loss": -0.0145,
      "step": 125
    },
    {
      "clip_ratio": 9.491267701378092e-05,
      "epoch": 0.00034980760581680074,
      "grad_norm": 0.08867970108985901,
      "kl": 0.0012800320982933044,
      "learning_rate": 1.0470914127423823e-06,
      "loss": -0.014,
      "step": 126
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00035258385665661664,
      "grad_norm": 0.05335162952542305,
      "kl": 0.0010596674110274762,
      "learning_rate": 1.0554016620498616e-06,
      "loss": -0.0142,
      "step": 127
    },
    {
      "clip_ratio": 0.00040634788456372917,
      "epoch": 0.00035536010749643253,
      "grad_norm": 0.08012160658836365,
      "kl": 0.0014414938050322235,
      "learning_rate": 1.0637119113573407e-06,
      "loss": -0.0138,
      "step": 128
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00035813635833624837,
      "grad_norm": 0.13474638760089874,
      "kl": 0.0013173840707167983,
      "learning_rate": 1.0720221606648198e-06,
      "loss": -0.0142,
      "step": 129
    },
    {
      "clip_ratio": 0.00018023690790869296,
      "epoch": 0.00036091260917606426,
      "grad_norm": 0.05427733436226845,
      "kl": 0.0011062846169807017,
      "learning_rate": 1.0803324099722992e-06,
      "loss": -0.0142,
      "step": 130
    },
    {
      "clip_ratio": 9.307520667789504e-05,
      "epoch": 0.00036368886001588016,
      "grad_norm": 0.0898558497428894,
      "kl": 0.0013623088016174734,
      "learning_rate": 1.0886426592797783e-06,
      "loss": -0.0141,
      "step": 131
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00036646511085569605,
      "grad_norm": 0.07763069868087769,
      "kl": 0.0013295715907588601,
      "learning_rate": 1.0969529085872576e-06,
      "loss": -0.0142,
      "step": 132
    },
    {
      "clip_ratio": 0.00016427145601483062,
      "completion_length": 248.33333587646484,
      "epoch": 0.0003692413616955119,
      "grad_norm": 0.07501526921987534,
      "kl": 0.0011433768086135387,
      "learning_rate": 1.1052631578947369e-06,
      "loss": 0.0024,
      "reward": 0.2458333671092987,
      "reward_std": 0.23012542724609375,
      "rewards/countdown_reward_func": 0.2458333522081375,
      "step": 133,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0001751334930304438,
      "epoch": 0.0003720176125353278,
      "grad_norm": 0.12185314297676086,
      "kl": 0.0012197635951451957,
      "learning_rate": 1.1135734072022162e-06,
      "loss": 0.0026,
      "step": 134
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0003747938633751437,
      "grad_norm": 0.08777681738138199,
      "kl": 0.0011197520070709288,
      "learning_rate": 1.1218836565096953e-06,
      "loss": 0.0021,
      "step": 135
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00037757011421495957,
      "grad_norm": 0.08989999443292618,
      "kl": 0.0010376300197094679,
      "learning_rate": 1.1301939058171746e-06,
      "loss": 0.0023,
      "step": 136
    },
    {
      "clip_ratio": 8.223684562835842e-05,
      "epoch": 0.0003803463650547754,
      "grad_norm": 0.08074529469013214,
      "kl": 0.0010572479804977775,
      "learning_rate": 1.1385041551246537e-06,
      "loss": 0.0019,
      "step": 137
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0003831226158945913,
      "grad_norm": 0.08292272686958313,
      "kl": 0.0011392622254788876,
      "learning_rate": 1.146814404432133e-06,
      "loss": 0.0022,
      "step": 138
    },
    {
      "clip_ratio": 8.289124525617808e-05,
      "epoch": 0.0003858988667344072,
      "grad_norm": 0.0767727866768837,
      "kl": 0.001106983982026577,
      "learning_rate": 1.1551246537396122e-06,
      "loss": 0.0019,
      "step": 139
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0003886751175742231,
      "grad_norm": 0.0795488953590393,
      "kl": 0.0012870717328041792,
      "learning_rate": 1.1634349030470915e-06,
      "loss": 0.003,
      "step": 140
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00039145136841403893,
      "grad_norm": 0.0856185331940651,
      "kl": 0.001159544801339507,
      "learning_rate": 1.1717451523545706e-06,
      "loss": 0.0018,
      "step": 141
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0003942276192538548,
      "grad_norm": 0.08445297926664352,
      "kl": 0.0011639554286375642,
      "learning_rate": 1.1800554016620499e-06,
      "loss": 0.0025,
      "step": 142
    },
    {
      "clip_ratio": 8.223684562835842e-05,
      "epoch": 0.0003970038700936707,
      "grad_norm": 0.07916339486837387,
      "kl": 0.0011218629661016166,
      "learning_rate": 1.188365650969529e-06,
      "loss": 0.0025,
      "step": 143
    },
    {
      "clip_ratio": 0.00024620501790195704,
      "epoch": 0.00039978012093348656,
      "grad_norm": 0.0729660838842392,
      "kl": 0.0012778243399225175,
      "learning_rate": 1.1966759002770083e-06,
      "loss": 0.0024,
      "step": 144
    },
    {
      "clip_ratio": 8.967001485871151e-05,
      "completion_length": 232.02083587646484,
      "epoch": 0.00040255637177330245,
      "grad_norm": 0.16677485406398773,
      "kl": 0.0013882183120585978,
      "learning_rate": 1.2049861495844876e-06,
      "loss": 0.014,
      "reward": 0.30416667461395264,
      "reward_std": 0.36280614137649536,
      "rewards/countdown_reward_func": 0.30416667461395264,
      "step": 145,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 0.00010926573304459453,
      "epoch": 0.00040533262261311834,
      "grad_norm": 0.17154261469841003,
      "kl": 0.0015535103739239275,
      "learning_rate": 1.213296398891967e-06,
      "loss": 0.0145,
      "step": 146
    },
    {
      "clip_ratio": 8.526603050995618e-05,
      "epoch": 0.00040810887345293424,
      "grad_norm": 0.11442252993583679,
      "kl": 0.001147409901022911,
      "learning_rate": 1.221606648199446e-06,
      "loss": 0.015,
      "step": 147
    },
    {
      "clip_ratio": 8.967001485871151e-05,
      "epoch": 0.0004108851242927501,
      "grad_norm": 0.11827956140041351,
      "kl": 0.0013873514835722744,
      "learning_rate": 1.2299168975069254e-06,
      "loss": 0.0147,
      "step": 148
    },
    {
      "clip_ratio": 0.0008980906713986769,
      "epoch": 0.00041366137513256597,
      "grad_norm": 0.10520009696483612,
      "kl": 0.001313833869062364,
      "learning_rate": 1.2382271468144045e-06,
      "loss": 0.0145,
      "step": 149
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00041643762597238186,
      "grad_norm": 0.11322156339883804,
      "kl": 0.0012296418426558375,
      "learning_rate": 1.2465373961218836e-06,
      "loss": 0.0146,
      "step": 150
    },
    {
      "clip_ratio": 0.00025922466011252254,
      "epoch": 0.00041921387681219776,
      "grad_norm": 0.16956517100334167,
      "kl": 0.0012730106245726347,
      "learning_rate": 1.2548476454293629e-06,
      "loss": 0.0134,
      "step": 151
    },
    {
      "clip_ratio": 0.00010926573304459453,
      "epoch": 0.0004219901276520136,
      "grad_norm": 0.1768382489681244,
      "kl": 0.0016011170810088515,
      "learning_rate": 1.263157894736842e-06,
      "loss": 0.0138,
      "step": 152
    },
    {
      "clip_ratio": 8.428860746789724e-05,
      "epoch": 0.0004247663784918295,
      "grad_norm": 0.11108796298503876,
      "kl": 0.0012242573429830372,
      "learning_rate": 1.2714681440443213e-06,
      "loss": 0.0142,
      "step": 153
    },
    {
      "clip_ratio": 0.00016696537932148203,
      "epoch": 0.0004275426293316454,
      "grad_norm": 0.13032646477222443,
      "kl": 0.0015075987321324646,
      "learning_rate": 1.2797783933518006e-06,
      "loss": 0.014,
      "step": 154
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0004303188801714613,
      "grad_norm": 0.09640121459960938,
      "kl": 0.0013893723953515291,
      "learning_rate": 1.28808864265928e-06,
      "loss": 0.0138,
      "step": 155
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0004330951310112771,
      "grad_norm": 0.10653083771467209,
      "kl": 0.0013818400911986828,
      "learning_rate": 1.296398891966759e-06,
      "loss": 0.014,
      "step": 156
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "completion_length": 235.75,
      "epoch": 0.000435871381851093,
      "grad_norm": 0.05519864708185196,
      "kl": 0.001432242221198976,
      "learning_rate": 1.3047091412742383e-06,
      "loss": 0.0092,
      "reward": 0.24583333730697632,
      "reward_std": 0.12264448031783104,
      "rewards/countdown_reward_func": 0.24583333730697632,
      "step": 157,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0004386476326909089,
      "grad_norm": 0.09035968035459518,
      "kl": 0.0013828090741299093,
      "learning_rate": 1.3130193905817175e-06,
      "loss": 0.0092,
      "step": 158
    },
    {
      "clip_ratio": 0.0005962533032288775,
      "epoch": 0.00044142388353072474,
      "grad_norm": 0.1271812468767166,
      "kl": 0.0021160971373319626,
      "learning_rate": 1.3213296398891968e-06,
      "loss": 0.0093,
      "step": 159
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00044420013437054064,
      "grad_norm": 0.0863696038722992,
      "kl": 0.001588326005730778,
      "learning_rate": 1.3296398891966759e-06,
      "loss": 0.0096,
      "step": 160
    },
    {
      "clip_ratio": 0.00016276042151730508,
      "epoch": 0.00044697638521035653,
      "grad_norm": 0.061348333954811096,
      "kl": 0.0016502806101925671,
      "learning_rate": 1.3379501385041552e-06,
      "loss": 0.0092,
      "step": 161
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0004497526360501724,
      "grad_norm": 0.04854971170425415,
      "kl": 0.0016185293206945062,
      "learning_rate": 1.3462603878116343e-06,
      "loss": 0.0094,
      "step": 162
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00045252888688998826,
      "grad_norm": 0.060902826488018036,
      "kl": 0.0025005778297781944,
      "learning_rate": 1.3545706371191136e-06,
      "loss": 0.009,
      "step": 163
    },
    {
      "clip_ratio": 0.00011261261533945799,
      "epoch": 0.00045530513772980416,
      "grad_norm": 0.08311949670314789,
      "kl": 0.0018348235171288252,
      "learning_rate": 1.3628808864265927e-06,
      "loss": 0.0094,
      "step": 164
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.00045808138856962005,
      "grad_norm": 0.1301003396511078,
      "kl": 0.0029168009059503675,
      "learning_rate": 1.371191135734072e-06,
      "loss": 0.01,
      "step": 165
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00046085763940943594,
      "grad_norm": 0.07615955919027328,
      "kl": 0.002113637514412403,
      "learning_rate": 1.3795013850415513e-06,
      "loss": 0.009,
      "step": 166
    },
    {
      "clip_ratio": 9.67492233030498e-05,
      "epoch": 0.0004636338902492518,
      "grad_norm": 0.061112839728593826,
      "kl": 0.001958071778062731,
      "learning_rate": 1.3878116343490307e-06,
      "loss": 0.0092,
      "step": 167
    },
    {
      "clip_ratio": 0.00017946927982848138,
      "epoch": 0.0004664101410890677,
      "grad_norm": 0.04938596114516258,
      "kl": 0.002263071248307824,
      "learning_rate": 1.3961218836565098e-06,
      "loss": 0.0092,
      "step": 168
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 212.87500762939453,
      "epoch": 0.00046918639192888357,
      "grad_norm": 0.08904451131820679,
      "kl": 0.0021521300077438354,
      "learning_rate": 1.404432132963989e-06,
      "loss": -0.0054,
      "reward": 0.3541666716337204,
      "reward_std": 0.3927241712808609,
      "rewards/countdown_reward_func": 0.3541666716337204,
      "step": 169,
      "zero_std_ratio": 0.0
    },
    {
      "clip_ratio": 0.000244140625,
      "epoch": 0.00047196264276869946,
      "grad_norm": 0.11747531592845917,
      "kl": 0.0023746880469843745,
      "learning_rate": 1.4127423822714682e-06,
      "loss": -0.0057,
      "step": 170
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0004747388936085153,
      "grad_norm": 0.10242423415184021,
      "kl": 0.0022907587699592113,
      "learning_rate": 1.4210526315789473e-06,
      "loss": -0.0055,
      "step": 171
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0004775151444483312,
      "grad_norm": 0.09900445491075516,
      "kl": 0.0021405539009720087,
      "learning_rate": 1.4293628808864266e-06,
      "loss": -0.0054,
      "step": 172
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0004802913952881471,
      "grad_norm": 0.13611117005348206,
      "kl": 0.0026756724109873176,
      "learning_rate": 1.4376731301939057e-06,
      "loss": -0.0062,
      "step": 173
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00048306764612796293,
      "grad_norm": 0.11884411424398422,
      "kl": 0.0026626097969710827,
      "learning_rate": 1.445983379501385e-06,
      "loss": -0.0055,
      "step": 174
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0004858438969677788,
      "grad_norm": 0.09348145127296448,
      "kl": 0.002659423043951392,
      "learning_rate": 1.4542936288088643e-06,
      "loss": -0.0048,
      "step": 175
    },
    {
      "clip_ratio": 0.000244140625,
      "epoch": 0.0004886201478075947,
      "grad_norm": 0.11439940333366394,
      "kl": 0.0028921624179929495,
      "learning_rate": 1.4626038781163436e-06,
      "loss": -0.0065,
      "step": 176
    },
    {
      "clip_ratio": 9.476876584812999e-05,
      "epoch": 0.0004913963986474106,
      "grad_norm": 0.21524833142757416,
      "kl": 0.002880470361560583,
      "learning_rate": 1.4709141274238228e-06,
      "loss": -0.0057,
      "step": 177
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0004941726494872264,
      "grad_norm": 0.10695669800043106,
      "kl": 0.0028178965440019965,
      "learning_rate": 1.479224376731302e-06,
      "loss": -0.006,
      "step": 178
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0004969489003270424,
      "grad_norm": 0.12360948324203491,
      "kl": 0.0034589068964123726,
      "learning_rate": 1.4875346260387812e-06,
      "loss": -0.006,
      "step": 179
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0004997251511668582,
      "grad_norm": 0.1427268236875534,
      "kl": 0.0033028185134753585,
      "learning_rate": 1.4958448753462605e-06,
      "loss": -0.006,
      "step": 180
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 237.1041717529297,
      "epoch": 0.0005025014020066741,
      "grad_norm": 0.06570961326360703,
      "kl": 0.0035237747943028808,
      "learning_rate": 1.5041551246537396e-06,
      "loss": 0.0151,
      "reward": 0.166666679084301,
      "reward_std": 0.19649019464850426,
      "rewards/countdown_reward_func": 0.1666666716337204,
      "step": 181,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00050527765284649,
      "grad_norm": 0.07061938941478729,
      "kl": 0.003632499254308641,
      "learning_rate": 1.5124653739612187e-06,
      "loss": 0.0151,
      "step": 182
    },
    {
      "clip_ratio": 0.00017770155682228506,
      "epoch": 0.0005080539036863059,
      "grad_norm": 0.09591417759656906,
      "kl": 0.004101799800992012,
      "learning_rate": 1.5207756232686982e-06,
      "loss": 0.0149,
      "step": 183
    },
    {
      "clip_ratio": 8.169934881152585e-05,
      "epoch": 0.0005108301545261217,
      "grad_norm": 0.0730750560760498,
      "kl": 0.003928871126845479,
      "learning_rate": 1.5290858725761773e-06,
      "loss": 0.015,
      "step": 184
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0005136064053659377,
      "grad_norm": 0.08014266192913055,
      "kl": 0.003984199371188879,
      "learning_rate": 1.5373961218836564e-06,
      "loss": 0.0144,
      "step": 185
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0005163826562057535,
      "grad_norm": 0.08353135734796524,
      "kl": 0.004139116033911705,
      "learning_rate": 1.5457063711911357e-06,
      "loss": 0.0146,
      "step": 186
    },
    {
      "clip_ratio": 0.00018080235895467922,
      "epoch": 0.0005191589070455693,
      "grad_norm": 0.07052478194236755,
      "kl": 0.004334121011197567,
      "learning_rate": 1.554016620498615e-06,
      "loss": 0.0151,
      "step": 187
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0005219351578853853,
      "grad_norm": 0.12264339625835419,
      "kl": 0.004643098916858435,
      "learning_rate": 1.5623268698060944e-06,
      "loss": 0.0148,
      "step": 188
    },
    {
      "clip_ratio": 8.7596352386754e-05,
      "epoch": 0.0005247114087252011,
      "grad_norm": 0.10199768096208572,
      "kl": 0.00538562866859138,
      "learning_rate": 1.5706371191135735e-06,
      "loss": 0.014,
      "step": 189
    },
    {
      "clip_ratio": 8.169934881152585e-05,
      "epoch": 0.0005274876595650171,
      "grad_norm": 0.07175012677907944,
      "kl": 0.005290654953569174,
      "learning_rate": 1.5789473684210526e-06,
      "loss": 0.0147,
      "step": 190
    },
    {
      "clip_ratio": 0.00017524592112749815,
      "epoch": 0.0005302639104048329,
      "grad_norm": 0.08007644861936569,
      "kl": 0.0052713199984282255,
      "learning_rate": 1.5872576177285321e-06,
      "loss": 0.0146,
      "step": 191
    },
    {
      "clip_ratio": 8.538251131540164e-05,
      "epoch": 0.0005330401612446487,
      "grad_norm": 0.08058490604162216,
      "kl": 0.005752292228862643,
      "learning_rate": 1.5955678670360112e-06,
      "loss": 0.0142,
      "step": 192
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 224.25000762939453,
      "epoch": 0.0005358164120844647,
      "grad_norm": 0.07866574823856354,
      "kl": 0.006615105550736189,
      "learning_rate": 1.6038781163434903e-06,
      "loss": -0.005,
      "reward": 0.1458333507180214,
      "reward_std": 0.1595480851829052,
      "rewards/countdown_reward_func": 0.1458333432674408,
      "step": 193,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0005385926629242805,
      "grad_norm": 0.10931258648633957,
      "kl": 0.006460321601480246,
      "learning_rate": 1.6121883656509694e-06,
      "loss": -0.0049,
      "step": 194
    },
    {
      "clip_ratio": 8.37240440887399e-05,
      "epoch": 0.0005413689137640964,
      "grad_norm": 0.08174652606248856,
      "kl": 0.0075355947483330965,
      "learning_rate": 1.6204986149584487e-06,
      "loss": -0.0045,
      "step": 195
    },
    {
      "clip_ratio": 8.37240440887399e-05,
      "epoch": 0.0005441451646039123,
      "grad_norm": 0.08151189982891083,
      "kl": 0.006791774649173021,
      "learning_rate": 1.628808864265928e-06,
      "loss": -0.0046,
      "step": 196
    },
    {
      "clip_ratio": 9.077705180970952e-05,
      "epoch": 0.0005469214154437282,
      "grad_norm": 0.06075465306639671,
      "kl": 0.007927711587399244,
      "learning_rate": 1.6371191135734074e-06,
      "loss": -0.005,
      "step": 197
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0005496976662835441,
      "grad_norm": 0.06800781190395355,
      "kl": 0.007068223087117076,
      "learning_rate": 1.6454293628808865e-06,
      "loss": -0.005,
      "step": 198
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0005524739171233599,
      "grad_norm": 0.08043520897626877,
      "kl": 0.008283360861241817,
      "learning_rate": 1.6537396121883656e-06,
      "loss": -0.0051,
      "step": 199
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0005552501679631758,
      "grad_norm": 0.08258956670761108,
      "kl": 0.007602859288454056,
      "learning_rate": 1.662049861495845e-06,
      "loss": -0.0056,
      "step": 200
    },
    {
      "clip_ratio": 8.37240440887399e-05,
      "epoch": 0.0005580264188029917,
      "grad_norm": 0.08197806775569916,
      "kl": 0.009075871668756008,
      "learning_rate": 1.6703601108033242e-06,
      "loss": -0.0042,
      "step": 201
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0005608026696428076,
      "grad_norm": 0.08293389528989792,
      "kl": 0.0072558128740638494,
      "learning_rate": 1.6786703601108033e-06,
      "loss": -0.0044,
      "step": 202
    },
    {
      "clip_ratio": 0.00018364480638410896,
      "epoch": 0.0005635789204826234,
      "grad_norm": 0.0675339549779892,
      "kl": 0.008414975367486477,
      "learning_rate": 1.6869806094182824e-06,
      "loss": -0.0051,
      "step": 203
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0005663551713224394,
      "grad_norm": 0.0665825828909874,
      "kl": 0.007153096608817577,
      "learning_rate": 1.695290858725762e-06,
      "loss": -0.0048,
      "step": 204
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 216.5416717529297,
      "epoch": 0.0005691314221622552,
      "grad_norm": 0.0898197814822197,
      "kl": 0.0062689268961548805,
      "learning_rate": 1.703601108033241e-06,
      "loss": -0.0212,
      "reward": 0.24375002086162567,
      "reward_std": 0.23638741672039032,
      "rewards/countdown_reward_func": 0.24375001341104507,
      "step": 205,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.0007762892200844362,
      "epoch": 0.000571907673002071,
      "grad_norm": 0.11292041838169098,
      "kl": 0.0070912938099354506,
      "learning_rate": 1.7119113573407201e-06,
      "loss": -0.0204,
      "step": 206
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.000574683923841887,
      "grad_norm": 0.1421259045600891,
      "kl": 0.006881088018417358,
      "learning_rate": 1.7202216066481995e-06,
      "loss": -0.0206,
      "step": 207
    },
    {
      "clip_ratio": 0.00012413108197506517,
      "epoch": 0.0005774601746817028,
      "grad_norm": 0.12615558505058289,
      "kl": 0.006374599179252982,
      "learning_rate": 1.7285318559556788e-06,
      "loss": -0.022,
      "step": 208
    },
    {
      "clip_ratio": 0.0001857691750046797,
      "epoch": 0.0005802364255215188,
      "grad_norm": 0.11529010534286499,
      "kl": 0.007537527941167355,
      "learning_rate": 1.736842105263158e-06,
      "loss": -0.0217,
      "step": 209
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0005830126763613346,
      "grad_norm": 0.12297216802835464,
      "kl": 0.006081829313188791,
      "learning_rate": 1.7451523545706372e-06,
      "loss": -0.0219,
      "step": 210
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0005857889272011504,
      "grad_norm": 0.09268545359373093,
      "kl": 0.005436737323179841,
      "learning_rate": 1.7534626038781163e-06,
      "loss": -0.0217,
      "step": 211
    },
    {
      "clip_ratio": 0.00048481223348062485,
      "epoch": 0.0005885651780409664,
      "grad_norm": 0.09842047840356827,
      "kl": 0.005601341603323817,
      "learning_rate": 1.7617728531855958e-06,
      "loss": -0.021,
      "step": 212
    },
    {
      "clip_ratio": 0.00019936203898396343,
      "epoch": 0.0005913414288807822,
      "grad_norm": 0.13735024631023407,
      "kl": 0.005223254906013608,
      "learning_rate": 1.770083102493075e-06,
      "loss": -0.022,
      "step": 213
    },
    {
      "clip_ratio": 0.00022600556985707954,
      "epoch": 0.0005941176797205981,
      "grad_norm": 0.11505624651908875,
      "kl": 0.004508420126512647,
      "learning_rate": 1.778393351800554e-06,
      "loss": -0.0228,
      "step": 214
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.000596893930560414,
      "grad_norm": 0.10706788301467896,
      "kl": 0.005065717967227101,
      "learning_rate": 1.7867036011080331e-06,
      "loss": -0.022,
      "step": 215
    },
    {
      "clip_ratio": 8.6088155512698e-05,
      "epoch": 0.0005996701814002299,
      "grad_norm": 0.11780065298080444,
      "kl": 0.004196330206468701,
      "learning_rate": 1.7950138504155125e-06,
      "loss": -0.0226,
      "step": 216
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 229.7291717529297,
      "epoch": 0.0006024464322400457,
      "grad_norm": 0.07949130237102509,
      "kl": 0.004571998957544565,
      "learning_rate": 1.8033240997229918e-06,
      "loss": 0.0072,
      "reward": 0.24583334475755692,
      "reward_std": 0.22264151275157928,
      "rewards/countdown_reward_func": 0.24583334475755692,
      "step": 217,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0006052226830798616,
      "grad_norm": 0.10650928318500519,
      "kl": 0.004555474733933806,
      "learning_rate": 1.811634349030471e-06,
      "loss": 0.0074,
      "step": 218
    },
    {
      "clip_ratio": 9.95222944766283e-05,
      "epoch": 0.0006079989339196775,
      "grad_norm": 0.10155003517866135,
      "kl": 0.0035600599367171526,
      "learning_rate": 1.8199445983379502e-06,
      "loss": 0.0071,
      "step": 219
    },
    {
      "clip_ratio": 0.00018531362002249807,
      "epoch": 0.0006107751847594934,
      "grad_norm": 0.09107557684183121,
      "kl": 0.0034167662961408496,
      "learning_rate": 1.8282548476454293e-06,
      "loss": 0.0078,
      "step": 220
    },
    {
      "clip_ratio": 9.95222944766283e-05,
      "epoch": 0.0006135514355993093,
      "grad_norm": 0.13129286468029022,
      "kl": 0.003628110629506409,
      "learning_rate": 1.8365650969529088e-06,
      "loss": 0.008,
      "step": 221
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0006163276864391251,
      "grad_norm": 0.07515281438827515,
      "kl": 0.0033600571332499385,
      "learning_rate": 1.844875346260388e-06,
      "loss": 0.0079,
      "step": 222
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0006191039372789411,
      "grad_norm": 0.07991602271795273,
      "kl": 0.0036217205924913287,
      "learning_rate": 1.853185595567867e-06,
      "loss": 0.0073,
      "step": 223
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0006218801881187569,
      "grad_norm": 0.1082308441400528,
      "kl": 0.0038258974673226476,
      "learning_rate": 1.8614958448753461e-06,
      "loss": 0.0074,
      "step": 224
    },
    {
      "clip_ratio": 0.00037994710146449506,
      "epoch": 0.0006246564389585727,
      "grad_norm": 0.09276928007602692,
      "kl": 0.0031863467302173376,
      "learning_rate": 1.8698060941828257e-06,
      "loss": 0.0073,
      "step": 225
    },
    {
      "clip_ratio": 8.25082533992827e-05,
      "epoch": 0.0006274326897983887,
      "grad_norm": 0.09415683150291443,
      "kl": 0.002969763823784888,
      "learning_rate": 1.8781163434903048e-06,
      "loss": 0.0079,
      "step": 226
    },
    {
      "clip_ratio": 0.00026924171834252775,
      "epoch": 0.0006302089406382045,
      "grad_norm": 0.12183138728141785,
      "kl": 0.003270600689575076,
      "learning_rate": 1.8864265927977839e-06,
      "loss": 0.0078,
      "step": 227
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0006329851914780204,
      "grad_norm": 0.07732009142637253,
      "kl": 0.002818679786287248,
      "learning_rate": 1.8947368421052632e-06,
      "loss": 0.0079,
      "step": 228
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "completion_length": 227.75000762939453,
      "epoch": 0.0006357614423178363,
      "grad_norm": 0.10973645746707916,
      "kl": 0.003377788234502077,
      "learning_rate": 1.9030470914127425e-06,
      "loss": 0.0024,
      "reward": 0.2458333522081375,
      "reward_std": 0.3024734854698181,
      "rewards/countdown_reward_func": 0.24583334475755692,
      "step": 229,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.0001726210830383934,
      "epoch": 0.0006385376931576522,
      "grad_norm": 0.0772981122136116,
      "kl": 0.003124785842373967,
      "learning_rate": 1.911357340720222e-06,
      "loss": 0.0019,
      "step": 230
    },
    {
      "clip_ratio": 9.286775457439944e-05,
      "epoch": 0.0006413139439974681,
      "grad_norm": 0.2130415439605713,
      "kl": 0.003285923390649259,
      "learning_rate": 1.919667590027701e-06,
      "loss": 0.003,
      "step": 231
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0006440901948372839,
      "grad_norm": 0.09580881893634796,
      "kl": 0.002900973428040743,
      "learning_rate": 1.92797783933518e-06,
      "loss": 0.0023,
      "step": 232
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0006468664456770998,
      "grad_norm": 0.09705420583486557,
      "kl": 0.003031375934369862,
      "learning_rate": 1.9362880886426595e-06,
      "loss": 0.002,
      "step": 233
    },
    {
      "clip_ratio": 9.177679748972878e-05,
      "epoch": 0.0006496426965169157,
      "grad_norm": 0.08564502000808716,
      "kl": 0.0033782614627853036,
      "learning_rate": 1.9445983379501387e-06,
      "loss": 0.0024,
      "step": 234
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0006524189473567316,
      "grad_norm": 0.11757489293813705,
      "kl": 0.003013497218489647,
      "learning_rate": 1.9529085872576178e-06,
      "loss": 0.0018,
      "step": 235
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0006551951981965474,
      "grad_norm": 0.08156201988458633,
      "kl": 0.0028692481573671103,
      "learning_rate": 1.961218836565097e-06,
      "loss": 0.0022,
      "step": 236
    },
    {
      "clip_ratio": 9.124087227974087e-05,
      "epoch": 0.0006579714490363634,
      "grad_norm": 0.20828205347061157,
      "kl": 0.00310861156322062,
      "learning_rate": 1.969529085872576e-06,
      "loss": 0.0024,
      "step": 237
    },
    {
      "clip_ratio": 0.0011520737316459417,
      "epoch": 0.0006607476998761792,
      "grad_norm": 0.09965456277132034,
      "kl": 0.0033284203382208943,
      "learning_rate": 1.9778393351800555e-06,
      "loss": 0.0024,
      "step": 238
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0006635239507159951,
      "grad_norm": 0.08843494206666946,
      "kl": 0.0030406563309952617,
      "learning_rate": 1.9861495844875346e-06,
      "loss": 0.0018,
      "step": 239
    },
    {
      "clip_ratio": 0.00018355359497945756,
      "epoch": 0.000666300201555811,
      "grad_norm": 0.08795179426670074,
      "kl": 0.0030062462901696563,
      "learning_rate": 1.9944598337950137e-06,
      "loss": 0.0022,
      "step": 240
    },
    {
      "clip_ratio": 8.698677993379533e-05,
      "completion_length": 239.81250762939453,
      "epoch": 0.0006690764523956268,
      "grad_norm": 0.06673049926757812,
      "kl": 0.002854794613085687,
      "learning_rate": 2.002770083102493e-06,
      "loss": -0.0041,
      "reward": 0.1875000149011612,
      "reward_std": 0.16496698185801506,
      "rewards/countdown_reward_func": 0.1875000149011612,
      "step": 241,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0006718527032354428,
      "grad_norm": 0.05837683007121086,
      "kl": 0.0029842500807717443,
      "learning_rate": 2.0110803324099723e-06,
      "loss": -0.0044,
      "step": 242
    },
    {
      "clip_ratio": 8.698677993379533e-05,
      "epoch": 0.0006746289540752586,
      "grad_norm": 0.05153853818774223,
      "kl": 0.0027709035202860832,
      "learning_rate": 2.0193905817174514e-06,
      "loss": -0.0041,
      "step": 243
    },
    {
      "clip_ratio": 0.00016836699796840549,
      "epoch": 0.0006774052049150744,
      "grad_norm": 0.09485623240470886,
      "kl": 0.003231698414310813,
      "learning_rate": 2.027700831024931e-06,
      "loss": -0.0041,
      "step": 244
    },
    {
      "clip_ratio": 9.104151831706986e-05,
      "epoch": 0.0006801814557548904,
      "grad_norm": 0.06381552666425705,
      "kl": 0.003178777056746185,
      "learning_rate": 2.03601108033241e-06,
      "loss": -0.0044,
      "step": 245
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0006829577065947062,
      "grad_norm": 0.06970212608575821,
      "kl": 0.0039439547108486295,
      "learning_rate": 2.044321329639889e-06,
      "loss": -0.0044,
      "step": 246
    },
    {
      "clip_ratio": 0.0007053310982882977,
      "epoch": 0.0006857339574345221,
      "grad_norm": 0.05486287549138069,
      "kl": 0.0030296844197437167,
      "learning_rate": 2.0526315789473687e-06,
      "loss": -0.0041,
      "step": 247
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.000688510208274338,
      "grad_norm": 0.05788696929812431,
      "kl": 0.0034874266711995006,
      "learning_rate": 2.060941828254848e-06,
      "loss": -0.0043,
      "step": 248
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0006912864591141539,
      "grad_norm": 0.05053585022687912,
      "kl": 0.003134583937935531,
      "learning_rate": 2.069252077562327e-06,
      "loss": -0.0045,
      "step": 249
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0006940627099539698,
      "grad_norm": 0.08815469592809677,
      "kl": 0.0033259709598496556,
      "learning_rate": 2.0775623268698064e-06,
      "loss": -0.0048,
      "step": 250
    },
    {
      "clip_ratio": 9.104151831706986e-05,
      "epoch": 0.0006968389607937856,
      "grad_norm": 0.06786980479955673,
      "kl": 0.00334134662989527,
      "learning_rate": 2.0858725761772855e-06,
      "loss": -0.0048,
      "step": 251
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0006996152116336015,
      "grad_norm": 0.06675417721271515,
      "kl": 0.0038671550573781133,
      "learning_rate": 2.0941828254847646e-06,
      "loss": -0.0048,
      "step": 252
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 234.33334350585938,
      "epoch": 0.0007023914624734174,
      "grad_norm": 0.10852495580911636,
      "kl": 0.003565722843632102,
      "learning_rate": 2.1024930747922437e-06,
      "loss": 0.0025,
      "reward": 0.24791669100522995,
      "reward_std": 0.22883932292461395,
      "rewards/countdown_reward_func": 0.24791669100522995,
      "step": 253,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.00017266585200559348,
      "epoch": 0.0007051677133132333,
      "grad_norm": 0.10487343370914459,
      "kl": 0.0033710359130054712,
      "learning_rate": 2.1108033240997233e-06,
      "loss": 0.0034,
      "step": 254
    },
    {
      "clip_ratio": 0.00030323654209496453,
      "epoch": 0.0007079439641530491,
      "grad_norm": 0.10246977210044861,
      "kl": 0.003350336686708033,
      "learning_rate": 2.1191135734072024e-06,
      "loss": 0.0024,
      "step": 255
    },
    {
      "clip_ratio": 0.0002672308764886111,
      "epoch": 0.0007107202149928651,
      "grad_norm": 0.11946409195661545,
      "kl": 0.0036674703005701303,
      "learning_rate": 2.1274238227146815e-06,
      "loss": 0.0023,
      "step": 256
    },
    {
      "clip_ratio": 0.0001684882226982154,
      "epoch": 0.0007134964658326809,
      "grad_norm": 0.0756080374121666,
      "kl": 0.0034829488722607493,
      "learning_rate": 2.1357340720221606e-06,
      "loss": 0.0024,
      "step": 257
    },
    {
      "clip_ratio": 0.00016693805810064077,
      "epoch": 0.0007162727166724967,
      "grad_norm": 0.11272068321704865,
      "kl": 0.0037394753890112042,
      "learning_rate": 2.1440443213296397e-06,
      "loss": 0.0026,
      "step": 258
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0007190489675123127,
      "grad_norm": 0.10757721215486526,
      "kl": 0.003679752117022872,
      "learning_rate": 2.152354570637119e-06,
      "loss": 0.0022,
      "step": 259
    },
    {
      "clip_ratio": 0.00017266585200559348,
      "epoch": 0.0007218252183521285,
      "grad_norm": 0.07714775204658508,
      "kl": 0.003283996251411736,
      "learning_rate": 2.1606648199445983e-06,
      "loss": 0.0029,
      "step": 260
    },
    {
      "clip_ratio": 0.00020028267317684367,
      "epoch": 0.0007246014691919445,
      "grad_norm": 0.08071676641702652,
      "kl": 0.0033569439547136426,
      "learning_rate": 2.1689750692520774e-06,
      "loss": 0.0022,
      "step": 261
    },
    {
      "clip_ratio": 0.0005443122354336083,
      "epoch": 0.0007273777200317603,
      "grad_norm": 0.11737988144159317,
      "kl": 0.004172776127234101,
      "learning_rate": 2.1772853185595565e-06,
      "loss": 0.0027,
      "step": 262
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0007301539708715762,
      "grad_norm": 0.0743941143155098,
      "kl": 0.003631085390225053,
      "learning_rate": 2.185595567867036e-06,
      "loss": 0.0024,
      "step": 263
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0007329302217113921,
      "grad_norm": 0.10840493440628052,
      "kl": 0.003976256353780627,
      "learning_rate": 2.193905817174515e-06,
      "loss": 0.0021,
      "step": 264
    },
    {
      "clip_ratio": 0.00016276042151730508,
      "completion_length": 217.4791717529297,
      "epoch": 0.0007357064725512079,
      "grad_norm": 0.08730115741491318,
      "kl": 0.0036497570108622313,
      "learning_rate": 2.2022160664819947e-06,
      "loss": -0.0134,
      "reward": 0.24375002086162567,
      "reward_std": 0.19520405679941177,
      "rewards/countdown_reward_func": 0.24375002086162567,
      "step": 265,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0007384827233910238,
      "grad_norm": 0.07678765058517456,
      "kl": 0.003502070438116789,
      "learning_rate": 2.2105263157894738e-06,
      "loss": -0.0136,
      "step": 266
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0007412589742308397,
      "grad_norm": 0.10239151120185852,
      "kl": 0.004613874014467001,
      "learning_rate": 2.218836565096953e-06,
      "loss": -0.0133,
      "step": 267
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0007440352250706556,
      "grad_norm": 0.09318691492080688,
      "kl": 0.003522416460327804,
      "learning_rate": 2.2271468144044324e-06,
      "loss": -0.0137,
      "step": 268
    },
    {
      "clip_ratio": 0.00012007684563286602,
      "epoch": 0.0007468114759104715,
      "grad_norm": 0.10666222870349884,
      "kl": 0.0034163352102041245,
      "learning_rate": 2.2354570637119115e-06,
      "loss": -0.0136,
      "step": 269
    },
    {
      "clip_ratio": 0.00018601190822664648,
      "epoch": 0.0007495877267502874,
      "grad_norm": 0.07045263051986694,
      "kl": 0.003822776139713824,
      "learning_rate": 2.2437673130193906e-06,
      "loss": -0.0135,
      "step": 270
    },
    {
      "clip_ratio": 0.00016324006719514728,
      "epoch": 0.0007523639775901032,
      "grad_norm": 0.08979963511228561,
      "kl": 0.003612265922129154,
      "learning_rate": 2.25207756232687e-06,
      "loss": -0.0136,
      "step": 271
    },
    {
      "clip_ratio": 9.300595411332324e-05,
      "epoch": 0.0007551402284299191,
      "grad_norm": 0.07281750440597534,
      "kl": 0.0034800268476828933,
      "learning_rate": 2.2603878116343493e-06,
      "loss": -0.014,
      "step": 272
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.000757916479269735,
      "grad_norm": 0.1016596257686615,
      "kl": 0.004669018788263202,
      "learning_rate": 2.2686980609418284e-06,
      "loss": -0.0135,
      "step": 273
    },
    {
      "clip_ratio": 0.00021114865376148373,
      "epoch": 0.0007606927301095508,
      "grad_norm": 0.09079304337501526,
      "kl": 0.0033616855507716537,
      "learning_rate": 2.2770083102493075e-06,
      "loss": -0.0134,
      "step": 274
    },
    {
      "clip_ratio": 0.00019769607024500147,
      "epoch": 0.0007634689809493668,
      "grad_norm": 0.11171045154333115,
      "kl": 0.0033946483163163066,
      "learning_rate": 2.285318559556787e-06,
      "loss": -0.0136,
      "step": 275
    },
    {
      "clip_ratio": 0.00032530390308238566,
      "epoch": 0.0007662452317891826,
      "grad_norm": 0.06686889380216599,
      "kl": 0.004112225957214832,
      "learning_rate": 2.293628808864266e-06,
      "loss": -0.0133,
      "step": 276
    },
    {
      "clip_ratio": 0.000663017388433218,
      "completion_length": 232.50000762939453,
      "epoch": 0.0007690214826289984,
      "grad_norm": 0.09006670117378235,
      "kl": 0.003324171178974211,
      "learning_rate": 2.301939058171745e-06,
      "loss": 0.0007,
      "reward": 0.3541666865348816,
      "reward_std": 0.34993430972099304,
      "rewards/countdown_reward_func": 0.3541666865348816,
      "step": 277,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0007717977334688144,
      "grad_norm": 0.10477007925510406,
      "kl": 0.003485492314212024,
      "learning_rate": 2.3102493074792243e-06,
      "loss": -0.0001,
      "step": 278
    },
    {
      "clip_ratio": 0.0001145737842307426,
      "epoch": 0.0007745739843086302,
      "grad_norm": 0.11382175236940384,
      "kl": 0.003511702874675393,
      "learning_rate": 2.3185595567867034e-06,
      "loss": 0.0,
      "step": 279
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0007773502351484462,
      "grad_norm": 0.09704895317554474,
      "kl": 0.0034232381731271744,
      "learning_rate": 2.326869806094183e-06,
      "loss": 0.0006,
      "step": 280
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.000780126485988262,
      "grad_norm": 0.11391977220773697,
      "kl": 0.0033790983725339174,
      "learning_rate": 2.335180055401662e-06,
      "loss": 0.0002,
      "step": 281
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0007829027368280779,
      "grad_norm": 0.3949395418167114,
      "kl": 0.0036244012881070375,
      "learning_rate": 2.343490304709141e-06,
      "loss": 0.0005,
      "step": 282
    },
    {
      "clip_ratio": 0.0006535947904922068,
      "epoch": 0.0007856789876678938,
      "grad_norm": 0.10197897255420685,
      "kl": 0.00319200346712023,
      "learning_rate": 2.3518005540166202e-06,
      "loss": 0.0001,
      "step": 283
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0007884552385077096,
      "grad_norm": 0.10884040594100952,
      "kl": 0.0036248579854145646,
      "learning_rate": 2.3601108033240998e-06,
      "loss": -0.0003,
      "step": 284
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0007912314893475255,
      "grad_norm": 0.1121264100074768,
      "kl": 0.003715887665748596,
      "learning_rate": 2.368421052631579e-06,
      "loss": -0.0012,
      "step": 285
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0007940077401873414,
      "grad_norm": 0.10213881731033325,
      "kl": 0.003711581346578896,
      "learning_rate": 2.376731301939058e-06,
      "loss": -0.0,
      "step": 286
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0007967839910271573,
      "grad_norm": 0.15096428990364075,
      "kl": 0.004820869769901037,
      "learning_rate": 2.3850415512465375e-06,
      "loss": -0.0011,
      "step": 287
    },
    {
      "clip_ratio": 0.0001633986976230517,
      "epoch": 0.0007995602418669731,
      "grad_norm": 0.12257353216409683,
      "kl": 0.004069758579134941,
      "learning_rate": 2.3933518005540166e-06,
      "loss": -0.0001,
      "step": 288
    },
    {
      "clip_ratio": 0.00025150904548354447,
      "completion_length": 229.6041717529297,
      "epoch": 0.0008023364927067891,
      "grad_norm": 0.1052643209695816,
      "kl": 0.004225482931360602,
      "learning_rate": 2.401662049861496e-06,
      "loss": 0.0125,
      "reward": 0.22500000149011612,
      "reward_std": 0.2582135424017906,
      "rewards/countdown_reward_func": 0.22500000149011612,
      "step": 289,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0008051127435466049,
      "grad_norm": 0.08905075490474701,
      "kl": 0.004490195075049996,
      "learning_rate": 2.4099722991689752e-06,
      "loss": 0.0132,
      "step": 290
    },
    {
      "clip_ratio": 0.0005068937316536903,
      "epoch": 0.0008078889943864208,
      "grad_norm": 0.07550480216741562,
      "kl": 0.0038030524738132954,
      "learning_rate": 2.4182825484764543e-06,
      "loss": 0.012,
      "step": 291
    },
    {
      "clip_ratio": 0.00012575452274177223,
      "epoch": 0.0008106652452262367,
      "grad_norm": 0.08875437080860138,
      "kl": 0.00425003154668957,
      "learning_rate": 2.426592797783934e-06,
      "loss": 0.0121,
      "step": 292
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0008134414960660525,
      "grad_norm": 0.08870893716812134,
      "kl": 0.004312893608585,
      "learning_rate": 2.434903047091413e-06,
      "loss": 0.0117,
      "step": 293
    },
    {
      "clip_ratio": 0.0008138020639307797,
      "epoch": 0.0008162177469058685,
      "grad_norm": 0.08667095005512238,
      "kl": 0.004525796044617891,
      "learning_rate": 2.443213296398892e-06,
      "loss": 0.0117,
      "step": 294
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0008189939977456843,
      "grad_norm": 0.10581125319004059,
      "kl": 0.004761378280818462,
      "learning_rate": 2.451523545706371e-06,
      "loss": 0.0118,
      "step": 295
    },
    {
      "clip_ratio": 0.00016276042151730508,
      "epoch": 0.0008217702485855002,
      "grad_norm": 0.08822281658649445,
      "kl": 0.005203166743740439,
      "learning_rate": 2.4598337950138507e-06,
      "loss": 0.0123,
      "step": 296
    },
    {
      "clip_ratio": 0.0010381632746430114,
      "epoch": 0.0008245464994253161,
      "grad_norm": 0.07447604089975357,
      "kl": 0.00508764130063355,
      "learning_rate": 2.46814404432133e-06,
      "loss": 0.012,
      "step": 297
    },
    {
      "clip_ratio": 0.00037726358277723193,
      "epoch": 0.0008273227502651319,
      "grad_norm": 0.087750643491745,
      "kl": 0.005412213504314423,
      "learning_rate": 2.476454293628809e-06,
      "loss": 0.0111,
      "step": 298
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0008300990011049479,
      "grad_norm": 0.08693079650402069,
      "kl": 0.005746564595028758,
      "learning_rate": 2.484764542936288e-06,
      "loss": 0.0114,
      "step": 299
    },
    {
      "clip_ratio": 0.0007324218604480848,
      "epoch": 0.0008328752519447637,
      "grad_norm": 0.09118447452783585,
      "kl": 0.0058568131644278765,
      "learning_rate": 2.493074792243767e-06,
      "loss": 0.0116,
      "step": 300
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 223.0,
      "epoch": 0.0008356515027845796,
      "grad_norm": 0.09235308319330215,
      "kl": 0.006070051807910204,
      "learning_rate": 2.5013850415512467e-06,
      "loss": -0.0069,
      "reward": 0.20000001043081284,
      "reward_std": 0.2610399127006531,
      "rewards/countdown_reward_func": 0.20000001043081284,
      "step": 301,
      "zero_std_ratio": 0.0
    },
    {
      "clip_ratio": 8.361203799722716e-05,
      "epoch": 0.0008384277536243955,
      "grad_norm": 0.08406054228544235,
      "kl": 0.0060172725934535265,
      "learning_rate": 2.5096952908587258e-06,
      "loss": -0.0069,
      "step": 302
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0008412040044642114,
      "grad_norm": 0.0954718291759491,
      "kl": 0.006230462109670043,
      "learning_rate": 2.518005540166205e-06,
      "loss": -0.0066,
      "step": 303
    },
    {
      "clip_ratio": 0.00010575295891612768,
      "epoch": 0.0008439802553040272,
      "grad_norm": 0.09710827469825745,
      "kl": 0.00680284365080297,
      "learning_rate": 2.526315789473684e-06,
      "loss": -0.007,
      "step": 304
    },
    {
      "clip_ratio": 0.0002602904787636362,
      "epoch": 0.0008467565061438431,
      "grad_norm": 0.09757263213396072,
      "kl": 0.007144181756302714,
      "learning_rate": 2.5346260387811635e-06,
      "loss": -0.0062,
      "step": 305
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.000849532756983659,
      "grad_norm": 0.07679014652967453,
      "kl": 0.008339704247191548,
      "learning_rate": 2.5429362880886426e-06,
      "loss": -0.0076,
      "step": 306
    },
    {
      "clip_ratio": 8.361203799722716e-05,
      "epoch": 0.0008523090078234748,
      "grad_norm": 0.09126242995262146,
      "kl": 0.0085701416246593,
      "learning_rate": 2.5512465373961217e-06,
      "loss": -0.0071,
      "step": 307
    },
    {
      "clip_ratio": 8.361203799722716e-05,
      "epoch": 0.0008550852586632908,
      "grad_norm": 0.08423109352588654,
      "kl": 0.007637211121618748,
      "learning_rate": 2.5595567867036012e-06,
      "loss": -0.0073,
      "step": 308
    },
    {
      "clip_ratio": 8.833922038320452e-05,
      "epoch": 0.0008578615095031066,
      "grad_norm": 0.09705601632595062,
      "kl": 0.007593115558847785,
      "learning_rate": 2.5678670360110803e-06,
      "loss": -0.0067,
      "step": 309
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0008606377603429226,
      "grad_norm": 0.09621502459049225,
      "kl": 0.008113941876217723,
      "learning_rate": 2.57617728531856e-06,
      "loss": -0.0075,
      "step": 310
    },
    {
      "clip_ratio": 0.00020451023010537028,
      "epoch": 0.0008634140111827384,
      "grad_norm": 0.10358273983001709,
      "kl": 0.008504221215844154,
      "learning_rate": 2.584487534626039e-06,
      "loss": -0.0073,
      "step": 311
    },
    {
      "clip_ratio": 0.00016722407599445432,
      "epoch": 0.0008661902620225542,
      "grad_norm": 0.08098059147596359,
      "kl": 0.009506989270448685,
      "learning_rate": 2.592797783933518e-06,
      "loss": -0.0075,
      "step": 312
    },
    {
      "clip_ratio": 0.00016914748994167894,
      "completion_length": 226.37500762939453,
      "epoch": 0.0008689665128623702,
      "grad_norm": 0.0626518577337265,
      "kl": 0.008252686355262995,
      "learning_rate": 2.6011080332409976e-06,
      "loss": 0.0071,
      "reward": 0.2291666716337204,
      "reward_std": 0.21667250245809555,
      "rewards/countdown_reward_func": 0.2291666641831398,
      "step": 313,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.000871742763702186,
      "grad_norm": 0.0984596535563469,
      "kl": 0.010352411307394505,
      "learning_rate": 2.6094182825484767e-06,
      "loss": 0.0072,
      "step": 314
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0008745190145420019,
      "grad_norm": 0.09516298025846481,
      "kl": 0.011239033192396164,
      "learning_rate": 2.617728531855956e-06,
      "loss": 0.0077,
      "step": 315
    },
    {
      "clip_ratio": 9.09090886125341e-05,
      "epoch": 0.0008772952653818178,
      "grad_norm": 0.10682272166013718,
      "kl": 0.010467468295246363,
      "learning_rate": 2.626038781163435e-06,
      "loss": 0.0069,
      "step": 316
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0008800715162216336,
      "grad_norm": 0.08754424750804901,
      "kl": 0.010327389929443598,
      "learning_rate": 2.6343490304709144e-06,
      "loss": 0.0072,
      "step": 317
    },
    {
      "clip_ratio": 9.09090886125341e-05,
      "epoch": 0.0008828477670614495,
      "grad_norm": 0.12904123961925507,
      "kl": 0.010680126026272774,
      "learning_rate": 2.6426592797783935e-06,
      "loss": 0.0072,
      "step": 318
    },
    {
      "clip_ratio": 0.00025372125674039125,
      "epoch": 0.0008856240179012654,
      "grad_norm": 0.06477981805801392,
      "kl": 0.009791890624910593,
      "learning_rate": 2.6509695290858726e-06,
      "loss": 0.0066,
      "step": 319
    },
    {
      "clip_ratio": 0.00040258895023725927,
      "epoch": 0.0008884002687410813,
      "grad_norm": 0.185244619846344,
      "kl": 0.010858574416488409,
      "learning_rate": 2.6592797783933517e-06,
      "loss": 0.0068,
      "step": 320
    },
    {
      "clip_ratio": 9.968101949198171e-05,
      "epoch": 0.0008911765195808972,
      "grad_norm": 0.08614979684352875,
      "kl": 0.014307011850178242,
      "learning_rate": 2.667590027700831e-06,
      "loss": 0.0067,
      "step": 321
    },
    {
      "clip_ratio": 0.00027272728038951755,
      "epoch": 0.0008939527704207131,
      "grad_norm": 0.10753199458122253,
      "kl": 0.013140483759343624,
      "learning_rate": 2.6759002770083104e-06,
      "loss": 0.0054,
      "step": 322
    },
    {
      "clip_ratio": 0.00029475959308911115,
      "epoch": 0.0008967290212605289,
      "grad_norm": 0.10408802330493927,
      "kl": 0.01228410704061389,
      "learning_rate": 2.6842105263157895e-06,
      "loss": 0.0061,
      "step": 323
    },
    {
      "clip_ratio": 0.00019936203898396343,
      "epoch": 0.0008995052721003448,
      "grad_norm": 0.12455835938453674,
      "kl": 0.014058600179851055,
      "learning_rate": 2.6925207756232686e-06,
      "loss": 0.0061,
      "step": 324
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "completion_length": 237.64583587646484,
      "epoch": 0.0009022815229401607,
      "grad_norm": 0.08178848773241043,
      "kl": 0.011353591922670603,
      "learning_rate": 2.7008310249307477e-06,
      "loss": 0.0089,
      "reward": 0.1937500163912773,
      "reward_std": 0.1958785615861416,
      "rewards/countdown_reward_func": 0.1937500163912773,
      "step": 325,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio": 0.0006667007837677374,
      "epoch": 0.0009050577737799765,
      "grad_norm": 0.08745010197162628,
      "kl": 0.015723693184554577,
      "learning_rate": 2.709141274238227e-06,
      "loss": 0.0086,
      "step": 326
    },
    {
      "clip_ratio": 0.00018234216258861125,
      "epoch": 0.0009078340246197925,
      "grad_norm": 0.08209048956632614,
      "kl": 0.013638208620250225,
      "learning_rate": 2.7174515235457063e-06,
      "loss": 0.0081,
      "step": 327
    },
    {
      "clip_ratio": 8.741259080125019e-05,
      "epoch": 0.0009106102754596083,
      "grad_norm": 0.07701068371534348,
      "kl": 0.014465087559074163,
      "learning_rate": 2.7257617728531854e-06,
      "loss": 0.0088,
      "step": 328
    },
    {
      "clip_ratio": 0.00025351856311317533,
      "epoch": 0.0009133865262994242,
      "grad_norm": 0.08792005479335785,
      "kl": 0.014732198789715767,
      "learning_rate": 2.734072022160665e-06,
      "loss": 0.0089,
      "step": 329
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0009161627771392401,
      "grad_norm": 0.07523926347494125,
      "kl": 0.014703777618706226,
      "learning_rate": 2.742382271468144e-06,
      "loss": 0.0082,
      "step": 330
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0009189390279790559,
      "grad_norm": 0.08062107861042023,
      "kl": 0.013506517745554447,
      "learning_rate": 2.7506925207756236e-06,
      "loss": 0.0086,
      "step": 331
    },
    {
      "clip_ratio": 0.0006983240018598735,
      "epoch": 0.0009217152788188719,
      "grad_norm": 0.07194234430789948,
      "kl": 0.019093542359769344,
      "learning_rate": 2.7590027700831027e-06,
      "loss": 0.0082,
      "step": 332
    },
    {
      "clip_ratio": 0.00025065265799639747,
      "epoch": 0.0009244915296586877,
      "grad_norm": 0.08615561574697495,
      "kl": 0.015138544142246246,
      "learning_rate": 2.7673130193905818e-06,
      "loss": 0.0077,
      "step": 333
    },
    {
      "clip_ratio": 8.741259080125019e-05,
      "epoch": 0.0009272677804985036,
      "grad_norm": 0.09492068737745285,
      "kl": 0.01689472934231162,
      "learning_rate": 2.7756232686980613e-06,
      "loss": 0.0082,
      "step": 334
    },
    {
      "clip_ratio": 0.0004922889638692141,
      "epoch": 0.0009300440313383195,
      "grad_norm": 0.10922621935606003,
      "kl": 0.017271476797759533,
      "learning_rate": 2.7839335180055404e-06,
      "loss": 0.007,
      "step": 335
    },
    {
      "clip_ratio": 0.00010048231342807412,
      "epoch": 0.0009328202821781354,
      "grad_norm": 0.0761309564113617,
      "kl": 0.01699853641912341,
      "learning_rate": 2.7922437673130195e-06,
      "loss": 0.0081,
      "step": 336
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 227.08334350585938,
      "epoch": 0.0009355965330179512,
      "grad_norm": 0.05785919353365898,
      "kl": 0.01666484959423542,
      "learning_rate": 2.8005540166204986e-06,
      "loss": 0.01,
      "reward": 0.15416667610406876,
      "reward_std": 0.1091257855296135,
      "rewards/countdown_reward_func": 0.15416667610406876,
      "step": 337,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio": 0.00018317173817194998,
      "epoch": 0.0009383727838577671,
      "grad_norm": 0.05232284963130951,
      "kl": 0.01712951622903347,
      "learning_rate": 2.808864265927978e-06,
      "loss": 0.0099,
      "step": 338
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.000941149034697583,
      "grad_norm": 0.09636948257684708,
      "kl": 0.019584951922297478,
      "learning_rate": 2.8171745152354573e-06,
      "loss": 0.0101,
      "step": 339
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0009439252855373989,
      "grad_norm": 0.06418822705745697,
      "kl": 0.0171346515417099,
      "learning_rate": 2.8254847645429364e-06,
      "loss": 0.0094,
      "step": 340
    },
    {
      "clip_ratio": 0.000244140625,
      "epoch": 0.0009467015363772148,
      "grad_norm": 0.05186130478978157,
      "kl": 0.018314712680876255,
      "learning_rate": 2.8337950138504155e-06,
      "loss": 0.0096,
      "step": 341
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0009494777872170306,
      "grad_norm": 0.05359478294849396,
      "kl": 0.022239549085497856,
      "learning_rate": 2.8421052631578946e-06,
      "loss": 0.0094,
      "step": 342
    },
    {
      "clip_ratio": 8.60289073898457e-05,
      "epoch": 0.0009522540380568466,
      "grad_norm": 0.05925218388438225,
      "kl": 0.020320788025856018,
      "learning_rate": 2.850415512465374e-06,
      "loss": 0.0096,
      "step": 343
    },
    {
      "clip_ratio": 0.00016276042151730508,
      "epoch": 0.0009550302888966624,
      "grad_norm": 0.05631886422634125,
      "kl": 0.02036405447870493,
      "learning_rate": 2.858725761772853e-06,
      "loss": 0.0094,
      "step": 344
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0009578065397364782,
      "grad_norm": 0.1005195826292038,
      "kl": 0.023968802765011787,
      "learning_rate": 2.8670360110803323e-06,
      "loss": 0.0093,
      "step": 345
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0009605827905762942,
      "grad_norm": 0.06793226301670074,
      "kl": 0.02041178196668625,
      "learning_rate": 2.8753462603878114e-06,
      "loss": 0.0087,
      "step": 346
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00096335904141611,
      "grad_norm": 0.048542365431785583,
      "kl": 0.021388364024460316,
      "learning_rate": 2.883656509695291e-06,
      "loss": 0.0093,
      "step": 347
    },
    {
      "clip_ratio": 0.0002645519489306025,
      "epoch": 0.0009661352922559259,
      "grad_norm": 0.08362864702939987,
      "kl": 0.027280261740088463,
      "learning_rate": 2.89196675900277e-06,
      "loss": 0.0093,
      "step": 348
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 228.02084350585938,
      "epoch": 0.0009689115430957418,
      "grad_norm": 0.0892510712146759,
      "kl": 0.02360483817756176,
      "learning_rate": 2.900277008310249e-06,
      "loss": -0.0109,
      "reward": 0.16875001043081284,
      "reward_std": 0.1314988099038601,
      "rewards/countdown_reward_func": 0.16875001043081284,
      "step": 349,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0009716877939355576,
      "grad_norm": 0.09001202881336212,
      "kl": 0.023638173937797546,
      "learning_rate": 2.9085872576177287e-06,
      "loss": -0.0107,
      "step": 350
    },
    {
      "clip_ratio": 0.0001959434594027698,
      "epoch": 0.0009744640447753736,
      "grad_norm": 0.113184355199337,
      "kl": 0.02315935306251049,
      "learning_rate": 2.9168975069252078e-06,
      "loss": -0.0112,
      "step": 351
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0009772402956151893,
      "grad_norm": 0.07550473511219025,
      "kl": 0.02309281285852194,
      "learning_rate": 2.9252077562326873e-06,
      "loss": -0.0108,
      "step": 352
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0009800165464550054,
      "grad_norm": 0.11282236129045486,
      "kl": 0.024476034566760063,
      "learning_rate": 2.9335180055401664e-06,
      "loss": -0.0112,
      "step": 353
    },
    {
      "clip_ratio": 0.00018355359497945756,
      "epoch": 0.0009827927972948212,
      "grad_norm": 0.11352071166038513,
      "kl": 0.02151069976389408,
      "learning_rate": 2.9418282548476455e-06,
      "loss": -0.0114,
      "step": 354
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.000985569048134637,
      "grad_norm": 0.08133033663034439,
      "kl": 0.021808774210512638,
      "learning_rate": 2.950138504155125e-06,
      "loss": -0.0114,
      "step": 355
    },
    {
      "clip_ratio": 0.00010416666918899864,
      "epoch": 0.000988345298974453,
      "grad_norm": 0.08895107358694077,
      "kl": 0.02102847583591938,
      "learning_rate": 2.958448753462604e-06,
      "loss": -0.0113,
      "step": 356
    },
    {
      "clip_ratio": 9.177679748972878e-05,
      "epoch": 0.0009911215498142687,
      "grad_norm": 0.11091752350330353,
      "kl": 0.01951050851494074,
      "learning_rate": 2.9667590027700832e-06,
      "loss": -0.0118,
      "step": 357
    },
    {
      "clip_ratio": 0.00028880476020276546,
      "epoch": 0.0009938978006540848,
      "grad_norm": 0.08083935081958771,
      "kl": 0.019494274631142616,
      "learning_rate": 2.9750692520775623e-06,
      "loss": -0.0118,
      "step": 358
    },
    {
      "clip_ratio": 0.00018355359497945756,
      "epoch": 0.0009966740514939006,
      "grad_norm": 0.09999022632837296,
      "kl": 0.019018066115677357,
      "learning_rate": 2.983379501385042e-06,
      "loss": -0.012,
      "step": 359
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0009994503023337165,
      "grad_norm": 0.09490374475717545,
      "kl": 0.01723548863083124,
      "learning_rate": 2.991689750692521e-06,
      "loss": -0.013,
      "step": 360
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 226.5625,
      "epoch": 0.0010022265531735323,
      "grad_norm": 0.09549269825220108,
      "kl": 0.019508808851242065,
      "learning_rate": 3e-06,
      "loss": 0.0015,
      "reward": 0.2250000238418579,
      "reward_std": 0.23222807794809341,
      "rewards/countdown_reward_func": 0.22500000894069672,
      "step": 361,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.00017572575598023832,
      "epoch": 0.0010050028040133482,
      "grad_norm": 0.10312195122241974,
      "kl": 0.01787219289690256,
      "learning_rate": 3e-06,
      "loss": 0.0021,
      "step": 362
    },
    {
      "clip_ratio": 0.0007824726053513587,
      "epoch": 0.001007779054853164,
      "grad_norm": 0.12467264384031296,
      "kl": 0.017863546032458544,
      "learning_rate": 3e-06,
      "loss": 0.0019,
      "step": 363
    },
    {
      "clip_ratio": 8.37240440887399e-05,
      "epoch": 0.00101055530569298,
      "grad_norm": 0.2627834379673004,
      "kl": 0.01767959538847208,
      "learning_rate": 3e-06,
      "loss": 0.0021,
      "step": 364
    },
    {
      "clip_ratio": 9.038322605192661e-05,
      "epoch": 0.0010133315565327959,
      "grad_norm": 0.0790000930428505,
      "kl": 0.01718911249190569,
      "learning_rate": 3e-06,
      "loss": 0.0016,
      "step": 365
    },
    {
      "clip_ratio": 0.0001808318484108895,
      "epoch": 0.0010161078073726117,
      "grad_norm": 0.07705968618392944,
      "kl": 0.01755279116332531,
      "learning_rate": 3e-06,
      "loss": 0.0022,
      "step": 366
    },
    {
      "clip_ratio": 0.0001741072628647089,
      "epoch": 0.0010188840582124276,
      "grad_norm": 0.10953141748905182,
      "kl": 0.01750816684216261,
      "learning_rate": 3e-06,
      "loss": 0.0021,
      "step": 367
    },
    {
      "clip_ratio": 0.0004197447851765901,
      "epoch": 0.0010216603090522434,
      "grad_norm": 0.2015552967786789,
      "kl": 0.017061928287148476,
      "learning_rate": 3e-06,
      "loss": 0.0016,
      "step": 368
    },
    {
      "clip_ratio": 0.000489045400172472,
      "epoch": 0.0010244365598920595,
      "grad_norm": 0.1368420273065567,
      "kl": 0.017408848274499178,
      "learning_rate": 3e-06,
      "loss": 0.0013,
      "step": 369
    },
    {
      "clip_ratio": 0.00018153311975765973,
      "epoch": 0.0010272128107318753,
      "grad_norm": 0.09191857278347015,
      "kl": 0.016698247753083706,
      "learning_rate": 3e-06,
      "loss": 0.0017,
      "step": 370
    },
    {
      "clip_ratio": 8.394895849050954e-05,
      "epoch": 0.0010299890615716911,
      "grad_norm": 0.07735387235879898,
      "kl": 0.017519176937639713,
      "learning_rate": 3e-06,
      "loss": 0.0017,
      "step": 371
    },
    {
      "clip_ratio": 0.0002578312996774912,
      "epoch": 0.001032765312411507,
      "grad_norm": 0.07390379160642624,
      "kl": 0.018353909254074097,
      "learning_rate": 3e-06,
      "loss": 0.0019,
      "step": 372
    },
    {
      "clip_ratio": 8.196721319109201e-05,
      "completion_length": 228.68750762939453,
      "epoch": 0.0010355415632513228,
      "grad_norm": 0.10989905893802643,
      "kl": 0.017692445777356625,
      "learning_rate": 3e-06,
      "loss": -0.0103,
      "reward": 0.2500000223517418,
      "reward_std": 0.2080453634262085,
      "rewards/countdown_reward_func": 0.2500000223517418,
      "step": 373,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio": 0.00010279605339746922,
      "epoch": 0.0010383178140911387,
      "grad_norm": 0.15112467110157013,
      "kl": 0.03264212794601917,
      "learning_rate": 3e-06,
      "loss": -0.0084,
      "step": 374
    },
    {
      "clip_ratio": 0.0011789826894528233,
      "epoch": 0.0010410940649309547,
      "grad_norm": 0.1216062679886818,
      "kl": 0.021678635850548744,
      "learning_rate": 3e-06,
      "loss": -0.0091,
      "step": 375
    },
    {
      "clip_ratio": 9.15080527192913e-05,
      "epoch": 0.0010438703157707706,
      "grad_norm": 0.08477499336004257,
      "kl": 0.016693778336048126,
      "learning_rate": 3e-06,
      "loss": -0.0099,
      "step": 376
    },
    {
      "clip_ratio": 0.0002832566970027983,
      "epoch": 0.0010466465666105864,
      "grad_norm": 0.10167127847671509,
      "kl": 0.017779462039470673,
      "learning_rate": 3e-06,
      "loss": -0.0093,
      "step": 377
    },
    {
      "clip_ratio": 9.184423106489703e-05,
      "epoch": 0.0010494228174504022,
      "grad_norm": 0.09984228760004044,
      "kl": 0.017343958839774132,
      "learning_rate": 3e-06,
      "loss": -0.0096,
      "step": 378
    },
    {
      "clip_ratio": 0.00017322444182354957,
      "epoch": 0.001052199068290218,
      "grad_norm": 0.10843722522258759,
      "kl": 0.018563530407845974,
      "learning_rate": 3e-06,
      "loss": -0.0104,
      "step": 379
    },
    {
      "clip_ratio": 0.0003475236881058663,
      "epoch": 0.0010549753191300341,
      "grad_norm": 0.137297123670578,
      "kl": 0.03769372217357159,
      "learning_rate": 3e-06,
      "loss": -0.0097,
      "step": 380
    },
    {
      "clip_ratio": 0.0006742323748767376,
      "epoch": 0.00105775156996985,
      "grad_norm": 0.13010026514530182,
      "kl": 0.024436946026980877,
      "learning_rate": 3e-06,
      "loss": -0.0105,
      "step": 381
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0010605278208096658,
      "grad_norm": 0.09109175205230713,
      "kl": 0.01653394289314747,
      "learning_rate": 3e-06,
      "loss": -0.0106,
      "step": 382
    },
    {
      "clip_ratio": 0.0001830161054385826,
      "epoch": 0.0010633040716494816,
      "grad_norm": 0.08129965513944626,
      "kl": 0.017563740722835064,
      "learning_rate": 3e-06,
      "loss": -0.01,
      "step": 383
    },
    {
      "clip_ratio": 0.000265068665612489,
      "epoch": 0.0010660803224892975,
      "grad_norm": 0.09801369905471802,
      "kl": 0.01689669769257307,
      "learning_rate": 3e-06,
      "loss": -0.0115,
      "step": 384
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 240.64583587646484,
      "epoch": 0.0010688565733291133,
      "grad_norm": 0.07832996547222137,
      "kl": 0.02487264759838581,
      "learning_rate": 3e-06,
      "loss": 0.0146,
      "reward": 0.19166668504476547,
      "reward_std": 0.15476077422499657,
      "rewards/countdown_reward_func": 0.19166668504476547,
      "step": 385,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio": 8.716875890968367e-05,
      "epoch": 0.0010716328241689294,
      "grad_norm": 0.07169502228498459,
      "kl": 0.016012447886168957,
      "learning_rate": 3e-06,
      "loss": 0.0138,
      "step": 386
    },
    {
      "clip_ratio": 9.005763422464952e-05,
      "epoch": 0.0010744090750087452,
      "grad_norm": 0.08705933392047882,
      "kl": 0.016524842474609613,
      "learning_rate": 3e-06,
      "loss": 0.0143,
      "step": 387
    },
    {
      "clip_ratio": 9.61538462433964e-05,
      "epoch": 0.001077185325848561,
      "grad_norm": 0.0732106938958168,
      "kl": 0.015910383313894272,
      "learning_rate": 3e-06,
      "loss": 0.0142,
      "step": 388
    },
    {
      "clip_ratio": 0.0002589142677607015,
      "epoch": 0.001079961576688377,
      "grad_norm": 0.07911605387926102,
      "kl": 0.01897307112812996,
      "learning_rate": 3e-06,
      "loss": 0.0151,
      "step": 389
    },
    {
      "clip_ratio": 0.00017143784498330206,
      "epoch": 0.0010827378275281927,
      "grad_norm": 0.0664311945438385,
      "kl": 0.016379999462515116,
      "learning_rate": 3e-06,
      "loss": 0.0145,
      "step": 390
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0010855140783680088,
      "grad_norm": 0.08451702445745468,
      "kl": 0.0259824451059103,
      "learning_rate": 3e-06,
      "loss": 0.0146,
      "step": 391
    },
    {
      "clip_ratio": 9.005763422464952e-05,
      "epoch": 0.0010882903292078246,
      "grad_norm": 0.07148966938257217,
      "kl": 0.016325827687978745,
      "learning_rate": 3e-06,
      "loss": 0.0138,
      "step": 392
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0010910665800476405,
      "grad_norm": 0.09877816587686539,
      "kl": 0.01728895679116249,
      "learning_rate": 3e-06,
      "loss": 0.0141,
      "step": 393
    },
    {
      "clip_ratio": 0.0001923076924867928,
      "epoch": 0.0010938428308874563,
      "grad_norm": 0.08034543693065643,
      "kl": 0.016465777531266212,
      "learning_rate": 3e-06,
      "loss": 0.0138,
      "step": 394
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0010966190817272722,
      "grad_norm": 0.0796988308429718,
      "kl": 0.020429577212780714,
      "learning_rate": 3e-06,
      "loss": 0.0139,
      "step": 395
    },
    {
      "clip_ratio": 0.0005143135640537366,
      "epoch": 0.0010993953325670882,
      "grad_norm": 0.06786444783210754,
      "kl": 0.01705406652763486,
      "learning_rate": 3e-06,
      "loss": 0.013,
      "step": 396
    },
    {
      "clip_ratio": 0.00019317050464451313,
      "completion_length": 222.3541717529297,
      "epoch": 0.001102171583406904,
      "grad_norm": 0.1809883564710617,
      "kl": 0.02103044930845499,
      "learning_rate": 3e-06,
      "loss": 0.0201,
      "reward": 0.32500000298023224,
      "reward_std": 0.28887902945280075,
      "rewards/countdown_reward_func": 0.32499998807907104,
      "step": 397,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0003668378631118685,
      "epoch": 0.0011049478342467199,
      "grad_norm": 0.13064327836036682,
      "kl": 0.019325342029333115,
      "learning_rate": 3e-06,
      "loss": 0.0188,
      "step": 398
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0011077240850865357,
      "grad_norm": 0.134730726480484,
      "kl": 0.018743818625807762,
      "learning_rate": 3e-06,
      "loss": 0.018,
      "step": 399
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0011105003359263516,
      "grad_norm": 0.11716154217720032,
      "kl": 0.03218943625688553,
      "learning_rate": 3e-06,
      "loss": 0.0188,
      "step": 400
    },
    {
      "clip_ratio": 0.0003666364573291503,
      "epoch": 0.0011132765867661674,
      "grad_norm": 0.1279788464307785,
      "kl": 0.02369430661201477,
      "learning_rate": 3e-06,
      "loss": 0.0183,
      "step": 401
    },
    {
      "clip_ratio": 0.00018321751849725842,
      "epoch": 0.0011160528376059835,
      "grad_norm": 0.1514863669872284,
      "kl": 0.02853427268564701,
      "learning_rate": 3e-06,
      "loss": 0.0172,
      "step": 402
    },
    {
      "clip_ratio": 0.0005473888304550201,
      "epoch": 0.0011188290884457993,
      "grad_norm": 0.17688550055027008,
      "kl": 0.02879752404987812,
      "learning_rate": 3e-06,
      "loss": 0.0159,
      "step": 403
    },
    {
      "clip_ratio": 0.0003765894507523626,
      "epoch": 0.0011216053392856151,
      "grad_norm": 0.1264808475971222,
      "kl": 0.028363430872559547,
      "learning_rate": 3e-06,
      "loss": 0.0154,
      "step": 404
    },
    {
      "clip_ratio": 0.0002071251074085012,
      "epoch": 0.001124381590125431,
      "grad_norm": 0.13313868641853333,
      "kl": 0.028233185410499573,
      "learning_rate": 3e-06,
      "loss": 0.0148,
      "step": 405
    },
    {
      "clip_ratio": 0.00029883457318646833,
      "epoch": 0.0011271578409652468,
      "grad_norm": 0.11772707849740982,
      "kl": 0.0557091049849987,
      "learning_rate": 3e-06,
      "loss": 0.0158,
      "step": 406
    },
    {
      "clip_ratio": 0.0008593437087256461,
      "epoch": 0.0011299340918050629,
      "grad_norm": 0.1197652816772461,
      "kl": 0.03832645434886217,
      "learning_rate": 3e-06,
      "loss": 0.0146,
      "step": 407
    },
    {
      "clip_ratio": 0.0008315023733302951,
      "epoch": 0.0011327103426448787,
      "grad_norm": 0.11358743906021118,
      "kl": 0.049091488122940063,
      "learning_rate": 3e-06,
      "loss": 0.0139,
      "step": 408
    },
    {
      "clip_ratio": 0.00010860121983569115,
      "completion_length": 212.20833587646484,
      "epoch": 0.0011354865934846946,
      "grad_norm": 0.13000725209712982,
      "kl": 0.03431596327573061,
      "learning_rate": 3e-06,
      "loss": -0.0011,
      "reward": 0.239583358168602,
      "reward_std": 0.3074723482131958,
      "rewards/countdown_reward_func": 0.2395833507180214,
      "step": 409,
      "zero_std_ratio": 0.0
    },
    {
      "clip_ratio": 0.00010860121983569115,
      "epoch": 0.0011382628443245104,
      "grad_norm": 0.12621691823005676,
      "kl": 0.03901347145438194,
      "learning_rate": 3e-06,
      "loss": -0.0018,
      "step": 410
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0011410390951643262,
      "grad_norm": 0.15552707016468048,
      "kl": 0.05270291492342949,
      "learning_rate": 3e-06,
      "loss": -0.0012,
      "step": 411
    },
    {
      "clip_ratio": 0.0003161708955303766,
      "epoch": 0.001143815346004142,
      "grad_norm": 0.1499052792787552,
      "kl": 0.04181257076561451,
      "learning_rate": 3e-06,
      "loss": -0.0025,
      "step": 412
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0011465915968439581,
      "grad_norm": 0.11208264529705048,
      "kl": 0.04561302810907364,
      "learning_rate": 3e-06,
      "loss": -0.0011,
      "step": 413
    },
    {
      "clip_ratio": 8.514986257068813e-05,
      "epoch": 0.001149367847683774,
      "grad_norm": 0.117032989859581,
      "kl": 0.0457566250115633,
      "learning_rate": 3e-06,
      "loss": -0.002,
      "step": 414
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0011521440985235898,
      "grad_norm": 0.1544887125492096,
      "kl": 0.04844648204743862,
      "learning_rate": 3e-06,
      "loss": -0.0009,
      "step": 415
    },
    {
      "clip_ratio": 0.00010860121983569115,
      "epoch": 0.0011549203493634056,
      "grad_norm": 0.1259896457195282,
      "kl": 0.05372907593846321,
      "learning_rate": 3e-06,
      "loss": -0.0026,
      "step": 416
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0011576966002032215,
      "grad_norm": 0.15457068383693695,
      "kl": 0.06770433485507965,
      "learning_rate": 3e-06,
      "loss": -0.0032,
      "step": 417
    },
    {
      "clip_ratio": 0.00022890920809004456,
      "epoch": 0.0011604728510430375,
      "grad_norm": 0.1197003573179245,
      "kl": 0.0525658018887043,
      "learning_rate": 3e-06,
      "loss": -0.0043,
      "step": 418
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0011632491018828534,
      "grad_norm": 0.10684943944215775,
      "kl": 0.053456977009773254,
      "learning_rate": 3e-06,
      "loss": -0.0021,
      "step": 419
    },
    {
      "clip_ratio": 0.00021355503849918023,
      "epoch": 0.0011660253527226692,
      "grad_norm": 0.10173308104276657,
      "kl": 0.05415925942361355,
      "learning_rate": 3e-06,
      "loss": -0.0027,
      "step": 420
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 237.62500762939453,
      "epoch": 0.001168801603562485,
      "grad_norm": 0.11933105438947678,
      "kl": 0.06486517563462257,
      "learning_rate": 3e-06,
      "loss": 0.0027,
      "reward": 0.27916668355464935,
      "reward_std": 0.29565654695034027,
      "rewards/countdown_reward_func": 0.27916668355464935,
      "step": 421,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.001171577854402301,
      "grad_norm": 0.20322704315185547,
      "kl": 0.08543343096971512,
      "learning_rate": 3e-06,
      "loss": 0.0032,
      "step": 422
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0011743541052421167,
      "grad_norm": 0.19525644183158875,
      "kl": 0.13222120702266693,
      "learning_rate": 3e-06,
      "loss": 0.0045,
      "step": 423
    },
    {
      "clip_ratio": 0.0002695980292628519,
      "epoch": 0.0011771303560819328,
      "grad_norm": 0.13731662929058075,
      "kl": 0.0626319907605648,
      "learning_rate": 3e-06,
      "loss": 0.0031,
      "step": 424
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0011799066069217486,
      "grad_norm": 0.13071469962596893,
      "kl": 0.05833687447011471,
      "learning_rate": 3e-06,
      "loss": 0.0014,
      "step": 425
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0011826828577615645,
      "grad_norm": 0.23276741802692413,
      "kl": 0.06872095167636871,
      "learning_rate": 3e-06,
      "loss": 0.0012,
      "step": 426
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0011854591086013803,
      "grad_norm": 0.11540886759757996,
      "kl": 0.053551726043224335,
      "learning_rate": 3e-06,
      "loss": 0.0002,
      "step": 427
    },
    {
      "clip_ratio": 0.00016356298874597996,
      "epoch": 0.0011882353594411962,
      "grad_norm": 0.19618336856365204,
      "kl": 0.06640568189322948,
      "learning_rate": 3e-06,
      "loss": -0.0006,
      "step": 428
    },
    {
      "clip_ratio": 8.218277798732743e-05,
      "epoch": 0.0011910116102810122,
      "grad_norm": 0.18861764669418335,
      "kl": 0.1051701121032238,
      "learning_rate": 3e-06,
      "loss": 0.0011,
      "step": 429
    },
    {
      "clip_ratio": 0.000244140625,
      "epoch": 0.001193787861120828,
      "grad_norm": 0.14318421483039856,
      "kl": 0.04604991525411606,
      "learning_rate": 3e-06,
      "loss": 0.0016,
      "step": 430
    },
    {
      "clip_ratio": 0.0003451758166193031,
      "epoch": 0.0011965641119606439,
      "grad_norm": 0.14576469361782074,
      "kl": 0.04200077801942825,
      "learning_rate": 3e-06,
      "loss": -0.0002,
      "step": 431
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0011993403628004597,
      "grad_norm": 0.2222023755311966,
      "kl": 0.04660602658987045,
      "learning_rate": 3e-06,
      "loss": -0.004,
      "step": 432
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 236.81250762939453,
      "epoch": 0.0012021166136402756,
      "grad_norm": 0.08921542763710022,
      "kl": 0.04309102147817612,
      "learning_rate": 3e-06,
      "loss": -0.0024,
      "reward": 0.15208333730697632,
      "reward_std": 0.11422888934612274,
      "rewards/countdown_reward_func": 0.15208332985639572,
      "step": 433,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0012048928644800914,
      "grad_norm": 0.1347406804561615,
      "kl": 0.04063236340880394,
      "learning_rate": 3e-06,
      "loss": -0.0032,
      "step": 434
    },
    {
      "clip_ratio": 0.0002881626787711866,
      "epoch": 0.0012076691153199075,
      "grad_norm": 0.11881622672080994,
      "kl": 0.0401871744543314,
      "learning_rate": 3e-06,
      "loss": -0.0035,
      "step": 435
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0012104453661597233,
      "grad_norm": 0.06465653330087662,
      "kl": 0.03999011963605881,
      "learning_rate": 3e-06,
      "loss": -0.003,
      "step": 436
    },
    {
      "clip_ratio": 9.13075273274444e-05,
      "epoch": 0.0012132216169995391,
      "grad_norm": 0.07897289842367172,
      "kl": 0.03400629200041294,
      "learning_rate": 3e-06,
      "loss": -0.0031,
      "step": 437
    },
    {
      "clip_ratio": 0.00045705895172432065,
      "epoch": 0.001215997867839355,
      "grad_norm": 0.07360372692346573,
      "kl": 0.03901753947138786,
      "learning_rate": 3e-06,
      "loss": -0.0032,
      "step": 438
    },
    {
      "clip_ratio": 0.000265190057689324,
      "epoch": 0.0012187741186791708,
      "grad_norm": 0.07743567228317261,
      "kl": 0.030041148886084557,
      "learning_rate": 3e-06,
      "loss": -0.0041,
      "step": 439
    },
    {
      "clip_ratio": 0.000703962694387883,
      "epoch": 0.0012215503695189869,
      "grad_norm": 0.13309620320796967,
      "kl": 0.029125919565558434,
      "learning_rate": 3e-06,
      "loss": -0.0056,
      "step": 440
    },
    {
      "clip_ratio": 0.0005505228764377534,
      "epoch": 0.0012243266203588027,
      "grad_norm": 0.0934651717543602,
      "kl": 0.028855517506599426,
      "learning_rate": 3e-06,
      "loss": -0.0049,
      "step": 441
    },
    {
      "clip_ratio": 0.00017098593525588512,
      "epoch": 0.0012271028711986186,
      "grad_norm": 0.060086771845817566,
      "kl": 0.028184207156300545,
      "learning_rate": 3e-06,
      "loss": -0.0043,
      "step": 442
    },
    {
      "clip_ratio": 0.002318943908903748,
      "epoch": 0.0012298791220384344,
      "grad_norm": 0.07434255629777908,
      "kl": 0.024150204844772816,
      "learning_rate": 3e-06,
      "loss": -0.0043,
      "step": 443
    },
    {
      "clip_ratio": 0.0017171713116113096,
      "epoch": 0.0012326553728782502,
      "grad_norm": 0.06684679538011551,
      "kl": 0.028608722612261772,
      "learning_rate": 3e-06,
      "loss": -0.0045,
      "step": 444
    },
    {
      "clip_ratio": 0.00026747502852231264,
      "completion_length": 234.14584350585938,
      "epoch": 0.001235431623718066,
      "grad_norm": 0.07466662675142288,
      "kl": 0.024345185607671738,
      "learning_rate": 3e-06,
      "loss": 0.0042,
      "reward": 0.16875001043081284,
      "reward_std": 0.1619665026664734,
      "rewards/countdown_reward_func": 0.16875001043081284,
      "step": 445,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0012382078745578821,
      "grad_norm": 0.0632314682006836,
      "kl": 0.02845650538802147,
      "learning_rate": 3e-06,
      "loss": 0.0042,
      "step": 446
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.001240984125397698,
      "grad_norm": 0.064049132168293,
      "kl": 0.0239101629704237,
      "learning_rate": 3e-06,
      "loss": 0.0045,
      "step": 447
    },
    {
      "clip_ratio": 0.0002663619234226644,
      "epoch": 0.0012437603762375138,
      "grad_norm": 0.11490896344184875,
      "kl": 0.02280531730502844,
      "learning_rate": 3e-06,
      "loss": 0.0038,
      "step": 448
    },
    {
      "clip_ratio": 0.00027501455042511225,
      "epoch": 0.0012465366270773296,
      "grad_norm": 0.0676225870847702,
      "kl": 0.02224800456315279,
      "learning_rate": 3e-06,
      "loss": 0.0042,
      "step": 449
    },
    {
      "clip_ratio": 0.000460914452560246,
      "epoch": 0.0012493128779171455,
      "grad_norm": 0.07294591516256332,
      "kl": 0.023516996763646603,
      "learning_rate": 3e-06,
      "loss": 0.0047,
      "step": 450
    },
    {
      "clip_ratio": 0.00017831669538281858,
      "epoch": 0.0012520891287569615,
      "grad_norm": 0.07453124225139618,
      "kl": 0.021801339462399483,
      "learning_rate": 3e-06,
      "loss": 0.0039,
      "step": 451
    },
    {
      "clip_ratio": 0.0003460512016317807,
      "epoch": 0.0012548653795967774,
      "grad_norm": 0.06052190437912941,
      "kl": 0.026570623740553856,
      "learning_rate": 3e-06,
      "loss": 0.004,
      "step": 452
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0012576416304365932,
      "grad_norm": 0.06681171804666519,
      "kl": 0.022533809766173363,
      "learning_rate": 3e-06,
      "loss": 0.004,
      "step": 453
    },
    {
      "clip_ratio": 0.00017356310854665935,
      "epoch": 0.001260417881276409,
      "grad_norm": 0.1189865693449974,
      "kl": 0.02223300840705633,
      "learning_rate": 3e-06,
      "loss": 0.0031,
      "step": 454
    },
    {
      "clip_ratio": 0.00018221575010102242,
      "epoch": 0.001263194132116225,
      "grad_norm": 0.06577113270759583,
      "kl": 0.021816913969814777,
      "learning_rate": 3e-06,
      "loss": 0.0037,
      "step": 455
    },
    {
      "clip_ratio": 0.0005418355867732316,
      "epoch": 0.0012659703829560407,
      "grad_norm": 0.13180898129940033,
      "kl": 0.0227721044793725,
      "learning_rate": 3e-06,
      "loss": 0.0036,
      "step": 456
    },
    {
      "clip_ratio": 0.00020764119108207524,
      "completion_length": 221.70833587646484,
      "epoch": 0.0012687466337958568,
      "grad_norm": 0.109416663646698,
      "kl": 0.022198159247636795,
      "learning_rate": 3e-06,
      "loss": 0.0299,
      "reward": 0.32500001788139343,
      "reward_std": 0.28183095902204514,
      "rewards/countdown_reward_func": 0.32500001788139343,
      "step": 457,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0012715228846356726,
      "grad_norm": 0.12022383511066437,
      "kl": 0.02293353620916605,
      "learning_rate": 3e-06,
      "loss": 0.0297,
      "step": 458
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0012742991354754885,
      "grad_norm": 0.10432354360818863,
      "kl": 0.024400770664215088,
      "learning_rate": 3e-06,
      "loss": 0.0295,
      "step": 459
    },
    {
      "clip_ratio": 9.13075273274444e-05,
      "epoch": 0.0012770753863153043,
      "grad_norm": 0.10826432704925537,
      "kl": 0.022371195256710052,
      "learning_rate": 3e-06,
      "loss": 0.0295,
      "step": 460
    },
    {
      "clip_ratio": 0.00018136516155209392,
      "epoch": 0.0012798516371551201,
      "grad_norm": 0.1814108043909073,
      "kl": 0.02964179776608944,
      "learning_rate": 3e-06,
      "loss": 0.029,
      "step": 461
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0012826278879949362,
      "grad_norm": 0.1098814606666565,
      "kl": 0.02524806559085846,
      "learning_rate": 3e-06,
      "loss": 0.0288,
      "step": 462
    },
    {
      "clip_ratio": 0.00020764119108207524,
      "epoch": 0.001285404138834752,
      "grad_norm": 0.10370974242687225,
      "kl": 0.023861460387706757,
      "learning_rate": 3e-06,
      "loss": 0.0284,
      "step": 463
    },
    {
      "clip_ratio": 0.00010382059554103762,
      "epoch": 0.0012881803896745679,
      "grad_norm": 0.1190827265381813,
      "kl": 0.024492272175848484,
      "learning_rate": 3e-06,
      "loss": 0.0278,
      "step": 464
    },
    {
      "clip_ratio": 0.00046262028627097607,
      "epoch": 0.0012909566405143837,
      "grad_norm": 0.10446801036596298,
      "kl": 0.027533382177352905,
      "learning_rate": 3e-06,
      "loss": 0.0282,
      "step": 465
    },
    {
      "clip_ratio": 0.0005552970251301304,
      "epoch": 0.0012937328913541996,
      "grad_norm": 0.10887821763753891,
      "kl": 0.024930739775300026,
      "learning_rate": 3e-06,
      "loss": 0.0271,
      "step": 466
    },
    {
      "clip_ratio": 0.0009011498477775604,
      "epoch": 0.0012965091421940156,
      "grad_norm": 0.15684081614017487,
      "kl": 0.03675767965614796,
      "learning_rate": 3e-06,
      "loss": 0.026,
      "step": 467
    },
    {
      "clip_ratio": 0.0004027693357784301,
      "epoch": 0.0012992853930338315,
      "grad_norm": 0.11377524584531784,
      "kl": 0.03045613970607519,
      "learning_rate": 3e-06,
      "loss": 0.0264,
      "step": 468
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 239.1875,
      "epoch": 0.0013020616438736473,
      "grad_norm": 0.10519543290138245,
      "kl": 0.03355495072901249,
      "learning_rate": 3e-06,
      "loss": 0.0046,
      "reward": 0.20625001192092896,
      "reward_std": 0.2200612723827362,
      "rewards/countdown_reward_func": 0.20625001192092896,
      "step": 469,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0013048378947134631,
      "grad_norm": 0.09295057505369186,
      "kl": 0.03722470626235008,
      "learning_rate": 3e-06,
      "loss": 0.0037,
      "step": 470
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.001307614145553279,
      "grad_norm": 0.07510315626859665,
      "kl": 0.035189252346754074,
      "learning_rate": 3e-06,
      "loss": 0.0038,
      "step": 471
    },
    {
      "clip_ratio": 0.000260960339801386,
      "epoch": 0.0013103903963930948,
      "grad_norm": 0.1088418960571289,
      "kl": 0.03208626061677933,
      "learning_rate": 3e-06,
      "loss": 0.0039,
      "step": 472
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0013131666472329109,
      "grad_norm": 0.08343033492565155,
      "kl": 0.03320677764713764,
      "learning_rate": 3e-06,
      "loss": 0.0031,
      "step": 473
    },
    {
      "clip_ratio": 0.0002764667078736238,
      "epoch": 0.0013159428980727267,
      "grad_norm": 0.11299256235361099,
      "kl": 0.0330524817109108,
      "learning_rate": 3e-06,
      "loss": 0.0035,
      "step": 474
    },
    {
      "clip_ratio": 0.00027988169313175604,
      "epoch": 0.0013187191489125426,
      "grad_norm": 0.10915958881378174,
      "kl": 0.04382970742881298,
      "learning_rate": 3e-06,
      "loss": 0.0033,
      "step": 475
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0013214953997523584,
      "grad_norm": 0.08877313882112503,
      "kl": 0.04925047419965267,
      "learning_rate": 3e-06,
      "loss": 0.0027,
      "step": 476
    },
    {
      "clip_ratio": 0.00017536517407279462,
      "epoch": 0.0013242716505921742,
      "grad_norm": 0.08160615712404251,
      "kl": 0.0415896400809288,
      "learning_rate": 3e-06,
      "loss": 0.0036,
      "step": 477
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0013270479014319903,
      "grad_norm": 0.14993789792060852,
      "kl": 0.036379581317305565,
      "learning_rate": 3e-06,
      "loss": 0.0034,
      "step": 478
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0013298241522718061,
      "grad_norm": 0.08193667978048325,
      "kl": 0.03808531537652016,
      "learning_rate": 3e-06,
      "loss": 0.0028,
      "step": 479
    },
    {
      "clip_ratio": 0.0002638619553181343,
      "epoch": 0.001332600403111622,
      "grad_norm": 0.10145576298236847,
      "kl": 0.0362465288490057,
      "learning_rate": 3e-06,
      "loss": 0.0028,
      "step": 480
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 232.0625,
      "epoch": 0.0013353766539514378,
      "grad_norm": 0.11346805095672607,
      "kl": 0.038885802030563354,
      "learning_rate": 3e-06,
      "loss": 0.0224,
      "reward": 0.19166667014360428,
      "reward_std": 0.20098165795207024,
      "rewards/countdown_reward_func": 0.19166667014360428,
      "step": 481,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0013381529047912536,
      "grad_norm": 0.0885414257645607,
      "kl": 0.03647511638700962,
      "learning_rate": 3e-06,
      "loss": 0.0223,
      "step": 482
    },
    {
      "clip_ratio": 0.0002071251074085012,
      "epoch": 0.0013409291556310695,
      "grad_norm": 0.09309430420398712,
      "kl": 0.03728037141263485,
      "learning_rate": 3e-06,
      "loss": 0.0219,
      "step": 483
    },
    {
      "clip_ratio": 0.00018494276446290314,
      "epoch": 0.0013437054064708855,
      "grad_norm": 0.08568699657917023,
      "kl": 0.053651321679353714,
      "learning_rate": 3e-06,
      "loss": 0.0231,
      "step": 484
    },
    {
      "clip_ratio": 0.00026833474839804694,
      "epoch": 0.0013464816573107014,
      "grad_norm": 0.12132450193166733,
      "kl": 0.04722470976412296,
      "learning_rate": 3e-06,
      "loss": 0.0222,
      "step": 485
    },
    {
      "clip_ratio": 8.532423089491203e-05,
      "epoch": 0.0013492579081505172,
      "grad_norm": 0.09739983081817627,
      "kl": 0.03846907988190651,
      "learning_rate": 3e-06,
      "loss": 0.0218,
      "step": 486
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.001352034158990333,
      "grad_norm": 0.1051330417394638,
      "kl": 0.04565967619419098,
      "learning_rate": 3e-06,
      "loss": 0.0213,
      "step": 487
    },
    {
      "clip_ratio": 0.0003686312338686548,
      "epoch": 0.001354810409830149,
      "grad_norm": 0.08285031467676163,
      "kl": 0.04506035894155502,
      "learning_rate": 3e-06,
      "loss": 0.0212,
      "step": 488
    },
    {
      "clip_ratio": 0.0008285004296340048,
      "epoch": 0.001357586660669965,
      "grad_norm": 0.0878533124923706,
      "kl": 0.04655962623655796,
      "learning_rate": 3e-06,
      "loss": 0.0206,
      "step": 489
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0013603629115097808,
      "grad_norm": 0.07896918803453445,
      "kl": 0.06781511753797531,
      "learning_rate": 3e-06,
      "loss": 0.0215,
      "step": 490
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0013631391623495966,
      "grad_norm": 0.10892756283283234,
      "kl": 0.06482304260134697,
      "learning_rate": 3e-06,
      "loss": 0.0204,
      "step": 491
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0013659154131894125,
      "grad_norm": 0.09697525948286057,
      "kl": 0.052130842581391335,
      "learning_rate": 3e-06,
      "loss": 0.0204,
      "step": 492
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 230.4791717529297,
      "epoch": 0.0013686916640292283,
      "grad_norm": 0.12033633142709732,
      "kl": 0.05308363772928715,
      "learning_rate": 3e-06,
      "loss": -0.0028,
      "reward": 0.21041666716337204,
      "reward_std": 0.17074457183480263,
      "rewards/countdown_reward_func": 0.21041666716337204,
      "step": 493,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio": 0.0003349836479173973,
      "epoch": 0.0013714679148690441,
      "grad_norm": 0.09749346226453781,
      "kl": 0.05684215575456619,
      "learning_rate": 3e-06,
      "loss": -0.0034,
      "step": 494
    },
    {
      "clip_ratio": 9.025270992424339e-05,
      "epoch": 0.0013742441657088602,
      "grad_norm": 0.08872581273317337,
      "kl": 0.05968480557203293,
      "learning_rate": 3e-06,
      "loss": -0.003,
      "step": 495
    },
    {
      "clip_ratio": 0.00028852681134594604,
      "epoch": 0.001377020416548676,
      "grad_norm": 0.11848699301481247,
      "kl": 0.06252800300717354,
      "learning_rate": 3e-06,
      "loss": -0.0026,
      "step": 496
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0013797966673884919,
      "grad_norm": 0.24924242496490479,
      "kl": 0.06530779227614403,
      "learning_rate": 3e-06,
      "loss": -0.0012,
      "step": 497
    },
    {
      "clip_ratio": 0.0003621914656832814,
      "epoch": 0.0013825729182283077,
      "grad_norm": 0.11978671699762344,
      "kl": 0.06549267843365669,
      "learning_rate": 3e-06,
      "loss": -0.0024,
      "step": 498
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0013853491690681236,
      "grad_norm": 0.08640985190868378,
      "kl": 0.06214660406112671,
      "learning_rate": 3e-06,
      "loss": -0.0034,
      "step": 499
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0013881254199079396,
      "grad_norm": 0.10044127702713013,
      "kl": 0.061088208109140396,
      "learning_rate": 3e-06,
      "loss": -0.0035,
      "step": 500
    },
    {
      "clip_ratio": 9.025270992424339e-05,
      "epoch": 0.0013909016707477555,
      "grad_norm": 0.09724526852369308,
      "kl": 0.06105843186378479,
      "learning_rate": 3e-06,
      "loss": -0.0036,
      "step": 501
    },
    {
      "clip_ratio": 0.00047674778033979237,
      "epoch": 0.0013936779215875713,
      "grad_norm": 0.1106707900762558,
      "kl": 0.05963746830821037,
      "learning_rate": 3e-06,
      "loss": -0.0037,
      "step": 502
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0013964541724273871,
      "grad_norm": 0.28701546788215637,
      "kl": 0.058546338230371475,
      "learning_rate": 3e-06,
      "loss": -0.0043,
      "step": 503
    },
    {
      "clip_ratio": 0.0002860495515051298,
      "epoch": 0.001399230423267203,
      "grad_norm": 0.11430586874485016,
      "kl": 0.05745413526892662,
      "learning_rate": 3e-06,
      "loss": -0.0037,
      "step": 504
    },
    {
      "clip_ratio": 0.0004231043276377022,
      "completion_length": 207.95833587646484,
      "epoch": 0.0014020066741070188,
      "grad_norm": 0.1265731304883957,
      "kl": 0.06725966557860374,
      "learning_rate": 3e-06,
      "loss": -0.0148,
      "reward": 0.10833334177732468,
      "reward_std": 0.0599165465682745,
      "rewards/countdown_reward_func": 0.10833334177732468,
      "step": 505,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio": 0.0001892800792120397,
      "epoch": 0.0014047829249468349,
      "grad_norm": 0.07790236175060272,
      "kl": 0.06213269755244255,
      "learning_rate": 3e-06,
      "loss": -0.0152,
      "step": 506
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0014075591757866507,
      "grad_norm": 0.10863670706748962,
      "kl": 0.05851333029568195,
      "learning_rate": 3e-06,
      "loss": -0.0152,
      "step": 507
    },
    {
      "clip_ratio": 0.00011870844900840893,
      "epoch": 0.0014103354266264666,
      "grad_norm": 0.11995543539524078,
      "kl": 0.0762905403971672,
      "learning_rate": 3e-06,
      "loss": -0.0152,
      "step": 508
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0014131116774662824,
      "grad_norm": 0.10893931984901428,
      "kl": 0.07419506646692753,
      "learning_rate": 3e-06,
      "loss": -0.0151,
      "step": 509
    },
    {
      "clip_ratio": 0.0002796600092551671,
      "epoch": 0.0014158879283060982,
      "grad_norm": 0.11752152442932129,
      "kl": 0.05459407716989517,
      "learning_rate": 3e-06,
      "loss": -0.0165,
      "step": 510
    },
    {
      "clip_ratio": 0.00028186137205921113,
      "epoch": 0.0014186641791459143,
      "grad_norm": 0.10197150707244873,
      "kl": 0.0478304848074913,
      "learning_rate": 3e-06,
      "loss": -0.0171,
      "step": 511
    },
    {
      "clip_ratio": 0.0010716330725699663,
      "epoch": 0.0014214404299857301,
      "grad_norm": 0.06922204792499542,
      "kl": 0.042900703847408295,
      "learning_rate": 3e-06,
      "loss": -0.0171,
      "step": 512
    },
    {
      "clip_ratio": 0.00039922940777614713,
      "epoch": 0.001424216680825546,
      "grad_norm": 0.09703180193901062,
      "kl": 0.03880976140499115,
      "learning_rate": 3e-06,
      "loss": -0.0185,
      "step": 513
    },
    {
      "clip_ratio": 0.0022654032800346613,
      "epoch": 0.0014269929316653618,
      "grad_norm": 0.10023833066225052,
      "kl": 0.04965279810130596,
      "learning_rate": 3e-06,
      "loss": -0.0183,
      "step": 514
    },
    {
      "clip_ratio": 0.0015507703792536631,
      "epoch": 0.0014297691825051776,
      "grad_norm": 0.0990394875407219,
      "kl": 0.04965635947883129,
      "learning_rate": 3e-06,
      "loss": -0.0188,
      "step": 515
    },
    {
      "clip_ratio": 0.0020738598541356623,
      "epoch": 0.0014325454333449935,
      "grad_norm": 0.1003248319029808,
      "kl": 0.03512744698673487,
      "learning_rate": 3e-06,
      "loss": -0.0204,
      "step": 516
    },
    {
      "clip_ratio": 0.00010129659494850785,
      "completion_length": 233.4166717529297,
      "epoch": 0.0014353216841848095,
      "grad_norm": 0.09848001599311829,
      "kl": 0.026790697127580643,
      "learning_rate": 3e-06,
      "loss": -0.0064,
      "reward": 0.2250000238418579,
      "reward_std": 0.2915927767753601,
      "rewards/countdown_reward_func": 0.22500000894069672,
      "step": 517,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.0002025931898970157,
      "epoch": 0.0014380979350246254,
      "grad_norm": 0.10145186632871628,
      "kl": 0.024240675382316113,
      "learning_rate": 3e-06,
      "loss": -0.0064,
      "step": 518
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0014408741858644412,
      "grad_norm": 0.09955999255180359,
      "kl": 0.02282124198973179,
      "learning_rate": 3e-06,
      "loss": -0.0068,
      "step": 519
    },
    {
      "clip_ratio": 0.0001803751802071929,
      "epoch": 0.001443650436704257,
      "grad_norm": 0.11280816793441772,
      "kl": 0.024487541057169437,
      "learning_rate": 3e-06,
      "loss": -0.0059,
      "step": 520
    },
    {
      "clip_ratio": 0.0003646711993496865,
      "epoch": 0.001446426687544073,
      "grad_norm": 0.10844244807958603,
      "kl": 0.025233074091374874,
      "learning_rate": 3e-06,
      "loss": -0.0072,
      "step": 521
    },
    {
      "clip_ratio": 8.722958591533825e-05,
      "epoch": 0.001449202938383889,
      "grad_norm": 0.16373442113399506,
      "kl": 0.024108163081109524,
      "learning_rate": 3e-06,
      "loss": -0.0069,
      "step": 522
    },
    {
      "clip_ratio": 0.0006587411771761253,
      "epoch": 0.0014519791892237048,
      "grad_norm": 0.09821134060621262,
      "kl": 0.022494456730782986,
      "learning_rate": 3e-06,
      "loss": -0.0072,
      "step": 523
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0014547554400635206,
      "grad_norm": 0.1120600625872612,
      "kl": 0.021051418967545033,
      "learning_rate": 3e-06,
      "loss": -0.0086,
      "step": 524
    },
    {
      "clip_ratio": 0.00027575576677918434,
      "epoch": 0.0014575316909033365,
      "grad_norm": 0.09990929067134857,
      "kl": 0.020683609880506992,
      "learning_rate": 3e-06,
      "loss": -0.0081,
      "step": 525
    },
    {
      "clip_ratio": 0.0008699278841959313,
      "epoch": 0.0014603079417431523,
      "grad_norm": 0.1135360524058342,
      "kl": 0.02212886419147253,
      "learning_rate": 3e-06,
      "loss": -0.0077,
      "step": 526
    },
    {
      "clip_ratio": 0.0009819942351896316,
      "epoch": 0.0014630841925829681,
      "grad_norm": 0.09932764619588852,
      "kl": 0.02397053875029087,
      "learning_rate": 3e-06,
      "loss": -0.008,
      "step": 527
    },
    {
      "clip_ratio": 0.0010553163010627031,
      "epoch": 0.0014658604434227842,
      "grad_norm": 0.17817182838916779,
      "kl": 0.02317346353083849,
      "learning_rate": 3e-06,
      "loss": -0.008,
      "step": 528
    },
    {
      "clip_ratio": 0.00024488919734722003,
      "completion_length": 238.56250762939453,
      "epoch": 0.0014686366942626,
      "grad_norm": 0.06560882925987244,
      "kl": 0.025266059674322605,
      "learning_rate": 3e-06,
      "loss": -0.0058,
      "reward": 0.19166668504476547,
      "reward_std": 0.15476077422499657,
      "rewards/countdown_reward_func": 0.19166668504476547,
      "step": 529,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0014714129451024159,
      "grad_norm": 0.06739655882120132,
      "kl": 0.02375541441142559,
      "learning_rate": 3e-06,
      "loss": -0.0055,
      "step": 530
    },
    {
      "clip_ratio": 0.00017831669538281858,
      "epoch": 0.0014741891959422317,
      "grad_norm": 0.10260723531246185,
      "kl": 0.028817658312618732,
      "learning_rate": 3e-06,
      "loss": -0.0052,
      "step": 531
    },
    {
      "clip_ratio": 0.00017053855845006183,
      "epoch": 0.0014769654467820476,
      "grad_norm": 0.0644664391875267,
      "kl": 0.028914256952703,
      "learning_rate": 3e-06,
      "loss": -0.0053,
      "step": 532
    },
    {
      "clip_ratio": 9.03179170563817e-05,
      "epoch": 0.0014797416976218636,
      "grad_norm": 0.11235247552394867,
      "kl": 0.024057154543697834,
      "learning_rate": 3e-06,
      "loss": -0.0055,
      "step": 533
    },
    {
      "clip_ratio": 0.0001695947503321804,
      "epoch": 0.0014825179484616795,
      "grad_norm": 0.20861512422561646,
      "kl": 0.027661575004458427,
      "learning_rate": 3e-06,
      "loss": -0.0074,
      "step": 534
    },
    {
      "clip_ratio": 0.00025097496109083295,
      "epoch": 0.0014852941993014953,
      "grad_norm": 0.06489825993776321,
      "kl": 0.026025223545730114,
      "learning_rate": 3e-06,
      "loss": -0.0062,
      "step": 535
    },
    {
      "clip_ratio": 0.0007896195165812969,
      "epoch": 0.0014880704501413111,
      "grad_norm": 0.07131274044513702,
      "kl": 0.025113885290920734,
      "learning_rate": 3e-06,
      "loss": -0.0059,
      "step": 536
    },
    {
      "clip_ratio": 0.000509865116328001,
      "epoch": 0.001490846700981127,
      "grad_norm": 0.10190945118665695,
      "kl": 0.028591866604983807,
      "learning_rate": 3e-06,
      "loss": -0.0073,
      "step": 537
    },
    {
      "clip_ratio": 0.0011356433387845755,
      "epoch": 0.001493622951820943,
      "grad_norm": 0.07074970752000809,
      "kl": 0.027634769678115845,
      "learning_rate": 3e-06,
      "loss": -0.0058,
      "step": 538
    },
    {
      "clip_ratio": 0.000357331897248514,
      "epoch": 0.0014963992026607589,
      "grad_norm": 0.09467112272977829,
      "kl": 0.02522993925958872,
      "learning_rate": 3e-06,
      "loss": -0.0074,
      "step": 539
    },
    {
      "clip_ratio": 0.0012593485007528216,
      "epoch": 0.0014991754535005747,
      "grad_norm": 0.22211527824401855,
      "kl": 0.027548625133931637,
      "learning_rate": 3e-06,
      "loss": -0.0112,
      "step": 540
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 235.39584350585938,
      "epoch": 0.0015019517043403905,
      "grad_norm": 0.07119705528020859,
      "kl": 0.02431309036910534,
      "learning_rate": 3e-06,
      "loss": 0.0065,
      "reward": 0.30000002682209015,
      "reward_std": 0.22795327007770538,
      "rewards/countdown_reward_func": 0.30000001192092896,
      "step": 541,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.00016297261754516512,
      "epoch": 0.0015047279551802064,
      "grad_norm": 0.09539937227964401,
      "kl": 0.025972411036491394,
      "learning_rate": 3e-06,
      "loss": 0.006,
      "step": 542
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0015075042060200222,
      "grad_norm": 0.1664063185453415,
      "kl": 0.024256471544504166,
      "learning_rate": 3e-06,
      "loss": 0.0062,
      "step": 543
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0015102804568598383,
      "grad_norm": 0.07563801109790802,
      "kl": 0.02507698815315962,
      "learning_rate": 3e-06,
      "loss": 0.0052,
      "step": 544
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0015130567076996541,
      "grad_norm": 0.07607442140579224,
      "kl": 0.02424045093357563,
      "learning_rate": 3e-06,
      "loss": 0.0058,
      "step": 545
    },
    {
      "clip_ratio": 0.0006382791907526553,
      "epoch": 0.00151583295853947,
      "grad_norm": 0.15590515732765198,
      "kl": 0.02763733733445406,
      "learning_rate": 3e-06,
      "loss": 0.0064,
      "step": 546
    },
    {
      "clip_ratio": 0.0003597766626626253,
      "epoch": 0.0015186092093792858,
      "grad_norm": 0.07136835902929306,
      "kl": 0.025670908391475677,
      "learning_rate": 3e-06,
      "loss": 0.0057,
      "step": 547
    },
    {
      "clip_ratio": 0.0004282992740627378,
      "epoch": 0.0015213854602191016,
      "grad_norm": 0.08294139057397842,
      "kl": 0.027309386059641838,
      "learning_rate": 3e-06,
      "loss": 0.0059,
      "step": 548
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0015241617110589177,
      "grad_norm": 0.1601613610982895,
      "kl": 0.025316720828413963,
      "learning_rate": 3e-06,
      "loss": 0.0056,
      "step": 549
    },
    {
      "clip_ratio": 0.0002441406322759576,
      "epoch": 0.0015269379618987335,
      "grad_norm": 0.0745542123913765,
      "kl": 0.025896431878209114,
      "learning_rate": 3e-06,
      "loss": 0.0051,
      "step": 550
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0015297142127385494,
      "grad_norm": 0.06619153916835785,
      "kl": 0.024953342974185944,
      "learning_rate": 3e-06,
      "loss": 0.005,
      "step": 551
    },
    {
      "clip_ratio": 0.000708240841049701,
      "epoch": 0.0015324904635783652,
      "grad_norm": 0.13466133177280426,
      "kl": 0.028307722881436348,
      "learning_rate": 3e-06,
      "loss": 0.0054,
      "step": 552
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 234.81250762939453,
      "epoch": 0.001535266714418181,
      "grad_norm": 0.09829782694578171,
      "kl": 0.02855612989515066,
      "learning_rate": 3e-06,
      "loss": 0.0059,
      "reward": 0.19166667014360428,
      "reward_std": 0.19716466218233109,
      "rewards/countdown_reward_func": 0.19166666269302368,
      "step": 553,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio": 0.00018876852846005931,
      "epoch": 0.001538042965257997,
      "grad_norm": 0.07418268173933029,
      "kl": 0.026963720098137856,
      "learning_rate": 3e-06,
      "loss": 0.0054,
      "step": 554
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.001540819216097813,
      "grad_norm": 0.07878899574279785,
      "kl": 0.028570099733769894,
      "learning_rate": 3e-06,
      "loss": 0.006,
      "step": 555
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0015435954669376288,
      "grad_norm": 0.10893654823303223,
      "kl": 0.027858249843120575,
      "learning_rate": 3e-06,
      "loss": 0.0057,
      "step": 556
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0015463717177774446,
      "grad_norm": 0.07773488014936447,
      "kl": 0.02769723255187273,
      "learning_rate": 3e-06,
      "loss": 0.0057,
      "step": 557
    },
    {
      "clip_ratio": 0.00019869584502885118,
      "epoch": 0.0015491479686172605,
      "grad_norm": 0.08286737650632858,
      "kl": 0.02732114028185606,
      "learning_rate": 3e-06,
      "loss": 0.0053,
      "step": 558
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0015519242194570763,
      "grad_norm": 0.08795492351055145,
      "kl": 0.027023627422749996,
      "learning_rate": 3e-06,
      "loss": 0.0049,
      "step": 559
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0015547004702968924,
      "grad_norm": 0.07500362396240234,
      "kl": 0.02461559884250164,
      "learning_rate": 3e-06,
      "loss": 0.0044,
      "step": 560
    },
    {
      "clip_ratio": 0.00018876852846005931,
      "epoch": 0.0015574767211367082,
      "grad_norm": 0.08417128771543503,
      "kl": 0.026552588678896427,
      "learning_rate": 3e-06,
      "loss": 0.0052,
      "step": 561
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.001560252971976524,
      "grad_norm": 0.07278092950582504,
      "kl": 0.024690870195627213,
      "learning_rate": 3e-06,
      "loss": 0.0046,
      "step": 562
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0015630292228163399,
      "grad_norm": 0.07505667209625244,
      "kl": 0.02481877151876688,
      "learning_rate": 3e-06,
      "loss": 0.0054,
      "step": 563
    },
    {
      "clip_ratio": 0.00010738831770140678,
      "epoch": 0.0015658054736561557,
      "grad_norm": 0.08289463818073273,
      "kl": 0.024328703992068768,
      "learning_rate": 3e-06,
      "loss": 0.0044,
      "step": 564
    },
    {
      "clip_ratio": 0.00019328358030179515,
      "completion_length": 228.62500762939453,
      "epoch": 0.0015685817244959716,
      "grad_norm": 0.09321510046720505,
      "kl": 0.02534264326095581,
      "learning_rate": 3e-06,
      "loss": -0.0105,
      "reward": 0.27916668355464935,
      "reward_std": 0.28499096632003784,
      "rewards/countdown_reward_func": 0.27916668355464935,
      "step": 565,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0015713579753357876,
      "grad_norm": 0.09789825975894928,
      "kl": 0.023500449024140835,
      "learning_rate": 3e-06,
      "loss": -0.0099,
      "step": 566
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0015741342261756035,
      "grad_norm": 0.12602142989635468,
      "kl": 0.024465853348374367,
      "learning_rate": 3e-06,
      "loss": -0.01,
      "step": 567
    },
    {
      "clip_ratio": 0.00026942871045321226,
      "epoch": 0.0015769104770154193,
      "grad_norm": 0.07588708400726318,
      "kl": 0.021980220451951027,
      "learning_rate": 3e-06,
      "loss": -0.0094,
      "step": 568
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0015796867278552351,
      "grad_norm": 0.10751640051603317,
      "kl": 0.02414284460246563,
      "learning_rate": 3e-06,
      "loss": -0.0093,
      "step": 569
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.001582462978695051,
      "grad_norm": 0.12433457374572754,
      "kl": 0.02209012396633625,
      "learning_rate": 3e-06,
      "loss": -0.0097,
      "step": 570
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.001585239229534867,
      "grad_norm": 0.1603914499282837,
      "kl": 0.02457761950790882,
      "learning_rate": 3e-06,
      "loss": -0.0102,
      "step": 571
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0015880154803746829,
      "grad_norm": 0.09691650420427322,
      "kl": 0.023993924260139465,
      "learning_rate": 3e-06,
      "loss": -0.0106,
      "step": 572
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0015907917312144987,
      "grad_norm": 0.13051147758960724,
      "kl": 0.02514663338661194,
      "learning_rate": 3e-06,
      "loss": -0.0105,
      "step": 573
    },
    {
      "clip_ratio": 0.00018027036276180297,
      "epoch": 0.0015935679820543145,
      "grad_norm": 0.14570623636245728,
      "kl": 0.023281600326299667,
      "learning_rate": 3e-06,
      "loss": -0.0104,
      "step": 574
    },
    {
      "clip_ratio": 8.698677993379533e-05,
      "epoch": 0.0015963442328941304,
      "grad_norm": 0.10116558521986008,
      "kl": 0.026070833206176758,
      "learning_rate": 3e-06,
      "loss": -0.0109,
      "step": 575
    },
    {
      "clip_ratio": 8.698677993379533e-05,
      "epoch": 0.0015991204837339462,
      "grad_norm": 0.1298682689666748,
      "kl": 0.02464818675071001,
      "learning_rate": 3e-06,
      "loss": -0.0109,
      "step": 576
    },
    {
      "clip_ratio": 0.0005351027357392013,
      "completion_length": 240.83333587646484,
      "epoch": 0.0016018967345737623,
      "grad_norm": 0.05101247504353523,
      "kl": 0.02285961899906397,
      "learning_rate": 3e-06,
      "loss": 0.0031,
      "reward": 0.2083333507180214,
      "reward_std": 0.16015682369470596,
      "rewards/countdown_reward_func": 0.2083333507180214,
      "step": 577,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0016046729854135781,
      "grad_norm": 0.07094918191432953,
      "kl": 0.023334643803536892,
      "learning_rate": 3e-06,
      "loss": 0.0029,
      "step": 578
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.001607449236253394,
      "grad_norm": 0.047830481082201004,
      "kl": 0.0244672242552042,
      "learning_rate": 3e-06,
      "loss": 0.003,
      "step": 579
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0016102254870932098,
      "grad_norm": 0.06464120745658875,
      "kl": 0.024777245707809925,
      "learning_rate": 3e-06,
      "loss": 0.0036,
      "step": 580
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0016130017379330256,
      "grad_norm": 0.062469832599163055,
      "kl": 0.02657880913466215,
      "learning_rate": 3e-06,
      "loss": 0.0032,
      "step": 581
    },
    {
      "clip_ratio": 0.0007138257715268992,
      "epoch": 0.0016157779887728417,
      "grad_norm": 0.07282475382089615,
      "kl": 0.026876126416027546,
      "learning_rate": 3e-06,
      "loss": 0.0027,
      "step": 582
    },
    {
      "clip_ratio": 0.00042808218859136105,
      "epoch": 0.0016185542396126575,
      "grad_norm": 0.057990189641714096,
      "kl": 0.025540747679769993,
      "learning_rate": 3e-06,
      "loss": 0.0029,
      "step": 583
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0016213304904524734,
      "grad_norm": 0.07290374487638474,
      "kl": 0.02545579057186842,
      "learning_rate": 3e-06,
      "loss": 0.0028,
      "step": 584
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0016241067412922892,
      "grad_norm": 0.04659357666969299,
      "kl": 0.026573507115244865,
      "learning_rate": 3e-06,
      "loss": 0.0028,
      "step": 585
    },
    {
      "clip_ratio": 0.0006510416860692203,
      "epoch": 0.001626882992132105,
      "grad_norm": 0.05872870981693268,
      "kl": 0.02712887153029442,
      "learning_rate": 3e-06,
      "loss": 0.0031,
      "step": 586
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.001629659242971921,
      "grad_norm": 0.05923086777329445,
      "kl": 0.02828708291053772,
      "learning_rate": 3e-06,
      "loss": 0.0029,
      "step": 587
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.001632435493811737,
      "grad_norm": 0.07497607171535492,
      "kl": 0.02832121029496193,
      "learning_rate": 3e-06,
      "loss": 0.002,
      "step": 588
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 240.33333587646484,
      "epoch": 0.0016352117446515528,
      "grad_norm": 0.12389404326677322,
      "kl": 0.028777985833585262,
      "learning_rate": 3e-06,
      "loss": 0.0163,
      "reward": 0.3583333417773247,
      "reward_std": 0.36252936720848083,
      "rewards/countdown_reward_func": 0.3583333268761635,
      "step": 589,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0016379879954913686,
      "grad_norm": 0.16146422922611237,
      "kl": 0.029465525411069393,
      "learning_rate": 3e-06,
      "loss": 0.0157,
      "step": 590
    },
    {
      "clip_ratio": 0.0002460629912093282,
      "epoch": 0.0016407642463311845,
      "grad_norm": 0.11740246415138245,
      "kl": 0.02861588355153799,
      "learning_rate": 3e-06,
      "loss": 0.0157,
      "step": 591
    },
    {
      "clip_ratio": 8.674531272845343e-05,
      "epoch": 0.0016435404971710003,
      "grad_norm": 0.11579007655382156,
      "kl": 0.02733410894870758,
      "learning_rate": 3e-06,
      "loss": 0.0159,
      "step": 592
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0016463167480108164,
      "grad_norm": 0.09282111376523972,
      "kl": 0.029231702908873558,
      "learning_rate": 3e-06,
      "loss": 0.0151,
      "step": 593
    },
    {
      "clip_ratio": 0.0002602359454613179,
      "epoch": 0.0016490929988506322,
      "grad_norm": 0.15394672751426697,
      "kl": 0.029871191829442978,
      "learning_rate": 3e-06,
      "loss": 0.0157,
      "step": 594
    },
    {
      "clip_ratio": 8.202099706977606e-05,
      "epoch": 0.001651869249690448,
      "grad_norm": 0.12174469977617264,
      "kl": 0.027274416759610176,
      "learning_rate": 3e-06,
      "loss": 0.0142,
      "step": 595
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0016546455005302639,
      "grad_norm": 0.20378051698207855,
      "kl": 0.026893497444689274,
      "learning_rate": 3e-06,
      "loss": 0.0129,
      "step": 596
    },
    {
      "clip_ratio": 0.00016404199413955212,
      "epoch": 0.0016574217513700797,
      "grad_norm": 0.11612839996814728,
      "kl": 0.027020023204386234,
      "learning_rate": 3e-06,
      "loss": 0.0138,
      "step": 597
    },
    {
      "clip_ratio": 8.555784006603062e-05,
      "epoch": 0.0016601980022098958,
      "grad_norm": 0.10161249339580536,
      "kl": 0.025793558917939663,
      "learning_rate": 3e-06,
      "loss": 0.013,
      "step": 598
    },
    {
      "clip_ratio": 8.555784006603062e-05,
      "epoch": 0.0016629742530497116,
      "grad_norm": 0.09491390734910965,
      "kl": 0.027327225543558598,
      "learning_rate": 3e-06,
      "loss": 0.0133,
      "step": 599
    },
    {
      "clip_ratio": 0.000265891409071628,
      "epoch": 0.0016657505038895275,
      "grad_norm": 0.13563629984855652,
      "kl": 0.027993053197860718,
      "learning_rate": 3e-06,
      "loss": 0.0133,
      "step": 600
    },
    {
      "clip_ratio": 8.322236681124195e-05,
      "completion_length": 238.75000762939453,
      "epoch": 0.0016685267547293433,
      "grad_norm": 0.09000701457262039,
      "kl": 0.028913519345223904,
      "learning_rate": 3e-06,
      "loss": 0.0272,
      "reward": 0.2083333432674408,
      "reward_std": 0.24819570034742355,
      "rewards/countdown_reward_func": 0.2083333283662796,
      "step": 601,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0016713030055691591,
      "grad_norm": 0.08314099907875061,
      "kl": 0.025995067320764065,
      "learning_rate": 3e-06,
      "loss": 0.0262,
      "step": 602
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.001674079256408975,
      "grad_norm": 0.1207355260848999,
      "kl": 0.03479589242488146,
      "learning_rate": 3e-06,
      "loss": 0.0271,
      "step": 603
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.001676855507248791,
      "grad_norm": 0.11306949704885483,
      "kl": 0.030153939500451088,
      "learning_rate": 3e-06,
      "loss": 0.0267,
      "step": 604
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0016796317580886069,
      "grad_norm": 0.08935578912496567,
      "kl": 0.028079458512365818,
      "learning_rate": 3e-06,
      "loss": 0.0258,
      "step": 605
    },
    {
      "clip_ratio": 0.00024782493710517883,
      "epoch": 0.0016824080089284227,
      "grad_norm": 0.0978025496006012,
      "kl": 0.029070213437080383,
      "learning_rate": 3e-06,
      "loss": 0.0253,
      "step": 606
    },
    {
      "clip_ratio": 0.0005404279145295732,
      "epoch": 0.0016851842597682385,
      "grad_norm": 0.0878811702132225,
      "kl": 0.03100433386862278,
      "learning_rate": 3e-06,
      "loss": 0.025,
      "step": 607
    },
    {
      "clip_ratio": 0.00018288222781848162,
      "epoch": 0.0016879605106080544,
      "grad_norm": 0.09215917438268661,
      "kl": 0.02795230783522129,
      "learning_rate": 3e-06,
      "loss": 0.0248,
      "step": 608
    },
    {
      "clip_ratio": 9.144111390924081e-05,
      "epoch": 0.0016907367614478704,
      "grad_norm": 0.12471023947000504,
      "kl": 0.03957472741603851,
      "learning_rate": 3e-06,
      "loss": 0.0252,
      "step": 609
    },
    {
      "clip_ratio": 8.322236681124195e-05,
      "epoch": 0.0016935130122876863,
      "grad_norm": 0.1100461557507515,
      "kl": 0.0361151285469532,
      "learning_rate": 3e-06,
      "loss": 0.0234,
      "step": 610
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0016962892631275021,
      "grad_norm": 0.08403483778238297,
      "kl": 0.03325536102056503,
      "learning_rate": 3e-06,
      "loss": 0.0233,
      "step": 611
    },
    {
      "clip_ratio": 0.0007063246885081753,
      "epoch": 0.001699065513967318,
      "grad_norm": 0.09051875025033951,
      "kl": 0.034811416640877724,
      "learning_rate": 3e-06,
      "loss": 0.0225,
      "step": 612
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "completion_length": 243.9791717529297,
      "epoch": 0.0017018417648071338,
      "grad_norm": 0.09752815216779709,
      "kl": 0.0324931014329195,
      "learning_rate": 3e-06,
      "loss": 0.0035,
      "reward": 0.20625001937150955,
      "reward_std": 0.2455231510102749,
      "rewards/countdown_reward_func": 0.20625000447034836,
      "step": 613,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.00018611777340993285,
      "epoch": 0.0017046180156469496,
      "grad_norm": 0.10532078891992569,
      "kl": 0.03395223990082741,
      "learning_rate": 3e-06,
      "loss": 0.0041,
      "step": 614
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0017073942664867657,
      "grad_norm": 0.0800807774066925,
      "kl": 0.034072598442435265,
      "learning_rate": 3e-06,
      "loss": 0.0038,
      "step": 615
    },
    {
      "clip_ratio": 0.00025160193035844713,
      "epoch": 0.0017101705173265815,
      "grad_norm": 0.11874743551015854,
      "kl": 0.03573352470993996,
      "learning_rate": 3e-06,
      "loss": 0.0044,
      "step": 616
    },
    {
      "clip_ratio": 8.164597966242582e-05,
      "epoch": 0.0017129467681663974,
      "grad_norm": 0.08836469054222107,
      "kl": 0.06380523927509785,
      "learning_rate": 3e-06,
      "loss": 0.0045,
      "step": 617
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0017157230190062132,
      "grad_norm": 0.08665862679481506,
      "kl": 0.045055605471134186,
      "learning_rate": 3e-06,
      "loss": 0.0032,
      "step": 618
    },
    {
      "clip_ratio": 0.0003530426474753767,
      "epoch": 0.001718499269846029,
      "grad_norm": 0.12727470695972443,
      "kl": 0.040752191096544266,
      "learning_rate": 3e-06,
      "loss": 0.004,
      "step": 619
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0017212755206858451,
      "grad_norm": 0.11716008186340332,
      "kl": 0.042310649529099464,
      "learning_rate": 3e-06,
      "loss": 0.0028,
      "step": 620
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.001724051771525661,
      "grad_norm": 0.07381536811590195,
      "kl": 0.041947562247514725,
      "learning_rate": 3e-06,
      "loss": 0.0031,
      "step": 621
    },
    {
      "clip_ratio": 0.0001702217195997946,
      "epoch": 0.0017268280223654768,
      "grad_norm": 0.09706465154886246,
      "kl": 0.042636461555957794,
      "learning_rate": 3e-06,
      "loss": 0.0047,
      "step": 622
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0017296042732052926,
      "grad_norm": 0.08298773318529129,
      "kl": 0.07458402588963509,
      "learning_rate": 3e-06,
      "loss": 0.0036,
      "step": 623
    },
    {
      "clip_ratio": 0.00016276042151730508,
      "epoch": 0.0017323805240451085,
      "grad_norm": 0.08479683846235275,
      "kl": 0.05383196100592613,
      "learning_rate": 3e-06,
      "loss": 0.0025,
      "step": 624
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 227.9166717529297,
      "epoch": 0.0017351567748849243,
      "grad_norm": 0.09818287193775177,
      "kl": 0.049956170842051506,
      "learning_rate": 3e-06,
      "loss": 0.0324,
      "reward": 0.20625001937150955,
      "reward_std": 0.20693624019622803,
      "rewards/countdown_reward_func": 0.20625000447034836,
      "step": 625,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0003646973054856062,
      "epoch": 0.0017379330257247404,
      "grad_norm": 0.11653796583414078,
      "kl": 0.0570333506911993,
      "learning_rate": 3e-06,
      "loss": 0.0323,
      "step": 626
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0017407092765645562,
      "grad_norm": 0.10893271118402481,
      "kl": 0.05227653309702873,
      "learning_rate": 3e-06,
      "loss": 0.0315,
      "step": 627
    },
    {
      "clip_ratio": 0.00045520756975747645,
      "epoch": 0.001743485527404372,
      "grad_norm": 0.1160278171300888,
      "kl": 0.05303039960563183,
      "learning_rate": 3e-06,
      "loss": 0.0313,
      "step": 628
    },
    {
      "clip_ratio": 8.159269054885954e-05,
      "epoch": 0.0017462617782441879,
      "grad_norm": 0.10207342356443405,
      "kl": 0.05501401796936989,
      "learning_rate": 3e-06,
      "loss": 0.0314,
      "step": 629
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0017490380290840037,
      "grad_norm": 0.08500054478645325,
      "kl": 0.06589866057038307,
      "learning_rate": 3e-06,
      "loss": 0.0306,
      "step": 630
    },
    {
      "clip_ratio": 9.117432637140155e-05,
      "epoch": 0.0017518142799238198,
      "grad_norm": 0.09498704224824905,
      "kl": 0.061905499547719955,
      "learning_rate": 3e-06,
      "loss": 0.0309,
      "step": 631
    },
    {
      "clip_ratio": 0.00027352297911420465,
      "epoch": 0.0017545905307636356,
      "grad_norm": 0.10427255183458328,
      "kl": 0.07243792712688446,
      "learning_rate": 3e-06,
      "loss": 0.0298,
      "step": 632
    },
    {
      "clip_ratio": 0.00017255453713005409,
      "epoch": 0.0017573667816034515,
      "grad_norm": 0.11053306609392166,
      "kl": 0.07044854387640953,
      "learning_rate": 3e-06,
      "loss": 0.0295,
      "step": 633
    },
    {
      "clip_ratio": 0.00010434056457597762,
      "epoch": 0.0017601430324432673,
      "grad_norm": 0.11823248863220215,
      "kl": 0.07041215151548386,
      "learning_rate": 3e-06,
      "loss": 0.0289,
      "step": 634
    },
    {
      "clip_ratio": 0.0005901292752241716,
      "epoch": 0.0017629192832830831,
      "grad_norm": 0.09512235224246979,
      "kl": 0.07614094018936157,
      "learning_rate": 3e-06,
      "loss": 0.0288,
      "step": 635
    },
    {
      "clip_ratio": 0.00029720802558586,
      "epoch": 0.001765695534122899,
      "grad_norm": 0.07722548395395279,
      "kl": 0.09275055304169655,
      "learning_rate": 3e-06,
      "loss": 0.0284,
      "step": 636
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 237.89583587646484,
      "epoch": 0.001768471784962715,
      "grad_norm": 0.0991094708442688,
      "kl": 0.10344970971345901,
      "learning_rate": 3e-06,
      "loss": 0.0205,
      "reward": 0.19166666269302368,
      "reward_std": 0.1672205552458763,
      "rewards/countdown_reward_func": 0.19166666269302368,
      "step": 637,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio": 8.896797226043418e-05,
      "epoch": 0.0017712480358025309,
      "grad_norm": 0.10992806404829025,
      "kl": 0.10585097968578339,
      "learning_rate": 3e-06,
      "loss": 0.0203,
      "step": 638
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0017740242866423467,
      "grad_norm": 0.07519710063934326,
      "kl": 0.1175805889070034,
      "learning_rate": 3e-06,
      "loss": 0.0194,
      "step": 639
    },
    {
      "clip_ratio": 0.00017600550199858844,
      "epoch": 0.0017768005374821625,
      "grad_norm": 0.15158335864543915,
      "kl": 0.1538476049900055,
      "learning_rate": 3e-06,
      "loss": 0.0215,
      "step": 640
    },
    {
      "clip_ratio": 9.462528396397829e-05,
      "epoch": 0.0017795767883219784,
      "grad_norm": 0.07322803884744644,
      "kl": 0.14574335515499115,
      "learning_rate": 3e-06,
      "loss": 0.021,
      "step": 641
    },
    {
      "clip_ratio": 0.0005492736818268895,
      "epoch": 0.0017823530391617944,
      "grad_norm": 0.12571430206298828,
      "kl": 0.15892429277300835,
      "learning_rate": 3e-06,
      "loss": 0.0215,
      "step": 642
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0017851292900016103,
      "grad_norm": 0.0937129482626915,
      "kl": 0.14000995457172394,
      "learning_rate": 3e-06,
      "loss": 0.0194,
      "step": 643
    },
    {
      "clip_ratio": 0.00017034818301908672,
      "epoch": 0.0017879055408414261,
      "grad_norm": 0.10581913590431213,
      "kl": 0.13933642953634262,
      "learning_rate": 3e-06,
      "loss": 0.0194,
      "step": 644
    },
    {
      "clip_ratio": 0.00018925056792795658,
      "epoch": 0.001790681791681242,
      "grad_norm": 0.09151846915483475,
      "kl": 0.14892030507326126,
      "learning_rate": 3e-06,
      "loss": 0.0194,
      "step": 645
    },
    {
      "clip_ratio": 0.0002573857200331986,
      "epoch": 0.0017934580425210578,
      "grad_norm": 0.16274204850196838,
      "kl": 0.1857452318072319,
      "learning_rate": 3e-06,
      "loss": 0.0201,
      "step": 646
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0017962342933608736,
      "grad_norm": 0.07416386157274246,
      "kl": 0.17207730561494827,
      "learning_rate": 3e-06,
      "loss": 0.0196,
      "step": 647
    },
    {
      "clip_ratio": 0.0008967332323663868,
      "epoch": 0.0017990105442006897,
      "grad_norm": 0.11084357649087906,
      "kl": 0.17409475147724152,
      "learning_rate": 3e-06,
      "loss": 0.02,
      "step": 648
    },
    {
      "clip_ratio": 9.999999747378752e-05,
      "completion_length": 226.64584350585938,
      "epoch": 0.0018017867950405055,
      "grad_norm": 0.15701425075531006,
      "kl": 0.16254248470067978,
      "learning_rate": 3e-06,
      "loss": 0.0062,
      "reward": 0.2770833596587181,
      "reward_std": 0.3268149420619011,
      "rewards/countdown_reward_func": 0.2770833596587181,
      "step": 649,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 9.999999747378752e-05,
      "epoch": 0.0018045630458803214,
      "grad_norm": 0.14769934117794037,
      "kl": 0.15972411632537842,
      "learning_rate": 3e-06,
      "loss": 0.0047,
      "step": 650
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0018073392967201372,
      "grad_norm": 0.15065068006515503,
      "kl": 0.15974202752113342,
      "learning_rate": 3e-06,
      "loss": 0.0055,
      "step": 651
    },
    {
      "clip_ratio": 0.00018544086924521253,
      "epoch": 0.001810115547559953,
      "grad_norm": 0.25048333406448364,
      "kl": 0.1500309482216835,
      "learning_rate": 3e-06,
      "loss": 0.0031,
      "step": 652
    },
    {
      "clip_ratio": 8.638562576379627e-05,
      "epoch": 0.0018128917983997691,
      "grad_norm": 0.16509313881397247,
      "kl": 0.14627444744110107,
      "learning_rate": 3e-06,
      "loss": 0.0038,
      "step": 653
    },
    {
      "clip_ratio": 0.00018638561596162617,
      "epoch": 0.001815668049239585,
      "grad_norm": 0.2661731243133545,
      "kl": 0.1369135081768036,
      "learning_rate": 3e-06,
      "loss": 0.0038,
      "step": 654
    },
    {
      "clip_ratio": 9.999999747378752e-05,
      "epoch": 0.0018184443000794008,
      "grad_norm": 0.14947877824306488,
      "kl": 0.13618455082178116,
      "learning_rate": 3e-06,
      "loss": 0.0042,
      "step": 655
    },
    {
      "clip_ratio": 0.00019307520415168256,
      "epoch": 0.0018212205509192166,
      "grad_norm": 0.16721273958683014,
      "kl": 0.12803955748677254,
      "learning_rate": 3e-06,
      "loss": 0.0031,
      "step": 656
    },
    {
      "clip_ratio": 0.00010548523277975619,
      "epoch": 0.0018239968017590325,
      "grad_norm": 0.13844816386699677,
      "kl": 0.12379540503025055,
      "learning_rate": 3e-06,
      "loss": 0.0029,
      "step": 657
    },
    {
      "clip_ratio": 0.0003572673595044762,
      "epoch": 0.0018267730525988483,
      "grad_norm": 0.24204890429973602,
      "kl": 0.10973387956619263,
      "learning_rate": 3e-06,
      "loss": -0.003,
      "step": 658
    },
    {
      "clip_ratio": 0.0006522906478494406,
      "epoch": 0.0018295493034386644,
      "grad_norm": 0.1522047072649002,
      "kl": 0.10548713058233261,
      "learning_rate": 3e-06,
      "loss": -0.0003,
      "step": 659
    },
    {
      "clip_ratio": 0.001921730930916965,
      "epoch": 0.0018323255542784802,
      "grad_norm": 0.25724494457244873,
      "kl": 0.09767122194170952,
      "learning_rate": 3e-06,
      "loss": -0.0013,
      "step": 660
    },
    {
      "clip_ratio": 9.051412052940577e-05,
      "completion_length": 218.7916717529297,
      "epoch": 0.001835101805118296,
      "grad_norm": 0.11165298521518707,
      "kl": 0.0836155079305172,
      "learning_rate": 3e-06,
      "loss": 0.007,
      "reward": 0.2604166716337204,
      "reward_std": 0.26486171036958694,
      "rewards/countdown_reward_func": 0.2604166716337204,
      "step": 661,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0018378780559581119,
      "grad_norm": 0.10689469426870346,
      "kl": 0.09136947244405746,
      "learning_rate": 3e-06,
      "loss": 0.0088,
      "step": 662
    },
    {
      "clip_ratio": 0.00019087179680354893,
      "epoch": 0.0018406543067979277,
      "grad_norm": 0.12443461269140244,
      "kl": 0.07553870230913162,
      "learning_rate": 3e-06,
      "loss": 0.0075,
      "step": 663
    },
    {
      "clip_ratio": 8.383634849451482e-05,
      "epoch": 0.0018434305576377438,
      "grad_norm": 0.1088162511587143,
      "kl": 0.07327612116932869,
      "learning_rate": 3e-06,
      "loss": 0.0079,
      "step": 664
    },
    {
      "clip_ratio": 0.0001824945939006284,
      "epoch": 0.0018462068084775596,
      "grad_norm": 0.09485284239053726,
      "kl": 0.07073798030614853,
      "learning_rate": 3e-06,
      "loss": 0.0066,
      "step": 665
    },
    {
      "clip_ratio": 0.00030053697992116213,
      "epoch": 0.0018489830593173755,
      "grad_norm": 0.13278953731060028,
      "kl": 0.06900656968355179,
      "learning_rate": 3e-06,
      "loss": 0.0067,
      "step": 666
    },
    {
      "clip_ratio": 9.865824540611356e-05,
      "epoch": 0.0018517593101571913,
      "grad_norm": 0.10497219115495682,
      "kl": 0.06802161037921906,
      "learning_rate": 3e-06,
      "loss": 0.0067,
      "step": 667
    },
    {
      "clip_ratio": 0.00021725406986661255,
      "epoch": 0.0018545355609970071,
      "grad_norm": 0.11255109310150146,
      "kl": 0.07535571232438087,
      "learning_rate": 3e-06,
      "loss": 0.0089,
      "step": 668
    },
    {
      "clip_ratio": 0.0005888359155505896,
      "epoch": 0.0018573118118368232,
      "grad_norm": 0.10095056146383286,
      "kl": 0.06463643535971642,
      "learning_rate": 3e-06,
      "loss": 0.0064,
      "step": 669
    },
    {
      "clip_ratio": 0.0005477168742800131,
      "epoch": 0.001860088062676639,
      "grad_norm": 0.09998640418052673,
      "kl": 0.06509601883590221,
      "learning_rate": 3e-06,
      "loss": 0.007,
      "step": 670
    },
    {
      "clip_ratio": 0.0001824945939006284,
      "epoch": 0.0018628643135164549,
      "grad_norm": 0.1530102789402008,
      "kl": 0.06189018301665783,
      "learning_rate": 3e-06,
      "loss": 0.0052,
      "step": 671
    },
    {
      "clip_ratio": 0.0002959747507702559,
      "epoch": 0.0018656405643562707,
      "grad_norm": 0.13745903968811035,
      "kl": 0.06357544660568237,
      "learning_rate": 3e-06,
      "loss": 0.0064,
      "step": 672
    },
    {
      "clip_ratio": 8.486082515446469e-05,
      "completion_length": 220.12500762939453,
      "epoch": 0.0018684168151960865,
      "grad_norm": 0.17126166820526123,
      "kl": 0.060750387609004974,
      "learning_rate": 3e-06,
      "loss": -0.0094,
      "reward": 0.26250001788139343,
      "reward_std": 0.2778630629181862,
      "rewards/countdown_reward_func": 0.26250001043081284,
      "step": 673,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0018711930660359024,
      "grad_norm": 0.11247212439775467,
      "kl": 0.059839921072125435,
      "learning_rate": 3e-06,
      "loss": -0.0094,
      "step": 674
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0018739693168757184,
      "grad_norm": 0.2667171359062195,
      "kl": 0.06426364183425903,
      "learning_rate": 3e-06,
      "loss": -0.0092,
      "step": 675
    },
    {
      "clip_ratio": 9.36329597607255e-05,
      "epoch": 0.0018767455677155343,
      "grad_norm": 0.1640663743019104,
      "kl": 0.06390438973903656,
      "learning_rate": 3e-06,
      "loss": -0.0087,
      "step": 676
    },
    {
      "clip_ratio": 0.0001101321613532491,
      "epoch": 0.0018795218185553501,
      "grad_norm": 0.11860151588916779,
      "kl": 0.054045552387833595,
      "learning_rate": 3e-06,
      "loss": -0.0108,
      "step": 677
    },
    {
      "clip_ratio": 0.00019816032727248967,
      "epoch": 0.001882298069395166,
      "grad_norm": 0.1296575963497162,
      "kl": 0.059720540419220924,
      "learning_rate": 3e-06,
      "loss": -0.0096,
      "step": 678
    },
    {
      "clip_ratio": 0.00033944330061785877,
      "epoch": 0.0018850743202349818,
      "grad_norm": 0.1466340571641922,
      "kl": 0.05133458413183689,
      "learning_rate": 3e-06,
      "loss": -0.0108,
      "step": 679
    },
    {
      "clip_ratio": 8.486082515446469e-05,
      "epoch": 0.0018878505710747979,
      "grad_norm": 0.10148210823535919,
      "kl": 0.049093831330537796,
      "learning_rate": 3e-06,
      "loss": -0.0103,
      "step": 680
    },
    {
      "clip_ratio": 9.476876584812999e-05,
      "epoch": 0.0018906268219146137,
      "grad_norm": 0.20377697050571442,
      "kl": 0.05094917304813862,
      "learning_rate": 3e-06,
      "loss": -0.0118,
      "step": 681
    },
    {
      "clip_ratio": 0.0003458706341916695,
      "epoch": 0.0018934030727544295,
      "grad_norm": 0.15611247718334198,
      "kl": 0.050493909046053886,
      "learning_rate": 3e-06,
      "loss": -0.0106,
      "step": 682
    },
    {
      "clip_ratio": 0.0002511018610675819,
      "epoch": 0.0018961793235942454,
      "grad_norm": 0.1299794614315033,
      "kl": 0.04327445663511753,
      "learning_rate": 3e-06,
      "loss": -0.0112,
      "step": 683
    },
    {
      "clip_ratio": 0.0003479021688690409,
      "epoch": 0.0018989555744340612,
      "grad_norm": 0.12855900824069977,
      "kl": 0.04604472406208515,
      "learning_rate": 3e-06,
      "loss": -0.0124,
      "step": 684
    },
    {
      "clip_ratio": 0.00042530003702268004,
      "completion_length": 235.7916717529297,
      "epoch": 0.001901731825273877,
      "grad_norm": 0.13944292068481445,
      "kl": 0.04754863306879997,
      "learning_rate": 3e-06,
      "loss": 0.0191,
      "reward": 0.24791667610406876,
      "reward_std": 0.23748211562633514,
      "rewards/countdown_reward_func": 0.24791667610406876,
      "step": 685,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.001904508076113693,
      "grad_norm": 0.10677601397037506,
      "kl": 0.041842538863420486,
      "learning_rate": 3e-06,
      "loss": 0.0195,
      "step": 686
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.001907284326953509,
      "grad_norm": 0.08806996792554855,
      "kl": 0.04203086532652378,
      "learning_rate": 3e-06,
      "loss": 0.019,
      "step": 687
    },
    {
      "clip_ratio": 0.00018215480668004602,
      "epoch": 0.0019100605777933248,
      "grad_norm": 0.1156989261507988,
      "kl": 0.042531004175543785,
      "learning_rate": 3e-06,
      "loss": 0.0186,
      "step": 688
    },
    {
      "clip_ratio": 0.00018691782315727323,
      "epoch": 0.0019128368286331406,
      "grad_norm": 0.09394989162683487,
      "kl": 0.041263919323682785,
      "learning_rate": 3e-06,
      "loss": 0.0194,
      "step": 689
    },
    {
      "clip_ratio": 0.00026606619940139353,
      "epoch": 0.0019156130794729565,
      "grad_norm": 0.10816143453121185,
      "kl": 0.041256049647927284,
      "learning_rate": 3e-06,
      "loss": 0.0201,
      "step": 690
    },
    {
      "clip_ratio": 0.0005038198505644687,
      "epoch": 0.0019183893303127725,
      "grad_norm": 0.16718047857284546,
      "kl": 0.04300625994801521,
      "learning_rate": 3e-06,
      "loss": 0.0206,
      "step": 691
    },
    {
      "clip_ratio": 0.0001649922487558797,
      "epoch": 0.0019211655811525884,
      "grad_norm": 0.10943221300840378,
      "kl": 0.04007406160235405,
      "learning_rate": 3e-06,
      "loss": 0.0195,
      "step": 692
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0019239418319924042,
      "grad_norm": 0.08448076248168945,
      "kl": 0.040019482374191284,
      "learning_rate": 3e-06,
      "loss": 0.0188,
      "step": 693
    },
    {
      "clip_ratio": 0.00025433551491005346,
      "epoch": 0.00192671808283222,
      "grad_norm": 0.11321469396352768,
      "kl": 0.04307270236313343,
      "learning_rate": 3e-06,
      "loss": 0.0183,
      "step": 694
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0019294943336720359,
      "grad_norm": 0.09902040660381317,
      "kl": 0.042439911514520645,
      "learning_rate": 3e-06,
      "loss": 0.0188,
      "step": 695
    },
    {
      "clip_ratio": 0.0001846859959186986,
      "epoch": 0.0019322705845118517,
      "grad_norm": 0.11367753148078918,
      "kl": 0.04280451126396656,
      "learning_rate": 3e-06,
      "loss": 0.0197,
      "step": 696
    },
    {
      "clip_ratio": 0.00010229132749373093,
      "completion_length": 230.8541717529297,
      "epoch": 0.0019350468353516678,
      "grad_norm": 0.13595792651176453,
      "kl": 0.04034610837697983,
      "learning_rate": 3e-06,
      "loss": 0.0264,
      "reward": 0.26250001788139343,
      "reward_std": 0.2784983515739441,
      "rewards/countdown_reward_func": 0.26250000298023224,
      "step": 697,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0019378230861914836,
      "grad_norm": 0.1038818508386612,
      "kl": 0.040640873834490776,
      "learning_rate": 3e-06,
      "loss": 0.0261,
      "step": 698
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0019405993370312995,
      "grad_norm": 0.0944518968462944,
      "kl": 0.04026073403656483,
      "learning_rate": 3e-06,
      "loss": 0.0269,
      "step": 699
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0019433755878711153,
      "grad_norm": 0.10623365640640259,
      "kl": 0.03956114687025547,
      "learning_rate": 3e-06,
      "loss": 0.0262,
      "step": 700
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0019461518387109311,
      "grad_norm": 0.09136571735143661,
      "kl": 0.044885121285915375,
      "learning_rate": 3e-06,
      "loss": 0.0263,
      "step": 701
    },
    {
      "clip_ratio": 8.915834769140929e-05,
      "epoch": 0.0019489280895507472,
      "grad_norm": 0.12446179240942001,
      "kl": 0.04785134270787239,
      "learning_rate": 3e-06,
      "loss": 0.0254,
      "step": 702
    },
    {
      "clip_ratio": 0.0003792484931182116,
      "epoch": 0.001951704340390563,
      "grad_norm": 0.1446102112531662,
      "kl": 0.04690235108137131,
      "learning_rate": 3e-06,
      "loss": 0.0251,
      "step": 703
    },
    {
      "clip_ratio": 8.915834769140929e-05,
      "epoch": 0.0019544805912303786,
      "grad_norm": 0.11323653906583786,
      "kl": 0.048219822347164154,
      "learning_rate": 3e-06,
      "loss": 0.0248,
      "step": 704
    },
    {
      "clip_ratio": 0.00016276042151730508,
      "epoch": 0.001957256842070195,
      "grad_norm": 0.09300023317337036,
      "kl": 0.04829951003193855,
      "learning_rate": 3e-06,
      "loss": 0.0253,
      "step": 705
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0019600330929100108,
      "grad_norm": 0.09304996579885483,
      "kl": 0.04665645770728588,
      "learning_rate": 3e-06,
      "loss": 0.0243,
      "step": 706
    },
    {
      "clip_ratio": 0.0004711005021817982,
      "epoch": 0.0019628093437498266,
      "grad_norm": 0.10053187608718872,
      "kl": 0.056041302159428596,
      "learning_rate": 3e-06,
      "loss": 0.0249,
      "step": 707
    },
    {
      "clip_ratio": 8.915834769140929e-05,
      "epoch": 0.0019655855945896424,
      "grad_norm": 0.11689490079879761,
      "kl": 0.05928555130958557,
      "learning_rate": 3e-06,
      "loss": 0.023,
      "step": 708
    },
    {
      "clip_ratio": 0.0001020408162730746,
      "completion_length": 217.87500762939453,
      "epoch": 0.0019683618454294583,
      "grad_norm": 0.12107165902853012,
      "kl": 0.059857327491045,
      "learning_rate": 3e-06,
      "loss": 0.0018,
      "reward": 0.2875000312924385,
      "reward_std": 0.26614009588956833,
      "rewards/countdown_reward_func": 0.2875000312924385,
      "step": 709,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.00016276042151730508,
      "epoch": 0.001971138096269274,
      "grad_norm": 0.1230737566947937,
      "kl": 0.06688201427459717,
      "learning_rate": 3e-06,
      "loss": 0.0023,
      "step": 710
    },
    {
      "clip_ratio": 0.0003445422335062176,
      "epoch": 0.00197391434710909,
      "grad_norm": 0.09549138695001602,
      "kl": 0.06407090276479721,
      "learning_rate": 3e-06,
      "loss": 0.0025,
      "step": 711
    },
    {
      "clip_ratio": 0.00010593220213195309,
      "epoch": 0.001976690597948906,
      "grad_norm": 0.1953248232603073,
      "kl": 0.06954507902264595,
      "learning_rate": 3e-06,
      "loss": 0.0026,
      "step": 712
    },
    {
      "clip_ratio": 0.000314092067128513,
      "epoch": 0.0019794668487887216,
      "grad_norm": 0.09191843122243881,
      "kl": 0.0671706348657608,
      "learning_rate": 3e-06,
      "loss": 0.0025,
      "step": 713
    },
    {
      "clip_ratio": 9.191176650347188e-05,
      "epoch": 0.0019822430996285375,
      "grad_norm": 0.12914390861988068,
      "kl": 0.06849226728081703,
      "learning_rate": 3e-06,
      "loss": 0.0013,
      "step": 714
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0019850193504683533,
      "grad_norm": 0.1148599311709404,
      "kl": 0.06904346123337746,
      "learning_rate": 3e-06,
      "loss": 0.0014,
      "step": 715
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0019877956013081696,
      "grad_norm": 0.12667132914066315,
      "kl": 0.07613038271665573,
      "learning_rate": 3e-06,
      "loss": 0.002,
      "step": 716
    },
    {
      "clip_ratio": 0.00017349517293041572,
      "epoch": 0.0019905718521479854,
      "grad_norm": 0.09524544328451157,
      "kl": 0.07107832655310631,
      "learning_rate": 3e-06,
      "loss": 0.0013,
      "step": 717
    },
    {
      "clip_ratio": 0.00010593220213195309,
      "epoch": 0.0019933481029878013,
      "grad_norm": 0.13948820531368256,
      "kl": 0.07579323649406433,
      "learning_rate": 3e-06,
      "loss": 0.0023,
      "step": 718
    },
    {
      "clip_ratio": 0.00030192390113370493,
      "epoch": 0.001996124353827617,
      "grad_norm": 0.0964130386710167,
      "kl": 0.07115180417895317,
      "learning_rate": 3e-06,
      "loss": 0.0018,
      "step": 719
    },
    {
      "clip_ratio": 0.00019251657067798078,
      "epoch": 0.001998900604667433,
      "grad_norm": 0.13039907813072205,
      "kl": 0.07175230234861374,
      "learning_rate": 3e-06,
      "loss": 0.0002,
      "step": 720
    },
    {
      "clip_ratio": 0.00028670569736277685,
      "completion_length": 223.6875,
      "epoch": 0.002001676855507249,
      "grad_norm": 0.22225071489810944,
      "kl": 0.07658716291189194,
      "learning_rate": 3e-06,
      "loss": 0.0017,
      "reward": 0.15208333730697632,
      "reward_std": 0.14035604149103165,
      "rewards/countdown_reward_func": 0.15208333730697632,
      "step": 721,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0020044531063470646,
      "grad_norm": 0.07475583255290985,
      "kl": 0.06551392935216427,
      "learning_rate": 3e-06,
      "loss": 0.0021,
      "step": 722
    },
    {
      "clip_ratio": 9.742790280142799e-05,
      "epoch": 0.0020072293571868805,
      "grad_norm": 0.08553145080804825,
      "kl": 0.06234482862055302,
      "learning_rate": 3e-06,
      "loss": 0.0017,
      "step": 723
    },
    {
      "clip_ratio": 0.00019808593060588464,
      "epoch": 0.0020100056080266963,
      "grad_norm": 0.05904084071516991,
      "kl": 0.06513764709234238,
      "learning_rate": 3e-06,
      "loss": 0.0017,
      "step": 724
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.002012781858866512,
      "grad_norm": 0.07201807200908661,
      "kl": 0.06482937932014465,
      "learning_rate": 3e-06,
      "loss": 0.0019,
      "step": 725
    },
    {
      "clip_ratio": 9.984025382436812e-05,
      "epoch": 0.002015558109706328,
      "grad_norm": 0.07085428386926651,
      "kl": 0.06466732174158096,
      "learning_rate": 3e-06,
      "loss": 0.0018,
      "step": 726
    },
    {
      "clip_ratio": 0.00018783042469294742,
      "epoch": 0.0020183343605461443,
      "grad_norm": 0.24391846358776093,
      "kl": 0.06686633080244064,
      "learning_rate": 3e-06,
      "loss": -0.001,
      "step": 727
    },
    {
      "clip_ratio": 0.00016469038382638246,
      "epoch": 0.00202111061138596,
      "grad_norm": 0.08265461027622223,
      "kl": 0.056683897972106934,
      "learning_rate": 3e-06,
      "loss": 0.0008,
      "step": 728
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.002023886862225776,
      "grad_norm": 0.08346160501241684,
      "kl": 0.05314911529421806,
      "learning_rate": 3e-06,
      "loss": 0.0008,
      "step": 729
    },
    {
      "clip_ratio": 0.00019712094945134595,
      "epoch": 0.0020266631130655918,
      "grad_norm": 0.057865407317876816,
      "kl": 0.05542047321796417,
      "learning_rate": 3e-06,
      "loss": 0.0012,
      "step": 730
    },
    {
      "clip_ratio": 0.0002478523238096386,
      "epoch": 0.0020294393639054076,
      "grad_norm": 0.07208742946386337,
      "kl": 0.054563652724027634,
      "learning_rate": 3e-06,
      "loss": 0.0008,
      "step": 731
    },
    {
      "clip_ratio": 0.0003381839778739959,
      "epoch": 0.0020322156147452235,
      "grad_norm": 0.06954418867826462,
      "kl": 0.05391604080796242,
      "learning_rate": 3e-06,
      "loss": 0.0007,
      "step": 732
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 209.27083587646484,
      "epoch": 0.0020349918655850393,
      "grad_norm": 0.12923622131347656,
      "kl": 0.054487695917487144,
      "learning_rate": 3e-06,
      "loss": 0.0177,
      "reward": 0.24791669100522995,
      "reward_std": 0.3050043284893036,
      "rewards/countdown_reward_func": 0.24791667610406876,
      "step": 733,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 0.0001197317978949286,
      "epoch": 0.002037768116424855,
      "grad_norm": 0.10722116380929947,
      "kl": 0.05108374170958996,
      "learning_rate": 3e-06,
      "loss": 0.0178,
      "step": 734
    },
    {
      "clip_ratio": 0.0001197317978949286,
      "epoch": 0.002040544367264671,
      "grad_norm": 0.11042473465204239,
      "kl": 0.052645549178123474,
      "learning_rate": 3e-06,
      "loss": 0.0183,
      "step": 735
    },
    {
      "clip_ratio": 9.104151831706986e-05,
      "epoch": 0.002043320618104487,
      "grad_norm": 0.1352359801530838,
      "kl": 0.049059027805924416,
      "learning_rate": 3e-06,
      "loss": 0.0169,
      "step": 736
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0020460968689443026,
      "grad_norm": 0.11228242516517639,
      "kl": 0.056157197803258896,
      "learning_rate": 3e-06,
      "loss": 0.0182,
      "step": 737
    },
    {
      "clip_ratio": 0.00029813701985403895,
      "epoch": 0.002048873119784119,
      "grad_norm": 0.09875979274511337,
      "kl": 0.04752412252128124,
      "learning_rate": 3e-06,
      "loss": 0.0174,
      "step": 738
    },
    {
      "clip_ratio": 8.922198321670294e-05,
      "epoch": 0.0020516493706239348,
      "grad_norm": 0.12920497357845306,
      "kl": 0.04895847663283348,
      "learning_rate": 3e-06,
      "loss": 0.0167,
      "step": 739
    },
    {
      "clip_ratio": 0.0003326056757941842,
      "epoch": 0.0020544256214637506,
      "grad_norm": 0.10440492630004883,
      "kl": 0.04799576476216316,
      "learning_rate": 3e-06,
      "loss": 0.0173,
      "step": 740
    },
    {
      "clip_ratio": 0.00021150236716493964,
      "epoch": 0.0020572018723035664,
      "grad_norm": 0.10824400186538696,
      "kl": 0.050282422453165054,
      "learning_rate": 3e-06,
      "loss": 0.0179,
      "step": 741
    },
    {
      "clip_ratio": 0.0002083023136947304,
      "epoch": 0.0020599781231433823,
      "grad_norm": 0.17329491674900055,
      "kl": 0.04743002541363239,
      "learning_rate": 3e-06,
      "loss": 0.0154,
      "step": 742
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.002062754373983198,
      "grad_norm": 0.11108177900314331,
      "kl": 0.05471383221447468,
      "learning_rate": 3e-06,
      "loss": 0.017,
      "step": 743
    },
    {
      "clip_ratio": 0.00021150236716493964,
      "epoch": 0.002065530624823014,
      "grad_norm": 0.09756101667881012,
      "kl": 0.048802973702549934,
      "learning_rate": 3e-06,
      "loss": 0.0166,
      "step": 744
    },
    {
      "clip_ratio": 8.650519157527015e-05,
      "completion_length": 231.375,
      "epoch": 0.00206830687566283,
      "grad_norm": 0.11797621846199036,
      "kl": 0.05034934915602207,
      "learning_rate": 3e-06,
      "loss": 0.0265,
      "reward": 0.32500001043081284,
      "reward_std": 0.33128294348716736,
      "rewards/countdown_reward_func": 0.32500001043081284,
      "step": 745,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0020710831265026456,
      "grad_norm": 0.1360631287097931,
      "kl": 0.049324722960591316,
      "learning_rate": 3e-06,
      "loss": 0.0259,
      "step": 746
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0020738593773424615,
      "grad_norm": 0.11022765189409256,
      "kl": 0.04972606897354126,
      "learning_rate": 3e-06,
      "loss": 0.0263,
      "step": 747
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0020766356281822773,
      "grad_norm": 0.11825685203075409,
      "kl": 0.05200809799134731,
      "learning_rate": 3e-06,
      "loss": 0.0255,
      "step": 748
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0020794118790220936,
      "grad_norm": 0.15757912397384644,
      "kl": 0.05278201401233673,
      "learning_rate": 3e-06,
      "loss": 0.0252,
      "step": 749
    },
    {
      "clip_ratio": 9.335325012216344e-05,
      "epoch": 0.0020821881298619094,
      "grad_norm": 0.12597443163394928,
      "kl": 0.05840662308037281,
      "learning_rate": 3e-06,
      "loss": 0.0252,
      "step": 750
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0020849643807017253,
      "grad_norm": 0.12119497358798981,
      "kl": 0.061403946951031685,
      "learning_rate": 3e-06,
      "loss": 0.0255,
      "step": 751
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.002087740631541541,
      "grad_norm": 0.15548032522201538,
      "kl": 0.06312582828104496,
      "learning_rate": 3e-06,
      "loss": 0.024,
      "step": 752
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.002090516882381357,
      "grad_norm": 0.09560614824295044,
      "kl": 0.06370921805500984,
      "learning_rate": 3e-06,
      "loss": 0.0251,
      "step": 753
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.002093293133221173,
      "grad_norm": 0.11291124671697617,
      "kl": 0.06948983296751976,
      "learning_rate": 3e-06,
      "loss": 0.023,
      "step": 754
    },
    {
      "clip_ratio": 9.328358282800764e-05,
      "epoch": 0.0020960693840609886,
      "grad_norm": 0.14481361210346222,
      "kl": 0.07263422012329102,
      "learning_rate": 3e-06,
      "loss": 0.0218,
      "step": 755
    },
    {
      "clip_ratio": 9.13075273274444e-05,
      "epoch": 0.0020988456349008045,
      "grad_norm": 0.11197732388973236,
      "kl": 0.08130589872598648,
      "learning_rate": 3e-06,
      "loss": 0.0229,
      "step": 756
    },
    {
      "clip_ratio": 9.144111390924081e-05,
      "completion_length": 233.5416717529297,
      "epoch": 0.0021016218857406203,
      "grad_norm": 0.08032705634832382,
      "kl": 0.0910085029900074,
      "learning_rate": 3e-06,
      "loss": 0.0084,
      "reward": 0.20625000447034836,
      "reward_std": 0.245523139834404,
      "rewards/countdown_reward_func": 0.20625000447034836,
      "step": 757,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.00019638649246189743,
      "epoch": 0.002104398136580436,
      "grad_norm": 0.08426212519407272,
      "kl": 0.0998266153037548,
      "learning_rate": 3e-06,
      "loss": 0.0084,
      "step": 758
    },
    {
      "clip_ratio": 0.0001748835711623542,
      "epoch": 0.002107174387420252,
      "grad_norm": 0.09099453687667847,
      "kl": 0.09878429025411606,
      "learning_rate": 3e-06,
      "loss": 0.0081,
      "step": 759
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0021099506382600683,
      "grad_norm": 0.10690543055534363,
      "kl": 0.10915743559598923,
      "learning_rate": 3e-06,
      "loss": 0.0094,
      "step": 760
    },
    {
      "clip_ratio": 0.00026571148191578686,
      "epoch": 0.002112726889099884,
      "grad_norm": 0.08678600192070007,
      "kl": 0.11478978767991066,
      "learning_rate": 3e-06,
      "loss": 0.0084,
      "step": 761
    },
    {
      "clip_ratio": 0.0001645819575060159,
      "epoch": 0.0021155031399397,
      "grad_norm": 0.09271499514579773,
      "kl": 0.1079954281449318,
      "learning_rate": 3e-06,
      "loss": 0.0083,
      "step": 762
    },
    {
      "clip_ratio": 9.144111390924081e-05,
      "epoch": 0.0021182793907795158,
      "grad_norm": 0.08949034661054611,
      "kl": 0.11407085880637169,
      "learning_rate": 3e-06,
      "loss": 0.0076,
      "step": 763
    },
    {
      "clip_ratio": 0.00028337843832559884,
      "epoch": 0.0021210556416193316,
      "grad_norm": 0.09156984835863113,
      "kl": 0.12308426946401596,
      "learning_rate": 3e-06,
      "loss": 0.0082,
      "step": 764
    },
    {
      "clip_ratio": 0.0001748835711623542,
      "epoch": 0.0021238318924591475,
      "grad_norm": 0.08866655081510544,
      "kl": 0.11775670573115349,
      "learning_rate": 3e-06,
      "loss": 0.0079,
      "step": 765
    },
    {
      "clip_ratio": 9.819324623094872e-05,
      "epoch": 0.0021266081432989633,
      "grad_norm": 0.10422717034816742,
      "kl": 0.12401585280895233,
      "learning_rate": 3e-06,
      "loss": 0.0084,
      "step": 766
    },
    {
      "clip_ratio": 9.259259240934625e-05,
      "epoch": 0.002129384394138779,
      "grad_norm": 0.08140372484922409,
      "kl": 0.12763554602861404,
      "learning_rate": 3e-06,
      "loss": 0.008,
      "step": 767
    },
    {
      "clip_ratio": 0.000270773540250957,
      "epoch": 0.002132160644978595,
      "grad_norm": 0.09283492714166641,
      "kl": 0.11555681005120277,
      "learning_rate": 3e-06,
      "loss": 0.0076,
      "step": 768
    },
    {
      "clip_ratio": 0.00017576066602487117,
      "completion_length": 214.00000762939453,
      "epoch": 0.002134936895818411,
      "grad_norm": 0.1142767146229744,
      "kl": 0.10155479237437248,
      "learning_rate": 3e-06,
      "loss": 0.0015,
      "reward": 0.20625000447034836,
      "reward_std": 0.20823679491877556,
      "rewards/countdown_reward_func": 0.20625000447034836,
      "step": 769,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0021377131466582266,
      "grad_norm": 0.12313186377286911,
      "kl": 0.11426293849945068,
      "learning_rate": 3e-06,
      "loss": 0.001,
      "step": 770
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.002140489397498043,
      "grad_norm": 0.10338300466537476,
      "kl": 0.10406426340341568,
      "learning_rate": 3e-06,
      "loss": 0.0004,
      "step": 771
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0021432656483378588,
      "grad_norm": 0.12133070081472397,
      "kl": 0.09835099801421165,
      "learning_rate": 3e-06,
      "loss": 0.0006,
      "step": 772
    },
    {
      "clip_ratio": 0.00040839538269210607,
      "epoch": 0.0021460418991776746,
      "grad_norm": 0.13222798705101013,
      "kl": 0.10666431114077568,
      "learning_rate": 3e-06,
      "loss": 0.0006,
      "step": 773
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0021488181500174904,
      "grad_norm": 0.1232038140296936,
      "kl": 0.09488913416862488,
      "learning_rate": 3e-06,
      "loss": 0.0003,
      "step": 774
    },
    {
      "clip_ratio": 9.164222865365446e-05,
      "epoch": 0.0021515944008573063,
      "grad_norm": 0.10721003264188766,
      "kl": 0.08875302597880363,
      "learning_rate": 3e-06,
      "loss": -0.0009,
      "step": 775
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.002154370651697122,
      "grad_norm": 0.12701751291751862,
      "kl": 0.09637927263975143,
      "learning_rate": 3e-06,
      "loss": -0.0006,
      "step": 776
    },
    {
      "clip_ratio": 0.0003659786016214639,
      "epoch": 0.002157146902536938,
      "grad_norm": 0.10145313292741776,
      "kl": 0.08755913749337196,
      "learning_rate": 3e-06,
      "loss": -0.001,
      "step": 777
    },
    {
      "clip_ratio": 0.00018341893155593425,
      "epoch": 0.002159923153376754,
      "grad_norm": 0.11642103642225266,
      "kl": 0.08162051066756248,
      "learning_rate": 3e-06,
      "loss": -0.0017,
      "step": 778
    },
    {
      "clip_ratio": 0.0006324425921775401,
      "epoch": 0.0021626994042165696,
      "grad_norm": 0.1388908177614212,
      "kl": 0.08449938148260117,
      "learning_rate": 3e-06,
      "loss": -0.0006,
      "step": 779
    },
    {
      "clip_ratio": 0.00041412835707888007,
      "epoch": 0.0021654756550563855,
      "grad_norm": 0.12610669434070587,
      "kl": 0.07518958300352097,
      "learning_rate": 3e-06,
      "loss": -0.001,
      "step": 780
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 227.58333587646484,
      "epoch": 0.0021682519058962017,
      "grad_norm": 0.11886297166347504,
      "kl": 0.08037593588232994,
      "learning_rate": 3e-06,
      "loss": 0.0063,
      "reward": 0.26250000298023224,
      "reward_std": 0.27733949571847916,
      "rewards/countdown_reward_func": 0.26249999552965164,
      "step": 781,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.00010702054714784026,
      "epoch": 0.0021710281567360176,
      "grad_norm": 0.14289286732673645,
      "kl": 0.07106167078018188,
      "learning_rate": 3e-06,
      "loss": 0.0047,
      "step": 782
    },
    {
      "clip_ratio": 0.0001884007579064928,
      "epoch": 0.0021738044075758334,
      "grad_norm": 0.12329453229904175,
      "kl": 0.07392378151416779,
      "learning_rate": 3e-06,
      "loss": 0.0059,
      "step": 783
    },
    {
      "clip_ratio": 0.000244140625,
      "epoch": 0.0021765806584156493,
      "grad_norm": 0.11597783863544464,
      "kl": 0.06427717953920364,
      "learning_rate": 3e-06,
      "loss": 0.0049,
      "step": 784
    },
    {
      "clip_ratio": 8.37801635498181e-05,
      "epoch": 0.002179356909255465,
      "grad_norm": 0.12029743194580078,
      "kl": 0.06854793429374695,
      "learning_rate": 3e-06,
      "loss": 0.0063,
      "step": 785
    },
    {
      "clip_ratio": 0.0002638931109686382,
      "epoch": 0.002182133160095281,
      "grad_norm": 0.11125701665878296,
      "kl": 0.06303357519209385,
      "learning_rate": 3e-06,
      "loss": 0.0061,
      "step": 786
    },
    {
      "clip_ratio": 0.0005720614135498181,
      "epoch": 0.002184909410935097,
      "grad_norm": 0.11483973264694214,
      "kl": 0.06457911431789398,
      "learning_rate": 3e-06,
      "loss": 0.0044,
      "step": 787
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0021876856617749126,
      "grad_norm": 0.1489003449678421,
      "kl": 0.057757457718253136,
      "learning_rate": 3e-06,
      "loss": 0.0044,
      "step": 788
    },
    {
      "clip_ratio": 0.00045818173384759575,
      "epoch": 0.0021904619126147285,
      "grad_norm": 0.1282450258731842,
      "kl": 0.06039944291114807,
      "learning_rate": 3e-06,
      "loss": 0.0042,
      "step": 789
    },
    {
      "clip_ratio": 0.00048828125,
      "epoch": 0.0021932381634545443,
      "grad_norm": 0.12330767512321472,
      "kl": 0.055448392406105995,
      "learning_rate": 3e-06,
      "loss": 0.0036,
      "step": 790
    },
    {
      "clip_ratio": 0.0007235735538415611,
      "epoch": 0.00219601441429436,
      "grad_norm": 0.12403135746717453,
      "kl": 0.058060334995388985,
      "learning_rate": 3e-06,
      "loss": 0.0048,
      "step": 791
    },
    {
      "clip_ratio": 0.00037331361090764403,
      "epoch": 0.0021987906651341764,
      "grad_norm": 0.10532835870981216,
      "kl": 0.05376404523849487,
      "learning_rate": 3e-06,
      "loss": 0.0047,
      "step": 792
    },
    {
      "clip_ratio": 0.00010984182881657034,
      "completion_length": 222.31250762939453,
      "epoch": 0.0022015669159739923,
      "grad_norm": 0.04651214927434921,
      "kl": 0.05401436612010002,
      "learning_rate": 3e-06,
      "loss": 0.003,
      "reward": 0.15000001341104507,
      "reward_std": 0.11558077484369278,
      "rewards/countdown_reward_func": 0.15000000223517418,
      "step": 793,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio": 0.000742982214433141,
      "epoch": 0.002204343166813808,
      "grad_norm": 0.07730413973331451,
      "kl": 0.0580199658870697,
      "learning_rate": 3e-06,
      "loss": 0.0033,
      "step": 794
    },
    {
      "clip_ratio": 0.00010984182881657034,
      "epoch": 0.002207119417653624,
      "grad_norm": 0.051809556782245636,
      "kl": 0.057794030755758286,
      "learning_rate": 3e-06,
      "loss": 0.0035,
      "step": 795
    },
    {
      "clip_ratio": 0.0007241156417876482,
      "epoch": 0.0022098956684934398,
      "grad_norm": 0.0728415995836258,
      "kl": 0.05362066254019737,
      "learning_rate": 3e-06,
      "loss": 0.0025,
      "step": 796
    },
    {
      "clip_ratio": 0.0002787023695418611,
      "epoch": 0.0022126719193332556,
      "grad_norm": 0.06317047029733658,
      "kl": 0.05362400598824024,
      "learning_rate": 3e-06,
      "loss": 0.0032,
      "step": 797
    },
    {
      "clip_ratio": 0.00017235546692973003,
      "epoch": 0.0022154481701730715,
      "grad_norm": 0.06358745694160461,
      "kl": 0.05262857303023338,
      "learning_rate": 3e-06,
      "loss": 0.0024,
      "step": 798
    },
    {
      "clip_ratio": 9.097525617107749e-05,
      "epoch": 0.0022182244210128873,
      "grad_norm": 0.05143192410469055,
      "kl": 0.049039315432310104,
      "learning_rate": 3e-06,
      "loss": 0.0028,
      "step": 799
    },
    {
      "clip_ratio": 0.00010984182881657034,
      "epoch": 0.002221000671852703,
      "grad_norm": 0.0856727808713913,
      "kl": 0.052018070593476295,
      "learning_rate": 3e-06,
      "loss": 0.0027,
      "step": 800
    },
    {
      "clip_ratio": 0.00038732589746359736,
      "epoch": 0.002223776922692519,
      "grad_norm": 0.05116529390215874,
      "kl": 0.05213000252842903,
      "learning_rate": 3e-06,
      "loss": 0.003,
      "step": 801
    },
    {
      "clip_ratio": 0.0006221811127034016,
      "epoch": 0.002226553173532335,
      "grad_norm": 0.07559063285589218,
      "kl": 0.04888819716870785,
      "learning_rate": 3e-06,
      "loss": 0.0019,
      "step": 802
    },
    {
      "clip_ratio": 0.0006752755725756288,
      "epoch": 0.002229329424372151,
      "grad_norm": 0.06661748886108398,
      "kl": 0.04938088357448578,
      "learning_rate": 3e-06,
      "loss": 0.0025,
      "step": 803
    },
    {
      "clip_ratio": 0.0006201023061294109,
      "epoch": 0.002232105675211967,
      "grad_norm": 0.06247713044285774,
      "kl": 0.04884421452879906,
      "learning_rate": 3e-06,
      "loss": 0.0018,
      "step": 804
    },
    {
      "clip_ratio": 0.00010399334132671356,
      "completion_length": 228.52083587646484,
      "epoch": 0.0022348819260517828,
      "grad_norm": 0.10135756433010101,
      "kl": 0.043619923293590546,
      "learning_rate": 3e-06,
      "loss": 0.0141,
      "reward": 0.2875000014901161,
      "reward_std": 0.28183095902204514,
      "rewards/countdown_reward_func": 0.2875000014901161,
      "step": 805,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0003937007859349251,
      "epoch": 0.0022376581768915986,
      "grad_norm": 0.10308714210987091,
      "kl": 0.0447169654071331,
      "learning_rate": 3e-06,
      "loss": 0.014,
      "step": 806
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0022404344277314144,
      "grad_norm": 0.12397963553667068,
      "kl": 0.04130372405052185,
      "learning_rate": 3e-06,
      "loss": 0.0146,
      "step": 807
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0022432106785712303,
      "grad_norm": 0.0904117226600647,
      "kl": 0.04326653108000755,
      "learning_rate": 3e-06,
      "loss": 0.0137,
      "step": 808
    },
    {
      "clip_ratio": 8.747375977691263e-05,
      "epoch": 0.002245986929411046,
      "grad_norm": 0.11636728793382645,
      "kl": 0.04562844708561897,
      "learning_rate": 3e-06,
      "loss": 0.0135,
      "step": 809
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.002248763180250862,
      "grad_norm": 0.10902263969182968,
      "kl": 0.04578346759080887,
      "learning_rate": 3e-06,
      "loss": 0.0137,
      "step": 810
    },
    {
      "clip_ratio": 8.747375977691263e-05,
      "epoch": 0.002251539431090678,
      "grad_norm": 0.10831674933433533,
      "kl": 0.04260227829217911,
      "learning_rate": 3e-06,
      "loss": 0.0132,
      "step": 811
    },
    {
      "clip_ratio": 0.0003889085492119193,
      "epoch": 0.0022543156819304936,
      "grad_norm": 0.11185412853956223,
      "kl": 0.04456242546439171,
      "learning_rate": 3e-06,
      "loss": 0.0142,
      "step": 812
    },
    {
      "clip_ratio": 0.00047818864550208673,
      "epoch": 0.0022570919327703095,
      "grad_norm": 0.10440458357334137,
      "kl": 0.04222998023033142,
      "learning_rate": 3e-06,
      "loss": 0.0138,
      "step": 813
    },
    {
      "clip_ratio": 0.00018712585733737797,
      "epoch": 0.0022598681836101257,
      "grad_norm": 0.0939682200551033,
      "kl": 0.04461575858294964,
      "learning_rate": 3e-06,
      "loss": 0.0124,
      "step": 814
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0022626444344499416,
      "grad_norm": 0.09211880713701248,
      "kl": 0.047047100961208344,
      "learning_rate": 3e-06,
      "loss": 0.0121,
      "step": 815
    },
    {
      "clip_ratio": 0.00036837265361100435,
      "epoch": 0.0022654206852897574,
      "grad_norm": 0.12892583012580872,
      "kl": 0.04755326174199581,
      "learning_rate": 3e-06,
      "loss": 0.0134,
      "step": 816
    },
    {
      "clip_ratio": 0.00047789004747755826,
      "completion_length": 220.1666717529297,
      "epoch": 0.0022681969361295733,
      "grad_norm": 0.18188029527664185,
      "kl": 0.04710717685520649,
      "learning_rate": 3e-06,
      "loss": 0.0458,
      "reward": 0.34166669845581055,
      "reward_std": 0.27476726472377777,
      "rewards/countdown_reward_func": 0.34166666865348816,
      "step": 817,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.00017529455362819135,
      "epoch": 0.002270973186969389,
      "grad_norm": 0.1454269289970398,
      "kl": 0.05038350075483322,
      "learning_rate": 3e-06,
      "loss": 0.0453,
      "step": 818
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.002273749437809205,
      "grad_norm": 0.13133502006530762,
      "kl": 0.04751100763678551,
      "learning_rate": 3e-06,
      "loss": 0.045,
      "step": 819
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.002276525688649021,
      "grad_norm": 0.1250462830066681,
      "kl": 0.04578425548970699,
      "learning_rate": 3e-06,
      "loss": 0.0449,
      "step": 820
    },
    {
      "clip_ratio": 0.0001035625537042506,
      "epoch": 0.0022793019394888366,
      "grad_norm": 0.14770175516605377,
      "kl": 0.05231819488108158,
      "learning_rate": 3e-06,
      "loss": 0.0442,
      "step": 821
    },
    {
      "clip_ratio": 0.0004608447488863021,
      "epoch": 0.0022820781903286525,
      "grad_norm": 0.14028239250183105,
      "kl": 0.05149639584124088,
      "learning_rate": 3e-06,
      "loss": 0.0435,
      "step": 822
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0022848544411684683,
      "grad_norm": 0.17355775833129883,
      "kl": 0.056207796558737755,
      "learning_rate": 3e-06,
      "loss": 0.0419,
      "step": 823
    },
    {
      "clip_ratio": 0.0001035625537042506,
      "epoch": 0.002287630692008284,
      "grad_norm": 0.14540952444076538,
      "kl": 0.060877278447151184,
      "learning_rate": 3e-06,
      "loss": 0.042,
      "step": 824
    },
    {
      "clip_ratio": 0.00018522187019698322,
      "epoch": 0.0022904069428481004,
      "grad_norm": 0.1283130794763565,
      "kl": 0.060043616220355034,
      "learning_rate": 3e-06,
      "loss": 0.042,
      "step": 825
    },
    {
      "clip_ratio": 9.104151831706986e-05,
      "epoch": 0.0022931831936879163,
      "grad_norm": 0.13025355339050293,
      "kl": 0.06011705473065376,
      "learning_rate": 3e-06,
      "loss": 0.0415,
      "step": 826
    },
    {
      "clip_ratio": 0.0006889307842357084,
      "epoch": 0.002295959444527732,
      "grad_norm": 0.1317429095506668,
      "kl": 0.07226631790399551,
      "learning_rate": 3e-06,
      "loss": 0.0392,
      "step": 827
    },
    {
      "clip_ratio": 0.0011769172851927578,
      "epoch": 0.002298735695367548,
      "grad_norm": 0.12668536603450775,
      "kl": 0.07085844874382019,
      "learning_rate": 3e-06,
      "loss": 0.0392,
      "step": 828
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 217.25,
      "epoch": 0.0023015119462073638,
      "grad_norm": 0.0859948992729187,
      "kl": 0.07980459555983543,
      "learning_rate": 3e-06,
      "loss": 0.0234,
      "reward": 0.21250000596046448,
      "reward_std": 0.2080453746020794,
      "rewards/countdown_reward_func": 0.21250000596046448,
      "step": 829,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio": 9.593246068106964e-05,
      "epoch": 0.0023042881970471796,
      "grad_norm": 0.08892592787742615,
      "kl": 0.08402768895030022,
      "learning_rate": 3e-06,
      "loss": 0.0227,
      "step": 830
    },
    {
      "clip_ratio": 9.15080527192913e-05,
      "epoch": 0.0023070644478869955,
      "grad_norm": 0.07939447462558746,
      "kl": 0.08712485805153847,
      "learning_rate": 3e-06,
      "loss": 0.0224,
      "step": 831
    },
    {
      "clip_ratio": 9.15080527192913e-05,
      "epoch": 0.0023098406987268113,
      "grad_norm": 0.09546811133623123,
      "kl": 0.09581883251667023,
      "learning_rate": 3e-06,
      "loss": 0.0229,
      "step": 832
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.002312616949566627,
      "grad_norm": 0.1163158193230629,
      "kl": 0.10002047568559647,
      "learning_rate": 3e-06,
      "loss": 0.0228,
      "step": 833
    },
    {
      "clip_ratio": 0.00019186492136213928,
      "epoch": 0.002315393200406443,
      "grad_norm": 0.1022767499089241,
      "kl": 0.10018961131572723,
      "learning_rate": 3e-06,
      "loss": 0.0225,
      "step": 834
    },
    {
      "clip_ratio": 9.15080527192913e-05,
      "epoch": 0.002318169451246259,
      "grad_norm": 0.08190451562404633,
      "kl": 0.11375463381409645,
      "learning_rate": 3e-06,
      "loss": 0.0222,
      "step": 835
    },
    {
      "clip_ratio": 0.0001830161054385826,
      "epoch": 0.002320945702086075,
      "grad_norm": 0.1015833392739296,
      "kl": 0.11530355364084244,
      "learning_rate": 3e-06,
      "loss": 0.0219,
      "step": 836
    },
    {
      "clip_ratio": 0.00027506323385750875,
      "epoch": 0.002323721952925891,
      "grad_norm": 0.08156981319189072,
      "kl": 0.11573269963264465,
      "learning_rate": 3e-06,
      "loss": 0.0224,
      "step": 837
    },
    {
      "clip_ratio": 9.15080527192913e-05,
      "epoch": 0.0023264982037657068,
      "grad_norm": 0.08199756592512131,
      "kl": 0.12695779651403427,
      "learning_rate": 3e-06,
      "loss": 0.0217,
      "step": 838
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0023292744546055226,
      "grad_norm": 0.1132536232471466,
      "kl": 0.12991363927721977,
      "learning_rate": 3e-06,
      "loss": 0.0228,
      "step": 839
    },
    {
      "clip_ratio": 0.00036751566221937537,
      "epoch": 0.0023320507054453384,
      "grad_norm": 0.10609925538301468,
      "kl": 0.1261560171842575,
      "learning_rate": 3e-06,
      "loss": 0.0212,
      "step": 840
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 216.02083587646484,
      "epoch": 0.0023348269562851543,
      "grad_norm": 0.13725493848323822,
      "kl": 0.12262994423508644,
      "learning_rate": 3e-06,
      "loss": 0.0071,
      "reward": 0.24375002086162567,
      "reward_std": 0.26503098011016846,
      "rewards/countdown_reward_func": 0.24375002086162567,
      "step": 841,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00233760320712497,
      "grad_norm": 0.13868065178394318,
      "kl": 0.13203585147857666,
      "learning_rate": 3e-06,
      "loss": 0.007,
      "step": 842
    },
    {
      "clip_ratio": 9.03179170563817e-05,
      "epoch": 0.002340379457964786,
      "grad_norm": 0.13435383141040802,
      "kl": 0.13662400841712952,
      "learning_rate": 3e-06,
      "loss": 0.007,
      "step": 843
    },
    {
      "clip_ratio": 9.038322605192661e-05,
      "epoch": 0.002343155708804602,
      "grad_norm": 0.158269003033638,
      "kl": 0.12493979930877686,
      "learning_rate": 3e-06,
      "loss": 0.0079,
      "step": 844
    },
    {
      "clip_ratio": 8.662508480483666e-05,
      "epoch": 0.0023459319596444176,
      "grad_norm": 0.13201403617858887,
      "kl": 0.14266318082809448,
      "learning_rate": 3e-06,
      "loss": 0.0071,
      "step": 845
    },
    {
      "clip_ratio": 9.03179170563817e-05,
      "epoch": 0.0023487082104842335,
      "grad_norm": 0.11519859731197357,
      "kl": 0.13292869180440903,
      "learning_rate": 3e-06,
      "loss": 0.0078,
      "step": 846
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0023514844613240497,
      "grad_norm": 0.1352914720773697,
      "kl": 0.12666785717010498,
      "learning_rate": 3e-06,
      "loss": 0.0055,
      "step": 847
    },
    {
      "clip_ratio": 9.07111752894707e-05,
      "epoch": 0.0023542607121638656,
      "grad_norm": 0.13024933636188507,
      "kl": 0.12938842177391052,
      "learning_rate": 3e-06,
      "loss": 0.0058,
      "step": 848
    },
    {
      "clip_ratio": 0.0003490790259093046,
      "epoch": 0.0023570369630036814,
      "grad_norm": 0.13579486310482025,
      "kl": 0.13479531556367874,
      "learning_rate": 3e-06,
      "loss": 0.0064,
      "step": 849
    },
    {
      "clip_ratio": 0.0009073439359781332,
      "epoch": 0.0023598132138434973,
      "grad_norm": 0.15496942400932312,
      "kl": 0.1218239888548851,
      "learning_rate": 3e-06,
      "loss": 0.0063,
      "step": 850
    },
    {
      "clip_ratio": 0.0004516682820394635,
      "epoch": 0.002362589464683313,
      "grad_norm": 0.13003304600715637,
      "kl": 0.13300446420907974,
      "learning_rate": 3e-06,
      "loss": 0.0044,
      "step": 851
    },
    {
      "clip_ratio": 0.00022212152543943375,
      "epoch": 0.002365365715523129,
      "grad_norm": 0.11777352541685104,
      "kl": 0.12342452257871628,
      "learning_rate": 3e-06,
      "loss": 0.0056,
      "step": 852
    },
    {
      "clip_ratio": 0.00018712575547397137,
      "completion_length": 218.52083587646484,
      "epoch": 0.0023681419663629448,
      "grad_norm": 0.12623701989650726,
      "kl": 0.12132728099822998,
      "learning_rate": 3e-06,
      "loss": -0.0039,
      "reward": 0.16875000298023224,
      "reward_std": 0.16144294291734695,
      "rewards/countdown_reward_func": 0.16874999552965164,
      "step": 853,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 9.238728671334684e-05,
      "epoch": 0.0023709182172027606,
      "grad_norm": 0.12842930853366852,
      "kl": 0.11158213764429092,
      "learning_rate": 3e-06,
      "loss": -0.0044,
      "step": 854
    },
    {
      "clip_ratio": 0.0005675574648194015,
      "epoch": 0.0023736944680425765,
      "grad_norm": 0.16828866302967072,
      "kl": 0.10891519114375114,
      "learning_rate": 3e-06,
      "loss": -0.0048,
      "step": 855
    },
    {
      "clip_ratio": 9.245562250725925e-05,
      "epoch": 0.0023764707188823923,
      "grad_norm": 0.15191693603992462,
      "kl": 0.11035206541419029,
      "learning_rate": 3e-06,
      "loss": -0.0052,
      "step": 856
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.002379246969722208,
      "grad_norm": 0.09676827490329742,
      "kl": 0.10043661668896675,
      "learning_rate": 3e-06,
      "loss": -0.0061,
      "step": 857
    },
    {
      "clip_ratio": 0.00019535439787432551,
      "epoch": 0.0023820232205620244,
      "grad_norm": 0.13413938879966736,
      "kl": 0.0950070358812809,
      "learning_rate": 3e-06,
      "loss": -0.0066,
      "step": 858
    },
    {
      "clip_ratio": 0.00039087486220523715,
      "epoch": 0.0023847994714018403,
      "grad_norm": 0.12939363718032837,
      "kl": 0.09167426824569702,
      "learning_rate": 3e-06,
      "loss": -0.0072,
      "step": 859
    },
    {
      "clip_ratio": 0.0017128197359852493,
      "epoch": 0.002387575722241656,
      "grad_norm": 0.11775387078523636,
      "kl": 0.08235512301325798,
      "learning_rate": 3e-06,
      "loss": -0.0086,
      "step": 860
    },
    {
      "clip_ratio": 0.0017219405272044241,
      "epoch": 0.002390351973081472,
      "grad_norm": 0.10675527900457382,
      "kl": 0.07996315136551857,
      "learning_rate": 3e-06,
      "loss": -0.0083,
      "step": 861
    },
    {
      "clip_ratio": 0.0017481384566053748,
      "epoch": 0.0023931282239212878,
      "grad_norm": 0.14627479016780853,
      "kl": 0.07891392335295677,
      "learning_rate": 3e-06,
      "loss": -0.0086,
      "step": 862
    },
    {
      "clip_ratio": 0.003889568499289453,
      "epoch": 0.0023959044747611036,
      "grad_norm": 0.09009343385696411,
      "kl": 0.07284262031316757,
      "learning_rate": 3e-06,
      "loss": -0.0076,
      "step": 863
    },
    {
      "clip_ratio": 0.006949112517759204,
      "epoch": 0.0023986807256009194,
      "grad_norm": 0.11177929490804672,
      "kl": 0.06860150396823883,
      "learning_rate": 3e-06,
      "loss": -0.0095,
      "step": 864
    },
    {
      "clip_ratio": 8.520791016053408e-05,
      "completion_length": 221.2916717529297,
      "epoch": 0.0024014569764407353,
      "grad_norm": 0.0992504358291626,
      "kl": 0.07187891751527786,
      "learning_rate": 3e-06,
      "loss": 0.024,
      "reward": 0.24791669100522995,
      "reward_std": 0.22883931919932365,
      "rewards/countdown_reward_func": 0.24791667610406876,
      "step": 865,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.002404233227280551,
      "grad_norm": 0.10073084384202957,
      "kl": 0.06338691711425781,
      "learning_rate": 3e-06,
      "loss": 0.0226,
      "step": 866
    },
    {
      "clip_ratio": 9.811617201194167e-05,
      "epoch": 0.002407009478120367,
      "grad_norm": 0.10182993113994598,
      "kl": 0.061187781393527985,
      "learning_rate": 3e-06,
      "loss": 0.0235,
      "step": 867
    },
    {
      "clip_ratio": 0.00030962208984419703,
      "epoch": 0.002409785728960183,
      "grad_norm": 0.1328115165233612,
      "kl": 0.06472436338663101,
      "learning_rate": 3e-06,
      "loss": 0.0248,
      "step": 868
    },
    {
      "clip_ratio": 9.811617201194167e-05,
      "epoch": 0.002412561979799999,
      "grad_norm": 0.12169165909290314,
      "kl": 0.06141612492501736,
      "learning_rate": 3e-06,
      "loss": 0.0244,
      "step": 869
    },
    {
      "clip_ratio": 0.00019770623475778848,
      "epoch": 0.002415338230639815,
      "grad_norm": 0.11152414232492447,
      "kl": 0.06216576509177685,
      "learning_rate": 3e-06,
      "loss": 0.0251,
      "step": 870
    },
    {
      "clip_ratio": 8.520791016053408e-05,
      "epoch": 0.0024181144814796308,
      "grad_norm": 0.08779774606227875,
      "kl": 0.06478393822908401,
      "learning_rate": 3e-06,
      "loss": 0.0248,
      "step": 871
    },
    {
      "clip_ratio": 0.00018243287922814488,
      "epoch": 0.0024208907323194466,
      "grad_norm": 0.10080928355455399,
      "kl": 0.05965164303779602,
      "learning_rate": 3e-06,
      "loss": 0.0239,
      "step": 872
    },
    {
      "clip_ratio": 0.0002776125547825359,
      "epoch": 0.0024236669831592624,
      "grad_norm": 0.09574668854475021,
      "kl": 0.05859038606286049,
      "learning_rate": 3e-06,
      "loss": 0.0238,
      "step": 873
    },
    {
      "clip_ratio": 0.00010575295891612768,
      "epoch": 0.0024264432339990783,
      "grad_norm": 0.1512366235256195,
      "kl": 0.06465988978743553,
      "learning_rate": 3e-06,
      "loss": 0.0241,
      "step": 874
    },
    {
      "clip_ratio": 0.0002814402541844174,
      "epoch": 0.002429219484838894,
      "grad_norm": 0.11698296666145325,
      "kl": 0.06264219433069229,
      "learning_rate": 3e-06,
      "loss": 0.0225,
      "step": 875
    },
    {
      "clip_ratio": 0.0004044586094096303,
      "epoch": 0.00243199573567871,
      "grad_norm": 0.1288166046142578,
      "kl": 0.06408961862325668,
      "learning_rate": 3e-06,
      "loss": 0.0247,
      "step": 876
    },
    {
      "clip_ratio": 9.765625145519152e-05,
      "completion_length": 228.87500762939453,
      "epoch": 0.002434771986518526,
      "grad_norm": 0.12193048000335693,
      "kl": 0.06197246536612511,
      "learning_rate": 3e-06,
      "loss": -0.0093,
      "reward": 0.3229166716337204,
      "reward_std": 0.3930432200431824,
      "rewards/countdown_reward_func": 0.3229166716337204,
      "step": 877,
      "zero_std_ratio": 0.0
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0024375482373583416,
      "grad_norm": 0.10932080447673798,
      "kl": 0.06703280657529831,
      "learning_rate": 3e-06,
      "loss": -0.0094,
      "step": 878
    },
    {
      "clip_ratio": 0.00034134124871343374,
      "epoch": 0.0024403244881981575,
      "grad_norm": 0.13418442010879517,
      "kl": 0.06639807298779488,
      "learning_rate": 3e-06,
      "loss": -0.0095,
      "step": 879
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0024431007390379737,
      "grad_norm": 0.15062019228935242,
      "kl": 0.06595133803784847,
      "learning_rate": 3e-06,
      "loss": -0.0089,
      "step": 880
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0024458769898777896,
      "grad_norm": 0.16575033962726593,
      "kl": 0.0656869113445282,
      "learning_rate": 3e-06,
      "loss": -0.0098,
      "step": 881
    },
    {
      "clip_ratio": 0.000244140625,
      "epoch": 0.0024486532407176054,
      "grad_norm": 0.1262388676404953,
      "kl": 0.06567086651921272,
      "learning_rate": 3e-06,
      "loss": -0.0092,
      "step": 882
    },
    {
      "clip_ratio": 0.00017017313075484708,
      "epoch": 0.0024514294915574213,
      "grad_norm": 0.11113395541906357,
      "kl": 0.06466436572372913,
      "learning_rate": 3e-06,
      "loss": -0.0093,
      "step": 883
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.002454205742397237,
      "grad_norm": 0.10662338137626648,
      "kl": 0.0688115581870079,
      "learning_rate": 3e-06,
      "loss": -0.0096,
      "step": 884
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.002456981993237053,
      "grad_norm": 0.1345973163843155,
      "kl": 0.06801817566156387,
      "learning_rate": 3e-06,
      "loss": -0.0096,
      "step": 885
    },
    {
      "clip_ratio": 9.720062371343374e-05,
      "epoch": 0.0024597582440768688,
      "grad_norm": 0.144433856010437,
      "kl": 0.06568440422415733,
      "learning_rate": 3e-06,
      "loss": -0.0107,
      "step": 886
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0024625344949166846,
      "grad_norm": 0.16164414584636688,
      "kl": 0.06407869979739189,
      "learning_rate": 3e-06,
      "loss": -0.0106,
      "step": 887
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0024653107457565005,
      "grad_norm": 0.11676425486803055,
      "kl": 0.06534116342663765,
      "learning_rate": 3e-06,
      "loss": -0.0106,
      "step": 888
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 220.02084350585938,
      "epoch": 0.0024680869965963163,
      "grad_norm": 0.1673632115125656,
      "kl": 0.061935342848300934,
      "learning_rate": 3e-06,
      "loss": -0.0087,
      "reward": 0.22291667014360428,
      "reward_std": 0.26279305666685104,
      "rewards/countdown_reward_func": 0.22291666269302368,
      "step": 889,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.00020538756507448852,
      "epoch": 0.002470863247436132,
      "grad_norm": 0.11077067255973816,
      "kl": 0.06396340392529964,
      "learning_rate": 3e-06,
      "loss": -0.0089,
      "step": 890
    },
    {
      "clip_ratio": 0.00019290123600512743,
      "epoch": 0.0024736394982759484,
      "grad_norm": 0.15075094997882843,
      "kl": 0.06460290402173996,
      "learning_rate": 3e-06,
      "loss": -0.0083,
      "step": 891
    },
    {
      "clip_ratio": 9.272996976505965e-05,
      "epoch": 0.0024764157491157643,
      "grad_norm": 0.10630524158477783,
      "kl": 0.0635167695581913,
      "learning_rate": 3e-06,
      "loss": -0.0087,
      "step": 892
    },
    {
      "clip_ratio": 0.0002878382147173397,
      "epoch": 0.00247919199995558,
      "grad_norm": 0.11911377310752869,
      "kl": 0.06044987216591835,
      "learning_rate": 3e-06,
      "loss": -0.0086,
      "step": 893
    },
    {
      "clip_ratio": 0.00030706560937687755,
      "epoch": 0.002481968250795396,
      "grad_norm": 0.11326951533555984,
      "kl": 0.06032133474946022,
      "learning_rate": 3e-06,
      "loss": -0.0098,
      "step": 894
    },
    {
      "clip_ratio": 0.00010088780982187018,
      "epoch": 0.0024847445016352118,
      "grad_norm": 0.17850473523139954,
      "kl": 0.05723920278251171,
      "learning_rate": 3e-06,
      "loss": -0.0116,
      "step": 895
    },
    {
      "clip_ratio": 0.00030706560937687755,
      "epoch": 0.0024875207524750276,
      "grad_norm": 0.11720696091651917,
      "kl": 0.05883798375725746,
      "learning_rate": 3e-06,
      "loss": -0.0106,
      "step": 896
    },
    {
      "clip_ratio": 0.0002937890458269976,
      "epoch": 0.0024902970033148434,
      "grad_norm": 0.1409841924905777,
      "kl": 0.0597931444644928,
      "learning_rate": 3e-06,
      "loss": -0.0107,
      "step": 897
    },
    {
      "clip_ratio": 0.00028926064987899736,
      "epoch": 0.0024930732541546593,
      "grad_norm": 0.1104314997792244,
      "kl": 0.05723441019654274,
      "learning_rate": 3e-06,
      "loss": -0.0106,
      "step": 898
    },
    {
      "clip_ratio": 0.0003814154479186982,
      "epoch": 0.002495849504994475,
      "grad_norm": 0.1183939203619957,
      "kl": 0.05589612200856209,
      "learning_rate": 3e-06,
      "loss": -0.0112,
      "step": 899
    },
    {
      "clip_ratio": 0.0010607101139612496,
      "epoch": 0.002498625755834291,
      "grad_norm": 0.12353586405515671,
      "kl": 0.05687661096453667,
      "learning_rate": 3e-06,
      "loss": -0.0124,
      "step": 900
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 232.25,
      "epoch": 0.002501402006674107,
      "grad_norm": 0.09452740103006363,
      "kl": 0.06010809168219566,
      "learning_rate": 3e-06,
      "loss": 0.0025,
      "reward": 0.3020833358168602,
      "reward_std": 0.2922677993774414,
      "rewards/countdown_reward_func": 0.3020833358168602,
      "step": 901,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 9.834775846684352e-05,
      "epoch": 0.002504178257513923,
      "grad_norm": 0.11706841737031937,
      "kl": 0.06478729844093323,
      "learning_rate": 3e-06,
      "loss": 0.0024,
      "step": 902
    },
    {
      "clip_ratio": 8.680555765749887e-05,
      "epoch": 0.002506954508353739,
      "grad_norm": 0.09865949302911758,
      "kl": 0.062279969453811646,
      "learning_rate": 3e-06,
      "loss": 0.0019,
      "step": 903
    },
    {
      "clip_ratio": 0.00018655490566743538,
      "epoch": 0.0025097307591935548,
      "grad_norm": 0.13041819632053375,
      "kl": 0.05919046886265278,
      "learning_rate": 3e-06,
      "loss": 0.0029,
      "step": 904
    },
    {
      "clip_ratio": 9.834775846684352e-05,
      "epoch": 0.0025125070100333706,
      "grad_norm": 0.14748121798038483,
      "kl": 0.059419430792331696,
      "learning_rate": 3e-06,
      "loss": 0.0023,
      "step": 905
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0025152832608731864,
      "grad_norm": 0.09640306979417801,
      "kl": 0.05946239456534386,
      "learning_rate": 3e-06,
      "loss": 0.0024,
      "step": 906
    },
    {
      "clip_ratio": 0.00036380960227688774,
      "epoch": 0.0025180595117130023,
      "grad_norm": 0.08679016679525375,
      "kl": 0.05758332274854183,
      "learning_rate": 3e-06,
      "loss": 0.0021,
      "step": 907
    },
    {
      "clip_ratio": 0.0004546546551864594,
      "epoch": 0.002520835762552818,
      "grad_norm": 0.10903292149305344,
      "kl": 0.06176324933767319,
      "learning_rate": 3e-06,
      "loss": 0.0007,
      "step": 908
    },
    {
      "clip_ratio": 0.0001810974645195529,
      "epoch": 0.002523612013392634,
      "grad_norm": 0.10386759042739868,
      "kl": 0.06048583798110485,
      "learning_rate": 3e-06,
      "loss": 0.0009,
      "step": 909
    },
    {
      "clip_ratio": 0.0005492813070304692,
      "epoch": 0.00252638826423245,
      "grad_norm": 0.10254993289709091,
      "kl": 0.057429682463407516,
      "learning_rate": 3e-06,
      "loss": 0.0013,
      "step": 910
    },
    {
      "clip_ratio": 9.834775846684352e-05,
      "epoch": 0.0025291645150722656,
      "grad_norm": 0.14101967215538025,
      "kl": 0.05812258459627628,
      "learning_rate": 3e-06,
      "loss": 0.0015,
      "step": 911
    },
    {
      "clip_ratio": 0.0004883037181571126,
      "epoch": 0.0025319407659120815,
      "grad_norm": 0.08682620525360107,
      "kl": 0.059369875118136406,
      "learning_rate": 3e-06,
      "loss": 0.0012,
      "step": 912
    },
    {
      "clip_ratio": 0.0005338219925761223,
      "completion_length": 228.77083587646484,
      "epoch": 0.0025347170167518977,
      "grad_norm": 0.11657971143722534,
      "kl": 0.05467422492802143,
      "learning_rate": 3e-06,
      "loss": 0.0358,
      "reward": 0.23125001043081284,
      "reward_std": 0.2115694098174572,
      "rewards/countdown_reward_func": 0.23125001043081284,
      "step": 913,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0025374932675917136,
      "grad_norm": 0.12091764807701111,
      "kl": 0.05531150847673416,
      "learning_rate": 3e-06,
      "loss": 0.0353,
      "step": 914
    },
    {
      "clip_ratio": 0.0012001376890111715,
      "epoch": 0.0025402695184315294,
      "grad_norm": 0.13323377072811127,
      "kl": 0.05716773122549057,
      "learning_rate": 3e-06,
      "loss": 0.0353,
      "step": 915
    },
    {
      "clip_ratio": 0.0001666462339926511,
      "epoch": 0.0025430457692713453,
      "grad_norm": 0.08699194341897964,
      "kl": 0.058137066662311554,
      "learning_rate": 3e-06,
      "loss": 0.0353,
      "step": 916
    },
    {
      "clip_ratio": 0.0005200195519137196,
      "epoch": 0.002545822020111161,
      "grad_norm": 0.10792747139930725,
      "kl": 0.06085832789540291,
      "learning_rate": 3e-06,
      "loss": 0.0362,
      "step": 917
    },
    {
      "clip_ratio": 8.60289073898457e-05,
      "epoch": 0.002548598270950977,
      "grad_norm": 0.08918231725692749,
      "kl": 0.05700236186385155,
      "learning_rate": 3e-06,
      "loss": 0.0347,
      "step": 918
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0025513745217907928,
      "grad_norm": 0.11331060528755188,
      "kl": 0.05938470549881458,
      "learning_rate": 3e-06,
      "loss": 0.035,
      "step": 919
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0025541507726306086,
      "grad_norm": 0.12028547376394272,
      "kl": 0.060254018753767014,
      "learning_rate": 3e-06,
      "loss": 0.0339,
      "step": 920
    },
    {
      "clip_ratio": 0.0003332924679853022,
      "epoch": 0.0025569270234704245,
      "grad_norm": 0.12606388330459595,
      "kl": 0.06682027131319046,
      "learning_rate": 3e-06,
      "loss": 0.0338,
      "step": 921
    },
    {
      "clip_ratio": 8.526603050995618e-05,
      "epoch": 0.0025597032743102403,
      "grad_norm": 0.08522091805934906,
      "kl": 0.06515257433056831,
      "learning_rate": 3e-06,
      "loss": 0.0341,
      "step": 922
    },
    {
      "clip_ratio": 0.0005214190459810197,
      "epoch": 0.0025624795251500566,
      "grad_norm": 0.10933967679738998,
      "kl": 0.06955453753471375,
      "learning_rate": 3e-06,
      "loss": 0.0342,
      "step": 923
    },
    {
      "clip_ratio": 8.60289073898457e-05,
      "epoch": 0.0025652557759898724,
      "grad_norm": 0.08516346663236618,
      "kl": 0.06566111743450165,
      "learning_rate": 3e-06,
      "loss": 0.0332,
      "step": 924
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 238.70834350585938,
      "epoch": 0.0025680320268296883,
      "grad_norm": 0.051000598818063736,
      "kl": 0.06130129098892212,
      "learning_rate": 3e-06,
      "loss": 0.008,
      "reward": 0.19166667759418488,
      "reward_std": 0.1091257855296135,
      "rewards/countdown_reward_func": 0.19166666269302368,
      "step": 925,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio": 9.704969124868512e-05,
      "epoch": 0.002570808277669504,
      "grad_norm": 0.06743042171001434,
      "kl": 0.06677010655403137,
      "learning_rate": 3e-06,
      "loss": 0.0085,
      "step": 926
    },
    {
      "clip_ratio": 0.0002635217970237136,
      "epoch": 0.00257358452850932,
      "grad_norm": 0.05554405599832535,
      "kl": 0.06589578464627266,
      "learning_rate": 3e-06,
      "loss": 0.0079,
      "step": 927
    },
    {
      "clip_ratio": 8.520791016053408e-05,
      "epoch": 0.0025763607793491358,
      "grad_norm": 0.06437760591506958,
      "kl": 0.06517062336206436,
      "learning_rate": 3e-06,
      "loss": 0.0083,
      "step": 928
    },
    {
      "clip_ratio": 0.00024796833167783916,
      "epoch": 0.0025791370301889516,
      "grad_norm": 0.056508298963308334,
      "kl": 0.0656033419072628,
      "learning_rate": 3e-06,
      "loss": 0.0079,
      "step": 929
    },
    {
      "clip_ratio": 0.0002620808663778007,
      "epoch": 0.0025819132810287674,
      "grad_norm": 0.05479772761464119,
      "kl": 0.07185834646224976,
      "learning_rate": 3e-06,
      "loss": 0.0084,
      "step": 930
    },
    {
      "clip_ratio": 9.704969124868512e-05,
      "epoch": 0.0025846895318685833,
      "grad_norm": 0.05194047465920448,
      "kl": 0.06922575458884239,
      "learning_rate": 3e-06,
      "loss": 0.008,
      "step": 931
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.002587465782708399,
      "grad_norm": 0.06964042782783508,
      "kl": 0.07410523667931557,
      "learning_rate": 3e-06,
      "loss": 0.008,
      "step": 932
    },
    {
      "clip_ratio": 9.704969124868512e-05,
      "epoch": 0.002590242033548215,
      "grad_norm": 0.04855266585946083,
      "kl": 0.07128038257360458,
      "learning_rate": 3e-06,
      "loss": 0.0079,
      "step": 933
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0025930182843880312,
      "grad_norm": 0.07284444570541382,
      "kl": 0.0707533061504364,
      "learning_rate": 3e-06,
      "loss": 0.0085,
      "step": 934
    },
    {
      "clip_ratio": 0.00016276042151730508,
      "epoch": 0.002595794535227847,
      "grad_norm": 0.06308530271053314,
      "kl": 0.07013512030243874,
      "learning_rate": 3e-06,
      "loss": 0.0075,
      "step": 935
    },
    {
      "clip_ratio": 0.00040690103196538985,
      "epoch": 0.002598570786067663,
      "grad_norm": 0.05616437643766403,
      "kl": 0.0736420564353466,
      "learning_rate": 3e-06,
      "loss": 0.0074,
      "step": 936
    },
    {
      "clip_ratio": 0.00017452477186452597,
      "completion_length": 236.02084350585938,
      "epoch": 0.0026013470369074788,
      "grad_norm": 0.09799813479185104,
      "kl": 0.08096648007631302,
      "learning_rate": 3e-06,
      "loss": -0.0021,
      "reward": 0.24583333730697632,
      "reward_std": 0.214500330388546,
      "rewards/countdown_reward_func": 0.24583332985639572,
      "step": 937,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.00016611294995527714,
      "epoch": 0.0026041232877472946,
      "grad_norm": 0.19633734226226807,
      "kl": 0.08069871366024017,
      "learning_rate": 3e-06,
      "loss": -0.0012,
      "step": 938
    },
    {
      "clip_ratio": 0.00033192152477568015,
      "epoch": 0.0026068995385871104,
      "grad_norm": 0.09509781002998352,
      "kl": 0.07796993106603622,
      "learning_rate": 3e-06,
      "loss": -0.0019,
      "step": 939
    },
    {
      "clip_ratio": 9.865824540611356e-05,
      "epoch": 0.0026096757894269263,
      "grad_norm": 0.09252092242240906,
      "kl": 0.08357620239257812,
      "learning_rate": 3e-06,
      "loss": -0.002,
      "step": 940
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.002612452040266742,
      "grad_norm": 0.1239117830991745,
      "kl": 0.0816221609711647,
      "learning_rate": 3e-06,
      "loss": -0.002,
      "step": 941
    },
    {
      "clip_ratio": 0.00018341893155593425,
      "epoch": 0.002615228291106558,
      "grad_norm": 0.09028197079896927,
      "kl": 0.0803113654255867,
      "learning_rate": 3e-06,
      "loss": -0.0015,
      "step": 942
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.002618004541946374,
      "grad_norm": 0.09007646143436432,
      "kl": 0.07762451097369194,
      "learning_rate": 3e-06,
      "loss": -0.0025,
      "step": 943
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0026207807927861896,
      "grad_norm": 0.21453428268432617,
      "kl": 0.07529747486114502,
      "learning_rate": 3e-06,
      "loss": -0.004,
      "step": 944
    },
    {
      "clip_ratio": 0.0001691611105343327,
      "epoch": 0.002623557043626006,
      "grad_norm": 0.08801465481519699,
      "kl": 0.07271482422947884,
      "learning_rate": 3e-06,
      "loss": -0.002,
      "step": 945
    },
    {
      "clip_ratio": 0.00018643914518179372,
      "epoch": 0.0026263332944658217,
      "grad_norm": 0.09710396826267242,
      "kl": 0.0768899917602539,
      "learning_rate": 3e-06,
      "loss": -0.003,
      "step": 946
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0026291095453056376,
      "grad_norm": 0.1289989948272705,
      "kl": 0.07622705027461052,
      "learning_rate": 3e-06,
      "loss": -0.0033,
      "step": 947
    },
    {
      "clip_ratio": 0.0005273459973977879,
      "epoch": 0.0026318857961454534,
      "grad_norm": 0.08834261447191238,
      "kl": 0.07354174181818962,
      "learning_rate": 3e-06,
      "loss": -0.0024,
      "step": 948
    },
    {
      "clip_ratio": 8.890469325706363e-05,
      "completion_length": 230.70834350585938,
      "epoch": 0.0026346620469852693,
      "grad_norm": 0.07417230308055878,
      "kl": 0.06710755452513695,
      "learning_rate": 3e-06,
      "loss": -0.0056,
      "reward": 0.16875001788139343,
      "reward_std": 0.15433254092931747,
      "rewards/countdown_reward_func": 0.16875001788139343,
      "step": 949,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio": 0.00036710136919282377,
      "epoch": 0.002637438297825085,
      "grad_norm": 0.10319241136312485,
      "kl": 0.0669807717204094,
      "learning_rate": 3e-06,
      "loss": -0.0052,
      "step": 950
    },
    {
      "clip_ratio": 8.890469325706363e-05,
      "epoch": 0.002640214548664901,
      "grad_norm": 0.08117877691984177,
      "kl": 0.06458613276481628,
      "learning_rate": 3e-06,
      "loss": -0.0062,
      "step": 951
    },
    {
      "clip_ratio": 0.0001982553512789309,
      "epoch": 0.0026429907995047168,
      "grad_norm": 0.08867120742797852,
      "kl": 0.06354731135070324,
      "learning_rate": 3e-06,
      "loss": -0.0049,
      "step": 952
    },
    {
      "clip_ratio": 9.65996878221631e-05,
      "epoch": 0.0026457670503445326,
      "grad_norm": 0.07009001821279526,
      "kl": 0.06276779621839523,
      "learning_rate": 3e-06,
      "loss": -0.0057,
      "step": 953
    },
    {
      "clip_ratio": 0.0001789265952538699,
      "epoch": 0.0026485433011843485,
      "grad_norm": 0.08066268265247345,
      "kl": 0.061625886708498,
      "learning_rate": 3e-06,
      "loss": -0.0056,
      "step": 954
    },
    {
      "clip_ratio": 0.00026640027499524876,
      "epoch": 0.0026513195520241643,
      "grad_norm": 0.07269750535488129,
      "kl": 0.058654628694057465,
      "learning_rate": 3e-06,
      "loss": -0.0063,
      "step": 955
    },
    {
      "clip_ratio": 0.0012421588180586696,
      "epoch": 0.0026540958028639806,
      "grad_norm": 0.08015119284391403,
      "kl": 0.05745803192257881,
      "learning_rate": 3e-06,
      "loss": -0.0057,
      "step": 956
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0026568720537037964,
      "grad_norm": 0.07885020971298218,
      "kl": 0.056764453649520874,
      "learning_rate": 3e-06,
      "loss": -0.007,
      "step": 957
    },
    {
      "clip_ratio": 0.00027389177557779476,
      "epoch": 0.0026596483045436123,
      "grad_norm": 0.08463863283395767,
      "kl": 0.056341877207159996,
      "learning_rate": 3e-06,
      "loss": -0.0064,
      "step": 958
    },
    {
      "clip_ratio": 0.001056087960023433,
      "epoch": 0.002662424555383428,
      "grad_norm": 0.07922295480966568,
      "kl": 0.05454845167696476,
      "learning_rate": 3e-06,
      "loss": -0.007,
      "step": 959
    },
    {
      "clip_ratio": 0.0007927687838673592,
      "epoch": 0.002665200806223244,
      "grad_norm": 0.0849594920873642,
      "kl": 0.05426880158483982,
      "learning_rate": 3e-06,
      "loss": -0.006,
      "step": 960
    },
    {
      "clip_ratio": 0.00043250381713733077,
      "completion_length": 235.6666717529297,
      "epoch": 0.0026679770570630598,
      "grad_norm": 0.09037365019321442,
      "kl": 0.05271290987730026,
      "learning_rate": 3e-06,
      "loss": 0.0065,
      "reward": 0.229166679084301,
      "reward_std": 0.18291139230132103,
      "rewards/countdown_reward_func": 0.229166679084301,
      "step": 961,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0026707533079028756,
      "grad_norm": 0.10632222890853882,
      "kl": 0.05197305604815483,
      "learning_rate": 3e-06,
      "loss": 0.0064,
      "step": 962
    },
    {
      "clip_ratio": 0.0001750104856910184,
      "epoch": 0.0026735295587426914,
      "grad_norm": 0.06896176934242249,
      "kl": 0.04834933951497078,
      "learning_rate": 3e-06,
      "loss": 0.0068,
      "step": 963
    },
    {
      "clip_ratio": 0.0002540506684454158,
      "epoch": 0.0026763058095825073,
      "grad_norm": 0.07026878744363785,
      "kl": 0.051994968205690384,
      "learning_rate": 3e-06,
      "loss": 0.007,
      "step": 964
    },
    {
      "clip_ratio": 8.821453957352787e-05,
      "epoch": 0.002679082060422323,
      "grad_norm": 0.06770803779363632,
      "kl": 0.05004582740366459,
      "learning_rate": 3e-06,
      "loss": 0.0068,
      "step": 965
    },
    {
      "clip_ratio": 8.722958591533825e-05,
      "epoch": 0.002681858311262139,
      "grad_norm": 0.06422404199838638,
      "kl": 0.04954391345381737,
      "learning_rate": 3e-06,
      "loss": 0.0067,
      "step": 966
    },
    {
      "clip_ratio": 8.821453957352787e-05,
      "epoch": 0.0026846345621019552,
      "grad_norm": 0.09136969596147537,
      "kl": 0.04910329729318619,
      "learning_rate": 3e-06,
      "loss": 0.006,
      "step": 967
    },
    {
      "clip_ratio": 0.00027080931613454595,
      "epoch": 0.002687410812941771,
      "grad_norm": 0.10625612735748291,
      "kl": 0.047768834978342056,
      "learning_rate": 3e-06,
      "loss": 0.0058,
      "step": 968
    },
    {
      "clip_ratio": 0.0003528630913933739,
      "epoch": 0.002690187063781587,
      "grad_norm": 0.06184415891766548,
      "kl": 0.04515623487532139,
      "learning_rate": 3e-06,
      "loss": 0.0063,
      "step": 969
    },
    {
      "clip_ratio": 0.00016860979667399079,
      "epoch": 0.0026929633146214028,
      "grad_norm": 0.06915763020515442,
      "kl": 0.0480178352445364,
      "learning_rate": 3e-06,
      "loss": 0.006,
      "step": 970
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0026957395654612186,
      "grad_norm": 0.06491398066282272,
      "kl": 0.047293346375226974,
      "learning_rate": 3e-06,
      "loss": 0.0062,
      "step": 971
    },
    {
      "clip_ratio": 0.00016860979667399079,
      "epoch": 0.0026985158163010344,
      "grad_norm": 0.07768521457910538,
      "kl": 0.04707910679280758,
      "learning_rate": 3e-06,
      "loss": 0.0061,
      "step": 972
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 241.56250762939453,
      "epoch": 0.0027012920671408503,
      "grad_norm": 0.10156456381082535,
      "kl": 0.04707324132323265,
      "learning_rate": 3e-06,
      "loss": 0.0114,
      "reward": 0.2666666805744171,
      "reward_std": 0.2661244869232178,
      "rewards/countdown_reward_func": 0.2666666731238365,
      "step": 973,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.00025393527175765485,
      "epoch": 0.002704068317980666,
      "grad_norm": 0.08211185038089752,
      "kl": 0.044957129284739494,
      "learning_rate": 3e-06,
      "loss": 0.0122,
      "step": 974
    },
    {
      "clip_ratio": 9.391435014549643e-05,
      "epoch": 0.002706844568820482,
      "grad_norm": 0.080048106610775,
      "kl": 0.047641387209296227,
      "learning_rate": 3e-06,
      "loss": 0.0124,
      "step": 975
    },
    {
      "clip_ratio": 0.00024569814559072256,
      "epoch": 0.002709620819660298,
      "grad_norm": 0.1037638708949089,
      "kl": 0.04482552409172058,
      "learning_rate": 3e-06,
      "loss": 0.0119,
      "step": 976
    },
    {
      "clip_ratio": 0.00017228929937118664,
      "epoch": 0.0027123970705001136,
      "grad_norm": 0.08106601238250732,
      "kl": 0.04870855435729027,
      "learning_rate": 3e-06,
      "loss": 0.0119,
      "step": 977
    },
    {
      "clip_ratio": 0.0002454323766869493,
      "epoch": 0.00271517332133993,
      "grad_norm": 0.09446461498737335,
      "kl": 0.04764538258314133,
      "learning_rate": 3e-06,
      "loss": 0.0117,
      "step": 978
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0027179495721797457,
      "grad_norm": 0.0833747461438179,
      "kl": 0.04668613523244858,
      "learning_rate": 3e-06,
      "loss": 0.0109,
      "step": 979
    },
    {
      "clip_ratio": 9.09090886125341e-05,
      "epoch": 0.0027207258230195616,
      "grad_norm": 0.07825154811143875,
      "kl": 0.045210424810647964,
      "learning_rate": 3e-06,
      "loss": 0.0113,
      "step": 980
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0027235020738593774,
      "grad_norm": 0.0840124562382698,
      "kl": 0.04799755476415157,
      "learning_rate": 3e-06,
      "loss": 0.0107,
      "step": 981
    },
    {
      "clip_ratio": 0.0004264606104698032,
      "epoch": 0.0027262783246991933,
      "grad_norm": 0.11435186117887497,
      "kl": 0.04499867558479309,
      "learning_rate": 3e-06,
      "loss": 0.0109,
      "step": 982
    },
    {
      "clip_ratio": 0.00037225715641397983,
      "epoch": 0.002729054575539009,
      "grad_norm": 0.08535240590572357,
      "kl": 0.049062151461839676,
      "learning_rate": 3e-06,
      "loss": 0.011,
      "step": 983
    },
    {
      "clip_ratio": 0.00016276042151730508,
      "epoch": 0.002731830826378825,
      "grad_norm": 0.09356574714183807,
      "kl": 0.04856777377426624,
      "learning_rate": 3e-06,
      "loss": 0.0105,
      "step": 984
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 222.45833587646484,
      "epoch": 0.0027346070772186408,
      "grad_norm": 0.11254774034023285,
      "kl": 0.048314955085515976,
      "learning_rate": 3e-06,
      "loss": 0.0166,
      "reward": 0.22291667759418488,
      "reward_std": 0.26681750267744064,
      "rewards/countdown_reward_func": 0.22291667759418488,
      "step": 985,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0027373833280584566,
      "grad_norm": 0.128098264336586,
      "kl": 0.0460913497954607,
      "learning_rate": 3e-06,
      "loss": 0.0151,
      "step": 986
    },
    {
      "clip_ratio": 0.00010339123400626704,
      "epoch": 0.0027401595788982725,
      "grad_norm": 0.09424342215061188,
      "kl": 0.0468437634408474,
      "learning_rate": 3e-06,
      "loss": 0.0159,
      "step": 987
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0027429358297380883,
      "grad_norm": 0.11079585552215576,
      "kl": 0.04501481167972088,
      "learning_rate": 3e-06,
      "loss": 0.0165,
      "step": 988
    },
    {
      "clip_ratio": 0.0026023527534562163,
      "epoch": 0.0027457120805779046,
      "grad_norm": 0.08553663641214371,
      "kl": 0.04914248362183571,
      "learning_rate": 3e-06,
      "loss": 0.0171,
      "step": 989
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0027484883314177204,
      "grad_norm": 0.0737449899315834,
      "kl": 0.04859176091849804,
      "learning_rate": 3e-06,
      "loss": 0.016,
      "step": 990
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0027512645822575362,
      "grad_norm": 0.1108751967549324,
      "kl": 0.05102384462952614,
      "learning_rate": 3e-06,
      "loss": 0.0155,
      "step": 991
    },
    {
      "clip_ratio": 9.15080527192913e-05,
      "epoch": 0.002754040833097352,
      "grad_norm": 0.10651170462369919,
      "kl": 0.04962940514087677,
      "learning_rate": 3e-06,
      "loss": 0.0151,
      "step": 992
    },
    {
      "clip_ratio": 0.00029216046823421493,
      "epoch": 0.002756817083937168,
      "grad_norm": 0.08815775066614151,
      "kl": 0.050901319831609726,
      "learning_rate": 3e-06,
      "loss": 0.0147,
      "step": 993
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0027595933347769838,
      "grad_norm": 0.1047065481543541,
      "kl": 0.04865125194191933,
      "learning_rate": 3e-06,
      "loss": 0.0157,
      "step": 994
    },
    {
      "clip_ratio": 0.0027102040985482745,
      "epoch": 0.0027623695856167996,
      "grad_norm": 0.08856725692749023,
      "kl": 0.053303858265280724,
      "learning_rate": 3e-06,
      "loss": 0.0166,
      "step": 995
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0027651458364566154,
      "grad_norm": 0.07977961748838425,
      "kl": 0.0532270185649395,
      "learning_rate": 3e-06,
      "loss": 0.0156,
      "step": 996
    },
    {
      "clip_ratio": 0.00010254306835122406,
      "completion_length": 225.8541717529297,
      "epoch": 0.0027679220872964313,
      "grad_norm": 0.11937547475099564,
      "kl": 0.0554725993424654,
      "learning_rate": 3e-06,
      "loss": 0.0133,
      "reward": 0.30000001937150955,
      "reward_std": 0.27453966438770294,
      "rewards/countdown_reward_func": 0.30000001937150955,
      "step": 997,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.002770698338136247,
      "grad_norm": 0.45567768812179565,
      "kl": 0.05745168775320053,
      "learning_rate": 3e-06,
      "loss": 0.0133,
      "step": 998
    },
    {
      "clip_ratio": 0.0001995211496250704,
      "epoch": 0.002773474588976063,
      "grad_norm": 0.1074514389038086,
      "kl": 0.060051169246435165,
      "learning_rate": 3e-06,
      "loss": 0.0124,
      "step": 999
    },
    {
      "epoch": 0.0027762508398158792,
      "grad_norm": 0.12824182212352753,
      "learning_rate": 3e-06,
      "loss": 0.012,
      "step": 1000
    },
    {
      "clip_ratio": 0.00012577813322423026,
      "epoch": 0.002779027090655695,
      "grad_norm": 0.09932482987642288,
      "kl": 0.060510776937007904,
      "learning_rate": 3e-06,
      "loss": 0.0125,
      "step": 1001
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.002781803341495511,
      "grad_norm": 0.09447323530912399,
      "kl": 0.06021983176469803,
      "learning_rate": 3e-06,
      "loss": 0.0126,
      "step": 1002
    },
    {
      "clip_ratio": 0.0002693641581572592,
      "epoch": 0.0027845795923353268,
      "grad_norm": 0.11810337752103806,
      "kl": 0.06065908633172512,
      "learning_rate": 3e-06,
      "loss": 0.0114,
      "step": 1003
    },
    {
      "clip_ratio": 9.97605748125352e-05,
      "epoch": 0.0027873558431751426,
      "grad_norm": 0.15187525749206543,
      "kl": 0.06130600720643997,
      "learning_rate": 3e-06,
      "loss": 0.012,
      "step": 1004
    },
    {
      "clip_ratio": 0.00019688567408593372,
      "epoch": 0.0027901320940149584,
      "grad_norm": 0.09934880584478378,
      "kl": 0.06536610797047615,
      "learning_rate": 3e-06,
      "loss": 0.0109,
      "step": 1005
    },
    {
      "clip_ratio": 0.0002693641581572592,
      "epoch": 0.0027929083448547743,
      "grad_norm": 0.1290806382894516,
      "kl": 0.06462856568396091,
      "learning_rate": 3e-06,
      "loss": 0.0118,
      "step": 1006
    },
    {
      "clip_ratio": 0.00017373176524415612,
      "epoch": 0.00279568459569459,
      "grad_norm": 0.09871116280555725,
      "kl": 0.06735832616686821,
      "learning_rate": 3e-06,
      "loss": 0.0116,
      "step": 1007
    },
    {
      "clip_ratio": 9.97605748125352e-05,
      "epoch": 0.002798460846534406,
      "grad_norm": 0.09537402540445328,
      "kl": 0.06406117230653763,
      "learning_rate": 3e-06,
      "loss": 0.0114,
      "step": 1008
    },
    {
      "clip_ratio": 0.00011488970631035045,
      "completion_length": 231.37500762939453,
      "epoch": 0.002801237097374222,
      "grad_norm": 0.3545832335948944,
      "kl": 0.05917428806424141,
      "learning_rate": 3e-06,
      "loss": 0.0063,
      "reward": 0.208333358168602,
      "reward_std": 0.18449045717716217,
      "rewards/countdown_reward_func": 0.2083333507180214,
      "step": 1009,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0004897922772215679,
      "epoch": 0.0028040133482140376,
      "grad_norm": 0.0879272073507309,
      "kl": 0.06689955294132233,
      "learning_rate": 3e-06,
      "loss": 0.0063,
      "step": 1010
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.002806789599053854,
      "grad_norm": 0.06225398927927017,
      "kl": 0.06652659550309181,
      "learning_rate": 3e-06,
      "loss": 0.007,
      "step": 1011
    },
    {
      "clip_ratio": 0.0002790517173707485,
      "epoch": 0.0028095658498936697,
      "grad_norm": 0.0772099494934082,
      "kl": 0.07098489999771118,
      "learning_rate": 3e-06,
      "loss": 0.0065,
      "step": 1012
    },
    {
      "clip_ratio": 0.00016578249051235616,
      "epoch": 0.0028123421007334856,
      "grad_norm": 0.09716866910457611,
      "kl": 0.06839455664157867,
      "learning_rate": 3e-06,
      "loss": 0.007,
      "step": 1013
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0028151183515733014,
      "grad_norm": 0.08656153082847595,
      "kl": 0.06978728249669075,
      "learning_rate": 3e-06,
      "loss": 0.0069,
      "step": 1014
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0028178946024131173,
      "grad_norm": 0.07657860964536667,
      "kl": 0.0637618862092495,
      "learning_rate": 3e-06,
      "loss": 0.0061,
      "step": 1015
    },
    {
      "clip_ratio": 0.00025825316697591916,
      "epoch": 0.002820670853252933,
      "grad_norm": 0.09020351618528366,
      "kl": 0.0716327540576458,
      "learning_rate": 3e-06,
      "loss": 0.0058,
      "step": 1016
    },
    {
      "clip_ratio": 0.0004928143753204495,
      "epoch": 0.002823447104092749,
      "grad_norm": 0.07356920838356018,
      "kl": 0.06976194307208061,
      "learning_rate": 3e-06,
      "loss": 0.0068,
      "step": 1017
    },
    {
      "clip_ratio": 0.00035517166543286294,
      "epoch": 0.0028262233549325648,
      "grad_norm": 0.08907874673604965,
      "kl": 0.07387241721153259,
      "learning_rate": 3e-06,
      "loss": 0.0067,
      "step": 1018
    },
    {
      "clip_ratio": 0.00017687295621726662,
      "epoch": 0.0028289996057723806,
      "grad_norm": 0.08614590018987656,
      "kl": 0.07145676389336586,
      "learning_rate": 3e-06,
      "loss": 0.0071,
      "step": 1019
    },
    {
      "clip_ratio": 0.0001792114635463804,
      "epoch": 0.0028317758566121965,
      "grad_norm": 0.10110598802566528,
      "kl": 0.07314455509185791,
      "learning_rate": 3e-06,
      "loss": 0.0059,
      "step": 1020
    },
    {
      "clip_ratio": 0.00019379844889044762,
      "completion_length": 232.77084350585938,
      "epoch": 0.0028345521074520123,
      "grad_norm": 0.07683458924293518,
      "kl": 0.07031066715717316,
      "learning_rate": 3e-06,
      "loss": 0.0068,
      "reward": 0.2291666716337204,
      "reward_std": 0.21667250245809555,
      "rewards/countdown_reward_func": 0.2291666641831398,
      "step": 1021,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0002766043471638113,
      "epoch": 0.0028373283582918286,
      "grad_norm": 0.08281517773866653,
      "kl": 0.07224087789654732,
      "learning_rate": 3e-06,
      "loss": 0.0075,
      "step": 1022
    },
    {
      "clip_ratio": 0.0003614198212744668,
      "epoch": 0.0028401046091316444,
      "grad_norm": 0.13180024921894073,
      "kl": 0.07020625099539757,
      "learning_rate": 3e-06,
      "loss": 0.0074,
      "step": 1023
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0028428808599714602,
      "grad_norm": 0.09534606337547302,
      "kl": 0.07347037643194199,
      "learning_rate": 3e-06,
      "loss": 0.0071,
      "step": 1024
    },
    {
      "clip_ratio": 0.0003760801919270307,
      "epoch": 0.002845657110811276,
      "grad_norm": 0.11282859742641449,
      "kl": 0.07506215572357178,
      "learning_rate": 3e-06,
      "loss": 0.0069,
      "step": 1025
    },
    {
      "clip_ratio": 9.124087227974087e-05,
      "epoch": 0.002848433361651092,
      "grad_norm": 0.08544313162565231,
      "kl": 0.07066556811332703,
      "learning_rate": 3e-06,
      "loss": 0.0062,
      "step": 1026
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0028512096124909078,
      "grad_norm": 0.07569853216409683,
      "kl": 0.07081609964370728,
      "learning_rate": 3e-06,
      "loss": 0.0069,
      "step": 1027
    },
    {
      "clip_ratio": 0.00036229201941750944,
      "epoch": 0.0028539858633307236,
      "grad_norm": 0.09049994498491287,
      "kl": 0.07069279998540878,
      "learning_rate": 3e-06,
      "loss": 0.0068,
      "step": 1028
    },
    {
      "clip_ratio": 0.0009208531701005995,
      "epoch": 0.0028567621141705394,
      "grad_norm": 0.15014901757240295,
      "kl": 0.06979522109031677,
      "learning_rate": 3e-06,
      "loss": 0.0065,
      "step": 1029
    },
    {
      "clip_ratio": 0.00026231101946905255,
      "epoch": 0.0028595383650103553,
      "grad_norm": 0.10039456933736801,
      "kl": 0.0724063590168953,
      "learning_rate": 3e-06,
      "loss": 0.006,
      "step": 1030
    },
    {
      "clip_ratio": 0.0007212661657831632,
      "epoch": 0.002862314615850171,
      "grad_norm": 0.10597745329141617,
      "kl": 0.07448813319206238,
      "learning_rate": 3e-06,
      "loss": 0.005,
      "step": 1031
    },
    {
      "clip_ratio": 0.00026949287712341174,
      "epoch": 0.002865090866689987,
      "grad_norm": 0.08808151632547379,
      "kl": 0.06819097325205803,
      "learning_rate": 3e-06,
      "loss": 0.0057,
      "step": 1032
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 220.4791717529297,
      "epoch": 0.0028678671175298032,
      "grad_norm": 0.12043464183807373,
      "kl": 0.06623350828886032,
      "learning_rate": 3e-06,
      "loss": -0.0042,
      "reward": 0.35625001788139343,
      "reward_std": 0.29714028537273407,
      "rewards/countdown_reward_func": 0.35625001788139343,
      "step": 1033,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 9.448223863728344e-05,
      "epoch": 0.002870643368369619,
      "grad_norm": 0.10195807367563248,
      "kl": 0.07157684862613678,
      "learning_rate": 3e-06,
      "loss": -0.0033,
      "step": 1034
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.002873419619209435,
      "grad_norm": 0.09431300312280655,
      "kl": 0.06604013964533806,
      "learning_rate": 3e-06,
      "loss": -0.0042,
      "step": 1035
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0028761958700492508,
      "grad_norm": 0.10502111911773682,
      "kl": 0.06586654111742973,
      "learning_rate": 3e-06,
      "loss": -0.0037,
      "step": 1036
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0028789721208890666,
      "grad_norm": 0.17106521129608154,
      "kl": 0.06683254614472389,
      "learning_rate": 3e-06,
      "loss": -0.0046,
      "step": 1037
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0028817483717288824,
      "grad_norm": 0.1259961724281311,
      "kl": 0.06468446552753448,
      "learning_rate": 3e-06,
      "loss": -0.0038,
      "step": 1038
    },
    {
      "clip_ratio": 9.448223863728344e-05,
      "epoch": 0.0028845246225686983,
      "grad_norm": 0.12149322032928467,
      "kl": 0.06319839507341385,
      "learning_rate": 3e-06,
      "loss": -0.0049,
      "step": 1039
    },
    {
      "clip_ratio": 9.448223863728344e-05,
      "epoch": 0.002887300873408514,
      "grad_norm": 0.09481094032526016,
      "kl": 0.0666964054107666,
      "learning_rate": 3e-06,
      "loss": -0.0039,
      "step": 1040
    },
    {
      "clip_ratio": 8.491847984259948e-05,
      "epoch": 0.00289007712424833,
      "grad_norm": 0.09099525958299637,
      "kl": 0.061311766505241394,
      "learning_rate": 3e-06,
      "loss": -0.005,
      "step": 1041
    },
    {
      "clip_ratio": 0.00018680129142012447,
      "epoch": 0.002892853375088146,
      "grad_norm": 0.11290211975574493,
      "kl": 0.061817897483706474,
      "learning_rate": 3e-06,
      "loss": -0.0053,
      "step": 1042
    },
    {
      "clip_ratio": 0.00027388295711716637,
      "epoch": 0.0028956296259279616,
      "grad_norm": 0.13663621246814728,
      "kl": 0.06613102555274963,
      "learning_rate": 3e-06,
      "loss": -0.0062,
      "step": 1043
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.002898405876767778,
      "grad_norm": 0.10643167048692703,
      "kl": 0.06023329682648182,
      "learning_rate": 3e-06,
      "loss": -0.0055,
      "step": 1044
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 232.56250762939453,
      "epoch": 0.0029011821276075937,
      "grad_norm": 0.07880929112434387,
      "kl": 0.06694615818560123,
      "learning_rate": 3e-06,
      "loss": 0.0099,
      "reward": 0.21250002086162567,
      "reward_std": 0.16564146801829338,
      "rewards/countdown_reward_func": 0.21250002086162567,
      "step": 1045,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio": 0.00028686892619589344,
      "epoch": 0.0029039583784474096,
      "grad_norm": 0.08016741275787354,
      "kl": 0.06679723039269447,
      "learning_rate": 3e-06,
      "loss": 0.0107,
      "step": 1046
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0029067346292872254,
      "grad_norm": 0.07099456340074539,
      "kl": 0.06291750818490982,
      "learning_rate": 3e-06,
      "loss": 0.0097,
      "step": 1047
    },
    {
      "clip_ratio": 0.00010056315659312531,
      "epoch": 0.0029095108801270413,
      "grad_norm": 0.07919389754533768,
      "kl": 0.06845093332231045,
      "learning_rate": 3e-06,
      "loss": 0.0109,
      "step": 1048
    },
    {
      "clip_ratio": 0.00010080645006382838,
      "epoch": 0.002912287130966857,
      "grad_norm": 0.07597494125366211,
      "kl": 0.06439798139035702,
      "learning_rate": 3e-06,
      "loss": 0.0106,
      "step": 1049
    },
    {
      "clip_ratio": 0.00017322444182354957,
      "epoch": 0.002915063381806673,
      "grad_norm": 0.09772368520498276,
      "kl": 0.0649341493844986,
      "learning_rate": 3e-06,
      "loss": 0.0107,
      "step": 1050
    },
    {
      "clip_ratio": 0.001496979035437107,
      "epoch": 0.0029178396326464888,
      "grad_norm": 0.08392099291086197,
      "kl": 0.06803803145885468,
      "learning_rate": 3e-06,
      "loss": 0.0101,
      "step": 1051
    },
    {
      "clip_ratio": 0.000669743909384124,
      "epoch": 0.0029206158834863046,
      "grad_norm": 0.08624348044395447,
      "kl": 0.06706851720809937,
      "learning_rate": 3e-06,
      "loss": 0.01,
      "step": 1052
    },
    {
      "clip_ratio": 0.001187364658107981,
      "epoch": 0.0029233921343261205,
      "grad_norm": 0.07946142554283142,
      "kl": 0.06465382128953934,
      "learning_rate": 3e-06,
      "loss": 0.0091,
      "step": 1053
    },
    {
      "clip_ratio": 0.0002712814530241303,
      "epoch": 0.0029261683851659363,
      "grad_norm": 0.1127048060297966,
      "kl": 0.07115714997053146,
      "learning_rate": 3e-06,
      "loss": 0.0101,
      "step": 1054
    },
    {
      "clip_ratio": 0.00016687953029759228,
      "epoch": 0.0029289446360057526,
      "grad_norm": 0.07184568047523499,
      "kl": 0.06631535664200783,
      "learning_rate": 3e-06,
      "loss": 0.0091,
      "step": 1055
    },
    {
      "clip_ratio": 0.00020161290012765676,
      "epoch": 0.0029317208868455684,
      "grad_norm": 0.107190802693367,
      "kl": 0.06775633245706558,
      "learning_rate": 3e-06,
      "loss": 0.0092,
      "step": 1056
    },
    {
      "clip_ratio": 0.00026758435706142336,
      "completion_length": 230.25000762939453,
      "epoch": 0.0029344971376853842,
      "grad_norm": 0.08019760996103287,
      "kl": 0.07465282827615738,
      "learning_rate": 3e-06,
      "loss": 0.0048,
      "reward": 0.22708334773778915,
      "reward_std": 0.17232364416122437,
      "rewards/countdown_reward_func": 0.22708334773778915,
      "step": 1057,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 8.294625149574131e-05,
      "epoch": 0.0029372733885252,
      "grad_norm": 0.08832976967096329,
      "kl": 0.07460768148303032,
      "learning_rate": 3e-06,
      "loss": 0.0056,
      "step": 1058
    },
    {
      "clip_ratio": 0.00026728439843282104,
      "epoch": 0.002940049639365016,
      "grad_norm": 0.07996435463428497,
      "kl": 0.07490583881735802,
      "learning_rate": 3e-06,
      "loss": 0.0055,
      "step": 1059
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0029428258902048318,
      "grad_norm": 0.06566757708787918,
      "kl": 0.07495003193616867,
      "learning_rate": 3e-06,
      "loss": 0.0056,
      "step": 1060
    },
    {
      "clip_ratio": 0.0001822225167416036,
      "epoch": 0.0029456021410446476,
      "grad_norm": 0.07333634048700333,
      "kl": 0.07876036688685417,
      "learning_rate": 3e-06,
      "loss": 0.0054,
      "step": 1061
    },
    {
      "clip_ratio": 9.72762645687908e-05,
      "epoch": 0.0029483783918844634,
      "grad_norm": 0.08647799491882324,
      "kl": 0.0771314725279808,
      "learning_rate": 3e-06,
      "loss": 0.005,
      "step": 1062
    },
    {
      "clip_ratio": 8.928571332944557e-05,
      "epoch": 0.0029511546427242793,
      "grad_norm": 0.07851536571979523,
      "kl": 0.07513091340661049,
      "learning_rate": 3e-06,
      "loss": 0.0045,
      "step": 1063
    },
    {
      "clip_ratio": 0.0002651687682373449,
      "epoch": 0.002953930893564095,
      "grad_norm": 0.12982326745986938,
      "kl": 0.07557458430528641,
      "learning_rate": 3e-06,
      "loss": 0.0051,
      "step": 1064
    },
    {
      "clip_ratio": 0.00026728439843282104,
      "epoch": 0.0029567071444039114,
      "grad_norm": 0.07488778233528137,
      "kl": 0.07366466149687767,
      "learning_rate": 3e-06,
      "loss": 0.0045,
      "step": 1065
    },
    {
      "clip_ratio": 0.00026775128208100796,
      "epoch": 0.0029594833952437272,
      "grad_norm": 0.07061980664730072,
      "kl": 0.0725603848695755,
      "learning_rate": 3e-06,
      "loss": 0.0048,
      "step": 1066
    },
    {
      "clip_ratio": 0.00024727272102609277,
      "epoch": 0.002962259646083543,
      "grad_norm": 0.07645444571971893,
      "kl": 0.07657236978411674,
      "learning_rate": 3e-06,
      "loss": 0.0046,
      "step": 1067
    },
    {
      "clip_ratio": 0.0003571428533177823,
      "epoch": 0.002965035896923359,
      "grad_norm": 0.07127691805362701,
      "kl": 0.07353055477142334,
      "learning_rate": 3e-06,
      "loss": 0.0045,
      "step": 1068
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 222.81250762939453,
      "epoch": 0.0029678121477631748,
      "grad_norm": 0.0665845200419426,
      "kl": 0.06168382987380028,
      "learning_rate": 3e-06,
      "loss": 0.0032,
      "reward": 0.17500000447034836,
      "reward_std": 0.14995060861110687,
      "rewards/countdown_reward_func": 0.17500000074505806,
      "step": 1069,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio": 0.00035589729668572545,
      "epoch": 0.0029705883986029906,
      "grad_norm": 0.08601254224777222,
      "kl": 0.05869180150330067,
      "learning_rate": 3e-06,
      "loss": 0.0034,
      "step": 1070
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0029733646494428064,
      "grad_norm": 0.08124065399169922,
      "kl": 0.05769502557814121,
      "learning_rate": 3e-06,
      "loss": 0.0034,
      "step": 1071
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0029761409002826223,
      "grad_norm": 0.062196120619773865,
      "kl": 0.0583453718572855,
      "learning_rate": 3e-06,
      "loss": 0.0034,
      "step": 1072
    },
    {
      "clip_ratio": 0.0002649337984621525,
      "epoch": 0.002978917151122438,
      "grad_norm": 0.05779939889907837,
      "kl": 0.056453775614500046,
      "learning_rate": 3e-06,
      "loss": 0.0038,
      "step": 1073
    },
    {
      "clip_ratio": 0.00030795554630458355,
      "epoch": 0.002981693401962254,
      "grad_norm": 0.05982575565576553,
      "kl": 0.055925050750374794,
      "learning_rate": 3e-06,
      "loss": 0.0027,
      "step": 1074
    },
    {
      "clip_ratio": 0.00030165042699081823,
      "epoch": 0.00298446965280207,
      "grad_norm": 0.07068102061748505,
      "kl": 0.057197773829102516,
      "learning_rate": 3e-06,
      "loss": 0.0028,
      "step": 1075
    },
    {
      "clip_ratio": 0.0004265254028723575,
      "epoch": 0.002987245903641886,
      "grad_norm": 0.09287840127944946,
      "kl": 0.05506822466850281,
      "learning_rate": 3e-06,
      "loss": 0.0024,
      "step": 1076
    },
    {
      "clip_ratio": 0.0004502255469560623,
      "epoch": 0.002990022154481702,
      "grad_norm": 0.07072468101978302,
      "kl": 0.053443435579538345,
      "learning_rate": 3e-06,
      "loss": 0.0028,
      "step": 1077
    },
    {
      "clip_ratio": 0.00017390426364727318,
      "epoch": 0.0029927984053215177,
      "grad_norm": 0.07123330235481262,
      "kl": 0.05358114279806614,
      "learning_rate": 3e-06,
      "loss": 0.003,
      "step": 1078
    },
    {
      "clip_ratio": 0.0006227242993190885,
      "epoch": 0.0029955746561613336,
      "grad_norm": 0.063522107899189,
      "kl": 0.052388763055205345,
      "learning_rate": 3e-06,
      "loss": 0.0032,
      "step": 1079
    },
    {
      "clip_ratio": 0.0002694505383260548,
      "epoch": 0.0029983509070011494,
      "grad_norm": 0.058882538229227066,
      "kl": 0.05159541964530945,
      "learning_rate": 3e-06,
      "loss": 0.0024,
      "step": 1080
    },
    {
      "clip_ratio": 0.00018450184143148363,
      "completion_length": 226.64584350585938,
      "epoch": 0.0030011271578409653,
      "grad_norm": 0.1080564484000206,
      "kl": 0.05418804846704006,
      "learning_rate": 3e-06,
      "loss": -0.0019,
      "reward": 0.28333334624767303,
      "reward_std": 0.3219813033938408,
      "rewards/countdown_reward_func": 0.28333333879709244,
      "step": 1081,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 0.0006435593895730563,
      "epoch": 0.003003903408680781,
      "grad_norm": 0.12120220065116882,
      "kl": 0.05108523927628994,
      "learning_rate": 3e-06,
      "loss": -0.0019,
      "step": 1082
    },
    {
      "clip_ratio": 0.0006201210926519707,
      "epoch": 0.003006679659520597,
      "grad_norm": 0.10109229385852814,
      "kl": 0.052261438220739365,
      "learning_rate": 3e-06,
      "loss": -0.0024,
      "step": 1083
    },
    {
      "clip_ratio": 0.00017445541743654758,
      "epoch": 0.0030094559103604128,
      "grad_norm": 0.09882961213588715,
      "kl": 0.05041038617491722,
      "learning_rate": 3e-06,
      "loss": -0.0026,
      "step": 1084
    },
    {
      "clip_ratio": 0.00020145705639151856,
      "epoch": 0.0030122321612002286,
      "grad_norm": 0.10909520089626312,
      "kl": 0.0491649005562067,
      "learning_rate": 3e-06,
      "loss": -0.0027,
      "step": 1085
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0030150084120400445,
      "grad_norm": 0.1672067940235138,
      "kl": 0.049838531762361526,
      "learning_rate": 3e-06,
      "loss": -0.0029,
      "step": 1086
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0030177846628798607,
      "grad_norm": 0.12344881147146225,
      "kl": 0.05135510489344597,
      "learning_rate": 3e-06,
      "loss": -0.0028,
      "step": 1087
    },
    {
      "clip_ratio": 0.001279033807804808,
      "epoch": 0.0030205609137196766,
      "grad_norm": 0.11550983041524887,
      "kl": 0.04587686434388161,
      "learning_rate": 3e-06,
      "loss": -0.0032,
      "step": 1088
    },
    {
      "clip_ratio": 0.0004374117561383173,
      "epoch": 0.0030233371645594924,
      "grad_norm": 0.10007118433713913,
      "kl": 0.048211125656962395,
      "learning_rate": 3e-06,
      "loss": -0.0032,
      "step": 1089
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0030261134153993082,
      "grad_norm": 0.10254620760679245,
      "kl": 0.04644571617245674,
      "learning_rate": 3e-06,
      "loss": -0.004,
      "step": 1090
    },
    {
      "clip_ratio": 0.00016276042151730508,
      "epoch": 0.003028889666239124,
      "grad_norm": 0.11051363497972488,
      "kl": 0.044875843450427055,
      "learning_rate": 3e-06,
      "loss": -0.0043,
      "step": 1091
    },
    {
      "clip_ratio": 0.0014370106218848377,
      "epoch": 0.00303166591707894,
      "grad_norm": 0.15505895018577576,
      "kl": 0.0460149310529232,
      "learning_rate": 3e-06,
      "loss": -0.0041,
      "step": 1092
    },
    {
      "clip_ratio": 8.698677993379533e-05,
      "completion_length": 237.5416717529297,
      "epoch": 0.0030344421679187558,
      "grad_norm": 0.10442124307155609,
      "kl": 0.04337725602090359,
      "learning_rate": 3e-06,
      "loss": 0.0063,
      "reward": 0.2854166701436043,
      "reward_std": 0.31687821447849274,
      "rewards/countdown_reward_func": 0.2854166701436043,
      "step": 1093,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.0001680653658695519,
      "epoch": 0.0030372184187585716,
      "grad_norm": 0.1659352034330368,
      "kl": 0.04275266453623772,
      "learning_rate": 3e-06,
      "loss": 0.0053,
      "step": 1094
    },
    {
      "clip_ratio": 0.00017313426360487938,
      "epoch": 0.0030399946695983874,
      "grad_norm": 0.13962288200855255,
      "kl": 0.04703891836106777,
      "learning_rate": 3e-06,
      "loss": 0.0056,
      "step": 1095
    },
    {
      "clip_ratio": 0.00017009561270242557,
      "epoch": 0.0030427709204382033,
      "grad_norm": 0.13137014210224152,
      "kl": 0.044395437464118004,
      "learning_rate": 3e-06,
      "loss": 0.0057,
      "step": 1096
    },
    {
      "clip_ratio": 0.0002661462058313191,
      "epoch": 0.003045547171278019,
      "grad_norm": 0.10747912526130676,
      "kl": 0.04033567197620869,
      "learning_rate": 3e-06,
      "loss": 0.0051,
      "step": 1097
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0030483234221178354,
      "grad_norm": 0.10946369171142578,
      "kl": 0.042647797614336014,
      "learning_rate": 3e-06,
      "loss": 0.0051,
      "step": 1098
    },
    {
      "clip_ratio": 9.704969124868512e-05,
      "epoch": 0.0030510996729576512,
      "grad_norm": 0.10199002921581268,
      "kl": 0.04471134953200817,
      "learning_rate": 3e-06,
      "loss": 0.0047,
      "step": 1099
    },
    {
      "clip_ratio": 0.0002724502555793151,
      "epoch": 0.003053875923797467,
      "grad_norm": 0.10533545166254044,
      "kl": 0.044932011514902115,
      "learning_rate": 3e-06,
      "loss": 0.0036,
      "step": 1100
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.003056652174637283,
      "grad_norm": 0.14148595929145813,
      "kl": 0.047867994755506516,
      "learning_rate": 3e-06,
      "loss": 0.0031,
      "step": 1101
    },
    {
      "clip_ratio": 0.00046344404108822346,
      "epoch": 0.0030594284254770988,
      "grad_norm": 0.12489910423755646,
      "kl": 0.047186482697725296,
      "learning_rate": 3e-06,
      "loss": 0.0034,
      "step": 1102
    },
    {
      "clip_ratio": 0.0002588110146461986,
      "epoch": 0.0030622046763169146,
      "grad_norm": 0.13311858475208282,
      "kl": 0.04366219788789749,
      "learning_rate": 3e-06,
      "loss": 0.0042,
      "step": 1103
    },
    {
      "clip_ratio": 9.704969124868512e-05,
      "epoch": 0.0030649809271567304,
      "grad_norm": 0.09973665326833725,
      "kl": 0.045309677720069885,
      "learning_rate": 3e-06,
      "loss": 0.0039,
      "step": 1104
    },
    {
      "clip_ratio": 0.0007995735504664481,
      "completion_length": 233.06250762939453,
      "epoch": 0.0030677571779965463,
      "grad_norm": 0.12320923805236816,
      "kl": 0.04813423752784729,
      "learning_rate": 3e-06,
      "loss": 0.0091,
      "reward": 0.24791669100522995,
      "reward_std": 0.22883931919932365,
      "rewards/countdown_reward_func": 0.24791667610406876,
      "step": 1105,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 8.884150884114206e-05,
      "epoch": 0.003070533428836362,
      "grad_norm": 0.09392733871936798,
      "kl": 0.04717006906867027,
      "learning_rate": 3e-06,
      "loss": 0.0082,
      "step": 1106
    },
    {
      "clip_ratio": 0.00026870113651966676,
      "epoch": 0.003073309679676178,
      "grad_norm": 0.10014389455318451,
      "kl": 0.05198243260383606,
      "learning_rate": 3e-06,
      "loss": 0.0081,
      "step": 1107
    },
    {
      "clip_ratio": 8.934953802963719e-05,
      "epoch": 0.003076085930515994,
      "grad_norm": 0.06945673376321793,
      "kl": 0.048388589173555374,
      "learning_rate": 3e-06,
      "loss": 0.0083,
      "step": 1108
    },
    {
      "clip_ratio": 8.350033749593422e-05,
      "epoch": 0.00307886218135581,
      "grad_norm": 0.07839835435152054,
      "kl": 0.05126038379967213,
      "learning_rate": 3e-06,
      "loss": 0.0088,
      "step": 1109
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.003081638432195626,
      "grad_norm": 0.07928033918142319,
      "kl": 0.05163071118295193,
      "learning_rate": 3e-06,
      "loss": 0.0077,
      "step": 1110
    },
    {
      "clip_ratio": 0.00010382059554103762,
      "epoch": 0.0030844146830354417,
      "grad_norm": 0.09869455546140671,
      "kl": 0.05214657075703144,
      "learning_rate": 3e-06,
      "loss": 0.008,
      "step": 1111
    },
    {
      "clip_ratio": 0.00017819104687077925,
      "epoch": 0.0030871909338752576,
      "grad_norm": 0.08661158382892609,
      "kl": 0.05166606977581978,
      "learning_rate": 3e-06,
      "loss": 0.0079,
      "step": 1112
    },
    {
      "clip_ratio": 0.00043943087075604126,
      "epoch": 0.0030899671847150734,
      "grad_norm": 0.08715176582336426,
      "kl": 0.05564386770129204,
      "learning_rate": 3e-06,
      "loss": 0.0077,
      "step": 1113
    },
    {
      "clip_ratio": 0.0001715863836579956,
      "epoch": 0.0030927434355548893,
      "grad_norm": 0.06843938678503036,
      "kl": 0.0525389164686203,
      "learning_rate": 3e-06,
      "loss": 0.0073,
      "step": 1114
    },
    {
      "clip_ratio": 0.00018732093303697184,
      "epoch": 0.003095519686394705,
      "grad_norm": 0.08035042136907578,
      "kl": 0.05591611564159393,
      "learning_rate": 3e-06,
      "loss": 0.008,
      "step": 1115
    },
    {
      "clip_ratio": 8.934953802963719e-05,
      "epoch": 0.003098295937234521,
      "grad_norm": 0.0761677697300911,
      "kl": 0.056424250826239586,
      "learning_rate": 3e-06,
      "loss": 0.0069,
      "step": 1116
    },
    {
      "clip_ratio": 0.00025893703423207626,
      "completion_length": 221.83333587646484,
      "epoch": 0.0031010721880743368,
      "grad_norm": 0.08327441662549973,
      "kl": 0.05536172166466713,
      "learning_rate": 3e-06,
      "loss": -0.0039,
      "reward": 0.2645833492279053,
      "reward_std": 0.21911749243736267,
      "rewards/countdown_reward_func": 0.2645833343267441,
      "step": 1117,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0031038484389141526,
      "grad_norm": 0.100023552775383,
      "kl": 0.05755512788891792,
      "learning_rate": 3e-06,
      "loss": -0.004,
      "step": 1118
    },
    {
      "clip_ratio": 0.00018813963106367737,
      "epoch": 0.0031066246897539685,
      "grad_norm": 0.139785036444664,
      "kl": 0.05882943049073219,
      "learning_rate": 3e-06,
      "loss": -0.0037,
      "step": 1119
    },
    {
      "clip_ratio": 0.0001152073746197857,
      "epoch": 0.0031094009405937847,
      "grad_norm": 0.09004988521337509,
      "kl": 0.055969417095184326,
      "learning_rate": 3e-06,
      "loss": -0.0038,
      "step": 1120
    },
    {
      "clip_ratio": 9.191176650347188e-05,
      "epoch": 0.0031121771914336006,
      "grad_norm": 0.06995224207639694,
      "kl": 0.05503300577402115,
      "learning_rate": 3e-06,
      "loss": -0.0039,
      "step": 1121
    },
    {
      "clip_ratio": 0.00032552084303461015,
      "epoch": 0.0031149534422734164,
      "grad_norm": 0.060606665909290314,
      "kl": 0.05465872026979923,
      "learning_rate": 3e-06,
      "loss": -0.0036,
      "step": 1122
    },
    {
      "clip_ratio": 0.00017755682347342372,
      "epoch": 0.0031177296931132322,
      "grad_norm": 0.07822634279727936,
      "kl": 0.056107934564352036,
      "learning_rate": 3e-06,
      "loss": -0.0044,
      "step": 1123
    },
    {
      "clip_ratio": 0.0001152073746197857,
      "epoch": 0.003120505943953048,
      "grad_norm": 0.08555705845355988,
      "kl": 0.05655817873775959,
      "learning_rate": 3e-06,
      "loss": -0.0041,
      "step": 1124
    },
    {
      "clip_ratio": 0.00019658758537843823,
      "epoch": 0.003123282194792864,
      "grad_norm": 0.13074277341365814,
      "kl": 0.05860498920083046,
      "learning_rate": 3e-06,
      "loss": -0.005,
      "step": 1125
    },
    {
      "clip_ratio": 0.00038114040944492444,
      "epoch": 0.0031260584456326798,
      "grad_norm": 0.0899885818362236,
      "kl": 0.054890843108296394,
      "learning_rate": 3e-06,
      "loss": -0.005,
      "step": 1126
    },
    {
      "clip_ratio": 0.0006433823728002608,
      "epoch": 0.0031288346964724956,
      "grad_norm": 0.06713841110467911,
      "kl": 0.05289384722709656,
      "learning_rate": 3e-06,
      "loss": -0.005,
      "step": 1127
    },
    {
      "clip_ratio": 0.00027796779613709077,
      "epoch": 0.0031316109473123114,
      "grad_norm": 0.06459382176399231,
      "kl": 0.05130323953926563,
      "learning_rate": 3e-06,
      "loss": -0.0041,
      "step": 1128
    },
    {
      "clip_ratio": 0.0010648140450939536,
      "completion_length": 236.02083587646484,
      "epoch": 0.0031343871981521273,
      "grad_norm": 0.13045701384544373,
      "kl": 0.052938977256417274,
      "learning_rate": 3e-06,
      "loss": 0.0017,
      "reward": 0.2500000149011612,
      "reward_std": 0.29990123212337494,
      "rewards/countdown_reward_func": 0.2500000149011612,
      "step": 1129,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.00026675376284401864,
      "epoch": 0.003137163448991943,
      "grad_norm": 0.09997519105672836,
      "kl": 0.05317997932434082,
      "learning_rate": 3e-06,
      "loss": 0.0011,
      "step": 1130
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0031399396998317594,
      "grad_norm": 0.08752911537885666,
      "kl": 0.052106352522969246,
      "learning_rate": 3e-06,
      "loss": 0.001,
      "step": 1131
    },
    {
      "clip_ratio": 0.00019378491560928524,
      "epoch": 0.0031427159506715752,
      "grad_norm": 0.11155866086483002,
      "kl": 0.052127305418252945,
      "learning_rate": 3e-06,
      "loss": 0.0009,
      "step": 1132
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.003145492201511391,
      "grad_norm": 0.0927848219871521,
      "kl": 0.053360870108008385,
      "learning_rate": 3e-06,
      "loss": 0.0009,
      "step": 1133
    },
    {
      "clip_ratio": 0.000244140625,
      "epoch": 0.003148268452351207,
      "grad_norm": 0.09402063488960266,
      "kl": 0.053240761160850525,
      "learning_rate": 3e-06,
      "loss": 0.001,
      "step": 1134
    },
    {
      "clip_ratio": 0.0004069010537932627,
      "epoch": 0.0031510447031910228,
      "grad_norm": 0.12245853245258331,
      "kl": 0.050540367141366005,
      "learning_rate": 3e-06,
      "loss": 0.0015,
      "step": 1135
    },
    {
      "clip_ratio": 0.0005173567624296993,
      "epoch": 0.0031538209540308386,
      "grad_norm": 0.10049139708280563,
      "kl": 0.05162609927356243,
      "learning_rate": 3e-06,
      "loss": 0.0006,
      "step": 1136
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0031565972048706544,
      "grad_norm": 0.10036790370941162,
      "kl": 0.051185326650738716,
      "learning_rate": 3e-06,
      "loss": -0.0004,
      "step": 1137
    },
    {
      "clip_ratio": 0.00037915847497060895,
      "epoch": 0.0031593734557104703,
      "grad_norm": 0.10827629268169403,
      "kl": 0.05169490538537502,
      "learning_rate": 3e-06,
      "loss": 0.001,
      "step": 1138
    },
    {
      "clip_ratio": 0.0004089097637915984,
      "epoch": 0.003162149706550286,
      "grad_norm": 0.0900021344423294,
      "kl": 0.052112411707639694,
      "learning_rate": 3e-06,
      "loss": 0.0002,
      "step": 1139
    },
    {
      "clip_ratio": 0.0002506030141375959,
      "epoch": 0.003164925957390102,
      "grad_norm": 0.09494884312152863,
      "kl": 0.05369776301085949,
      "learning_rate": 3e-06,
      "loss": -0.0003,
      "step": 1140
    },
    {
      "clip_ratio": 8.316699677379802e-05,
      "completion_length": 241.08334350585938,
      "epoch": 0.003167702208229918,
      "grad_norm": 0.14724688231945038,
      "kl": 0.050434716045856476,
      "learning_rate": 3e-06,
      "loss": 0.0052,
      "reward": 0.3750000149011612,
      "reward_std": 0.40321892499923706,
      "rewards/countdown_reward_func": 0.375,
      "step": 1141,
      "zero_std_ratio": 0.0
    },
    {
      "clip_ratio": 0.0008169302745955065,
      "epoch": 0.003170478459069734,
      "grad_norm": 0.10081308335065842,
      "kl": 0.05200190842151642,
      "learning_rate": 3e-06,
      "loss": 0.005,
      "step": 1142
    },
    {
      "clip_ratio": 0.0002557880652602762,
      "epoch": 0.00317325470990955,
      "grad_norm": 0.11964285373687744,
      "kl": 0.05052047781646252,
      "learning_rate": 3e-06,
      "loss": 0.0048,
      "step": 1143
    },
    {
      "clip_ratio": 0.0005074728833278641,
      "epoch": 0.0031760309607493657,
      "grad_norm": 0.20048139989376068,
      "kl": 0.056764667853713036,
      "learning_rate": 3e-06,
      "loss": 0.006,
      "step": 1144
    },
    {
      "clip_ratio": 0.00027372263139113784,
      "epoch": 0.0031788072115891816,
      "grad_norm": 0.10537584125995636,
      "kl": 0.04961631819605827,
      "learning_rate": 3e-06,
      "loss": 0.0046,
      "step": 1145
    },
    {
      "clip_ratio": 0.00033895507658598945,
      "epoch": 0.0031815834624289974,
      "grad_norm": 0.11174456030130386,
      "kl": 0.0519944503903389,
      "learning_rate": 3e-06,
      "loss": 0.0037,
      "step": 1146
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0031843597132688133,
      "grad_norm": 0.11552406847476959,
      "kl": 0.05385969392955303,
      "learning_rate": 3e-06,
      "loss": 0.0032,
      "step": 1147
    },
    {
      "clip_ratio": 0.0011523118009790778,
      "epoch": 0.003187135964108629,
      "grad_norm": 0.10118366032838821,
      "kl": 0.05519482307136059,
      "learning_rate": 3e-06,
      "loss": 0.0034,
      "step": 1148
    },
    {
      "clip_ratio": 0.0002575748658273369,
      "epoch": 0.003189912214948445,
      "grad_norm": 0.12076838314533234,
      "kl": 0.05428927019238472,
      "learning_rate": 3e-06,
      "loss": 0.0032,
      "step": 1149
    },
    {
      "clip_ratio": 0.0005272936105029657,
      "epoch": 0.0031926884657882608,
      "grad_norm": 0.1505252718925476,
      "kl": 0.06170363910496235,
      "learning_rate": 3e-06,
      "loss": 0.004,
      "step": 1150
    },
    {
      "clip_ratio": 0.00017786595708457753,
      "epoch": 0.0031954647166280766,
      "grad_norm": 0.0998905599117279,
      "kl": 0.054063014686107635,
      "learning_rate": 3e-06,
      "loss": 0.0033,
      "step": 1151
    },
    {
      "clip_ratio": 0.0005830957088619471,
      "epoch": 0.0031982409674678925,
      "grad_norm": 0.11353659629821777,
      "kl": 0.055830128490924835,
      "learning_rate": 3e-06,
      "loss": 0.003,
      "step": 1152
    },
    {
      "clip_ratio": 8.26719551696442e-05,
      "completion_length": 232.8541717529297,
      "epoch": 0.0032010172183077087,
      "grad_norm": 0.12211279571056366,
      "kl": 0.07155881449580193,
      "learning_rate": 3e-06,
      "loss": 0.0302,
      "reward": 0.3229166865348816,
      "reward_std": 0.24453017860651016,
      "rewards/countdown_reward_func": 0.3229166865348816,
      "step": 1153,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.00018164265202358365,
      "epoch": 0.0032037934691475246,
      "grad_norm": 0.12927870452404022,
      "kl": 0.064654640853405,
      "learning_rate": 3e-06,
      "loss": 0.0298,
      "step": 1154
    },
    {
      "clip_ratio": 0.0001653439103392884,
      "epoch": 0.0032065697199873404,
      "grad_norm": 0.10473229736089706,
      "kl": 0.07196308299899101,
      "learning_rate": 3e-06,
      "loss": 0.0299,
      "step": 1155
    },
    {
      "clip_ratio": 0.00024557957658544183,
      "epoch": 0.0032093459708271562,
      "grad_norm": 0.1307552605867386,
      "kl": 0.06787708401679993,
      "learning_rate": 3e-06,
      "loss": 0.0292,
      "step": 1156
    },
    {
      "clip_ratio": 0.00028894259594380856,
      "epoch": 0.003212122221666972,
      "grad_norm": 0.1252485066652298,
      "kl": 0.07557385787367821,
      "learning_rate": 3e-06,
      "loss": 0.03,
      "step": 1157
    },
    {
      "clip_ratio": 0.0008342704823007807,
      "epoch": 0.003214898472506788,
      "grad_norm": 0.11512560397386551,
      "kl": 0.07205545529723167,
      "learning_rate": 3e-06,
      "loss": 0.0282,
      "step": 1158
    },
    {
      "clip_ratio": 8.406186680076644e-05,
      "epoch": 0.0032176747233466038,
      "grad_norm": 0.13281333446502686,
      "kl": 0.07855755090713501,
      "learning_rate": 3e-06,
      "loss": 0.0279,
      "step": 1159
    },
    {
      "clip_ratio": 0.0001690331264398992,
      "epoch": 0.0032204509741864196,
      "grad_norm": 0.12156619131565094,
      "kl": 0.07363288104534149,
      "learning_rate": 3e-06,
      "loss": 0.0271,
      "step": 1160
    },
    {
      "clip_ratio": 0.00041335978312417865,
      "epoch": 0.0032232272250262354,
      "grad_norm": 0.10087471455335617,
      "kl": 0.08428899571299553,
      "learning_rate": 3e-06,
      "loss": 0.0275,
      "step": 1161
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0032260034758660513,
      "grad_norm": 0.14251810312271118,
      "kl": 0.07908787578344345,
      "learning_rate": 3e-06,
      "loss": 0.0252,
      "step": 1162
    },
    {
      "clip_ratio": 0.0003094059356953949,
      "epoch": 0.003228779726705867,
      "grad_norm": 0.11256741732358932,
      "kl": 0.08613381907343864,
      "learning_rate": 3e-06,
      "loss": 0.026,
      "step": 1163
    },
    {
      "clip_ratio": 0.00026637538394425064,
      "epoch": 0.0032315559775456834,
      "grad_norm": 0.1097400113940239,
      "kl": 0.08562665432691574,
      "learning_rate": 3e-06,
      "loss": 0.0253,
      "step": 1164
    },
    {
      "clip_ratio": 0.0001998317675315775,
      "completion_length": 213.89584350585938,
      "epoch": 0.0032343322283854992,
      "grad_norm": 0.087900809943676,
      "kl": 0.0852268636226654,
      "learning_rate": 3e-06,
      "loss": 0.0202,
      "reward": 0.4166666865348816,
      "reward_std": 0.3476388454437256,
      "rewards/countdown_reward_func": 0.4166666567325592,
      "step": 1165,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.003237108479225315,
      "grad_norm": 0.1092759519815445,
      "kl": 0.08935586363077164,
      "learning_rate": 3e-06,
      "loss": 0.02,
      "step": 1166
    },
    {
      "clip_ratio": 0.00037480452738236636,
      "epoch": 0.003239884730065131,
      "grad_norm": 0.1183471605181694,
      "kl": 0.09479113668203354,
      "learning_rate": 3e-06,
      "loss": 0.0208,
      "step": 1167
    },
    {
      "clip_ratio": 0.0002687285596039146,
      "epoch": 0.0032426609809049468,
      "grad_norm": 0.09400281310081482,
      "kl": 0.09442057460546494,
      "learning_rate": 3e-06,
      "loss": 0.0193,
      "step": 1168
    },
    {
      "clip_ratio": 0.00017308967653661966,
      "epoch": 0.0032454372317447626,
      "grad_norm": 0.14759588241577148,
      "kl": 0.09654445201158524,
      "learning_rate": 3e-06,
      "loss": 0.0188,
      "step": 1169
    },
    {
      "clip_ratio": 0.00020166092144791037,
      "epoch": 0.0032482134825845784,
      "grad_norm": 0.10944969207048416,
      "kl": 0.09353556111454964,
      "learning_rate": 3e-06,
      "loss": 0.0197,
      "step": 1170
    },
    {
      "clip_ratio": 0.00010879024921450764,
      "epoch": 0.0032509897334243943,
      "grad_norm": 0.10797934979200363,
      "kl": 0.10146218538284302,
      "learning_rate": 3e-06,
      "loss": 0.0192,
      "step": 1171
    },
    {
      "clip_ratio": 0.0008672036346979439,
      "epoch": 0.00325376598426421,
      "grad_norm": 0.11213571578264236,
      "kl": 0.1059374175965786,
      "learning_rate": 3e-06,
      "loss": 0.0183,
      "step": 1172
    },
    {
      "clip_ratio": 0.0018254909082315862,
      "epoch": 0.003256542235104026,
      "grad_norm": 0.1145535483956337,
      "kl": 0.1167490966618061,
      "learning_rate": 3e-06,
      "loss": 0.0203,
      "step": 1173
    },
    {
      "clip_ratio": 0.0010834443964995444,
      "epoch": 0.003259318485943842,
      "grad_norm": 0.08255753666162491,
      "kl": 0.11234157159924507,
      "learning_rate": 3e-06,
      "loss": 0.0177,
      "step": 1174
    },
    {
      "clip_ratio": 0.0012724358239211142,
      "epoch": 0.003262094736783658,
      "grad_norm": 0.10409360378980637,
      "kl": 0.11117475107312202,
      "learning_rate": 3e-06,
      "loss": 0.0181,
      "step": 1175
    },
    {
      "clip_ratio": 0.0022241021506488323,
      "epoch": 0.003264870987623474,
      "grad_norm": 0.09691368043422699,
      "kl": 0.10797763615846634,
      "learning_rate": 3e-06,
      "loss": 0.018,
      "step": 1176
    },
    {
      "clip_ratio": 0.0002880159590858966,
      "completion_length": 233.7291717529297,
      "epoch": 0.0032676472384632897,
      "grad_norm": 0.08466751873493195,
      "kl": 0.12909483164548874,
      "learning_rate": 3e-06,
      "loss": 0.0086,
      "reward": 0.2291666939854622,
      "reward_std": 0.2128555178642273,
      "rewards/countdown_reward_func": 0.2291666865348816,
      "step": 1177,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio": 0.0003561055054888129,
      "epoch": 0.0032704234893031056,
      "grad_norm": 0.11577208340167999,
      "kl": 0.12361054494976997,
      "learning_rate": 3e-06,
      "loss": 0.0078,
      "step": 1178
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0032731997401429214,
      "grad_norm": 0.09205642342567444,
      "kl": 0.13038898259401321,
      "learning_rate": 3e-06,
      "loss": 0.0089,
      "step": 1179
    },
    {
      "clip_ratio": 0.000333176227286458,
      "epoch": 0.0032759759909827373,
      "grad_norm": 0.10060107707977295,
      "kl": 0.13378220051527023,
      "learning_rate": 3e-06,
      "loss": 0.0083,
      "step": 1180
    },
    {
      "clip_ratio": 0.00010113268945133314,
      "epoch": 0.003278752241822553,
      "grad_norm": 0.09801634401082993,
      "kl": 0.1360284462571144,
      "learning_rate": 3e-06,
      "loss": 0.0089,
      "step": 1181
    },
    {
      "clip_ratio": 0.0002441406322759576,
      "epoch": 0.003281528492662369,
      "grad_norm": 0.08726264536380768,
      "kl": 0.131295807659626,
      "learning_rate": 3e-06,
      "loss": 0.0089,
      "step": 1182
    },
    {
      "clip_ratio": 0.00017773196304915473,
      "epoch": 0.0032843047435021848,
      "grad_norm": 0.08691044896841049,
      "kl": 0.13674616813659668,
      "learning_rate": 3e-06,
      "loss": 0.0085,
      "step": 1183
    },
    {
      "clip_ratio": 0.00018315018678549677,
      "epoch": 0.0032870809943420006,
      "grad_norm": 0.11604570597410202,
      "kl": 0.1295301541686058,
      "learning_rate": 3e-06,
      "loss": 0.0073,
      "step": 1184
    },
    {
      "clip_ratio": 0.0007019351178314537,
      "epoch": 0.0032898572451818165,
      "grad_norm": 0.09062675386667252,
      "kl": 0.13159853219985962,
      "learning_rate": 3e-06,
      "loss": 0.0084,
      "step": 1185
    },
    {
      "clip_ratio": 0.0004526742995949462,
      "epoch": 0.0032926334960216327,
      "grad_norm": 0.10843745619058609,
      "kl": 0.13263515383005142,
      "learning_rate": 3e-06,
      "loss": 0.0079,
      "step": 1186
    },
    {
      "clip_ratio": 0.0006641855870839208,
      "epoch": 0.0032954097468614486,
      "grad_norm": 0.08221055567264557,
      "kl": 0.13184264674782753,
      "learning_rate": 3e-06,
      "loss": 0.0083,
      "step": 1187
    },
    {
      "clip_ratio": 0.0004921089857816696,
      "epoch": 0.0032981859977012644,
      "grad_norm": 0.08339028805494308,
      "kl": 0.1247389204800129,
      "learning_rate": 3e-06,
      "loss": 0.0072,
      "step": 1188
    },
    {
      "clip_ratio": 0.00020850708824582398,
      "completion_length": 234.0416717529297,
      "epoch": 0.0033009622485410802,
      "grad_norm": 0.11418038606643677,
      "kl": 0.10565116629004478,
      "learning_rate": 3e-06,
      "loss": -0.0015,
      "reward": 0.3437500298023224,
      "reward_std": 0.28535500913858414,
      "rewards/countdown_reward_func": 0.3437500149011612,
      "step": 1189,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.003303738499380896,
      "grad_norm": 0.12243412435054779,
      "kl": 0.09964673221111298,
      "learning_rate": 3e-06,
      "loss": -0.0021,
      "step": 1190
    },
    {
      "clip_ratio": 0.00028036253206664696,
      "epoch": 0.003306514750220712,
      "grad_norm": 0.17603905498981476,
      "kl": 0.10419896617531776,
      "learning_rate": 3e-06,
      "loss": -0.0001,
      "step": 1191
    },
    {
      "clip_ratio": 0.0001840942568378523,
      "epoch": 0.0033092910010605278,
      "grad_norm": 0.11945787817239761,
      "kl": 0.09894903376698494,
      "learning_rate": 3e-06,
      "loss": -0.0027,
      "step": 1192
    },
    {
      "clip_ratio": 8.406186680076644e-05,
      "epoch": 0.0033120672519003436,
      "grad_norm": 0.14571459591388702,
      "kl": 0.09924932196736336,
      "learning_rate": 3e-06,
      "loss": -0.0023,
      "step": 1193
    },
    {
      "clip_ratio": 0.000245529918174725,
      "epoch": 0.0033148435027401594,
      "grad_norm": 0.1262568235397339,
      "kl": 0.09315666556358337,
      "learning_rate": 3e-06,
      "loss": -0.0041,
      "step": 1194
    },
    {
      "clip_ratio": 0.00029078290390316397,
      "epoch": 0.0033176197535799753,
      "grad_norm": 0.12053867429494858,
      "kl": 0.09638616442680359,
      "learning_rate": 3e-06,
      "loss": -0.0028,
      "step": 1195
    },
    {
      "clip_ratio": 0.0008103728177957237,
      "epoch": 0.0033203960044197916,
      "grad_norm": 0.14339543879032135,
      "kl": 0.089625783264637,
      "learning_rate": 3e-06,
      "loss": -0.0034,
      "step": 1196
    },
    {
      "clip_ratio": 0.0003617427501012571,
      "epoch": 0.0033231722552596074,
      "grad_norm": 0.17292562127113342,
      "kl": 0.09256411343812943,
      "learning_rate": 3e-06,
      "loss": -0.0038,
      "step": 1197
    },
    {
      "clip_ratio": 0.0008070902986219153,
      "epoch": 0.0033259485060994232,
      "grad_norm": 0.11857297271490097,
      "kl": 0.08773430064320564,
      "learning_rate": 3e-06,
      "loss": -0.0032,
      "step": 1198
    },
    {
      "clip_ratio": 0.00044789022649638355,
      "epoch": 0.003328724756939239,
      "grad_norm": 0.1319698989391327,
      "kl": 0.08959746360778809,
      "learning_rate": 3e-06,
      "loss": -0.0056,
      "step": 1199
    },
    {
      "clip_ratio": 0.0012999755563214421,
      "epoch": 0.003331501007779055,
      "grad_norm": 0.1194002777338028,
      "kl": 0.08413466811180115,
      "learning_rate": 3e-06,
      "loss": -0.0052,
      "step": 1200
    },
    {
      "clip_ratio": 0.00024771419703029096,
      "completion_length": 225.64584350585938,
      "epoch": 0.0033342772586188708,
      "grad_norm": 0.07703583687543869,
      "kl": 0.09083598852157593,
      "learning_rate": 3e-06,
      "loss": 0.0078,
      "reward": 0.2291666939854622,
      "reward_std": 0.18291139975190163,
      "rewards/countdown_reward_func": 0.2291666865348816,
      "step": 1201,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio": 0.00020563387079164386,
      "epoch": 0.0033370535094586866,
      "grad_norm": 0.0965060293674469,
      "kl": 0.09030065685510635,
      "learning_rate": 3e-06,
      "loss": 0.0065,
      "step": 1202
    },
    {
      "clip_ratio": 0.00019009599782293662,
      "epoch": 0.0033398297602985024,
      "grad_norm": 0.07806777209043503,
      "kl": 0.09229482710361481,
      "learning_rate": 3e-06,
      "loss": 0.0073,
      "step": 1203
    },
    {
      "clip_ratio": 0.00033928067568922415,
      "epoch": 0.0033426060111383183,
      "grad_norm": 0.08718756586313248,
      "kl": 0.08507254719734192,
      "learning_rate": 3e-06,
      "loss": 0.0068,
      "step": 1204
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.003345382261978134,
      "grad_norm": 0.08955029398202896,
      "kl": 0.08796864375472069,
      "learning_rate": 3e-06,
      "loss": 0.0063,
      "step": 1205
    },
    {
      "clip_ratio": 8.316699677379802e-05,
      "epoch": 0.00334815851281795,
      "grad_norm": 0.11132470518350601,
      "kl": 0.09032916277647018,
      "learning_rate": 3e-06,
      "loss": 0.007,
      "step": 1206
    },
    {
      "clip_ratio": 0.0002561136716394685,
      "epoch": 0.0033509347636577662,
      "grad_norm": 0.08454351872205734,
      "kl": 0.08703029155731201,
      "learning_rate": 3e-06,
      "loss": 0.0064,
      "step": 1207
    },
    {
      "clip_ratio": 9.137426968663931e-05,
      "epoch": 0.003353711014497582,
      "grad_norm": 0.08685325086116791,
      "kl": 0.08563976734876633,
      "learning_rate": 3e-06,
      "loss": 0.006,
      "step": 1208
    },
    {
      "clip_ratio": 0.0001069290010491386,
      "epoch": 0.003356487265337398,
      "grad_norm": 0.08751443773508072,
      "kl": 0.0876334123313427,
      "learning_rate": 3e-06,
      "loss": 0.007,
      "step": 1209
    },
    {
      "clip_ratio": 0.00042423446575412527,
      "epoch": 0.0033592635161772137,
      "grad_norm": 0.07404050976037979,
      "kl": 0.07978618890047073,
      "learning_rate": 3e-06,
      "loss": 0.0063,
      "step": 1210
    },
    {
      "clip_ratio": 0.0005211606621742249,
      "epoch": 0.0033620397670170296,
      "grad_norm": 0.1123705729842186,
      "kl": 0.08384502306580544,
      "learning_rate": 3e-06,
      "loss": 0.006,
      "step": 1211
    },
    {
      "clip_ratio": 0.00043046276550740004,
      "epoch": 0.0033648160178568454,
      "grad_norm": 0.09377264976501465,
      "kl": 0.08539973199367523,
      "learning_rate": 3e-06,
      "loss": 0.0065,
      "step": 1212
    },
    {
      "clip_ratio": 0.00017872503667604178,
      "completion_length": 238.9375,
      "epoch": 0.0033675922686966613,
      "grad_norm": 0.08966858685016632,
      "kl": 0.08921418339014053,
      "learning_rate": 3e-06,
      "loss": 0.0032,
      "reward": 0.26458335667848587,
      "reward_std": 0.2634518966078758,
      "rewards/countdown_reward_func": 0.26458335667848587,
      "step": 1213,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0003323842174722813,
      "epoch": 0.003370368519536477,
      "grad_norm": 0.11603780835866928,
      "kl": 0.09003105014562607,
      "learning_rate": 3e-06,
      "loss": 0.0033,
      "step": 1214
    },
    {
      "clip_ratio": 9.505703201284632e-05,
      "epoch": 0.003373144770376293,
      "grad_norm": 0.12451639026403427,
      "kl": 0.08492600917816162,
      "learning_rate": 3e-06,
      "loss": 0.0038,
      "step": 1215
    },
    {
      "clip_ratio": 8.234519191319123e-05,
      "epoch": 0.0033759210212161088,
      "grad_norm": 0.09685535728931427,
      "kl": 0.08527249097824097,
      "learning_rate": 3e-06,
      "loss": 0.0028,
      "step": 1216
    },
    {
      "clip_ratio": 0.0005166697083041072,
      "epoch": 0.0033786972720559246,
      "grad_norm": 0.1449529230594635,
      "kl": 0.08535202592611313,
      "learning_rate": 3e-06,
      "loss": 0.0034,
      "step": 1217
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.003381473522895741,
      "grad_norm": 0.08282797783613205,
      "kl": 0.0859798900783062,
      "learning_rate": 3e-06,
      "loss": 0.0036,
      "step": 1218
    },
    {
      "clip_ratio": 0.00018502863531466573,
      "epoch": 0.0033842497737355567,
      "grad_norm": 0.09159551560878754,
      "kl": 0.08557010814547539,
      "learning_rate": 3e-06,
      "loss": 0.0027,
      "step": 1219
    },
    {
      "clip_ratio": 0.0002536034444347024,
      "epoch": 0.0033870260245753726,
      "grad_norm": 0.10851403325796127,
      "kl": 0.08524026349186897,
      "learning_rate": 3e-06,
      "loss": 0.0024,
      "step": 1220
    },
    {
      "clip_ratio": 0.0005241364997345954,
      "epoch": 0.0033898022754151884,
      "grad_norm": 0.1287156045436859,
      "kl": 0.08104551210999489,
      "learning_rate": 3e-06,
      "loss": 0.0025,
      "step": 1221
    },
    {
      "clip_ratio": 0.00017740222392603755,
      "epoch": 0.0033925785262550042,
      "grad_norm": 0.10037072002887726,
      "kl": 0.08082518354058266,
      "learning_rate": 3e-06,
      "loss": 0.002,
      "step": 1222
    },
    {
      "clip_ratio": 0.00017451102030463517,
      "epoch": 0.00339535477709482,
      "grad_norm": 0.13647525012493134,
      "kl": 0.08178085088729858,
      "learning_rate": 3e-06,
      "loss": 0.0009,
      "step": 1223
    },
    {
      "clip_ratio": 0.000411814013205003,
      "epoch": 0.003398131027934636,
      "grad_norm": 0.08645734190940857,
      "kl": 0.08200101554393768,
      "learning_rate": 3e-06,
      "loss": 0.0031,
      "step": 1224
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 241.62500762939453,
      "epoch": 0.0034009072787744518,
      "grad_norm": 0.10560040175914764,
      "kl": 0.07796743884682655,
      "learning_rate": 3e-06,
      "loss": 0.0136,
      "reward": 0.26875001937150955,
      "reward_std": 0.2272602580487728,
      "rewards/countdown_reward_func": 0.26875001937150955,
      "step": 1225,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio": 0.00016276042151730508,
      "epoch": 0.0034036835296142676,
      "grad_norm": 0.15348504483699799,
      "kl": 0.07507862150669098,
      "learning_rate": 3e-06,
      "loss": 0.014,
      "step": 1226
    },
    {
      "clip_ratio": 0.00033518215059302747,
      "epoch": 0.0034064597804540834,
      "grad_norm": 0.08251035213470459,
      "kl": 0.08217736333608627,
      "learning_rate": 3e-06,
      "loss": 0.015,
      "step": 1227
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0034092360312938993,
      "grad_norm": 0.11358001828193665,
      "kl": 0.07504938542842865,
      "learning_rate": 3e-06,
      "loss": 0.0148,
      "step": 1228
    },
    {
      "clip_ratio": 0.0003684598486870527,
      "epoch": 0.0034120122821337156,
      "grad_norm": 0.1041935384273529,
      "kl": 0.08010926097631454,
      "learning_rate": 3e-06,
      "loss": 0.0143,
      "step": 1229
    },
    {
      "clip_ratio": 0.0005087452591396868,
      "epoch": 0.0034147885329735314,
      "grad_norm": 0.09497429430484772,
      "kl": 0.07363765686750412,
      "learning_rate": 3e-06,
      "loss": 0.0139,
      "step": 1230
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0034175647838133472,
      "grad_norm": 0.08915600925683975,
      "kl": 0.07781890407204628,
      "learning_rate": 3e-06,
      "loss": 0.0131,
      "step": 1231
    },
    {
      "clip_ratio": 0.0003355148946866393,
      "epoch": 0.003420341034653163,
      "grad_norm": 0.0972413569688797,
      "kl": 0.07709966227412224,
      "learning_rate": 3e-06,
      "loss": 0.0131,
      "step": 1232
    },
    {
      "clip_ratio": 0.000244140625,
      "epoch": 0.003423117285492979,
      "grad_norm": 0.08852770924568176,
      "kl": 0.08478394895792007,
      "learning_rate": 3e-06,
      "loss": 0.0147,
      "step": 1233
    },
    {
      "clip_ratio": 0.0008937545935623348,
      "epoch": 0.0034258935363327947,
      "grad_norm": 0.11425919830799103,
      "kl": 0.07852787896990776,
      "learning_rate": 3e-06,
      "loss": 0.0144,
      "step": 1234
    },
    {
      "clip_ratio": 0.000438432558439672,
      "epoch": 0.0034286697871726106,
      "grad_norm": 0.09492716938257217,
      "kl": 0.08431283012032509,
      "learning_rate": 3e-06,
      "loss": 0.013,
      "step": 1235
    },
    {
      "clip_ratio": 0.0004982753162039444,
      "epoch": 0.0034314460380124264,
      "grad_norm": 0.10918039828538895,
      "kl": 0.07764238864183426,
      "learning_rate": 3e-06,
      "loss": 0.012,
      "step": 1236
    },
    {
      "clip_ratio": 8.821453957352787e-05,
      "completion_length": 239.8541717529297,
      "epoch": 0.0034342222888522423,
      "grad_norm": 0.1400684416294098,
      "kl": 0.0789659395813942,
      "learning_rate": 3e-06,
      "loss": 0.0171,
      "reward": 0.4375000149011612,
      "reward_std": 0.398020476102829,
      "rewards/countdown_reward_func": 0.4375000149011612,
      "step": 1237,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 8.790435822447762e-05,
      "epoch": 0.003436998539692058,
      "grad_norm": 0.117515429854393,
      "kl": 0.08153605833649635,
      "learning_rate": 3e-06,
      "loss": 0.0183,
      "step": 1238
    },
    {
      "clip_ratio": 0.0001731122611090541,
      "epoch": 0.003439774790531874,
      "grad_norm": 0.12731020152568817,
      "kl": 0.08211733773350716,
      "learning_rate": 3e-06,
      "loss": 0.0186,
      "step": 1239
    },
    {
      "clip_ratio": 0.00017580871644895524,
      "epoch": 0.0034425510413716902,
      "grad_norm": 0.1385928988456726,
      "kl": 0.07959434390068054,
      "learning_rate": 3e-06,
      "loss": 0.0168,
      "step": 1240
    },
    {
      "clip_ratio": 0.00017041582032106817,
      "epoch": 0.003445327292211506,
      "grad_norm": 0.12391113489866257,
      "kl": 0.08285382017493248,
      "learning_rate": 3e-06,
      "loss": 0.0182,
      "step": 1241
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.003448103543051322,
      "grad_norm": 0.10620970278978348,
      "kl": 0.0919383093714714,
      "learning_rate": 3e-06,
      "loss": 0.0182,
      "step": 1242
    },
    {
      "clip_ratio": 8.821453957352787e-05,
      "epoch": 0.0034508797938911377,
      "grad_norm": 0.1453152298927307,
      "kl": 0.08631188422441483,
      "learning_rate": 3e-06,
      "loss": 0.0174,
      "step": 1243
    },
    {
      "clip_ratio": 8.884150884114206e-05,
      "epoch": 0.0034536560447309536,
      "grad_norm": 0.11560267210006714,
      "kl": 0.08962936699390411,
      "learning_rate": 3e-06,
      "loss": 0.0167,
      "step": 1244
    },
    {
      "clip_ratio": 0.000705406149791088,
      "epoch": 0.0034564322955707694,
      "grad_norm": 0.13734284043312073,
      "kl": 0.09047690033912659,
      "learning_rate": 3e-06,
      "loss": 0.0165,
      "step": 1245
    },
    {
      "clip_ratio": 8.234519191319123e-05,
      "epoch": 0.0034592085464105853,
      "grad_norm": 0.11958130449056625,
      "kl": 0.08676190301775932,
      "learning_rate": 3e-06,
      "loss": 0.0159,
      "step": 1246
    },
    {
      "clip_ratio": 0.00025863035989459604,
      "epoch": 0.003461984797250401,
      "grad_norm": 0.13450902700424194,
      "kl": 0.09278702363371849,
      "learning_rate": 3e-06,
      "loss": 0.0169,
      "step": 1247
    },
    {
      "clip_ratio": 8.884150884114206e-05,
      "epoch": 0.003464761048090217,
      "grad_norm": 0.11801480501890182,
      "kl": 0.1030571274459362,
      "learning_rate": 3e-06,
      "loss": 0.0163,
      "step": 1248
    },
    {
      "clip_ratio": 9.231905278284103e-05,
      "completion_length": 225.1041717529297,
      "epoch": 0.0034675372989300328,
      "grad_norm": 0.13714726269245148,
      "kl": 0.09993145614862442,
      "learning_rate": 3e-06,
      "loss": 0.0016,
      "reward": 0.30000002682209015,
      "reward_std": 0.30601368844509125,
      "rewards/countdown_reward_func": 0.30000000447034836,
      "step": 1249,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 0.00034739852708298713,
      "epoch": 0.0034703135497698486,
      "grad_norm": 0.11648391932249069,
      "kl": 0.10220260173082352,
      "learning_rate": 3e-06,
      "loss": 0.0002,
      "step": 1250
    },
    {
      "clip_ratio": 0.00016276042151730508,
      "epoch": 0.003473089800609665,
      "grad_norm": 0.0945407897233963,
      "kl": 0.11158350110054016,
      "learning_rate": 3e-06,
      "loss": 0.0013,
      "step": 1251
    },
    {
      "clip_ratio": 0.00010683760774554685,
      "epoch": 0.0034758660514494807,
      "grad_norm": 0.1068953201174736,
      "kl": 0.1054844819009304,
      "learning_rate": 3e-06,
      "loss": 0.001,
      "step": 1252
    },
    {
      "clip_ratio": 0.00010416666918899864,
      "epoch": 0.0034786423022892966,
      "grad_norm": 0.1203155517578125,
      "kl": 0.11958234012126923,
      "learning_rate": 3e-06,
      "loss": 0.0009,
      "step": 1253
    },
    {
      "clip_ratio": 0.00018602220370667055,
      "epoch": 0.0034814185531291124,
      "grad_norm": 0.13715901970863342,
      "kl": 0.11173819750547409,
      "learning_rate": 3e-06,
      "loss": 0.0016,
      "step": 1254
    },
    {
      "clip_ratio": 9.231905278284103e-05,
      "epoch": 0.0034841948039689282,
      "grad_norm": 0.14036564528942108,
      "kl": 0.10752937197685242,
      "learning_rate": 3e-06,
      "loss": 0.0002,
      "step": 1255
    },
    {
      "clip_ratio": 0.0004339037259342149,
      "epoch": 0.003486971054808744,
      "grad_norm": 0.1094101294875145,
      "kl": 0.10794253274798393,
      "learning_rate": 3e-06,
      "loss": -0.0001,
      "step": 1256
    },
    {
      "clip_ratio": 0.0002743252844084054,
      "epoch": 0.00348974730564856,
      "grad_norm": 0.1030493676662445,
      "kl": 0.11747561022639275,
      "learning_rate": 3e-06,
      "loss": 0.0004,
      "step": 1257
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0034925235564883758,
      "grad_norm": 0.09440533816814423,
      "kl": 0.11023146659135818,
      "learning_rate": 3e-06,
      "loss": 0.0007,
      "step": 1258
    },
    {
      "clip_ratio": 0.000180825540155638,
      "epoch": 0.0034952998073281916,
      "grad_norm": 0.11672715842723846,
      "kl": 0.1223120354115963,
      "learning_rate": 3e-06,
      "loss": -0.0004,
      "step": 1259
    },
    {
      "clip_ratio": 0.00018602220370667055,
      "epoch": 0.0034980760581680074,
      "grad_norm": 0.14074575901031494,
      "kl": 0.11307071894407272,
      "learning_rate": 3e-06,
      "loss": -0.0001,
      "step": 1260
    },
    {
      "clip_ratio": 0.00016666666488163173,
      "completion_length": 237.3125,
      "epoch": 0.0035008523090078233,
      "grad_norm": 0.10606394708156586,
      "kl": 0.10512935742735863,
      "learning_rate": 3e-06,
      "loss": -0.0002,
      "reward": 0.23125001043081284,
      "reward_std": 0.2202121764421463,
      "rewards/countdown_reward_func": 0.23125001043081284,
      "step": 1261,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio": 9.412650251761079e-05,
      "epoch": 0.0035036285598476396,
      "grad_norm": 0.11450087279081345,
      "kl": 0.10360938310623169,
      "learning_rate": 3e-06,
      "loss": 0.0004,
      "step": 1262
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0035064048106874554,
      "grad_norm": 0.09911279380321503,
      "kl": 0.11000372096896172,
      "learning_rate": 3e-06,
      "loss": -0.0002,
      "step": 1263
    },
    {
      "clip_ratio": 0.00035296654095873237,
      "epoch": 0.0035091810615272712,
      "grad_norm": 0.09363103657960892,
      "kl": 0.10553918406367302,
      "learning_rate": 3e-06,
      "loss": -0.0005,
      "step": 1264
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.003511957312367087,
      "grad_norm": 0.1504822075366974,
      "kl": 0.10735897347331047,
      "learning_rate": 3e-06,
      "loss": 0.0005,
      "step": 1265
    },
    {
      "clip_ratio": 0.00025724637089297175,
      "epoch": 0.003514733563206903,
      "grad_norm": 0.08245550096035004,
      "kl": 0.10223409533500671,
      "learning_rate": 3e-06,
      "loss": 0.0002,
      "step": 1266
    },
    {
      "clip_ratio": 0.00024871622008504346,
      "epoch": 0.0035175098140467187,
      "grad_norm": 0.09745687991380692,
      "kl": 0.1023399755358696,
      "learning_rate": 3e-06,
      "loss": -0.001,
      "step": 1267
    },
    {
      "clip_ratio": 0.0002794273314066231,
      "epoch": 0.0035202860648865346,
      "grad_norm": 0.09371484816074371,
      "kl": 0.09988303855061531,
      "learning_rate": 3e-06,
      "loss": -0.0003,
      "step": 1268
    },
    {
      "clip_ratio": 8.896797226043418e-05,
      "epoch": 0.0035230623157263504,
      "grad_norm": 0.12038465589284897,
      "kl": 0.10891411453485489,
      "learning_rate": 3e-06,
      "loss": -0.0019,
      "step": 1269
    },
    {
      "clip_ratio": 0.0005968022960587405,
      "epoch": 0.0035258385665661663,
      "grad_norm": 0.10312320291996002,
      "kl": 0.1005222424864769,
      "learning_rate": 3e-06,
      "loss": -0.0011,
      "step": 1270
    },
    {
      "clip_ratio": 0.0006010157812852412,
      "epoch": 0.003528614817405982,
      "grad_norm": 0.16399547457695007,
      "kl": 0.10354287177324295,
      "learning_rate": 3e-06,
      "loss": -0.0024,
      "step": 1271
    },
    {
      "clip_ratio": 0.00042980091529898345,
      "epoch": 0.003531391068245798,
      "grad_norm": 0.08301949501037598,
      "kl": 0.09836991503834724,
      "learning_rate": 3e-06,
      "loss": -0.0004,
      "step": 1272
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "completion_length": 243.81250762939453,
      "epoch": 0.0035341673190856142,
      "grad_norm": 0.08151868730783463,
      "kl": 0.09731181338429451,
      "learning_rate": 3e-06,
      "loss": 0.0097,
      "reward": 0.19375000149011612,
      "reward_std": 0.15347465127706528,
      "rewards/countdown_reward_func": 0.19375000149011612,
      "step": 1273,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio": 0.00016276042151730508,
      "epoch": 0.00353694356992543,
      "grad_norm": 0.08378417789936066,
      "kl": 0.09494262933731079,
      "learning_rate": 3e-06,
      "loss": 0.0091,
      "step": 1274
    },
    {
      "clip_ratio": 0.00034852556564146653,
      "epoch": 0.003539719820765246,
      "grad_norm": 0.06453592330217361,
      "kl": 0.09252246841788292,
      "learning_rate": 3e-06,
      "loss": 0.009,
      "step": 1275
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0035424960716050617,
      "grad_norm": 0.059850361198186874,
      "kl": 0.09488331153988838,
      "learning_rate": 3e-06,
      "loss": 0.009,
      "step": 1276
    },
    {
      "clip_ratio": 0.0005926662124693394,
      "epoch": 0.0035452723224448776,
      "grad_norm": 0.08670902252197266,
      "kl": 0.09074336290359497,
      "learning_rate": 3e-06,
      "loss": 0.0087,
      "step": 1277
    },
    {
      "clip_ratio": 0.0004440398042788729,
      "epoch": 0.0035480485732846934,
      "grad_norm": 0.14446234703063965,
      "kl": 0.09088549762964249,
      "learning_rate": 3e-06,
      "loss": 0.0092,
      "step": 1278
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0035508248241245093,
      "grad_norm": 0.07895587384700775,
      "kl": 0.09484604373574257,
      "learning_rate": 3e-06,
      "loss": 0.0086,
      "step": 1279
    },
    {
      "clip_ratio": 0.000244140625,
      "epoch": 0.003553601074964325,
      "grad_norm": 0.07360237091779709,
      "kl": 0.09300564974546432,
      "learning_rate": 3e-06,
      "loss": 0.0093,
      "step": 1280
    },
    {
      "clip_ratio": 0.00032552084303461015,
      "epoch": 0.003556377325804141,
      "grad_norm": 0.06718261539936066,
      "kl": 0.09006435796618462,
      "learning_rate": 3e-06,
      "loss": 0.0091,
      "step": 1281
    },
    {
      "clip_ratio": 0.00025739153352333233,
      "epoch": 0.0035591535766439568,
      "grad_norm": 0.06159327179193497,
      "kl": 0.09368356689810753,
      "learning_rate": 3e-06,
      "loss": 0.0087,
      "step": 1282
    },
    {
      "clip_ratio": 0.00044258125126361847,
      "epoch": 0.0035619298274837726,
      "grad_norm": 0.1282200962305069,
      "kl": 0.08870528638362885,
      "learning_rate": 3e-06,
      "loss": 0.0088,
      "step": 1283
    },
    {
      "clip_ratio": 0.0005040145115344785,
      "epoch": 0.003564706078323589,
      "grad_norm": 0.13320301473140717,
      "kl": 0.0886475220322609,
      "learning_rate": 3e-06,
      "loss": 0.0076,
      "step": 1284
    },
    {
      "clip_ratio": 0.0005532470531761646,
      "completion_length": 223.64584350585938,
      "epoch": 0.0035674823291634047,
      "grad_norm": 0.11603424698114395,
      "kl": 0.08919220045208931,
      "learning_rate": 3e-06,
      "loss": -0.0095,
      "reward": 0.27916668355464935,
      "reward_std": 0.3674810379743576,
      "rewards/countdown_reward_func": 0.27916668355464935,
      "step": 1285,
      "zero_std_ratio": 0.0
    },
    {
      "clip_ratio": 0.0001726210757624358,
      "epoch": 0.0035702585800032206,
      "grad_norm": 0.11266878992319107,
      "kl": 0.09426239505410194,
      "learning_rate": 3e-06,
      "loss": -0.0086,
      "step": 1286
    },
    {
      "clip_ratio": 0.00018248174455948174,
      "epoch": 0.0035730348308430364,
      "grad_norm": 0.1493123471736908,
      "kl": 0.10469094663858414,
      "learning_rate": 3e-06,
      "loss": -0.0097,
      "step": 1287
    },
    {
      "clip_ratio": 9.272996976505965e-05,
      "epoch": 0.0035758110816828522,
      "grad_norm": 0.1302638202905655,
      "kl": 0.09487400203943253,
      "learning_rate": 3e-06,
      "loss": -0.0106,
      "step": 1288
    },
    {
      "clip_ratio": 0.0001050420178216882,
      "epoch": 0.003578587332522668,
      "grad_norm": 0.11855723708868027,
      "kl": 0.09268858656287193,
      "learning_rate": 3e-06,
      "loss": -0.0108,
      "step": 1289
    },
    {
      "clip_ratio": 9.124087227974087e-05,
      "epoch": 0.003581363583362484,
      "grad_norm": 0.0906294658780098,
      "kl": 0.10145439580082893,
      "learning_rate": 3e-06,
      "loss": -0.009,
      "step": 1290
    },
    {
      "clip_ratio": 0.00035833738365909085,
      "epoch": 0.0035841398342022998,
      "grad_norm": 0.10448364913463593,
      "kl": 0.0870097205042839,
      "learning_rate": 3e-06,
      "loss": -0.0114,
      "step": 1291
    },
    {
      "clip_ratio": 0.0001726210757624358,
      "epoch": 0.0035869160850421156,
      "grad_norm": 0.12192343175411224,
      "kl": 0.09259336069226265,
      "learning_rate": 3e-06,
      "loss": -0.0107,
      "step": 1292
    },
    {
      "clip_ratio": 0.00018248174455948174,
      "epoch": 0.0035896923358819314,
      "grad_norm": 0.12048960477113724,
      "kl": 0.10295253619551659,
      "learning_rate": 3e-06,
      "loss": -0.0115,
      "step": 1293
    },
    {
      "clip_ratio": 9.272996976505965e-05,
      "epoch": 0.0035924685867217473,
      "grad_norm": 0.12977741658687592,
      "kl": 0.0927329771220684,
      "learning_rate": 3e-06,
      "loss": -0.0141,
      "step": 1294
    },
    {
      "clip_ratio": 0.0003834426242974587,
      "epoch": 0.0035952448375615636,
      "grad_norm": 0.1129441037774086,
      "kl": 0.09166432544589043,
      "learning_rate": 3e-06,
      "loss": -0.0121,
      "step": 1295
    },
    {
      "clip_ratio": 0.0004474218876566738,
      "epoch": 0.0035980210884013794,
      "grad_norm": 0.10428842157125473,
      "kl": 0.10321545228362083,
      "learning_rate": 3e-06,
      "loss": -0.0114,
      "step": 1296
    },
    {
      "clip_ratio": 0.00017595020472072065,
      "completion_length": 240.0625,
      "epoch": 0.0036007973392411952,
      "grad_norm": 0.11313264816999435,
      "kl": 0.08579898625612259,
      "learning_rate": 3e-06,
      "loss": 0.0105,
      "reward": 0.18958333879709244,
      "reward_std": 0.20279134064912796,
      "rewards/countdown_reward_func": 0.18958333134651184,
      "step": 1297,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.00016954136663116515,
      "epoch": 0.003603573590081011,
      "grad_norm": 0.08754347264766693,
      "kl": 0.07732414454221725,
      "learning_rate": 3e-06,
      "loss": 0.0108,
      "step": 1298
    },
    {
      "clip_ratio": 0.00021079258294776082,
      "epoch": 0.003606349840920827,
      "grad_norm": 0.09487965703010559,
      "kl": 0.07977383211255074,
      "learning_rate": 3e-06,
      "loss": 0.0106,
      "step": 1299
    },
    {
      "clip_ratio": 8.468834857922047e-05,
      "epoch": 0.0036091260917606427,
      "grad_norm": 0.08399857580661774,
      "kl": 0.08700349926948547,
      "learning_rate": 3e-06,
      "loss": 0.0118,
      "step": 1300
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0036119023426004586,
      "grad_norm": 0.06855960935354233,
      "kl": 0.07815397530794144,
      "learning_rate": 3e-06,
      "loss": 0.0115,
      "step": 1301
    },
    {
      "clip_ratio": 0.0002462810225551948,
      "epoch": 0.0036146785934402744,
      "grad_norm": 0.0689474567770958,
      "kl": 0.07884856685996056,
      "learning_rate": 3e-06,
      "loss": 0.0115,
      "step": 1302
    },
    {
      "clip_ratio": 0.00018677650223253295,
      "epoch": 0.0036174548442800903,
      "grad_norm": 0.10885361582040787,
      "kl": 0.08625783771276474,
      "learning_rate": 3e-06,
      "loss": 0.0108,
      "step": 1303
    },
    {
      "clip_ratio": 0.00017632621165830642,
      "epoch": 0.003620231095119906,
      "grad_norm": 0.07666079699993134,
      "kl": 0.07669586688280106,
      "learning_rate": 3e-06,
      "loss": 0.01,
      "step": 1304
    },
    {
      "clip_ratio": 0.0002925453591160476,
      "epoch": 0.003623007345959722,
      "grad_norm": 0.08154989778995514,
      "kl": 0.07938069105148315,
      "learning_rate": 3e-06,
      "loss": 0.01,
      "step": 1305
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0036257835967995382,
      "grad_norm": 0.0733746886253357,
      "kl": 0.08602719753980637,
      "learning_rate": 3e-06,
      "loss": 0.0106,
      "step": 1306
    },
    {
      "clip_ratio": 0.0002507569151930511,
      "epoch": 0.003628559847639354,
      "grad_norm": 0.06607773900032043,
      "kl": 0.07838872075080872,
      "learning_rate": 3e-06,
      "loss": 0.0107,
      "step": 1307
    },
    {
      "clip_ratio": 0.0004108092689421028,
      "epoch": 0.00363133609847917,
      "grad_norm": 0.07465820014476776,
      "kl": 0.07868605107069016,
      "learning_rate": 3e-06,
      "loss": 0.0115,
      "step": 1308
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "completion_length": 249.0416717529297,
      "epoch": 0.0036341123493189857,
      "grad_norm": 0.10679054260253906,
      "kl": 0.07724452763795853,
      "learning_rate": 3e-06,
      "loss": -0.0055,
      "reward": 0.32500002533197403,
      "reward_std": 0.27318819612264633,
      "rewards/countdown_reward_func": 0.32499999552965164,
      "step": 1309,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 9.645061800256371e-05,
      "epoch": 0.0036368886001588016,
      "grad_norm": 0.08431645482778549,
      "kl": 0.07454115152359009,
      "learning_rate": 3e-06,
      "loss": -0.0059,
      "step": 1310
    },
    {
      "clip_ratio": 0.00041751094977371395,
      "epoch": 0.0036396648509986174,
      "grad_norm": 0.10150210559368134,
      "kl": 0.07631824165582657,
      "learning_rate": 3e-06,
      "loss": -0.0058,
      "step": 1311
    },
    {
      "clip_ratio": 0.00016276042151730508,
      "epoch": 0.0036424411018384333,
      "grad_norm": 0.08061017096042633,
      "kl": 0.07825561240315437,
      "learning_rate": 3e-06,
      "loss": -0.0056,
      "step": 1312
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.003645217352678249,
      "grad_norm": 0.1347588747739792,
      "kl": 0.07661932334303856,
      "learning_rate": 3e-06,
      "loss": -0.0053,
      "step": 1313
    },
    {
      "clip_ratio": 0.00016276042151730508,
      "epoch": 0.003647993603518065,
      "grad_norm": 0.09667013585567474,
      "kl": 0.07734496891498566,
      "learning_rate": 3e-06,
      "loss": -0.0057,
      "step": 1314
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0036507698543578808,
      "grad_norm": 0.11161034554243088,
      "kl": 0.07669401913881302,
      "learning_rate": 3e-06,
      "loss": -0.0062,
      "step": 1315
    },
    {
      "clip_ratio": 0.00016276042151730508,
      "epoch": 0.0036535461051976966,
      "grad_norm": 0.08601905405521393,
      "kl": 0.07428323850035667,
      "learning_rate": 3e-06,
      "loss": -0.0066,
      "step": 1316
    },
    {
      "clip_ratio": 0.00032552084303461015,
      "epoch": 0.003656322356037513,
      "grad_norm": 0.11961396038532257,
      "kl": 0.07602384686470032,
      "learning_rate": 3e-06,
      "loss": -0.0068,
      "step": 1317
    },
    {
      "clip_ratio": 0.0009102527255890891,
      "epoch": 0.0036590986068773287,
      "grad_norm": 0.08503416925668716,
      "kl": 0.0780373215675354,
      "learning_rate": 3e-06,
      "loss": -0.0059,
      "step": 1318
    },
    {
      "clip_ratio": 0.000244140625,
      "epoch": 0.0036618748577171446,
      "grad_norm": 0.13413764536380768,
      "kl": 0.07871726900339127,
      "learning_rate": 3e-06,
      "loss": -0.0072,
      "step": 1319
    },
    {
      "clip_ratio": 0.00017783083603717387,
      "epoch": 0.0036646511085569604,
      "grad_norm": 0.16159453988075256,
      "kl": 0.07949946075677872,
      "learning_rate": 3e-06,
      "loss": -0.0074,
      "step": 1320
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 224.3541717529297,
      "epoch": 0.0036674273593967762,
      "grad_norm": 0.12504050135612488,
      "kl": 0.08029628917574883,
      "learning_rate": 3e-06,
      "loss": 0.0344,
      "reward": 0.3583333492279053,
      "reward_std": 0.3268071115016937,
      "rewards/countdown_reward_func": 0.3583333492279053,
      "step": 1321,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.0005550031710299663,
      "epoch": 0.003670203610236592,
      "grad_norm": 0.14056633412837982,
      "kl": 0.08753347396850586,
      "learning_rate": 3e-06,
      "loss": 0.0343,
      "step": 1322
    },
    {
      "clip_ratio": 0.0009008992274175398,
      "epoch": 0.003672979861076408,
      "grad_norm": 0.1351320594549179,
      "kl": 0.0846715196967125,
      "learning_rate": 3e-06,
      "loss": 0.0351,
      "step": 1323
    },
    {
      "clip_ratio": 0.00020169082563370466,
      "epoch": 0.0036757561119162238,
      "grad_norm": 0.16348907351493835,
      "kl": 0.07923517376184464,
      "learning_rate": 3e-06,
      "loss": 0.0341,
      "step": 1324
    },
    {
      "clip_ratio": 0.0011630582448560745,
      "epoch": 0.0036785323627560396,
      "grad_norm": 0.1238764300942421,
      "kl": 0.07697004824876785,
      "learning_rate": 3e-06,
      "loss": 0.0332,
      "step": 1325
    },
    {
      "clip_ratio": 0.00018680757784750313,
      "epoch": 0.0036813086135958554,
      "grad_norm": 0.13604602217674255,
      "kl": 0.08331404998898506,
      "learning_rate": 3e-06,
      "loss": 0.0335,
      "step": 1326
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0036840848644356713,
      "grad_norm": 0.12275619804859161,
      "kl": 0.08377550542354584,
      "learning_rate": 3e-06,
      "loss": 0.0324,
      "step": 1327
    },
    {
      "clip_ratio": 0.0008797913324087858,
      "epoch": 0.0036868611152754876,
      "grad_norm": 0.13052096962928772,
      "kl": 0.09020379930734634,
      "learning_rate": 3e-06,
      "loss": 0.032,
      "step": 1328
    },
    {
      "clip_ratio": 0.0005085435113869607,
      "epoch": 0.0036896373661153034,
      "grad_norm": 0.1253458559513092,
      "kl": 0.0886102207005024,
      "learning_rate": 3e-06,
      "loss": 0.0316,
      "step": 1329
    },
    {
      "clip_ratio": 0.000303602428175509,
      "epoch": 0.0036924136169551192,
      "grad_norm": 0.16655124723911285,
      "kl": 0.08304303884506226,
      "learning_rate": 3e-06,
      "loss": 0.0295,
      "step": 1330
    },
    {
      "clip_ratio": 0.0007008319735177793,
      "epoch": 0.003695189867794935,
      "grad_norm": 0.11321963369846344,
      "kl": 0.08307855576276779,
      "learning_rate": 3e-06,
      "loss": 0.0305,
      "step": 1331
    },
    {
      "clip_ratio": 0.0001017087051877752,
      "epoch": 0.003697966118634751,
      "grad_norm": 0.12860439717769623,
      "kl": 0.09165726974606514,
      "learning_rate": 3e-06,
      "loss": 0.0293,
      "step": 1332
    },
    {
      "clip_ratio": 0.00017229790682904422,
      "completion_length": 233.58333587646484,
      "epoch": 0.0037007423694745667,
      "grad_norm": 0.12028127908706665,
      "kl": 0.10655560344457626,
      "learning_rate": 3e-06,
      "loss": 0.0049,
      "reward": 0.2666666731238365,
      "reward_std": 0.2709502577781677,
      "rewards/countdown_reward_func": 0.2666666731238365,
      "step": 1333,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0037035186203143826,
      "grad_norm": 0.09744270890951157,
      "kl": 0.11018062755465508,
      "learning_rate": 3e-06,
      "loss": 0.0061,
      "step": 1334
    },
    {
      "clip_ratio": 8.579272252973169e-05,
      "epoch": 0.0037062948711541984,
      "grad_norm": 0.09601444751024246,
      "kl": 0.10575228929519653,
      "learning_rate": 3e-06,
      "loss": 0.0058,
      "step": 1335
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0037090711219940143,
      "grad_norm": 0.11352216452360153,
      "kl": 0.10562450438737869,
      "learning_rate": 3e-06,
      "loss": 0.0053,
      "step": 1336
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.00371184737283383,
      "grad_norm": 0.0934673398733139,
      "kl": 0.11037517711520195,
      "learning_rate": 3e-06,
      "loss": 0.0042,
      "step": 1337
    },
    {
      "clip_ratio": 0.0005623652250505984,
      "epoch": 0.0037146236236736464,
      "grad_norm": 0.09364868700504303,
      "kl": 0.10772473365068436,
      "learning_rate": 3e-06,
      "loss": 0.0057,
      "step": 1338
    },
    {
      "clip_ratio": 9.377344395034015e-05,
      "epoch": 0.0037173998745134622,
      "grad_norm": 0.2115117907524109,
      "kl": 0.12028898671269417,
      "learning_rate": 3e-06,
      "loss": 0.0033,
      "step": 1339
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.003720176125353278,
      "grad_norm": 0.10891567170619965,
      "kl": 0.12114686891436577,
      "learning_rate": 3e-06,
      "loss": 0.0054,
      "step": 1340
    },
    {
      "clip_ratio": 8.650519157527015e-05,
      "epoch": 0.003722952376193094,
      "grad_norm": 0.09488851577043533,
      "kl": 0.11187062785029411,
      "learning_rate": 3e-06,
      "loss": 0.0057,
      "step": 1341
    },
    {
      "clip_ratio": 0.000176635745447129,
      "epoch": 0.0037257286270329097,
      "grad_norm": 0.1026405617594719,
      "kl": 0.10968057066202164,
      "learning_rate": 3e-06,
      "loss": 0.0054,
      "step": 1342
    },
    {
      "clip_ratio": 0.0003423265879973769,
      "epoch": 0.0037285048778727256,
      "grad_norm": 0.10193421691656113,
      "kl": 0.11749966815114021,
      "learning_rate": 3e-06,
      "loss": 0.0035,
      "step": 1343
    },
    {
      "clip_ratio": 0.0011813296005129814,
      "epoch": 0.0037312811287125414,
      "grad_norm": 0.09785063564777374,
      "kl": 0.10901345312595367,
      "learning_rate": 3e-06,
      "loss": 0.0051,
      "step": 1344
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "completion_length": 224.70834350585938,
      "epoch": 0.0037340573795523573,
      "grad_norm": 0.15201179683208466,
      "kl": 0.11200324073433876,
      "learning_rate": 3e-06,
      "loss": 0.0035,
      "reward": 0.33750002086162567,
      "reward_std": 0.29471276700496674,
      "rewards/countdown_reward_func": 0.33750002086162567,
      "step": 1345,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.003736833630392173,
      "grad_norm": 0.12798374891281128,
      "kl": 0.11272676661610603,
      "learning_rate": 3e-06,
      "loss": 0.0036,
      "step": 1346
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.003739609881231989,
      "grad_norm": 0.12021691352128983,
      "kl": 0.11317825317382812,
      "learning_rate": 3e-06,
      "loss": 0.0036,
      "step": 1347
    },
    {
      "clip_ratio": 9.448223863728344e-05,
      "epoch": 0.0037423861320718048,
      "grad_norm": 0.11852817982435226,
      "kl": 0.11661011353135109,
      "learning_rate": 3e-06,
      "loss": 0.0027,
      "step": 1348
    },
    {
      "clip_ratio": 0.00016276042151730508,
      "epoch": 0.003745162382911621,
      "grad_norm": 0.15811295807361603,
      "kl": 0.1139235608279705,
      "learning_rate": 3e-06,
      "loss": 0.0029,
      "step": 1349
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.003747938633751437,
      "grad_norm": 0.14151591062545776,
      "kl": 0.10737059265375137,
      "learning_rate": 3e-06,
      "loss": 0.0033,
      "step": 1350
    },
    {
      "clip_ratio": 0.0002975761817651801,
      "epoch": 0.0037507148845912527,
      "grad_norm": 0.11291375011205673,
      "kl": 0.10967541486024857,
      "learning_rate": 3e-06,
      "loss": 0.0032,
      "step": 1351
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0037534911354310686,
      "grad_norm": 0.1173899918794632,
      "kl": 0.10674367100000381,
      "learning_rate": 3e-06,
      "loss": 0.0013,
      "step": 1352
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0037562673862708844,
      "grad_norm": 0.13352568447589874,
      "kl": 0.10739367455244064,
      "learning_rate": 3e-06,
      "loss": 0.0019,
      "step": 1353
    },
    {
      "clip_ratio": 8.229097875300795e-05,
      "epoch": 0.0037590436371107002,
      "grad_norm": 0.10833993554115295,
      "kl": 0.11009395867586136,
      "learning_rate": 3e-06,
      "loss": 0.0012,
      "step": 1354
    },
    {
      "clip_ratio": 0.0004950694419676438,
      "epoch": 0.003761819887950516,
      "grad_norm": 0.1272728592157364,
      "kl": 0.106673963367939,
      "learning_rate": 3e-06,
      "loss": 0.0015,
      "step": 1355
    },
    {
      "clip_ratio": 0.0003583999059628695,
      "epoch": 0.003764596138790332,
      "grad_norm": 0.13600032031536102,
      "kl": 0.10148491337895393,
      "learning_rate": 3e-06,
      "loss": 0.0016,
      "step": 1356
    },
    {
      "clip_ratio": 0.00035957014188170433,
      "completion_length": 235.6875,
      "epoch": 0.0037673723896301478,
      "grad_norm": 0.1315588802099228,
      "kl": 0.11271170154213905,
      "learning_rate": 3e-06,
      "loss": 0.0092,
      "reward": 0.2854166775941849,
      "reward_std": 0.23748210817575455,
      "rewards/countdown_reward_func": 0.2854166775941849,
      "step": 1357,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 8.37240440887399e-05,
      "epoch": 0.0037701486404699636,
      "grad_norm": 0.10195645689964294,
      "kl": 0.11273347586393356,
      "learning_rate": 3e-06,
      "loss": 0.0096,
      "step": 1358
    },
    {
      "clip_ratio": 0.000245323171839118,
      "epoch": 0.0037729248913097794,
      "grad_norm": 0.10797201097011566,
      "kl": 0.10941316187381744,
      "learning_rate": 3e-06,
      "loss": 0.0089,
      "step": 1359
    },
    {
      "clip_ratio": 0.00018649760022526607,
      "epoch": 0.0037757011421495957,
      "grad_norm": 0.092195063829422,
      "kl": 0.10438445582985878,
      "learning_rate": 3e-06,
      "loss": 0.0092,
      "step": 1360
    },
    {
      "clip_ratio": 0.0003678085922729224,
      "epoch": 0.0037784773929894115,
      "grad_norm": 0.1453506350517273,
      "kl": 0.10365356132388115,
      "learning_rate": 3e-06,
      "loss": 0.0098,
      "step": 1361
    },
    {
      "clip_ratio": 0.00017496491636848077,
      "epoch": 0.0037812536438292274,
      "grad_norm": 0.12429957091808319,
      "kl": 0.10129589587450027,
      "learning_rate": 3e-06,
      "loss": 0.0095,
      "step": 1362
    },
    {
      "clip_ratio": 9.272996976505965e-05,
      "epoch": 0.0037840298946690432,
      "grad_norm": 0.2589426636695862,
      "kl": 0.10026075318455696,
      "learning_rate": 3e-06,
      "loss": 0.008,
      "step": 1363
    },
    {
      "clip_ratio": 0.00017496491636848077,
      "epoch": 0.003786806145508859,
      "grad_norm": 0.09787080436944962,
      "kl": 0.09990701079368591,
      "learning_rate": 3e-06,
      "loss": 0.009,
      "step": 1364
    },
    {
      "clip_ratio": 0.0006807727040722966,
      "epoch": 0.003789582396348675,
      "grad_norm": 0.1061621904373169,
      "kl": 0.09625846892595291,
      "learning_rate": 3e-06,
      "loss": 0.0082,
      "step": 1365
    },
    {
      "clip_ratio": 0.0002560440043453127,
      "epoch": 0.0037923586471884907,
      "grad_norm": 0.09652417153120041,
      "kl": 0.0922749936580658,
      "learning_rate": 3e-06,
      "loss": 0.0085,
      "step": 1366
    },
    {
      "clip_ratio": 0.0006315374630503356,
      "epoch": 0.0037951348980283066,
      "grad_norm": 0.1492643803358078,
      "kl": 0.09287701919674873,
      "learning_rate": 3e-06,
      "loss": 0.0089,
      "step": 1367
    },
    {
      "clip_ratio": 0.0005454363708849996,
      "epoch": 0.0037979111488681224,
      "grad_norm": 0.12842383980751038,
      "kl": 0.0905197411775589,
      "learning_rate": 3e-06,
      "loss": 0.0081,
      "step": 1368
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 230.39583587646484,
      "epoch": 0.0038006873997079383,
      "grad_norm": 0.11756516993045807,
      "kl": 0.08745963126420975,
      "learning_rate": 3e-06,
      "loss": -0.01,
      "reward": 0.3229166865348816,
      "reward_std": 0.29132401943206787,
      "rewards/countdown_reward_func": 0.3229166865348816,
      "step": 1369,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.00017399911303073168,
      "epoch": 0.003803463650547754,
      "grad_norm": 0.12292972207069397,
      "kl": 0.09213906899094582,
      "learning_rate": 3e-06,
      "loss": -0.0102,
      "step": 1370
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0038062399013875704,
      "grad_norm": 0.10542229562997818,
      "kl": 0.08154939487576485,
      "learning_rate": 3e-06,
      "loss": -0.0099,
      "step": 1371
    },
    {
      "clip_ratio": 0.0002737229297054,
      "epoch": 0.003809016152227386,
      "grad_norm": 0.13350439071655273,
      "kl": 0.08433039486408234,
      "learning_rate": 3e-06,
      "loss": -0.0102,
      "step": 1372
    },
    {
      "clip_ratio": 0.000268786505330354,
      "epoch": 0.003811792403067202,
      "grad_norm": 0.11377749592065811,
      "kl": 0.08954215422272682,
      "learning_rate": 3e-06,
      "loss": -0.0088,
      "step": 1373
    },
    {
      "clip_ratio": 0.00017508336168248206,
      "epoch": 0.003814568653907018,
      "grad_norm": 0.1519603431224823,
      "kl": 0.07980604469776154,
      "learning_rate": 3e-06,
      "loss": -0.0121,
      "step": 1374
    },
    {
      "clip_ratio": 0.00018208303663413972,
      "epoch": 0.0038173449047468337,
      "grad_norm": 0.11631568521261215,
      "kl": 0.07712772488594055,
      "learning_rate": 3e-06,
      "loss": -0.0112,
      "step": 1375
    },
    {
      "clip_ratio": 0.00018214939336758107,
      "epoch": 0.0038201211555866496,
      "grad_norm": 0.10698658227920532,
      "kl": 0.08292505145072937,
      "learning_rate": 3e-06,
      "loss": -0.0116,
      "step": 1376
    },
    {
      "clip_ratio": 0.0005369195641833358,
      "epoch": 0.0038228974064264654,
      "grad_norm": 0.1011744886636734,
      "kl": 0.07199421525001526,
      "learning_rate": 3e-06,
      "loss": -0.0121,
      "step": 1377
    },
    {
      "clip_ratio": 0.000721444986993447,
      "epoch": 0.0038256736572662813,
      "grad_norm": 0.11945533007383347,
      "kl": 0.07399719953536987,
      "learning_rate": 3e-06,
      "loss": -0.0122,
      "step": 1378
    },
    {
      "clip_ratio": 0.0008790974534349516,
      "epoch": 0.003828449908106097,
      "grad_norm": 0.11251705139875412,
      "kl": 0.07932459190487862,
      "learning_rate": 3e-06,
      "loss": -0.0119,
      "step": 1379
    },
    {
      "clip_ratio": 0.000354837131453678,
      "epoch": 0.003831226158945913,
      "grad_norm": 0.15868762135505676,
      "kl": 0.07149028778076172,
      "learning_rate": 3e-06,
      "loss": -0.0146,
      "step": 1380
    },
    {
      "clip_ratio": 0.00017755682347342372,
      "completion_length": 225.33333587646484,
      "epoch": 0.0038340024097857288,
      "grad_norm": 0.11194837838411331,
      "kl": 0.07465841248631477,
      "learning_rate": 3e-06,
      "loss": -0.0171,
      "reward": 0.28541669249534607,
      "reward_std": 0.31687821447849274,
      "rewards/countdown_reward_func": 0.2854166850447655,
      "step": 1381,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.0005127792246639729,
      "epoch": 0.003836778660625545,
      "grad_norm": 0.1955379843711853,
      "kl": 0.07209612429141998,
      "learning_rate": 3e-06,
      "loss": -0.0169,
      "step": 1382
    },
    {
      "clip_ratio": 8.877841173671186e-05,
      "epoch": 0.003839554911465361,
      "grad_norm": 0.11438043415546417,
      "kl": 0.06967338174581528,
      "learning_rate": 3e-06,
      "loss": -0.0187,
      "step": 1383
    },
    {
      "clip_ratio": 0.00017607717745704576,
      "epoch": 0.0038423311623051767,
      "grad_norm": 0.11129593104124069,
      "kl": 0.07491909712553024,
      "learning_rate": 3e-06,
      "loss": -0.0179,
      "step": 1384
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0038451074131449926,
      "grad_norm": 0.0992979034781456,
      "kl": 0.07310023903846741,
      "learning_rate": 3e-06,
      "loss": -0.0178,
      "step": 1385
    },
    {
      "clip_ratio": 0.0002843922993633896,
      "epoch": 0.0038478836639848084,
      "grad_norm": 0.12313024699687958,
      "kl": 0.0734870582818985,
      "learning_rate": 3e-06,
      "loss": -0.0174,
      "step": 1386
    },
    {
      "clip_ratio": 0.0005009273590985686,
      "epoch": 0.0038506599148246242,
      "grad_norm": 0.12103430181741714,
      "kl": 0.06940187513828278,
      "learning_rate": 3e-06,
      "loss": -0.0189,
      "step": 1387
    },
    {
      "clip_ratio": 0.0008541795250494033,
      "epoch": 0.00385343616566444,
      "grad_norm": 0.20041851699352264,
      "kl": 0.06606732122600079,
      "learning_rate": 3e-06,
      "loss": -0.0202,
      "step": 1388
    },
    {
      "clip_ratio": 0.00030615586729254574,
      "epoch": 0.003856212416504256,
      "grad_norm": 0.09863422811031342,
      "kl": 0.06485064327716827,
      "learning_rate": 3e-06,
      "loss": -0.0203,
      "step": 1389
    },
    {
      "clip_ratio": 0.0005214700649958104,
      "epoch": 0.0038589886673440718,
      "grad_norm": 0.11281978338956833,
      "kl": 0.07150644063949585,
      "learning_rate": 3e-06,
      "loss": -0.021,
      "step": 1390
    },
    {
      "clip_ratio": 0.0021090117224957794,
      "epoch": 0.0038617649181838876,
      "grad_norm": 0.09994053095579147,
      "kl": 0.07051489502191544,
      "learning_rate": 3e-06,
      "loss": -0.0197,
      "step": 1391
    },
    {
      "clip_ratio": 0.0022545086685568094,
      "epoch": 0.0038645411690237034,
      "grad_norm": 0.12088467925786972,
      "kl": 0.07186911255121231,
      "learning_rate": 3e-06,
      "loss": -0.0191,
      "step": 1392
    },
    {
      "clip_ratio": 0.0001826150546548888,
      "completion_length": 240.52083587646484,
      "epoch": 0.0038673174198635197,
      "grad_norm": 0.0785083919763565,
      "kl": 0.06684068590402603,
      "learning_rate": 3e-06,
      "loss": 0.0088,
      "reward": 0.21250000596046448,
      "reward_std": 0.2080453708767891,
      "rewards/countdown_reward_func": 0.21250000596046448,
      "step": 1393,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio": 0.000263084439211525,
      "epoch": 0.0038700936707033355,
      "grad_norm": 0.07907012850046158,
      "kl": 0.062989991158247,
      "learning_rate": 3e-06,
      "loss": 0.0086,
      "step": 1394
    },
    {
      "clip_ratio": 0.0002763880111160688,
      "epoch": 0.0038728699215431514,
      "grad_norm": 0.070924311876297,
      "kl": 0.07118378207087517,
      "learning_rate": 3e-06,
      "loss": 0.0093,
      "step": 1395
    },
    {
      "clip_ratio": 0.0004090342263225466,
      "epoch": 0.0038756461723829672,
      "grad_norm": 0.06483456492424011,
      "kl": 0.062327247112989426,
      "learning_rate": 3e-06,
      "loss": 0.0086,
      "step": 1396
    },
    {
      "clip_ratio": 0.0008118404075503349,
      "epoch": 0.003878422423222783,
      "grad_norm": 0.06826819479465485,
      "kl": 0.06643109023571014,
      "learning_rate": 3e-06,
      "loss": 0.009,
      "step": 1397
    },
    {
      "clip_ratio": 0.00035210512578487396,
      "epoch": 0.003881198674062599,
      "grad_norm": 0.07230556011199951,
      "kl": 0.0697788167744875,
      "learning_rate": 3e-06,
      "loss": 0.009,
      "step": 1398
    },
    {
      "clip_ratio": 0.0006263371469685808,
      "epoch": 0.0038839749249024147,
      "grad_norm": 0.08306799083948135,
      "kl": 0.06783925369381905,
      "learning_rate": 3e-06,
      "loss": 0.009,
      "step": 1399
    },
    {
      "clip_ratio": 0.00025315712264273316,
      "epoch": 0.0038867511757422306,
      "grad_norm": 0.08525695651769638,
      "kl": 0.06320414319634438,
      "learning_rate": 3e-06,
      "loss": 0.0081,
      "step": 1400
    },
    {
      "clip_ratio": 0.00045900307304691523,
      "epoch": 0.0038895274265820464,
      "grad_norm": 0.09048361331224442,
      "kl": 0.07014712691307068,
      "learning_rate": 3e-06,
      "loss": 0.0092,
      "step": 1401
    },
    {
      "clip_ratio": 0.0009449293720535934,
      "epoch": 0.0038923036774218623,
      "grad_norm": 0.06213811784982681,
      "kl": 0.06285431608557701,
      "learning_rate": 3e-06,
      "loss": 0.0082,
      "step": 1402
    },
    {
      "clip_ratio": 0.0002441406322759576,
      "epoch": 0.003895079928261678,
      "grad_norm": 0.061364803463220596,
      "kl": 0.0679735615849495,
      "learning_rate": 3e-06,
      "loss": 0.009,
      "step": 1403
    },
    {
      "clip_ratio": 0.0007923201483208686,
      "epoch": 0.0038978561791014944,
      "grad_norm": 0.07901846617460251,
      "kl": 0.06937158480286598,
      "learning_rate": 3e-06,
      "loss": 0.0082,
      "step": 1404
    },
    {
      "clip_ratio": 0.00017844396643340588,
      "completion_length": 237.95833587646484,
      "epoch": 0.00390063242994131,
      "grad_norm": 0.09061950445175171,
      "kl": 0.06311015225946903,
      "learning_rate": 3e-06,
      "loss": 0.0072,
      "reward": 0.2083333507180214,
      "reward_std": 0.21443458646535873,
      "rewards/countdown_reward_func": 0.2083333432674408,
      "step": 1405,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.00017060219397535548,
      "epoch": 0.003903408680781126,
      "grad_norm": 0.06206053867936134,
      "kl": 0.055555472150444984,
      "learning_rate": 3e-06,
      "loss": 0.007,
      "step": 1406
    },
    {
      "clip_ratio": 9.266123379347846e-05,
      "epoch": 0.003906184931620942,
      "grad_norm": 0.10189791768789291,
      "kl": 0.05937515199184418,
      "learning_rate": 3e-06,
      "loss": 0.0071,
      "step": 1407
    },
    {
      "clip_ratio": 0.0004294630925869569,
      "epoch": 0.003908961182460757,
      "grad_norm": 0.07696424424648285,
      "kl": 0.060661833733320236,
      "learning_rate": 3e-06,
      "loss": 0.0079,
      "step": 1408
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.003911737433300574,
      "grad_norm": 0.0714978277683258,
      "kl": 0.06452694535255432,
      "learning_rate": 3e-06,
      "loss": 0.0071,
      "step": 1409
    },
    {
      "clip_ratio": 0.000443236087448895,
      "epoch": 0.00391451368414039,
      "grad_norm": 0.1540229171514511,
      "kl": 0.06495703011751175,
      "learning_rate": 3e-06,
      "loss": 0.007,
      "step": 1410
    },
    {
      "clip_ratio": 9.266123379347846e-05,
      "epoch": 0.003917289934980205,
      "grad_norm": 0.09125962853431702,
      "kl": 0.06371764466166496,
      "learning_rate": 3e-06,
      "loss": 0.0066,
      "step": 1411
    },
    {
      "clip_ratio": 0.0002598241771920584,
      "epoch": 0.0039200661858200215,
      "grad_norm": 0.0668187364935875,
      "kl": 0.05546574853360653,
      "learning_rate": 3e-06,
      "loss": 0.0062,
      "step": 1412
    },
    {
      "clip_ratio": 9.266123379347846e-05,
      "epoch": 0.003922842436659837,
      "grad_norm": 0.1012551337480545,
      "kl": 0.05864310637116432,
      "learning_rate": 3e-06,
      "loss": 0.0062,
      "step": 1413
    },
    {
      "clip_ratio": 9.266123379347846e-05,
      "epoch": 0.003925618687499653,
      "grad_norm": 0.07103273272514343,
      "kl": 0.06015484221279621,
      "learning_rate": 3e-06,
      "loss": 0.0071,
      "step": 1414
    },
    {
      "clip_ratio": 0.0005025817663408816,
      "epoch": 0.003928394938339469,
      "grad_norm": 0.06943207234144211,
      "kl": 0.06314868479967117,
      "learning_rate": 3e-06,
      "loss": 0.0062,
      "step": 1415
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.003931171189179285,
      "grad_norm": 0.10475575923919678,
      "kl": 0.06454218924045563,
      "learning_rate": 3e-06,
      "loss": 0.0056,
      "step": 1416
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 236.2291717529297,
      "epoch": 0.0039339474400191,
      "grad_norm": 0.0840207114815712,
      "kl": 0.06996531412005424,
      "learning_rate": 3e-06,
      "loss": -0.0023,
      "reward": 0.24583332985639572,
      "reward_std": 0.3068140149116516,
      "rewards/countdown_reward_func": 0.24583332985639572,
      "step": 1417,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 0.000174452448845841,
      "epoch": 0.0039367236908589166,
      "grad_norm": 0.09196259081363678,
      "kl": 0.08129844069480896,
      "learning_rate": 3e-06,
      "loss": -0.0026,
      "step": 1418
    },
    {
      "clip_ratio": 0.0006801302515668795,
      "epoch": 0.003939499941698732,
      "grad_norm": 0.08758815377950668,
      "kl": 0.0710601955652237,
      "learning_rate": 3e-06,
      "loss": -0.0027,
      "step": 1419
    },
    {
      "clip_ratio": 0.0006429070708691142,
      "epoch": 0.003942276192538548,
      "grad_norm": 0.07932908087968826,
      "kl": 0.06780187785625458,
      "learning_rate": 3e-06,
      "loss": -0.0034,
      "step": 1420
    },
    {
      "clip_ratio": 0.00027745863917516544,
      "epoch": 0.0039450524433783645,
      "grad_norm": 0.10550674796104431,
      "kl": 0.0699809230864048,
      "learning_rate": 3e-06,
      "loss": -0.003,
      "step": 1421
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00394782869421818,
      "grad_norm": 0.1297944188117981,
      "kl": 0.0661407969892025,
      "learning_rate": 3e-06,
      "loss": -0.0031,
      "step": 1422
    },
    {
      "clip_ratio": 0.00017994409427046776,
      "epoch": 0.003950604945057996,
      "grad_norm": 0.08687513321638107,
      "kl": 0.06938713788986206,
      "learning_rate": 3e-06,
      "loss": -0.0033,
      "step": 1423
    },
    {
      "clip_ratio": 8.185985643649474e-05,
      "epoch": 0.003953381195897812,
      "grad_norm": 0.09864569455385208,
      "kl": 0.08142375946044922,
      "learning_rate": 3e-06,
      "loss": -0.0026,
      "step": 1424
    },
    {
      "clip_ratio": 0.00047662161523476243,
      "epoch": 0.003956157446737628,
      "grad_norm": 0.10956618189811707,
      "kl": 0.07189195230603218,
      "learning_rate": 3e-06,
      "loss": -0.0031,
      "step": 1425
    },
    {
      "clip_ratio": 0.000174703003722243,
      "epoch": 0.003958933697577443,
      "grad_norm": 0.08014890551567078,
      "kl": 0.0699024386703968,
      "learning_rate": 3e-06,
      "loss": -0.0037,
      "step": 1426
    },
    {
      "clip_ratio": 0.0002667709268280305,
      "epoch": 0.0039617099484172595,
      "grad_norm": 0.10431994497776031,
      "kl": 0.07111196964979172,
      "learning_rate": 3e-06,
      "loss": -0.0033,
      "step": 1427
    },
    {
      "clip_ratio": 8.73515018611215e-05,
      "epoch": 0.003964486199257075,
      "grad_norm": 0.13337118923664093,
      "kl": 0.06857022643089294,
      "learning_rate": 3e-06,
      "loss": -0.0047,
      "step": 1428
    },
    {
      "clip_ratio": 0.00010064412344945595,
      "completion_length": 244.58334350585938,
      "epoch": 0.003967262450096891,
      "grad_norm": 0.07497075945138931,
      "kl": 0.058519456535577774,
      "learning_rate": 3e-06,
      "loss": -0.0042,
      "reward": 0.22708335518836975,
      "reward_std": 0.25706911087036133,
      "rewards/countdown_reward_func": 0.22708334028720856,
      "step": 1429,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0003266488783992827,
      "epoch": 0.003970038700936707,
      "grad_norm": 0.0831906795501709,
      "kl": 0.06262272223830223,
      "learning_rate": 3e-06,
      "loss": -0.0035,
      "step": 1430
    },
    {
      "clip_ratio": 0.000244140625,
      "epoch": 0.003972814951776523,
      "grad_norm": 0.13976958394050598,
      "kl": 0.05951959639787674,
      "learning_rate": 3e-06,
      "loss": -0.0041,
      "step": 1431
    },
    {
      "clip_ratio": 0.0022837779542896897,
      "epoch": 0.003975591202616339,
      "grad_norm": 0.09820695966482162,
      "kl": 0.059285424649715424,
      "learning_rate": 3e-06,
      "loss": -0.0036,
      "step": 1432
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.003978367453456155,
      "grad_norm": 0.08077849447727203,
      "kl": 0.05986793525516987,
      "learning_rate": 3e-06,
      "loss": -0.0047,
      "step": 1433
    },
    {
      "clip_ratio": 0.00032552084303461015,
      "epoch": 0.003981143704295971,
      "grad_norm": 0.091177798807621,
      "kl": 0.06356241181492805,
      "learning_rate": 3e-06,
      "loss": -0.0043,
      "step": 1434
    },
    {
      "clip_ratio": 0.0005404362455010414,
      "epoch": 0.003983919955135786,
      "grad_norm": 0.08302978426218033,
      "kl": 0.06063215062022209,
      "learning_rate": 3e-06,
      "loss": -0.0047,
      "step": 1435
    },
    {
      "clip_ratio": 0.0004089395733899437,
      "epoch": 0.0039866962059756025,
      "grad_norm": 0.08915877342224121,
      "kl": 0.0654546320438385,
      "learning_rate": 3e-06,
      "loss": -0.0043,
      "step": 1436
    },
    {
      "clip_ratio": 0.000328657595673576,
      "epoch": 0.003989472456815418,
      "grad_norm": 0.1467614471912384,
      "kl": 0.062131211161613464,
      "learning_rate": 3e-06,
      "loss": -0.0064,
      "step": 1437
    },
    {
      "clip_ratio": 0.0012416826793923974,
      "epoch": 0.003992248707655234,
      "grad_norm": 0.11402739584445953,
      "kl": 0.06069890968501568,
      "learning_rate": 3e-06,
      "loss": -0.0043,
      "step": 1438
    },
    {
      "clip_ratio": 0.00041950895683839917,
      "epoch": 0.00399502495849505,
      "grad_norm": 0.08940764516592026,
      "kl": 0.06148812361061573,
      "learning_rate": 3e-06,
      "loss": -0.0055,
      "step": 1439
    },
    {
      "clip_ratio": 0.0012557760928757489,
      "epoch": 0.003997801209334866,
      "grad_norm": 0.08653771132230759,
      "kl": 0.06552222743630409,
      "learning_rate": 3e-06,
      "loss": -0.0057,
      "step": 1440
    },
    {
      "clip_ratio": 0.00010434056457597762,
      "completion_length": 232.9791717529297,
      "epoch": 0.004000577460174681,
      "grad_norm": 0.11438572406768799,
      "kl": 0.07905929908156395,
      "learning_rate": 3e-06,
      "loss": 0.0201,
      "reward": 0.4375,
      "reward_std": 0.3626646399497986,
      "rewards/countdown_reward_func": 0.4375,
      "step": 1441,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.00016276042151730508,
      "epoch": 0.004003353711014498,
      "grad_norm": 0.13407845795154572,
      "kl": 0.07975272089242935,
      "learning_rate": 3e-06,
      "loss": 0.0212,
      "step": 1442
    },
    {
      "clip_ratio": 0.0002699867181945592,
      "epoch": 0.004006129961854314,
      "grad_norm": 0.18041589856147766,
      "kl": 0.08294541388750076,
      "learning_rate": 3e-06,
      "loss": 0.0206,
      "step": 1443
    },
    {
      "clip_ratio": 0.00037823428283445537,
      "epoch": 0.004008906212694129,
      "grad_norm": 0.10812884569168091,
      "kl": 0.07889501005411148,
      "learning_rate": 3e-06,
      "loss": 0.0205,
      "step": 1444
    },
    {
      "clip_ratio": 0.00018572076805867255,
      "epoch": 0.0040116824635339455,
      "grad_norm": 0.10284543037414551,
      "kl": 0.07939217984676361,
      "learning_rate": 3e-06,
      "loss": 0.0199,
      "step": 1445
    },
    {
      "clip_ratio": 0.000244140625,
      "epoch": 0.004014458714373761,
      "grad_norm": 0.1508864164352417,
      "kl": 0.07976921647787094,
      "learning_rate": 3e-06,
      "loss": 0.02,
      "step": 1446
    },
    {
      "clip_ratio": 0.0001915093234856613,
      "epoch": 0.004017234965213577,
      "grad_norm": 0.11611857265233994,
      "kl": 0.08126603439450264,
      "learning_rate": 3e-06,
      "loss": 0.0194,
      "step": 1447
    },
    {
      "clip_ratio": 0.00018572077533463016,
      "epoch": 0.004020011216053393,
      "grad_norm": 0.1259177178144455,
      "kl": 0.08240155875682831,
      "learning_rate": 3e-06,
      "loss": 0.0193,
      "step": 1448
    },
    {
      "clip_ratio": 0.0006606255192309618,
      "epoch": 0.004022787466893209,
      "grad_norm": 0.15441936254501343,
      "kl": 0.08467500656843185,
      "learning_rate": 3e-06,
      "loss": 0.0179,
      "step": 1449
    },
    {
      "clip_ratio": 0.00018383923452347517,
      "epoch": 0.004025563717733024,
      "grad_norm": 0.11314117908477783,
      "kl": 0.0805022120475769,
      "learning_rate": 3e-06,
      "loss": 0.0187,
      "step": 1450
    },
    {
      "clip_ratio": 0.0006165863596834242,
      "epoch": 0.0040283399685728406,
      "grad_norm": 0.11447932571172714,
      "kl": 0.08336110040545464,
      "learning_rate": 3e-06,
      "loss": 0.0186,
      "step": 1451
    },
    {
      "clip_ratio": 0.0005323204240994528,
      "epoch": 0.004031116219412656,
      "grad_norm": 0.15156546235084534,
      "kl": 0.0839230939745903,
      "learning_rate": 3e-06,
      "loss": 0.0175,
      "step": 1452
    },
    {
      "clip_ratio": 8.278145833173767e-05,
      "completion_length": 235.14583587646484,
      "epoch": 0.004033892470252472,
      "grad_norm": 0.1193864643573761,
      "kl": 0.08672641590237617,
      "learning_rate": 3e-06,
      "loss": 0.0317,
      "reward": 0.35625001788139343,
      "reward_std": 0.3211328834295273,
      "rewards/countdown_reward_func": 0.35625001788139343,
      "step": 1453,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0040366687210922885,
      "grad_norm": 0.12705117464065552,
      "kl": 0.08588536828756332,
      "learning_rate": 3e-06,
      "loss": 0.0305,
      "step": 1454
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004039444971932104,
      "grad_norm": 0.10421835631132126,
      "kl": 0.08824677020311356,
      "learning_rate": 3e-06,
      "loss": 0.0301,
      "step": 1455
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00404222122277192,
      "grad_norm": 0.12364216893911362,
      "kl": 0.08488886058330536,
      "learning_rate": 3e-06,
      "loss": 0.0296,
      "step": 1456
    },
    {
      "clip_ratio": 9.170946577796713e-05,
      "epoch": 0.004044997473611736,
      "grad_norm": 0.11285992711782455,
      "kl": 0.08128321543335915,
      "learning_rate": 3e-06,
      "loss": 0.0301,
      "step": 1457
    },
    {
      "clip_ratio": 9.137426968663931e-05,
      "epoch": 0.004047773724451552,
      "grad_norm": 0.09293954819440842,
      "kl": 0.0865498036146164,
      "learning_rate": 3e-06,
      "loss": 0.0286,
      "step": 1458
    },
    {
      "clip_ratio": 0.00026412875740788877,
      "epoch": 0.004050549975291367,
      "grad_norm": 0.11421343684196472,
      "kl": 0.09296088293194771,
      "learning_rate": 3e-06,
      "loss": 0.0293,
      "step": 1459
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0040533262261311835,
      "grad_norm": 0.11324277520179749,
      "kl": 0.09277210384607315,
      "learning_rate": 3e-06,
      "loss": 0.0278,
      "step": 1460
    },
    {
      "clip_ratio": 0.0001641616690903902,
      "epoch": 0.004056102476970999,
      "grad_norm": 0.11188522726297379,
      "kl": 0.09390882402658463,
      "learning_rate": 3e-06,
      "loss": 0.0275,
      "step": 1461
    },
    {
      "clip_ratio": 0.0001641616690903902,
      "epoch": 0.004058878727810815,
      "grad_norm": 0.11014936119318008,
      "kl": 0.09254594147205353,
      "learning_rate": 3e-06,
      "loss": 0.0265,
      "step": 1462
    },
    {
      "clip_ratio": 0.0001744909241097048,
      "epoch": 0.004061654978650631,
      "grad_norm": 0.11003927886486053,
      "kl": 0.08989466726779938,
      "learning_rate": 3e-06,
      "loss": 0.0281,
      "step": 1463
    },
    {
      "clip_ratio": 8.532423089491203e-05,
      "epoch": 0.004064431229490447,
      "grad_norm": 0.10190403461456299,
      "kl": 0.09649864211678505,
      "learning_rate": 3e-06,
      "loss": 0.026,
      "step": 1464
    },
    {
      "clip_ratio": 0.00018395879305899143,
      "completion_length": 239.625,
      "epoch": 0.004067207480330263,
      "grad_norm": 0.09303663671016693,
      "kl": 0.09736296534538269,
      "learning_rate": 3e-06,
      "loss": 0.0134,
      "reward": 0.30625003576278687,
      "reward_std": 0.2696641683578491,
      "rewards/countdown_reward_func": 0.30625003576278687,
      "step": 1465,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 9.211496217176318e-05,
      "epoch": 0.004069983731170079,
      "grad_norm": 0.1130358949303627,
      "kl": 0.08961380645632744,
      "learning_rate": 3e-06,
      "loss": 0.0126,
      "step": 1466
    },
    {
      "clip_ratio": 0.00017335961456410587,
      "epoch": 0.004072759982009895,
      "grad_norm": 0.0847916454076767,
      "kl": 0.10012925043702126,
      "learning_rate": 3e-06,
      "loss": 0.013,
      "step": 1467
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00407553623284971,
      "grad_norm": 0.09342540800571442,
      "kl": 0.09443873539566994,
      "learning_rate": 3e-06,
      "loss": 0.0127,
      "step": 1468
    },
    {
      "clip_ratio": 0.000447636324679479,
      "epoch": 0.0040783124836895265,
      "grad_norm": 0.10130415111780167,
      "kl": 0.09730666130781174,
      "learning_rate": 3e-06,
      "loss": 0.0124,
      "step": 1469
    },
    {
      "clip_ratio": 8.60289073898457e-05,
      "epoch": 0.004081088734529342,
      "grad_norm": 0.09560680389404297,
      "kl": 0.10042464733123779,
      "learning_rate": 3e-06,
      "loss": 0.0124,
      "step": 1470
    },
    {
      "clip_ratio": 0.00025938851467799395,
      "epoch": 0.004083864985369158,
      "grad_norm": 0.1030803993344307,
      "kl": 0.10246840491890907,
      "learning_rate": 3e-06,
      "loss": 0.0117,
      "step": 1471
    },
    {
      "clip_ratio": 0.00017924292478710413,
      "epoch": 0.004086641236208974,
      "grad_norm": 0.10079611092805862,
      "kl": 0.09707974642515182,
      "learning_rate": 3e-06,
      "loss": 0.0117,
      "step": 1472
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00408941748704879,
      "grad_norm": 0.0831371396780014,
      "kl": 0.10519356653094292,
      "learning_rate": 3e-06,
      "loss": 0.0114,
      "step": 1473
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004092193737888605,
      "grad_norm": 0.1005701944231987,
      "kl": 0.09982095658779144,
      "learning_rate": 3e-06,
      "loss": 0.0109,
      "step": 1474
    },
    {
      "clip_ratio": 0.0009021220030263066,
      "epoch": 0.004094969988728422,
      "grad_norm": 0.08770573884248734,
      "kl": 0.10152734071016312,
      "learning_rate": 3e-06,
      "loss": 0.0107,
      "step": 1475
    },
    {
      "clip_ratio": 0.0006180326745379716,
      "epoch": 0.004097746239568238,
      "grad_norm": 0.10259686410427094,
      "kl": 0.10511034727096558,
      "learning_rate": 3e-06,
      "loss": 0.0115,
      "step": 1476
    },
    {
      "clip_ratio": 8.747375977691263e-05,
      "completion_length": 225.6875,
      "epoch": 0.004100522490408053,
      "grad_norm": 0.11141236126422882,
      "kl": 0.10264391079545021,
      "learning_rate": 3e-06,
      "loss": 0.0114,
      "reward": 0.26875001937150955,
      "reward_std": 0.2696641534566879,
      "rewards/countdown_reward_func": 0.26875001937150955,
      "step": 1477,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0041032987412478695,
      "grad_norm": 0.10607676953077316,
      "kl": 0.10159710422158241,
      "learning_rate": 3e-06,
      "loss": 0.0115,
      "step": 1478
    },
    {
      "clip_ratio": 0.00045908106403658167,
      "epoch": 0.004106074992087685,
      "grad_norm": 0.12285490334033966,
      "kl": 0.09724065288901329,
      "learning_rate": 3e-06,
      "loss": 0.0121,
      "step": 1479
    },
    {
      "clip_ratio": 8.915834769140929e-05,
      "epoch": 0.004108851242927501,
      "grad_norm": 0.10112053155899048,
      "kl": 0.09738883748650551,
      "learning_rate": 3e-06,
      "loss": 0.0111,
      "step": 1480
    },
    {
      "clip_ratio": 0.000543587921129074,
      "epoch": 0.004111627493767317,
      "grad_norm": 0.11109442263841629,
      "kl": 0.10089835524559021,
      "learning_rate": 3e-06,
      "loss": 0.0117,
      "step": 1481
    },
    {
      "clip_ratio": 8.7596352386754e-05,
      "epoch": 0.004114403744607133,
      "grad_norm": 0.09819518029689789,
      "kl": 0.10074819624423981,
      "learning_rate": 3e-06,
      "loss": 0.0116,
      "step": 1482
    },
    {
      "clip_ratio": 0.00010557432688074186,
      "epoch": 0.004117179995446948,
      "grad_norm": 0.10844285786151886,
      "kl": 0.1009376123547554,
      "learning_rate": 3e-06,
      "loss": 0.0101,
      "step": 1483
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0041199562462867646,
      "grad_norm": 0.1014895886182785,
      "kl": 0.1020418331027031,
      "learning_rate": 3e-06,
      "loss": 0.0103,
      "step": 1484
    },
    {
      "clip_ratio": 0.0003863678648485802,
      "epoch": 0.00412273249712658,
      "grad_norm": 0.11585193872451782,
      "kl": 0.09769553691148758,
      "learning_rate": 3e-06,
      "loss": 0.0107,
      "step": 1485
    },
    {
      "clip_ratio": 0.0004875522281508893,
      "epoch": 0.004125508747966396,
      "grad_norm": 0.09965824335813522,
      "kl": 0.09749152138829231,
      "learning_rate": 3e-06,
      "loss": 0.0099,
      "step": 1486
    },
    {
      "clip_ratio": 0.0005232414114288986,
      "epoch": 0.0041282849988062125,
      "grad_norm": 0.11992790549993515,
      "kl": 0.09825573861598969,
      "learning_rate": 3e-06,
      "loss": 0.0098,
      "step": 1487
    },
    {
      "clip_ratio": 0.00016897656314540654,
      "epoch": 0.004131061249646028,
      "grad_norm": 0.09515371173620224,
      "kl": 0.09882111474871635,
      "learning_rate": 3e-06,
      "loss": 0.0099,
      "step": 1488
    },
    {
      "clip_ratio": 9.170946577796713e-05,
      "completion_length": 212.7916717529297,
      "epoch": 0.004133837500485844,
      "grad_norm": 0.06438077986240387,
      "kl": 0.1108875423669815,
      "learning_rate": 3e-06,
      "loss": 0.005,
      "reward": 0.16874999925494194,
      "reward_std": 0.11910479329526424,
      "rewards/countdown_reward_func": 0.16874999925494194,
      "step": 1489,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio": 0.0001817244410631247,
      "epoch": 0.00413661375132566,
      "grad_norm": 0.07629207521677017,
      "kl": 0.1037943847477436,
      "learning_rate": 3e-06,
      "loss": 0.0059,
      "step": 1490
    },
    {
      "clip_ratio": 9.170946577796713e-05,
      "epoch": 0.004139390002165476,
      "grad_norm": 0.07570668309926987,
      "kl": 0.09897951036691666,
      "learning_rate": 3e-06,
      "loss": 0.0049,
      "step": 1491
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004142166253005291,
      "grad_norm": 0.06962047517299652,
      "kl": 0.1040000282227993,
      "learning_rate": 3e-06,
      "loss": 0.0055,
      "step": 1492
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0041449425038451075,
      "grad_norm": 0.07831595093011856,
      "kl": 0.09726639091968536,
      "learning_rate": 3e-06,
      "loss": 0.0045,
      "step": 1493
    },
    {
      "clip_ratio": 0.0001838923490140587,
      "epoch": 0.004147718754684923,
      "grad_norm": 0.06859088689088821,
      "kl": 0.10122519358992577,
      "learning_rate": 3e-06,
      "loss": 0.0049,
      "step": 1494
    },
    {
      "clip_ratio": 0.0007797929574735463,
      "epoch": 0.004150495005524739,
      "grad_norm": 0.07352989166975021,
      "kl": 0.10665607079863548,
      "learning_rate": 3e-06,
      "loss": 0.0048,
      "step": 1495
    },
    {
      "clip_ratio": 8.954155055107549e-05,
      "epoch": 0.004153271256364555,
      "grad_norm": 0.0793951153755188,
      "kl": 0.09884294867515564,
      "learning_rate": 3e-06,
      "loss": 0.0047,
      "step": 1496
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004156047507204371,
      "grad_norm": 0.07391367107629776,
      "kl": 0.09311042353510857,
      "learning_rate": 3e-06,
      "loss": 0.0046,
      "step": 1497
    },
    {
      "clip_ratio": 0.0002756018075160682,
      "epoch": 0.004158823758044187,
      "grad_norm": 0.06575078517198563,
      "kl": 0.09845591336488724,
      "learning_rate": 3e-06,
      "loss": 0.0047,
      "step": 1498
    },
    {
      "clip_ratio": 0.00036945813917554915,
      "epoch": 0.004161600008884003,
      "grad_norm": 0.08721313625574112,
      "kl": 0.09197269007563591,
      "learning_rate": 3e-06,
      "loss": 0.0037,
      "step": 1499
    },
    {
      "clip_ratio": 0.0002712659916142002,
      "epoch": 0.004164376259723819,
      "grad_norm": 0.07341030240058899,
      "kl": 0.09440727904438972,
      "learning_rate": 3e-06,
      "loss": 0.0046,
      "step": 1500
    },
    {
      "clip_ratio": 0.00010064412344945595,
      "completion_length": 220.5416717529297,
      "epoch": 0.004167152510563634,
      "grad_norm": 0.11328040063381195,
      "kl": 0.09348515421152115,
      "learning_rate": 3e-06,
      "loss": 0.0053,
      "reward": 0.35625001788139343,
      "reward_std": 0.31346985697746277,
      "rewards/countdown_reward_func": 0.35625001788139343,
      "step": 1501,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0041699287614034505,
      "grad_norm": 0.11099179089069366,
      "kl": 0.09527323395013809,
      "learning_rate": 3e-06,
      "loss": 0.0064,
      "step": 1502
    },
    {
      "clip_ratio": 0.00023474179033655673,
      "epoch": 0.004172705012243266,
      "grad_norm": 0.10442166030406952,
      "kl": 0.08588381856679916,
      "learning_rate": 3e-06,
      "loss": 0.0054,
      "step": 1503
    },
    {
      "clip_ratio": 8.992805669549853e-05,
      "epoch": 0.004175481263083082,
      "grad_norm": 0.13162118196487427,
      "kl": 0.08620287477970123,
      "learning_rate": 3e-06,
      "loss": 0.005,
      "step": 1504
    },
    {
      "clip_ratio": 0.000326107838191092,
      "epoch": 0.004178257513922898,
      "grad_norm": 0.11255912482738495,
      "kl": 0.08432386443018913,
      "learning_rate": 3e-06,
      "loss": 0.0055,
      "step": 1505
    },
    {
      "clip_ratio": 0.0004599874228006229,
      "epoch": 0.004181033764762714,
      "grad_norm": 0.11156965047121048,
      "kl": 0.08439022302627563,
      "learning_rate": 3e-06,
      "loss": 0.0056,
      "step": 1506
    },
    {
      "clip_ratio": 9.750390017870814e-05,
      "epoch": 0.004183810015602529,
      "grad_norm": 0.11413870751857758,
      "kl": 0.08592484891414642,
      "learning_rate": 3e-06,
      "loss": 0.0051,
      "step": 1507
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.004186586266442346,
      "grad_norm": 0.12828348577022552,
      "kl": 0.08880098164081573,
      "learning_rate": 3e-06,
      "loss": 0.0044,
      "step": 1508
    },
    {
      "clip_ratio": 0.0002807183118420653,
      "epoch": 0.004189362517282162,
      "grad_norm": 0.10022372007369995,
      "kl": 0.07961970940232277,
      "learning_rate": 3e-06,
      "loss": 0.0039,
      "step": 1509
    },
    {
      "clip_ratio": 0.0004508365091169253,
      "epoch": 0.004192138768121977,
      "grad_norm": 0.1420833021402359,
      "kl": 0.08037307858467102,
      "learning_rate": 3e-06,
      "loss": 0.0048,
      "step": 1510
    },
    {
      "clip_ratio": 0.0011079075193265453,
      "epoch": 0.0041949150189617935,
      "grad_norm": 0.11328054964542389,
      "kl": 0.07894135639071465,
      "learning_rate": 3e-06,
      "loss": 0.0045,
      "step": 1511
    },
    {
      "clip_ratio": 0.0002834866172634065,
      "epoch": 0.004197691269801609,
      "grad_norm": 0.12972882390022278,
      "kl": 0.07961436733603477,
      "learning_rate": 3e-06,
      "loss": 0.004,
      "step": 1512
    },
    {
      "clip_ratio": 0.0002615062694530934,
      "completion_length": 231.58333587646484,
      "epoch": 0.004200467520641425,
      "grad_norm": 0.09790017455816269,
      "kl": 0.08236554637551308,
      "learning_rate": 3e-06,
      "loss": 0.0149,
      "reward": 0.229166679084301,
      "reward_std": 0.19716466218233109,
      "rewards/countdown_reward_func": 0.229166679084301,
      "step": 1513,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio": 0.00018543919577496126,
      "epoch": 0.004203243771481241,
      "grad_norm": 0.07614653557538986,
      "kl": 0.09001684933900833,
      "learning_rate": 3e-06,
      "loss": 0.0156,
      "step": 1514
    },
    {
      "clip_ratio": 0.00016276042151730508,
      "epoch": 0.004206020022321057,
      "grad_norm": 0.08829134702682495,
      "kl": 0.0819174312055111,
      "learning_rate": 3e-06,
      "loss": 0.0153,
      "step": 1515
    },
    {
      "clip_ratio": 0.0001734273391775787,
      "epoch": 0.004208796273160872,
      "grad_norm": 0.05850941315293312,
      "kl": 0.0837816521525383,
      "learning_rate": 3e-06,
      "loss": 0.0154,
      "step": 1516
    },
    {
      "clip_ratio": 0.00029112402262398973,
      "epoch": 0.0042115725240006886,
      "grad_norm": 0.07008534669876099,
      "kl": 0.08121475949883461,
      "learning_rate": 3e-06,
      "loss": 0.0152,
      "step": 1517
    },
    {
      "clip_ratio": 0.0001695139508228749,
      "epoch": 0.004214348774840504,
      "grad_norm": 0.07618328183889389,
      "kl": 0.08276839926838875,
      "learning_rate": 3e-06,
      "loss": 0.0155,
      "step": 1518
    },
    {
      "clip_ratio": 0.00034776486427290365,
      "epoch": 0.00421712502568032,
      "grad_norm": 0.10261756926774979,
      "kl": 0.082290880382061,
      "learning_rate": 3e-06,
      "loss": 0.0155,
      "step": 1519
    },
    {
      "clip_ratio": 0.00016469038382638246,
      "epoch": 0.0042199012765201365,
      "grad_norm": 0.06574001908302307,
      "kl": 0.09078622981905937,
      "learning_rate": 3e-06,
      "loss": 0.0156,
      "step": 1520
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004222677527359952,
      "grad_norm": 0.08486269414424896,
      "kl": 0.08278439193964005,
      "learning_rate": 3e-06,
      "loss": 0.0151,
      "step": 1521
    },
    {
      "clip_ratio": 0.0001796506403479725,
      "epoch": 0.004225453778199768,
      "grad_norm": 0.05522237718105316,
      "kl": 0.08544733002781868,
      "learning_rate": 3e-06,
      "loss": 0.0154,
      "step": 1522
    },
    {
      "clip_ratio": 0.00026453185273567215,
      "epoch": 0.004228230029039584,
      "grad_norm": 0.08137688040733337,
      "kl": 0.08243484795093536,
      "learning_rate": 3e-06,
      "loss": 0.0151,
      "step": 1523
    },
    {
      "clip_ratio": 0.00016276042151730508,
      "epoch": 0.0042310062798794,
      "grad_norm": 0.06071337312459946,
      "kl": 0.08487970381975174,
      "learning_rate": 3e-06,
      "loss": 0.0142,
      "step": 1524
    },
    {
      "clip_ratio": 0.00026819700724445283,
      "completion_length": 232.87500762939453,
      "epoch": 0.004233782530719215,
      "grad_norm": 0.066123828291893,
      "kl": 0.08086292445659637,
      "learning_rate": 3e-06,
      "loss": -0.0006,
      "reward": 0.17083334922790527,
      "reward_std": 0.11393594369292259,
      "rewards/countdown_reward_func": 0.17083334922790527,
      "step": 1525,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio": 0.00016953254089457914,
      "epoch": 0.0042365587815590315,
      "grad_norm": 0.19338181614875793,
      "kl": 0.08114173635840416,
      "learning_rate": 3e-06,
      "loss": -0.0,
      "step": 1526
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.004239335032398847,
      "grad_norm": 0.08377951383590698,
      "kl": 0.07633576914668083,
      "learning_rate": 3e-06,
      "loss": -0.0006,
      "step": 1527
    },
    {
      "clip_ratio": 0.0001913186424644664,
      "epoch": 0.004242111283238663,
      "grad_norm": 0.06688714772462845,
      "kl": 0.0815008170902729,
      "learning_rate": 3e-06,
      "loss": -0.0004,
      "step": 1528
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.004244887534078479,
      "grad_norm": 0.05774759501218796,
      "kl": 0.07802558317780495,
      "learning_rate": 3e-06,
      "loss": -0.0003,
      "step": 1529
    },
    {
      "clip_ratio": 0.0004640174884116277,
      "epoch": 0.004247663784918295,
      "grad_norm": 0.09573258459568024,
      "kl": 0.07392871007323265,
      "learning_rate": 3e-06,
      "loss": -0.001,
      "step": 1530
    },
    {
      "clip_ratio": 0.00025327454204671085,
      "epoch": 0.004250440035758111,
      "grad_norm": 0.07546308636665344,
      "kl": 0.07837391272187233,
      "learning_rate": 3e-06,
      "loss": -0.0008,
      "step": 1531
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.004253216286597927,
      "grad_norm": 0.21055151522159576,
      "kl": 0.07768003270030022,
      "learning_rate": 3e-06,
      "loss": -0.0019,
      "step": 1532
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.004255992537437743,
      "grad_norm": 0.0746055543422699,
      "kl": 0.07167375087738037,
      "learning_rate": 3e-06,
      "loss": -0.0011,
      "step": 1533
    },
    {
      "clip_ratio": 0.0003103909839410335,
      "epoch": 0.004258768788277558,
      "grad_norm": 0.07092703133821487,
      "kl": 0.076376061886549,
      "learning_rate": 3e-06,
      "loss": -0.0013,
      "step": 1534
    },
    {
      "clip_ratio": 0.0007744758331682533,
      "epoch": 0.0042615450391173745,
      "grad_norm": 0.058808572590351105,
      "kl": 0.07276593148708344,
      "learning_rate": 3e-06,
      "loss": -0.0009,
      "step": 1535
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.00426432128995719,
      "grad_norm": 0.07845474034547806,
      "kl": 0.06798744946718216,
      "learning_rate": 3e-06,
      "loss": -0.0021,
      "step": 1536
    },
    {
      "clip_ratio": 0.0003383140719961375,
      "completion_length": 234.5,
      "epoch": 0.004267097540797006,
      "grad_norm": 0.11504442989826202,
      "kl": 0.06899916008114815,
      "learning_rate": 3e-06,
      "loss": 0.0036,
      "reward": 0.2854166775941849,
      "reward_std": 0.2782912850379944,
      "rewards/countdown_reward_func": 0.2854166552424431,
      "step": 1537,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.00025050100521184504,
      "epoch": 0.004269873791636822,
      "grad_norm": 0.08852005749940872,
      "kl": 0.06766302511096,
      "learning_rate": 3e-06,
      "loss": 0.0039,
      "step": 1538
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004272650042476638,
      "grad_norm": 0.08253919333219528,
      "kl": 0.06684885919094086,
      "learning_rate": 3e-06,
      "loss": 0.0035,
      "step": 1539
    },
    {
      "clip_ratio": 0.0003363504074513912,
      "epoch": 0.004275426293316453,
      "grad_norm": 0.09681417047977448,
      "kl": 0.062055185437202454,
      "learning_rate": 3e-06,
      "loss": 0.0033,
      "step": 1540
    },
    {
      "clip_ratio": 0.0004321035448811017,
      "epoch": 0.00427820254415627,
      "grad_norm": 0.09291265904903412,
      "kl": 0.06612430512905121,
      "learning_rate": 3e-06,
      "loss": 0.0039,
      "step": 1541
    },
    {
      "clip_ratio": 8.520791016053408e-05,
      "epoch": 0.004280978794996086,
      "grad_norm": 0.09506966918706894,
      "kl": 0.06933099403977394,
      "learning_rate": 3e-06,
      "loss": 0.0042,
      "step": 1542
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004283755045835901,
      "grad_norm": 0.11389172077178955,
      "kl": 0.0656692385673523,
      "learning_rate": 3e-06,
      "loss": 0.0028,
      "step": 1543
    },
    {
      "clip_ratio": 0.00033449956390541047,
      "epoch": 0.0042865312966757175,
      "grad_norm": 0.08597692847251892,
      "kl": 0.0642948318272829,
      "learning_rate": 3e-06,
      "loss": 0.0026,
      "step": 1544
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.004289307547515533,
      "grad_norm": 0.09447018057107925,
      "kl": 0.06415480002760887,
      "learning_rate": 3e-06,
      "loss": 0.0023,
      "step": 1545
    },
    {
      "clip_ratio": 0.0008052970515564084,
      "epoch": 0.004292083798355349,
      "grad_norm": 0.08938866853713989,
      "kl": 0.06041167676448822,
      "learning_rate": 3e-06,
      "loss": 0.0018,
      "step": 1546
    },
    {
      "clip_ratio": 0.00062086500111036,
      "epoch": 0.004294860049195165,
      "grad_norm": 0.5281029939651489,
      "kl": 0.06464917957782745,
      "learning_rate": 3e-06,
      "loss": 0.0026,
      "step": 1547
    },
    {
      "clip_ratio": 0.0005138094347785227,
      "epoch": 0.004297636300034981,
      "grad_norm": 0.08887960761785507,
      "kl": 0.06742081791162491,
      "learning_rate": 3e-06,
      "loss": 0.0035,
      "step": 1548
    },
    {
      "clip_ratio": 0.0002517917600926012,
      "completion_length": 237.14584350585938,
      "epoch": 0.004300412550874796,
      "grad_norm": 0.10000678896903992,
      "kl": 0.05254969373345375,
      "learning_rate": 3e-06,
      "loss": 0.0067,
      "reward": 0.2666666954755783,
      "reward_std": 0.2623074799776077,
      "rewards/countdown_reward_func": 0.2666666880249977,
      "step": 1549,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0003343000134918839,
      "epoch": 0.0043031888017146126,
      "grad_norm": 0.09186454117298126,
      "kl": 0.056838370859622955,
      "learning_rate": 3e-06,
      "loss": 0.0056,
      "step": 1550
    },
    {
      "clip_ratio": 0.0007483240551664494,
      "epoch": 0.004305965052554429,
      "grad_norm": 0.09988920390605927,
      "kl": 0.05866162106394768,
      "learning_rate": 3e-06,
      "loss": 0.0075,
      "step": 1551
    },
    {
      "clip_ratio": 0.00024526867491658777,
      "epoch": 0.004308741303394244,
      "grad_norm": 0.07087462395429611,
      "kl": 0.055921848863363266,
      "learning_rate": 3e-06,
      "loss": 0.0053,
      "step": 1552
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0043115175542340605,
      "grad_norm": 0.07110070437192917,
      "kl": 0.05466669611632824,
      "learning_rate": 3e-06,
      "loss": 0.0062,
      "step": 1553
    },
    {
      "clip_ratio": 0.00032777692831587046,
      "epoch": 0.004314293805073876,
      "grad_norm": 0.09251809120178223,
      "kl": 0.05664430558681488,
      "learning_rate": 3e-06,
      "loss": 0.0071,
      "step": 1554
    },
    {
      "clip_ratio": 0.00032552083575865254,
      "epoch": 0.004317070055913692,
      "grad_norm": 0.09662747383117676,
      "kl": 0.05329587869346142,
      "learning_rate": 3e-06,
      "loss": 0.0064,
      "step": 1555
    },
    {
      "clip_ratio": 0.00032552083575865254,
      "epoch": 0.004319846306753508,
      "grad_norm": 0.10306031256914139,
      "kl": 0.056288715451955795,
      "learning_rate": 3e-06,
      "loss": 0.0048,
      "step": 1556
    },
    {
      "clip_ratio": 0.0003680418885778636,
      "epoch": 0.004322622557593324,
      "grad_norm": 0.08361591398715973,
      "kl": 0.06024627387523651,
      "learning_rate": 3e-06,
      "loss": 0.0068,
      "step": 1557
    },
    {
      "clip_ratio": 0.00032552083575865254,
      "epoch": 0.004325398808433139,
      "grad_norm": 0.06961996108293533,
      "kl": 0.05571894347667694,
      "learning_rate": 3e-06,
      "loss": 0.0052,
      "step": 1558
    },
    {
      "clip_ratio": 9.051412052940577e-05,
      "epoch": 0.0043281750592729555,
      "grad_norm": 0.20629696547985077,
      "kl": 0.05542530678212643,
      "learning_rate": 3e-06,
      "loss": 0.0058,
      "step": 1559
    },
    {
      "clip_ratio": 0.0005872198089491576,
      "epoch": 0.004330951310112771,
      "grad_norm": 0.09624534100294113,
      "kl": 0.05719461478292942,
      "learning_rate": 3e-06,
      "loss": 0.0056,
      "step": 1560
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 233.52084350585938,
      "epoch": 0.004333727560952587,
      "grad_norm": 0.07015831768512726,
      "kl": 0.06173841841518879,
      "learning_rate": 3e-06,
      "loss": -0.0015,
      "reward": 0.2666666954755783,
      "reward_std": 0.22149831801652908,
      "rewards/countdown_reward_func": 0.2666666805744171,
      "step": 1561,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio": 0.00018502863531466573,
      "epoch": 0.0043365038117924035,
      "grad_norm": 0.07204274088144302,
      "kl": 0.0598264392465353,
      "learning_rate": 3e-06,
      "loss": -0.0018,
      "step": 1562
    },
    {
      "clip_ratio": 0.00016556291666347533,
      "epoch": 0.004339280062632219,
      "grad_norm": 0.08235902339220047,
      "kl": 0.06111270561814308,
      "learning_rate": 3e-06,
      "loss": -0.0016,
      "step": 1563
    },
    {
      "clip_ratio": 0.000179314476554282,
      "epoch": 0.004342056313472035,
      "grad_norm": 0.08542285859584808,
      "kl": 0.06232334300875664,
      "learning_rate": 3e-06,
      "loss": -0.002,
      "step": 1564
    },
    {
      "clip_ratio": 0.0001975396226043813,
      "epoch": 0.004344832564311851,
      "grad_norm": 0.08785772323608398,
      "kl": 0.06148502230644226,
      "learning_rate": 3e-06,
      "loss": -0.0023,
      "step": 1565
    },
    {
      "clip_ratio": 0.0001065643664333038,
      "epoch": 0.004347608815151667,
      "grad_norm": 0.06917145103216171,
      "kl": 0.06312652491033077,
      "learning_rate": 3e-06,
      "loss": -0.0017,
      "step": 1566
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004350385065991482,
      "grad_norm": 0.07437288761138916,
      "kl": 0.0617559514939785,
      "learning_rate": 3e-06,
      "loss": -0.0019,
      "step": 1567
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0043531613168312985,
      "grad_norm": 0.07470710575580597,
      "kl": 0.0611141100525856,
      "learning_rate": 3e-06,
      "loss": -0.0026,
      "step": 1568
    },
    {
      "clip_ratio": 0.00016971942386589944,
      "epoch": 0.004355937567671114,
      "grad_norm": 0.10094926506280899,
      "kl": 0.06114407069981098,
      "learning_rate": 3e-06,
      "loss": -0.0028,
      "step": 1569
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00435871381851093,
      "grad_norm": 0.08059901744127274,
      "kl": 0.06223343499004841,
      "learning_rate": 3e-06,
      "loss": -0.0026,
      "step": 1570
    },
    {
      "clip_ratio": 0.0002885148787754588,
      "epoch": 0.004361490069350746,
      "grad_norm": 0.07476944476366043,
      "kl": 0.06176266819238663,
      "learning_rate": 3e-06,
      "loss": -0.0027,
      "step": 1571
    },
    {
      "clip_ratio": 0.0001713502424536273,
      "epoch": 0.004364266320190562,
      "grad_norm": 0.07194018363952637,
      "kl": 0.06364806741476059,
      "learning_rate": 3e-06,
      "loss": -0.0024,
      "step": 1572
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 238.9791717529297,
      "epoch": 0.004367042571030378,
      "grad_norm": 0.06975971907377243,
      "kl": 0.07021701335906982,
      "learning_rate": 3e-06,
      "loss": 0.0159,
      "reward": 0.22500000894069672,
      "reward_std": 0.18065783381462097,
      "rewards/countdown_reward_func": 0.22500000149011612,
      "step": 1573,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004369818821870194,
      "grad_norm": 0.06149615719914436,
      "kl": 0.07013114541769028,
      "learning_rate": 3e-06,
      "loss": 0.0162,
      "step": 1574
    },
    {
      "clip_ratio": 0.0001724217290757224,
      "epoch": 0.00437259507271001,
      "grad_norm": 0.06460988521575928,
      "kl": 0.07087487727403641,
      "learning_rate": 3e-06,
      "loss": 0.0164,
      "step": 1575
    },
    {
      "clip_ratio": 0.0001953798928298056,
      "epoch": 0.004375371323549825,
      "grad_norm": 0.08817639201879501,
      "kl": 0.07189563289284706,
      "learning_rate": 3e-06,
      "loss": 0.0164,
      "step": 1576
    },
    {
      "clip_ratio": 0.00010486577230039984,
      "epoch": 0.0043781475743896415,
      "grad_norm": 0.0764622911810875,
      "kl": 0.06396012753248215,
      "learning_rate": 3e-06,
      "loss": 0.0164,
      "step": 1577
    },
    {
      "clip_ratio": 0.0002441406322759576,
      "epoch": 0.004380923825229457,
      "grad_norm": 0.06788492202758789,
      "kl": 0.06740446761250496,
      "learning_rate": 3e-06,
      "loss": 0.0158,
      "step": 1578
    },
    {
      "clip_ratio": 0.00016276042151730508,
      "epoch": 0.004383700076069273,
      "grad_norm": 0.06214107945561409,
      "kl": 0.07128945738077164,
      "learning_rate": 3e-06,
      "loss": 0.0165,
      "step": 1579
    },
    {
      "clip_ratio": 0.00024570666573708877,
      "epoch": 0.004386476326909089,
      "grad_norm": 0.06276597827672958,
      "kl": 0.0706591084599495,
      "learning_rate": 3e-06,
      "loss": 0.0157,
      "step": 1580
    },
    {
      "clip_ratio": 0.00033518214331706986,
      "epoch": 0.004389252577748905,
      "grad_norm": 0.06644298136234283,
      "kl": 0.07160716131329536,
      "learning_rate": 3e-06,
      "loss": 0.0156,
      "step": 1581
    },
    {
      "clip_ratio": 0.00033465474552940577,
      "epoch": 0.00439202882858872,
      "grad_norm": 0.09485163539648056,
      "kl": 0.07267293706536293,
      "learning_rate": 3e-06,
      "loss": 0.0154,
      "step": 1582
    },
    {
      "clip_ratio": 9.104151831706986e-05,
      "epoch": 0.0043948050794285366,
      "grad_norm": 0.07193972915410995,
      "kl": 0.06612110696732998,
      "learning_rate": 3e-06,
      "loss": 0.0152,
      "step": 1583
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.004397581330268353,
      "grad_norm": 0.06304839253425598,
      "kl": 0.06994717195630074,
      "learning_rate": 3e-06,
      "loss": 0.0155,
      "step": 1584
    },
    {
      "clip_ratio": 0.00011627907224465162,
      "completion_length": 229.9166717529297,
      "epoch": 0.004400357581108168,
      "grad_norm": 0.16353268921375275,
      "kl": 0.07109616324305534,
      "learning_rate": 3e-06,
      "loss": 0.012,
      "reward": 0.3437500298023224,
      "reward_std": 0.33480696380138397,
      "rewards/countdown_reward_func": 0.3437500298023224,
      "step": 1585,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0044031338319479845,
      "grad_norm": 0.09060167521238327,
      "kl": 0.06814149767160416,
      "learning_rate": 3e-06,
      "loss": 0.0127,
      "step": 1586
    },
    {
      "clip_ratio": 0.00019944607629440725,
      "epoch": 0.0044059100827878,
      "grad_norm": 0.11436658352613449,
      "kl": 0.06723786145448685,
      "learning_rate": 3e-06,
      "loss": 0.0122,
      "step": 1587
    },
    {
      "clip_ratio": 0.00029042133246548474,
      "epoch": 0.004408686333627616,
      "grad_norm": 0.09583216160535812,
      "kl": 0.07507549971342087,
      "learning_rate": 3e-06,
      "loss": 0.0116,
      "step": 1588
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004411462584467432,
      "grad_norm": 0.08748263865709305,
      "kl": 0.07277511805295944,
      "learning_rate": 3e-06,
      "loss": 0.0118,
      "step": 1589
    },
    {
      "clip_ratio": 0.0005822835373692214,
      "epoch": 0.004414238835307248,
      "grad_norm": 0.10275944322347641,
      "kl": 0.06728723272681236,
      "learning_rate": 3e-06,
      "loss": 0.0114,
      "step": 1590
    },
    {
      "clip_ratio": 0.00011627907224465162,
      "epoch": 0.004417015086147063,
      "grad_norm": 0.15305157005786896,
      "kl": 0.0735839381814003,
      "learning_rate": 3e-06,
      "loss": 0.0102,
      "step": 1591
    },
    {
      "clip_ratio": 0.00043021741294069216,
      "epoch": 0.0044197913369868795,
      "grad_norm": 0.1069241613149643,
      "kl": 0.07472379878163338,
      "learning_rate": 3e-06,
      "loss": 0.0111,
      "step": 1592
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004422567587826695,
      "grad_norm": 0.09179497510194778,
      "kl": 0.07055623456835747,
      "learning_rate": 3e-06,
      "loss": 0.0105,
      "step": 1593
    },
    {
      "clip_ratio": 0.00019944607629440725,
      "epoch": 0.004425343838666511,
      "grad_norm": 0.09448972344398499,
      "kl": 0.08183940127491951,
      "learning_rate": 3e-06,
      "loss": 0.0114,
      "step": 1594
    },
    {
      "clip_ratio": 8.316699677379802e-05,
      "epoch": 0.0044281200895063275,
      "grad_norm": 0.12228698283433914,
      "kl": 0.07783747464418411,
      "learning_rate": 3e-06,
      "loss": 0.01,
      "step": 1595
    },
    {
      "clip_ratio": 0.0004985033592674881,
      "epoch": 0.004430896340346143,
      "grad_norm": 0.13479016721248627,
      "kl": 0.07179780304431915,
      "learning_rate": 3e-06,
      "loss": 0.0101,
      "step": 1596
    },
    {
      "clip_ratio": 0.0002777284535113722,
      "completion_length": 230.43750762939453,
      "epoch": 0.004433672591185959,
      "grad_norm": 0.11454746127128601,
      "kl": 0.06979512795805931,
      "learning_rate": 3e-06,
      "loss": 0.0316,
      "reward": 0.30416668951511383,
      "reward_std": 0.2991009056568146,
      "rewards/countdown_reward_func": 0.30416667461395264,
      "step": 1597,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.00035156837839167565,
      "epoch": 0.004436448842025775,
      "grad_norm": 0.14501981437206268,
      "kl": 0.07220923155546188,
      "learning_rate": 3e-06,
      "loss": 0.0309,
      "step": 1598
    },
    {
      "clip_ratio": 9.484066686127335e-05,
      "epoch": 0.004439225092865591,
      "grad_norm": 0.09493423253297806,
      "kl": 0.07610296085476875,
      "learning_rate": 3e-06,
      "loss": 0.0313,
      "step": 1599
    },
    {
      "clip_ratio": 0.0005380346046877094,
      "epoch": 0.004442001343705406,
      "grad_norm": 0.10318285971879959,
      "kl": 0.07600558176636696,
      "learning_rate": 3e-06,
      "loss": 0.0309,
      "step": 1600
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0044447775945452225,
      "grad_norm": 0.1465279459953308,
      "kl": 0.07435224205255508,
      "learning_rate": 3e-06,
      "loss": 0.0301,
      "step": 1601
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004447553845385038,
      "grad_norm": 0.132258802652359,
      "kl": 0.07526106759905815,
      "learning_rate": 3e-06,
      "loss": 0.0296,
      "step": 1602
    },
    {
      "clip_ratio": 0.0003190559073118493,
      "epoch": 0.004450330096224854,
      "grad_norm": 0.11824331432580948,
      "kl": 0.07644139230251312,
      "learning_rate": 3e-06,
      "loss": 0.0287,
      "step": 1603
    },
    {
      "clip_ratio": 0.00036518805427476764,
      "epoch": 0.00445310634706467,
      "grad_norm": 0.12642835080623627,
      "kl": 0.08045900240540504,
      "learning_rate": 3e-06,
      "loss": 0.028,
      "step": 1604
    },
    {
      "clip_ratio": 9.484066686127335e-05,
      "epoch": 0.004455882597904486,
      "grad_norm": 0.08899746090173721,
      "kl": 0.08476192504167557,
      "learning_rate": 3e-06,
      "loss": 0.029,
      "step": 1605
    },
    {
      "clip_ratio": 0.00035244174068793654,
      "epoch": 0.004458658848744302,
      "grad_norm": 0.09743326157331467,
      "kl": 0.08727932721376419,
      "learning_rate": 3e-06,
      "loss": 0.0277,
      "step": 1606
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004461435099584118,
      "grad_norm": 0.13737818598747253,
      "kl": 0.08706852421164513,
      "learning_rate": 3e-06,
      "loss": 0.0263,
      "step": 1607
    },
    {
      "clip_ratio": 0.0005535711825359613,
      "epoch": 0.004464211350423934,
      "grad_norm": 0.12821559607982635,
      "kl": 0.08946230262517929,
      "learning_rate": 3e-06,
      "loss": 0.0255,
      "step": 1608
    },
    {
      "clip_ratio": 0.0003540786055964418,
      "completion_length": 227.70833587646484,
      "epoch": 0.004466987601263749,
      "grad_norm": 0.11783099174499512,
      "kl": 0.10899167135357857,
      "learning_rate": 3e-06,
      "loss": -0.0008,
      "reward": 0.30416668951511383,
      "reward_std": 0.3719724863767624,
      "rewards/countdown_reward_func": 0.30416668951511383,
      "step": 1609,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 9.704969124868512e-05,
      "epoch": 0.0044697638521035655,
      "grad_norm": 0.10847090929746628,
      "kl": 0.10389678180217743,
      "learning_rate": 3e-06,
      "loss": -0.0007,
      "step": 1610
    },
    {
      "clip_ratio": 0.0006049876101315022,
      "epoch": 0.004472540102943381,
      "grad_norm": 0.11665836721658707,
      "kl": 0.10452697053551674,
      "learning_rate": 3e-06,
      "loss": -0.0001,
      "step": 1611
    },
    {
      "clip_ratio": 0.000750809209421277,
      "epoch": 0.004475316353783197,
      "grad_norm": 0.1398657262325287,
      "kl": 0.12372316420078278,
      "learning_rate": 3e-06,
      "loss": -0.0002,
      "step": 1612
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004478092604623013,
      "grad_norm": 0.09913130849599838,
      "kl": 0.11288458853960037,
      "learning_rate": 3e-06,
      "loss": -0.0017,
      "step": 1613
    },
    {
      "clip_ratio": 0.00017424796533305198,
      "epoch": 0.004480868855462829,
      "grad_norm": 0.1439204066991806,
      "kl": 0.1121748648583889,
      "learning_rate": 3e-06,
      "loss": -0.0008,
      "step": 1614
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.004483645106302644,
      "grad_norm": 0.11864045262336731,
      "kl": 0.12259047850966454,
      "learning_rate": 3e-06,
      "loss": 0.0006,
      "step": 1615
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0044864213571424606,
      "grad_norm": 0.10626986622810364,
      "kl": 0.11897166818380356,
      "learning_rate": 3e-06,
      "loss": -0.0003,
      "step": 1616
    },
    {
      "clip_ratio": 0.0007439094333676621,
      "epoch": 0.004489197607982277,
      "grad_norm": 0.10574115067720413,
      "kl": 0.1181485652923584,
      "learning_rate": 3e-06,
      "loss": 0.0004,
      "step": 1617
    },
    {
      "clip_ratio": 0.000587511618505232,
      "epoch": 0.004491973858822092,
      "grad_norm": 0.12936261296272278,
      "kl": 0.13641805946826935,
      "learning_rate": 3e-06,
      "loss": -0.0005,
      "step": 1618
    },
    {
      "clip_ratio": 0.0001871362328529358,
      "epoch": 0.0044947501096619085,
      "grad_norm": 0.09758912771940231,
      "kl": 0.12104825302958488,
      "learning_rate": 3e-06,
      "loss": -0.0026,
      "step": 1619
    },
    {
      "clip_ratio": 0.00016276042151730508,
      "epoch": 0.004497526360501724,
      "grad_norm": 0.10167786478996277,
      "kl": 0.11872199177742004,
      "learning_rate": 3e-06,
      "loss": -0.0008,
      "step": 1620
    },
    {
      "clip_ratio": 0.0004541333037195727,
      "completion_length": 230.33333587646484,
      "epoch": 0.00450030261134154,
      "grad_norm": 0.09969217330217361,
      "kl": 0.13528293371200562,
      "learning_rate": 3e-06,
      "loss": 0.0064,
      "reward": 0.2875000238418579,
      "reward_std": 0.2661401033401489,
      "rewards/countdown_reward_func": 0.2875000163912773,
      "step": 1621,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004503078862181356,
      "grad_norm": 0.11825183779001236,
      "kl": 0.11860142275691032,
      "learning_rate": 3e-06,
      "loss": 0.0048,
      "step": 1622
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004505855113021172,
      "grad_norm": 0.09762841463088989,
      "kl": 0.1295238509774208,
      "learning_rate": 3e-06,
      "loss": 0.0057,
      "step": 1623
    },
    {
      "clip_ratio": 0.00018274853937327862,
      "epoch": 0.004508631363860987,
      "grad_norm": 0.1245664656162262,
      "kl": 0.12989997118711472,
      "learning_rate": 3e-06,
      "loss": 0.0048,
      "step": 1624
    },
    {
      "clip_ratio": 0.0004756918060593307,
      "epoch": 0.0045114076147008035,
      "grad_norm": 0.09959601610898972,
      "kl": 0.13353952765464783,
      "learning_rate": 3e-06,
      "loss": 0.0055,
      "step": 1625
    },
    {
      "clip_ratio": 0.0004383840714581311,
      "epoch": 0.004514183865540619,
      "grad_norm": 0.10265988856554031,
      "kl": 0.1284654214978218,
      "learning_rate": 3e-06,
      "loss": 0.0058,
      "step": 1626
    },
    {
      "clip_ratio": 0.0007191235781647265,
      "epoch": 0.004516960116380435,
      "grad_norm": 0.09630168974399567,
      "kl": 0.13352051377296448,
      "learning_rate": 3e-06,
      "loss": 0.0057,
      "step": 1627
    },
    {
      "clip_ratio": 0.0001932617014972493,
      "epoch": 0.0045197363672202515,
      "grad_norm": 0.1130925789475441,
      "kl": 0.11547379568219185,
      "learning_rate": 3e-06,
      "loss": 0.0042,
      "step": 1628
    },
    {
      "clip_ratio": 0.00018102824105881155,
      "epoch": 0.004522512618060067,
      "grad_norm": 0.10465371608734131,
      "kl": 0.12602712959051132,
      "learning_rate": 3e-06,
      "loss": 0.0042,
      "step": 1629
    },
    {
      "clip_ratio": 0.0002716227754717693,
      "epoch": 0.004525288868899883,
      "grad_norm": 0.11785195767879486,
      "kl": 0.12726370617747307,
      "learning_rate": 3e-06,
      "loss": 0.003,
      "step": 1630
    },
    {
      "clip_ratio": 0.00018102824105881155,
      "epoch": 0.004528065119739699,
      "grad_norm": 0.1045258566737175,
      "kl": 0.12971790879964828,
      "learning_rate": 3e-06,
      "loss": 0.0042,
      "step": 1631
    },
    {
      "clip_ratio": 0.0006421779398806393,
      "epoch": 0.004530841370579515,
      "grad_norm": 0.10235866159200668,
      "kl": 0.12507328763604164,
      "learning_rate": 3e-06,
      "loss": 0.004,
      "step": 1632
    },
    {
      "clip_ratio": 9.61538462433964e-05,
      "completion_length": 237.31250762939453,
      "epoch": 0.00453361762141933,
      "grad_norm": 0.10965386778116226,
      "kl": 0.11101473122835159,
      "learning_rate": 3e-06,
      "loss": 0.0247,
      "reward": 0.34166668355464935,
      "reward_std": 0.3452594429254532,
      "rewards/countdown_reward_func": 0.34166668355464935,
      "step": 1633,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.0002707348467083648,
      "epoch": 0.0045363938722591465,
      "grad_norm": 0.10254738479852676,
      "kl": 0.11892389506101608,
      "learning_rate": 3e-06,
      "loss": 0.0251,
      "step": 1634
    },
    {
      "clip_ratio": 0.00018317173817194998,
      "epoch": 0.004539170123098962,
      "grad_norm": 0.1211012601852417,
      "kl": 0.11001782864332199,
      "learning_rate": 3e-06,
      "loss": 0.0254,
      "step": 1635
    },
    {
      "clip_ratio": 0.0007125435367925093,
      "epoch": 0.004541946373938778,
      "grad_norm": 0.10486630350351334,
      "kl": 0.11037512868642807,
      "learning_rate": 3e-06,
      "loss": 0.025,
      "step": 1636
    },
    {
      "clip_ratio": 0.0001923076924867928,
      "epoch": 0.004544722624778594,
      "grad_norm": 0.11679045855998993,
      "kl": 0.11175461858510971,
      "learning_rate": 3e-06,
      "loss": 0.025,
      "step": 1637
    },
    {
      "clip_ratio": 0.00017780938651412725,
      "epoch": 0.00454749887561841,
      "grad_norm": 0.11241600662469864,
      "kl": 0.11684262380003929,
      "learning_rate": 3e-06,
      "loss": 0.0248,
      "step": 1638
    },
    {
      "clip_ratio": 8.383634849451482e-05,
      "epoch": 0.004550275126458226,
      "grad_norm": 0.12549988925457,
      "kl": 0.1113254725933075,
      "learning_rate": 3e-06,
      "loss": 0.0236,
      "step": 1639
    },
    {
      "clip_ratio": 0.0003387995529919863,
      "epoch": 0.004553051377298042,
      "grad_norm": 0.09110260754823685,
      "kl": 0.11747504025697708,
      "learning_rate": 3e-06,
      "loss": 0.0238,
      "step": 1640
    },
    {
      "clip_ratio": 0.00026678377616917714,
      "epoch": 0.004555827628137858,
      "grad_norm": 0.12658299505710602,
      "kl": 0.11034082993865013,
      "learning_rate": 3e-06,
      "loss": 0.0242,
      "step": 1641
    },
    {
      "clip_ratio": 0.0003627651822171174,
      "epoch": 0.004558603878977673,
      "grad_norm": 0.1093655377626419,
      "kl": 0.11209141090512276,
      "learning_rate": 3e-06,
      "loss": 0.0244,
      "step": 1642
    },
    {
      "clip_ratio": 9.61538462433964e-05,
      "epoch": 0.0045613801298174895,
      "grad_norm": 0.11235036700963974,
      "kl": 0.11257147789001465,
      "learning_rate": 3e-06,
      "loss": 0.0237,
      "step": 1643
    },
    {
      "clip_ratio": 8.890469325706363e-05,
      "epoch": 0.004564156380657305,
      "grad_norm": 0.11150780320167542,
      "kl": 0.11817445978522301,
      "learning_rate": 3e-06,
      "loss": 0.024,
      "step": 1644
    },
    {
      "clip_ratio": 0.00017182034207507968,
      "completion_length": 218.52083587646484,
      "epoch": 0.004566932631497121,
      "grad_norm": 0.11840204149484634,
      "kl": 0.13194195181131363,
      "learning_rate": 3e-06,
      "loss": 0.0075,
      "reward": 0.3791666626930237,
      "reward_std": 0.36280614137649536,
      "rewards/countdown_reward_func": 0.3791666626930237,
      "step": 1645,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 0.0001797049117158167,
      "epoch": 0.004569708882336937,
      "grad_norm": 0.10418499261140823,
      "kl": 0.13568759709596634,
      "learning_rate": 3e-06,
      "loss": 0.0073,
      "step": 1646
    },
    {
      "clip_ratio": 0.00017531556659378111,
      "epoch": 0.004572485133176753,
      "grad_norm": 0.13569895923137665,
      "kl": 0.1283918097615242,
      "learning_rate": 3e-06,
      "loss": 0.0067,
      "step": 1647
    },
    {
      "clip_ratio": 0.00020443140238057822,
      "epoch": 0.004575261384016568,
      "grad_norm": 0.13669107854366302,
      "kl": 0.13637378439307213,
      "learning_rate": 3e-06,
      "loss": 0.0073,
      "step": 1648
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0045780376348563846,
      "grad_norm": 0.10260222107172012,
      "kl": 0.1373545378446579,
      "learning_rate": 3e-06,
      "loss": 0.0077,
      "step": 1649
    },
    {
      "clip_ratio": 0.0004510040016612038,
      "epoch": 0.004580813885696201,
      "grad_norm": 0.1269139051437378,
      "kl": 0.14053868502378464,
      "learning_rate": 3e-06,
      "loss": 0.0081,
      "step": 1650
    },
    {
      "clip_ratio": 0.0006953472329769284,
      "epoch": 0.004583590136536016,
      "grad_norm": 0.11335631459951401,
      "kl": 0.12832976132631302,
      "learning_rate": 3e-06,
      "loss": 0.0068,
      "step": 1651
    },
    {
      "clip_ratio": 0.0004522776580415666,
      "epoch": 0.0045863663873758325,
      "grad_norm": 0.12118861079216003,
      "kl": 0.13261255621910095,
      "learning_rate": 3e-06,
      "loss": 0.0063,
      "step": 1652
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004589142638215648,
      "grad_norm": 0.130536749958992,
      "kl": 0.12578046321868896,
      "learning_rate": 3e-06,
      "loss": 0.0049,
      "step": 1653
    },
    {
      "clip_ratio": 0.00020579837291734293,
      "epoch": 0.004591918889055464,
      "grad_norm": 0.13175055384635925,
      "kl": 0.13097375631332397,
      "learning_rate": 3e-06,
      "loss": 0.0062,
      "step": 1654
    },
    {
      "clip_ratio": 0.00038550328463315964,
      "epoch": 0.00459469513989528,
      "grad_norm": 0.0984969213604927,
      "kl": 0.13125114515423775,
      "learning_rate": 3e-06,
      "loss": 0.0059,
      "step": 1655
    },
    {
      "clip_ratio": 0.0004875610757153481,
      "epoch": 0.004597471390735096,
      "grad_norm": 0.11173491179943085,
      "kl": 0.13282914832234383,
      "learning_rate": 3e-06,
      "loss": 0.0056,
      "step": 1656
    },
    {
      "clip_ratio": 0.0001720578147796914,
      "completion_length": 235.7916717529297,
      "epoch": 0.004600247641574911,
      "grad_norm": 0.13065429031848907,
      "kl": 0.10866416990756989,
      "learning_rate": 3e-06,
      "loss": 0.015,
      "reward": 0.30625002831220627,
      "reward_std": 0.31206804513931274,
      "rewards/countdown_reward_func": 0.3062500134110451,
      "step": 1657,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 8.650519157527015e-05,
      "epoch": 0.0046030238924147275,
      "grad_norm": 0.09855504333972931,
      "kl": 0.10450971871614456,
      "learning_rate": 3e-06,
      "loss": 0.0145,
      "step": 1658
    },
    {
      "clip_ratio": 0.0006019261782057583,
      "epoch": 0.004605800143254543,
      "grad_norm": 0.08144383132457733,
      "kl": 0.11103589460253716,
      "learning_rate": 3e-06,
      "loss": 0.0156,
      "step": 1659
    },
    {
      "clip_ratio": 0.00030096308910287917,
      "epoch": 0.004608576394094359,
      "grad_norm": 0.09432277828454971,
      "kl": 0.10549774393439293,
      "learning_rate": 3e-06,
      "loss": 0.0161,
      "step": 1660
    },
    {
      "clip_ratio": 0.0004300602595321834,
      "epoch": 0.0046113526449341755,
      "grad_norm": 0.09225940704345703,
      "kl": 0.108955267816782,
      "learning_rate": 3e-06,
      "loss": 0.0163,
      "step": 1661
    },
    {
      "clip_ratio": 0.00045322794176172465,
      "epoch": 0.004614128895773991,
      "grad_norm": 0.12361875921487808,
      "kl": 0.1058424860239029,
      "learning_rate": 3e-06,
      "loss": 0.0149,
      "step": 1662
    },
    {
      "clip_ratio": 8.60289073898457e-05,
      "epoch": 0.004616905146613807,
      "grad_norm": 0.11309605091810226,
      "kl": 0.10573908686637878,
      "learning_rate": 3e-06,
      "loss": 0.0146,
      "step": 1663
    },
    {
      "clip_ratio": 0.00016788540233392268,
      "epoch": 0.004619681397453623,
      "grad_norm": 0.09809679538011551,
      "kl": 0.10098403319716454,
      "learning_rate": 3e-06,
      "loss": 0.0145,
      "step": 1664
    },
    {
      "clip_ratio": 9.09090886125341e-05,
      "epoch": 0.004622457648293439,
      "grad_norm": 0.07421303540468216,
      "kl": 0.10688856989145279,
      "learning_rate": 3e-06,
      "loss": 0.0152,
      "step": 1665
    },
    {
      "clip_ratio": 0.00010032102727564052,
      "epoch": 0.004625233899133254,
      "grad_norm": 0.09634650498628616,
      "kl": 0.10281075537204742,
      "learning_rate": 3e-06,
      "loss": 0.0153,
      "step": 1666
    },
    {
      "clip_ratio": 0.0004270849240128882,
      "epoch": 0.0046280101499730705,
      "grad_norm": 0.10299887508153915,
      "kl": 0.10465852543711662,
      "learning_rate": 3e-06,
      "loss": 0.0147,
      "step": 1667
    },
    {
      "clip_ratio": 0.0006324783898890018,
      "epoch": 0.004630786400812886,
      "grad_norm": 0.11212204396724701,
      "kl": 0.10365309193730354,
      "learning_rate": 3e-06,
      "loss": 0.0127,
      "step": 1668
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 232.75,
      "epoch": 0.004633562651652702,
      "grad_norm": 0.06845124810934067,
      "kl": 0.09933054447174072,
      "learning_rate": 3e-06,
      "loss": -0.008,
      "reward": 0.15416667610406876,
      "reward_std": 0.1091257855296135,
      "rewards/countdown_reward_func": 0.15416667610406876,
      "step": 1669,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio": 8.45165632199496e-05,
      "epoch": 0.004636338902492518,
      "grad_norm": 0.0753096267580986,
      "kl": 0.10704569146037102,
      "learning_rate": 3e-06,
      "loss": -0.0078,
      "step": 1670
    },
    {
      "clip_ratio": 0.00035820446646539494,
      "epoch": 0.004639115153332334,
      "grad_norm": 0.056882914155721664,
      "kl": 0.10202248394489288,
      "learning_rate": 3e-06,
      "loss": -0.008,
      "step": 1671
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00464189140417215,
      "grad_norm": 0.0649522915482521,
      "kl": 0.0977204330265522,
      "learning_rate": 3e-06,
      "loss": -0.008,
      "step": 1672
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004644667655011966,
      "grad_norm": 0.06856367737054825,
      "kl": 0.09061568230390549,
      "learning_rate": 3e-06,
      "loss": -0.0085,
      "step": 1673
    },
    {
      "clip_ratio": 0.0005575706818490289,
      "epoch": 0.004647443905851782,
      "grad_norm": 0.09739033877849579,
      "kl": 0.09403853490948677,
      "learning_rate": 3e-06,
      "loss": -0.0085,
      "step": 1674
    },
    {
      "clip_ratio": 9.412650251761079e-05,
      "epoch": 0.004650220156691597,
      "grad_norm": 0.06736251711845398,
      "kl": 0.09165050461888313,
      "learning_rate": 3e-06,
      "loss": -0.0087,
      "step": 1675
    },
    {
      "clip_ratio": 0.00017174614913528785,
      "epoch": 0.0046529964075314135,
      "grad_norm": 0.0773087590932846,
      "kl": 0.09735730290412903,
      "learning_rate": 3e-06,
      "loss": -0.0086,
      "step": 1676
    },
    {
      "clip_ratio": 0.001088498393073678,
      "epoch": 0.004655772658371229,
      "grad_norm": 0.05607219412922859,
      "kl": 0.0904586911201477,
      "learning_rate": 3e-06,
      "loss": -0.009,
      "step": 1677
    },
    {
      "clip_ratio": 0.0009852994699031115,
      "epoch": 0.004658548909211045,
      "grad_norm": 0.07360868901014328,
      "kl": 0.08624210581183434,
      "learning_rate": 3e-06,
      "loss": -0.0091,
      "step": 1678
    },
    {
      "clip_ratio": 0.0013295641401782632,
      "epoch": 0.004661325160050861,
      "grad_norm": 0.06193244829773903,
      "kl": 0.07950897514820099,
      "learning_rate": 3e-06,
      "loss": -0.0099,
      "step": 1679
    },
    {
      "clip_ratio": 0.001959454733878374,
      "epoch": 0.004664101410890677,
      "grad_norm": 0.06660173088312149,
      "kl": 0.0817050151526928,
      "learning_rate": 3e-06,
      "loss": -0.0093,
      "step": 1680
    },
    {
      "clip_ratio": 0.00019872814300470054,
      "completion_length": 224.27083587646484,
      "epoch": 0.004666877661730492,
      "grad_norm": 0.11621461063623428,
      "kl": 0.09354111552238464,
      "learning_rate": 3e-06,
      "loss": -0.0011,
      "reward": 0.3020833432674408,
      "reward_std": 0.2798703759908676,
      "rewards/countdown_reward_func": 0.3020833358168602,
      "step": 1681,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 8.644536865176633e-05,
      "epoch": 0.0046696539125703086,
      "grad_norm": 0.09669879823923111,
      "kl": 0.07669191434979439,
      "learning_rate": 3e-06,
      "loss": -0.0009,
      "step": 1682
    },
    {
      "clip_ratio": 0.0003749770621652715,
      "epoch": 0.004672430163410125,
      "grad_norm": 0.10819410532712936,
      "kl": 0.08729588240385056,
      "learning_rate": 3e-06,
      "loss": -0.001,
      "step": 1683
    },
    {
      "clip_ratio": 8.716875890968367e-05,
      "epoch": 0.00467520641424994,
      "grad_norm": 0.08888690173625946,
      "kl": 0.07762591540813446,
      "learning_rate": 3e-06,
      "loss": -0.0003,
      "step": 1684
    },
    {
      "clip_ratio": 8.704735228093341e-05,
      "epoch": 0.0046779826650897565,
      "grad_norm": 0.08127215504646301,
      "kl": 0.08648081123828888,
      "learning_rate": 3e-06,
      "loss": -0.0002,
      "step": 1685
    },
    {
      "clip_ratio": 8.704735228093341e-05,
      "epoch": 0.004680758915929572,
      "grad_norm": 0.08273155987262726,
      "kl": 0.0694781243801117,
      "learning_rate": 3e-06,
      "loss": -0.0011,
      "step": 1686
    },
    {
      "clip_ratio": 0.0006585983792319894,
      "epoch": 0.004683535166769388,
      "grad_norm": 0.1248389482498169,
      "kl": 0.08379602804780006,
      "learning_rate": 3e-06,
      "loss": -0.0014,
      "step": 1687
    },
    {
      "clip_ratio": 0.00037935634463792667,
      "epoch": 0.004686311417609204,
      "grad_norm": 0.09269773215055466,
      "kl": 0.06929580494761467,
      "learning_rate": 3e-06,
      "loss": -0.0026,
      "step": 1688
    },
    {
      "clip_ratio": 0.0001979566877707839,
      "epoch": 0.00468908766844902,
      "grad_norm": 0.10713022947311401,
      "kl": 0.08045705035328865,
      "learning_rate": 3e-06,
      "loss": -0.0023,
      "step": 1689
    },
    {
      "clip_ratio": 0.0003845375686069019,
      "epoch": 0.004691863919288835,
      "grad_norm": 0.093922920525074,
      "kl": 0.07109234854578972,
      "learning_rate": 3e-06,
      "loss": -0.001,
      "step": 1690
    },
    {
      "clip_ratio": 0.000683231744915247,
      "epoch": 0.0046946401701286515,
      "grad_norm": 0.08436852693557739,
      "kl": 0.07955209910869598,
      "learning_rate": 3e-06,
      "loss": -0.0011,
      "step": 1691
    },
    {
      "clip_ratio": 0.0004670836788136512,
      "epoch": 0.004697416420968467,
      "grad_norm": 0.07506747543811798,
      "kl": 0.06490103155374527,
      "learning_rate": 3e-06,
      "loss": -0.0018,
      "step": 1692
    },
    {
      "clip_ratio": 0.0004444956357474439,
      "completion_length": 228.9791717529297,
      "epoch": 0.004700192671808283,
      "grad_norm": 0.08811832219362259,
      "kl": 0.07729529216885567,
      "learning_rate": 3e-06,
      "loss": -0.0021,
      "reward": 0.2645833492279053,
      "reward_std": 0.23099135607481003,
      "rewards/countdown_reward_func": 0.2645833417773247,
      "step": 1693,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0047029689226480995,
      "grad_norm": 0.13022494316101074,
      "kl": 0.07385160028934479,
      "learning_rate": 3e-06,
      "loss": -0.0017,
      "step": 1694
    },
    {
      "clip_ratio": 0.0014592632069252431,
      "epoch": 0.004705745173487915,
      "grad_norm": 0.18172861635684967,
      "kl": 0.07337892800569534,
      "learning_rate": 3e-06,
      "loss": -0.0003,
      "step": 1695
    },
    {
      "clip_ratio": 0.0005964136798866093,
      "epoch": 0.004708521424327731,
      "grad_norm": 0.0807371437549591,
      "kl": 0.07234909385442734,
      "learning_rate": 3e-06,
      "loss": -0.0011,
      "step": 1696
    },
    {
      "clip_ratio": 0.0004422512211021967,
      "epoch": 0.004711297675167547,
      "grad_norm": 0.07396223396062851,
      "kl": 0.07509288936853409,
      "learning_rate": 3e-06,
      "loss": -0.002,
      "step": 1697
    },
    {
      "clip_ratio": 0.0003952159022446722,
      "epoch": 0.004714073926007363,
      "grad_norm": 0.09103484451770782,
      "kl": 0.08038389682769775,
      "learning_rate": 3e-06,
      "loss": -0.0014,
      "step": 1698
    },
    {
      "clip_ratio": 9.873617818811908e-05,
      "epoch": 0.004716850176847178,
      "grad_norm": 0.1258394420146942,
      "kl": 0.07613859698176384,
      "learning_rate": 3e-06,
      "loss": -0.0025,
      "step": 1699
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0047196264276869945,
      "grad_norm": 0.12652093172073364,
      "kl": 0.07367661595344543,
      "learning_rate": 3e-06,
      "loss": -0.0034,
      "step": 1700
    },
    {
      "clip_ratio": 0.0015260791988112032,
      "epoch": 0.00472240267852681,
      "grad_norm": 0.07796800136566162,
      "kl": 0.07423160970211029,
      "learning_rate": 3e-06,
      "loss": -0.0021,
      "step": 1701
    },
    {
      "clip_ratio": 0.00018615041335579008,
      "epoch": 0.004725178929366626,
      "grad_norm": 0.0833539217710495,
      "kl": 0.07227658852934837,
      "learning_rate": 3e-06,
      "loss": -0.0026,
      "step": 1702
    },
    {
      "clip_ratio": 0.0006381928396876901,
      "epoch": 0.004727955180206442,
      "grad_norm": 0.08434394001960754,
      "kl": 0.07486193627119064,
      "learning_rate": 3e-06,
      "loss": -0.0027,
      "step": 1703
    },
    {
      "clip_ratio": 0.0013060531928204,
      "epoch": 0.004730731431046258,
      "grad_norm": 0.07230468839406967,
      "kl": 0.08221058547496796,
      "learning_rate": 3e-06,
      "loss": -0.0026,
      "step": 1704
    },
    {
      "clip_ratio": 0.0002603422835818492,
      "completion_length": 229.4166717529297,
      "epoch": 0.004733507681886074,
      "grad_norm": 0.10037482529878616,
      "kl": 0.06833513081073761,
      "learning_rate": 3e-06,
      "loss": 0.0286,
      "reward": 0.36249999701976776,
      "reward_std": 0.33992572128772736,
      "rewards/countdown_reward_func": 0.36249999701976776,
      "step": 1705,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0047362839327258896,
      "grad_norm": 0.10078221559524536,
      "kl": 0.06954888440668583,
      "learning_rate": 3e-06,
      "loss": 0.0288,
      "step": 1706
    },
    {
      "clip_ratio": 0.00036663911305367947,
      "epoch": 0.004739060183565706,
      "grad_norm": 0.08536067605018616,
      "kl": 0.07179216295480728,
      "learning_rate": 3e-06,
      "loss": 0.028,
      "step": 1707
    },
    {
      "clip_ratio": 0.0001773409530869685,
      "epoch": 0.004741836434405521,
      "grad_norm": 0.11112114787101746,
      "kl": 0.061546871438622475,
      "learning_rate": 3e-06,
      "loss": 0.0281,
      "step": 1708
    },
    {
      "clip_ratio": 0.00025751072098501027,
      "epoch": 0.0047446126852453375,
      "grad_norm": 0.24011927843093872,
      "kl": 0.07169570401310921,
      "learning_rate": 3e-06,
      "loss": 0.0287,
      "step": 1709
    },
    {
      "clip_ratio": 0.0001065643664333038,
      "epoch": 0.004747388936085153,
      "grad_norm": 0.09349286556243896,
      "kl": 0.07777471095323563,
      "learning_rate": 3e-06,
      "loss": 0.029,
      "step": 1710
    },
    {
      "clip_ratio": 9.433962259208784e-05,
      "epoch": 0.004750165186924969,
      "grad_norm": 0.10448563098907471,
      "kl": 0.07240425795316696,
      "learning_rate": 3e-06,
      "loss": 0.0277,
      "step": 1711
    },
    {
      "clip_ratio": 9.211496217176318e-05,
      "epoch": 0.004752941437764785,
      "grad_norm": 0.09998632222414017,
      "kl": 0.07552265375852585,
      "learning_rate": 3e-06,
      "loss": 0.0273,
      "step": 1712
    },
    {
      "clip_ratio": 9.433962259208784e-05,
      "epoch": 0.004755717688604601,
      "grad_norm": 0.09559866786003113,
      "kl": 0.07743804901838303,
      "learning_rate": 3e-06,
      "loss": 0.0268,
      "step": 1713
    },
    {
      "clip_ratio": 0.0007562700993730687,
      "epoch": 0.004758493939444416,
      "grad_norm": 0.10372816026210785,
      "kl": 0.06693735346198082,
      "learning_rate": 3e-06,
      "loss": 0.0273,
      "step": 1714
    },
    {
      "clip_ratio": 0.0005361997973523103,
      "epoch": 0.0047612701902842326,
      "grad_norm": 0.13456197082996368,
      "kl": 0.07971575111150742,
      "learning_rate": 3e-06,
      "loss": 0.0266,
      "step": 1715
    },
    {
      "clip_ratio": 0.00019023237109649926,
      "epoch": 0.004764046441124049,
      "grad_norm": 0.10750555992126465,
      "kl": 0.08734529465436935,
      "learning_rate": 3e-06,
      "loss": 0.0272,
      "step": 1716
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 239.375,
      "epoch": 0.004766822691963864,
      "grad_norm": 0.10056161880493164,
      "kl": 0.0764843225479126,
      "learning_rate": 3e-06,
      "loss": 0.0052,
      "reward": 0.3229166939854622,
      "reward_std": 0.3206951767206192,
      "rewards/countdown_reward_func": 0.3229166939854622,
      "step": 1717,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 0.00016971943114185706,
      "epoch": 0.0047695989428036805,
      "grad_norm": 0.0818033367395401,
      "kl": 0.0784323550760746,
      "learning_rate": 3e-06,
      "loss": 0.0056,
      "step": 1718
    },
    {
      "clip_ratio": 0.000244140625,
      "epoch": 0.004772375193643496,
      "grad_norm": 0.11514274030923843,
      "kl": 0.08092424646019936,
      "learning_rate": 3e-06,
      "loss": 0.0055,
      "step": 1719
    },
    {
      "clip_ratio": 0.00044265526230446994,
      "epoch": 0.004775151444483312,
      "grad_norm": 0.08406610041856766,
      "kl": 0.07746245339512825,
      "learning_rate": 3e-06,
      "loss": 0.0053,
      "step": 1720
    },
    {
      "clip_ratio": 0.00016971943114185706,
      "epoch": 0.004777927695323128,
      "grad_norm": 0.0868329182267189,
      "kl": 0.08470290526747704,
      "learning_rate": 3e-06,
      "loss": 0.0054,
      "step": 1721
    },
    {
      "clip_ratio": 9.07111752894707e-05,
      "epoch": 0.004780703946162944,
      "grad_norm": 0.11065360903739929,
      "kl": 0.08721869066357613,
      "learning_rate": 3e-06,
      "loss": 0.0062,
      "step": 1722
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004783480197002759,
      "grad_norm": 0.09971947222948074,
      "kl": 0.08457087725400925,
      "learning_rate": 3e-06,
      "loss": 0.0049,
      "step": 1723
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0047862564478425755,
      "grad_norm": 0.0916123166680336,
      "kl": 0.08624141290783882,
      "learning_rate": 3e-06,
      "loss": 0.0043,
      "step": 1724
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004789032698682391,
      "grad_norm": 0.09791970252990723,
      "kl": 0.08757534250617027,
      "learning_rate": 3e-06,
      "loss": 0.0052,
      "step": 1725
    },
    {
      "clip_ratio": 0.00026201604487141594,
      "epoch": 0.004791808949522207,
      "grad_norm": 0.11371166259050369,
      "kl": 0.08409228920936584,
      "learning_rate": 3e-06,
      "loss": 0.0047,
      "step": 1726
    },
    {
      "clip_ratio": 8.833922038320452e-05,
      "epoch": 0.0047945852003620235,
      "grad_norm": 0.08852825313806534,
      "kl": 0.09211602807044983,
      "learning_rate": 3e-06,
      "loss": 0.0046,
      "step": 1727
    },
    {
      "clip_ratio": 0.00016276042151730508,
      "epoch": 0.004797361451201839,
      "grad_norm": 0.08812572062015533,
      "kl": 0.0936342217028141,
      "learning_rate": 3e-06,
      "loss": 0.005,
      "step": 1728
    },
    {
      "clip_ratio": 8.333333244081587e-05,
      "completion_length": 226.58334350585938,
      "epoch": 0.004800137702041655,
      "grad_norm": 0.08161304146051407,
      "kl": 0.10271313786506653,
      "learning_rate": 3e-06,
      "loss": 0.0168,
      "reward": 0.28333336114883423,
      "reward_std": 0.21825158596038818,
      "rewards/countdown_reward_func": 0.28333333134651184,
      "step": 1729,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.00017384745297022164,
      "epoch": 0.004802913952881471,
      "grad_norm": 0.09780708700418472,
      "kl": 0.0883183516561985,
      "learning_rate": 3e-06,
      "loss": 0.0166,
      "step": 1730
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004805690203721287,
      "grad_norm": 0.18682555854320526,
      "kl": 0.09598564729094505,
      "learning_rate": 3e-06,
      "loss": 0.0164,
      "step": 1731
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004808466454561102,
      "grad_norm": 0.06634990125894547,
      "kl": 0.1002919152379036,
      "learning_rate": 3e-06,
      "loss": 0.0164,
      "step": 1732
    },
    {
      "clip_ratio": 8.947744936449453e-05,
      "epoch": 0.0048112427054009185,
      "grad_norm": 0.07831425219774246,
      "kl": 0.10307565331459045,
      "learning_rate": 3e-06,
      "loss": 0.0172,
      "step": 1733
    },
    {
      "clip_ratio": 0.0004471343127079308,
      "epoch": 0.004814018956240734,
      "grad_norm": 0.09252594411373138,
      "kl": 0.09812160581350327,
      "learning_rate": 3e-06,
      "loss": 0.0166,
      "step": 1734
    },
    {
      "clip_ratio": 0.00027552236133487895,
      "epoch": 0.00481679520708055,
      "grad_norm": 0.08623359352350235,
      "kl": 0.10652950406074524,
      "learning_rate": 3e-06,
      "loss": 0.0157,
      "step": 1735
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004819571457920366,
      "grad_norm": 0.08916810154914856,
      "kl": 0.0917881578207016,
      "learning_rate": 3e-06,
      "loss": 0.016,
      "step": 1736
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004822347708760182,
      "grad_norm": 0.11515611410140991,
      "kl": 0.09990720450878143,
      "learning_rate": 3e-06,
      "loss": 0.0151,
      "step": 1737
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004825123959599998,
      "grad_norm": 0.06613731384277344,
      "kl": 0.10350186005234718,
      "learning_rate": 3e-06,
      "loss": 0.0154,
      "step": 1738
    },
    {
      "clip_ratio": 9.051412052940577e-05,
      "epoch": 0.0048279002104398136,
      "grad_norm": 0.08245322108268738,
      "kl": 0.10670360550284386,
      "learning_rate": 3e-06,
      "loss": 0.0161,
      "step": 1739
    },
    {
      "clip_ratio": 8.947744936449453e-05,
      "epoch": 0.00483067646127963,
      "grad_norm": 0.08258913457393646,
      "kl": 0.10113772377371788,
      "learning_rate": 3e-06,
      "loss": 0.0151,
      "step": 1740
    },
    {
      "clip_ratio": 9.999999747378752e-05,
      "completion_length": 238.39584350585938,
      "epoch": 0.004833452712119445,
      "grad_norm": 0.08632933348417282,
      "kl": 0.10816295444965363,
      "learning_rate": 3e-06,
      "loss": 0.0146,
      "reward": 0.3395833671092987,
      "reward_std": 0.3079586625099182,
      "rewards/countdown_reward_func": 0.3395833522081375,
      "step": 1741,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.000183780153747648,
      "epoch": 0.0048362289629592615,
      "grad_norm": 0.10708604753017426,
      "kl": 0.10778484493494034,
      "learning_rate": 3e-06,
      "loss": 0.0143,
      "step": 1742
    },
    {
      "clip_ratio": 8.468834857922047e-05,
      "epoch": 0.004839005213799077,
      "grad_norm": 0.11992800235748291,
      "kl": 0.10616076365113258,
      "learning_rate": 3e-06,
      "loss": 0.0148,
      "step": 1743
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004841781464638893,
      "grad_norm": 0.08565586805343628,
      "kl": 0.11514625698328018,
      "learning_rate": 3e-06,
      "loss": 0.0148,
      "step": 1744
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004844557715478709,
      "grad_norm": 0.10972116142511368,
      "kl": 0.11541270092129707,
      "learning_rate": 3e-06,
      "loss": 0.0141,
      "step": 1745
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004847333966318525,
      "grad_norm": 0.11946596950292587,
      "kl": 0.11952852457761765,
      "learning_rate": 3e-06,
      "loss": 0.0145,
      "step": 1746
    },
    {
      "clip_ratio": 0.0003465483241598122,
      "epoch": 0.00485011021715834,
      "grad_norm": 0.1060066968202591,
      "kl": 0.11223845556378365,
      "learning_rate": 3e-06,
      "loss": 0.0138,
      "step": 1747
    },
    {
      "clip_ratio": 8.468834857922047e-05,
      "epoch": 0.0048528864679981566,
      "grad_norm": 0.09957956522703171,
      "kl": 0.11351840943098068,
      "learning_rate": 3e-06,
      "loss": 0.0139,
      "step": 1748
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004855662718837973,
      "grad_norm": 0.10374125838279724,
      "kl": 0.11228137835860252,
      "learning_rate": 3e-06,
      "loss": 0.0126,
      "step": 1749
    },
    {
      "clip_ratio": 0.0001720578147796914,
      "epoch": 0.004858438969677788,
      "grad_norm": 0.08856089413166046,
      "kl": 0.121010672301054,
      "learning_rate": 3e-06,
      "loss": 0.0141,
      "step": 1750
    },
    {
      "clip_ratio": 0.0001767400826793164,
      "epoch": 0.0048612152205176045,
      "grad_norm": 0.13084955513477325,
      "kl": 0.12098020315170288,
      "learning_rate": 3e-06,
      "loss": 0.0122,
      "step": 1751
    },
    {
      "clip_ratio": 0.00024654832668602467,
      "epoch": 0.00486399147135742,
      "grad_norm": 0.10557901859283447,
      "kl": 0.12543104588985443,
      "learning_rate": 3e-06,
      "loss": 0.0119,
      "step": 1752
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 231.7916717529297,
      "epoch": 0.004866767722197236,
      "grad_norm": 0.09326691925525665,
      "kl": 0.1041104681789875,
      "learning_rate": 3e-06,
      "loss": -0.0005,
      "reward": 0.3229166865348816,
      "reward_std": 0.2270909696817398,
      "rewards/countdown_reward_func": 0.3229166716337204,
      "step": 1753,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio": 0.00018162409833166748,
      "epoch": 0.004869543973037052,
      "grad_norm": 0.1563946008682251,
      "kl": 0.10441932827234268,
      "learning_rate": 3e-06,
      "loss": 0.001,
      "step": 1754
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004872320223876868,
      "grad_norm": 0.09415509551763535,
      "kl": 0.10655347630381584,
      "learning_rate": 3e-06,
      "loss": 0.0006,
      "step": 1755
    },
    {
      "clip_ratio": 0.0001735860641929321,
      "epoch": 0.004875096474716683,
      "grad_norm": 0.07791784405708313,
      "kl": 0.11189163848757744,
      "learning_rate": 3e-06,
      "loss": 0.001,
      "step": 1756
    },
    {
      "clip_ratio": 0.0005474452627822757,
      "epoch": 0.0048778727255564995,
      "grad_norm": 0.07700037211179733,
      "kl": 0.11374571919441223,
      "learning_rate": 3e-06,
      "loss": 0.0006,
      "step": 1757
    },
    {
      "clip_ratio": 0.0004572612378979102,
      "epoch": 0.004880648976396315,
      "grad_norm": 0.09134937077760696,
      "kl": 0.1076822318136692,
      "learning_rate": 3e-06,
      "loss": -0.0005,
      "step": 1758
    },
    {
      "clip_ratio": 0.0005507961686816998,
      "epoch": 0.004883425227236131,
      "grad_norm": 0.11843130737543106,
      "kl": 0.1056605763733387,
      "learning_rate": 3e-06,
      "loss": -0.0003,
      "step": 1759
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0048862014780759475,
      "grad_norm": 0.16378162801265717,
      "kl": 0.10637029260396957,
      "learning_rate": 3e-06,
      "loss": 0.0001,
      "step": 1760
    },
    {
      "clip_ratio": 0.00017188674246426672,
      "epoch": 0.004888977728915763,
      "grad_norm": 0.09075536578893661,
      "kl": 0.10630379244685173,
      "learning_rate": 3e-06,
      "loss": -0.0009,
      "step": 1761
    },
    {
      "clip_ratio": 0.00036519287823466584,
      "epoch": 0.004891753979755579,
      "grad_norm": 0.07516534626483917,
      "kl": 0.11119216680526733,
      "learning_rate": 3e-06,
      "loss": -0.0004,
      "step": 1762
    },
    {
      "clip_ratio": 0.00018491214723326266,
      "epoch": 0.004894530230595395,
      "grad_norm": 0.07346532493829727,
      "kl": 0.1081731989979744,
      "learning_rate": 3e-06,
      "loss": -0.0006,
      "step": 1763
    },
    {
      "clip_ratio": 0.0012658692721743137,
      "epoch": 0.004897306481435211,
      "grad_norm": 0.09767357259988785,
      "kl": 0.10378069430589676,
      "learning_rate": 3e-06,
      "loss": -0.0013,
      "step": 1764
    },
    {
      "clip_ratio": 0.000739818497095257,
      "completion_length": 228.83333587646484,
      "epoch": 0.004900082732275026,
      "grad_norm": 0.04857170954346657,
      "kl": 0.11081210523843765,
      "learning_rate": 3e-06,
      "loss": 0.0008,
      "reward": 0.13333334028720856,
      "reward_std": 0.09824509173631668,
      "rewards/countdown_reward_func": 0.13333333283662796,
      "step": 1765,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio": 0.001252438290975988,
      "epoch": 0.0049028589831148425,
      "grad_norm": 0.06208095699548721,
      "kl": 0.11198806017637253,
      "learning_rate": 3e-06,
      "loss": 0.0006,
      "step": 1766
    },
    {
      "clip_ratio": 0.00036127932253293693,
      "epoch": 0.004905635233954658,
      "grad_norm": 0.06156253442168236,
      "kl": 0.11192040145397186,
      "learning_rate": 3e-06,
      "loss": 0.0009,
      "step": 1767
    },
    {
      "clip_ratio": 0.0002693784481380135,
      "epoch": 0.004908411484794474,
      "grad_norm": 0.05420385301113129,
      "kl": 0.10709425061941147,
      "learning_rate": 3e-06,
      "loss": 0.001,
      "step": 1768
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.00491118773563429,
      "grad_norm": 0.054946351796388626,
      "kl": 0.10676027089357376,
      "learning_rate": 3e-06,
      "loss": 0.0004,
      "step": 1769
    },
    {
      "clip_ratio": 0.0008511265332344919,
      "epoch": 0.004913963986474106,
      "grad_norm": 0.03839809074997902,
      "kl": 0.10732469335198402,
      "learning_rate": 3e-06,
      "loss": 0.0007,
      "step": 1770
    },
    {
      "clip_ratio": 0.0003610541461966932,
      "epoch": 0.004916740237313922,
      "grad_norm": 0.046104252338409424,
      "kl": 0.10469045117497444,
      "learning_rate": 3e-06,
      "loss": 0.0,
      "step": 1771
    },
    {
      "clip_ratio": 0.0015279441722668707,
      "epoch": 0.0049195164881537376,
      "grad_norm": 0.06267226487398148,
      "kl": 0.10644160956144333,
      "learning_rate": 3e-06,
      "loss": 0.0,
      "step": 1772
    },
    {
      "clip_ratio": 0.0001966938652913086,
      "epoch": 0.004922292738993554,
      "grad_norm": 0.05547748878598213,
      "kl": 0.10656990110874176,
      "learning_rate": 3e-06,
      "loss": 0.0001,
      "step": 1773
    },
    {
      "clip_ratio": 0.0008054111385717988,
      "epoch": 0.004925068989833369,
      "grad_norm": 0.0485399030148983,
      "kl": 0.10373036935925484,
      "learning_rate": 3e-06,
      "loss": 0.0,
      "step": 1774
    },
    {
      "clip_ratio": 0.0011505663569550961,
      "epoch": 0.0049278452406731855,
      "grad_norm": 0.09875226765871048,
      "kl": 0.10132811218500137,
      "learning_rate": 3e-06,
      "loss": -0.0,
      "step": 1775
    },
    {
      "clip_ratio": 0.0020636909175664186,
      "epoch": 0.004930621491513001,
      "grad_norm": 0.042701300233602524,
      "kl": 0.10125269740819931,
      "learning_rate": 3e-06,
      "loss": 0.0002,
      "step": 1776
    },
    {
      "clip_ratio": 0.0004316127669881098,
      "completion_length": 223.7291717529297,
      "epoch": 0.004933397742352817,
      "grad_norm": 0.37167254090309143,
      "kl": 0.10485289990901947,
      "learning_rate": 3e-06,
      "loss": 0.0251,
      "reward": 0.3812500238418579,
      "reward_std": 0.350497841835022,
      "rewards/countdown_reward_func": 0.3812499940395355,
      "step": 1777,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004936173993192633,
      "grad_norm": 0.1220475360751152,
      "kl": 0.10859641060233116,
      "learning_rate": 3e-06,
      "loss": 0.0243,
      "step": 1778
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004938950244032449,
      "grad_norm": 0.11971476674079895,
      "kl": 0.10970620810985565,
      "learning_rate": 3e-06,
      "loss": 0.0248,
      "step": 1779
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004941726494872264,
      "grad_norm": 0.11429105699062347,
      "kl": 0.09786089137196541,
      "learning_rate": 3e-06,
      "loss": 0.0239,
      "step": 1780
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0049445027457120806,
      "grad_norm": 0.10652303695678711,
      "kl": 0.10274695977568626,
      "learning_rate": 3e-06,
      "loss": 0.0252,
      "step": 1781
    },
    {
      "clip_ratio": 0.000254739832598716,
      "epoch": 0.004947278996551897,
      "grad_norm": 0.12241549789905548,
      "kl": 0.10112031176686287,
      "learning_rate": 3e-06,
      "loss": 0.0244,
      "step": 1782
    },
    {
      "clip_ratio": 0.00017335961456410587,
      "epoch": 0.004950055247391712,
      "grad_norm": 0.1391787976026535,
      "kl": 0.10613827779889107,
      "learning_rate": 3e-06,
      "loss": 0.0237,
      "step": 1783
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0049528314982315285,
      "grad_norm": 0.17388495802879333,
      "kl": 0.10883147642016411,
      "learning_rate": 3e-06,
      "loss": 0.0232,
      "step": 1784
    },
    {
      "clip_ratio": 0.00029968634044053033,
      "epoch": 0.004955607749071344,
      "grad_norm": 0.11082387715578079,
      "kl": 0.11147769913077354,
      "learning_rate": 3e-06,
      "loss": 0.0236,
      "step": 1785
    },
    {
      "clip_ratio": 0.0001846859959186986,
      "epoch": 0.00495838399991116,
      "grad_norm": 0.11685911566019058,
      "kl": 0.09964372962713242,
      "learning_rate": 3e-06,
      "loss": 0.0226,
      "step": 1786
    },
    {
      "clip_ratio": 0.00011281588376732543,
      "epoch": 0.004961160250750976,
      "grad_norm": 0.09485632926225662,
      "kl": 0.10617783665657043,
      "learning_rate": 3e-06,
      "loss": 0.0236,
      "step": 1787
    },
    {
      "clip_ratio": 0.00043700785317923874,
      "epoch": 0.004963936501590792,
      "grad_norm": 0.11837552487850189,
      "kl": 0.10385986790060997,
      "learning_rate": 3e-06,
      "loss": 0.0227,
      "step": 1788
    },
    {
      "clip_ratio": 0.00016897656314540654,
      "completion_length": 237.0416717529297,
      "epoch": 0.004966712752430607,
      "grad_norm": 0.12939836084842682,
      "kl": 0.1072714664041996,
      "learning_rate": 3e-06,
      "loss": 0.0052,
      "reward": 0.3437500298023224,
      "reward_std": 0.37016281485557556,
      "rewards/countdown_reward_func": 0.3437500298023224,
      "step": 1789,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0049694890032704235,
      "grad_norm": 0.11611343175172806,
      "kl": 0.11076124012470245,
      "learning_rate": 3e-06,
      "loss": 0.0048,
      "step": 1790
    },
    {
      "clip_ratio": 0.0003420523353270255,
      "epoch": 0.004972265254110239,
      "grad_norm": 0.11028125882148743,
      "kl": 0.11113661155104637,
      "learning_rate": 3e-06,
      "loss": 0.0043,
      "step": 1791
    },
    {
      "clip_ratio": 0.00017754628788679838,
      "epoch": 0.004975041504950055,
      "grad_norm": 0.09910908341407776,
      "kl": 0.11355148255825043,
      "learning_rate": 3e-06,
      "loss": 0.0056,
      "step": 1792
    },
    {
      "clip_ratio": 0.00017655367264524102,
      "epoch": 0.0049778177557898715,
      "grad_norm": 0.0909920260310173,
      "kl": 0.115181814879179,
      "learning_rate": 3e-06,
      "loss": 0.0045,
      "step": 1793
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.004980594006629687,
      "grad_norm": 0.08922834694385529,
      "kl": 0.10475178062915802,
      "learning_rate": 3e-06,
      "loss": 0.0044,
      "step": 1794
    },
    {
      "clip_ratio": 8.7596352386754e-05,
      "epoch": 0.004983370257469503,
      "grad_norm": 0.08417384326457977,
      "kl": 0.11183957010507584,
      "learning_rate": 3e-06,
      "loss": 0.0046,
      "step": 1795
    },
    {
      "clip_ratio": 8.7596352386754e-05,
      "epoch": 0.004986146508309319,
      "grad_norm": 0.11513616889715195,
      "kl": 0.11391586065292358,
      "learning_rate": 3e-06,
      "loss": 0.0035,
      "step": 1796
    },
    {
      "clip_ratio": 0.00017655367264524102,
      "epoch": 0.004988922759149135,
      "grad_norm": 0.09690085053443909,
      "kl": 0.1138366088271141,
      "learning_rate": 3e-06,
      "loss": 0.0038,
      "step": 1797
    },
    {
      "clip_ratio": 0.0008805023971945047,
      "epoch": 0.00499169900998895,
      "grad_norm": 0.1030769795179367,
      "kl": 0.11408108472824097,
      "learning_rate": 3e-06,
      "loss": 0.0052,
      "step": 1798
    },
    {
      "clip_ratio": 0.0005165053516975604,
      "epoch": 0.0049944752608287665,
      "grad_norm": 0.08906270563602448,
      "kl": 0.11490156501531601,
      "learning_rate": 3e-06,
      "loss": 0.0045,
      "step": 1799
    },
    {
      "clip_ratio": 8.827683632262051e-05,
      "epoch": 0.004997251511668582,
      "grad_norm": 0.09603746235370636,
      "kl": 0.10548309236764908,
      "learning_rate": 3e-06,
      "loss": 0.0031,
      "step": 1800
    },
    {
      "clip_ratio": 0.0002600554726086557,
      "completion_length": 222.20834350585938,
      "epoch": 0.005000027762508398,
      "grad_norm": 0.0949837937951088,
      "kl": 0.10473751276731491,
      "learning_rate": 3e-06,
      "loss": 0.0252,
      "reward": 0.302083358168602,
      "reward_std": 0.2531573101878166,
      "rewards/countdown_reward_func": 0.302083358168602,
      "step": 1801,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 8.716875890968367e-05,
      "epoch": 0.005002804013348214,
      "grad_norm": 0.13353735208511353,
      "kl": 0.09882612898945808,
      "learning_rate": 3e-06,
      "loss": 0.0252,
      "step": 1802
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00500558026418803,
      "grad_norm": 0.12187909334897995,
      "kl": 0.10175449773669243,
      "learning_rate": 3e-06,
      "loss": 0.0253,
      "step": 1803
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.005008356515027846,
      "grad_norm": 0.13179193437099457,
      "kl": 0.10488756746053696,
      "learning_rate": 3e-06,
      "loss": 0.0246,
      "step": 1804
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0050111327658676616,
      "grad_norm": 0.10637587308883667,
      "kl": 0.10015484690666199,
      "learning_rate": 3e-06,
      "loss": 0.0246,
      "step": 1805
    },
    {
      "clip_ratio": 8.668516238685697e-05,
      "epoch": 0.005013909016707478,
      "grad_norm": 0.10470699518918991,
      "kl": 0.10087545961141586,
      "learning_rate": 3e-06,
      "loss": 0.0246,
      "step": 1806
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.005016685267547293,
      "grad_norm": 0.1398010551929474,
      "kl": 0.10762974619865417,
      "learning_rate": 3e-06,
      "loss": 0.0242,
      "step": 1807
    },
    {
      "clip_ratio": 0.00020646479970309883,
      "epoch": 0.0050194615183871095,
      "grad_norm": 0.19034115970134735,
      "kl": 0.10274334251880646,
      "learning_rate": 3e-06,
      "loss": 0.0227,
      "step": 1808
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.005022237769226925,
      "grad_norm": 0.11217880249023438,
      "kl": 0.10773766785860062,
      "learning_rate": 3e-06,
      "loss": 0.023,
      "step": 1809
    },
    {
      "clip_ratio": 0.00010822511103469878,
      "epoch": 0.005025014020066741,
      "grad_norm": 0.1465480774641037,
      "kl": 0.11147905141115189,
      "learning_rate": 3e-06,
      "loss": 0.0215,
      "step": 1810
    },
    {
      "clip_ratio": 0.0004162504119449295,
      "epoch": 0.005027790270906557,
      "grad_norm": 0.0986005887389183,
      "kl": 0.10899289697408676,
      "learning_rate": 3e-06,
      "loss": 0.0235,
      "step": 1811
    },
    {
      "clip_ratio": 9.788566967472434e-05,
      "epoch": 0.005030566521746373,
      "grad_norm": 0.13088902831077576,
      "kl": 0.1097114235162735,
      "learning_rate": 3e-06,
      "loss": 0.023,
      "step": 1812
    },
    {
      "clip_ratio": 0.0002538930275477469,
      "completion_length": 213.83333587646484,
      "epoch": 0.005033342772586188,
      "grad_norm": 0.09580644220113754,
      "kl": 0.10691358521580696,
      "learning_rate": 3e-06,
      "loss": -0.001,
      "reward": 0.3958333432674408,
      "reward_std": 0.2887437641620636,
      "rewards/countdown_reward_func": 0.3958333283662796,
      "step": 1813,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0050361190234260046,
      "grad_norm": 0.08536209911108017,
      "kl": 0.11967135220766068,
      "learning_rate": 3e-06,
      "loss": -0.001,
      "step": 1814
    },
    {
      "clip_ratio": 0.00019878626335412264,
      "epoch": 0.005038895274265821,
      "grad_norm": 0.0959966704249382,
      "kl": 0.11832546070218086,
      "learning_rate": 3e-06,
      "loss": -0.0003,
      "step": 1815
    },
    {
      "clip_ratio": 0.0021427306346595287,
      "epoch": 0.005041671525105636,
      "grad_norm": 0.11329105496406555,
      "kl": 0.12149709090590477,
      "learning_rate": 3e-06,
      "loss": 0.0015,
      "step": 1816
    },
    {
      "clip_ratio": 0.00026538767997408286,
      "epoch": 0.0050444477759454525,
      "grad_norm": 0.12283118814229965,
      "kl": 0.12353919818997383,
      "learning_rate": 3e-06,
      "loss": -0.0004,
      "step": 1817
    },
    {
      "clip_ratio": 0.0004591475590132177,
      "epoch": 0.005047224026785268,
      "grad_norm": 0.11809934675693512,
      "kl": 0.11520658433437347,
      "learning_rate": 3e-06,
      "loss": -0.002,
      "step": 1818
    },
    {
      "clip_ratio": 0.00027188926469534636,
      "epoch": 0.005050000277625084,
      "grad_norm": 0.10198795795440674,
      "kl": 0.11487729102373123,
      "learning_rate": 3e-06,
      "loss": -0.001,
      "step": 1819
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0050527765284649,
      "grad_norm": 0.09649267792701721,
      "kl": 0.12504612654447556,
      "learning_rate": 3e-06,
      "loss": -0.0014,
      "step": 1820
    },
    {
      "clip_ratio": 0.0004089476424269378,
      "epoch": 0.005055552779304716,
      "grad_norm": 0.2582434117794037,
      "kl": 0.12240016460418701,
      "learning_rate": 3e-06,
      "loss": -0.0011,
      "step": 1821
    },
    {
      "clip_ratio": 0.0019799493966274895,
      "epoch": 0.005058329030144531,
      "grad_norm": 0.12102940678596497,
      "kl": 0.12317222356796265,
      "learning_rate": 3e-06,
      "loss": -0.0002,
      "step": 1822
    },
    {
      "clip_ratio": 0.00018400746921543032,
      "epoch": 0.0050611052809843475,
      "grad_norm": 0.12739579379558563,
      "kl": 0.12464824318885803,
      "learning_rate": 3e-06,
      "loss": -0.0008,
      "step": 1823
    },
    {
      "clip_ratio": 0.00020525451691355556,
      "epoch": 0.005063881531824163,
      "grad_norm": 0.11777213960886002,
      "kl": 0.11386393383145332,
      "learning_rate": 3e-06,
      "loss": -0.0031,
      "step": 1824
    },
    {
      "clip_ratio": 0.0003442906090640463,
      "completion_length": 233.89584350585938,
      "epoch": 0.005066657782663979,
      "grad_norm": 0.07550974935293198,
      "kl": 0.13742565363645554,
      "learning_rate": 3e-06,
      "loss": 0.014,
      "reward": 0.24791669100522995,
      "reward_std": 0.2587834596633911,
      "rewards/countdown_reward_func": 0.24791667610406876,
      "step": 1825,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.00044292007805779576,
      "epoch": 0.0050694340335037955,
      "grad_norm": 0.09571371972560883,
      "kl": 0.13907401263713837,
      "learning_rate": 3e-06,
      "loss": 0.0135,
      "step": 1826
    },
    {
      "clip_ratio": 0.00018455334065947682,
      "epoch": 0.005072210284343611,
      "grad_norm": 0.11037315428256989,
      "kl": 0.13096477836370468,
      "learning_rate": 3e-06,
      "loss": 0.0139,
      "step": 1827
    },
    {
      "clip_ratio": 0.0003636363544501364,
      "epoch": 0.005074986535183427,
      "grad_norm": 0.08234788477420807,
      "kl": 0.12375519797205925,
      "learning_rate": 3e-06,
      "loss": 0.0135,
      "step": 1828
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.005077762786023243,
      "grad_norm": 0.12358090281486511,
      "kl": 0.1380329132080078,
      "learning_rate": 3e-06,
      "loss": 0.0136,
      "step": 1829
    },
    {
      "clip_ratio": 0.0005509999464266002,
      "epoch": 0.005080539036863059,
      "grad_norm": 0.10022760182619095,
      "kl": 0.12769127637147903,
      "learning_rate": 3e-06,
      "loss": 0.0132,
      "step": 1830
    },
    {
      "clip_ratio": 0.00044850878475699574,
      "epoch": 0.005083315287702874,
      "grad_norm": 0.07287811487913132,
      "kl": 0.1334082931280136,
      "learning_rate": 3e-06,
      "loss": 0.0134,
      "step": 1831
    },
    {
      "clip_ratio": 0.0004338581129559316,
      "epoch": 0.0050860915385426905,
      "grad_norm": 0.07464426755905151,
      "kl": 0.13454636931419373,
      "learning_rate": 3e-06,
      "loss": 0.0137,
      "step": 1832
    },
    {
      "clip_ratio": 0.00045034874347038567,
      "epoch": 0.005088867789382506,
      "grad_norm": 0.12014682590961456,
      "kl": 0.12987623363733292,
      "learning_rate": 3e-06,
      "loss": 0.0129,
      "step": 1833
    },
    {
      "clip_ratio": 0.00018235020252177492,
      "epoch": 0.005091644040222322,
      "grad_norm": 0.07914389669895172,
      "kl": 0.12284610792994499,
      "learning_rate": 3e-06,
      "loss": 0.0123,
      "step": 1834
    },
    {
      "clip_ratio": 0.00017130826745415106,
      "epoch": 0.0050944202910621385,
      "grad_norm": 0.11240369826555252,
      "kl": 0.13621504604816437,
      "learning_rate": 3e-06,
      "loss": 0.0131,
      "step": 1835
    },
    {
      "clip_ratio": 0.0006353274511639029,
      "epoch": 0.005097196541901954,
      "grad_norm": 0.09824000298976898,
      "kl": 0.12700487673282623,
      "learning_rate": 3e-06,
      "loss": 0.0133,
      "step": 1836
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 230.31250762939453,
      "epoch": 0.00509997279274177,
      "grad_norm": 0.12020771205425262,
      "kl": 0.1342645138502121,
      "learning_rate": 3e-06,
      "loss": -0.0,
      "reward": 0.35833336412906647,
      "reward_std": 0.37625907361507416,
      "rewards/countdown_reward_func": 0.3583333492279053,
      "step": 1837,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0051027490435815856,
      "grad_norm": 0.17950089275836945,
      "kl": 0.1457306444644928,
      "learning_rate": 3e-06,
      "loss": 0.0,
      "step": 1838
    },
    {
      "clip_ratio": 9.968101949198171e-05,
      "epoch": 0.005105525294421402,
      "grad_norm": 0.150423064827919,
      "kl": 0.1225435845553875,
      "learning_rate": 3e-06,
      "loss": -0.0004,
      "step": 1839
    },
    {
      "clip_ratio": 0.0003465738918748684,
      "epoch": 0.005108301545261217,
      "grad_norm": 0.17598502337932587,
      "kl": 0.12997641041874886,
      "learning_rate": 3e-06,
      "loss": -0.0014,
      "step": 1840
    },
    {
      "clip_ratio": 0.0003827196196652949,
      "epoch": 0.0051110777961010335,
      "grad_norm": 0.13662053644657135,
      "kl": 0.12417108565568924,
      "learning_rate": 3e-06,
      "loss": -0.0021,
      "step": 1841
    },
    {
      "clip_ratio": 9.904913167702034e-05,
      "epoch": 0.005113854046940849,
      "grad_norm": 0.1616424322128296,
      "kl": 0.12268723547458649,
      "learning_rate": 3e-06,
      "loss": -0.0027,
      "step": 1842
    },
    {
      "clip_ratio": 0.0006147185195004568,
      "epoch": 0.005116630297780665,
      "grad_norm": 0.1300649344921112,
      "kl": 0.12857786938548088,
      "learning_rate": 3e-06,
      "loss": -0.0024,
      "step": 1843
    },
    {
      "clip_ratio": 0.00027205952210351825,
      "epoch": 0.005119406548620481,
      "grad_norm": 0.1779082864522934,
      "kl": 0.14096860587596893,
      "learning_rate": 3e-06,
      "loss": -0.0021,
      "step": 1844
    },
    {
      "clip_ratio": 0.00035519967786967754,
      "epoch": 0.005122182799460297,
      "grad_norm": 0.13828246295452118,
      "kl": 0.1195240318775177,
      "learning_rate": 3e-06,
      "loss": -0.004,
      "step": 1845
    },
    {
      "clip_ratio": 0.001379472087137401,
      "epoch": 0.005124959050300113,
      "grad_norm": 0.16142214834690094,
      "kl": 0.12471278756856918,
      "learning_rate": 3e-06,
      "loss": -0.0059,
      "step": 1846
    },
    {
      "clip_ratio": 0.001573986024595797,
      "epoch": 0.0051277353011399286,
      "grad_norm": 0.1403919756412506,
      "kl": 0.11798427999019623,
      "learning_rate": 3e-06,
      "loss": -0.0067,
      "step": 1847
    },
    {
      "clip_ratio": 0.0015037042612675577,
      "epoch": 0.005130511551979745,
      "grad_norm": 0.1678413301706314,
      "kl": 0.1188434436917305,
      "learning_rate": 3e-06,
      "loss": -0.0067,
      "step": 1848
    },
    {
      "clip_ratio": 0.0005114346859045327,
      "completion_length": 231.4375,
      "epoch": 0.00513328780281956,
      "grad_norm": 0.06092618405818939,
      "kl": 0.11399786919355392,
      "learning_rate": 3e-06,
      "loss": -0.0017,
      "reward": 0.1937500163912773,
      "reward_std": 0.15347465500235558,
      "rewards/countdown_reward_func": 0.1937500163912773,
      "step": 1849,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio": 8.802816591924056e-05,
      "epoch": 0.0051360640536593765,
      "grad_norm": 0.08002404868602753,
      "kl": 0.10806835442781448,
      "learning_rate": 3e-06,
      "loss": -0.0025,
      "step": 1850
    },
    {
      "clip_ratio": 0.0001694083766778931,
      "epoch": 0.005138840304499192,
      "grad_norm": 0.07181860506534576,
      "kl": 0.10447893664240837,
      "learning_rate": 3e-06,
      "loss": -0.0033,
      "step": 1851
    },
    {
      "clip_ratio": 0.0006369501352310181,
      "epoch": 0.005141616555339008,
      "grad_norm": 0.0680135115981102,
      "kl": 0.10594388097524643,
      "learning_rate": 3e-06,
      "loss": -0.0029,
      "step": 1852
    },
    {
      "clip_ratio": 0.0009738111984916031,
      "epoch": 0.005144392806178824,
      "grad_norm": 0.05742673948407173,
      "kl": 0.10426938533782959,
      "learning_rate": 3e-06,
      "loss": -0.0026,
      "step": 1853
    },
    {
      "clip_ratio": 0.0006258258072193712,
      "epoch": 0.00514716905701864,
      "grad_norm": 0.06563463062047958,
      "kl": 0.10713209956884384,
      "learning_rate": 3e-06,
      "loss": -0.003,
      "step": 1854
    },
    {
      "clip_ratio": 0.001724798115901649,
      "epoch": 0.005149945307858455,
      "grad_norm": 0.05824752897024155,
      "kl": 0.10814163088798523,
      "learning_rate": 3e-06,
      "loss": -0.0022,
      "step": 1855
    },
    {
      "clip_ratio": 0.0009082746400963515,
      "epoch": 0.0051527215586982715,
      "grad_norm": 0.08871683478355408,
      "kl": 0.10223837941884995,
      "learning_rate": 3e-06,
      "loss": -0.0039,
      "step": 1856
    },
    {
      "clip_ratio": 0.0010515126050449908,
      "epoch": 0.005155497809538088,
      "grad_norm": 0.0660201832652092,
      "kl": 0.10071967542171478,
      "learning_rate": 3e-06,
      "loss": -0.004,
      "step": 1857
    },
    {
      "clip_ratio": 0.002384283463470638,
      "epoch": 0.005158274060377903,
      "grad_norm": 0.0644756555557251,
      "kl": 0.10329937189817429,
      "learning_rate": 3e-06,
      "loss": -0.0041,
      "step": 1858
    },
    {
      "clip_ratio": 0.0010630089382175356,
      "epoch": 0.0051610503112177195,
      "grad_norm": 0.05238967016339302,
      "kl": 0.09916985407471657,
      "learning_rate": 3e-06,
      "loss": -0.0034,
      "step": 1859
    },
    {
      "clip_ratio": 0.0028576954500749707,
      "epoch": 0.005163826562057535,
      "grad_norm": 0.06547994166612625,
      "kl": 0.1038946770131588,
      "learning_rate": 3e-06,
      "loss": -0.0043,
      "step": 1860
    },
    {
      "clip_ratio": 8.46310067572631e-05,
      "completion_length": 236.08333587646484,
      "epoch": 0.005166602812897351,
      "grad_norm": 0.1372431516647339,
      "kl": 0.1432003378868103,
      "learning_rate": 3e-06,
      "loss": 0.0302,
      "reward": 0.3041666969656944,
      "reward_std": 0.27210913598537445,
      "rewards/countdown_reward_func": 0.3041666969656944,
      "step": 1861,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 8.46310067572631e-05,
      "epoch": 0.005169379063737167,
      "grad_norm": 0.08329330384731293,
      "kl": 0.12102905288338661,
      "learning_rate": 3e-06,
      "loss": 0.0297,
      "step": 1862
    },
    {
      "clip_ratio": 0.00017888411093736067,
      "epoch": 0.005172155314576983,
      "grad_norm": 0.10118658095598221,
      "kl": 0.13169753551483154,
      "learning_rate": 3e-06,
      "loss": 0.0301,
      "step": 1863
    },
    {
      "clip_ratio": 0.0002753450389718637,
      "epoch": 0.005174931565416798,
      "grad_norm": 0.11060193926095963,
      "kl": 0.13022061809897423,
      "learning_rate": 3e-06,
      "loss": 0.0292,
      "step": 1864
    },
    {
      "clip_ratio": 0.00017968667816603556,
      "epoch": 0.0051777078162566145,
      "grad_norm": 0.10579267144203186,
      "kl": 0.12778804078698158,
      "learning_rate": 3e-06,
      "loss": 0.0302,
      "step": 1865
    },
    {
      "clip_ratio": 8.46310067572631e-05,
      "epoch": 0.00518048406709643,
      "grad_norm": 0.0976695865392685,
      "kl": 0.11770972609519958,
      "learning_rate": 3e-06,
      "loss": 0.0288,
      "step": 1866
    },
    {
      "clip_ratio": 0.0003618215851020068,
      "epoch": 0.005183260317936246,
      "grad_norm": 0.09765175729990005,
      "kl": 0.14232248812913895,
      "learning_rate": 3e-06,
      "loss": 0.0292,
      "step": 1867
    },
    {
      "clip_ratio": 0.00016601121751591563,
      "epoch": 0.0051860365687760625,
      "grad_norm": 0.0782550796866417,
      "kl": 0.1213303953409195,
      "learning_rate": 3e-06,
      "loss": 0.0291,
      "step": 1868
    },
    {
      "clip_ratio": 0.0002776276451186277,
      "epoch": 0.005188812819615878,
      "grad_norm": 0.10285110771656036,
      "kl": 0.13297457247972488,
      "learning_rate": 3e-06,
      "loss": 0.0283,
      "step": 1869
    },
    {
      "clip_ratio": 0.00018646800162969157,
      "epoch": 0.005191589070455694,
      "grad_norm": 0.11395300924777985,
      "kl": 0.1316339075565338,
      "learning_rate": 3e-06,
      "loss": 0.0278,
      "step": 1870
    },
    {
      "clip_ratio": 0.0005108660116093233,
      "epoch": 0.0051943653212955096,
      "grad_norm": 0.0980798751115799,
      "kl": 0.12793593108654022,
      "learning_rate": 3e-06,
      "loss": 0.0287,
      "step": 1871
    },
    {
      "clip_ratio": 9.097525617107749e-05,
      "epoch": 0.005197141572135326,
      "grad_norm": 0.09456781297922134,
      "kl": 0.12113503366708755,
      "learning_rate": 3e-06,
      "loss": 0.0272,
      "step": 1872
    },
    {
      "clip_ratio": 8.85896515683271e-05,
      "completion_length": 233.06250762939453,
      "epoch": 0.005199917822975141,
      "grad_norm": 0.15110257267951965,
      "kl": 0.11747819557785988,
      "learning_rate": 3e-06,
      "loss": 0.0183,
      "reward": 0.3437500298023224,
      "reward_std": 0.31206804513931274,
      "rewards/countdown_reward_func": 0.3437500149011612,
      "step": 1873,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 9.897070412989706e-05,
      "epoch": 0.0052026940738149575,
      "grad_norm": 0.11430589109659195,
      "kl": 0.1168769858777523,
      "learning_rate": 3e-06,
      "loss": 0.0183,
      "step": 1874
    },
    {
      "clip_ratio": 8.90313385752961e-05,
      "epoch": 0.005205470324654773,
      "grad_norm": 0.09721416234970093,
      "kl": 0.11751173809170723,
      "learning_rate": 3e-06,
      "loss": 0.0188,
      "step": 1875
    },
    {
      "clip_ratio": 0.0004218236426822841,
      "epoch": 0.005208246575494589,
      "grad_norm": 0.09081301838159561,
      "kl": 0.11737838387489319,
      "learning_rate": 3e-06,
      "loss": 0.0181,
      "step": 1876
    },
    {
      "clip_ratio": 0.0003606221798690967,
      "epoch": 0.005211022826334405,
      "grad_norm": 0.08658932149410248,
      "kl": 0.12061323970556259,
      "learning_rate": 3e-06,
      "loss": 0.0184,
      "step": 1877
    },
    {
      "clip_ratio": 0.0004417505406308919,
      "epoch": 0.005213799077174221,
      "grad_norm": 0.13034342229366302,
      "kl": 0.11869553104043007,
      "learning_rate": 3e-06,
      "loss": 0.0181,
      "step": 1878
    },
    {
      "clip_ratio": 0.0003327302838442847,
      "epoch": 0.005216575328014037,
      "grad_norm": 0.09448829293251038,
      "kl": 0.12387342751026154,
      "learning_rate": 3e-06,
      "loss": 0.0167,
      "step": 1879
    },
    {
      "clip_ratio": 0.0003734165584319271,
      "epoch": 0.0052193515788538525,
      "grad_norm": 0.10919336974620819,
      "kl": 0.12406548857688904,
      "learning_rate": 3e-06,
      "loss": 0.0164,
      "step": 1880
    },
    {
      "clip_ratio": 8.85896515683271e-05,
      "epoch": 0.005222127829693669,
      "grad_norm": 0.09010238200426102,
      "kl": 0.1251124069094658,
      "learning_rate": 3e-06,
      "loss": 0.0178,
      "step": 1881
    },
    {
      "clip_ratio": 0.0004870776829193346,
      "epoch": 0.005224904080533484,
      "grad_norm": 0.08950203657150269,
      "kl": 0.1254933625459671,
      "learning_rate": 3e-06,
      "loss": 0.016,
      "step": 1882
    },
    {
      "clip_ratio": 0.0005361769872251898,
      "epoch": 0.0052276803313733005,
      "grad_norm": 0.1491861492395401,
      "kl": 0.13046596199274063,
      "learning_rate": 3e-06,
      "loss": 0.0163,
      "step": 1883
    },
    {
      "clip_ratio": 0.0005558350239880383,
      "epoch": 0.005230456582213116,
      "grad_norm": 0.12711560726165771,
      "kl": 0.12579327076673508,
      "learning_rate": 3e-06,
      "loss": 0.0155,
      "step": 1884
    },
    {
      "clip_ratio": 0.0005470910182339139,
      "completion_length": 228.58333587646484,
      "epoch": 0.005233232833052932,
      "grad_norm": 0.18066106736660004,
      "kl": 0.14193446934223175,
      "learning_rate": 3e-06,
      "loss": 0.0218,
      "reward": 0.34166671335697174,
      "reward_std": 0.3784969747066498,
      "rewards/countdown_reward_func": 0.34166669845581055,
      "step": 1885,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 0.000280211737845093,
      "epoch": 0.005236009083892748,
      "grad_norm": 0.1274838149547577,
      "kl": 0.13073249906301498,
      "learning_rate": 3e-06,
      "loss": 0.0227,
      "step": 1886
    },
    {
      "clip_ratio": 0.00027726277767214924,
      "epoch": 0.005238785334732564,
      "grad_norm": 0.11818882077932358,
      "kl": 0.14046981185674667,
      "learning_rate": 3e-06,
      "loss": 0.0234,
      "step": 1887
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.005241561585572379,
      "grad_norm": 0.12307964265346527,
      "kl": 0.13810279220342636,
      "learning_rate": 3e-06,
      "loss": 0.022,
      "step": 1888
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0052443378364121955,
      "grad_norm": 0.12825541198253632,
      "kl": 0.13892988115549088,
      "learning_rate": 3e-06,
      "loss": 0.0217,
      "step": 1889
    },
    {
      "clip_ratio": 9.170946577796713e-05,
      "epoch": 0.005247114087252012,
      "grad_norm": 0.14815275371074677,
      "kl": 0.15306030213832855,
      "learning_rate": 3e-06,
      "loss": 0.0225,
      "step": 1890
    },
    {
      "clip_ratio": 8.890469325706363e-05,
      "epoch": 0.005249890338091827,
      "grad_norm": 0.12983007729053497,
      "kl": 0.1546284407377243,
      "learning_rate": 3e-06,
      "loss": 0.0203,
      "step": 1891
    },
    {
      "clip_ratio": 0.00017390426364727318,
      "epoch": 0.0052526665889316435,
      "grad_norm": 0.20150499045848846,
      "kl": 0.13978148996829987,
      "learning_rate": 3e-06,
      "loss": 0.0197,
      "step": 1892
    },
    {
      "clip_ratio": 0.0015431393767357804,
      "epoch": 0.005255442839771459,
      "grad_norm": 0.12668222188949585,
      "kl": 0.14928098767995834,
      "learning_rate": 3e-06,
      "loss": 0.0219,
      "step": 1893
    },
    {
      "clip_ratio": 0.00017780938651412725,
      "epoch": 0.005258219090611275,
      "grad_norm": 0.1326785534620285,
      "kl": 0.14815984666347504,
      "learning_rate": 3e-06,
      "loss": 0.0205,
      "step": 1894
    },
    {
      "clip_ratio": 0.000360042235115543,
      "epoch": 0.005260995341451091,
      "grad_norm": 0.11318658292293549,
      "kl": 0.14500422030687332,
      "learning_rate": 3e-06,
      "loss": 0.0193,
      "step": 1895
    },
    {
      "clip_ratio": 0.00046394005767069757,
      "epoch": 0.005263771592290907,
      "grad_norm": 0.17353534698486328,
      "kl": 0.1581486538052559,
      "learning_rate": 3e-06,
      "loss": 0.0198,
      "step": 1896
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 227.6666717529297,
      "epoch": 0.005266547843130722,
      "grad_norm": 0.09441768378019333,
      "kl": 0.14793231338262558,
      "learning_rate": 3e-06,
      "loss": 0.0268,
      "reward": 0.19375000149011612,
      "reward_std": 0.16211743652820587,
      "rewards/countdown_reward_func": 0.19375000149011612,
      "step": 1897,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio": 0.00019291546777822077,
      "epoch": 0.0052693240939705385,
      "grad_norm": 0.06794694811105728,
      "kl": 0.15445201843976974,
      "learning_rate": 3e-06,
      "loss": 0.0273,
      "step": 1898
    },
    {
      "clip_ratio": 9.084302291739732e-05,
      "epoch": 0.005272100344810354,
      "grad_norm": 0.07045197486877441,
      "kl": 0.15664274990558624,
      "learning_rate": 3e-06,
      "loss": 0.0278,
      "step": 1899
    },
    {
      "clip_ratio": 0.0002881958498619497,
      "epoch": 0.00527487659565017,
      "grad_norm": 0.0634378045797348,
      "kl": 0.15410824120044708,
      "learning_rate": 3e-06,
      "loss": 0.0281,
      "step": 1900
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0052776528464899865,
      "grad_norm": 0.07607164233922958,
      "kl": 0.14435160160064697,
      "learning_rate": 3e-06,
      "loss": 0.027,
      "step": 1901
    },
    {
      "clip_ratio": 0.0004623479617293924,
      "epoch": 0.005280429097329802,
      "grad_norm": 0.08263766020536423,
      "kl": 0.14478224515914917,
      "learning_rate": 3e-06,
      "loss": 0.0261,
      "step": 1902
    },
    {
      "clip_ratio": 0.0003508084482746199,
      "epoch": 0.005283205348169618,
      "grad_norm": 0.08324983716011047,
      "kl": 0.15262345969676971,
      "learning_rate": 3e-06,
      "loss": 0.0265,
      "step": 1903
    },
    {
      "clip_ratio": 0.0002769144412013702,
      "epoch": 0.0052859815990094336,
      "grad_norm": 0.0647958442568779,
      "kl": 0.16446733474731445,
      "learning_rate": 3e-06,
      "loss": 0.0268,
      "step": 1904
    },
    {
      "clip_ratio": 0.0002789492136798799,
      "epoch": 0.00528875784984925,
      "grad_norm": 0.06370534002780914,
      "kl": 0.165434792637825,
      "learning_rate": 3e-06,
      "loss": 0.0274,
      "step": 1905
    },
    {
      "clip_ratio": 0.0009251071896869689,
      "epoch": 0.005291534100689065,
      "grad_norm": 0.06624593585729599,
      "kl": 0.1653437316417694,
      "learning_rate": 3e-06,
      "loss": 0.0266,
      "step": 1906
    },
    {
      "clip_ratio": 0.0005657712754327804,
      "epoch": 0.0052943103515288815,
      "grad_norm": 0.06858285516500473,
      "kl": 0.15655828267335892,
      "learning_rate": 3e-06,
      "loss": 0.0265,
      "step": 1907
    },
    {
      "clip_ratio": 0.0008327158866450191,
      "epoch": 0.005297086602368697,
      "grad_norm": 0.08369874209165573,
      "kl": 0.15721774101257324,
      "learning_rate": 3e-06,
      "loss": 0.0254,
      "step": 1908
    },
    {
      "clip_ratio": 8.239947055699304e-05,
      "completion_length": 220.08334350585938,
      "epoch": 0.005299862853208513,
      "grad_norm": 0.07524816691875458,
      "kl": 0.16231046617031097,
      "learning_rate": 3e-06,
      "loss": 0.0145,
      "reward": 0.1937500163912773,
      "reward_std": 0.1958785466849804,
      "rewards/countdown_reward_func": 0.1937500163912773,
      "step": 1909,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio": 9.057971328729764e-05,
      "epoch": 0.005302639104048329,
      "grad_norm": 0.06150933727622032,
      "kl": 0.17525289952754974,
      "learning_rate": 3e-06,
      "loss": 0.0154,
      "step": 1910
    },
    {
      "clip_ratio": 0.00010024057701230049,
      "epoch": 0.005305415354888145,
      "grad_norm": 0.06644804775714874,
      "kl": 0.17246182262897491,
      "learning_rate": 3e-06,
      "loss": 0.0141,
      "step": 1911
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.005308191605727961,
      "grad_norm": 0.11811656504869461,
      "kl": 0.18016871809959412,
      "learning_rate": 3e-06,
      "loss": 0.0154,
      "step": 1912
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0053109678565677765,
      "grad_norm": 0.06376418471336365,
      "kl": 0.16709844022989273,
      "learning_rate": 3e-06,
      "loss": 0.0137,
      "step": 1913
    },
    {
      "clip_ratio": 0.00010469011613167822,
      "epoch": 0.005313744107407593,
      "grad_norm": 0.055688947439193726,
      "kl": 0.16722818464040756,
      "learning_rate": 3e-06,
      "loss": 0.0141,
      "step": 1914
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.005316520358247408,
      "grad_norm": 0.0814003273844719,
      "kl": 0.16963288933038712,
      "learning_rate": 3e-06,
      "loss": 0.0142,
      "step": 1915
    },
    {
      "clip_ratio": 0.00029551039915531874,
      "epoch": 0.0053192966090872245,
      "grad_norm": 0.06685936450958252,
      "kl": 0.18109215795993805,
      "learning_rate": 3e-06,
      "loss": 0.0147,
      "step": 1916
    },
    {
      "clip_ratio": 0.00018481432925909758,
      "epoch": 0.00532207285992704,
      "grad_norm": 0.07375549525022507,
      "kl": 0.17562759667634964,
      "learning_rate": 3e-06,
      "loss": 0.0138,
      "step": 1917
    },
    {
      "clip_ratio": 0.0001017087051877752,
      "epoch": 0.005324849110766856,
      "grad_norm": 0.12521782517433167,
      "kl": 0.18078292161226273,
      "learning_rate": 3e-06,
      "loss": 0.0154,
      "step": 1918
    },
    {
      "clip_ratio": 0.00018628245015861467,
      "epoch": 0.005327625361606672,
      "grad_norm": 0.06385212391614914,
      "kl": 0.1704089492559433,
      "learning_rate": 3e-06,
      "loss": 0.0135,
      "step": 1919
    },
    {
      "clip_ratio": 0.00010469011613167822,
      "epoch": 0.005330401612446488,
      "grad_norm": 0.05945051088929176,
      "kl": 0.16913989931344986,
      "learning_rate": 3e-06,
      "loss": 0.014,
      "step": 1920
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 226.56250762939453,
      "epoch": 0.005333177863286303,
      "grad_norm": 0.06853771954774857,
      "kl": 0.1720345988869667,
      "learning_rate": 3e-06,
      "loss": 0.0216,
      "reward": 0.2500000149011612,
      "reward_std": 0.2080453597009182,
      "rewards/countdown_reward_func": 0.2500000149011612,
      "step": 1921,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio": 0.00026880551740759984,
      "epoch": 0.0053359541141261195,
      "grad_norm": 0.08787719905376434,
      "kl": 0.16743547469377518,
      "learning_rate": 3e-06,
      "loss": 0.0207,
      "step": 1922
    },
    {
      "clip_ratio": 8.567512122681364e-05,
      "epoch": 0.005338730364965936,
      "grad_norm": 0.1024947464466095,
      "kl": 0.16995961219072342,
      "learning_rate": 3e-06,
      "loss": 0.0217,
      "step": 1923
    },
    {
      "clip_ratio": 0.0003480182640487328,
      "epoch": 0.005341506615805751,
      "grad_norm": 0.0839357003569603,
      "kl": 0.18032675236463547,
      "learning_rate": 3e-06,
      "loss": 0.0214,
      "step": 1924
    },
    {
      "clip_ratio": 0.0002771032159216702,
      "epoch": 0.0053442828666455675,
      "grad_norm": 0.08104819059371948,
      "kl": 0.18133548647165298,
      "learning_rate": 3e-06,
      "loss": 0.0211,
      "step": 1925
    },
    {
      "clip_ratio": 0.00025895826547639444,
      "epoch": 0.005347059117485383,
      "grad_norm": 0.10513396561145782,
      "kl": 0.16761326789855957,
      "learning_rate": 3e-06,
      "loss": 0.0211,
      "step": 1926
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.005349835368325199,
      "grad_norm": 0.0935124084353447,
      "kl": 0.17295867204666138,
      "learning_rate": 3e-06,
      "loss": 0.0209,
      "step": 1927
    },
    {
      "clip_ratio": 0.000300093786790967,
      "epoch": 0.005352611619165015,
      "grad_norm": 0.090276338160038,
      "kl": 0.16908128559589386,
      "learning_rate": 3e-06,
      "loss": 0.0201,
      "step": 1928
    },
    {
      "clip_ratio": 8.555784006603062e-05,
      "epoch": 0.005355387870004831,
      "grad_norm": 0.09899937361478806,
      "kl": 0.17253682017326355,
      "learning_rate": 3e-06,
      "loss": 0.0213,
      "step": 1929
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.005358164120844646,
      "grad_norm": 0.09567968547344208,
      "kl": 0.18297704309225082,
      "learning_rate": 3e-06,
      "loss": 0.0212,
      "step": 1930
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0053609403716844625,
      "grad_norm": 0.07854413986206055,
      "kl": 0.18571214377880096,
      "learning_rate": 3e-06,
      "loss": 0.0203,
      "step": 1931
    },
    {
      "clip_ratio": 0.0004300739456084557,
      "epoch": 0.005363716622524278,
      "grad_norm": 0.08686288446187973,
      "kl": 0.17234565317630768,
      "learning_rate": 3e-06,
      "loss": 0.0197,
      "step": 1932
    },
    {
      "clip_ratio": 0.00018758241640171036,
      "completion_length": 219.52083587646484,
      "epoch": 0.005366492873364094,
      "grad_norm": 0.11458230018615723,
      "kl": 0.18364715576171875,
      "learning_rate": 3e-06,
      "loss": 0.0036,
      "reward": 0.21250002086162567,
      "reward_std": 0.2080453634262085,
      "rewards/countdown_reward_func": 0.21250002086162567,
      "step": 1933,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio": 0.00010620220564305782,
      "epoch": 0.0053692691242039105,
      "grad_norm": 0.22106598317623138,
      "kl": 0.1838528960943222,
      "learning_rate": 3e-06,
      "loss": 0.0045,
      "step": 1934
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.005372045375043726,
      "grad_norm": 0.11077378690242767,
      "kl": 0.18076221644878387,
      "learning_rate": 3e-06,
      "loss": 0.0035,
      "step": 1935
    },
    {
      "clip_ratio": 0.0006542036862811074,
      "epoch": 0.005374821625883542,
      "grad_norm": 0.10001291334629059,
      "kl": 0.17611156404018402,
      "learning_rate": 3e-06,
      "loss": 0.0038,
      "step": 1936
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0053775978767233576,
      "grad_norm": 0.09241759777069092,
      "kl": 0.17259087413549423,
      "learning_rate": 3e-06,
      "loss": 0.0032,
      "step": 1937
    },
    {
      "clip_ratio": 0.00039494471275247633,
      "epoch": 0.005380374127563174,
      "grad_norm": 0.11034689098596573,
      "kl": 0.17264589667320251,
      "learning_rate": 3e-06,
      "loss": 0.0028,
      "step": 1938
    },
    {
      "clip_ratio": 8.196721319109201e-05,
      "epoch": 0.005383150378402989,
      "grad_norm": 0.11598069220781326,
      "kl": 0.1707867681980133,
      "learning_rate": 3e-06,
      "loss": 0.0018,
      "step": 1939
    },
    {
      "clip_ratio": 0.0003757518425118178,
      "epoch": 0.0053859266292428055,
      "grad_norm": 0.23157401382923126,
      "kl": 0.16873134672641754,
      "learning_rate": 3e-06,
      "loss": 0.002,
      "step": 1940
    },
    {
      "clip_ratio": 0.0005139116401551291,
      "epoch": 0.005388702880082621,
      "grad_norm": 0.11090226471424103,
      "kl": 0.1620839387178421,
      "learning_rate": 3e-06,
      "loss": 0.0012,
      "step": 1941
    },
    {
      "clip_ratio": 0.0010267609904985875,
      "epoch": 0.005391479130922437,
      "grad_norm": 0.09474062919616699,
      "kl": 0.15658225119113922,
      "learning_rate": 3e-06,
      "loss": 0.0019,
      "step": 1942
    },
    {
      "clip_ratio": 0.000244140625,
      "epoch": 0.005394255381762253,
      "grad_norm": 0.08938335627317429,
      "kl": 0.1493806093931198,
      "learning_rate": 3e-06,
      "loss": 0.0012,
      "step": 1943
    },
    {
      "clip_ratio": 0.0003643684176495299,
      "epoch": 0.005397031632602069,
      "grad_norm": 0.14408725500106812,
      "kl": 0.15142924338579178,
      "learning_rate": 3e-06,
      "loss": 0.0002,
      "step": 1944
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "completion_length": 229.0416717529297,
      "epoch": 0.005399807883441885,
      "grad_norm": 0.12846609950065613,
      "kl": 0.14687072485685349,
      "learning_rate": 3e-06,
      "loss": 0.0124,
      "reward": 0.3020833730697632,
      "reward_std": 0.2922677993774414,
      "rewards/countdown_reward_func": 0.3020833432674408,
      "step": 1945,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0054025841342817005,
      "grad_norm": 0.10064707696437836,
      "kl": 0.13952406495809555,
      "learning_rate": 3e-06,
      "loss": 0.012,
      "step": 1946
    },
    {
      "clip_ratio": 0.0007594309572596103,
      "epoch": 0.005405360385121517,
      "grad_norm": 0.10994094610214233,
      "kl": 0.13837341219186783,
      "learning_rate": 3e-06,
      "loss": 0.0116,
      "step": 1947
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.005408136635961332,
      "grad_norm": 0.09539451450109482,
      "kl": 0.1353205442428589,
      "learning_rate": 3e-06,
      "loss": 0.0122,
      "step": 1948
    },
    {
      "clip_ratio": 0.00025521605130052194,
      "epoch": 0.0054109128868011485,
      "grad_norm": 0.11715108156204224,
      "kl": 0.13513853400945663,
      "learning_rate": 3e-06,
      "loss": 0.0127,
      "step": 1949
    },
    {
      "clip_ratio": 0.0005876675131730735,
      "epoch": 0.005413689137640964,
      "grad_norm": 0.11635179072618484,
      "kl": 0.12926241755485535,
      "learning_rate": 3e-06,
      "loss": 0.0114,
      "step": 1950
    },
    {
      "clip_ratio": 0.00033659624023130164,
      "epoch": 0.00541646538848078,
      "grad_norm": 0.13250043988227844,
      "kl": 0.1317070946097374,
      "learning_rate": 3e-06,
      "loss": 0.0108,
      "step": 1951
    },
    {
      "clip_ratio": 0.0002992119698319584,
      "epoch": 0.005419241639320596,
      "grad_norm": 0.10431980341672897,
      "kl": 0.12609682232141495,
      "learning_rate": 3e-06,
      "loss": 0.0105,
      "step": 1952
    },
    {
      "clip_ratio": 0.0017799893976189196,
      "epoch": 0.005422017890160412,
      "grad_norm": 0.1222839429974556,
      "kl": 0.12637890875339508,
      "learning_rate": 3e-06,
      "loss": 0.0103,
      "step": 1953
    },
    {
      "clip_ratio": 0.0005579154822044075,
      "epoch": 0.005424794141000227,
      "grad_norm": 0.09429076313972473,
      "kl": 0.12555726990103722,
      "learning_rate": 3e-06,
      "loss": 0.0103,
      "step": 1954
    },
    {
      "clip_ratio": 0.0017903645930346102,
      "epoch": 0.0054275703918400435,
      "grad_norm": 0.11087554693222046,
      "kl": 0.1258564032614231,
      "learning_rate": 3e-06,
      "loss": 0.0102,
      "step": 1955
    },
    {
      "clip_ratio": 0.002133891510311514,
      "epoch": 0.00543034664267986,
      "grad_norm": 0.13041195273399353,
      "kl": 0.12211589515209198,
      "learning_rate": 3e-06,
      "loss": 0.0104,
      "step": 1956
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 222.1041717529297,
      "epoch": 0.005433122893519675,
      "grad_norm": 0.09448617696762085,
      "kl": 0.11988238245248795,
      "learning_rate": 3e-06,
      "loss": 0.0148,
      "reward": 0.2666666805744171,
      "reward_std": 0.22970523685216904,
      "rewards/countdown_reward_func": 0.2666666656732559,
      "step": 1957,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio": 0.00017508336168248206,
      "epoch": 0.0054358991443594915,
      "grad_norm": 0.09333960711956024,
      "kl": 0.12271144986152649,
      "learning_rate": 3e-06,
      "loss": 0.0156,
      "step": 1958
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.005438675395199307,
      "grad_norm": 0.10355954617261887,
      "kl": 0.11948385834693909,
      "learning_rate": 3e-06,
      "loss": 0.0148,
      "step": 1959
    },
    {
      "clip_ratio": 0.00018209777772426605,
      "epoch": 0.005441451646039123,
      "grad_norm": 0.10874044895172119,
      "kl": 0.12741681933403015,
      "learning_rate": 3e-06,
      "loss": 0.0161,
      "step": 1960
    },
    {
      "clip_ratio": 0.0004606101065292023,
      "epoch": 0.005444227896878939,
      "grad_norm": 0.11931112408638,
      "kl": 0.11942564323544502,
      "learning_rate": 3e-06,
      "loss": 0.0144,
      "step": 1961
    },
    {
      "clip_ratio": 0.00018340400856686756,
      "epoch": 0.005447004147718755,
      "grad_norm": 0.13169850409030914,
      "kl": 0.11675844341516495,
      "learning_rate": 3e-06,
      "loss": 0.0149,
      "step": 1962
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.00544978039855857,
      "grad_norm": 0.09862768650054932,
      "kl": 0.12140379846096039,
      "learning_rate": 3e-06,
      "loss": 0.0147,
      "step": 1963
    },
    {
      "clip_ratio": 0.0002775423854473047,
      "epoch": 0.0054525566493983865,
      "grad_norm": 0.10945676267147064,
      "kl": 0.12424385920166969,
      "learning_rate": 3e-06,
      "loss": 0.0146,
      "step": 1964
    },
    {
      "clip_ratio": 0.0005308387917466462,
      "epoch": 0.005455332900238202,
      "grad_norm": 0.09942898899316788,
      "kl": 0.12193005159497261,
      "learning_rate": 3e-06,
      "loss": 0.0137,
      "step": 1965
    },
    {
      "clip_ratio": 0.00037315295776352286,
      "epoch": 0.005458109151078018,
      "grad_norm": 0.1435389667749405,
      "kl": 0.12972797825932503,
      "learning_rate": 3e-06,
      "loss": 0.0155,
      "step": 1966
    },
    {
      "clip_ratio": 0.0011096491580246948,
      "epoch": 0.0054608854019178345,
      "grad_norm": 0.118125319480896,
      "kl": 0.12346281483769417,
      "learning_rate": 3e-06,
      "loss": 0.0133,
      "step": 1967
    },
    {
      "clip_ratio": 0.00108197086956352,
      "epoch": 0.00546366165275765,
      "grad_norm": 0.13901278376579285,
      "kl": 0.12123304605484009,
      "learning_rate": 3e-06,
      "loss": 0.0131,
      "step": 1968
    },
    {
      "clip_ratio": 0.00017531556659378111,
      "completion_length": 226.37500762939453,
      "epoch": 0.005466437903597466,
      "grad_norm": 0.1171063482761383,
      "kl": 0.11619845405220985,
      "learning_rate": 3e-06,
      "loss": 0.0091,
      "reward": 0.3395833522081375,
      "reward_std": 0.35036255419254303,
      "rewards/countdown_reward_func": 0.3395833373069763,
      "step": 1969,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 9.645061800256371e-05,
      "epoch": 0.0054692141544372816,
      "grad_norm": 0.1672900915145874,
      "kl": 0.12332340329885483,
      "learning_rate": 3e-06,
      "loss": 0.0087,
      "step": 1970
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.005471990405277098,
      "grad_norm": 0.12007147073745728,
      "kl": 0.12126722559332848,
      "learning_rate": 3e-06,
      "loss": 0.0096,
      "step": 1971
    },
    {
      "clip_ratio": 9.645061800256371e-05,
      "epoch": 0.005474766656116913,
      "grad_norm": 0.1385733038187027,
      "kl": 0.11890603601932526,
      "learning_rate": 3e-06,
      "loss": 0.0084,
      "step": 1972
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0054775429069567295,
      "grad_norm": 0.122285395860672,
      "kl": 0.12061040103435516,
      "learning_rate": 3e-06,
      "loss": 0.0079,
      "step": 1973
    },
    {
      "clip_ratio": 8.765778329689056e-05,
      "epoch": 0.005480319157796545,
      "grad_norm": 0.11869718879461288,
      "kl": 0.12436644360423088,
      "learning_rate": 3e-06,
      "loss": 0.0088,
      "step": 1974
    },
    {
      "clip_ratio": 8.765778329689056e-05,
      "epoch": 0.005483095408636361,
      "grad_norm": 0.11378361284732819,
      "kl": 0.11800763756036758,
      "learning_rate": 3e-06,
      "loss": 0.007,
      "step": 1975
    },
    {
      "clip_ratio": 0.00017234613187611103,
      "epoch": 0.005485871659476177,
      "grad_norm": 0.1709638386964798,
      "kl": 0.1264483742415905,
      "learning_rate": 3e-06,
      "loss": 0.0078,
      "step": 1976
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.005488647910315993,
      "grad_norm": 0.118524931371212,
      "kl": 0.12333150953054428,
      "learning_rate": 3e-06,
      "loss": 0.0078,
      "step": 1977
    },
    {
      "clip_ratio": 9.645061800256371e-05,
      "epoch": 0.005491424161155809,
      "grad_norm": 0.15198098123073578,
      "kl": 0.12151569873094559,
      "learning_rate": 3e-06,
      "loss": 0.0068,
      "step": 1978
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0054942004119956245,
      "grad_norm": 0.13697625696659088,
      "kl": 0.12277953699231148,
      "learning_rate": 3e-06,
      "loss": 0.0065,
      "step": 1979
    },
    {
      "clip_ratio": 0.0002635204582475126,
      "epoch": 0.005496976662835441,
      "grad_norm": 0.1165957823395729,
      "kl": 0.12548280879855156,
      "learning_rate": 3e-06,
      "loss": 0.0077,
      "step": 1980
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 220.08334350585938,
      "epoch": 0.005499752913675256,
      "grad_norm": 0.1365557312965393,
      "kl": 0.11501205712556839,
      "learning_rate": 3e-06,
      "loss": -0.0015,
      "reward": 0.36250002682209015,
      "reward_std": 0.33128294348716736,
      "rewards/countdown_reward_func": 0.36250001192092896,
      "step": 1981,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.00026728439843282104,
      "epoch": 0.0055025291645150725,
      "grad_norm": 0.1341644525527954,
      "kl": 0.11507071927189827,
      "learning_rate": 3e-06,
      "loss": -0.0015,
      "step": 1982
    },
    {
      "clip_ratio": 0.00010775862028822303,
      "epoch": 0.005505305415354888,
      "grad_norm": 0.1273004114627838,
      "kl": 0.12233321368694305,
      "learning_rate": 3e-06,
      "loss": -0.0006,
      "step": 1983
    },
    {
      "clip_ratio": 0.0005858765362063423,
      "epoch": 0.005508081666194704,
      "grad_norm": 0.12228009104728699,
      "kl": 0.12594753503799438,
      "learning_rate": 3e-06,
      "loss": -0.0006,
      "step": 1984
    },
    {
      "clip_ratio": 0.00029870675643905997,
      "epoch": 0.00551085791703452,
      "grad_norm": 0.1282241940498352,
      "kl": 0.12114127725362778,
      "learning_rate": 3e-06,
      "loss": -0.002,
      "step": 1985
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.005513634167874336,
      "grad_norm": 0.18000636994838715,
      "kl": 0.12271250784397125,
      "learning_rate": 3e-06,
      "loss": -0.0024,
      "step": 1986
    },
    {
      "clip_ratio": 0.0003066251229029149,
      "epoch": 0.005516410418714151,
      "grad_norm": 0.11781095713376999,
      "kl": 0.11188013106584549,
      "learning_rate": 3e-06,
      "loss": -0.0023,
      "step": 1987
    },
    {
      "clip_ratio": 0.0005269177490845323,
      "epoch": 0.0055191866695539675,
      "grad_norm": 0.13404454290866852,
      "kl": 0.1126963309943676,
      "learning_rate": 3e-06,
      "loss": -0.0036,
      "step": 1988
    },
    {
      "clip_ratio": 0.0003066251229029149,
      "epoch": 0.005521962920393784,
      "grad_norm": 0.11764881014823914,
      "kl": 0.11812717467546463,
      "learning_rate": 3e-06,
      "loss": -0.0026,
      "step": 1989
    },
    {
      "clip_ratio": 0.0011090568150393665,
      "epoch": 0.005524739171233599,
      "grad_norm": 0.14475159347057343,
      "kl": 0.12093603238463402,
      "learning_rate": 3e-06,
      "loss": -0.0037,
      "step": 1990
    },
    {
      "clip_ratio": 0.0010156702192034572,
      "epoch": 0.0055275154220734155,
      "grad_norm": 0.11484024673700333,
      "kl": 0.11511994898319244,
      "learning_rate": 3e-06,
      "loss": -0.0032,
      "step": 1991
    },
    {
      "clip_ratio": 0.0007695165404584259,
      "epoch": 0.005530291672913231,
      "grad_norm": 0.18602021038532257,
      "kl": 0.11601589620113373,
      "learning_rate": 3e-06,
      "loss": -0.0049,
      "step": 1992
    },
    {
      "clip_ratio": 0.0005422846879810095,
      "completion_length": 225.81250762939453,
      "epoch": 0.005533067923753047,
      "grad_norm": 0.10769283771514893,
      "kl": 0.11326644197106361,
      "learning_rate": 3e-06,
      "loss": 0.0156,
      "reward": 0.2854166850447655,
      "reward_std": 0.24453017115592957,
      "rewards/countdown_reward_func": 0.2854166701436043,
      "step": 1993,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0003762734049814753,
      "epoch": 0.005535844174592863,
      "grad_norm": 0.12362723052501678,
      "kl": 0.11308754608035088,
      "learning_rate": 3e-06,
      "loss": 0.0153,
      "step": 1994
    },
    {
      "clip_ratio": 0.0005234857235336676,
      "epoch": 0.005538620425432679,
      "grad_norm": 0.0922532007098198,
      "kl": 0.10114821046590805,
      "learning_rate": 3e-06,
      "loss": 0.0145,
      "step": 1995
    },
    {
      "clip_ratio": 8.73515018611215e-05,
      "epoch": 0.005541396676272494,
      "grad_norm": 0.10349724441766739,
      "kl": 0.10615367442369461,
      "learning_rate": 3e-06,
      "loss": 0.016,
      "step": 1996
    },
    {
      "clip_ratio": 0.00021079258294776082,
      "epoch": 0.0055441729271123105,
      "grad_norm": 0.13311462104320526,
      "kl": 0.10724882781505585,
      "learning_rate": 3e-06,
      "loss": 0.0144,
      "step": 1997
    },
    {
      "clip_ratio": 0.0001151012911577709,
      "epoch": 0.005546949177952126,
      "grad_norm": 0.09960220754146576,
      "kl": 0.10265998914837837,
      "learning_rate": 3e-06,
      "loss": 0.0153,
      "step": 1998
    },
    {
      "clip_ratio": 0.0007085143588483334,
      "epoch": 0.005549725428791942,
      "grad_norm": 0.10948970168828964,
      "kl": 0.10900342464447021,
      "learning_rate": 3e-06,
      "loss": 0.0154,
      "step": 1999
    },
    {
      "epoch": 0.0055525016796317585,
      "grad_norm": 0.09006191045045853,
      "learning_rate": 3e-06,
      "loss": 0.0145,
      "step": 2000
    },
    {
      "clip_ratio": 0.0005361156981962267,
      "epoch": 0.005555277930471574,
      "grad_norm": 0.10735763609409332,
      "kl": 0.1035256776958704,
      "learning_rate": 3e-06,
      "loss": 0.0144,
      "step": 2001
    },
    {
      "clip_ratio": 0.0004768256621900946,
      "epoch": 0.00555805418131139,
      "grad_norm": 0.09122443944215775,
      "kl": 0.10310684517025948,
      "learning_rate": 3e-06,
      "loss": 0.0147,
      "step": 2002
    },
    {
      "clip_ratio": 0.0020118614193052053,
      "epoch": 0.0055608304321512056,
      "grad_norm": 0.21058741211891174,
      "kl": 0.10488555207848549,
      "learning_rate": 3e-06,
      "loss": 0.0131,
      "step": 2003
    },
    {
      "clip_ratio": 0.0006240379370865412,
      "epoch": 0.005563606682991022,
      "grad_norm": 0.09514111280441284,
      "kl": 0.10193825513124466,
      "learning_rate": 3e-06,
      "loss": 0.0136,
      "step": 2004
    },
    {
      "clip_ratio": 0.0002617926656967029,
      "completion_length": 223.08333587646484,
      "epoch": 0.005566382933830837,
      "grad_norm": 0.08725622296333313,
      "kl": 0.11390889436006546,
      "learning_rate": 3e-06,
      "loss": 0.0051,
      "reward": 0.2291666641831398,
      "reward_std": 0.16722052544355392,
      "rewards/countdown_reward_func": 0.2291666641831398,
      "step": 2005,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio": 0.00020610057981684804,
      "epoch": 0.0055691591846706535,
      "grad_norm": 0.08379297703504562,
      "kl": 0.11956554651260376,
      "learning_rate": 3e-06,
      "loss": 0.0049,
      "step": 2006
    },
    {
      "clip_ratio": 0.0002910737384809181,
      "epoch": 0.005571935435510469,
      "grad_norm": 0.0656660795211792,
      "kl": 0.10992233455181122,
      "learning_rate": 3e-06,
      "loss": 0.0051,
      "step": 2007
    },
    {
      "clip_ratio": 8.840169903123751e-05,
      "epoch": 0.005574711686350285,
      "grad_norm": 0.07707367092370987,
      "kl": 0.11566202342510223,
      "learning_rate": 3e-06,
      "loss": 0.0054,
      "step": 2008
    },
    {
      "clip_ratio": 0.0005598522620857693,
      "epoch": 0.005577487937190101,
      "grad_norm": 0.0825631394982338,
      "kl": 0.11144465208053589,
      "learning_rate": 3e-06,
      "loss": 0.0042,
      "step": 2009
    },
    {
      "clip_ratio": 8.333333244081587e-05,
      "epoch": 0.005580264188029917,
      "grad_norm": 0.07131849229335785,
      "kl": 0.11648586019873619,
      "learning_rate": 3e-06,
      "loss": 0.0044,
      "step": 2010
    },
    {
      "clip_ratio": 9.204712841892615e-05,
      "epoch": 0.005583040438869733,
      "grad_norm": 0.07822870463132858,
      "kl": 0.11300336197018623,
      "learning_rate": 3e-06,
      "loss": 0.0043,
      "step": 2011
    },
    {
      "clip_ratio": 0.00019509741832735017,
      "epoch": 0.0055858166897095485,
      "grad_norm": 0.08241821080446243,
      "kl": 0.11744438856840134,
      "learning_rate": 3e-06,
      "loss": 0.0046,
      "step": 2012
    },
    {
      "clip_ratio": 0.0005299464974086732,
      "epoch": 0.005588592940549365,
      "grad_norm": 0.07519717514514923,
      "kl": 0.1080629974603653,
      "learning_rate": 3e-06,
      "loss": 0.0044,
      "step": 2013
    },
    {
      "clip_ratio": 9.578544268151745e-05,
      "epoch": 0.00559136919138918,
      "grad_norm": 0.07436896860599518,
      "kl": 0.11372564733028412,
      "learning_rate": 3e-06,
      "loss": 0.0053,
      "step": 2014
    },
    {
      "clip_ratio": 0.0011513839708641171,
      "epoch": 0.0055941454422289965,
      "grad_norm": 0.07369540631771088,
      "kl": 0.10970757156610489,
      "learning_rate": 3e-06,
      "loss": 0.0035,
      "step": 2015
    },
    {
      "clip_ratio": 0.0004363836196716875,
      "epoch": 0.005596921693068812,
      "grad_norm": 0.06370438635349274,
      "kl": 0.1145344153046608,
      "learning_rate": 3e-06,
      "loss": 0.0037,
      "step": 2016
    },
    {
      "clip_ratio": 0.0002770941355265677,
      "completion_length": 221.2291717529297,
      "epoch": 0.005599697943908628,
      "grad_norm": 0.07984435558319092,
      "kl": 0.10508502274751663,
      "learning_rate": 3e-06,
      "loss": 0.0025,
      "reward": 0.21250000596046448,
      "reward_std": 0.17428425326943398,
      "rewards/countdown_reward_func": 0.21250000596046448,
      "step": 2017,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio": 9.21828905120492e-05,
      "epoch": 0.005602474194748444,
      "grad_norm": 0.07923725992441177,
      "kl": 0.11465127393603325,
      "learning_rate": 3e-06,
      "loss": 0.0021,
      "step": 2018
    },
    {
      "clip_ratio": 0.00029520990210585296,
      "epoch": 0.00560525044558826,
      "grad_norm": 0.08113577216863632,
      "kl": 0.10587036609649658,
      "learning_rate": 3e-06,
      "loss": 0.0026,
      "step": 2019
    },
    {
      "clip_ratio": 0.00039488900802098215,
      "epoch": 0.005608026696428075,
      "grad_norm": 0.09565461426973343,
      "kl": 0.11312683299183846,
      "learning_rate": 3e-06,
      "loss": 0.0021,
      "step": 2020
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0056108029472678915,
      "grad_norm": 0.10134981572628021,
      "kl": 0.10707443952560425,
      "learning_rate": 3e-06,
      "loss": 0.0027,
      "step": 2021
    },
    {
      "clip_ratio": 0.000533617683686316,
      "epoch": 0.005613579198107708,
      "grad_norm": 0.0789194330573082,
      "kl": 0.1022312305867672,
      "learning_rate": 3e-06,
      "loss": 0.0019,
      "step": 2022
    },
    {
      "clip_ratio": 0.0005019560339860618,
      "epoch": 0.005616355448947523,
      "grad_norm": 0.08137885481119156,
      "kl": 0.10183565318584442,
      "learning_rate": 3e-06,
      "loss": 0.0022,
      "step": 2023
    },
    {
      "clip_ratio": 9.21828905120492e-05,
      "epoch": 0.0056191316997873395,
      "grad_norm": 0.0739808902144432,
      "kl": 0.11150963604450226,
      "learning_rate": 3e-06,
      "loss": 0.0012,
      "step": 2024
    },
    {
      "clip_ratio": 0.0001953362807398662,
      "epoch": 0.005621907950627155,
      "grad_norm": 0.07564988732337952,
      "kl": 0.10209690034389496,
      "learning_rate": 3e-06,
      "loss": 0.0017,
      "step": 2025
    },
    {
      "clip_ratio": 0.0004712695226771757,
      "epoch": 0.005624684201466971,
      "grad_norm": 0.14252831041812897,
      "kl": 0.10544009134173393,
      "learning_rate": 3e-06,
      "loss": 0.0015,
      "step": 2026
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.005627460452306787,
      "grad_norm": 0.0803026333451271,
      "kl": 0.10230910778045654,
      "learning_rate": 3e-06,
      "loss": 0.0012,
      "step": 2027
    },
    {
      "clip_ratio": 0.0007070228894008324,
      "epoch": 0.005630236703146603,
      "grad_norm": 0.08244466781616211,
      "kl": 0.0964125283062458,
      "learning_rate": 3e-06,
      "loss": 0.001,
      "step": 2028
    },
    {
      "clip_ratio": 0.0004911591531708837,
      "completion_length": 220.95833587646484,
      "epoch": 0.005633012953986418,
      "grad_norm": 0.10779600590467453,
      "kl": 0.11588352173566818,
      "learning_rate": 3e-06,
      "loss": 0.0112,
      "reward": 0.30416668206453323,
      "reward_std": 0.3633297234773636,
      "rewards/countdown_reward_func": 0.30416668206453323,
      "step": 2029,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 0.00043572409049374983,
      "epoch": 0.0056357892048262345,
      "grad_norm": 0.11869516223669052,
      "kl": 0.09700796753168106,
      "learning_rate": 3e-06,
      "loss": 0.0108,
      "step": 2030
    },
    {
      "clip_ratio": 0.00020404949464136735,
      "epoch": 0.00563856545566605,
      "grad_norm": 0.09410764276981354,
      "kl": 0.10727613791823387,
      "learning_rate": 3e-06,
      "loss": 0.0105,
      "step": 2031
    },
    {
      "clip_ratio": 0.00020356984896352515,
      "epoch": 0.005641341706505866,
      "grad_norm": 0.11350057274103165,
      "kl": 0.10674090310931206,
      "learning_rate": 3e-06,
      "loss": 0.0096,
      "step": 2032
    },
    {
      "clip_ratio": 0.0002443792764097452,
      "epoch": 0.0056441179573456825,
      "grad_norm": 0.12127961963415146,
      "kl": 0.10875796526670456,
      "learning_rate": 3e-06,
      "loss": 0.0101,
      "step": 2033
    },
    {
      "clip_ratio": 0.0003601440985221416,
      "epoch": 0.005646894208185498,
      "grad_norm": 0.1095552071928978,
      "kl": 0.09716677665710449,
      "learning_rate": 3e-06,
      "loss": 0.01,
      "step": 2034
    },
    {
      "clip_ratio": 0.0002456332149449736,
      "epoch": 0.005649670459025314,
      "grad_norm": 0.12184404581785202,
      "kl": 0.11015952378511429,
      "learning_rate": 3e-06,
      "loss": 0.01,
      "step": 2035
    },
    {
      "clip_ratio": 8.191350207198411e-05,
      "epoch": 0.0056524467098651296,
      "grad_norm": 0.11499220877885818,
      "kl": 0.09228203445672989,
      "learning_rate": 3e-06,
      "loss": 0.0097,
      "step": 2036
    },
    {
      "clip_ratio": 0.0001221896382048726,
      "epoch": 0.005655222960704946,
      "grad_norm": 0.12085867673158646,
      "kl": 0.10267635807394981,
      "learning_rate": 3e-06,
      "loss": 0.0094,
      "step": 2037
    },
    {
      "clip_ratio": 0.0004097962155356072,
      "epoch": 0.005657999211544761,
      "grad_norm": 0.12743261456489563,
      "kl": 0.10104519873857498,
      "learning_rate": 3e-06,
      "loss": 0.0086,
      "step": 2038
    },
    {
      "clip_ratio": 0.0006971483817324042,
      "epoch": 0.0056607754623845775,
      "grad_norm": 0.12285710871219635,
      "kl": 0.10486527159810066,
      "learning_rate": 3e-06,
      "loss": 0.0099,
      "step": 2039
    },
    {
      "clip_ratio": 0.0005238101875875145,
      "epoch": 0.005663551713224393,
      "grad_norm": 0.1023198664188385,
      "kl": 0.09349857643246651,
      "learning_rate": 3e-06,
      "loss": 0.0078,
      "step": 2040
    },
    {
      "clip_ratio": 0.0009150805417448282,
      "completion_length": 226.89583587646484,
      "epoch": 0.005666327964064209,
      "grad_norm": 0.09265701472759247,
      "kl": 0.09547650068998337,
      "learning_rate": 3e-06,
      "loss": 0.0079,
      "reward": 0.26875001937150955,
      "reward_std": 0.15347465127706528,
      "rewards/countdown_reward_func": 0.26875000447034836,
      "step": 2041,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.005669104214904025,
      "grad_norm": 0.050366487354040146,
      "kl": 0.10091326385736465,
      "learning_rate": 3e-06,
      "loss": 0.0086,
      "step": 2042
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.005671880465743841,
      "grad_norm": 0.06937897950410843,
      "kl": 0.10959598422050476,
      "learning_rate": 3e-06,
      "loss": 0.0086,
      "step": 2043
    },
    {
      "clip_ratio": 0.0001739728031679988,
      "epoch": 0.005674656716583657,
      "grad_norm": 0.06653722375631332,
      "kl": 0.09630914404988289,
      "learning_rate": 3e-06,
      "loss": 0.0084,
      "step": 2044
    },
    {
      "clip_ratio": 0.00018076645210385323,
      "epoch": 0.0056774329674234725,
      "grad_norm": 0.04911844804883003,
      "kl": 0.08973881602287292,
      "learning_rate": 3e-06,
      "loss": 0.0084,
      "step": 2045
    },
    {
      "clip_ratio": 0.0002787739722407423,
      "epoch": 0.005680209218263289,
      "grad_norm": 0.08200076222419739,
      "kl": 0.0952768363058567,
      "learning_rate": 3e-06,
      "loss": 0.0077,
      "step": 2046
    },
    {
      "clip_ratio": 0.0003649073769338429,
      "epoch": 0.005682985469103104,
      "grad_norm": 0.06790746748447418,
      "kl": 0.09479077905416489,
      "learning_rate": 3e-06,
      "loss": 0.0078,
      "step": 2047
    },
    {
      "clip_ratio": 9.15080527192913e-05,
      "epoch": 0.0056857617199429205,
      "grad_norm": 0.05466790497303009,
      "kl": 0.10251379758119583,
      "learning_rate": 3e-06,
      "loss": 0.0084,
      "step": 2048
    },
    {
      "clip_ratio": 0.0002632715040817857,
      "epoch": 0.005688537970782736,
      "grad_norm": 0.06806403398513794,
      "kl": 0.1096310168504715,
      "learning_rate": 3e-06,
      "loss": 0.0081,
      "step": 2049
    },
    {
      "clip_ratio": 0.0001739728031679988,
      "epoch": 0.005691314221622552,
      "grad_norm": 0.06310102343559265,
      "kl": 0.09831372275948524,
      "learning_rate": 3e-06,
      "loss": 0.0084,
      "step": 2050
    },
    {
      "clip_ratio": 0.0001840161858126521,
      "epoch": 0.005694090472462368,
      "grad_norm": 0.05473968759179115,
      "kl": 0.09245896711945534,
      "learning_rate": 3e-06,
      "loss": 0.0077,
      "step": 2051
    },
    {
      "clip_ratio": 9.36329597607255e-05,
      "epoch": 0.005696866723302184,
      "grad_norm": 0.06946438550949097,
      "kl": 0.0970330685377121,
      "learning_rate": 3e-06,
      "loss": 0.0074,
      "step": 2052
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 230.62500762939453,
      "epoch": 0.005699642974141999,
      "grad_norm": 0.1020989716053009,
      "kl": 0.09304828196763992,
      "learning_rate": 3e-06,
      "loss": 0.0123,
      "reward": 0.3395833522081375,
      "reward_std": 0.2922678142786026,
      "rewards/countdown_reward_func": 0.3395833522081375,
      "step": 2053,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0057024192249818155,
      "grad_norm": 0.08666249364614487,
      "kl": 0.09421010315418243,
      "learning_rate": 3e-06,
      "loss": 0.0129,
      "step": 2054
    },
    {
      "clip_ratio": 0.00016469038382638246,
      "epoch": 0.005705195475821632,
      "grad_norm": 0.11592990159988403,
      "kl": 0.09231939166784286,
      "learning_rate": 3e-06,
      "loss": 0.0128,
      "step": 2055
    },
    {
      "clip_ratio": 0.0002482597410562448,
      "epoch": 0.005707971726661447,
      "grad_norm": 0.1035403460264206,
      "kl": 0.09664532542228699,
      "learning_rate": 3e-06,
      "loss": 0.012,
      "step": 2056
    },
    {
      "clip_ratio": 0.00016687953029759228,
      "epoch": 0.0057107479775012635,
      "grad_norm": 0.1996307075023651,
      "kl": 0.09553533047437668,
      "learning_rate": 3e-06,
      "loss": 0.0118,
      "step": 2057
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.005713524228341079,
      "grad_norm": 0.09972159564495087,
      "kl": 0.10145417600870132,
      "learning_rate": 3e-06,
      "loss": 0.013,
      "step": 2058
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.005716300479180895,
      "grad_norm": 0.10573241114616394,
      "kl": 0.09803595021367073,
      "learning_rate": 3e-06,
      "loss": 0.0114,
      "step": 2059
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.005719076730020711,
      "grad_norm": 0.08910799026489258,
      "kl": 0.09924589842557907,
      "learning_rate": 3e-06,
      "loss": 0.012,
      "step": 2060
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.005721852980860527,
      "grad_norm": 0.11156439036130905,
      "kl": 0.09876829758286476,
      "learning_rate": 3e-06,
      "loss": 0.0116,
      "step": 2061
    },
    {
      "clip_ratio": 0.00035182230203645304,
      "epoch": 0.005724629231700342,
      "grad_norm": 0.11116062849760056,
      "kl": 0.10230807960033417,
      "learning_rate": 3e-06,
      "loss": 0.0107,
      "step": 2062
    },
    {
      "clip_ratio": 0.00016687953029759228,
      "epoch": 0.0057274054825401585,
      "grad_norm": 0.11640466749668121,
      "kl": 0.10202505066990852,
      "learning_rate": 3e-06,
      "loss": 0.0104,
      "step": 2063
    },
    {
      "clip_ratio": 0.0002663229824975133,
      "epoch": 0.005730181733379974,
      "grad_norm": 0.20945513248443604,
      "kl": 0.10810398682951927,
      "learning_rate": 3e-06,
      "loss": 0.0113,
      "step": 2064
    },
    {
      "clip_ratio": 8.223684562835842e-05,
      "completion_length": 233.02084350585938,
      "epoch": 0.00573295798421979,
      "grad_norm": 0.10434425622224808,
      "kl": 0.11239994317293167,
      "learning_rate": 3e-06,
      "loss": -0.0028,
      "reward": 0.32083335518836975,
      "reward_std": 0.3441803753376007,
      "rewards/countdown_reward_func": 0.32083334028720856,
      "step": 2065,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0057357342350596065,
      "grad_norm": 0.11738111078739166,
      "kl": 0.11531775444746017,
      "learning_rate": 3e-06,
      "loss": -0.0023,
      "step": 2066
    },
    {
      "clip_ratio": 0.0006976053700782359,
      "epoch": 0.005738510485899422,
      "grad_norm": 0.09144604206085205,
      "kl": 0.11997786909341812,
      "learning_rate": 3e-06,
      "loss": -0.002,
      "step": 2067
    },
    {
      "clip_ratio": 0.0002564237511251122,
      "epoch": 0.005741286736739238,
      "grad_norm": 0.15023760497570038,
      "kl": 0.11250534653663635,
      "learning_rate": 3e-06,
      "loss": -0.0033,
      "step": 2068
    },
    {
      "clip_ratio": 0.0003439874417381361,
      "epoch": 0.0057440629875790536,
      "grad_norm": 0.0950041189789772,
      "kl": 0.1112917885184288,
      "learning_rate": 3e-06,
      "loss": -0.0029,
      "step": 2069
    },
    {
      "clip_ratio": 8.327781688421965e-05,
      "epoch": 0.00574683923841887,
      "grad_norm": 0.1445959061384201,
      "kl": 0.12204555794596672,
      "learning_rate": 3e-06,
      "loss": -0.0021,
      "step": 2070
    },
    {
      "clip_ratio": 9.09090886125341e-05,
      "epoch": 0.005749615489258685,
      "grad_norm": 0.10922154039144516,
      "kl": 0.1177988089621067,
      "learning_rate": 3e-06,
      "loss": -0.0032,
      "step": 2071
    },
    {
      "clip_ratio": 0.0002721360942814499,
      "epoch": 0.0057523917400985015,
      "grad_norm": 0.10157906264066696,
      "kl": 0.11939038708806038,
      "learning_rate": 3e-06,
      "loss": -0.0028,
      "step": 2072
    },
    {
      "clip_ratio": 0.0011802471126429737,
      "epoch": 0.005755167990938317,
      "grad_norm": 0.10021565854549408,
      "kl": 0.12260624766349792,
      "learning_rate": 3e-06,
      "loss": -0.0032,
      "step": 2073
    },
    {
      "clip_ratio": 0.0005078699323348701,
      "epoch": 0.005757944241778133,
      "grad_norm": 0.14854243397712708,
      "kl": 0.11457186192274094,
      "learning_rate": 3e-06,
      "loss": -0.0052,
      "step": 2074
    },
    {
      "clip_ratio": 0.0005989633646095172,
      "epoch": 0.005760720492617949,
      "grad_norm": 0.10022560507059097,
      "kl": 0.11323418468236923,
      "learning_rate": 3e-06,
      "loss": -0.0044,
      "step": 2075
    },
    {
      "clip_ratio": 0.0011352297442499548,
      "epoch": 0.005763496743457765,
      "grad_norm": 0.14965884387493134,
      "kl": 0.12618472799658775,
      "learning_rate": 3e-06,
      "loss": -0.0045,
      "step": 2076
    },
    {
      "clip_ratio": 9.689922444522381e-05,
      "completion_length": 221.6041717529297,
      "epoch": 0.005766272994297581,
      "grad_norm": 0.11252445727586746,
      "kl": 0.11330411210656166,
      "learning_rate": 3e-06,
      "loss": 0.0338,
      "reward": 0.26875001937150955,
      "reward_std": 0.2272602580487728,
      "rewards/countdown_reward_func": 0.26875001937150955,
      "step": 2077,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio": 0.0002688652166398242,
      "epoch": 0.0057690492451373965,
      "grad_norm": 0.11611206829547882,
      "kl": 0.11921777203679085,
      "learning_rate": 3e-06,
      "loss": 0.0351,
      "step": 2078
    },
    {
      "clip_ratio": 0.00021403797290986404,
      "epoch": 0.005771825495977213,
      "grad_norm": 0.09081734716892242,
      "kl": 0.11672532185912132,
      "learning_rate": 3e-06,
      "loss": 0.0339,
      "step": 2079
    },
    {
      "clip_ratio": 0.0006553223647642881,
      "epoch": 0.005774601746817028,
      "grad_norm": 0.11462453752756119,
      "kl": 0.12759307771921158,
      "learning_rate": 3e-06,
      "loss": 0.0341,
      "step": 2080
    },
    {
      "clip_ratio": 0.0002815326370182447,
      "epoch": 0.0057773779976568445,
      "grad_norm": 0.11553678661584854,
      "kl": 0.11894625052809715,
      "learning_rate": 3e-06,
      "loss": 0.0344,
      "step": 2081
    },
    {
      "clip_ratio": 0.00038389285327866673,
      "epoch": 0.00578015424849666,
      "grad_norm": 0.13438205420970917,
      "kl": 0.13819441944360733,
      "learning_rate": 3e-06,
      "loss": 0.0343,
      "step": 2082
    },
    {
      "clip_ratio": 0.00019379844889044762,
      "epoch": 0.005782930499336476,
      "grad_norm": 0.09481407701969147,
      "kl": 0.12022696807980537,
      "learning_rate": 3e-06,
      "loss": 0.0335,
      "step": 2083
    },
    {
      "clip_ratio": 0.00046928575466154143,
      "epoch": 0.005785706750176292,
      "grad_norm": 0.11112938076257706,
      "kl": 0.12938351184129715,
      "learning_rate": 3e-06,
      "loss": 0.034,
      "step": 2084
    },
    {
      "clip_ratio": 0.0005152427984285168,
      "epoch": 0.005788483001016108,
      "grad_norm": 0.08531668782234192,
      "kl": 0.12835099548101425,
      "learning_rate": 3e-06,
      "loss": 0.0329,
      "step": 2085
    },
    {
      "clip_ratio": 0.0012749898305628449,
      "epoch": 0.005791259251855923,
      "grad_norm": 0.14292016625404358,
      "kl": 0.14164124429225922,
      "learning_rate": 3e-06,
      "loss": 0.0331,
      "step": 2086
    },
    {
      "clip_ratio": 0.0004016064340248704,
      "epoch": 0.0057940355026957395,
      "grad_norm": 0.10637033730745316,
      "kl": 0.1343921273946762,
      "learning_rate": 3e-06,
      "loss": 0.0319,
      "step": 2087
    },
    {
      "clip_ratio": 0.0011857394711114466,
      "epoch": 0.005796811753535556,
      "grad_norm": 0.10809402167797089,
      "kl": 0.15764878690242767,
      "learning_rate": 3e-06,
      "loss": 0.0319,
      "step": 2088
    },
    {
      "clip_ratio": 9.689922444522381e-05,
      "completion_length": 241.83334350585938,
      "epoch": 0.005799588004375371,
      "grad_norm": 0.16788090765476227,
      "kl": 0.1235123947262764,
      "learning_rate": 3e-06,
      "loss": 0.016,
      "reward": 0.23125001043081284,
      "reward_std": 0.2202121838927269,
      "rewards/countdown_reward_func": 0.23125001043081284,
      "step": 2089,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio": 0.00040937707672128454,
      "epoch": 0.0058023642552151875,
      "grad_norm": 0.08114562183618546,
      "kl": 0.14093566685914993,
      "learning_rate": 3e-06,
      "loss": 0.0153,
      "step": 2090
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.005805140506055003,
      "grad_norm": 0.09554962068796158,
      "kl": 0.1442393809556961,
      "learning_rate": 3e-06,
      "loss": 0.0148,
      "step": 2091
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.005807916756894819,
      "grad_norm": 0.090549536049366,
      "kl": 0.14015483856201172,
      "learning_rate": 3e-06,
      "loss": 0.0169,
      "step": 2092
    },
    {
      "clip_ratio": 0.0002713593712542206,
      "epoch": 0.005810693007734635,
      "grad_norm": 0.10590094327926636,
      "kl": 0.1370547115802765,
      "learning_rate": 3e-06,
      "loss": 0.0161,
      "step": 2093
    },
    {
      "clip_ratio": 0.00025648381415521726,
      "epoch": 0.005813469258574451,
      "grad_norm": 0.1045352965593338,
      "kl": 0.14967551827430725,
      "learning_rate": 3e-06,
      "loss": 0.016,
      "step": 2094
    },
    {
      "clip_ratio": 0.00045060378033667803,
      "epoch": 0.005816245509414266,
      "grad_norm": 0.08050543814897537,
      "kl": 0.13956885784864426,
      "learning_rate": 3e-06,
      "loss": 0.0143,
      "step": 2095
    },
    {
      "clip_ratio": 0.0002441406322759576,
      "epoch": 0.0058190217602540825,
      "grad_norm": 0.14037755131721497,
      "kl": 0.15493950247764587,
      "learning_rate": 3e-06,
      "loss": 0.015,
      "step": 2096
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.005821798011093898,
      "grad_norm": 0.09420851618051529,
      "kl": 0.15672826766967773,
      "learning_rate": 3e-06,
      "loss": 0.0139,
      "step": 2097
    },
    {
      "clip_ratio": 0.0003684598486870527,
      "epoch": 0.005824574261933714,
      "grad_norm": 0.08682415634393692,
      "kl": 0.15357310324907303,
      "learning_rate": 3e-06,
      "loss": 0.016,
      "step": 2098
    },
    {
      "clip_ratio": 0.0007170586031861603,
      "epoch": 0.0058273505127735305,
      "grad_norm": 0.09509247541427612,
      "kl": 0.149859219789505,
      "learning_rate": 3e-06,
      "loss": 0.0145,
      "step": 2099
    },
    {
      "clip_ratio": 0.0007866056985221803,
      "epoch": 0.005830126763613346,
      "grad_norm": 0.1083296537399292,
      "kl": 0.16195228695869446,
      "learning_rate": 3e-06,
      "loss": 0.0151,
      "step": 2100
    },
    {
      "clip_ratio": 0.00034024709020741284,
      "completion_length": 231.875,
      "epoch": 0.005832903014453162,
      "grad_norm": 0.06713063269853592,
      "kl": 0.16988279670476913,
      "learning_rate": 3e-06,
      "loss": 0.0083,
      "reward": 0.1937500163912773,
      "reward_std": 0.16211743280291557,
      "rewards/countdown_reward_func": 0.1937500163912773,
      "step": 2101,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio": 0.0007102232775650918,
      "epoch": 0.0058356792652929776,
      "grad_norm": 0.0946163609623909,
      "kl": 0.17318201810121536,
      "learning_rate": 3e-06,
      "loss": 0.0086,
      "step": 2102
    },
    {
      "clip_ratio": 0.000451006053481251,
      "epoch": 0.005838455516132794,
      "grad_norm": 0.08715200424194336,
      "kl": 0.17537237703800201,
      "learning_rate": 3e-06,
      "loss": 0.0084,
      "step": 2103
    },
    {
      "clip_ratio": 0.0002783698437269777,
      "epoch": 0.005841231766972609,
      "grad_norm": 0.09208168089389801,
      "kl": 0.18494432419538498,
      "learning_rate": 3e-06,
      "loss": 0.009,
      "step": 2104
    },
    {
      "clip_ratio": 0.0006052821408957243,
      "epoch": 0.0058440080178124255,
      "grad_norm": 0.08401837199926376,
      "kl": 0.18430516123771667,
      "learning_rate": 3e-06,
      "loss": 0.0085,
      "step": 2105
    },
    {
      "clip_ratio": 0.0005398763241828419,
      "epoch": 0.005846784268652241,
      "grad_norm": 0.09048160165548325,
      "kl": 0.17285944521427155,
      "learning_rate": 3e-06,
      "loss": 0.0083,
      "step": 2106
    },
    {
      "clip_ratio": 0.001236416690517217,
      "epoch": 0.005849560519492057,
      "grad_norm": 0.06877080351114273,
      "kl": 0.16914395987987518,
      "learning_rate": 3e-06,
      "loss": 0.0078,
      "step": 2107
    },
    {
      "clip_ratio": 0.0004467820399440825,
      "epoch": 0.005852336770331873,
      "grad_norm": 0.08058128505945206,
      "kl": 0.16808201372623444,
      "learning_rate": 3e-06,
      "loss": 0.0074,
      "step": 2108
    },
    {
      "clip_ratio": 0.0016149040893651545,
      "epoch": 0.005855113021171689,
      "grad_norm": 0.0833841860294342,
      "kl": 0.1688889116048813,
      "learning_rate": 3e-06,
      "loss": 0.0083,
      "step": 2109
    },
    {
      "clip_ratio": 0.0009857022087089717,
      "epoch": 0.005857889272011505,
      "grad_norm": 0.07885358482599258,
      "kl": 0.17396697402000427,
      "learning_rate": 3e-06,
      "loss": 0.0075,
      "step": 2110
    },
    {
      "clip_ratio": 0.0015709067229181528,
      "epoch": 0.0058606655228513205,
      "grad_norm": 0.07803882658481598,
      "kl": 0.17317739129066467,
      "learning_rate": 3e-06,
      "loss": 0.0079,
      "step": 2111
    },
    {
      "clip_ratio": 0.0007821381441317499,
      "epoch": 0.005863441773691137,
      "grad_norm": 0.09485685080289841,
      "kl": 0.1609647423028946,
      "learning_rate": 3e-06,
      "loss": 0.0074,
      "step": 2112
    },
    {
      "clip_ratio": 0.0007151653699111193,
      "completion_length": 230.875,
      "epoch": 0.005866218024530952,
      "grad_norm": 0.11236539483070374,
      "kl": 0.1636364832520485,
      "learning_rate": 3e-06,
      "loss": 0.0213,
      "reward": 0.40000002086162567,
      "reward_std": 0.4050685316324234,
      "rewards/countdown_reward_func": 0.40000002086162567,
      "step": 2113,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 0.00017586752073839307,
      "epoch": 0.0058689942753707685,
      "grad_norm": 0.1399078369140625,
      "kl": 0.14752335101366043,
      "learning_rate": 3e-06,
      "loss": 0.0204,
      "step": 2114
    },
    {
      "clip_ratio": 0.000263152651314158,
      "epoch": 0.005871770526210584,
      "grad_norm": 0.13787932693958282,
      "kl": 0.1409480944275856,
      "learning_rate": 3e-06,
      "loss": 0.0213,
      "step": 2115
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0058745467770504,
      "grad_norm": 0.12428930401802063,
      "kl": 0.14609377086162567,
      "learning_rate": 3e-06,
      "loss": 0.0202,
      "step": 2116
    },
    {
      "clip_ratio": 0.0003564371290849522,
      "epoch": 0.005877323027890216,
      "grad_norm": 0.1103990450501442,
      "kl": 0.1507614701986313,
      "learning_rate": 3e-06,
      "loss": 0.0211,
      "step": 2117
    },
    {
      "clip_ratio": 0.00045055238297209144,
      "epoch": 0.005880099278730032,
      "grad_norm": 0.13074712455272675,
      "kl": 0.14322231709957123,
      "learning_rate": 3e-06,
      "loss": 0.0215,
      "step": 2118
    },
    {
      "clip_ratio": 0.0006283415132202208,
      "epoch": 0.005882875529569848,
      "grad_norm": 0.11959907412528992,
      "kl": 0.1588028073310852,
      "learning_rate": 3e-06,
      "loss": 0.0208,
      "step": 2119
    },
    {
      "clip_ratio": 0.0003582945646485314,
      "epoch": 0.0058856517804096635,
      "grad_norm": 0.1354365348815918,
      "kl": 0.1482446938753128,
      "learning_rate": 3e-06,
      "loss": 0.0203,
      "step": 2120
    },
    {
      "clip_ratio": 0.00043741075205616653,
      "epoch": 0.00588842803124948,
      "grad_norm": 0.14199601113796234,
      "kl": 0.14358460903167725,
      "learning_rate": 3e-06,
      "loss": 0.02,
      "step": 2121
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.005891204282089295,
      "grad_norm": 0.11802490055561066,
      "kl": 0.15160782635211945,
      "learning_rate": 3e-06,
      "loss": 0.0193,
      "step": 2122
    },
    {
      "clip_ratio": 0.00018325866403756663,
      "epoch": 0.0058939805329291115,
      "grad_norm": 0.1145329475402832,
      "kl": 0.15987689793109894,
      "learning_rate": 3e-06,
      "loss": 0.0203,
      "step": 2123
    },
    {
      "clip_ratio": 0.0003541165206115693,
      "epoch": 0.005896756783768927,
      "grad_norm": 0.13508586585521698,
      "kl": 0.15324489772319794,
      "learning_rate": 3e-06,
      "loss": 0.0196,
      "step": 2124
    },
    {
      "clip_ratio": 0.0005099133632029407,
      "completion_length": 218.33333587646484,
      "epoch": 0.005899533034608743,
      "grad_norm": 0.11340119689702988,
      "kl": 0.1783110797405243,
      "learning_rate": 3e-06,
      "loss": 0.0107,
      "reward": 0.30416667461395264,
      "reward_std": 0.29580747336149216,
      "rewards/countdown_reward_func": 0.30416665971279144,
      "step": 2125,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.00028971848951186985,
      "epoch": 0.005902309285448559,
      "grad_norm": 0.11806675046682358,
      "kl": 0.19692577421665192,
      "learning_rate": 3e-06,
      "loss": 0.013,
      "step": 2126
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.005905085536288375,
      "grad_norm": 0.1320490837097168,
      "kl": 0.18967118114233017,
      "learning_rate": 3e-06,
      "loss": 0.0124,
      "step": 2127
    },
    {
      "clip_ratio": 0.00026122476992895827,
      "epoch": 0.00590786178712819,
      "grad_norm": 0.11485249549150467,
      "kl": 0.1766699254512787,
      "learning_rate": 3e-06,
      "loss": 0.0114,
      "step": 2128
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0059106380379680065,
      "grad_norm": 0.10251068323850632,
      "kl": 0.175718754529953,
      "learning_rate": 3e-06,
      "loss": 0.0119,
      "step": 2129
    },
    {
      "clip_ratio": 0.0002794336760416627,
      "epoch": 0.005913414288807823,
      "grad_norm": 0.1114218533039093,
      "kl": 0.1810135394334793,
      "learning_rate": 3e-06,
      "loss": 0.011,
      "step": 2130
    },
    {
      "clip_ratio": 9.314456110587344e-05,
      "epoch": 0.005916190539647638,
      "grad_norm": 0.12015023082494736,
      "kl": 0.1831328272819519,
      "learning_rate": 3e-06,
      "loss": 0.0098,
      "step": 2131
    },
    {
      "clip_ratio": 0.0003007580089615658,
      "epoch": 0.0059189667904874545,
      "grad_norm": 0.11531781405210495,
      "kl": 0.1972024142742157,
      "learning_rate": 3e-06,
      "loss": 0.0125,
      "step": 2132
    },
    {
      "clip_ratio": 0.0002856182763935067,
      "epoch": 0.00592174304132727,
      "grad_norm": 0.11725442856550217,
      "kl": 0.18396782875061035,
      "learning_rate": 3e-06,
      "loss": 0.01,
      "step": 2133
    },
    {
      "clip_ratio": 0.00017714993737172335,
      "epoch": 0.005924519292167086,
      "grad_norm": 0.11699375510215759,
      "kl": 0.17351438850164413,
      "learning_rate": 3e-06,
      "loss": 0.0097,
      "step": 2134
    },
    {
      "clip_ratio": 0.0002972428919747472,
      "epoch": 0.0059272955430069016,
      "grad_norm": 0.12814870476722717,
      "kl": 0.16928401589393616,
      "learning_rate": 3e-06,
      "loss": 0.0101,
      "step": 2135
    },
    {
      "clip_ratio": 0.00038728527579223737,
      "epoch": 0.005930071793846718,
      "grad_norm": 0.10801739990711212,
      "kl": 0.17271753400564194,
      "learning_rate": 3e-06,
      "loss": 0.0089,
      "step": 2136
    },
    {
      "clip_ratio": 0.0008804201061138883,
      "completion_length": 240.1666717529297,
      "epoch": 0.005932848044686533,
      "grad_norm": 0.11421734094619751,
      "kl": 0.169437974691391,
      "learning_rate": 3e-06,
      "loss": 0.0286,
      "reward": 0.30416667461395264,
      "reward_std": 0.2904581278562546,
      "rewards/countdown_reward_func": 0.30416667461395264,
      "step": 2137,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0059356242955263495,
      "grad_norm": 0.1658470183610916,
      "kl": 0.16381431370973587,
      "learning_rate": 3e-06,
      "loss": 0.0283,
      "step": 2138
    },
    {
      "clip_ratio": 8.765778329689056e-05,
      "epoch": 0.005938400546366165,
      "grad_norm": 0.1286601573228836,
      "kl": 0.16420385986566544,
      "learning_rate": 3e-06,
      "loss": 0.029,
      "step": 2139
    },
    {
      "clip_ratio": 0.000244140625,
      "epoch": 0.005941176797205981,
      "grad_norm": 0.09617311507463455,
      "kl": 0.15908657014369965,
      "learning_rate": 3e-06,
      "loss": 0.0279,
      "step": 2140
    },
    {
      "clip_ratio": 0.0004251676582498476,
      "epoch": 0.0059439530480457975,
      "grad_norm": 0.10428610444068909,
      "kl": 0.1709904745221138,
      "learning_rate": 3e-06,
      "loss": 0.029,
      "step": 2141
    },
    {
      "clip_ratio": 0.000519495370099321,
      "epoch": 0.005946729298885613,
      "grad_norm": 0.08956306427717209,
      "kl": 0.17576873302459717,
      "learning_rate": 3e-06,
      "loss": 0.0287,
      "step": 2142
    },
    {
      "clip_ratio": 0.00035832179128192365,
      "epoch": 0.005949505549725429,
      "grad_norm": 0.09938909113407135,
      "kl": 0.1722540631890297,
      "learning_rate": 3e-06,
      "loss": 0.0279,
      "step": 2143
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0059522818005652445,
      "grad_norm": 0.0916515365242958,
      "kl": 0.17020156979560852,
      "learning_rate": 3e-06,
      "loss": 0.0275,
      "step": 2144
    },
    {
      "clip_ratio": 0.00017531556659378111,
      "epoch": 0.005955058051405061,
      "grad_norm": 0.1481482982635498,
      "kl": 0.17428260296583176,
      "learning_rate": 3e-06,
      "loss": 0.0269,
      "step": 2145
    },
    {
      "clip_ratio": 0.0007600796525366604,
      "epoch": 0.005957834302244876,
      "grad_norm": 0.10551691800355911,
      "kl": 0.17008347064256668,
      "learning_rate": 3e-06,
      "loss": 0.0264,
      "step": 2146
    },
    {
      "clip_ratio": 0.00026695779524743557,
      "epoch": 0.0059606105530846925,
      "grad_norm": 0.1174376830458641,
      "kl": 0.18421828001737595,
      "learning_rate": 3e-06,
      "loss": 0.0284,
      "step": 2147
    },
    {
      "clip_ratio": 0.000535756757017225,
      "epoch": 0.005963386803924508,
      "grad_norm": 0.08853857964277267,
      "kl": 0.19289961457252502,
      "learning_rate": 3e-06,
      "loss": 0.0272,
      "step": 2148
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "completion_length": 232.875,
      "epoch": 0.005966163054764324,
      "grad_norm": 0.04596855491399765,
      "kl": 0.18686389178037643,
      "learning_rate": 3e-06,
      "loss": 0.0206,
      "reward": 0.17083335667848587,
      "reward_std": 0.11509480327367783,
      "rewards/countdown_reward_func": 0.17083334922790527,
      "step": 2149,
      "zero_std_ratio": 0.625
    },
    {
      "clip_ratio": 0.000768062163842842,
      "epoch": 0.00596893930560414,
      "grad_norm": 0.0615745410323143,
      "kl": 0.1883329302072525,
      "learning_rate": 3e-06,
      "loss": 0.0202,
      "step": 2150
    },
    {
      "clip_ratio": 0.0002582559172878973,
      "epoch": 0.005971715556443956,
      "grad_norm": 0.060142528265714645,
      "kl": 0.2043200358748436,
      "learning_rate": 3e-06,
      "loss": 0.0209,
      "step": 2151
    },
    {
      "clip_ratio": 0.00044469654676504433,
      "epoch": 0.005974491807283772,
      "grad_norm": 0.066607765853405,
      "kl": 0.22076967358589172,
      "learning_rate": 3e-06,
      "loss": 0.0211,
      "step": 2152
    },
    {
      "clip_ratio": 0.0004701242069131695,
      "epoch": 0.0059772680581235875,
      "grad_norm": 0.08139292895793915,
      "kl": 0.20817560702562332,
      "learning_rate": 3e-06,
      "loss": 0.0203,
      "step": 2153
    },
    {
      "clip_ratio": 0.0001826150546548888,
      "epoch": 0.005980044308963404,
      "grad_norm": 0.07108417898416519,
      "kl": 0.20678973197937012,
      "learning_rate": 3e-06,
      "loss": 0.02,
      "step": 2154
    },
    {
      "clip_ratio": 0.00027385010616853833,
      "epoch": 0.005982820559803219,
      "grad_norm": 0.044102661311626434,
      "kl": 0.2047039493918419,
      "learning_rate": 3e-06,
      "loss": 0.0201,
      "step": 2155
    },
    {
      "clip_ratio": 0.00040690103196538985,
      "epoch": 0.0059855968106430355,
      "grad_norm": 0.054487258195877075,
      "kl": 0.20282930880784988,
      "learning_rate": 3e-06,
      "loss": 0.0196,
      "step": 2156
    },
    {
      "clip_ratio": 0.00010032102727564052,
      "epoch": 0.005988373061482851,
      "grad_norm": 0.05873025581240654,
      "kl": 0.22172988951206207,
      "learning_rate": 3e-06,
      "loss": 0.0205,
      "step": 2157
    },
    {
      "clip_ratio": 0.00020064205455128103,
      "epoch": 0.005991149312322667,
      "grad_norm": 0.06830353289842606,
      "kl": 0.2312830686569214,
      "learning_rate": 3e-06,
      "loss": 0.0203,
      "step": 2158
    },
    {
      "clip_ratio": 0.00020064205455128103,
      "epoch": 0.005993925563162483,
      "grad_norm": 0.06709025055170059,
      "kl": 0.21746045351028442,
      "learning_rate": 3e-06,
      "loss": 0.0191,
      "step": 2159
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.005996701814002299,
      "grad_norm": 0.07886490970849991,
      "kl": 0.21650362014770508,
      "learning_rate": 3e-06,
      "loss": 0.0196,
      "step": 2160
    },
    {
      "clip_ratio": 9.391435014549643e-05,
      "completion_length": 214.43750762939453,
      "epoch": 0.005999478064842114,
      "grad_norm": 0.11006475239992142,
      "kl": 0.21128001064062119,
      "learning_rate": 3e-06,
      "loss": 0.0006,
      "reward": 0.28333335369825363,
      "reward_std": 0.2853127121925354,
      "rewards/countdown_reward_func": 0.28333333879709244,
      "step": 2161,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0001065643664333038,
      "epoch": 0.0060022543156819305,
      "grad_norm": 0.17413948476314545,
      "kl": 0.20346222817897797,
      "learning_rate": 3e-06,
      "loss": -0.0006,
      "step": 2162
    },
    {
      "clip_ratio": 0.00018050541984848678,
      "epoch": 0.006005030566521747,
      "grad_norm": 0.11314403265714645,
      "kl": 0.19823309034109116,
      "learning_rate": 3e-06,
      "loss": 0.0001,
      "step": 2163
    },
    {
      "clip_ratio": 0.0005011356552131474,
      "epoch": 0.006007806817361562,
      "grad_norm": 0.1388438194990158,
      "kl": 0.1954411193728447,
      "learning_rate": 3e-06,
      "loss": 0.0001,
      "step": 2164
    },
    {
      "clip_ratio": 0.00022549449931830168,
      "epoch": 0.0060105830682013785,
      "grad_norm": 0.1569499969482422,
      "kl": 0.18387842923402786,
      "learning_rate": 3e-06,
      "loss": -0.0012,
      "step": 2165
    },
    {
      "clip_ratio": 0.0001065643664333038,
      "epoch": 0.006013359319041194,
      "grad_norm": 0.14936763048171997,
      "kl": 0.19631676375865936,
      "learning_rate": 3e-06,
      "loss": -0.0009,
      "step": 2166
    },
    {
      "clip_ratio": 0.0001065643664333038,
      "epoch": 0.00601613556988101,
      "grad_norm": 0.10569910705089569,
      "kl": 0.19845695048570633,
      "learning_rate": 3e-06,
      "loss": -0.0009,
      "step": 2167
    },
    {
      "clip_ratio": 0.00045912877249065787,
      "epoch": 0.0060189118207208256,
      "grad_norm": 0.1597263067960739,
      "kl": 0.18819713592529297,
      "learning_rate": 3e-06,
      "loss": -0.0028,
      "step": 2168
    },
    {
      "clip_ratio": 0.000762759504141286,
      "epoch": 0.006021688071560642,
      "grad_norm": 0.1164880320429802,
      "kl": 0.17806652933359146,
      "learning_rate": 3e-06,
      "loss": -0.0014,
      "step": 2169
    },
    {
      "clip_ratio": 0.0007057104958221316,
      "epoch": 0.006024464322400457,
      "grad_norm": 0.12073780596256256,
      "kl": 0.17337379604578018,
      "learning_rate": 3e-06,
      "loss": -0.0033,
      "step": 2170
    },
    {
      "clip_ratio": 0.00038431792927440256,
      "epoch": 0.0060272405732402735,
      "grad_norm": 0.15236780047416687,
      "kl": 0.1608799397945404,
      "learning_rate": 3e-06,
      "loss": -0.0041,
      "step": 2171
    },
    {
      "clip_ratio": 0.0005894555361010134,
      "epoch": 0.006030016824080089,
      "grad_norm": 0.1417692005634308,
      "kl": 0.17029716074466705,
      "learning_rate": 3e-06,
      "loss": -0.0044,
      "step": 2172
    },
    {
      "clip_ratio": 0.00017841139197116718,
      "completion_length": 235.6041717529297,
      "epoch": 0.006032793074919905,
      "grad_norm": 0.09551423043012619,
      "kl": 0.1639445647597313,
      "learning_rate": 3e-06,
      "loss": -0.0062,
      "reward": 0.21250000596046448,
      "reward_std": 0.11618950217962265,
      "rewards/countdown_reward_func": 0.21250000596046448,
      "step": 2173,
      "zero_std_ratio": 0.75
    },
    {
      "clip_ratio": 0.0006238899368327111,
      "epoch": 0.0060355693257597215,
      "grad_norm": 0.11371457576751709,
      "kl": 0.15197165310382843,
      "learning_rate": 3e-06,
      "loss": -0.0061,
      "step": 2174
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.006038345576599537,
      "grad_norm": 0.09170381724834442,
      "kl": 0.15608438104391098,
      "learning_rate": 3e-06,
      "loss": -0.0057,
      "step": 2175
    },
    {
      "clip_ratio": 0.0005786644760519266,
      "epoch": 0.006041121827439353,
      "grad_norm": 0.08310035616159439,
      "kl": 0.15356775373220444,
      "learning_rate": 3e-06,
      "loss": -0.0068,
      "step": 2176
    },
    {
      "clip_ratio": 8.802816591924056e-05,
      "epoch": 0.0060438980782791685,
      "grad_norm": 0.10131369531154633,
      "kl": 0.148365817964077,
      "learning_rate": 3e-06,
      "loss": -0.0066,
      "step": 2177
    },
    {
      "clip_ratio": 0.0007334884430747479,
      "epoch": 0.006046674329118985,
      "grad_norm": 0.0910923108458519,
      "kl": 0.14135953783988953,
      "learning_rate": 3e-06,
      "loss": -0.0067,
      "step": 2178
    },
    {
      "clip_ratio": 0.0007884440710768104,
      "epoch": 0.0060494505799588,
      "grad_norm": 0.09182950854301453,
      "kl": 0.1402914896607399,
      "learning_rate": 3e-06,
      "loss": -0.0073,
      "step": 2179
    },
    {
      "clip_ratio": 0.0012513441615737975,
      "epoch": 0.0060522268307986165,
      "grad_norm": 0.11456513404846191,
      "kl": 0.13089902698993683,
      "learning_rate": 3e-06,
      "loss": -0.0089,
      "step": 2180
    },
    {
      "clip_ratio": 0.0005369534774217755,
      "epoch": 0.006055003081638432,
      "grad_norm": 0.09350922703742981,
      "kl": 0.1340053342282772,
      "learning_rate": 3e-06,
      "loss": -0.0084,
      "step": 2181
    },
    {
      "clip_ratio": 0.0014489490713458508,
      "epoch": 0.006057779332478248,
      "grad_norm": 0.08015953004360199,
      "kl": 0.13297784700989723,
      "learning_rate": 3e-06,
      "loss": -0.0091,
      "step": 2182
    },
    {
      "clip_ratio": 0.0017656179843470454,
      "epoch": 0.006060555583318064,
      "grad_norm": 0.08231103420257568,
      "kl": 0.130483016371727,
      "learning_rate": 3e-06,
      "loss": -0.0095,
      "step": 2183
    },
    {
      "clip_ratio": 0.002995648537762463,
      "epoch": 0.00606333183415788,
      "grad_norm": 0.0673069879412651,
      "kl": 0.12187189608812332,
      "learning_rate": 3e-06,
      "loss": -0.009,
      "step": 2184
    },
    {
      "clip_ratio": 0.0004267231997800991,
      "completion_length": 233.3541717529297,
      "epoch": 0.006066108084997696,
      "grad_norm": 0.07689964771270752,
      "kl": 0.1193951666355133,
      "learning_rate": 3e-06,
      "loss": 0.0086,
      "reward": 0.21250000596046448,
      "reward_std": 0.2418064922094345,
      "rewards/countdown_reward_func": 0.21250000596046448,
      "step": 2185,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0002802690723910928,
      "epoch": 0.0060688843358375115,
      "grad_norm": 0.07742653042078018,
      "kl": 0.1159132868051529,
      "learning_rate": 3e-06,
      "loss": 0.0085,
      "step": 2186
    },
    {
      "clip_ratio": 0.0004338699218351394,
      "epoch": 0.006071660586677328,
      "grad_norm": 0.11565269529819489,
      "kl": 0.11845757439732552,
      "learning_rate": 3e-06,
      "loss": 0.0084,
      "step": 2187
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.006074436837517143,
      "grad_norm": 0.07725950330495834,
      "kl": 0.11740844696760178,
      "learning_rate": 3e-06,
      "loss": 0.0082,
      "step": 2188
    },
    {
      "clip_ratio": 0.00018731241289060563,
      "epoch": 0.0060772130883569595,
      "grad_norm": 0.0754290521144867,
      "kl": 0.11156599596142769,
      "learning_rate": 3e-06,
      "loss": 0.008,
      "step": 2189
    },
    {
      "clip_ratio": 0.000675759933074005,
      "epoch": 0.006079989339196775,
      "grad_norm": 0.10546611994504929,
      "kl": 0.11662887409329414,
      "learning_rate": 3e-06,
      "loss": 0.0078,
      "step": 2190
    },
    {
      "clip_ratio": 0.00046884678886272013,
      "epoch": 0.006082765590036591,
      "grad_norm": 0.07807113230228424,
      "kl": 0.11078613996505737,
      "learning_rate": 3e-06,
      "loss": 0.0078,
      "step": 2191
    },
    {
      "clip_ratio": 0.00047955120680853724,
      "epoch": 0.0060855418408764066,
      "grad_norm": 0.0798504427075386,
      "kl": 0.10783285275101662,
      "learning_rate": 3e-06,
      "loss": 0.0075,
      "step": 2192
    },
    {
      "clip_ratio": 0.00029404355882434174,
      "epoch": 0.006088318091716223,
      "grad_norm": 0.09978339821100235,
      "kl": 0.11092406511306763,
      "learning_rate": 3e-06,
      "loss": 0.0074,
      "step": 2193
    },
    {
      "clip_ratio": 0.000244140625,
      "epoch": 0.006091094342556038,
      "grad_norm": 0.08134466409683228,
      "kl": 0.11090698093175888,
      "learning_rate": 3e-06,
      "loss": 0.008,
      "step": 2194
    },
    {
      "clip_ratio": 0.0006887645286042243,
      "epoch": 0.0060938705933958545,
      "grad_norm": 0.07155325263738632,
      "kl": 0.10384676232933998,
      "learning_rate": 3e-06,
      "loss": 0.0072,
      "step": 2195
    },
    {
      "clip_ratio": 0.0011016842036042362,
      "epoch": 0.006096646844235671,
      "grad_norm": 0.08413667976856232,
      "kl": 0.11054657027125359,
      "learning_rate": 3e-06,
      "loss": 0.007,
      "step": 2196
    },
    {
      "clip_ratio": 0.00044989935122430325,
      "completion_length": 235.1875,
      "epoch": 0.006099423095075486,
      "grad_norm": 0.07087922096252441,
      "kl": 0.1160878986120224,
      "learning_rate": 3e-06,
      "loss": 0.0124,
      "reward": 0.2083333507180214,
      "reward_std": 0.2182515673339367,
      "rewards/countdown_reward_func": 0.2083333358168602,
      "step": 2197,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0061021993459153025,
      "grad_norm": 0.07426546514034271,
      "kl": 0.11279673129320145,
      "learning_rate": 3e-06,
      "loss": 0.0122,
      "step": 2198
    },
    {
      "clip_ratio": 8.185985643649474e-05,
      "epoch": 0.006104975596755118,
      "grad_norm": 0.07759413123130798,
      "kl": 0.116276104003191,
      "learning_rate": 3e-06,
      "loss": 0.012,
      "step": 2199
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.006107751847594934,
      "grad_norm": 0.0708848312497139,
      "kl": 0.10642623901367188,
      "learning_rate": 3e-06,
      "loss": 0.0116,
      "step": 2200
    },
    {
      "clip_ratio": 0.00017426943668397143,
      "epoch": 0.0061105280984347496,
      "grad_norm": 0.06520801782608032,
      "kl": 0.12019915506243706,
      "learning_rate": 3e-06,
      "loss": 0.0127,
      "step": 2201
    },
    {
      "clip_ratio": 0.00034561891516204923,
      "epoch": 0.006113304349274566,
      "grad_norm": 0.07822219282388687,
      "kl": 0.10957641154527664,
      "learning_rate": 3e-06,
      "loss": 0.0129,
      "step": 2202
    },
    {
      "clip_ratio": 0.0004435717419255525,
      "epoch": 0.006116080600114381,
      "grad_norm": 0.07648086547851562,
      "kl": 0.11310628056526184,
      "learning_rate": 3e-06,
      "loss": 0.0122,
      "step": 2203
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0061188568509541975,
      "grad_norm": 0.08405080437660217,
      "kl": 0.11058183759450912,
      "learning_rate": 3e-06,
      "loss": 0.0112,
      "step": 2204
    },
    {
      "clip_ratio": 9.469696669839323e-05,
      "epoch": 0.006121633101794013,
      "grad_norm": 0.08572155982255936,
      "kl": 0.11485684290528297,
      "learning_rate": 3e-06,
      "loss": 0.0113,
      "step": 2205
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.006124409352633829,
      "grad_norm": 0.07442327588796616,
      "kl": 0.1044563390314579,
      "learning_rate": 3e-06,
      "loss": 0.0112,
      "step": 2206
    },
    {
      "clip_ratio": 0.0004602315020747483,
      "epoch": 0.0061271856034736455,
      "grad_norm": 0.18411950767040253,
      "kl": 0.11828203499317169,
      "learning_rate": 3e-06,
      "loss": 0.0119,
      "step": 2207
    },
    {
      "clip_ratio": 0.0005978553963359445,
      "epoch": 0.006129961854313461,
      "grad_norm": 0.0784916952252388,
      "kl": 0.10879363864660263,
      "learning_rate": 3e-06,
      "loss": 0.0119,
      "step": 2208
    },
    {
      "clip_ratio": 0.0002201182724093087,
      "completion_length": 233.75,
      "epoch": 0.006132738105153277,
      "grad_norm": 0.11505284905433655,
      "kl": 0.10442957654595375,
      "learning_rate": 3e-06,
      "loss": 0.024,
      "reward": 0.3812500089406967,
      "reward_std": 0.3858536630868912,
      "rewards/countdown_reward_func": 0.3812500089406967,
      "step": 2209,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 0.0003907842037733644,
      "epoch": 0.0061355143559930925,
      "grad_norm": 0.10098423808813095,
      "kl": 0.10408291965723038,
      "learning_rate": 3e-06,
      "loss": 0.0239,
      "step": 2210
    },
    {
      "clip_ratio": 0.0001870947889983654,
      "epoch": 0.006138290606832909,
      "grad_norm": 0.10101636499166489,
      "kl": 0.10832390189170837,
      "learning_rate": 3e-06,
      "loss": 0.0225,
      "step": 2211
    },
    {
      "clip_ratio": 0.0004268408374628052,
      "epoch": 0.006141066857672724,
      "grad_norm": 0.11006015539169312,
      "kl": 0.10417984053492546,
      "learning_rate": 3e-06,
      "loss": 0.0225,
      "step": 2212
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0061438431085125405,
      "grad_norm": 0.1787508875131607,
      "kl": 0.11693385615944862,
      "learning_rate": 3e-06,
      "loss": 0.0225,
      "step": 2213
    },
    {
      "clip_ratio": 0.0005292101996019483,
      "epoch": 0.006146619359352356,
      "grad_norm": 0.12185002118349075,
      "kl": 0.11256387084722519,
      "learning_rate": 3e-06,
      "loss": 0.0223,
      "step": 2214
    },
    {
      "clip_ratio": 0.00012230919674038887,
      "epoch": 0.006149395610192172,
      "grad_norm": 0.12709738314151764,
      "kl": 0.10908814519643784,
      "learning_rate": 3e-06,
      "loss": 0.023,
      "step": 2215
    },
    {
      "clip_ratio": 0.00012230919674038887,
      "epoch": 0.006152171861031988,
      "grad_norm": 0.10294611752033234,
      "kl": 0.11007463932037354,
      "learning_rate": 3e-06,
      "loss": 0.0217,
      "step": 2216
    },
    {
      "clip_ratio": 0.0002929751281044446,
      "epoch": 0.006154948111871804,
      "grad_norm": 0.09818847477436066,
      "kl": 0.11554765328764915,
      "learning_rate": 3e-06,
      "loss": 0.0212,
      "step": 2217
    },
    {
      "clip_ratio": 0.000292975120828487,
      "epoch": 0.00615772436271162,
      "grad_norm": 0.11660251021385193,
      "kl": 0.11222562938928604,
      "learning_rate": 3e-06,
      "loss": 0.0201,
      "step": 2218
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0061605006135514355,
      "grad_norm": 0.09623023122549057,
      "kl": 0.12738066911697388,
      "learning_rate": 3e-06,
      "loss": 0.0199,
      "step": 2219
    },
    {
      "clip_ratio": 0.0007260283455252647,
      "epoch": 0.006163276864391252,
      "grad_norm": 0.09498999267816544,
      "kl": 0.12235638499259949,
      "learning_rate": 3e-06,
      "loss": 0.0207,
      "step": 2220
    },
    {
      "clip_ratio": 0.00026175539096584544,
      "completion_length": 239.08333587646484,
      "epoch": 0.006166053115231067,
      "grad_norm": 0.08734557777643204,
      "kl": 0.10614108666777611,
      "learning_rate": 3e-06,
      "loss": 0.0156,
      "reward": 0.23124999552965164,
      "reward_std": 0.2115694098174572,
      "rewards/countdown_reward_func": 0.23124999552965164,
      "step": 2221,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio": 0.00016405216592829674,
      "epoch": 0.0061688293660708835,
      "grad_norm": 0.11606533825397491,
      "kl": 0.10931029170751572,
      "learning_rate": 3e-06,
      "loss": 0.0164,
      "step": 2222
    },
    {
      "clip_ratio": 0.0002663365739863366,
      "epoch": 0.006171605616910699,
      "grad_norm": 0.07539436966180801,
      "kl": 0.10711556673049927,
      "learning_rate": 3e-06,
      "loss": 0.0157,
      "step": 2223
    },
    {
      "clip_ratio": 0.0002645519489306025,
      "epoch": 0.006174381867750515,
      "grad_norm": 0.06568428874015808,
      "kl": 0.12526912242174149,
      "learning_rate": 3e-06,
      "loss": 0.0164,
      "step": 2224
    },
    {
      "clip_ratio": 0.00026455195620656013,
      "epoch": 0.0061771581185903306,
      "grad_norm": 0.07675132900476456,
      "kl": 0.11198921501636505,
      "learning_rate": 3e-06,
      "loss": 0.0158,
      "step": 2225
    },
    {
      "clip_ratio": 0.0007440476329065859,
      "epoch": 0.006179934369430147,
      "grad_norm": 0.07890935987234116,
      "kl": 0.11408694460988045,
      "learning_rate": 3e-06,
      "loss": 0.0169,
      "step": 2226
    },
    {
      "clip_ratio": 0.0004973536997567862,
      "epoch": 0.006182710620269962,
      "grad_norm": 0.08032620698213577,
      "kl": 0.11937838792800903,
      "learning_rate": 3e-06,
      "loss": 0.0155,
      "step": 2227
    },
    {
      "clip_ratio": 0.00019656029326142743,
      "epoch": 0.0061854868711097785,
      "grad_norm": 0.09635122865438461,
      "kl": 0.11930480599403381,
      "learning_rate": 3e-06,
      "loss": 0.0145,
      "step": 2228
    },
    {
      "clip_ratio": 9.476876584812999e-05,
      "epoch": 0.006188263121949595,
      "grad_norm": 0.21204665303230286,
      "kl": 0.11878373473882675,
      "learning_rate": 3e-06,
      "loss": 0.0142,
      "step": 2229
    },
    {
      "clip_ratio": 0.0009819945989875123,
      "epoch": 0.00619103937278941,
      "grad_norm": 0.06745634227991104,
      "kl": 0.13793090730905533,
      "learning_rate": 3e-06,
      "loss": 0.0158,
      "step": 2230
    },
    {
      "clip_ratio": 0.0004885463276877999,
      "epoch": 0.0061938156236292265,
      "grad_norm": 0.07236343622207642,
      "kl": 0.12467692419886589,
      "learning_rate": 3e-06,
      "loss": 0.0147,
      "step": 2231
    },
    {
      "clip_ratio": 0.0003369117039255798,
      "epoch": 0.006196591874469042,
      "grad_norm": 0.07333452254533768,
      "kl": 0.12604239955544472,
      "learning_rate": 3e-06,
      "loss": 0.0151,
      "step": 2232
    },
    {
      "clip_ratio": 0.00044176532537676394,
      "completion_length": 227.5,
      "epoch": 0.006199368125308858,
      "grad_norm": 0.11169209331274033,
      "kl": 0.14527644217014313,
      "learning_rate": 3e-06,
      "loss": 0.0172,
      "reward": 0.4520833492279053,
      "reward_std": 0.3492494821548462,
      "rewards/countdown_reward_func": 0.4520833194255829,
      "step": 2233,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 8.722958591533825e-05,
      "epoch": 0.0062021443761486736,
      "grad_norm": 0.12365715205669403,
      "kl": 0.15404719859361649,
      "learning_rate": 3e-06,
      "loss": 0.0176,
      "step": 2234
    },
    {
      "clip_ratio": 0.0003403676091693342,
      "epoch": 0.00620492062698849,
      "grad_norm": 0.10898413509130478,
      "kl": 0.15741366147994995,
      "learning_rate": 3e-06,
      "loss": 0.0166,
      "step": 2235
    },
    {
      "clip_ratio": 8.722958591533825e-05,
      "epoch": 0.006207696877828305,
      "grad_norm": 0.11201989650726318,
      "kl": 0.15081220865249634,
      "learning_rate": 3e-06,
      "loss": 0.0165,
      "step": 2236
    },
    {
      "clip_ratio": 0.00020764119108207524,
      "epoch": 0.0062104731286681215,
      "grad_norm": 0.2154986560344696,
      "kl": 0.15322324633598328,
      "learning_rate": 3e-06,
      "loss": 0.0158,
      "step": 2237
    },
    {
      "clip_ratio": 0.0006642429507337511,
      "epoch": 0.006213249379507937,
      "grad_norm": 0.0987052395939827,
      "kl": 0.15503258258104324,
      "learning_rate": 3e-06,
      "loss": 0.0166,
      "step": 2238
    },
    {
      "clip_ratio": 8.722958591533825e-05,
      "epoch": 0.006216025630347753,
      "grad_norm": 0.10490848124027252,
      "kl": 0.154758021235466,
      "learning_rate": 3e-06,
      "loss": 0.0157,
      "step": 2239
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0062188018811875695,
      "grad_norm": 0.1208440288901329,
      "kl": 0.16184867918491364,
      "learning_rate": 3e-06,
      "loss": 0.016,
      "step": 2240
    },
    {
      "clip_ratio": 0.0002552756923250854,
      "epoch": 0.006221578132027385,
      "grad_norm": 0.09863177686929703,
      "kl": 0.16634425520896912,
      "learning_rate": 3e-06,
      "loss": 0.0159,
      "step": 2241
    },
    {
      "clip_ratio": 0.00017934454808710143,
      "epoch": 0.006224354382867201,
      "grad_norm": 0.11939014494419098,
      "kl": 0.16019989550113678,
      "learning_rate": 3e-06,
      "loss": 0.0145,
      "step": 2242
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0062271306337070165,
      "grad_norm": 0.16073253750801086,
      "kl": 0.16025684773921967,
      "learning_rate": 3e-06,
      "loss": 0.0131,
      "step": 2243
    },
    {
      "clip_ratio": 0.0008758519834373146,
      "epoch": 0.006229906884546833,
      "grad_norm": 0.11048475652933121,
      "kl": 0.1635778620839119,
      "learning_rate": 3e-06,
      "loss": 0.0148,
      "step": 2244
    },
    {
      "clip_ratio": 8.90313385752961e-05,
      "completion_length": 217.1041717529297,
      "epoch": 0.006232683135386648,
      "grad_norm": 0.1347557008266449,
      "kl": 0.16955012828111649,
      "learning_rate": 3e-06,
      "loss": 0.0281,
      "reward": 0.35625001788139343,
      "reward_std": 0.3819515109062195,
      "rewards/countdown_reward_func": 0.35625001043081284,
      "step": 2245,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 9.204712841892615e-05,
      "epoch": 0.0062354593862264645,
      "grad_norm": 0.1472625583410263,
      "kl": 0.16981971263885498,
      "learning_rate": 3e-06,
      "loss": 0.0262,
      "step": 2246
    },
    {
      "clip_ratio": 0.0001900863426271826,
      "epoch": 0.00623823563706628,
      "grad_norm": 0.11650460213422775,
      "kl": 0.16689752787351608,
      "learning_rate": 3e-06,
      "loss": 0.0278,
      "step": 2247
    },
    {
      "clip_ratio": 0.0001804056082619354,
      "epoch": 0.006241011887906096,
      "grad_norm": 0.13127779960632324,
      "kl": 0.1765117198228836,
      "learning_rate": 3e-06,
      "loss": 0.028,
      "step": 2248
    },
    {
      "clip_ratio": 0.0005522827850654721,
      "epoch": 0.006243788138745912,
      "grad_norm": 0.12236137688159943,
      "kl": 0.16286692768335342,
      "learning_rate": 3e-06,
      "loss": 0.0261,
      "step": 2249
    },
    {
      "clip_ratio": 0.00018342139810556546,
      "epoch": 0.006246564389585728,
      "grad_norm": 0.13048216700553894,
      "kl": 0.16761357337236404,
      "learning_rate": 3e-06,
      "loss": 0.0269,
      "step": 2250
    },
    {
      "clip_ratio": 0.000550264201592654,
      "epoch": 0.006249340640425544,
      "grad_norm": 0.13221754133701324,
      "kl": 0.18436182290315628,
      "learning_rate": 3e-06,
      "loss": 0.0251,
      "step": 2251
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0062521168912653595,
      "grad_norm": 0.14144454896450043,
      "kl": 0.18143575638532639,
      "learning_rate": 3e-06,
      "loss": 0.0239,
      "step": 2252
    },
    {
      "clip_ratio": 0.00033115307451225817,
      "epoch": 0.006254893142105176,
      "grad_norm": 0.13937947154045105,
      "kl": 0.18208129703998566,
      "learning_rate": 3e-06,
      "loss": 0.0254,
      "step": 2253
    },
    {
      "clip_ratio": 0.00037381629226729274,
      "epoch": 0.006257669392944991,
      "grad_norm": 0.11771649122238159,
      "kl": 0.1911814734339714,
      "learning_rate": 3e-06,
      "loss": 0.0255,
      "step": 2254
    },
    {
      "clip_ratio": 0.000840340624563396,
      "epoch": 0.0062604456437848075,
      "grad_norm": 0.11015734076499939,
      "kl": 0.17935297638177872,
      "learning_rate": 3e-06,
      "loss": 0.0235,
      "step": 2255
    },
    {
      "clip_ratio": 0.0008332670404342934,
      "epoch": 0.006263221894624623,
      "grad_norm": 0.12103752791881561,
      "kl": 0.180360309779644,
      "learning_rate": 3e-06,
      "loss": 0.0231,
      "step": 2256
    },
    {
      "clip_ratio": 0.0001020408162730746,
      "completion_length": 203.4166717529297,
      "epoch": 0.006265998145464439,
      "grad_norm": 0.1085449829697609,
      "kl": 0.178614042699337,
      "learning_rate": 3e-06,
      "loss": 0.0057,
      "reward": 0.39791668951511383,
      "reward_std": 0.32604455202817917,
      "rewards/countdown_reward_func": 0.39791667461395264,
      "step": 2257,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.000700604279700201,
      "epoch": 0.0062687743963042546,
      "grad_norm": 0.12487374991178513,
      "kl": 0.18157772719860077,
      "learning_rate": 3e-06,
      "loss": 0.0065,
      "step": 2258
    },
    {
      "clip_ratio": 0.0005079238399048336,
      "epoch": 0.006271550647144071,
      "grad_norm": 0.16143116354942322,
      "kl": 0.19268468022346497,
      "learning_rate": 3e-06,
      "loss": 0.0063,
      "step": 2259
    },
    {
      "clip_ratio": 9.97605748125352e-05,
      "epoch": 0.006274326897983886,
      "grad_norm": 0.12245763093233109,
      "kl": 0.1898026168346405,
      "learning_rate": 3e-06,
      "loss": 0.0064,
      "step": 2260
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0062771031488237025,
      "grad_norm": 0.11813352257013321,
      "kl": 0.18366330116987228,
      "learning_rate": 3e-06,
      "loss": 0.0053,
      "step": 2261
    },
    {
      "clip_ratio": 0.0006266141863306984,
      "epoch": 0.006279879399663519,
      "grad_norm": 0.2579954266548157,
      "kl": 0.18229518085718155,
      "learning_rate": 3e-06,
      "loss": 0.0059,
      "step": 2262
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.006282655650503334,
      "grad_norm": 0.11579995602369308,
      "kl": 0.1908181607723236,
      "learning_rate": 3e-06,
      "loss": 0.0042,
      "step": 2263
    },
    {
      "clip_ratio": 0.0006184200756251812,
      "epoch": 0.0062854319013431505,
      "grad_norm": 0.1960228681564331,
      "kl": 0.1934300884604454,
      "learning_rate": 3e-06,
      "loss": 0.0054,
      "step": 2264
    },
    {
      "clip_ratio": 0.0012736761127598584,
      "epoch": 0.006288208152182966,
      "grad_norm": 0.12140205502510071,
      "kl": 0.20321929454803467,
      "learning_rate": 3e-06,
      "loss": 0.0057,
      "step": 2265
    },
    {
      "clip_ratio": 0.000426823906309437,
      "epoch": 0.006290984403022782,
      "grad_norm": 0.1191827654838562,
      "kl": 0.1980365291237831,
      "learning_rate": 3e-06,
      "loss": 0.0048,
      "step": 2266
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0062937606538625976,
      "grad_norm": 0.1335882693529129,
      "kl": 0.1887526884675026,
      "learning_rate": 3e-06,
      "loss": 0.004,
      "step": 2267
    },
    {
      "clip_ratio": 0.0004993882175767794,
      "epoch": 0.006296536904702414,
      "grad_norm": 0.11730194836854935,
      "kl": 0.18969019502401352,
      "learning_rate": 3e-06,
      "loss": 0.0042,
      "step": 2268
    },
    {
      "clip_ratio": 0.00018388705211691558,
      "completion_length": 227.70833587646484,
      "epoch": 0.006299313155542229,
      "grad_norm": 0.12374776601791382,
      "kl": 0.22037464380264282,
      "learning_rate": 3e-06,
      "loss": -0.0004,
      "reward": 0.2875000163912773,
      "reward_std": 0.2661401182413101,
      "rewards/countdown_reward_func": 0.2875000014901161,
      "step": 2269,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.00017997733084484935,
      "epoch": 0.0063020894063820455,
      "grad_norm": 0.13824217021465302,
      "kl": 0.2126154974102974,
      "learning_rate": 3e-06,
      "loss": -0.001,
      "step": 2270
    },
    {
      "clip_ratio": 8.802816591924056e-05,
      "epoch": 0.006304865657221861,
      "grad_norm": 0.18445436656475067,
      "kl": 0.2134125828742981,
      "learning_rate": 3e-06,
      "loss": -0.0016,
      "step": 2271
    },
    {
      "clip_ratio": 0.0001707650226308033,
      "epoch": 0.006307641908061677,
      "grad_norm": 0.13947488367557526,
      "kl": 0.2060968428850174,
      "learning_rate": 3e-06,
      "loss": -0.0013,
      "step": 2272
    },
    {
      "clip_ratio": 0.00017788648256100714,
      "epoch": 0.0063104181589014935,
      "grad_norm": 0.13269934058189392,
      "kl": 0.20021595805883408,
      "learning_rate": 3e-06,
      "loss": -0.0005,
      "step": 2273
    },
    {
      "clip_ratio": 0.0003868369967676699,
      "epoch": 0.006313194409741309,
      "grad_norm": 0.13080525398254395,
      "kl": 0.20501340180635452,
      "learning_rate": 3e-06,
      "loss": -0.0021,
      "step": 2274
    },
    {
      "clip_ratio": 0.0005433336482383311,
      "epoch": 0.006315970660581125,
      "grad_norm": 0.11887697875499725,
      "kl": 0.19895461946725845,
      "learning_rate": 3e-06,
      "loss": -0.0025,
      "step": 2275
    },
    {
      "clip_ratio": 0.0019091092544840649,
      "epoch": 0.0063187469114209405,
      "grad_norm": 0.13246318697929382,
      "kl": 0.18607723712921143,
      "learning_rate": 3e-06,
      "loss": -0.003,
      "step": 2276
    },
    {
      "clip_ratio": 0.0005245065985945985,
      "epoch": 0.006321523162260757,
      "grad_norm": 0.13839560747146606,
      "kl": 0.18259260058403015,
      "learning_rate": 3e-06,
      "loss": -0.0052,
      "step": 2277
    },
    {
      "clip_ratio": 0.001069007470505312,
      "epoch": 0.006324299413100572,
      "grad_norm": 0.126378133893013,
      "kl": 0.17721717059612274,
      "learning_rate": 3e-06,
      "loss": -0.0051,
      "step": 2278
    },
    {
      "clip_ratio": 0.0020173885859549046,
      "epoch": 0.0063270756639403885,
      "grad_norm": 0.13504590094089508,
      "kl": 0.16881190985441208,
      "learning_rate": 3e-06,
      "loss": -0.0043,
      "step": 2279
    },
    {
      "clip_ratio": 0.000982351542916149,
      "epoch": 0.006329851914780204,
      "grad_norm": 0.13827833533287048,
      "kl": 0.16988541185855865,
      "learning_rate": 3e-06,
      "loss": -0.0052,
      "step": 2280
    },
    {
      "clip_ratio": 9.057971328729764e-05,
      "completion_length": 228.64583587646484,
      "epoch": 0.00633262816562002,
      "grad_norm": 0.12721112370491028,
      "kl": 0.1614338904619217,
      "learning_rate": 3e-06,
      "loss": 0.053,
      "reward": 0.36250002682209015,
      "reward_std": 0.33128294348716736,
      "rewards/countdown_reward_func": 0.36250001192092896,
      "step": 2281,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.0003709889715537429,
      "epoch": 0.006335404416459836,
      "grad_norm": 0.20156148076057434,
      "kl": 0.16206463426351547,
      "learning_rate": 3e-06,
      "loss": 0.0541,
      "step": 2282
    },
    {
      "clip_ratio": 0.00019021370098926127,
      "epoch": 0.006338180667299652,
      "grad_norm": 0.16486519575119019,
      "kl": 0.15344135463237762,
      "learning_rate": 3e-06,
      "loss": 0.0543,
      "step": 2283
    },
    {
      "clip_ratio": 0.0008641848689876497,
      "epoch": 0.006340956918139468,
      "grad_norm": 0.16734211146831512,
      "kl": 0.15662110596895218,
      "learning_rate": 3e-06,
      "loss": 0.0541,
      "step": 2284
    },
    {
      "clip_ratio": 0.00026667342899600044,
      "epoch": 0.0063437331689792835,
      "grad_norm": 0.14003409445285797,
      "kl": 0.1625654399394989,
      "learning_rate": 3e-06,
      "loss": 0.0546,
      "step": 2285
    },
    {
      "clip_ratio": 8.239947055699304e-05,
      "epoch": 0.0063465094198191,
      "grad_norm": 0.14652083814144135,
      "kl": 0.15703174471855164,
      "learning_rate": 3e-06,
      "loss": 0.0539,
      "step": 2286
    },
    {
      "clip_ratio": 8.239947055699304e-05,
      "epoch": 0.006349285670658915,
      "grad_norm": 0.14092198014259338,
      "kl": 0.156203031539917,
      "learning_rate": 3e-06,
      "loss": 0.0527,
      "step": 2287
    },
    {
      "clip_ratio": 0.00041524558037053794,
      "epoch": 0.0063520619214987315,
      "grad_norm": 0.19670848548412323,
      "kl": 0.16062773764133453,
      "learning_rate": 3e-06,
      "loss": 0.0533,
      "step": 2288
    },
    {
      "clip_ratio": 0.0005093724466860294,
      "epoch": 0.006354838172338547,
      "grad_norm": 0.15894927084445953,
      "kl": 0.15653447806835175,
      "learning_rate": 3e-06,
      "loss": 0.0521,
      "step": 2289
    },
    {
      "clip_ratio": 0.0009049937652889639,
      "epoch": 0.006357614423178363,
      "grad_norm": 0.16536635160446167,
      "kl": 0.1637991964817047,
      "learning_rate": 3e-06,
      "loss": 0.051,
      "step": 2290
    },
    {
      "clip_ratio": 0.00036807394644711167,
      "epoch": 0.0063603906740181786,
      "grad_norm": 0.13223929703235626,
      "kl": 0.17576530575752258,
      "learning_rate": 3e-06,
      "loss": 0.052,
      "step": 2291
    },
    {
      "clip_ratio": 0.00010237510286970064,
      "epoch": 0.006363166924857995,
      "grad_norm": 0.13640287518501282,
      "kl": 0.16830559074878693,
      "learning_rate": 3e-06,
      "loss": 0.05,
      "step": 2292
    },
    {
      "clip_ratio": 0.0002590137009974569,
      "completion_length": 232.68750762939453,
      "epoch": 0.00636594317569781,
      "grad_norm": 0.0946778878569603,
      "kl": 0.18598677217960358,
      "learning_rate": 3e-06,
      "loss": 0.0195,
      "reward": 0.24583332985639572,
      "reward_std": 0.2609790861606598,
      "rewards/countdown_reward_func": 0.24583332985639572,
      "step": 2293,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0063687194265376265,
      "grad_norm": 0.0930834412574768,
      "kl": 0.18177608400583267,
      "learning_rate": 3e-06,
      "loss": 0.0184,
      "step": 2294
    },
    {
      "clip_ratio": 8.417508070124313e-05,
      "epoch": 0.006371495677377443,
      "grad_norm": 0.26509028673171997,
      "kl": 0.18570785969495773,
      "learning_rate": 3e-06,
      "loss": 0.0183,
      "step": 2295
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.006374271928217258,
      "grad_norm": 0.12472037225961685,
      "kl": 0.19241143763065338,
      "learning_rate": 3e-06,
      "loss": 0.0195,
      "step": 2296
    },
    {
      "clip_ratio": 0.0006900840235175565,
      "epoch": 0.0063770481790570745,
      "grad_norm": 0.1127580925822258,
      "kl": 0.19401460886001587,
      "learning_rate": 3e-06,
      "loss": 0.0185,
      "step": 2297
    },
    {
      "clip_ratio": 0.0001705010508885607,
      "epoch": 0.00637982442989689,
      "grad_norm": 0.1149328202009201,
      "kl": 0.20228491723537445,
      "learning_rate": 3e-06,
      "loss": 0.0181,
      "step": 2298
    },
    {
      "clip_ratio": 0.00026395946042612195,
      "epoch": 0.006382600680736706,
      "grad_norm": 0.09492156654596329,
      "kl": 0.21177734434604645,
      "learning_rate": 3e-06,
      "loss": 0.0185,
      "step": 2299
    },
    {
      "clip_ratio": 0.00018846639432013035,
      "epoch": 0.0063853769315765216,
      "grad_norm": 0.09327538311481476,
      "kl": 0.20418494194746017,
      "learning_rate": 3e-06,
      "loss": 0.0178,
      "step": 2300
    },
    {
      "clip_ratio": 0.00017548260802868754,
      "epoch": 0.006388153182416338,
      "grad_norm": 0.11833266168832779,
      "kl": 0.2049153670668602,
      "learning_rate": 3e-06,
      "loss": 0.0164,
      "step": 2301
    },
    {
      "clip_ratio": 0.00044657984835794196,
      "epoch": 0.006390929433256153,
      "grad_norm": 0.1208672747015953,
      "kl": 0.20857040584087372,
      "learning_rate": 3e-06,
      "loss": 0.018,
      "step": 2302
    },
    {
      "clip_ratio": 0.001321177463978529,
      "epoch": 0.0063937056840959695,
      "grad_norm": 0.09639300405979156,
      "kl": 0.20693976432085037,
      "learning_rate": 3e-06,
      "loss": 0.0173,
      "step": 2303
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.006396481934935785,
      "grad_norm": 0.1254264861345291,
      "kl": 0.21083715558052063,
      "learning_rate": 3e-06,
      "loss": 0.0165,
      "step": 2304
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 219.6666717529297,
      "epoch": 0.006399258185775601,
      "grad_norm": 0.20140250027179718,
      "kl": 0.22675880044698715,
      "learning_rate": 3e-06,
      "loss": 0.0569,
      "reward": 0.4375000298023224,
      "reward_std": 0.4317815750837326,
      "rewards/countdown_reward_func": 0.4375000149011612,
      "step": 2305,
      "zero_std_ratio": 0.0
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0064020344366154175,
      "grad_norm": 0.164041668176651,
      "kl": 0.22609523683786392,
      "learning_rate": 3e-06,
      "loss": 0.057,
      "step": 2306
    },
    {
      "clip_ratio": 0.0010027538519352674,
      "epoch": 0.006404810687455233,
      "grad_norm": 0.2000647336244583,
      "kl": 0.20753808319568634,
      "learning_rate": 3e-06,
      "loss": 0.0568,
      "step": 2307
    },
    {
      "clip_ratio": 0.00016891522682271898,
      "epoch": 0.006407586938295049,
      "grad_norm": 0.16357851028442383,
      "kl": 0.23235772550106049,
      "learning_rate": 3e-06,
      "loss": 0.0562,
      "step": 2308
    },
    {
      "clip_ratio": 8.941345004132017e-05,
      "epoch": 0.0064103631891348645,
      "grad_norm": 0.18721310794353485,
      "kl": 0.23869449645280838,
      "learning_rate": 3e-06,
      "loss": 0.055,
      "step": 2309
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.006413139439974681,
      "grad_norm": 0.1559915840625763,
      "kl": 0.26671281456947327,
      "learning_rate": 3e-06,
      "loss": 0.0561,
      "step": 2310
    },
    {
      "clip_ratio": 0.00017694846610538661,
      "epoch": 0.006415915690814496,
      "grad_norm": 0.15243524312973022,
      "kl": 0.25702714920043945,
      "learning_rate": 3e-06,
      "loss": 0.0538,
      "step": 2311
    },
    {
      "clip_ratio": 8.999279816634953e-05,
      "epoch": 0.0064186919416543125,
      "grad_norm": 0.1509992778301239,
      "kl": 0.26283329725265503,
      "learning_rate": 3e-06,
      "loss": 0.0525,
      "step": 2312
    },
    {
      "clip_ratio": 9.968101949198171e-05,
      "epoch": 0.006421468192494128,
      "grad_norm": 0.15100322663784027,
      "kl": 0.24412598460912704,
      "learning_rate": 3e-06,
      "loss": 0.0516,
      "step": 2313
    },
    {
      "clip_ratio": 0.000363269035005942,
      "epoch": 0.006424244443333944,
      "grad_norm": 0.1439571976661682,
      "kl": 0.2789086848497391,
      "learning_rate": 3e-06,
      "loss": 0.0529,
      "step": 2314
    },
    {
      "clip_ratio": 0.00019936203898396343,
      "epoch": 0.00642702069417376,
      "grad_norm": 0.15630877017974854,
      "kl": 0.29722483456134796,
      "learning_rate": 3e-06,
      "loss": 0.0506,
      "step": 2315
    },
    {
      "clip_ratio": 0.00027720882644644007,
      "epoch": 0.006429796945013576,
      "grad_norm": 0.15677094459533691,
      "kl": 0.32308197021484375,
      "learning_rate": 3e-06,
      "loss": 0.0526,
      "step": 2316
    },
    {
      "clip_ratio": 0.0004198152746539563,
      "completion_length": 203.25,
      "epoch": 0.006432573195853392,
      "grad_norm": 0.12152662873268127,
      "kl": 0.27852681279182434,
      "learning_rate": 3e-06,
      "loss": 0.0191,
      "reward": 0.3812500238418579,
      "reward_std": 0.37016279995441437,
      "rewards/countdown_reward_func": 0.3812499940395355,
      "step": 2317,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 0.00020843063975917175,
      "epoch": 0.0064353494466932075,
      "grad_norm": 0.139579176902771,
      "kl": 0.3006761074066162,
      "learning_rate": 3e-06,
      "loss": 0.019,
      "step": 2318
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.006438125697533024,
      "grad_norm": 0.14806227385997772,
      "kl": 0.3408869802951813,
      "learning_rate": 3e-06,
      "loss": 0.0213,
      "step": 2319
    },
    {
      "clip_ratio": 0.00039015276706777513,
      "epoch": 0.006440901948372839,
      "grad_norm": 0.17348290979862213,
      "kl": 0.34244829416275024,
      "learning_rate": 3e-06,
      "loss": 0.0215,
      "step": 2320
    },
    {
      "clip_ratio": 0.000316114688757807,
      "epoch": 0.0064436781992126555,
      "grad_norm": 0.15013259649276733,
      "kl": 0.34295883774757385,
      "learning_rate": 3e-06,
      "loss": 0.022,
      "step": 2321
    },
    {
      "clip_ratio": 0.0004764232726301998,
      "epoch": 0.006446454450052471,
      "grad_norm": 0.2142786681652069,
      "kl": 0.36009830236434937,
      "learning_rate": 3e-06,
      "loss": 0.023,
      "step": 2322
    },
    {
      "clip_ratio": 0.00029448137502186,
      "epoch": 0.006449230700892287,
      "grad_norm": 0.12087388336658478,
      "kl": 0.30946898460388184,
      "learning_rate": 3e-06,
      "loss": 0.0178,
      "step": 2323
    },
    {
      "clip_ratio": 0.00010347682109568268,
      "epoch": 0.0064520069517321026,
      "grad_norm": 0.15301726758480072,
      "kl": 0.3245188295841217,
      "learning_rate": 3e-06,
      "loss": 0.0184,
      "step": 2324
    },
    {
      "clip_ratio": 0.00010495381866348907,
      "epoch": 0.006454783202571919,
      "grad_norm": 0.15143437683582306,
      "kl": 0.3580729514360428,
      "learning_rate": 3e-06,
      "loss": 0.0202,
      "step": 2325
    },
    {
      "clip_ratio": 0.000572390272282064,
      "epoch": 0.006457559453411734,
      "grad_norm": 0.17135295271873474,
      "kl": 0.35064497590065,
      "learning_rate": 3e-06,
      "loss": 0.0181,
      "step": 2326
    },
    {
      "clip_ratio": 0.00043031001405324787,
      "epoch": 0.0064603357042515505,
      "grad_norm": 0.14325761795043945,
      "kl": 0.33868308365345,
      "learning_rate": 3e-06,
      "loss": 0.0201,
      "step": 2327
    },
    {
      "clip_ratio": 0.0015213553560897708,
      "epoch": 0.006463111955091367,
      "grad_norm": 0.158623605966568,
      "kl": 0.3448447734117508,
      "learning_rate": 3e-06,
      "loss": 0.022,
      "step": 2328
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 203.6666717529297,
      "epoch": 0.006465888205931182,
      "grad_norm": 0.17202085256576538,
      "kl": 0.2865896373987198,
      "learning_rate": 3e-06,
      "loss": 0.0367,
      "reward": 0.4541666954755783,
      "reward_std": 0.3947114050388336,
      "rewards/countdown_reward_func": 0.4541666954755783,
      "step": 2329,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 9.904913167702034e-05,
      "epoch": 0.0064686644567709985,
      "grad_norm": 0.14967620372772217,
      "kl": 0.2809770703315735,
      "learning_rate": 3e-06,
      "loss": 0.0355,
      "step": 2330
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.006471440707610814,
      "grad_norm": 0.1649092584848404,
      "kl": 0.2993633449077606,
      "learning_rate": 3e-06,
      "loss": 0.0373,
      "step": 2331
    },
    {
      "clip_ratio": 0.00019977435295004398,
      "epoch": 0.00647421695845063,
      "grad_norm": 0.1611592024564743,
      "kl": 0.2826383709907532,
      "learning_rate": 3e-06,
      "loss": 0.0346,
      "step": 2332
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0064769932092904456,
      "grad_norm": 0.13679081201553345,
      "kl": 0.28105440735816956,
      "learning_rate": 3e-06,
      "loss": 0.0359,
      "step": 2333
    },
    {
      "clip_ratio": 0.0002023549168370664,
      "epoch": 0.006479769460130262,
      "grad_norm": 0.15343226492404938,
      "kl": 0.2757682204246521,
      "learning_rate": 3e-06,
      "loss": 0.0357,
      "step": 2334
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.006482545710970077,
      "grad_norm": 0.17491856217384338,
      "kl": 0.2747037261724472,
      "learning_rate": 3e-06,
      "loss": 0.0344,
      "step": 2335
    },
    {
      "clip_ratio": 0.00014551804633811116,
      "epoch": 0.0064853219618098935,
      "grad_norm": 0.14823581278324127,
      "kl": 0.2763662040233612,
      "learning_rate": 3e-06,
      "loss": 0.0339,
      "step": 2336
    },
    {
      "clip_ratio": 9.391435014549643e-05,
      "epoch": 0.006488098212649709,
      "grad_norm": 0.15623505413532257,
      "kl": 0.30250681936740875,
      "learning_rate": 3e-06,
      "loss": 0.0355,
      "step": 2337
    },
    {
      "clip_ratio": 0.00010330578516004607,
      "epoch": 0.006490874463489525,
      "grad_norm": 0.15354587137699127,
      "kl": 0.2839023768901825,
      "learning_rate": 3e-06,
      "loss": 0.0342,
      "step": 2338
    },
    {
      "clip_ratio": 0.00020746888185385615,
      "epoch": 0.0064936507143293415,
      "grad_norm": 0.14572341740131378,
      "kl": 0.2831754833459854,
      "learning_rate": 3e-06,
      "loss": 0.0341,
      "step": 2339
    },
    {
      "clip_ratio": 9.391435014549643e-05,
      "epoch": 0.006496426965169157,
      "grad_norm": 0.1376647800207138,
      "kl": 0.28501126170158386,
      "learning_rate": 3e-06,
      "loss": 0.0341,
      "step": 2340
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 215.5416717529297,
      "epoch": 0.006499203216008973,
      "grad_norm": 0.2073156237602234,
      "kl": 0.3356934189796448,
      "learning_rate": 3e-06,
      "loss": 0.0187,
      "reward": 0.40000003576278687,
      "reward_std": 0.2904737517237663,
      "rewards/countdown_reward_func": 0.4000000059604645,
      "step": 2341,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0065019794668487885,
      "grad_norm": 0.14323925971984863,
      "kl": 0.322573721408844,
      "learning_rate": 3e-06,
      "loss": 0.0175,
      "step": 2342
    },
    {
      "clip_ratio": 0.00025773196830414236,
      "epoch": 0.006504755717688605,
      "grad_norm": 0.3021996021270752,
      "kl": 0.3119508624076843,
      "learning_rate": 3e-06,
      "loss": 0.0173,
      "step": 2343
    },
    {
      "clip_ratio": 0.0001781615719664842,
      "epoch": 0.00650753196852842,
      "grad_norm": 0.15104366838932037,
      "kl": 0.31141962110996246,
      "learning_rate": 3e-06,
      "loss": 0.0174,
      "step": 2344
    },
    {
      "clip_ratio": 0.0002609418734209612,
      "epoch": 0.0065103082193682365,
      "grad_norm": 0.1799515038728714,
      "kl": 0.3139701336622238,
      "learning_rate": 3e-06,
      "loss": 0.0174,
      "step": 2345
    },
    {
      "clip_ratio": 0.0003696906060213223,
      "epoch": 0.006513084470208052,
      "grad_norm": 0.15009325742721558,
      "kl": 0.29629264771938324,
      "learning_rate": 3e-06,
      "loss": 0.0184,
      "step": 2346
    },
    {
      "clip_ratio": 0.00026359809999121353,
      "epoch": 0.006515860721047868,
      "grad_norm": 0.20159399509429932,
      "kl": 0.30615514516830444,
      "learning_rate": 3e-06,
      "loss": 0.016,
      "step": 2347
    },
    {
      "clip_ratio": 8.591065125074238e-05,
      "epoch": 0.006518636971887684,
      "grad_norm": 0.15064962208271027,
      "kl": 0.28726859390735626,
      "learning_rate": 3e-06,
      "loss": 0.0158,
      "step": 2348
    },
    {
      "clip_ratio": 0.00031818235584069043,
      "epoch": 0.0065214132227275,
      "grad_norm": 0.15885993838310242,
      "kl": 0.2687215358018875,
      "learning_rate": 3e-06,
      "loss": 0.0144,
      "step": 2349
    },
    {
      "clip_ratio": 0.0012777996016666293,
      "epoch": 0.006524189473567316,
      "grad_norm": 0.15064121782779694,
      "kl": 0.26052138954401016,
      "learning_rate": 3e-06,
      "loss": 0.0151,
      "step": 2350
    },
    {
      "clip_ratio": 0.000863487075548619,
      "epoch": 0.0065269657244071315,
      "grad_norm": 0.17315149307250977,
      "kl": 0.2593861371278763,
      "learning_rate": 3e-06,
      "loss": 0.0123,
      "step": 2351
    },
    {
      "clip_ratio": 0.001552396803162992,
      "epoch": 0.006529741975246948,
      "grad_norm": 0.13436666131019592,
      "kl": 0.24121838808059692,
      "learning_rate": 3e-06,
      "loss": 0.0142,
      "step": 2352
    },
    {
      "clip_ratio": 0.00013102724915370345,
      "completion_length": 208.56250762939453,
      "epoch": 0.006532518226086763,
      "grad_norm": 0.16845515370368958,
      "kl": 0.2573952376842499,
      "learning_rate": 3e-06,
      "loss": 0.0076,
      "reward": 0.2854166701436043,
      "reward_std": 0.24690960347652435,
      "rewards/countdown_reward_func": 0.2854166701436043,
      "step": 2353,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.00021652656141668558,
      "epoch": 0.0065352944769265795,
      "grad_norm": 0.37916499376296997,
      "kl": 0.2523314207792282,
      "learning_rate": 3e-06,
      "loss": 0.0068,
      "step": 2354
    },
    {
      "clip_ratio": 9.293680341215804e-05,
      "epoch": 0.006538070727766395,
      "grad_norm": 0.16434593498706818,
      "kl": 0.24893923103809357,
      "learning_rate": 3e-06,
      "loss": 0.006,
      "step": 2355
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.006540846978606211,
      "grad_norm": 0.125566303730011,
      "kl": 0.23655416816473007,
      "learning_rate": 3e-06,
      "loss": 0.0069,
      "step": 2356
    },
    {
      "clip_ratio": 0.00024826216395013034,
      "epoch": 0.0065436232294460266,
      "grad_norm": 0.130938321352005,
      "kl": 0.220244362950325,
      "learning_rate": 3e-06,
      "loss": 0.0047,
      "step": 2357
    },
    {
      "clip_ratio": 0.00016687953029759228,
      "epoch": 0.006546399480285843,
      "grad_norm": 0.18719619512557983,
      "kl": 0.2183583378791809,
      "learning_rate": 3e-06,
      "loss": 0.0042,
      "step": 2358
    },
    {
      "clip_ratio": 0.0015653396840207279,
      "epoch": 0.006549175731125658,
      "grad_norm": 0.16384494304656982,
      "kl": 0.20466335862874985,
      "learning_rate": 3e-06,
      "loss": 0.0031,
      "step": 2359
    },
    {
      "clip_ratio": 0.0020808427361771464,
      "epoch": 0.0065519519819654745,
      "grad_norm": 0.1263015866279602,
      "kl": 0.1988421380519867,
      "learning_rate": 3e-06,
      "loss": 0.0029,
      "step": 2360
    },
    {
      "clip_ratio": 0.0030670628184452653,
      "epoch": 0.006554728232805291,
      "grad_norm": 0.13618893921375275,
      "kl": 0.20048832148313522,
      "learning_rate": 3e-06,
      "loss": 0.0016,
      "step": 2361
    },
    {
      "clip_ratio": 0.004884407157078385,
      "epoch": 0.006557504483645106,
      "grad_norm": 0.12469899654388428,
      "kl": 0.18725426495075226,
      "learning_rate": 3e-06,
      "loss": 0.0037,
      "step": 2362
    },
    {
      "clip_ratio": 0.006509467493742704,
      "epoch": 0.0065602807344849225,
      "grad_norm": 0.11395581811666489,
      "kl": 0.17675061523914337,
      "learning_rate": 3e-06,
      "loss": 0.0023,
      "step": 2363
    },
    {
      "clip_ratio": 0.010824932716786861,
      "epoch": 0.006563056985324738,
      "grad_norm": 0.14886018633842468,
      "kl": 0.17395079135894775,
      "learning_rate": 3e-06,
      "loss": -0.0004,
      "step": 2364
    },
    {
      "clip_ratio": 0.00041061273077502847,
      "completion_length": 225.1041717529297,
      "epoch": 0.006565833236164554,
      "grad_norm": 0.10056737810373306,
      "kl": 0.14540190249681473,
      "learning_rate": 3e-06,
      "loss": 0.0074,
      "reward": 0.3770833760499954,
      "reward_std": 0.3362167477607727,
      "rewards/countdown_reward_func": 0.37708336114883423,
      "step": 2365,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0065686094870043696,
      "grad_norm": 0.11986743658781052,
      "kl": 0.14874933660030365,
      "learning_rate": 3e-06,
      "loss": 0.0078,
      "step": 2366
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.006571385737844186,
      "grad_norm": 0.13911956548690796,
      "kl": 0.1324518918991089,
      "learning_rate": 3e-06,
      "loss": 0.0065,
      "step": 2367
    },
    {
      "clip_ratio": 0.00030378723749890924,
      "epoch": 0.006574161988684001,
      "grad_norm": 0.12894724309444427,
      "kl": 0.1367444545030594,
      "learning_rate": 3e-06,
      "loss": 0.0075,
      "step": 2368
    },
    {
      "clip_ratio": 0.0003762036649277434,
      "epoch": 0.0065769382395238175,
      "grad_norm": 0.10234810411930084,
      "kl": 0.1384653076529503,
      "learning_rate": 3e-06,
      "loss": 0.0064,
      "step": 2369
    },
    {
      "clip_ratio": 0.0003675257903523743,
      "epoch": 0.006579714490363633,
      "grad_norm": 0.11732344329357147,
      "kl": 0.12812578678131104,
      "learning_rate": 3e-06,
      "loss": 0.0075,
      "step": 2370
    },
    {
      "clip_ratio": 0.0004069010537932627,
      "epoch": 0.006582490741203449,
      "grad_norm": 0.09635314345359802,
      "kl": 0.1304508000612259,
      "learning_rate": 3e-06,
      "loss": 0.0069,
      "step": 2371
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0065852669920432655,
      "grad_norm": 0.11662594228982925,
      "kl": 0.13656722754240036,
      "learning_rate": 3e-06,
      "loss": 0.0065,
      "step": 2372
    },
    {
      "clip_ratio": 0.0008264775387942791,
      "epoch": 0.006588043242883081,
      "grad_norm": 0.11650729924440384,
      "kl": 0.12324058637022972,
      "learning_rate": 3e-06,
      "loss": 0.0059,
      "step": 2373
    },
    {
      "clip_ratio": 0.0005027086590416729,
      "epoch": 0.006590819493722897,
      "grad_norm": 0.12746286392211914,
      "kl": 0.12780633196234703,
      "learning_rate": 3e-06,
      "loss": 0.0055,
      "step": 2374
    },
    {
      "clip_ratio": 0.00035271809611003846,
      "epoch": 0.0065935957445627125,
      "grad_norm": 0.10422283411026001,
      "kl": 0.13124725595116615,
      "learning_rate": 3e-06,
      "loss": 0.0072,
      "step": 2375
    },
    {
      "clip_ratio": 0.0005158810163266025,
      "epoch": 0.006596371995402529,
      "grad_norm": 0.11545603722333908,
      "kl": 0.12114672735333443,
      "learning_rate": 3e-06,
      "loss": 0.0072,
      "step": 2376
    },
    {
      "clip_ratio": 0.00017926588043337688,
      "completion_length": 227.45834350585938,
      "epoch": 0.006599148246242344,
      "grad_norm": 0.09546420723199844,
      "kl": 0.1200266070663929,
      "learning_rate": 3e-06,
      "loss": 0.0108,
      "reward": 0.37708333134651184,
      "reward_std": 0.30922502279281616,
      "rewards/countdown_reward_func": 0.37708331644535065,
      "step": 2377,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0066019244970821605,
      "grad_norm": 0.11293301731348038,
      "kl": 0.11872411891818047,
      "learning_rate": 3e-06,
      "loss": 0.0105,
      "step": 2378
    },
    {
      "clip_ratio": 0.00032814600854180753,
      "epoch": 0.006604700747921976,
      "grad_norm": 0.10137040168046951,
      "kl": 0.1121567115187645,
      "learning_rate": 3e-06,
      "loss": 0.011,
      "step": 2379
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.006607476998761792,
      "grad_norm": 0.12760894000530243,
      "kl": 0.11916434392333031,
      "learning_rate": 3e-06,
      "loss": 0.0107,
      "step": 2380
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.006610253249601608,
      "grad_norm": 0.13065321743488312,
      "kl": 0.11660314351320267,
      "learning_rate": 3e-06,
      "loss": 0.0105,
      "step": 2381
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.006613029500441424,
      "grad_norm": 0.13641607761383057,
      "kl": 0.11466880142688751,
      "learning_rate": 3e-06,
      "loss": 0.0102,
      "step": 2382
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00661580575128124,
      "grad_norm": 0.09623085707426071,
      "kl": 0.11718219518661499,
      "learning_rate": 3e-06,
      "loss": 0.0105,
      "step": 2383
    },
    {
      "clip_ratio": 0.0006848488337709568,
      "epoch": 0.0066185820021210555,
      "grad_norm": 0.10395172983407974,
      "kl": 0.11643769219517708,
      "learning_rate": 3e-06,
      "loss": 0.0096,
      "step": 2384
    },
    {
      "clip_ratio": 0.0002559744389145635,
      "epoch": 0.006621358252960872,
      "grad_norm": 0.08755200356245041,
      "kl": 0.10955745726823807,
      "learning_rate": 3e-06,
      "loss": 0.0094,
      "step": 2385
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.006624134503800687,
      "grad_norm": 0.09580729156732559,
      "kl": 0.11723517999053001,
      "learning_rate": 3e-06,
      "loss": 0.0107,
      "step": 2386
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0066269107546405035,
      "grad_norm": 0.11413105577230453,
      "kl": 0.11416671797633171,
      "learning_rate": 3e-06,
      "loss": 0.0096,
      "step": 2387
    },
    {
      "clip_ratio": 0.00010495381866348907,
      "epoch": 0.006629687005480319,
      "grad_norm": 0.14028304815292358,
      "kl": 0.11132171005010605,
      "learning_rate": 3e-06,
      "loss": 0.0099,
      "step": 2388
    },
    {
      "clip_ratio": 0.0007585418788949028,
      "completion_length": 243.0,
      "epoch": 0.006632463256320135,
      "grad_norm": 0.12180110812187195,
      "kl": 0.11260923743247986,
      "learning_rate": 3e-06,
      "loss": 0.0024,
      "reward": 0.3229166716337204,
      "reward_std": 0.3841392993927002,
      "rewards/countdown_reward_func": 0.3229166716337204,
      "step": 2389,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0066352395071599506,
      "grad_norm": 0.10579289495944977,
      "kl": 0.12057583779096603,
      "learning_rate": 3e-06,
      "loss": 0.0038,
      "step": 2390
    },
    {
      "clip_ratio": 0.00044376778532750905,
      "epoch": 0.006638015757999767,
      "grad_norm": 0.1263391077518463,
      "kl": 0.11325542628765106,
      "learning_rate": 3e-06,
      "loss": 0.0029,
      "step": 2391
    },
    {
      "clip_ratio": 0.00017295530415140092,
      "epoch": 0.006640792008839583,
      "grad_norm": 0.11693087965250015,
      "kl": 0.1094004213809967,
      "learning_rate": 3e-06,
      "loss": 0.0033,
      "step": 2392
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.0066435682596793985,
      "grad_norm": 0.09953317046165466,
      "kl": 0.11280214041471481,
      "learning_rate": 3e-06,
      "loss": 0.0032,
      "step": 2393
    },
    {
      "clip_ratio": 0.00016556291666347533,
      "epoch": 0.006646344510519215,
      "grad_norm": 0.09708958119153976,
      "kl": 0.10897836834192276,
      "learning_rate": 3e-06,
      "loss": 0.0026,
      "step": 2394
    },
    {
      "clip_ratio": 0.00018115942657459527,
      "epoch": 0.00664912076135903,
      "grad_norm": 0.12472333759069443,
      "kl": 0.10994191840291023,
      "learning_rate": 3e-06,
      "loss": 0.0021,
      "step": 2395
    },
    {
      "clip_ratio": 0.0002494906075298786,
      "epoch": 0.0066518970121988465,
      "grad_norm": 0.10897238552570343,
      "kl": 0.1151459850370884,
      "learning_rate": 3e-06,
      "loss": 0.0034,
      "step": 2396
    },
    {
      "clip_ratio": 0.0005148796408320777,
      "epoch": 0.006654673263038662,
      "grad_norm": 0.12806881964206696,
      "kl": 0.10942655429244041,
      "learning_rate": 3e-06,
      "loss": 0.0017,
      "step": 2397
    },
    {
      "clip_ratio": 0.000273729907348752,
      "epoch": 0.006657449513878478,
      "grad_norm": 0.09647449851036072,
      "kl": 0.1066247895359993,
      "learning_rate": 3e-06,
      "loss": 0.003,
      "step": 2398
    },
    {
      "clip_ratio": 8.394895849050954e-05,
      "epoch": 0.0066602257647182935,
      "grad_norm": 0.11547134071588516,
      "kl": 0.10796680673956871,
      "learning_rate": 3e-06,
      "loss": 0.002,
      "step": 2399
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "epoch": 0.00666300201555811,
      "grad_norm": 0.10167829692363739,
      "kl": 0.10324173793196678,
      "learning_rate": 3e-06,
      "loss": 0.0014,
      "step": 2400
    },
    {
      "clip_ratio": 0.001098598149837926,
      "completion_length": 229.6041717529297,
      "epoch": 0.006665778266397925,
      "grad_norm": 0.12295004725456238,
      "kl": 0.10780546069145203,
      "learning_rate": 3e-06,
      "loss": 0.017,
      "reward": 0.28333336114883423,
      "reward_std": 0.2339424267411232,
      "rewards/countdown_reward_func": 0.28333333134651184,
      "step": 2401,
      "zero_std_ratio": 0.25
    },
    {
      "clip_ratio": 0.00017353141447529197,
      "epoch": 0.0066685545172377415,
      "grad_norm": 0.08983492106199265,
      "kl": 0.10426361858844757,
      "learning_rate": 3e-06,
      "loss": 0.0157,
      "step": 2402
    },
    {
      "clip_ratio": 8.196721319109201e-05,
      "epoch": 0.006671330768077558,
      "grad_norm": 0.06798221170902252,
      "kl": 0.09816591441631317,
      "learning_rate": 3e-06,
      "loss": 0.0156,
      "step": 2403
    },
    {
      "clip_ratio": 0.00025100400671362877,
      "epoch": 0.006674107018917373,
      "grad_norm": 0.08332613110542297,
      "kl": 0.10509132966399193,
      "learning_rate": 3e-06,
      "loss": 0.0153,
      "step": 2404
    },
    {
      "clip_ratio": 0.00019771909865085036,
      "epoch": 0.0066768832697571895,
      "grad_norm": 0.08753097057342529,
      "kl": 0.1027427539229393,
      "learning_rate": 3e-06,
      "loss": 0.016,
      "step": 2405
    },
    {
      "clip_ratio": 0.0006030112272128463,
      "epoch": 0.006679659520597005,
      "grad_norm": 0.06988681852817535,
      "kl": 0.10366249829530716,
      "learning_rate": 3e-06,
      "loss": 0.0161,
      "step": 2406
    },
    {
      "clip_ratio": 0.0005557472104555927,
      "epoch": 0.006682435771436821,
      "grad_norm": 0.09204717725515366,
      "kl": 0.10658174008131027,
      "learning_rate": 3e-06,
      "loss": 0.0154,
      "step": 2407
    },
    {
      "clip_ratio": 8.366800466319546e-05,
      "epoch": 0.0066852120222766365,
      "grad_norm": 0.08877309411764145,
      "kl": 0.10358146205544472,
      "learning_rate": 3e-06,
      "loss": 0.0147,
      "step": 2408
    },
    {
      "clip_ratio": 8.366800466319546e-05,
      "epoch": 0.006687988273116453,
      "grad_norm": 0.06935451179742813,
      "kl": 0.0986652821302414,
      "learning_rate": 3e-06,
      "loss": 0.0153,
      "step": 2409
    },
    {
      "clip_ratio": 0.0003415837127249688,
      "epoch": 0.006690764523956268,
      "grad_norm": 0.0854223370552063,
      "kl": 0.10557211935520172,
      "learning_rate": 3e-06,
      "loss": 0.0144,
      "step": 2410
    },
    {
      "clip_ratio": 0.0001140510939876549,
      "epoch": 0.0066935407747960845,
      "grad_norm": 0.08558721095323563,
      "kl": 0.10372918471693993,
      "learning_rate": 3e-06,
      "loss": 0.0151,
      "step": 2411
    },
    {
      "clip_ratio": 0.0006259864894673228,
      "epoch": 0.0066963170256359,
      "grad_norm": 0.06793802231550217,
      "kl": 0.10636503249406815,
      "learning_rate": 3e-06,
      "loss": 0.0152,
      "step": 2412
    },
    {
      "clip_ratio": 8.256275032181293e-05,
      "completion_length": 230.20834350585938,
      "epoch": 0.006699093276475716,
      "grad_norm": 0.07189203053712845,
      "kl": 0.11288557946681976,
      "learning_rate": 3e-06,
      "loss": 0.0073,
      "reward": 0.1875000074505806,
      "reward_std": 0.20407745242118835,
      "rewards/countdown_reward_func": 0.1875,
      "step": 2413,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.00019181969400960952,
      "epoch": 0.0067018695273155324,
      "grad_norm": 0.07376082241535187,
      "kl": 0.10653312504291534,
      "learning_rate": 3e-06,
      "loss": 0.0075,
      "step": 2414
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.006704645778155348,
      "grad_norm": 0.08378919959068298,
      "kl": 0.11886245012283325,
      "learning_rate": 3e-06,
      "loss": 0.0077,
      "step": 2415
    },
    {
      "clip_ratio": 9.245562250725925e-05,
      "epoch": 0.006707422028995164,
      "grad_norm": 0.06871742010116577,
      "kl": 0.11657249182462692,
      "learning_rate": 3e-06,
      "loss": 0.0075,
      "step": 2416
    },
    {
      "clip_ratio": 0.0008867568030836992,
      "epoch": 0.0067101982798349795,
      "grad_norm": 0.06403800845146179,
      "kl": 0.10760099440813065,
      "learning_rate": 3e-06,
      "loss": 0.0086,
      "step": 2417
    },
    {
      "clip_ratio": 9.225092071574181e-05,
      "epoch": 0.006712974530674796,
      "grad_norm": 0.06900127232074738,
      "kl": 0.10545783117413521,
      "learning_rate": 3e-06,
      "loss": 0.0067,
      "step": 2418
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.006715750781514611,
      "grad_norm": 0.07691039890050888,
      "kl": 0.11479473859071732,
      "learning_rate": 3e-06,
      "loss": 0.0072,
      "step": 2419
    },
    {
      "clip_ratio": 0.00036602198088075966,
      "epoch": 0.0067185270323544275,
      "grad_norm": 0.07924985140562057,
      "kl": 0.1063217967748642,
      "learning_rate": 3e-06,
      "loss": 0.0075,
      "step": 2420
    },
    {
      "clip_ratio": 0.0003559212273103185,
      "epoch": 0.006721303283194243,
      "grad_norm": 0.08351549506187439,
      "kl": 0.11921686306595802,
      "learning_rate": 3e-06,
      "loss": 0.0078,
      "step": 2421
    },
    {
      "clip_ratio": 0.00027179565222468227,
      "epoch": 0.006724079534034059,
      "grad_norm": 0.06077051907777786,
      "kl": 0.1158263087272644,
      "learning_rate": 3e-06,
      "loss": 0.0068,
      "step": 2422
    },
    {
      "clip_ratio": 0.0011344332597218454,
      "epoch": 0.0067268557848738746,
      "grad_norm": 0.06291055679321289,
      "kl": 0.10745818167924881,
      "learning_rate": 3e-06,
      "loss": 0.0076,
      "step": 2423
    },
    {
      "clip_ratio": 0.0003693613543873653,
      "epoch": 0.006729632035713691,
      "grad_norm": 0.0665401816368103,
      "kl": 0.10476551577448845,
      "learning_rate": 3e-06,
      "loss": 0.0065,
      "step": 2424
    },
    {
      "clip_ratio": 8.928571332944557e-05,
      "completion_length": 215.6875,
      "epoch": 0.006732408286553507,
      "grad_norm": 0.08517291396856308,
      "kl": 0.1074470691382885,
      "learning_rate": 3e-06,
      "loss": 0.0123,
      "reward": 0.3604166805744171,
      "reward_std": 0.28518571704626083,
      "rewards/countdown_reward_func": 0.3604166656732559,
      "step": 2425,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 0.0005277531163301319,
      "epoch": 0.0067351845373933225,
      "grad_norm": 0.12994913756847382,
      "kl": 0.11232437938451767,
      "learning_rate": 3e-06,
      "loss": 0.0127,
      "step": 2426
    },
    {
      "clip_ratio": 0.00017857142665889114,
      "epoch": 0.006737960788233139,
      "grad_norm": 0.20326417684555054,
      "kl": 0.11356884241104126,
      "learning_rate": 3e-06,
      "loss": 0.0123,
      "step": 2427
    },
    {
      "clip_ratio": 0.0001188212918350473,
      "epoch": 0.006740737039072954,
      "grad_norm": 0.078920379281044,
      "kl": 0.11158164963126183,
      "learning_rate": 3e-06,
      "loss": 0.0129,
      "step": 2428
    },
    {
      "clip_ratio": 0.0001050420178216882,
      "epoch": 0.0067435132899127705,
      "grad_norm": 0.07914809137582779,
      "kl": 0.11002899706363678,
      "learning_rate": 3e-06,
      "loss": 0.0122,
      "step": 2429
    },
    {
      "clip_ratio": 0.00022386331693269312,
      "epoch": 0.006746289540752586,
      "grad_norm": 0.10386265814304352,
      "kl": 0.1144975870847702,
      "learning_rate": 3e-06,
      "loss": 0.0123,
      "step": 2430
    },
    {
      "clip_ratio": 0.00030388979939743876,
      "epoch": 0.006749065791592402,
      "grad_norm": 0.09454360604286194,
      "kl": 0.10659191012382507,
      "learning_rate": 3e-06,
      "loss": 0.0127,
      "step": 2431
    },
    {
      "clip_ratio": 0.00039420771645382047,
      "epoch": 0.0067518420424322175,
      "grad_norm": 0.10722549259662628,
      "kl": 0.11116393283009529,
      "learning_rate": 3e-06,
      "loss": 0.0121,
      "step": 2432
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.006754618293272034,
      "grad_norm": 0.1120673343539238,
      "kl": 0.11296507716178894,
      "learning_rate": 3e-06,
      "loss": 0.0123,
      "step": 2433
    },
    {
      "clip_ratio": 0.0006201282667461783,
      "epoch": 0.006757394544111849,
      "grad_norm": 0.08440835773944855,
      "kl": 0.11203001067042351,
      "learning_rate": 3e-06,
      "loss": 0.012,
      "step": 2434
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0067601707949516655,
      "grad_norm": 0.07518981397151947,
      "kl": 0.10907341167330742,
      "learning_rate": 3e-06,
      "loss": 0.012,
      "step": 2435
    },
    {
      "clip_ratio": 0.0005205198394833133,
      "epoch": 0.006762947045791482,
      "grad_norm": 0.09459592401981354,
      "kl": 0.11349227651953697,
      "learning_rate": 3e-06,
      "loss": 0.0114,
      "step": 2436
    },
    {
      "clip_ratio": 0.00026166778843617067,
      "completion_length": 230.75000762939453,
      "epoch": 0.006765723296631297,
      "grad_norm": 0.07740966230630875,
      "kl": 0.10574423149228096,
      "learning_rate": 3e-06,
      "loss": 0.0033,
      "reward": 0.2666666731238365,
      "reward_std": 0.2285463623702526,
      "rewards/countdown_reward_func": 0.2666666731238365,
      "step": 2437,
      "zero_std_ratio": 0.5
    },
    {
      "clip_ratio": 8.802816591924056e-05,
      "epoch": 0.0067684995474711135,
      "grad_norm": 0.10261306911706924,
      "kl": 0.10881078243255615,
      "learning_rate": 3e-06,
      "loss": 0.0033,
      "step": 2438
    },
    {
      "clip_ratio": 0.00017181201837956905,
      "epoch": 0.006771275798310929,
      "grad_norm": 0.09754025936126709,
      "kl": 0.11322496458888054,
      "learning_rate": 3e-06,
      "loss": 0.0025,
      "step": 2439
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.006774052049150745,
      "grad_norm": 0.07824526727199554,
      "kl": 0.10903006792068481,
      "learning_rate": 3e-06,
      "loss": 0.0033,
      "step": 2440
    },
    {
      "clip_ratio": 0.0003595175876398571,
      "epoch": 0.0067768282999905605,
      "grad_norm": 0.18730951845645905,
      "kl": 0.11356868967413902,
      "learning_rate": 3e-06,
      "loss": 0.0022,
      "step": 2441
    },
    {
      "clip_ratio": 8.85896515683271e-05,
      "epoch": 0.006779604550830377,
      "grad_norm": 0.09276639670133591,
      "kl": 0.10244229808449745,
      "learning_rate": 3e-06,
      "loss": 0.0028,
      "step": 2442
    },
    {
      "clip_ratio": 0.00026698306464822963,
      "epoch": 0.006782380801670192,
      "grad_norm": 0.07316610962152481,
      "kl": 0.10353904590010643,
      "learning_rate": 3e-06,
      "loss": 0.0031,
      "step": 2443
    },
    {
      "clip_ratio": 8.802816591924056e-05,
      "epoch": 0.0067851570525100085,
      "grad_norm": 0.09989767521619797,
      "kl": 0.10630606859922409,
      "learning_rate": 3e-06,
      "loss": 0.0023,
      "step": 2444
    },
    {
      "clip_ratio": 0.0004429482505656779,
      "epoch": 0.006787933303349824,
      "grad_norm": 0.08019357174634933,
      "kl": 0.11003177613019943,
      "learning_rate": 3e-06,
      "loss": 0.0026,
      "step": 2445
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.00679070955418964,
      "grad_norm": 0.07700924575328827,
      "kl": 0.10536712780594826,
      "learning_rate": 3e-06,
      "loss": 0.0024,
      "step": 2446
    },
    {
      "clip_ratio": 0.0005259623358142562,
      "epoch": 0.0067934858050294564,
      "grad_norm": 0.18418793380260468,
      "kl": 0.11070561036467552,
      "learning_rate": 3e-06,
      "loss": -0.0002,
      "step": 2447
    },
    {
      "clip_ratio": 0.0001726998161757365,
      "epoch": 0.006796262055869272,
      "grad_norm": 0.10063984245061874,
      "kl": 0.1003638207912445,
      "learning_rate": 3e-06,
      "loss": 0.0023,
      "step": 2448
    },
    {
      "clip_ratio": 0.00025201612152159214,
      "completion_length": 229.39583587646484,
      "epoch": 0.006799038306709088,
      "grad_norm": 0.11044564843177795,
      "kl": 0.10882113873958588,
      "learning_rate": 3e-06,
      "loss": 0.022,
      "reward": 0.4333333671092987,
      "reward_std": 0.4348773956298828,
      "rewards/countdown_reward_func": 0.4333333671092987,
      "step": 2449,
      "zero_std_ratio": 0.0
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0068018145575489035,
      "grad_norm": 0.10996294021606445,
      "kl": 0.10753875970840454,
      "learning_rate": 3e-06,
      "loss": 0.0224,
      "step": 2450
    },
    {
      "clip_ratio": 0.00017908310110215098,
      "epoch": 0.00680459080838872,
      "grad_norm": 0.13970771431922913,
      "kl": 0.10632171854376793,
      "learning_rate": 3e-06,
      "loss": 0.0232,
      "step": 2451
    },
    {
      "clip_ratio": 0.0005040322430431843,
      "epoch": 0.006807367059228535,
      "grad_norm": 0.1698501855134964,
      "kl": 0.10705317929387093,
      "learning_rate": 3e-06,
      "loss": 0.0207,
      "step": 2452
    },
    {
      "clip_ratio": 8.585165051044896e-05,
      "epoch": 0.0068101433100683515,
      "grad_norm": 0.12471974641084671,
      "kl": 0.09810657054185867,
      "learning_rate": 3e-06,
      "loss": 0.0227,
      "step": 2453
    },
    {
      "clip_ratio": 9.571210102876648e-05,
      "epoch": 0.006812919560908167,
      "grad_norm": 0.12794393301010132,
      "kl": 0.097657959908247,
      "learning_rate": 3e-06,
      "loss": 0.0206,
      "step": 2454
    },
    {
      "clip_ratio": 0.0003692590180435218,
      "epoch": 0.006815695811747983,
      "grad_norm": 0.11906078457832336,
      "kl": 0.10667447000741959,
      "learning_rate": 3e-06,
      "loss": 0.0221,
      "step": 2455
    },
    {
      "clip_ratio": 9.999999747378752e-05,
      "epoch": 0.0068184720625877986,
      "grad_norm": 0.24868349730968475,
      "kl": 0.10781405493617058,
      "learning_rate": 3e-06,
      "loss": 0.0215,
      "step": 2456
    },
    {
      "clip_ratio": 0.00026493475888855755,
      "epoch": 0.006821248313427615,
      "grad_norm": 0.10990463197231293,
      "kl": 0.10687519982457161,
      "learning_rate": 3e-06,
      "loss": 0.0217,
      "step": 2457
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.006824024564267431,
      "grad_norm": 0.16284221410751343,
      "kl": 0.10830774903297424,
      "learning_rate": 3e-06,
      "loss": 0.0189,
      "step": 2458
    },
    {
      "clip_ratio": 0.00017539320106152445,
      "epoch": 0.0068268008151072465,
      "grad_norm": 0.11818123608827591,
      "kl": 0.09956466034054756,
      "learning_rate": 3e-06,
      "loss": 0.0216,
      "step": 2459
    },
    {
      "clip_ratio": 0.0002764785604085773,
      "epoch": 0.006829577065947063,
      "grad_norm": 0.1193012222647667,
      "kl": 0.09965555369853973,
      "learning_rate": 3e-06,
      "loss": 0.0197,
      "step": 2460
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 222.64583587646484,
      "epoch": 0.006832353316786878,
      "grad_norm": 0.09964413940906525,
      "kl": 0.11183221638202667,
      "learning_rate": 3e-06,
      "loss": 0.0273,
      "reward": 0.30625002086162567,
      "reward_std": 0.28535499423742294,
      "rewards/countdown_reward_func": 0.30625002086162567,
      "step": 2461,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 9.630200656829402e-05,
      "epoch": 0.0068351295676266945,
      "grad_norm": 0.09929607808589935,
      "kl": 0.112647145986557,
      "learning_rate": 3e-06,
      "loss": 0.0274,
      "step": 2462
    },
    {
      "clip_ratio": 0.0007768171490170062,
      "epoch": 0.00683790581846651,
      "grad_norm": 0.10724590718746185,
      "kl": 0.11417991295456886,
      "learning_rate": 3e-06,
      "loss": 0.0279,
      "step": 2463
    },
    {
      "clip_ratio": 0.000346161745255813,
      "epoch": 0.006840682069306326,
      "grad_norm": 0.0905621126294136,
      "kl": 0.11805011332035065,
      "learning_rate": 3e-06,
      "loss": 0.0279,
      "step": 2464
    },
    {
      "clip_ratio": 0.000299699509923812,
      "epoch": 0.0068434583201461415,
      "grad_norm": 0.09721238166093826,
      "kl": 0.11963087320327759,
      "learning_rate": 3e-06,
      "loss": 0.028,
      "step": 2465
    },
    {
      "clip_ratio": 8.614748367108405e-05,
      "epoch": 0.006846234570985958,
      "grad_norm": 0.09523501247167587,
      "kl": 0.12303963676095009,
      "learning_rate": 3e-06,
      "loss": 0.0261,
      "step": 2466
    },
    {
      "clip_ratio": 0.0002624175394885242,
      "epoch": 0.006849010821825773,
      "grad_norm": 0.09940183162689209,
      "kl": 0.12013134360313416,
      "learning_rate": 3e-06,
      "loss": 0.0264,
      "step": 2467
    },
    {
      "clip_ratio": 0.0002837783540599048,
      "epoch": 0.0068517870726655895,
      "grad_norm": 0.09632185846567154,
      "kl": 0.12204306945204735,
      "learning_rate": 3e-06,
      "loss": 0.0259,
      "step": 2468
    },
    {
      "clip_ratio": 0.0003724969647009857,
      "epoch": 0.006854563323505406,
      "grad_norm": 0.10749493539333344,
      "kl": 0.12405981495976448,
      "learning_rate": 3e-06,
      "loss": 0.0258,
      "step": 2469
    },
    {
      "clip_ratio": 8.771930151851848e-05,
      "epoch": 0.006857339574345221,
      "grad_norm": 0.084246926009655,
      "kl": 0.12915344536304474,
      "learning_rate": 3e-06,
      "loss": 0.0264,
      "step": 2470
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0068601158251850375,
      "grad_norm": 0.0897546112537384,
      "kl": 0.131361685693264,
      "learning_rate": 3e-06,
      "loss": 0.0264,
      "step": 2471
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.006862892076024853,
      "grad_norm": 0.07698703557252884,
      "kl": 0.1356644704937935,
      "learning_rate": 3e-06,
      "loss": 0.0253,
      "step": 2472
    },
    {
      "clip_ratio": 0.0003964965872000903,
      "completion_length": 210.31250762939453,
      "epoch": 0.006865668326864669,
      "grad_norm": 0.089408740401268,
      "kl": 0.12919557839632034,
      "learning_rate": 3e-06,
      "loss": 0.0279,
      "reward": 0.3604166805744171,
      "reward_std": 0.3206951916217804,
      "rewards/countdown_reward_func": 0.3604166805744171,
      "step": 2473,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 0.0005657835863530636,
      "epoch": 0.0068684445777044845,
      "grad_norm": 0.10280496627092361,
      "kl": 0.13355571776628494,
      "learning_rate": 3e-06,
      "loss": 0.0279,
      "step": 2474
    },
    {
      "clip_ratio": 0.00030004348809598014,
      "epoch": 0.006871220828544301,
      "grad_norm": 0.09122777730226517,
      "kl": 0.13717789202928543,
      "learning_rate": 3e-06,
      "loss": 0.0281,
      "step": 2475
    },
    {
      "clip_ratio": 9.578544268151745e-05,
      "epoch": 0.006873997079384116,
      "grad_norm": 0.08694249391555786,
      "kl": 0.1368713080883026,
      "learning_rate": 3e-06,
      "loss": 0.0283,
      "step": 2476
    },
    {
      "clip_ratio": 9.912767563946545e-05,
      "epoch": 0.0068767733302239325,
      "grad_norm": 0.10039347410202026,
      "kl": 0.1407921016216278,
      "learning_rate": 3e-06,
      "loss": 0.0278,
      "step": 2477
    },
    {
      "clip_ratio": 8.45165632199496e-05,
      "epoch": 0.006879549581063748,
      "grad_norm": 0.12389730662107468,
      "kl": 0.15291880071163177,
      "learning_rate": 3e-06,
      "loss": 0.028,
      "step": 2478
    },
    {
      "clip_ratio": 0.0001690331264398992,
      "epoch": 0.006882325831903564,
      "grad_norm": 0.09755761176347733,
      "kl": 0.14044442027807236,
      "learning_rate": 3e-06,
      "loss": 0.0272,
      "step": 2479
    },
    {
      "clip_ratio": 0.00030831367621431127,
      "epoch": 0.0068851020827433804,
      "grad_norm": 0.11109394580125809,
      "kl": 0.1446225941181183,
      "learning_rate": 3e-06,
      "loss": 0.0266,
      "step": 2480
    },
    {
      "clip_ratio": 0.0003083823903580196,
      "epoch": 0.006887878333583196,
      "grad_norm": 0.10463670641183853,
      "kl": 0.14909590035676956,
      "learning_rate": 3e-06,
      "loss": 0.0263,
      "step": 2481
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.006890654584423012,
      "grad_norm": 0.09704513847827911,
      "kl": 0.14730746299028397,
      "learning_rate": 3e-06,
      "loss": 0.0267,
      "step": 2482
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0068934308352628275,
      "grad_norm": 0.09785734117031097,
      "kl": 0.15231196582317352,
      "learning_rate": 3e-06,
      "loss": 0.0252,
      "step": 2483
    },
    {
      "clip_ratio": 8.45165632199496e-05,
      "epoch": 0.006896207086102644,
      "grad_norm": 0.08950638771057129,
      "kl": 0.1649750992655754,
      "learning_rate": 3e-06,
      "loss": 0.0262,
      "step": 2484
    },
    {
      "clip_ratio": 8.138021075865254e-05,
      "completion_length": 230.9166717529297,
      "epoch": 0.006898983336942459,
      "grad_norm": 0.08120502531528473,
      "kl": 0.12933644652366638,
      "learning_rate": 3e-06,
      "loss": 0.0153,
      "reward": 0.395833358168602,
      "reward_std": 0.3388310372829437,
      "rewards/countdown_reward_func": 0.3958333507180214,
      "step": 2485,
      "zero_std_ratio": 0.125
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0069017595877822755,
      "grad_norm": 0.12228546291589737,
      "kl": 0.13283831626176834,
      "learning_rate": 3e-06,
      "loss": 0.0166,
      "step": 2486
    },
    {
      "clip_ratio": 0.00025467218802077696,
      "epoch": 0.006904535838622091,
      "grad_norm": 0.11056458950042725,
      "kl": 0.1325739100575447,
      "learning_rate": 3e-06,
      "loss": 0.0158,
      "step": 2487
    },
    {
      "clip_ratio": 0.00018615041335579008,
      "epoch": 0.006907312089461907,
      "grad_norm": 0.09804553538560867,
      "kl": 0.13264501839876175,
      "learning_rate": 3e-06,
      "loss": 0.0154,
      "step": 2488
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0069100883403017226,
      "grad_norm": 0.10097736865282059,
      "kl": 0.1376706250011921,
      "learning_rate": 3e-06,
      "loss": 0.0157,
      "step": 2489
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.006912864591141539,
      "grad_norm": 0.09442541003227234,
      "kl": 0.13518304377794266,
      "learning_rate": 3e-06,
      "loss": 0.0159,
      "step": 2490
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.006915640841981355,
      "grad_norm": 0.08362875878810883,
      "kl": 0.13955392688512802,
      "learning_rate": 3e-06,
      "loss": 0.0152,
      "step": 2491
    },
    {
      "clip_ratio": 0.0002792256127577275,
      "epoch": 0.0069184170928211705,
      "grad_norm": 0.09364964812994003,
      "kl": 0.14495066553354263,
      "learning_rate": 3e-06,
      "loss": 0.0149,
      "step": 2492
    },
    {
      "clip_ratio": 0.00016947041149251163,
      "epoch": 0.006921193343660987,
      "grad_norm": 0.0926862508058548,
      "kl": 0.144854336977005,
      "learning_rate": 3e-06,
      "loss": 0.0147,
      "step": 2493
    },
    {
      "clip_ratio": 0.0006245378099265508,
      "epoch": 0.006923969594500802,
      "grad_norm": 0.09652328491210938,
      "kl": 0.1455134153366089,
      "learning_rate": 3e-06,
      "loss": 0.0148,
      "step": 2494
    },
    {
      "clip_ratio": 0.00016276042151730508,
      "epoch": 0.0069267458453406185,
      "grad_norm": 0.09675440937280655,
      "kl": 0.14825546741485596,
      "learning_rate": 3e-06,
      "loss": 0.0149,
      "step": 2495
    },
    {
      "clip_ratio": 0.000413611029216554,
      "epoch": 0.006929522096180434,
      "grad_norm": 0.11376820504665375,
      "kl": 0.14565087854862213,
      "learning_rate": 3e-06,
      "loss": 0.0144,
      "step": 2496
    },
    {
      "clip_ratio": 0.00010024057701230049,
      "completion_length": 216.45833587646484,
      "epoch": 0.00693229834702025,
      "grad_norm": 0.11657664179801941,
      "kl": 0.16808811575174332,
      "learning_rate": 3e-06,
      "loss": 0.0155,
      "reward": 0.28541669249534607,
      "reward_std": 0.21314848214387894,
      "rewards/countdown_reward_func": 0.2854166775941849,
      "step": 2497,
      "zero_std_ratio": 0.375
    },
    {
      "clip_ratio": 9.321401739725843e-05,
      "epoch": 0.0069350745978600655,
      "grad_norm": 0.0914500430226326,
      "kl": 0.18082208931446075,
      "learning_rate": 3e-06,
      "loss": 0.0155,
      "step": 2498
    },
    {
      "clip_ratio": 0.00028280332480790094,
      "epoch": 0.006937850848699882,
      "grad_norm": 0.10090383887290955,
      "kl": 0.17869839072227478,
      "learning_rate": 3e-06,
      "loss": 0.0146,
      "step": 2499
    },
    {
      "clip_ratio": 0.00018671700672712177,
      "epoch": 0.006940627099539697,
      "grad_norm": 0.07849971204996109,
      "kl": 0.1728534698486328,
      "learning_rate": 3e-06,
      "loss": 0.0154,
      "step": 2500
    }
  ],
  "logging_steps": 1,
  "max_steps": 3601980,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 10,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 6,
  "trial_name": null,
  "trial_params": null
}