{
  "best_metric": null,
  "best_model_checkpoint": null,
  "episode": 17920,
  "epoch": 0.24579258507413554,
  "eval_steps": 200.0,
  "global_step": 350,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "episode": 256,
      "epoch": 0.003511322643916222,
      "eps": 6,
      "loss/policy_avg": -0.07090990990400314,
      "loss/value_avg": 0.0,
      "lr": 3e-06,
      "objective/entropy": 49.42120361328125,
      "objective/kl": 0.006465356796979904,
      "objective/non_score_reward": -0.000646535714622587,
      "objective/rlhf_reward": -1.1137903928756714,
      "objective/scores": -1.109375,
      "policy/approxkl_avg": 27.096786499023438,
      "policy/clipfrac_avg": 0.732421875,
      "policy/entropy_avg": 0.92181396484375,
      "step": 5,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 12,
      "val/ratio": 1.0399832725524902,
      "val/ratio_var": 0.010045886039733887
    },
    {
      "episode": 512,
      "epoch": 0.007022645287832444,
      "eps": 6,
      "loss/policy_avg": -0.06497187167406082,
      "loss/value_avg": 0.0,
      "lr": 2.9923273657289e-06,
      "objective/entropy": 48.286014556884766,
      "objective/kl": 0.8119473457336426,
      "objective/non_score_reward": -0.08119472861289978,
      "objective/rlhf_reward": -1.266162633895874,
      "objective/scores": -1.1875,
      "policy/approxkl_avg": 18.666072845458984,
      "policy/clipfrac_avg": 0.7314453125,
      "policy/entropy_avg": 0.912261962890625,
      "step": 10,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 16,
      "val/ratio": 1.020957112312317,
      "val/ratio_var": 0.00411860179156065
    },
    {
      "episode": 768,
      "epoch": 0.010533967931748666,
      "eps": 6,
      "loss/policy_avg": -0.0872286781668663,
      "loss/value_avg": 0.0,
      "lr": 2.9846547314578008e-06,
      "objective/entropy": 49.34376525878906,
      "objective/kl": 1.9591996669769287,
      "objective/non_score_reward": -0.1959199756383896,
      "objective/rlhf_reward": -1.2858657836914062,
      "objective/scores": -1.09375,
      "policy/approxkl_avg": 20.772502899169922,
      "policy/clipfrac_avg": 0.73828125,
      "policy/entropy_avg": 0.927978515625,
      "step": 15,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 12,
      "val/ratio": 1.0191609859466553,
      "val/ratio_var": 0.00307083735242486
    },
    {
      "episode": 1024,
      "epoch": 0.014045290575664887,
      "eps": 6,
      "loss/policy_avg": -0.07566041499376297,
      "loss/value_avg": 0.0,
      "lr": 2.9769820971867007e-06,
      "objective/entropy": 53.13662338256836,
      "objective/kl": 2.4811532497406006,
      "objective/non_score_reward": -0.24811533093452454,
      "objective/rlhf_reward": -1.2548893690109253,
      "objective/scores": -1.0078125,
      "policy/approxkl_avg": 20.665164947509766,
      "policy/clipfrac_avg": 0.7314453125,
      "policy/entropy_avg": 0.989776611328125,
      "step": 20,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 11,
      "val/ratio": 1.011010766029358,
      "val/ratio_var": 0.004201602190732956
    },
    {
      "episode": 1280,
      "epoch": 0.01755661321958111,
      "eps": 6,
      "loss/policy_avg": -0.08593496680259705,
      "loss/value_avg": 0.0,
      "lr": 2.9693094629156014e-06,
      "objective/entropy": 53.72633743286133,
      "objective/kl": 3.3111624717712402,
      "objective/non_score_reward": -0.3311161994934082,
      "objective/rlhf_reward": -1.339456558227539,
      "objective/scores": -1.0078125,
      "policy/approxkl_avg": 25.559288024902344,
      "policy/clipfrac_avg": 0.7353515625,
      "policy/entropy_avg": 0.997894287109375,
      "step": 25,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 13,
      "val/ratio": 1.0134021043777466,
      "val/ratio_var": 0.0019979747012257576
    },
    {
      "episode": 1536,
      "epoch": 0.021067935863497332,
      "eps": 6,
      "loss/policy_avg": -0.09734417498111725,
      "loss/value_avg": 0.0,
      "lr": 2.9616368286445014e-06,
      "objective/entropy": 51.259735107421875,
      "objective/kl": 5.089182376861572,
      "objective/non_score_reward": -0.5089181661605835,
      "objective/rlhf_reward": -1.2202520370483398,
      "objective/scores": -0.7109375,
      "policy/approxkl_avg": 29.841636657714844,
      "policy/clipfrac_avg": 0.736328125,
      "policy/entropy_avg": 0.960479736328125,
      "step": 30,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 26,
      "val/ratio": 1.0178756713867188,
      "val/ratio_var": 0.009866585955023766
    },
    {
      "episode": 1792,
      "epoch": 0.024579258507413555,
      "eps": 6,
      "loss/policy_avg": -0.06831618398427963,
      "loss/value_avg": 0.0,
      "lr": 2.9539641943734013e-06,
      "objective/entropy": 40.643272399902344,
      "objective/kl": 6.974010944366455,
      "objective/non_score_reward": -0.6974011063575745,
      "objective/rlhf_reward": -1.2684605121612549,
      "objective/scores": -0.5703125,
      "policy/approxkl_avg": 35.33942413330078,
      "policy/clipfrac_avg": 0.6982421875,
      "policy/entropy_avg": 0.7505035400390625,
      "step": 35,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 16,
      "val/ratio": 1.00449800491333,
      "val/ratio_var": 0.0022142010275274515
    },
    {
      "episode": 2048,
      "epoch": 0.028090581151329775,
      "eps": 6,
      "loss/policy_avg": -0.04068079590797424,
      "loss/value_avg": 0.0,
      "lr": 2.946291560102302e-06,
      "objective/entropy": 23.142562866210938,
      "objective/kl": 8.180486679077148,
      "objective/non_score_reward": -0.8180487155914307,
      "objective/rlhf_reward": -1.0729957818984985,
      "objective/scores": -0.255859375,
      "policy/approxkl_avg": 23.68307876586914,
      "policy/clipfrac_avg": 0.5859375,
      "policy/entropy_avg": 0.4361400604248047,
      "step": 40,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 8,
      "val/ratio": 1.0077030658721924,
      "val/ratio_var": 0.0024766812566667795
    },
    {
      "episode": 2304,
      "epoch": 0.031601903795246,
      "eps": 6,
      "loss/policy_avg": -0.07307010889053345,
      "loss/value_avg": 0.0,
      "lr": 2.938618925831202e-06,
      "objective/entropy": 19.376842498779297,
      "objective/kl": 8.770210266113281,
      "objective/non_score_reward": -0.8770210146903992,
      "objective/rlhf_reward": -1.0002652406692505,
      "objective/scores": -0.12353515625,
      "policy/approxkl_avg": 31.00873565673828,
      "policy/clipfrac_avg": 0.5302734375,
      "policy/entropy_avg": 0.33237457275390625,
      "step": 45,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 20,
      "val/ratio": 0.996111273765564,
      "val/ratio_var": 0.001100091845728457
    },
    {
      "episode": 2560,
      "epoch": 0.03511322643916222,
      "eps": 6,
      "loss/policy_avg": -0.04584116116166115,
      "loss/value_avg": 0.0,
      "lr": 2.9309462915601027e-06,
      "objective/entropy": 11.984097480773926,
      "objective/kl": 8.4966402053833,
      "objective/non_score_reward": -0.849664032459259,
      "objective/rlhf_reward": -0.8017911911010742,
      "objective/scores": 0.0478515625,
      "policy/approxkl_avg": 22.561037063598633,
      "policy/clipfrac_avg": 0.451171875,
      "policy/entropy_avg": 0.19393539428710938,
      "step": 50,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 20,
      "val/ratio": 0.9952375888824463,
      "val/ratio_var": 0.000761833623982966
    },
    {
      "episode": 2816,
      "epoch": 0.03862454908307844,
      "eps": 5,
      "loss/policy_avg": -0.029720915481448174,
      "loss/value_avg": 0.0,
      "lr": 2.9232736572890026e-06,
      "objective/entropy": 4.9489898681640625,
      "objective/kl": 8.733837127685547,
      "objective/non_score_reward": -0.8733837604522705,
      "objective/rlhf_reward": -0.7492713928222656,
      "objective/scores": 0.1240234375,
      "policy/approxkl_avg": 16.253189086914062,
      "policy/clipfrac_avg": 0.341796875,
      "policy/entropy_avg": 0.07728099822998047,
      "step": 55,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 18,
      "val/ratio": 0.9972053170204163,
      "val/ratio_var": 0.00032430028659291565
    },
    {
      "episode": 3072,
      "epoch": 0.042135871726994664,
      "eps": 5,
      "loss/policy_avg": -0.01298562902957201,
      "loss/value_avg": 0.0,
      "lr": 2.9156010230179026e-06,
      "objective/entropy": 1.3101667165756226,
      "objective/kl": 8.699792861938477,
      "objective/non_score_reward": -0.8699792623519897,
      "objective/rlhf_reward": -0.5752952098846436,
      "objective/scores": 0.294921875,
      "policy/approxkl_avg": 2.27925968170166,
      "policy/clipfrac_avg": 0.236328125,
      "policy/entropy_avg": 0.02513742446899414,
      "step": 60,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 20,
      "val/ratio": 1.0017118453979492,
      "val/ratio_var": 0.00016639505338389426
    },
    {
      "episode": 3328,
      "epoch": 0.04564719437091089,
      "eps": 5,
      "loss/policy_avg": -0.02618303708732128,
      "loss/value_avg": 0.0,
      "lr": 2.9079283887468033e-06,
      "objective/entropy": 2.3685269355773926,
      "objective/kl": 9.208517074584961,
      "objective/non_score_reward": -0.9208516478538513,
      "objective/rlhf_reward": -0.5182289481163025,
      "objective/scores": 0.40234375,
      "policy/approxkl_avg": 2.6189699172973633,
      "policy/clipfrac_avg": 0.310546875,
      "policy/entropy_avg": 0.04020071029663086,
      "step": 65,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 20,
      "val/ratio": 1.003983497619629,
      "val/ratio_var": 0.0009448421187698841
    },
    {
      "episode": 3584,
      "epoch": 0.04915851701482711,
      "eps": 5,
      "loss/policy_avg": -0.02327096462249756,
      "loss/value_avg": 0.0,
      "lr": 2.9002557544757032e-06,
      "objective/entropy": 2.0416018962860107,
      "objective/kl": 9.701976776123047,
      "objective/non_score_reward": -0.9701976776123047,
      "objective/rlhf_reward": -0.49486449360847473,
      "objective/scores": 0.474609375,
      "policy/approxkl_avg": 1.271956443786621,
      "policy/clipfrac_avg": 0.2734375,
      "policy/entropy_avg": 0.041253089904785156,
      "step": 70,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 16,
      "val/ratio": 1.0039558410644531,
      "val/ratio_var": 0.00041477559716440737
    },
    {
      "episode": 3840,
      "epoch": 0.052669839658743334,
      "eps": 5,
      "loss/policy_avg": -0.033096276223659515,
      "loss/value_avg": 0.0,
      "lr": 2.892583120204604e-06,
      "objective/entropy": 2.7795495986938477,
      "objective/kl": 10.028523445129395,
      "objective/non_score_reward": -1.0028523206710815,
      "objective/rlhf_reward": -0.46555712819099426,
      "objective/scores": 0.5390625,
      "policy/approxkl_avg": 3.055203676223755,
      "policy/clipfrac_avg": 0.3427734375,
      "policy/entropy_avg": 0.053270816802978516,
      "step": 75,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 23,
      "val/ratio": 1.0012407302856445,
      "val/ratio_var": 0.00011274257121840492
    },
    {
      "episode": 4096,
      "epoch": 0.05618116230265955,
      "eps": 5,
      "loss/policy_avg": -0.01961323618888855,
      "loss/value_avg": 0.0,
      "lr": 2.884910485933504e-06,
      "objective/entropy": 2.5525641441345215,
      "objective/kl": 10.111019134521484,
      "objective/non_score_reward": -1.0111019611358643,
      "objective/rlhf_reward": -0.510233461856842,
      "objective/scores": 0.5,
      "policy/approxkl_avg": 1.331697940826416,
      "policy/clipfrac_avg": 0.2861328125,
      "policy/entropy_avg": 0.048857688903808594,
      "step": 80,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 25,
      "val/ratio": 1.011049509048462,
      "val/ratio_var": 0.004252108279615641
    },
    {
      "episode": 4352,
      "epoch": 0.05969248494657577,
      "eps": 5,
      "loss/policy_avg": -0.009127877652645111,
      "loss/value_avg": 0.0,
      "lr": 2.877237851662404e-06,
      "objective/entropy": 3.016789674758911,
      "objective/kl": 11.257818222045898,
      "objective/non_score_reward": -1.125781774520874,
      "objective/rlhf_reward": -0.4276960492134094,
      "objective/scores": 0.69921875,
      "policy/approxkl_avg": 1.4772686958312988,
      "policy/clipfrac_avg": 0.35546875,
      "policy/entropy_avg": 0.053719520568847656,
      "step": 85,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 6,
      "val/ratio": 1.0042904615402222,
      "val/ratio_var": 0.0008556774700991809
    },
    {
      "episode": 4608,
      "epoch": 0.063203807590492,
      "eps": 5,
      "loss/policy_avg": -0.025049656629562378,
      "loss/value_avg": 0.0,
      "lr": 2.8695652173913046e-06,
      "objective/entropy": 2.5907459259033203,
      "objective/kl": 10.457273483276367,
      "objective/non_score_reward": -1.0457274913787842,
      "objective/rlhf_reward": -0.3816419839859009,
      "objective/scores": 0.6640625,
      "policy/approxkl_avg": 2.3460922241210938,
      "policy/clipfrac_avg": 0.322265625,
      "policy/entropy_avg": 0.04626178741455078,
      "step": 90,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 11,
      "val/ratio": 1.0003862380981445,
      "val/ratio_var": 7.93520302977413e-05
    },
    {
      "episode": 4864,
      "epoch": 0.06671513023440821,
      "eps": 5,
      "loss/policy_avg": -0.01828361675143242,
      "loss/value_avg": 0.0,
      "lr": 2.8618925831202045e-06,
      "objective/entropy": 2.397810220718384,
      "objective/kl": 10.732559204101562,
      "objective/non_score_reward": -1.073256015777588,
      "objective/rlhf_reward": -0.35966813564300537,
      "objective/scores": 0.71484375,
      "policy/approxkl_avg": 1.1093428134918213,
      "policy/clipfrac_avg": 0.32421875,
      "policy/entropy_avg": 0.041881561279296875,
      "step": 95,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 15,
      "val/ratio": 1.0054664611816406,
      "val/ratio_var": 0.0017973663052543998
    },
    {
      "episode": 5120,
      "epoch": 0.07022645287832444,
      "eps": 5,
      "loss/policy_avg": -0.04088423401117325,
      "loss/value_avg": 0.0,
      "lr": 2.8542199488491053e-06,
      "objective/entropy": 2.343449592590332,
      "objective/kl": 11.780994415283203,
      "objective/non_score_reward": -1.1780993938446045,
      "objective/rlhf_reward": -0.4628324806690216,
      "objective/scores": 0.71484375,
      "policy/approxkl_avg": 0.894420325756073,
      "policy/clipfrac_avg": 0.46875,
      "policy/entropy_avg": 0.04486083984375,
      "step": 100,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 11,
      "val/ratio": 1.0009559392929077,
      "val/ratio_var": 4.804596756002866e-05
    },
    {
      "episode": 5376,
      "epoch": 0.07373777552224066,
      "eps": 5,
      "loss/policy_avg": -0.020697183907032013,
      "loss/value_avg": 0.0,
      "lr": 2.846547314578005e-06,
      "objective/entropy": 1.9023351669311523,
      "objective/kl": 10.29288101196289,
      "objective/non_score_reward": -1.0292882919311523,
      "objective/rlhf_reward": -0.29047834873199463,
      "objective/scores": 0.73828125,
      "policy/approxkl_avg": 0.9143690466880798,
      "policy/clipfrac_avg": 0.373046875,
      "policy/entropy_avg": 0.028568267822265625,
      "step": 105,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 10,
      "val/ratio": 1.000715732574463,
      "val/ratio_var": 4.201457340968773e-05
    },
    {
      "episode": 5632,
      "epoch": 0.07724909816615688,
      "eps": 5,
      "loss/policy_avg": -0.012633640319108963,
      "loss/value_avg": 0.0,
      "lr": 2.8388746803069055e-06,
      "objective/entropy": 1.3839142322540283,
      "objective/kl": 10.57151985168457,
      "objective/non_score_reward": -1.0571520328521729,
      "objective/rlhf_reward": -0.2935946583747864,
      "objective/scores": 0.765625,
      "policy/approxkl_avg": 0.6525547504425049,
      "policy/clipfrac_avg": 0.2646484375,
      "policy/entropy_avg": 0.0345916748046875,
      "step": 110,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 10,
      "val/ratio": 0.9999199509620667,
      "val/ratio_var": 2.6978697860613465e-05
    },
    {
      "episode": 5888,
      "epoch": 0.0807604208100731,
      "eps": 5,
      "loss/policy_avg": -0.026668714359402657,
      "loss/value_avg": 0.0,
      "lr": 2.831202046035806e-06,
      "objective/entropy": 2.17741322517395,
      "objective/kl": 11.39688491821289,
      "objective/non_score_reward": -1.139688491821289,
      "objective/rlhf_reward": -0.3027456998825073,
      "objective/scores": 0.8359375,
      "policy/approxkl_avg": 8.829752922058105,
      "policy/clipfrac_avg": 0.35546875,
      "policy/entropy_avg": 0.034277915954589844,
      "step": 115,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 8,
      "val/ratio": 1.0012441873550415,
      "val/ratio_var": 9.009366476675496e-05
    },
    {
      "episode": 6144,
      "epoch": 0.08427174345398933,
      "eps": 5,
      "loss/policy_avg": -0.011602860875427723,
      "loss/value_avg": 0.0,
      "lr": 2.823529411764706e-06,
      "objective/entropy": 1.418602466583252,
      "objective/kl": 10.246469497680664,
      "objective/non_score_reward": -1.0246469974517822,
      "objective/rlhf_reward": -0.22599510848522186,
      "objective/scores": 0.796875,
      "policy/approxkl_avg": 0.31790149211883545,
      "policy/clipfrac_avg": 0.2314453125,
      "policy/entropy_avg": 0.028847694396972656,
      "step": 120,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 9,
      "val/ratio": 1.0009679794311523,
      "val/ratio_var": 3.900106457876973e-05
    },
    {
      "episode": 6400,
      "epoch": 0.08778306609790555,
      "eps": 5,
      "loss/policy_avg": -0.0157505851238966,
      "loss/value_avg": 0.0,
      "lr": 2.8158567774936066e-06,
      "objective/entropy": 1.936393141746521,
      "objective/kl": 10.550077438354492,
      "objective/non_score_reward": -1.0550076961517334,
      "objective/rlhf_reward": -0.252943217754364,
      "objective/scores": 0.80078125,
      "policy/approxkl_avg": 6.545133113861084,
      "policy/clipfrac_avg": 0.341796875,
      "policy/entropy_avg": 0.039971351623535156,
      "step": 125,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 12,
      "val/ratio": 1.0001187324523926,
      "val/ratio_var": 0.00011527155584190041
    },
    {
      "episode": 6656,
      "epoch": 0.09129438874182177,
      "eps": 5,
      "loss/policy_avg": -0.00908716581761837,
      "loss/value_avg": 0.0,
      "lr": 2.8081841432225065e-06,
      "objective/entropy": 1.9167767763137817,
      "objective/kl": 10.831771850585938,
      "objective/non_score_reward": -1.0831772089004517,
      "objective/rlhf_reward": -0.24270595610141754,
      "objective/scores": 0.83984375,
      "policy/approxkl_avg": 13.507976531982422,
      "policy/clipfrac_avg": 0.25,
      "policy/entropy_avg": 0.034499168395996094,
      "step": 130,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 7,
      "val/ratio": 1.0004911422729492,
      "val/ratio_var": 0.00018595268193166703
    },
    {
      "episode": 6912,
      "epoch": 0.094805711385738,
      "eps": 5,
      "loss/policy_avg": -0.017197387292981148,
      "loss/value_avg": 0.0,
      "lr": 2.800511508951407e-06,
      "objective/entropy": 1.7237651348114014,
      "objective/kl": 11.095592498779297,
      "objective/non_score_reward": -1.1095592975616455,
      "objective/rlhf_reward": -0.21057555079460144,
      "objective/scores": 0.8984375,
      "policy/approxkl_avg": 2.7560040950775146,
      "policy/clipfrac_avg": 0.2841796875,
      "policy/entropy_avg": 0.032952308654785156,
      "step": 135,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 2,
      "val/ratio": 0.9994020462036133,
      "val/ratio_var": 3.074964843108319e-05
    },
    {
      "episode": 7168,
      "epoch": 0.09831703402965422,
      "eps": 5,
      "loss/policy_avg": -0.012010859325528145,
      "loss/value_avg": 0.0,
      "lr": 2.792838874680307e-06,
      "objective/entropy": 1.5862581729888916,
      "objective/kl": 10.674396514892578,
      "objective/non_score_reward": -1.0674396753311157,
      "objective/rlhf_reward": -0.14433012902736664,
      "objective/scores": 0.921875,
      "policy/approxkl_avg": 1.1186727285385132,
      "policy/clipfrac_avg": 0.2783203125,
      "policy/entropy_avg": 0.0295562744140625,
      "step": 140,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 13,
      "val/ratio": 1.0007727146148682,
      "val/ratio_var": 4.557183274300769e-05
    },
    {
      "episode": 7424,
      "epoch": 0.10182835667357044,
      "eps": 5,
      "loss/policy_avg": -0.013728385791182518,
      "loss/value_avg": 0.0,
      "lr": 2.785166240409207e-06,
      "objective/entropy": 1.5388869047164917,
      "objective/kl": 10.359582901000977,
      "objective/non_score_reward": -1.035958170890808,
      "objective/rlhf_reward": -0.14511710405349731,
      "objective/scores": 0.890625,
      "policy/approxkl_avg": 0.5204602479934692,
      "policy/clipfrac_avg": 0.283203125,
      "policy/entropy_avg": 0.028924942016601562,
      "step": 145,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 14,
      "val/ratio": 1.056097149848938,
      "val/ratio_var": 0.13372056186199188
    },
    {
      "episode": 7680,
      "epoch": 0.10533967931748667,
      "eps": 5,
      "loss/policy_avg": -0.014945434406399727,
      "loss/value_avg": 0.0,
      "lr": 2.7774936061381074e-06,
      "objective/entropy": 2.0769755840301514,
      "objective/kl": 11.147063255310059,
      "objective/non_score_reward": -1.11470627784729,
      "objective/rlhf_reward": -0.08940108120441437,
      "objective/scores": 1.0234375,
      "policy/approxkl_avg": 0.5961493253707886,
      "policy/clipfrac_avg": 0.3681640625,
      "policy/entropy_avg": 0.037804603576660156,
      "step": 150,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 13,
      "val/ratio": 1.0033739805221558,
      "val/ratio_var": 0.00030022990540601313
    },
    {
      "episode": 7936,
      "epoch": 0.10885100196140288,
      "eps": 5,
      "loss/policy_avg": -0.02276831492781639,
      "loss/value_avg": 0.0,
      "lr": 2.7698209718670078e-06,
      "objective/entropy": 2.1412830352783203,
      "objective/kl": 11.697949409484863,
      "objective/non_score_reward": -1.169795036315918,
      "objective/rlhf_reward": -0.13582009077072144,
      "objective/scores": 1.03125,
      "policy/approxkl_avg": 0.7155288457870483,
      "policy/clipfrac_avg": 0.3193359375,
      "policy/entropy_avg": 0.037835121154785156,
      "step": 155,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 13,
      "val/ratio": 1.0014090538024902,
      "val/ratio_var": 5.2470270020421594e-05
    },
    {
      "episode": 8192,
      "epoch": 0.1123623246053191,
      "eps": 5,
      "loss/policy_avg": -0.013076605275273323,
      "loss/value_avg": 0.0,
      "lr": 2.762148337595908e-06,
      "objective/entropy": 1.634714126586914,
      "objective/kl": 11.629154205322266,
      "objective/non_score_reward": -1.1629154682159424,
      "objective/rlhf_reward": -0.28488799929618835,
      "objective/scores": 0.87890625,
      "policy/approxkl_avg": 0.4181188941001892,
      "policy/clipfrac_avg": 0.3037109375,
      "policy/entropy_avg": 0.029273509979248047,
      "step": 160,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 15,
      "val/ratio": 1.0008339881896973,
      "val/ratio_var": 1.4662801731901709e-05
    },
    {
      "episode": 8448,
      "epoch": 0.11587364724923532,
      "eps": 5,
      "loss/policy_avg": -0.01651182770729065,
      "loss/value_avg": 0.0,
      "lr": 2.7544757033248085e-06,
      "objective/entropy": 1.9540742635726929,
      "objective/kl": 11.4830322265625,
      "objective/non_score_reward": -1.1483032703399658,
      "objective/rlhf_reward": -0.05983233451843262,
      "objective/scores": 1.0859375,
      "policy/approxkl_avg": 18.791297912597656,
      "policy/clipfrac_avg": 0.2880859375,
      "policy/entropy_avg": 0.03601264953613281,
      "step": 165,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 16,
      "val/ratio": 1.0220942497253418,
      "val/ratio_var": 0.02208283357322216
    },
    {
      "episode": 8704,
      "epoch": 0.11938496989315155,
      "eps": 5,
      "loss/policy_avg": -0.013821810483932495,
      "loss/value_avg": 0.0,
      "lr": 2.7468030690537084e-06,
      "objective/entropy": 1.6243339776992798,
      "objective/kl": 11.435280799865723,
      "objective/non_score_reward": -1.1435281038284302,
      "objective/rlhf_reward": -0.12443088740110397,
      "objective/scores": 1.015625,
      "policy/approxkl_avg": 0.29013216495513916,
      "policy/clipfrac_avg": 0.28125,
      "policy/entropy_avg": 0.03498649597167969,
      "step": 170,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 15,
      "val/ratio": 1.0027971267700195,
      "val/ratio_var": 0.0002298366161994636
    },
    {
      "episode": 8960,
      "epoch": 0.12289629253706777,
      "eps": 5,
      "loss/policy_avg": -0.011003649793565273,
      "loss/value_avg": 0.0,
      "lr": 2.7391304347826087e-06,
      "objective/entropy": 2.000375986099243,
      "objective/kl": 11.78514575958252,
      "objective/non_score_reward": -1.1785145998001099,
      "objective/rlhf_reward": -0.2609584331512451,
      "objective/scores": 0.91796875,
      "policy/approxkl_avg": 0.8603074550628662,
      "policy/clipfrac_avg": 0.2998046875,
      "policy/entropy_avg": 0.034775733947753906,
      "step": 175,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 19,
      "val/ratio": 1.0012288093566895,
      "val/ratio_var": 3.532394111971371e-05
    },
    {
      "episode": 9216,
      "epoch": 0.126407615180984,
      "eps": 5,
      "loss/policy_avg": -0.010885423980653286,
      "loss/value_avg": 0.0,
      "lr": 2.731457800511509e-06,
      "objective/entropy": 1.5240473747253418,
      "objective/kl": 12.420597076416016,
      "objective/non_score_reward": -1.2420598268508911,
      "objective/rlhf_reward": -0.16641265153884888,
      "objective/scores": 1.078125,
      "policy/approxkl_avg": 0.46217110753059387,
      "policy/clipfrac_avg": 0.2783203125,
      "policy/entropy_avg": 0.029424667358398438,
      "step": 180,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 20,
      "val/ratio": 1.0007582902908325,
      "val/ratio_var": 2.4759892767178826e-05
    },
    {
      "episode": 9472,
      "epoch": 0.12991893782490022,
      "eps": 5,
      "loss/policy_avg": -0.01097183395177126,
      "loss/value_avg": 0.0,
      "lr": 2.7237851662404094e-06,
      "objective/entropy": 1.6292238235473633,
      "objective/kl": 12.73173713684082,
      "objective/non_score_reward": -1.2731736898422241,
      "objective/rlhf_reward": -0.10916168242692947,
      "objective/scores": 1.1640625,
      "policy/approxkl_avg": 0.5525862574577332,
      "policy/clipfrac_avg": 0.310546875,
      "policy/entropy_avg": 0.031815528869628906,
      "step": 185,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 22,
      "val/ratio": 1.0027148723602295,
      "val/ratio_var": 0.00016600274830125272
    },
    {
      "episode": 9728,
      "epoch": 0.13343026046881643,
      "eps": 5,
      "loss/policy_avg": -0.010572239756584167,
      "loss/value_avg": 0.0,
      "lr": 2.7161125319693097e-06,
      "objective/entropy": 2.028618335723877,
      "objective/kl": 12.439943313598633,
      "objective/non_score_reward": -1.2439942359924316,
      "objective/rlhf_reward": -0.06748821586370468,
      "objective/scores": 1.171875,
      "policy/approxkl_avg": 0.4930054843425751,
      "policy/clipfrac_avg": 0.2841796875,
      "policy/entropy_avg": 0.03688812255859375,
      "step": 190,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 22,
      "val/ratio": 1.001340627670288,
      "val/ratio_var": 4.4035481550963596e-05
    },
    {
      "episode": 9984,
      "epoch": 0.13694158311273266,
      "eps": 5,
      "loss/policy_avg": -0.019254155457019806,
      "loss/value_avg": 0.0,
      "lr": 2.7084398976982097e-06,
      "objective/entropy": 2.295351266860962,
      "objective/kl": 13.32223892211914,
      "objective/non_score_reward": -1.332223892211914,
      "objective/rlhf_reward": -0.1836824268102646,
      "objective/scores": 1.1484375,
      "policy/approxkl_avg": 3.1426281929016113,
      "policy/clipfrac_avg": 0.3251953125,
      "policy/entropy_avg": 0.03939247131347656,
      "step": 195,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 17,
      "val/ratio": 1.0032271146774292,
      "val/ratio_var": 0.00019827872165478766
    },
    {
      "episode": 10240,
      "epoch": 0.14045290575664887,
      "eps": 5,
      "loss/policy_avg": -0.018122296780347824,
      "loss/value_avg": 0.0,
      "lr": 2.70076726342711e-06,
      "objective/entropy": 2.345075845718384,
      "objective/kl": 12.536066055297852,
      "objective/non_score_reward": -1.2536065578460693,
      "objective/rlhf_reward": -0.056986674666404724,
      "objective/scores": 1.1953125,
      "policy/approxkl_avg": 27.5201473236084,
      "policy/clipfrac_avg": 0.3046875,
      "policy/entropy_avg": 0.04156017303466797,
      "step": 200,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 20,
      "val/ratio": 0.9993807077407837,
      "val/ratio_var": 0.00011275127326371148
    },
    {
      "episode": 10496,
      "epoch": 0.1439642284005651,
      "eps": 5,
      "loss/policy_avg": -0.019295353442430496,
      "loss/value_avg": 0.0,
      "lr": 2.6930946291560103e-06,
      "objective/entropy": 2.091012477874756,
      "objective/kl": 12.746508598327637,
      "objective/non_score_reward": -1.2746508121490479,
      "objective/rlhf_reward": -0.09065462648868561,
      "objective/scores": 1.1875,
      "policy/approxkl_avg": 0.5554059743881226,
      "policy/clipfrac_avg": 0.2998046875,
      "policy/entropy_avg": 0.03620719909667969,
      "step": 205,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 17,
      "val/ratio": 1.001387119293213,
      "val/ratio_var": 3.4958156902575865e-05
    },
    {
      "episode": 10752,
      "epoch": 0.14747555104448132,
      "eps": 5,
      "loss/policy_avg": -0.010203800164163113,
      "loss/value_avg": 0.0,
      "lr": 2.6854219948849107e-06,
      "objective/entropy": 2.1808600425720215,
      "objective/kl": 12.404802322387695,
      "objective/non_score_reward": -1.2404803037643433,
      "objective/rlhf_reward": -0.059675075113773346,
      "objective/scores": 1.1796875,
      "policy/approxkl_avg": 0.5876989364624023,
      "policy/clipfrac_avg": 0.27734375,
      "policy/entropy_avg": 0.041385650634765625,
      "step": 210,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 14,
      "val/ratio": 1.0035128593444824,
      "val/ratio_var": 0.0004931605653837323
    },
    {
      "episode": 11008,
      "epoch": 0.15098687368839755,
      "eps": 5,
      "loss/policy_avg": -0.018955286592245102,
      "loss/value_avg": 0.0,
      "lr": 2.677749360613811e-06,
      "objective/entropy": 1.968322992324829,
      "objective/kl": 13.322561264038086,
      "objective/non_score_reward": -1.3322560787200928,
      "objective/rlhf_reward": -0.0670965313911438,
      "objective/scores": 1.265625,
      "policy/approxkl_avg": 0.39782679080963135,
      "policy/clipfrac_avg": 0.373046875,
      "policy/entropy_avg": 0.03279399871826172,
      "step": 215,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 14,
      "val/ratio": 1.0017969608306885,
      "val/ratio_var": 6.843745359219611e-05
    },
    {
      "episode": 11264,
      "epoch": 0.15449819633231376,
      "eps": 5,
      "loss/policy_avg": -0.014947709627449512,
      "loss/value_avg": 0.0,
      "lr": 2.670076726342711e-06,
      "objective/entropy": 1.7985560894012451,
      "objective/kl": 12.856376647949219,
      "objective/non_score_reward": -1.285637617111206,
      "objective/rlhf_reward": -0.03251491114497185,
      "objective/scores": 1.25,
      "policy/approxkl_avg": 0.4516296982765198,
      "policy/clipfrac_avg": 0.3671875,
      "policy/entropy_avg": 0.031859397888183594,
      "step": 220,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 13,
      "val/ratio": 1.0009992122650146,
      "val/ratio_var": 4.1194460209226236e-05
    },
    {
      "episode": 11520,
      "epoch": 0.15800951897623,
      "eps": 5,
      "loss/policy_avg": -0.019819077104330063,
      "loss/value_avg": 0.0,
      "lr": 2.6624040920716113e-06,
      "objective/entropy": 1.5284242630004883,
      "objective/kl": 14.283391952514648,
      "objective/non_score_reward": -1.4283392429351807,
      "objective/rlhf_reward": -0.014965277165174484,
      "objective/scores": 1.4140625,
      "policy/approxkl_avg": 1.5518393516540527,
      "policy/clipfrac_avg": 0.2744140625,
      "policy/entropy_avg": 0.026048660278320312,
      "step": 225,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 8,
      "val/ratio": 1.0007925033569336,
      "val/ratio_var": 3.721785105881281e-05
    },
    {
      "episode": 11776,
      "epoch": 0.1615208416201462,
      "eps": 5,
      "loss/policy_avg": -0.015632648020982742,
      "loss/value_avg": 0.0,
      "lr": 2.6547314578005116e-06,
      "objective/entropy": 1.5101430416107178,
      "objective/kl": 13.435927391052246,
      "objective/non_score_reward": -1.3435927629470825,
      "objective/rlhf_reward": 0.017792798578739166,
      "objective/scores": 1.359375,
      "policy/approxkl_avg": 0.22922199964523315,
      "policy/clipfrac_avg": 0.271484375,
      "policy/entropy_avg": 0.025536060333251953,
      "step": 230,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 11,
      "val/ratio": 1.0355898141860962,
      "val/ratio_var": 0.09009659290313721
    },
    {
      "episode": 12032,
      "epoch": 0.16503216426406245,
      "eps": 5,
      "loss/policy_avg": -0.014460040256381035,
      "loss/value_avg": 0.0,
      "lr": 2.647058823529412e-06,
      "objective/entropy": 1.412046194076538,
      "objective/kl": 13.981653213500977,
      "objective/non_score_reward": -1.3981653451919556,
      "objective/rlhf_reward": -0.19434592127799988,
      "objective/scores": 1.203125,
      "policy/approxkl_avg": 0.48358476161956787,
      "policy/clipfrac_avg": 0.287109375,
      "policy/entropy_avg": 0.027116775512695312,
      "step": 235,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 19,
      "val/ratio": 1.0002431869506836,
      "val/ratio_var": 1.317531587119447e-05
    },
    {
      "episode": 12288,
      "epoch": 0.16854348690797866,
      "eps": 5,
      "loss/policy_avg": -0.0144148338586092,
      "loss/value_avg": 0.0,
      "lr": 2.6393861892583123e-06,
      "objective/entropy": 1.5728825330734253,
      "objective/kl": 13.091099739074707,
      "objective/non_score_reward": -1.3091099262237549,
      "objective/rlhf_reward": -0.12438549101352692,
      "objective/scores": 1.1875,
      "policy/approxkl_avg": 0.5084937810897827,
      "policy/clipfrac_avg": 0.2587890625,
      "policy/entropy_avg": 0.028881072998046875,
      "step": 240,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 14,
      "val/ratio": 1.004534125328064,
      "val/ratio_var": 0.001037073670886457
    },
    {
      "episode": 12544,
      "epoch": 0.17205480955189487,
      "eps": 5,
      "loss/policy_avg": -0.02535724639892578,
      "loss/value_avg": 0.0,
      "lr": 2.6317135549872122e-06,
      "objective/entropy": 1.6895666122436523,
      "objective/kl": 13.036446571350098,
      "objective/non_score_reward": -1.3036446571350098,
      "objective/rlhf_reward": -0.08131173253059387,
      "objective/scores": 1.21875,
      "policy/approxkl_avg": 1.397173285484314,
      "policy/clipfrac_avg": 0.296875,
      "policy/entropy_avg": 0.025877952575683594,
      "step": 245,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 11,
      "val/ratio": 0.9993641972541809,
      "val/ratio_var": 3.554957584128715e-05
    },
    {
      "episode": 12800,
      "epoch": 0.1755661321958111,
      "eps": 5,
      "loss/policy_avg": -0.013989413157105446,
      "loss/value_avg": 0.0,
      "lr": 2.6240409207161126e-06,
      "objective/entropy": 1.4321318864822388,
      "objective/kl": 13.751260757446289,
      "objective/non_score_reward": -1.3751261234283447,
      "objective/rlhf_reward": 0.024946460500359535,
      "objective/scores": 1.3984375,
      "policy/approxkl_avg": 0.3265579044818878,
      "policy/clipfrac_avg": 0.3095703125,
      "policy/entropy_avg": 0.02507495880126953,
      "step": 250,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 14,
      "val/ratio": 1.0016669034957886,
      "val/ratio_var": 3.951354665332474e-05
    },
    {
      "episode": 13056,
      "epoch": 0.1790774548397273,
      "eps": 5,
      "loss/policy_avg": -0.01614242233335972,
      "loss/value_avg": 0.0,
      "lr": 2.616368286445013e-06,
      "objective/entropy": 1.2477443218231201,
      "objective/kl": 14.385757446289062,
      "objective/non_score_reward": -1.4385757446289062,
      "objective/rlhf_reward": -0.048571567982435226,
      "objective/scores": 1.390625,
      "policy/approxkl_avg": 0.38643181324005127,
      "policy/clipfrac_avg": 0.33203125,
      "policy/entropy_avg": 0.02417755126953125,
      "step": 255,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 19,
      "val/ratio": 1.0020815134048462,
      "val/ratio_var": 0.00012469623470678926
    },
    {
      "episode": 13312,
      "epoch": 0.18258877748364355,
      "eps": 5,
      "loss/policy_avg": -0.013632966205477715,
      "loss/value_avg": 0.0,
      "lr": 2.6086956521739132e-06,
      "objective/entropy": 1.5228471755981445,
      "objective/kl": 14.862211227416992,
      "objective/non_score_reward": -1.486221194267273,
      "objective/rlhf_reward": -0.07545565813779831,
      "objective/scores": 1.40625,
      "policy/approxkl_avg": 2.011383056640625,
      "policy/clipfrac_avg": 0.3359375,
      "policy/entropy_avg": 0.027433395385742188,
      "step": 260,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 17,
      "val/ratio": 0.9998003244400024,
      "val/ratio_var": 2.738163857429754e-05
    },
    {
      "episode": 13568,
      "epoch": 0.18610010012755976,
      "eps": 5,
      "loss/policy_avg": -0.020372817292809486,
      "loss/value_avg": 0.0,
      "lr": 2.6010230179028136e-06,
      "objective/entropy": 1.633180856704712,
      "objective/kl": 14.094629287719727,
      "objective/non_score_reward": -1.4094629287719727,
      "objective/rlhf_reward": -0.015713702887296677,
      "objective/scores": 1.390625,
      "policy/approxkl_avg": 0.47778478264808655,
      "policy/clipfrac_avg": 0.3701171875,
      "policy/entropy_avg": 0.02643442153930664,
      "step": 265,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 14,
      "val/ratio": 1.0079278945922852,
      "val/ratio_var": 0.0023215855471789837
    },
    {
      "episode": 13824,
      "epoch": 0.189611422771476,
      "eps": 5,
      "loss/policy_avg": -0.01413625106215477,
      "loss/value_avg": 0.0,
      "lr": 2.5933503836317135e-06,
      "objective/entropy": 1.2899070978164673,
      "objective/kl": 14.59975528717041,
      "objective/non_score_reward": -1.4599756002426147,
      "objective/rlhf_reward": -0.09675531834363937,
      "objective/scores": 1.359375,
      "policy/approxkl_avg": 0.4568091630935669,
      "policy/clipfrac_avg": 0.3076171875,
      "policy/entropy_avg": 0.02667713165283203,
      "step": 270,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 19,
      "val/ratio": 1.003631830215454,
      "val/ratio_var": 0.00041694415267556906
    },
    {
      "episode": 14080,
      "epoch": 0.1931227454153922,
      "eps": 5,
      "loss/policy_avg": -0.018304049968719482,
      "loss/value_avg": 0.0,
      "lr": 2.585677749360614e-06,
      "objective/entropy": 1.3464603424072266,
      "objective/kl": 14.915502548217773,
      "objective/non_score_reward": -1.491550326347351,
      "objective/rlhf_reward": -0.013601185753941536,
      "objective/scores": 1.4765625,
      "policy/approxkl_avg": 0.5154660940170288,
      "policy/clipfrac_avg": 0.3251953125,
      "policy/entropy_avg": 0.024927139282226562,
      "step": 275,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 20,
      "val/ratio": 1.0014865398406982,
      "val/ratio_var": 8.263294148491696e-05
    },
    {
      "episode": 14336,
      "epoch": 0.19663406805930844,
      "eps": 5,
      "loss/policy_avg": -0.009162629023194313,
      "loss/value_avg": 0.0,
      "lr": 2.578005115089514e-06,
      "objective/entropy": 1.3251242637634277,
      "objective/kl": 14.600137710571289,
      "objective/non_score_reward": -1.460013747215271,
      "objective/rlhf_reward": -0.10571230947971344,
      "objective/scores": 1.3515625,
      "policy/approxkl_avg": 0.4187917411327362,
      "policy/clipfrac_avg": 0.3046875,
      "policy/entropy_avg": 0.023657798767089844,
      "step": 280,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 16,
      "val/ratio": 1.008554458618164,
      "val/ratio_var": 0.001467635971494019
    },
    {
      "episode": 14592,
      "epoch": 0.20014539070322465,
      "eps": 5,
      "loss/policy_avg": -0.01609072834253311,
      "loss/value_avg": 0.0,
      "lr": 2.5703324808184145e-06,
      "objective/entropy": 1.3078004121780396,
      "objective/kl": 14.999523162841797,
      "objective/non_score_reward": -1.4999523162841797,
      "objective/rlhf_reward": -0.15238332748413086,
      "objective/scores": 1.3515625,
      "policy/approxkl_avg": 0.3968128561973572,
      "policy/clipfrac_avg": 0.36328125,
      "policy/entropy_avg": 0.023943424224853516,
      "step": 285,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 12,
      "val/ratio": 1.0019521713256836,
      "val/ratio_var": 8.228437945945188e-05
    },
    {
      "episode": 14848,
      "epoch": 0.2036567133471409,
      "eps": 5,
      "loss/policy_avg": -0.014186807908117771,
      "loss/value_avg": 0.0,
      "lr": 2.562659846547315e-06,
      "objective/entropy": 1.2583755254745483,
      "objective/kl": 15.623100280761719,
      "objective/non_score_reward": -1.5623100996017456,
      "objective/rlhf_reward": -0.09625323116779327,
      "objective/scores": 1.46875,
      "policy/approxkl_avg": 0.5678977370262146,
      "policy/clipfrac_avg": 0.3076171875,
      "policy/entropy_avg": 0.024990558624267578,
      "step": 290,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 13,
      "val/ratio": 1.0014848709106445,
      "val/ratio_var": 4.219371476210654e-05
    },
    {
      "episode": 15104,
      "epoch": 0.2071680359910571,
      "eps": 5,
      "loss/policy_avg": -0.013804701156914234,
      "loss/value_avg": 0.0,
      "lr": 2.5549872122762148e-06,
      "objective/entropy": 1.568720817565918,
      "objective/kl": 14.687668800354004,
      "objective/non_score_reward": -1.4687669277191162,
      "objective/rlhf_reward": -0.17009752988815308,
      "objective/scores": 1.296875,
      "policy/approxkl_avg": 0.3046334981918335,
      "policy/clipfrac_avg": 0.26953125,
      "policy/entropy_avg": 0.027862548828125,
      "step": 295,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 16,
      "val/ratio": 1.0005717277526855,
      "val/ratio_var": 1.1324932529532816e-05
    },
    {
      "episode": 15360,
      "epoch": 0.21067935863497333,
      "eps": 5,
      "loss/policy_avg": -0.018133502453565598,
      "loss/value_avg": 0.0,
      "lr": 2.547314578005115e-06,
      "objective/entropy": 1.2987349033355713,
      "objective/kl": 13.89183235168457,
      "objective/non_score_reward": -1.3891831636428833,
      "objective/rlhf_reward": -0.12500587105751038,
      "objective/scores": 1.265625,
      "policy/approxkl_avg": 0.31936001777648926,
      "policy/clipfrac_avg": 0.33984375,
      "policy/entropy_avg": 0.02469015121459961,
      "step": 300,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 18,
      "val/ratio": 1.0020601749420166,
      "val/ratio_var": 3.1293042411562055e-05
    },
    {
      "episode": 15616,
      "epoch": 0.21419068127888954,
      "eps": 5,
      "loss/policy_avg": -0.01710616797208786,
      "loss/value_avg": 0.0,
      "lr": 2.5396419437340155e-06,
      "objective/entropy": 1.4288297891616821,
      "objective/kl": 14.952780723571777,
      "objective/non_score_reward": -1.4952781200408936,
      "objective/rlhf_reward": -0.15792769193649292,
      "objective/scores": 1.3359375,
      "policy/approxkl_avg": 0.6461950540542603,
      "policy/clipfrac_avg": 0.3125,
      "policy/entropy_avg": 0.02637958526611328,
      "step": 305,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 19,
      "val/ratio": 1.0013606548309326,
      "val/ratio_var": 4.203971548122354e-05
    },
    {
      "episode": 15872,
      "epoch": 0.21770200392280575,
      "eps": 5,
      "loss/policy_avg": -0.016128187999129295,
      "loss/value_avg": 0.0,
      "lr": 2.531969309462916e-06,
      "objective/entropy": 1.3288850784301758,
      "objective/kl": 15.583921432495117,
      "objective/non_score_reward": -1.55839204788208,
      "objective/rlhf_reward": -0.07665687799453735,
      "objective/scores": 1.484375,
      "policy/approxkl_avg": 0.3284182548522949,
      "policy/clipfrac_avg": 0.328125,
      "policy/entropy_avg": 0.02404165267944336,
      "step": 310,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 16,
      "val/ratio": 1.0064442157745361,
      "val/ratio_var": 0.0015344778075814247
    },
    {
      "episode": 16128,
      "epoch": 0.221213326566722,
      "eps": 5,
      "loss/policy_avg": -0.015278931707143784,
      "loss/value_avg": 0.0,
      "lr": 2.524296675191816e-06,
      "objective/entropy": 1.3236112594604492,
      "objective/kl": 14.773448944091797,
      "objective/non_score_reward": -1.4773449897766113,
      "objective/rlhf_reward": -0.08708612620830536,
      "objective/scores": 1.390625,
      "policy/approxkl_avg": 0.2980467975139618,
      "policy/clipfrac_avg": 0.34765625,
      "policy/entropy_avg": 0.02494335174560547,
      "step": 315,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 7,
      "val/ratio": 1.0008249282836914,
      "val/ratio_var": 1.927867742779199e-05
    },
    {
      "episode": 16384,
      "epoch": 0.2247246492106382,
      "eps": 5,
      "loss/policy_avg": -0.020951703190803528,
      "loss/value_avg": 0.0,
      "lr": 2.516624040920716e-06,
      "objective/entropy": 1.2670817375183105,
      "objective/kl": 14.8348970413208,
      "objective/non_score_reward": -1.483489751815796,
      "objective/rlhf_reward": 0.03382519632577896,
      "objective/scores": 1.515625,
      "policy/approxkl_avg": 1.0973663330078125,
      "policy/clipfrac_avg": 0.361328125,
      "policy/entropy_avg": 0.02091073989868164,
      "step": 320,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 15,
      "val/ratio": 1.003138542175293,
      "val/ratio_var": 0.0001205340595333837
    },
    {
      "episode": 16640,
      "epoch": 0.22823597185455444,
      "eps": 5,
      "loss/policy_avg": -0.006676271557807922,
      "loss/value_avg": 0.0,
      "lr": 2.5089514066496164e-06,
      "objective/entropy": 1.191056728363037,
      "objective/kl": 16.46404457092285,
      "objective/non_score_reward": -1.646404504776001,
      "objective/rlhf_reward": -0.14990828931331635,
      "objective/scores": 1.5,
      "policy/approxkl_avg": 0.31475046277046204,
      "policy/clipfrac_avg": 0.2578125,
      "policy/entropy_avg": 0.021608352661132812,
      "step": 325,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 25,
      "val/ratio": 1.000575065612793,
      "val/ratio_var": 1.735856494633481e-05
    },
    {
      "episode": 16896,
      "epoch": 0.23174729449847065,
      "eps": 5,
      "loss/policy_avg": -0.02127978205680847,
      "loss/value_avg": 0.0,
      "lr": 2.5012787723785167e-06,
      "objective/entropy": 1.490570068359375,
      "objective/kl": 16.22044563293457,
      "objective/non_score_reward": -1.622044563293457,
      "objective/rlhf_reward": -0.015120631083846092,
      "objective/scores": 1.609375,
      "policy/approxkl_avg": 2.5871665477752686,
      "policy/clipfrac_avg": 0.35546875,
      "policy/entropy_avg": 0.027353286743164062,
      "step": 330,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 25,
      "val/ratio": 1.003063440322876,
      "val/ratio_var": 6.509448576252908e-05
    },
    {
      "episode": 17152,
      "epoch": 0.23525861714238688,
      "eps": 5,
      "loss/policy_avg": -0.013108542189002037,
      "loss/value_avg": 0.0,
      "lr": 2.493606138107417e-06,
      "objective/entropy": 1.2718842029571533,
      "objective/kl": 16.047882080078125,
      "objective/non_score_reward": -1.604788064956665,
      "objective/rlhf_reward": -0.12415145337581635,
      "objective/scores": 1.484375,
      "policy/approxkl_avg": 0.2758824825286865,
      "policy/clipfrac_avg": 0.287109375,
      "policy/entropy_avg": 0.024268627166748047,
      "step": 335,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 16,
      "val/ratio": 1.0036962032318115,
      "val/ratio_var": 0.0003905180492438376
    },
    {
      "episode": 17408,
      "epoch": 0.2387699397863031,
      "eps": 5,
      "loss/policy_avg": -0.014837839640676975,
      "loss/value_avg": 0.0,
      "lr": 2.4859335038363174e-06,
      "objective/entropy": 1.3406567573547363,
      "objective/kl": 16.428348541259766,
      "objective/non_score_reward": -1.642835021018982,
      "objective/rlhf_reward": 0.018702151253819466,
      "objective/scores": 1.6640625,
      "policy/approxkl_avg": 0.192110076546669,
      "policy/clipfrac_avg": 0.3388671875,
      "policy/entropy_avg": 0.025295734405517578,
      "step": 340,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 19,
      "val/ratio": 1.0008959770202637,
      "val/ratio_var": 1.205760781886056e-05
    },
    {
      "episode": 17664,
      "epoch": 0.24228126243021933,
      "eps": 5,
      "loss/policy_avg": -0.01899782381951809,
      "loss/value_avg": 0.0,
      "lr": 2.4782608695652173e-06,
      "objective/entropy": 1.5880205631256104,
      "objective/kl": 15.775943756103516,
      "objective/non_score_reward": -1.577594518661499,
      "objective/rlhf_reward": -0.08363974094390869,
      "objective/scores": 1.4921875,
      "policy/approxkl_avg": 0.9254180192947388,
      "policy/clipfrac_avg": 0.3251953125,
      "policy/entropy_avg": 0.024907588958740234,
      "step": 345,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 15,
      "val/ratio": 1.000749111175537,
      "val/ratio_var": 2.8957187168998644e-05
    },
    {
      "episode": 17920,
      "epoch": 0.24579258507413554,
      "eps": 5,
      "loss/policy_avg": -0.014669202268123627,
      "loss/value_avg": 0.0,
      "lr": 2.4705882352941177e-06,
      "objective/entropy": 1.2656543254852295,
      "objective/kl": 16.034730911254883,
      "objective/non_score_reward": -1.60347318649292,
      "objective/rlhf_reward": -0.011744961142539978,
      "objective/scores": 1.59375,
      "policy/approxkl_avg": 1.004683017730713,
      "policy/clipfrac_avg": 0.2646484375,
      "policy/entropy_avg": 0.023741722106933594,
      "step": 350,
      "val/clipfrac_avg": 0.0,
      "val/num_eos_tokens": 19,
      "val/ratio": 1.0009608268737793,
      "val/ratio_var": 4.649764014175162e-05
    }
  ],
  "logging_steps": 100,
  "max_steps": 391,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1.3716104077797742,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0,
  "train_batch_size": null,
  "trial_name": null,
  "trial_params": null
}