{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 33,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "debug/policy_chosen_logits": -2.4703779220581055,
      "debug/policy_chosen_logps": -179.5443115234375,
      "debug/policy_rejected_logits": -2.4724299907684326,
      "debug/policy_rejected_logps": -182.86181640625,
      "debug/reference_chosen_logps": -179.5443115234375,
      "debug/reference_rejected_logps": -182.86181640625,
      "epoch": 0.030303030303030304,
      "grad_norm": 2.750340847346066,
      "learning_rate": 1e-06,
      "logits/chosen": -2.4703779220581055,
      "logits/rejected": -2.4724299907684326,
      "logps/chosen": -179.5443115234375,
      "logps/rejected": -182.86181640625,
      "loss": 0.5,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "debug/policy_chosen_logits": -2.612560987472534,
      "debug/policy_chosen_logps": -187.98855590820312,
      "debug/policy_rejected_logits": -2.5251338481903076,
      "debug/policy_rejected_logps": -192.80223083496094,
      "debug/reference_chosen_logps": -187.92822265625,
      "debug/reference_rejected_logps": -192.76466369628906,
      "epoch": 0.06060606060606061,
      "grad_norm": 3.2222630610295133,
      "learning_rate": 1e-06,
      "logits/chosen": -2.612560987472534,
      "logits/rejected": -2.5251338481903076,
      "logps/chosen": -187.98855590820312,
      "logps/rejected": -192.80223083496094,
      "loss": 0.5,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0006031226948834956,
      "rewards/margins": -0.00022733688820153475,
      "rewards/rejected": -0.0003757858066819608,
      "step": 2
    },
    {
      "debug/policy_chosen_logits": -2.5202717781066895,
      "debug/policy_chosen_logps": -172.94644165039062,
      "debug/policy_rejected_logits": -2.446408748626709,
      "debug/policy_rejected_logps": -182.64926147460938,
      "debug/reference_chosen_logps": -172.74356079101562,
      "debug/reference_rejected_logps": -182.5146484375,
      "epoch": 0.09090909090909091,
      "grad_norm": 3.296231138476821,
      "learning_rate": 1e-06,
      "logits/chosen": -2.5202717781066895,
      "logits/rejected": -2.446408748626709,
      "logps/chosen": -172.94644165039062,
      "logps/rejected": -182.64926147460938,
      "loss": 0.4995,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.002028884831815958,
      "rewards/margins": -0.0006827162578701973,
      "rewards/rejected": -0.001346168457530439,
      "step": 3
    },
    {
      "debug/policy_chosen_logits": -2.5300447940826416,
      "debug/policy_chosen_logps": -180.87469482421875,
      "debug/policy_rejected_logits": -2.5812830924987793,
      "debug/policy_rejected_logps": -196.6584014892578,
      "debug/reference_chosen_logps": -180.65478515625,
      "debug/reference_rejected_logps": -196.28567504882812,
      "epoch": 0.12121212121212122,
      "grad_norm": 2.654852833868078,
      "learning_rate": 1e-06,
      "logits/chosen": -2.5300447940826416,
      "logits/rejected": -2.5812830924987793,
      "logps/chosen": -180.87469482421875,
      "logps/rejected": -196.6584014892578,
      "loss": 0.4987,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0021991347894072533,
      "rewards/margins": 0.0015281867235898972,
      "rewards/rejected": -0.0037273215129971504,
      "step": 4
    },
    {
      "debug/policy_chosen_logits": -2.443774700164795,
      "debug/policy_chosen_logps": -189.18003845214844,
      "debug/policy_rejected_logits": -2.4312195777893066,
      "debug/policy_rejected_logps": -172.596923828125,
      "debug/reference_chosen_logps": -188.87039184570312,
      "debug/reference_rejected_logps": -172.26124572753906,
      "epoch": 0.15151515151515152,
      "grad_norm": 2.765216242619178,
      "learning_rate": 1e-06,
      "logits/chosen": -2.443774700164795,
      "logits/rejected": -2.4312195777893066,
      "logps/chosen": -189.18003845214844,
      "logps/rejected": -172.596923828125,
      "loss": 0.4978,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0030963707249611616,
      "rewards/margins": 0.00026035308837890625,
      "rewards/rejected": -0.003356723580509424,
      "step": 5
    },
    {
      "debug/policy_chosen_logits": -2.5867741107940674,
      "debug/policy_chosen_logps": -191.50296020507812,
      "debug/policy_rejected_logits": -2.5049338340759277,
      "debug/policy_rejected_logps": -172.64727783203125,
      "debug/reference_chosen_logps": -191.35916137695312,
      "debug/reference_rejected_logps": -171.7898406982422,
      "epoch": 0.18181818181818182,
      "grad_norm": 2.98493127681766,
      "learning_rate": 1e-06,
      "logits/chosen": -2.5867741107940674,
      "logits/rejected": -2.5049338340759277,
      "logps/chosen": -191.50296020507812,
      "logps/rejected": -172.64727783203125,
      "loss": 0.4949,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0014381217770278454,
      "rewards/margins": 0.007136192638427019,
      "rewards/rejected": -0.008574314415454865,
      "step": 6
    },
    {
      "debug/policy_chosen_logits": -2.4890458583831787,
      "debug/policy_chosen_logps": -188.74664306640625,
      "debug/policy_rejected_logits": -2.503746747970581,
      "debug/policy_rejected_logps": -182.01272583007812,
      "debug/reference_chosen_logps": -188.08114624023438,
      "debug/reference_rejected_logps": -181.43170166015625,
      "epoch": 0.21212121212121213,
      "grad_norm": 2.565299522305327,
      "learning_rate": 1e-06,
      "logits/chosen": -2.4890458583831787,
      "logits/rejected": -2.503746747970581,
      "logps/chosen": -188.74664306640625,
      "logps/rejected": -182.01272583007812,
      "loss": 0.4962,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.006655101664364338,
      "rewards/margins": -0.0008448789012618363,
      "rewards/rejected": -0.005810222588479519,
      "step": 7
    },
    {
      "debug/policy_chosen_logits": -2.5022552013397217,
      "debug/policy_chosen_logps": -172.44332885742188,
      "debug/policy_rejected_logits": -2.38063907623291,
      "debug/policy_rejected_logps": -179.5426025390625,
      "debug/reference_chosen_logps": -171.3497772216797,
      "debug/reference_rejected_logps": -178.29132080078125,
      "epoch": 0.24242424242424243,
      "grad_norm": 2.683548775109707,
      "learning_rate": 1e-06,
      "logits/chosen": -2.5022552013397217,
      "logits/rejected": -2.38063907623291,
      "logps/chosen": -172.44332885742188,
      "logps/rejected": -179.5426025390625,
      "loss": 0.4946,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.010935591533780098,
      "rewards/margins": 0.001577300950884819,
      "rewards/rejected": -0.012512893415987492,
      "step": 8
    },
    {
      "debug/policy_chosen_logits": -2.495532989501953,
      "debug/policy_chosen_logps": -188.021728515625,
      "debug/policy_rejected_logits": -2.334559202194214,
      "debug/policy_rejected_logps": -184.0438690185547,
      "debug/reference_chosen_logps": -187.92709350585938,
      "debug/reference_rejected_logps": -183.24673461914062,
      "epoch": 0.2727272727272727,
      "grad_norm": 2.872420779341629,
      "learning_rate": 1e-06,
      "logits/chosen": -2.495532989501953,
      "logits/rejected": -2.334559202194214,
      "logps/chosen": -188.021728515625,
      "logps/rejected": -184.0438690185547,
      "loss": 0.4937,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0009464073227718472,
      "rewards/margins": 0.007024994120001793,
      "rewards/rejected": -0.007971401326358318,
      "step": 9
    },
    {
      "debug/policy_chosen_logits": -2.4814565181732178,
      "debug/policy_chosen_logps": -198.445556640625,
      "debug/policy_rejected_logits": -2.496356248855591,
      "debug/policy_rejected_logps": -177.85418701171875,
      "debug/reference_chosen_logps": -198.29083251953125,
      "debug/reference_rejected_logps": -175.91622924804688,
      "epoch": 0.30303030303030304,
      "grad_norm": 3.0807114531930426,
      "learning_rate": 1e-06,
      "logits/chosen": -2.4814565181732178,
      "logits/rejected": -2.496356248855591,
      "logps/chosen": -198.445556640625,
      "logps/rejected": -177.85418701171875,
      "loss": 0.4914,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0015472793020308018,
      "rewards/margins": 0.01783212646842003,
      "rewards/rejected": -0.019379405304789543,
      "step": 10
    },
    {
      "debug/policy_chosen_logits": -2.5934786796569824,
      "debug/policy_chosen_logps": -204.08248901367188,
      "debug/policy_rejected_logits": -2.4578843116760254,
      "debug/policy_rejected_logps": -186.8817138671875,
      "debug/reference_chosen_logps": -203.44688415527344,
      "debug/reference_rejected_logps": -187.01136779785156,
      "epoch": 0.3333333333333333,
      "grad_norm": 2.6552171041535537,
      "learning_rate": 1e-06,
      "logits/chosen": -2.5934786796569824,
      "logits/rejected": -2.4578843116760254,
      "logps/chosen": -204.08248901367188,
      "logps/rejected": -186.8817138671875,
      "loss": 0.4876,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.006356086581945419,
      "rewards/margins": -0.0076524922624230385,
      "rewards/rejected": 0.001296405796892941,
      "step": 11
    },
    {
      "debug/policy_chosen_logits": -2.5135629177093506,
      "debug/policy_chosen_logps": -177.83209228515625,
      "debug/policy_rejected_logits": -2.414395332336426,
      "debug/policy_rejected_logps": -174.76602172851562,
      "debug/reference_chosen_logps": -179.32247924804688,
      "debug/reference_rejected_logps": -171.77493286132812,
      "epoch": 0.36363636363636365,
      "grad_norm": 2.61493183338989,
      "learning_rate": 1e-06,
      "logits/chosen": -2.5135629177093506,
      "logits/rejected": -2.414395332336426,
      "logps/chosen": -177.83209228515625,
      "logps/rejected": -174.76602172851562,
      "loss": 0.4916,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.014903735369443893,
      "rewards/margins": 0.04481462389230728,
      "rewards/rejected": -0.029910888522863388,
      "step": 12
    },
    {
      "debug/policy_chosen_logits": -2.533503532409668,
      "debug/policy_chosen_logps": -183.77186584472656,
      "debug/policy_rejected_logits": -2.3716490268707275,
      "debug/policy_rejected_logps": -182.7346954345703,
      "debug/reference_chosen_logps": -186.3679656982422,
      "debug/reference_rejected_logps": -182.13394165039062,
      "epoch": 0.3939393939393939,
      "grad_norm": 2.664094734451926,
      "learning_rate": 1e-06,
      "logits/chosen": -2.533503532409668,
      "logits/rejected": -2.3716490268707275,
      "logps/chosen": -183.77186584472656,
      "logps/rejected": -182.7346954345703,
      "loss": 0.4855,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.025960901752114296,
      "rewards/margins": 0.03196847438812256,
      "rewards/rejected": -0.006007576361298561,
      "step": 13
    },
    {
      "debug/policy_chosen_logits": -2.594430923461914,
      "debug/policy_chosen_logps": -180.72747802734375,
      "debug/policy_rejected_logits": -2.5284759998321533,
      "debug/policy_rejected_logps": -187.50599670410156,
      "debug/reference_chosen_logps": -182.18939208984375,
      "debug/reference_rejected_logps": -188.8681182861328,
      "epoch": 0.42424242424242425,
      "grad_norm": 2.5123607464425324,
      "learning_rate": 1e-06,
      "logits/chosen": -2.594430923461914,
      "logits/rejected": -2.5284759998321533,
      "logps/chosen": -180.72747802734375,
      "logps/rejected": -187.50599670410156,
      "loss": 0.4808,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.014619007706642151,
      "rewards/margins": 0.0009978096932172775,
      "rewards/rejected": 0.013621198013424873,
      "step": 14
    },
    {
      "debug/policy_chosen_logits": -2.5420515537261963,
      "debug/policy_chosen_logps": -177.52317810058594,
      "debug/policy_rejected_logits": -2.487260580062866,
      "debug/policy_rejected_logps": -197.84640502929688,
      "debug/reference_chosen_logps": -178.9560089111328,
      "debug/reference_rejected_logps": -197.4163818359375,
      "epoch": 0.45454545454545453,
      "grad_norm": 2.680073567478649,
      "learning_rate": 1e-06,
      "logits/chosen": -2.5420515537261963,
      "logits/rejected": -2.487260580062866,
      "logps/chosen": -177.52317810058594,
      "logps/rejected": -197.84640502929688,
      "loss": 0.4794,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.014328251592814922,
      "rewards/margins": 0.018628500401973724,
      "rewards/rejected": -0.004300250671803951,
      "step": 15
    },
    {
      "debug/policy_chosen_logits": -2.5443243980407715,
      "debug/policy_chosen_logps": -175.14971923828125,
      "debug/policy_rejected_logits": -2.615178108215332,
      "debug/policy_rejected_logps": -195.92684936523438,
      "debug/reference_chosen_logps": -177.9427490234375,
      "debug/reference_rejected_logps": -192.58816528320312,
      "epoch": 0.48484848484848486,
      "grad_norm": 2.7557266784436454,
      "learning_rate": 1e-06,
      "logits/chosen": -2.5443243980407715,
      "logits/rejected": -2.615178108215332,
      "logps/chosen": -175.14971923828125,
      "logps/rejected": -195.92684936523438,
      "loss": 0.4813,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.027930183336138725,
      "rewards/margins": 0.061316829174757004,
      "rewards/rejected": -0.03338664770126343,
      "step": 16
    },
    {
      "debug/policy_chosen_logits": -2.4633026123046875,
      "debug/policy_chosen_logps": -189.36892700195312,
      "debug/policy_rejected_logits": -2.456451654434204,
      "debug/policy_rejected_logps": -171.43634033203125,
      "debug/reference_chosen_logps": -192.05772399902344,
      "debug/reference_rejected_logps": -170.10711669921875,
      "epoch": 0.5151515151515151,
      "grad_norm": 2.7126032387088785,
      "learning_rate": 1e-06,
      "logits/chosen": -2.4633026123046875,
      "logits/rejected": -2.456451654434204,
      "logps/chosen": -189.36892700195312,
      "logps/rejected": -171.43634033203125,
      "loss": 0.5013,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02688794955611229,
      "rewards/margins": 0.04018016904592514,
      "rewards/rejected": -0.013292218558490276,
      "step": 17
    },
    {
      "debug/policy_chosen_logits": -2.542309284210205,
      "debug/policy_chosen_logps": -178.5828857421875,
      "debug/policy_rejected_logits": -2.5415501594543457,
      "debug/policy_rejected_logps": -202.34744262695312,
      "debug/reference_chosen_logps": -180.57891845703125,
      "debug/reference_rejected_logps": -201.40545654296875,
      "epoch": 0.5454545454545454,
      "grad_norm": 2.604213651718002,
      "learning_rate": 1e-06,
      "logits/chosen": -2.542309284210205,
      "logits/rejected": -2.5415501594543457,
      "logps/chosen": -178.5828857421875,
      "logps/rejected": -202.34744262695312,
      "loss": 0.4857,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01996036432683468,
      "rewards/margins": 0.0293801911175251,
      "rewards/rejected": -0.009419821202754974,
      "step": 18
    },
    {
      "debug/policy_chosen_logits": -2.51920223236084,
      "debug/policy_chosen_logps": -173.57725524902344,
      "debug/policy_rejected_logits": -2.4803247451782227,
      "debug/policy_rejected_logps": -176.664306640625,
      "debug/reference_chosen_logps": -171.49952697753906,
      "debug/reference_rejected_logps": -174.46640014648438,
      "epoch": 0.5757575757575758,
      "grad_norm": 3.001719578710194,
      "learning_rate": 1e-06,
      "logits/chosen": -2.51920223236084,
      "logits/rejected": -2.4803247451782227,
      "logps/chosen": -173.57725524902344,
      "logps/rejected": -176.664306640625,
      "loss": 0.4888,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.020777244120836258,
      "rewards/margins": 0.0012017078697681427,
      "rewards/rejected": -0.02197895012795925,
      "step": 19
    },
    {
      "debug/policy_chosen_logits": -2.60622501373291,
      "debug/policy_chosen_logps": -195.92251586914062,
      "debug/policy_rejected_logits": -2.4916000366210938,
      "debug/policy_rejected_logps": -182.05645751953125,
      "debug/reference_chosen_logps": -196.1642608642578,
      "debug/reference_rejected_logps": -176.87411499023438,
      "epoch": 0.6060606060606061,
      "grad_norm": 2.8616257419732705,
      "learning_rate": 1e-06,
      "logits/chosen": -2.60622501373291,
      "logits/rejected": -2.4916000366210938,
      "logps/chosen": -195.92251586914062,
      "logps/rejected": -182.05645751953125,
      "loss": 0.4718,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.002417393261566758,
      "rewards/margins": 0.05424093082547188,
      "rewards/rejected": -0.05182354152202606,
      "step": 20
    },
    {
      "debug/policy_chosen_logits": -2.449052572250366,
      "debug/policy_chosen_logps": -171.43643188476562,
      "debug/policy_rejected_logits": -2.4381000995635986,
      "debug/policy_rejected_logps": -179.66990661621094,
      "debug/reference_chosen_logps": -172.74703979492188,
      "debug/reference_rejected_logps": -178.99798583984375,
      "epoch": 0.6363636363636364,
      "grad_norm": 2.7027165426616646,
      "learning_rate": 1e-06,
      "logits/chosen": -2.449052572250366,
      "logits/rejected": -2.4381000995635986,
      "logps/chosen": -171.43643188476562,
      "logps/rejected": -179.66990661621094,
      "loss": 0.4773,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.013106021098792553,
      "rewards/margins": 0.019825361669063568,
      "rewards/rejected": -0.006719340104609728,
      "step": 21
    },
    {
      "debug/policy_chosen_logits": -2.5421152114868164,
      "debug/policy_chosen_logps": -205.93255615234375,
      "debug/policy_rejected_logits": -2.630420446395874,
      "debug/policy_rejected_logps": -230.3187255859375,
      "debug/reference_chosen_logps": -208.52468872070312,
      "debug/reference_rejected_logps": -228.58242797851562,
      "epoch": 0.6666666666666666,
      "grad_norm": 2.7272923033258256,
      "learning_rate": 1e-06,
      "logits/chosen": -2.5421152114868164,
      "logits/rejected": -2.630420446395874,
      "logps/chosen": -205.93255615234375,
      "logps/rejected": -230.3187255859375,
      "loss": 0.4876,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02592136338353157,
      "rewards/margins": 0.0432843416929245,
      "rewards/rejected": -0.01736297458410263,
      "step": 22
    },
    {
      "debug/policy_chosen_logits": -2.6116299629211426,
      "debug/policy_chosen_logps": -183.0703125,
      "debug/policy_rejected_logits": -2.611067295074463,
      "debug/policy_rejected_logps": -210.6973876953125,
      "debug/reference_chosen_logps": -183.8590087890625,
      "debug/reference_rejected_logps": -211.12478637695312,
      "epoch": 0.696969696969697,
      "grad_norm": 2.5040109839505282,
      "learning_rate": 1e-06,
      "logits/chosen": -2.6116299629211426,
      "logits/rejected": -2.611067295074463,
      "logps/chosen": -183.0703125,
      "logps/rejected": -210.6973876953125,
      "loss": 0.4779,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.007887020707130432,
      "rewards/margins": 0.003613131120800972,
      "rewards/rejected": 0.004273890517652035,
      "step": 23
    },
    {
      "debug/policy_chosen_logits": -2.5641233921051025,
      "debug/policy_chosen_logps": -160.5648193359375,
      "debug/policy_rejected_logits": -2.555621862411499,
      "debug/policy_rejected_logps": -203.09788513183594,
      "debug/reference_chosen_logps": -165.14849853515625,
      "debug/reference_rejected_logps": -203.03428649902344,
      "epoch": 0.7272727272727273,
      "grad_norm": 2.8849926512874946,
      "learning_rate": 1e-06,
      "logits/chosen": -2.5641233921051025,
      "logits/rejected": -2.555621862411499,
      "logps/chosen": -160.5648193359375,
      "logps/rejected": -203.09788513183594,
      "loss": 0.4788,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04583675414323807,
      "rewards/margins": 0.046472661197185516,
      "rewards/rejected": -0.000635908916592598,
      "step": 24
    },
    {
      "debug/policy_chosen_logits": -2.4982876777648926,
      "debug/policy_chosen_logps": -182.13987731933594,
      "debug/policy_rejected_logits": -2.5158534049987793,
      "debug/policy_rejected_logps": -197.07666015625,
      "debug/reference_chosen_logps": -180.93638610839844,
      "debug/reference_rejected_logps": -195.06378173828125,
      "epoch": 0.7575757575757576,
      "grad_norm": 2.573419249626687,
      "learning_rate": 1e-06,
      "logits/chosen": -2.4982876777648926,
      "logits/rejected": -2.5158534049987793,
      "logps/chosen": -182.13987731933594,
      "logps/rejected": -197.07666015625,
      "loss": 0.4723,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.012034913524985313,
      "rewards/margins": 0.00809389166533947,
      "rewards/rejected": -0.020128803327679634,
      "step": 25
    },
    {
      "debug/policy_chosen_logits": -2.5644371509552,
      "debug/policy_chosen_logps": -190.55043029785156,
      "debug/policy_rejected_logits": -2.563399314880371,
      "debug/policy_rejected_logps": -203.17471313476562,
      "debug/reference_chosen_logps": -189.57899475097656,
      "debug/reference_rejected_logps": -200.677490234375,
      "epoch": 0.7878787878787878,
      "grad_norm": 2.6451303932135497,
      "learning_rate": 1e-06,
      "logits/chosen": -2.5644371509552,
      "logits/rejected": -2.563399314880371,
      "logps/chosen": -190.55043029785156,
      "logps/rejected": -203.17471313476562,
      "loss": 0.4921,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.009714392013847828,
      "rewards/margins": 0.015257682651281357,
      "rewards/rejected": -0.02497207559645176,
      "step": 26
    },
    {
      "debug/policy_chosen_logits": -2.5107343196868896,
      "debug/policy_chosen_logps": -157.84153747558594,
      "debug/policy_rejected_logits": -2.465113401412964,
      "debug/policy_rejected_logps": -193.6126708984375,
      "debug/reference_chosen_logps": -162.47134399414062,
      "debug/reference_rejected_logps": -187.2801971435547,
      "epoch": 0.8181818181818182,
      "grad_norm": 2.6477503407437504,
      "learning_rate": 1e-06,
      "logits/chosen": -2.5107343196868896,
      "logits/rejected": -2.465113401412964,
      "logps/chosen": -157.84153747558594,
      "logps/rejected": -193.6126708984375,
      "loss": 0.4575,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.04629802703857422,
      "rewards/margins": 0.10962266474962234,
      "rewards/rejected": -0.06332464516162872,
      "step": 27
    },
    {
      "debug/policy_chosen_logits": -2.6364948749542236,
      "debug/policy_chosen_logps": -170.59153747558594,
      "debug/policy_rejected_logits": -2.5919628143310547,
      "debug/policy_rejected_logps": -202.32334899902344,
      "debug/reference_chosen_logps": -173.39129638671875,
      "debug/reference_rejected_logps": -198.57765197753906,
      "epoch": 0.8484848484848485,
      "grad_norm": 3.545236381819129,
      "learning_rate": 1e-06,
      "logits/chosen": -2.6364948749542236,
      "logits/rejected": -2.5919628143310547,
      "logps/chosen": -170.59153747558594,
      "logps/rejected": -202.32334899902344,
      "loss": 0.4874,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02799762599170208,
      "rewards/margins": 0.06545457243919373,
      "rewards/rejected": -0.037456952035427094,
      "step": 28
    },
    {
      "debug/policy_chosen_logits": -2.543713331222534,
      "debug/policy_chosen_logps": -171.46937561035156,
      "debug/policy_rejected_logits": -2.5579335689544678,
      "debug/policy_rejected_logps": -209.11444091796875,
      "debug/reference_chosen_logps": -172.6361083984375,
      "debug/reference_rejected_logps": -201.7142333984375,
      "epoch": 0.8787878787878788,
      "grad_norm": 3.1668304273549825,
      "learning_rate": 1e-06,
      "logits/chosen": -2.543713331222534,
      "logits/rejected": -2.5579335689544678,
      "logps/chosen": -171.46937561035156,
      "logps/rejected": -209.11444091796875,
      "loss": 0.4725,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.011667365208268166,
      "rewards/margins": 0.08566955476999283,
      "rewards/rejected": -0.07400219142436981,
      "step": 29
    },
    {
      "debug/policy_chosen_logits": -2.5963022708892822,
      "debug/policy_chosen_logps": -185.55584716796875,
      "debug/policy_rejected_logits": -2.591168165206909,
      "debug/policy_rejected_logps": -198.93161010742188,
      "debug/reference_chosen_logps": -189.76705932617188,
      "debug/reference_rejected_logps": -198.31280517578125,
      "epoch": 0.9090909090909091,
      "grad_norm": 2.9274538153529783,
      "learning_rate": 1e-06,
      "logits/chosen": -2.5963022708892822,
      "logits/rejected": -2.591168165206909,
      "logps/chosen": -185.55584716796875,
      "logps/rejected": -198.93161010742188,
      "loss": 0.4708,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0421120822429657,
      "rewards/margins": 0.04830005392432213,
      "rewards/rejected": -0.0061879754066467285,
      "step": 30
    },
    {
      "debug/policy_chosen_logits": -2.540618419647217,
      "debug/policy_chosen_logps": -188.67941284179688,
      "debug/policy_rejected_logits": -2.491089105606079,
      "debug/policy_rejected_logps": -194.14166259765625,
      "debug/reference_chosen_logps": -196.98406982421875,
      "debug/reference_rejected_logps": -196.64881896972656,
      "epoch": 0.9393939393939394,
      "grad_norm": 2.76888166586592,
      "learning_rate": 1e-06,
      "logits/chosen": -2.540618419647217,
      "logits/rejected": -2.491089105606079,
      "logps/chosen": -188.67941284179688,
      "logps/rejected": -194.14166259765625,
      "loss": 0.474,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08304648846387863,
      "rewards/margins": 0.05797498673200607,
      "rewards/rejected": 0.025071505457162857,
      "step": 31
    },
    {
      "debug/policy_chosen_logits": -2.5790419578552246,
      "debug/policy_chosen_logps": -173.6497344970703,
      "debug/policy_rejected_logits": -2.4907901287078857,
      "debug/policy_rejected_logps": -215.8475799560547,
      "debug/reference_chosen_logps": -178.68959045410156,
      "debug/reference_rejected_logps": -208.4277801513672,
      "epoch": 0.9696969696969697,
      "grad_norm": 2.612964150864111,
      "learning_rate": 1e-06,
      "logits/chosen": -2.5790419578552246,
      "logits/rejected": -2.4907901287078857,
      "logps/chosen": -173.6497344970703,
      "logps/rejected": -215.8475799560547,
      "loss": 0.4595,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0503985770046711,
      "rewards/margins": 0.12459664791822433,
      "rewards/rejected": -0.07419807463884354,
      "step": 32
    },
    {
      "debug/policy_chosen_logits": -2.553539991378784,
      "debug/policy_chosen_logps": -181.01400756835938,
      "debug/policy_rejected_logits": -2.5499818325042725,
      "debug/policy_rejected_logps": -200.9937286376953,
      "debug/reference_chosen_logps": -185.49740600585938,
      "debug/reference_rejected_logps": -197.2001953125,
      "epoch": 1.0,
      "grad_norm": 2.5944736669447344,
      "learning_rate": 1e-06,
      "logits/chosen": -2.553539991378784,
      "logits/rejected": -2.5499818325042725,
      "logps/chosen": -181.01400756835938,
      "logps/rejected": -200.9937286376953,
      "loss": 0.429,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04483398422598839,
      "rewards/margins": 0.08276933431625366,
      "rewards/rejected": -0.03793535381555557,
      "step": 33
    },
    {
      "epoch": 1.0,
      "step": 33,
      "total_flos": 0.0,
      "train_loss": 0.48355864485104877,
      "train_runtime": 407.656,
      "train_samples_per_second": 5.078,
      "train_steps_per_second": 0.081
    }
  ],
  "logging_steps": 1,
  "max_steps": 33,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}