{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.998451213216314,
  "eval_steps": 100,
  "global_step": 1452,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 3.424657534246575e-09,
      "logits/chosen": -3.027578353881836,
      "logits/rejected": -3.0140559673309326,
      "logps/chosen": -274.68450927734375,
      "logps/rejected": -237.60464477539062,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.02,
      "learning_rate": 3.424657534246575e-08,
      "logits/chosen": -3.013417959213257,
      "logits/rejected": -3.0057966709136963,
      "logps/chosen": -295.5029602050781,
      "logps/rejected": -224.98101806640625,
      "loss": 0.6922,
      "rewards/accuracies": 0.4340277910232544,
      "rewards/chosen": -0.000507830292917788,
      "rewards/margins": -0.0004958626232109964,
      "rewards/rejected": -1.1967805221502203e-05,
      "step": 10
    },
    {
      "epoch": 0.04,
      "learning_rate": 6.84931506849315e-08,
      "logits/chosen": -3.0661492347717285,
      "logits/rejected": -3.032992362976074,
      "logps/chosen": -279.60821533203125,
      "logps/rejected": -237.44692993164062,
      "loss": 0.6747,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": 0.020145852118730545,
      "rewards/margins": 0.038568057119846344,
      "rewards/rejected": -0.0184222012758255,
      "step": 20
    },
    {
      "epoch": 0.06,
      "learning_rate": 1.0273972602739725e-07,
      "logits/chosen": -3.0230929851531982,
      "logits/rejected": -3.0147013664245605,
      "logps/chosen": -268.19683837890625,
      "logps/rejected": -226.419921875,
      "loss": 0.6379,
      "rewards/accuracies": 0.715624988079071,
      "rewards/chosen": 0.02288839966058731,
      "rewards/margins": 0.11917489767074585,
      "rewards/rejected": -0.09628649801015854,
      "step": 30
    },
    {
      "epoch": 0.08,
      "learning_rate": 1.36986301369863e-07,
      "logits/chosen": -3.0487260818481445,
      "logits/rejected": -3.0244030952453613,
      "logps/chosen": -298.54327392578125,
      "logps/rejected": -249.37460327148438,
      "loss": 0.5867,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.09271707385778427,
      "rewards/margins": 0.37581944465637207,
      "rewards/rejected": -0.2831023931503296,
      "step": 40
    },
    {
      "epoch": 0.1,
      "learning_rate": 1.7123287671232875e-07,
      "logits/chosen": -3.0150153636932373,
      "logits/rejected": -2.9876465797424316,
      "logps/chosen": -262.42645263671875,
      "logps/rejected": -225.4582977294922,
      "loss": 0.5406,
      "rewards/accuracies": 0.778124988079071,
      "rewards/chosen": 0.09807772189378738,
      "rewards/margins": 0.5801292657852173,
      "rewards/rejected": -0.4820515215396881,
      "step": 50
    },
    {
      "epoch": 0.12,
      "learning_rate": 2.054794520547945e-07,
      "logits/chosen": -3.0048201084136963,
      "logits/rejected": -2.9603819847106934,
      "logps/chosen": -291.26373291015625,
      "logps/rejected": -239.41476440429688,
      "loss": 0.5451,
      "rewards/accuracies": 0.7281249761581421,
      "rewards/chosen": -0.09130807220935822,
      "rewards/margins": 0.539149820804596,
      "rewards/rejected": -0.630457878112793,
      "step": 60
    },
    {
      "epoch": 0.14,
      "learning_rate": 2.3972602739726023e-07,
      "logits/chosen": -3.0534212589263916,
      "logits/rejected": -3.0245957374572754,
      "logps/chosen": -289.070068359375,
      "logps/rejected": -247.5080108642578,
      "loss": 0.5194,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.041085079312324524,
      "rewards/margins": 0.8055621385574341,
      "rewards/rejected": -0.7644770741462708,
      "step": 70
    },
    {
      "epoch": 0.17,
      "learning_rate": 2.73972602739726e-07,
      "logits/chosen": -3.015256404876709,
      "logits/rejected": -3.0051567554473877,
      "logps/chosen": -277.9364318847656,
      "logps/rejected": -248.8397979736328,
      "loss": 0.511,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.05333980172872543,
      "rewards/margins": 0.8779909014701843,
      "rewards/rejected": -0.9313308000564575,
      "step": 80
    },
    {
      "epoch": 0.19,
      "learning_rate": 3.0821917808219176e-07,
      "logits/chosen": -3.036608934402466,
      "logits/rejected": -3.020042896270752,
      "logps/chosen": -271.72698974609375,
      "logps/rejected": -236.72982788085938,
      "loss": 0.4956,
      "rewards/accuracies": 0.753125011920929,
      "rewards/chosen": -0.10287854820489883,
      "rewards/margins": 0.9299039840698242,
      "rewards/rejected": -1.0327825546264648,
      "step": 90
    },
    {
      "epoch": 0.21,
      "learning_rate": 3.424657534246575e-07,
      "logits/chosen": -3.065687417984009,
      "logits/rejected": -3.0368151664733887,
      "logps/chosen": -250.464599609375,
      "logps/rejected": -208.9635467529297,
      "loss": 0.5058,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.10430784523487091,
      "rewards/margins": 0.9304388165473938,
      "rewards/rejected": -1.0347466468811035,
      "step": 100
    },
    {
      "epoch": 0.21,
      "eval_logits/chosen": -3.0546915531158447,
      "eval_logits/rejected": -3.042543888092041,
      "eval_logps/chosen": -272.54913330078125,
      "eval_logps/rejected": -242.33013916015625,
      "eval_loss": 0.4910888373851776,
      "eval_rewards/accuracies": 0.7620000243186951,
      "eval_rewards/chosen": -0.11603058129549026,
      "eval_rewards/margins": 1.0332014560699463,
      "eval_rewards/rejected": -1.149232029914856,
      "eval_runtime": 348.8037,
      "eval_samples_per_second": 5.734,
      "eval_steps_per_second": 0.358,
      "step": 100
    },
    {
      "epoch": 0.23,
      "learning_rate": 3.767123287671233e-07,
      "logits/chosen": -3.0026986598968506,
      "logits/rejected": -2.969663381576538,
      "logps/chosen": -279.0836181640625,
      "logps/rejected": -242.3115234375,
      "loss": 0.4758,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.24072448909282684,
      "rewards/margins": 1.099170446395874,
      "rewards/rejected": -1.3398950099945068,
      "step": 110
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.10958904109589e-07,
      "logits/chosen": -3.0795905590057373,
      "logits/rejected": -3.0525641441345215,
      "logps/chosen": -283.3368835449219,
      "logps/rejected": -240.5817108154297,
      "loss": 0.4882,
      "rewards/accuracies": 0.7281249761581421,
      "rewards/chosen": -0.1642596572637558,
      "rewards/margins": 0.9456508755683899,
      "rewards/rejected": -1.1099107265472412,
      "step": 120
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.4520547945205477e-07,
      "logits/chosen": -3.062356472015381,
      "logits/rejected": -3.0561752319335938,
      "logps/chosen": -268.32696533203125,
      "logps/rejected": -242.36599731445312,
      "loss": 0.4935,
      "rewards/accuracies": 0.721875011920929,
      "rewards/chosen": -0.3437289297580719,
      "rewards/margins": 0.990301251411438,
      "rewards/rejected": -1.334030032157898,
      "step": 130
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.794520547945205e-07,
      "logits/chosen": -3.089620590209961,
      "logits/rejected": -3.0679385662078857,
      "logps/chosen": -282.35272216796875,
      "logps/rejected": -246.88681030273438,
      "loss": 0.5161,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.26912999153137207,
      "rewards/margins": 0.9498209953308105,
      "rewards/rejected": -1.2189509868621826,
      "step": 140
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.98468606431853e-07,
      "logits/chosen": -3.0892021656036377,
      "logits/rejected": -3.040349245071411,
      "logps/chosen": -261.6117248535156,
      "logps/rejected": -238.4809112548828,
      "loss": 0.484,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.35393625497817993,
      "rewards/margins": 1.0916435718536377,
      "rewards/rejected": -1.4455797672271729,
      "step": 150
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.946401225114854e-07,
      "logits/chosen": -3.070943832397461,
      "logits/rejected": -3.041159152984619,
      "logps/chosen": -273.7572326660156,
      "logps/rejected": -237.6653594970703,
      "loss": 0.4942,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.2099941074848175,
      "rewards/margins": 1.0625803470611572,
      "rewards/rejected": -1.2725746631622314,
      "step": 160
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.908116385911179e-07,
      "logits/chosen": -3.038595199584961,
      "logits/rejected": -3.008420467376709,
      "logps/chosen": -270.63311767578125,
      "logps/rejected": -250.5312957763672,
      "loss": 0.5107,
      "rewards/accuracies": 0.746874988079071,
      "rewards/chosen": -0.07959787547588348,
      "rewards/margins": 1.0289515256881714,
      "rewards/rejected": -1.1085494756698608,
      "step": 170
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.869831546707504e-07,
      "logits/chosen": -3.041801929473877,
      "logits/rejected": -3.0089993476867676,
      "logps/chosen": -302.63812255859375,
      "logps/rejected": -264.6396484375,
      "loss": 0.4786,
      "rewards/accuracies": 0.809374988079071,
      "rewards/chosen": -0.200510174036026,
      "rewards/margins": 1.2865705490112305,
      "rewards/rejected": -1.4870805740356445,
      "step": 180
    },
    {
      "epoch": 0.39,
      "learning_rate": 4.831546707503829e-07,
      "logits/chosen": -3.0655362606048584,
      "logits/rejected": -3.0381007194519043,
      "logps/chosen": -296.6184997558594,
      "logps/rejected": -271.54998779296875,
      "loss": 0.5105,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.27697867155075073,
      "rewards/margins": 1.28792142868042,
      "rewards/rejected": -1.5649000406265259,
      "step": 190
    },
    {
      "epoch": 0.41,
      "learning_rate": 4.793261868300153e-07,
      "logits/chosen": -2.926753282546997,
      "logits/rejected": -2.9210638999938965,
      "logps/chosen": -267.9990539550781,
      "logps/rejected": -241.55368041992188,
      "loss": 0.4946,
      "rewards/accuracies": 0.746874988079071,
      "rewards/chosen": -0.36534789204597473,
      "rewards/margins": 1.2018648386001587,
      "rewards/rejected": -1.567212700843811,
      "step": 200
    },
    {
      "epoch": 0.41,
      "eval_logits/chosen": -2.9312362670898438,
      "eval_logits/rejected": -2.917306423187256,
      "eval_logps/chosen": -274.8822021484375,
      "eval_logps/rejected": -246.8446044921875,
      "eval_loss": 0.48247700929641724,
      "eval_rewards/accuracies": 0.765999972820282,
      "eval_rewards/chosen": -0.30967435240745544,
      "eval_rewards/margins": 1.2142584323883057,
      "eval_rewards/rejected": -1.5239325761795044,
      "eval_runtime": 351.3731,
      "eval_samples_per_second": 5.692,
      "eval_steps_per_second": 0.356,
      "step": 200
    },
    {
      "epoch": 0.43,
      "learning_rate": 4.7549770290964776e-07,
      "logits/chosen": -2.970444679260254,
      "logits/rejected": -2.9305572509765625,
      "logps/chosen": -257.98101806640625,
      "logps/rejected": -241.5959014892578,
      "loss": 0.5164,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.3353632688522339,
      "rewards/margins": 0.9293055534362793,
      "rewards/rejected": -1.2646688222885132,
      "step": 210
    },
    {
      "epoch": 0.45,
      "learning_rate": 4.7166921898928023e-07,
      "logits/chosen": -3.0221638679504395,
      "logits/rejected": -2.9893369674682617,
      "logps/chosen": -271.72064208984375,
      "logps/rejected": -261.6221008300781,
      "loss": 0.4866,
      "rewards/accuracies": 0.8031250238418579,
      "rewards/chosen": -0.2942289710044861,
      "rewards/margins": 1.3097729682922363,
      "rewards/rejected": -1.6040019989013672,
      "step": 220
    },
    {
      "epoch": 0.47,
      "learning_rate": 4.678407350689127e-07,
      "logits/chosen": -3.0412347316741943,
      "logits/rejected": -3.0181870460510254,
      "logps/chosen": -271.87445068359375,
      "logps/rejected": -240.0888671875,
      "loss": 0.4794,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.28961092233657837,
      "rewards/margins": 1.12082839012146,
      "rewards/rejected": -1.4104394912719727,
      "step": 230
    },
    {
      "epoch": 0.5,
      "learning_rate": 4.640122511485451e-07,
      "logits/chosen": -2.983405590057373,
      "logits/rejected": -2.9718239307403564,
      "logps/chosen": -265.6470642089844,
      "logps/rejected": -240.9022216796875,
      "loss": 0.4808,
      "rewards/accuracies": 0.8031250238418579,
      "rewards/chosen": -0.389603853225708,
      "rewards/margins": 1.3018779754638672,
      "rewards/rejected": -1.6914818286895752,
      "step": 240
    },
    {
      "epoch": 0.52,
      "learning_rate": 4.6018376722817764e-07,
      "logits/chosen": -2.9728949069976807,
      "logits/rejected": -2.9602560997009277,
      "logps/chosen": -269.18231201171875,
      "logps/rejected": -248.4530487060547,
      "loss": 0.4663,
      "rewards/accuracies": 0.7718750238418579,
      "rewards/chosen": -0.14645107090473175,
      "rewards/margins": 1.2143638134002686,
      "rewards/rejected": -1.3608148097991943,
      "step": 250
    },
    {
      "epoch": 0.54,
      "learning_rate": 4.563552833078101e-07,
      "logits/chosen": -2.9226231575012207,
      "logits/rejected": -2.895167589187622,
      "logps/chosen": -291.22698974609375,
      "logps/rejected": -267.5406188964844,
      "loss": 0.4884,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.2955623269081116,
      "rewards/margins": 1.3867028951644897,
      "rewards/rejected": -1.682265043258667,
      "step": 260
    },
    {
      "epoch": 0.56,
      "learning_rate": 4.5252679938744253e-07,
      "logits/chosen": -2.998523235321045,
      "logits/rejected": -2.9929869174957275,
      "logps/chosen": -263.1947021484375,
      "logps/rejected": -246.89761352539062,
      "loss": 0.4983,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.4204261898994446,
      "rewards/margins": 1.0980994701385498,
      "rewards/rejected": -1.5185258388519287,
      "step": 270
    },
    {
      "epoch": 0.58,
      "learning_rate": 4.4869831546707505e-07,
      "logits/chosen": -2.968578338623047,
      "logits/rejected": -2.967036247253418,
      "logps/chosen": -280.832763671875,
      "logps/rejected": -258.5753479003906,
      "loss": 0.4878,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.4376456141471863,
      "rewards/margins": 1.1689329147338867,
      "rewards/rejected": -1.6065785884857178,
      "step": 280
    },
    {
      "epoch": 0.6,
      "learning_rate": 4.4486983154670747e-07,
      "logits/chosen": -2.953157901763916,
      "logits/rejected": -2.89455246925354,
      "logps/chosen": -281.18707275390625,
      "logps/rejected": -252.1344451904297,
      "loss": 0.4617,
      "rewards/accuracies": 0.784375011920929,
      "rewards/chosen": -0.4234972596168518,
      "rewards/margins": 1.4186747074127197,
      "rewards/rejected": -1.8421716690063477,
      "step": 290
    },
    {
      "epoch": 0.62,
      "learning_rate": 4.4104134762633994e-07,
      "logits/chosen": -2.9390437602996826,
      "logits/rejected": -2.9385740756988525,
      "logps/chosen": -250.908447265625,
      "logps/rejected": -226.9329071044922,
      "loss": 0.4777,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.42352238297462463,
      "rewards/margins": 1.06545090675354,
      "rewards/rejected": -1.4889730215072632,
      "step": 300
    },
    {
      "epoch": 0.62,
      "eval_logits/chosen": -2.970550060272217,
      "eval_logits/rejected": -2.9493422508239746,
      "eval_logps/chosen": -273.17779541015625,
      "eval_logps/rejected": -245.2894287109375,
      "eval_loss": 0.4709862172603607,
      "eval_rewards/accuracies": 0.7799999713897705,
      "eval_rewards/chosen": -0.16821104288101196,
      "eval_rewards/margins": 1.226639747619629,
      "eval_rewards/rejected": -1.3948508501052856,
      "eval_runtime": 350.9485,
      "eval_samples_per_second": 5.699,
      "eval_steps_per_second": 0.356,
      "step": 300
    },
    {
      "epoch": 0.64,
      "learning_rate": 4.372128637059724e-07,
      "logits/chosen": -3.0103516578674316,
      "logits/rejected": -2.9915904998779297,
      "logps/chosen": -284.6666259765625,
      "logps/rejected": -251.75927734375,
      "loss": 0.4733,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.3175427317619324,
      "rewards/margins": 1.2486777305603027,
      "rewards/rejected": -1.5662205219268799,
      "step": 310
    },
    {
      "epoch": 0.66,
      "learning_rate": 4.333843797856049e-07,
      "logits/chosen": -2.9826269149780273,
      "logits/rejected": -2.9733011722564697,
      "logps/chosen": -260.04168701171875,
      "logps/rejected": -246.3498992919922,
      "loss": 0.4868,
      "rewards/accuracies": 0.778124988079071,
      "rewards/chosen": -0.5097466707229614,
      "rewards/margins": 1.1551941633224487,
      "rewards/rejected": -1.6649408340454102,
      "step": 320
    },
    {
      "epoch": 0.68,
      "learning_rate": 4.295558958652373e-07,
      "logits/chosen": -2.9864468574523926,
      "logits/rejected": -2.9705405235290527,
      "logps/chosen": -281.12554931640625,
      "logps/rejected": -253.030029296875,
      "loss": 0.4621,
      "rewards/accuracies": 0.753125011920929,
      "rewards/chosen": -0.5258059501647949,
      "rewards/margins": 1.3014029264450073,
      "rewards/rejected": -1.8272088766098022,
      "step": 330
    },
    {
      "epoch": 0.7,
      "learning_rate": 4.257274119448698e-07,
      "logits/chosen": -2.961150646209717,
      "logits/rejected": -2.945072650909424,
      "logps/chosen": -290.82763671875,
      "logps/rejected": -250.4735565185547,
      "loss": 0.5215,
      "rewards/accuracies": 0.7593749761581421,
      "rewards/chosen": -0.5282103419303894,
      "rewards/margins": 1.072779655456543,
      "rewards/rejected": -1.6009900569915771,
      "step": 340
    },
    {
      "epoch": 0.72,
      "learning_rate": 4.218989280245023e-07,
      "logits/chosen": -2.999225378036499,
      "logits/rejected": -2.9767181873321533,
      "logps/chosen": -272.66912841796875,
      "logps/rejected": -241.7967987060547,
      "loss": 0.51,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.4329899847507477,
      "rewards/margins": 1.1193993091583252,
      "rewards/rejected": -1.552389144897461,
      "step": 350
    },
    {
      "epoch": 0.74,
      "learning_rate": 4.180704441041347e-07,
      "logits/chosen": -2.940605640411377,
      "logits/rejected": -2.918984889984131,
      "logps/chosen": -284.83880615234375,
      "logps/rejected": -248.2374725341797,
      "loss": 0.5095,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.42601585388183594,
      "rewards/margins": 1.0927661657333374,
      "rewards/rejected": -1.518782138824463,
      "step": 360
    },
    {
      "epoch": 0.76,
      "learning_rate": 4.1424196018376724e-07,
      "logits/chosen": -2.8906090259552,
      "logits/rejected": -2.8565895557403564,
      "logps/chosen": -270.98321533203125,
      "logps/rejected": -244.8457489013672,
      "loss": 0.4798,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.4540565609931946,
      "rewards/margins": 1.3409868478775024,
      "rewards/rejected": -1.7950433492660522,
      "step": 370
    },
    {
      "epoch": 0.78,
      "learning_rate": 4.1041347626339966e-07,
      "logits/chosen": -2.87612247467041,
      "logits/rejected": -2.8541007041931152,
      "logps/chosen": -268.45556640625,
      "logps/rejected": -254.88693237304688,
      "loss": 0.471,
      "rewards/accuracies": 0.746874988079071,
      "rewards/chosen": -0.5181501507759094,
      "rewards/margins": 1.3443554639816284,
      "rewards/rejected": -1.862505555152893,
      "step": 380
    },
    {
      "epoch": 0.81,
      "learning_rate": 4.065849923430321e-07,
      "logits/chosen": -2.910064935684204,
      "logits/rejected": -2.883251667022705,
      "logps/chosen": -279.72857666015625,
      "logps/rejected": -256.36566162109375,
      "loss": 0.4629,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.3664437234401703,
      "rewards/margins": 1.3844035863876343,
      "rewards/rejected": -1.7508472204208374,
      "step": 390
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.027565084226646e-07,
      "logits/chosen": -2.9096057415008545,
      "logits/rejected": -2.876736879348755,
      "logps/chosen": -274.341552734375,
      "logps/rejected": -249.8159942626953,
      "loss": 0.4795,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3804734945297241,
      "rewards/margins": 1.2859718799591064,
      "rewards/rejected": -1.6664454936981201,
      "step": 400
    },
    {
      "epoch": 0.83,
      "eval_logits/chosen": -2.8863630294799805,
      "eval_logits/rejected": -2.866253137588501,
      "eval_logps/chosen": -275.746826171875,
      "eval_logps/rejected": -249.8567657470703,
      "eval_loss": 0.46131572127342224,
      "eval_rewards/accuracies": 0.7879999876022339,
      "eval_rewards/chosen": -0.3814413547515869,
      "eval_rewards/margins": 1.3924998044967651,
      "eval_rewards/rejected": -1.7739408016204834,
      "eval_runtime": 350.0904,
      "eval_samples_per_second": 5.713,
      "eval_steps_per_second": 0.357,
      "step": 400
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.9892802450229707e-07,
      "logits/chosen": -2.908762216567993,
      "logits/rejected": -2.8779702186584473,
      "logps/chosen": -282.5458679199219,
      "logps/rejected": -249.31338500976562,
      "loss": 0.4513,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.36521783471107483,
      "rewards/margins": 1.4987428188323975,
      "rewards/rejected": -1.8639605045318604,
      "step": 410
    },
    {
      "epoch": 0.87,
      "learning_rate": 3.9509954058192954e-07,
      "logits/chosen": -2.9075565338134766,
      "logits/rejected": -2.8690285682678223,
      "logps/chosen": -272.22283935546875,
      "logps/rejected": -238.9677276611328,
      "loss": 0.4973,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.49386200308799744,
      "rewards/margins": 1.237187147140503,
      "rewards/rejected": -1.7310491800308228,
      "step": 420
    },
    {
      "epoch": 0.89,
      "learning_rate": 3.91271056661562e-07,
      "logits/chosen": -2.9533307552337646,
      "logits/rejected": -2.930840253829956,
      "logps/chosen": -278.5420227050781,
      "logps/rejected": -269.67047119140625,
      "loss": 0.4869,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.5343238711357117,
      "rewards/margins": 1.2090787887573242,
      "rewards/rejected": -1.7434027194976807,
      "step": 430
    },
    {
      "epoch": 0.91,
      "learning_rate": 3.874425727411945e-07,
      "logits/chosen": -2.8991990089416504,
      "logits/rejected": -2.883981227874756,
      "logps/chosen": -301.8912048339844,
      "logps/rejected": -258.8406677246094,
      "loss": 0.4767,
      "rewards/accuracies": 0.753125011920929,
      "rewards/chosen": -0.5268620252609253,
      "rewards/margins": 1.2819582223892212,
      "rewards/rejected": -1.808820128440857,
      "step": 440
    },
    {
      "epoch": 0.93,
      "learning_rate": 3.836140888208269e-07,
      "logits/chosen": -2.8849616050720215,
      "logits/rejected": -2.857882261276245,
      "logps/chosen": -275.40167236328125,
      "logps/rejected": -241.85580444335938,
      "loss": 0.4927,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.49572116136550903,
      "rewards/margins": 1.2914886474609375,
      "rewards/rejected": -1.7872097492218018,
      "step": 450
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.797856049004594e-07,
      "logits/chosen": -2.8969714641571045,
      "logits/rejected": -2.8453850746154785,
      "logps/chosen": -255.99887084960938,
      "logps/rejected": -229.887451171875,
      "loss": 0.4795,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.3662777841091156,
      "rewards/margins": 1.1540734767913818,
      "rewards/rejected": -1.5203514099121094,
      "step": 460
    },
    {
      "epoch": 0.97,
      "learning_rate": 3.7595712098009184e-07,
      "logits/chosen": -2.85526442527771,
      "logits/rejected": -2.82959246635437,
      "logps/chosen": -260.65997314453125,
      "logps/rejected": -234.6728973388672,
      "loss": 0.4751,
      "rewards/accuracies": 0.7718750238418579,
      "rewards/chosen": -0.42231401801109314,
      "rewards/margins": 1.306610107421875,
      "rewards/rejected": -1.7289241552352905,
      "step": 470
    },
    {
      "epoch": 0.99,
      "learning_rate": 3.7212863705972436e-07,
      "logits/chosen": -2.861299991607666,
      "logits/rejected": -2.861154794692993,
      "logps/chosen": -280.72894287109375,
      "logps/rejected": -251.1167755126953,
      "loss": 0.4664,
      "rewards/accuracies": 0.7281249761581421,
      "rewards/chosen": -0.4215695858001709,
      "rewards/margins": 1.0574934482574463,
      "rewards/rejected": -1.4790631532669067,
      "step": 480
    },
    {
      "epoch": 1.01,
      "learning_rate": 3.683001531393568e-07,
      "logits/chosen": -2.8483738899230957,
      "logits/rejected": -2.836601734161377,
      "logps/chosen": -271.0488586425781,
      "logps/rejected": -263.08831787109375,
      "loss": 0.254,
      "rewards/accuracies": 0.8968750238418579,
      "rewards/chosen": 0.15104517340660095,
      "rewards/margins": 2.5157012939453125,
      "rewards/rejected": -2.3646559715270996,
      "step": 490
    },
    {
      "epoch": 1.03,
      "learning_rate": 3.6447166921898925e-07,
      "logits/chosen": -2.8055942058563232,
      "logits/rejected": -2.795368194580078,
      "logps/chosen": -252.1892547607422,
      "logps/rejected": -278.5306396484375,
      "loss": 0.1195,
      "rewards/accuracies": 0.971875011920929,
      "rewards/chosen": 0.34003573656082153,
      "rewards/margins": 3.622701644897461,
      "rewards/rejected": -3.2826664447784424,
      "step": 500
    },
    {
      "epoch": 1.03,
      "eval_logits/chosen": -2.8173274993896484,
      "eval_logits/rejected": -2.788041114807129,
      "eval_logps/chosen": -279.2414245605469,
      "eval_logps/rejected": -256.60906982421875,
      "eval_loss": 0.46797943115234375,
      "eval_rewards/accuracies": 0.7879999876022339,
      "eval_rewards/chosen": -0.6714959144592285,
      "eval_rewards/margins": 1.662886142730713,
      "eval_rewards/rejected": -2.3343818187713623,
      "eval_runtime": 350.8393,
      "eval_samples_per_second": 5.701,
      "eval_steps_per_second": 0.356,
      "step": 500
    },
    {
      "epoch": 1.05,
      "learning_rate": 3.606431852986218e-07,
      "logits/chosen": -2.762050151824951,
      "logits/rejected": -2.752959728240967,
      "logps/chosen": -265.3427734375,
      "logps/rejected": -263.5534362792969,
      "loss": 0.1113,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.23612014949321747,
      "rewards/margins": 3.711341381072998,
      "rewards/rejected": -3.475221633911133,
      "step": 510
    },
    {
      "epoch": 1.07,
      "learning_rate": 3.568147013782542e-07,
      "logits/chosen": -2.8299038410186768,
      "logits/rejected": -2.769249439239502,
      "logps/chosen": -273.716796875,
      "logps/rejected": -283.7131652832031,
      "loss": 0.0966,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": 0.20140285789966583,
      "rewards/margins": 3.9053382873535156,
      "rewards/rejected": -3.703935146331787,
      "step": 520
    },
    {
      "epoch": 1.09,
      "learning_rate": 3.5298621745788666e-07,
      "logits/chosen": -2.751771926879883,
      "logits/rejected": -2.7126195430755615,
      "logps/chosen": -271.053466796875,
      "logps/rejected": -274.77874755859375,
      "loss": 0.1055,
      "rewards/accuracies": 0.971875011920929,
      "rewards/chosen": -0.04146752506494522,
      "rewards/margins": 4.163424968719482,
      "rewards/rejected": -4.204892158508301,
      "step": 530
    },
    {
      "epoch": 1.12,
      "learning_rate": 3.4915773353751913e-07,
      "logits/chosen": -2.810192584991455,
      "logits/rejected": -2.780327320098877,
      "logps/chosen": -273.2801513671875,
      "logps/rejected": -286.0684509277344,
      "loss": 0.1153,
      "rewards/accuracies": 0.9468749761581421,
      "rewards/chosen": -0.3729572594165802,
      "rewards/margins": 3.824183940887451,
      "rewards/rejected": -4.197141170501709,
      "step": 540
    },
    {
      "epoch": 1.14,
      "learning_rate": 3.453292496171516e-07,
      "logits/chosen": -2.795090436935425,
      "logits/rejected": -2.7538418769836426,
      "logps/chosen": -271.19036865234375,
      "logps/rejected": -272.4897766113281,
      "loss": 0.109,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.23404860496520996,
      "rewards/margins": 4.046402931213379,
      "rewards/rejected": -4.28045129776001,
      "step": 550
    },
    {
      "epoch": 1.16,
      "learning_rate": 3.41500765696784e-07,
      "logits/chosen": -2.815134286880493,
      "logits/rejected": -2.800909996032715,
      "logps/chosen": -267.2737731933594,
      "logps/rejected": -300.0193176269531,
      "loss": 0.1077,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.02004162408411503,
      "rewards/margins": 4.190427303314209,
      "rewards/rejected": -4.210468292236328,
      "step": 560
    },
    {
      "epoch": 1.18,
      "learning_rate": 3.3767228177641655e-07,
      "logits/chosen": -2.8094847202301025,
      "logits/rejected": -2.8050694465637207,
      "logps/chosen": -281.94866943359375,
      "logps/rejected": -298.3989562988281,
      "loss": 0.1133,
      "rewards/accuracies": 0.965624988079071,
      "rewards/chosen": -0.09466864168643951,
      "rewards/margins": 4.250113487243652,
      "rewards/rejected": -4.34478235244751,
      "step": 570
    },
    {
      "epoch": 1.2,
      "learning_rate": 3.33843797856049e-07,
      "logits/chosen": -2.793306350708008,
      "logits/rejected": -2.776901960372925,
      "logps/chosen": -277.68499755859375,
      "logps/rejected": -279.35272216796875,
      "loss": 0.0941,
      "rewards/accuracies": 0.965624988079071,
      "rewards/chosen": -0.047898270189762115,
      "rewards/margins": 4.4947590827941895,
      "rewards/rejected": -4.542656898498535,
      "step": 580
    },
    {
      "epoch": 1.22,
      "learning_rate": 3.3001531393568143e-07,
      "logits/chosen": -2.797222375869751,
      "logits/rejected": -2.761697292327881,
      "logps/chosen": -265.84979248046875,
      "logps/rejected": -280.93890380859375,
      "loss": 0.0913,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.3232390284538269,
      "rewards/margins": 4.295743465423584,
      "rewards/rejected": -4.618982791900635,
      "step": 590
    },
    {
      "epoch": 1.24,
      "learning_rate": 3.2618683001531396e-07,
      "logits/chosen": -2.7871365547180176,
      "logits/rejected": -2.7726902961730957,
      "logps/chosen": -257.52520751953125,
      "logps/rejected": -284.35504150390625,
      "loss": 0.1038,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.2683785855770111,
      "rewards/margins": 4.337276458740234,
      "rewards/rejected": -4.605654716491699,
      "step": 600
    },
    {
      "epoch": 1.24,
      "eval_logits/chosen": -2.8235371112823486,
      "eval_logits/rejected": -2.794421911239624,
      "eval_logps/chosen": -285.988525390625,
      "eval_logps/rejected": -267.02618408203125,
      "eval_loss": 0.48899853229522705,
      "eval_rewards/accuracies": 0.7739999890327454,
      "eval_rewards/chosen": -1.2315046787261963,
      "eval_rewards/margins": 1.9674973487854004,
      "eval_rewards/rejected": -3.199002265930176,
      "eval_runtime": 351.0766,
      "eval_samples_per_second": 5.697,
      "eval_steps_per_second": 0.356,
      "step": 600
    },
    {
      "epoch": 1.26,
      "learning_rate": 3.223583460949464e-07,
      "logits/chosen": -2.8070285320281982,
      "logits/rejected": -2.778954029083252,
      "logps/chosen": -268.57122802734375,
      "logps/rejected": -294.0079650878906,
      "loss": 0.1012,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.27337712049484253,
      "rewards/margins": 4.341578006744385,
      "rewards/rejected": -4.614955425262451,
      "step": 610
    },
    {
      "epoch": 1.28,
      "learning_rate": 3.1852986217457885e-07,
      "logits/chosen": -2.766972064971924,
      "logits/rejected": -2.756957530975342,
      "logps/chosen": -295.1747131347656,
      "logps/rejected": -305.98260498046875,
      "loss": 0.1166,
      "rewards/accuracies": 0.9593750238418579,
      "rewards/chosen": -0.40655288100242615,
      "rewards/margins": 4.126054286956787,
      "rewards/rejected": -4.532607078552246,
      "step": 620
    },
    {
      "epoch": 1.3,
      "learning_rate": 3.147013782542113e-07,
      "logits/chosen": -2.8336269855499268,
      "logits/rejected": -2.8039116859436035,
      "logps/chosen": -264.93963623046875,
      "logps/rejected": -276.555908203125,
      "loss": 0.1157,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.42949333786964417,
      "rewards/margins": 4.0998215675354,
      "rewards/rejected": -4.529314994812012,
      "step": 630
    },
    {
      "epoch": 1.32,
      "learning_rate": 3.108728943338438e-07,
      "logits/chosen": -2.8210394382476807,
      "logits/rejected": -2.8052823543548584,
      "logps/chosen": -275.5909729003906,
      "logps/rejected": -300.68292236328125,
      "loss": 0.1137,
      "rewards/accuracies": 0.971875011920929,
      "rewards/chosen": -0.2871936559677124,
      "rewards/margins": 4.335806846618652,
      "rewards/rejected": -4.623000621795654,
      "step": 640
    },
    {
      "epoch": 1.34,
      "learning_rate": 3.070444104134762e-07,
      "logits/chosen": -2.782514810562134,
      "logits/rejected": -2.755683422088623,
      "logps/chosen": -272.1896667480469,
      "logps/rejected": -282.8103332519531,
      "loss": 0.117,
      "rewards/accuracies": 0.965624988079071,
      "rewards/chosen": -0.4863522946834564,
      "rewards/margins": 4.17294979095459,
      "rewards/rejected": -4.6593017578125,
      "step": 650
    },
    {
      "epoch": 1.36,
      "learning_rate": 3.0321592649310873e-07,
      "logits/chosen": -2.8078765869140625,
      "logits/rejected": -2.771209955215454,
      "logps/chosen": -269.9143371582031,
      "logps/rejected": -273.5386047363281,
      "loss": 0.1127,
      "rewards/accuracies": 0.9593750238418579,
      "rewards/chosen": -0.5027432441711426,
      "rewards/margins": 4.127696990966797,
      "rewards/rejected": -4.630440711975098,
      "step": 660
    },
    {
      "epoch": 1.38,
      "learning_rate": 2.993874425727412e-07,
      "logits/chosen": -2.82597017288208,
      "logits/rejected": -2.777392625808716,
      "logps/chosen": -270.00823974609375,
      "logps/rejected": -289.9776306152344,
      "loss": 0.1074,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.7837791442871094,
      "rewards/margins": 4.425631999969482,
      "rewards/rejected": -5.209410667419434,
      "step": 670
    },
    {
      "epoch": 1.4,
      "learning_rate": 2.955589586523736e-07,
      "logits/chosen": -2.8265202045440674,
      "logits/rejected": -2.7997395992279053,
      "logps/chosen": -264.7549743652344,
      "logps/rejected": -275.26983642578125,
      "loss": 0.1172,
      "rewards/accuracies": 0.9468749761581421,
      "rewards/chosen": -0.597946047782898,
      "rewards/margins": 4.154361248016357,
      "rewards/rejected": -4.752306938171387,
      "step": 680
    },
    {
      "epoch": 1.42,
      "learning_rate": 2.9173047473200614e-07,
      "logits/chosen": -2.8278021812438965,
      "logits/rejected": -2.7613134384155273,
      "logps/chosen": -273.6087951660156,
      "logps/rejected": -278.0371398925781,
      "loss": 0.1085,
      "rewards/accuracies": 0.9593750238418579,
      "rewards/chosen": -0.5144752264022827,
      "rewards/margins": 4.272164821624756,
      "rewards/rejected": -4.786640644073486,
      "step": 690
    },
    {
      "epoch": 1.45,
      "learning_rate": 2.8790199081163856e-07,
      "logits/chosen": -2.8102574348449707,
      "logits/rejected": -2.8080575466156006,
      "logps/chosen": -293.1568908691406,
      "logps/rejected": -288.95648193359375,
      "loss": 0.1091,
      "rewards/accuracies": 0.965624988079071,
      "rewards/chosen": -0.3547598123550415,
      "rewards/margins": 4.377170085906982,
      "rewards/rejected": -4.731929779052734,
      "step": 700
    },
    {
      "epoch": 1.45,
      "eval_logits/chosen": -2.7946670055389404,
      "eval_logits/rejected": -2.769153594970703,
      "eval_logps/chosen": -289.4513854980469,
      "eval_logps/rejected": -270.27752685546875,
      "eval_loss": 0.49690696597099304,
      "eval_rewards/accuracies": 0.7720000147819519,
      "eval_rewards/chosen": -1.5189176797866821,
      "eval_rewards/margins": 1.9499460458755493,
      "eval_rewards/rejected": -3.4688637256622314,
      "eval_runtime": 350.9125,
      "eval_samples_per_second": 5.699,
      "eval_steps_per_second": 0.356,
      "step": 700
    },
    {
      "epoch": 1.47,
      "learning_rate": 2.8407350689127103e-07,
      "logits/chosen": -2.805912733078003,
      "logits/rejected": -2.7843575477600098,
      "logps/chosen": -268.52874755859375,
      "logps/rejected": -289.5450744628906,
      "loss": 0.1044,
      "rewards/accuracies": 0.9593750238418579,
      "rewards/chosen": -0.34271737933158875,
      "rewards/margins": 4.460578918457031,
      "rewards/rejected": -4.803296089172363,
      "step": 710
    },
    {
      "epoch": 1.49,
      "learning_rate": 2.802450229709035e-07,
      "logits/chosen": -2.788163423538208,
      "logits/rejected": -2.7605245113372803,
      "logps/chosen": -260.4649353027344,
      "logps/rejected": -290.8619689941406,
      "loss": 0.1069,
      "rewards/accuracies": 0.9593750238418579,
      "rewards/chosen": -0.521013617515564,
      "rewards/margins": 4.3092875480651855,
      "rewards/rejected": -4.830301284790039,
      "step": 720
    },
    {
      "epoch": 1.51,
      "learning_rate": 2.7641653905053597e-07,
      "logits/chosen": -2.8127048015594482,
      "logits/rejected": -2.767697811126709,
      "logps/chosen": -281.118408203125,
      "logps/rejected": -267.87408447265625,
      "loss": 0.1139,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.4672430157661438,
      "rewards/margins": 4.233705043792725,
      "rewards/rejected": -4.7009477615356445,
      "step": 730
    },
    {
      "epoch": 1.53,
      "learning_rate": 2.725880551301684e-07,
      "logits/chosen": -2.7799177169799805,
      "logits/rejected": -2.778627872467041,
      "logps/chosen": -270.40753173828125,
      "logps/rejected": -282.4443359375,
      "loss": 0.1028,
      "rewards/accuracies": 0.9593750238418579,
      "rewards/chosen": -0.4677719473838806,
      "rewards/margins": 4.507622718811035,
      "rewards/rejected": -4.975394248962402,
      "step": 740
    },
    {
      "epoch": 1.55,
      "learning_rate": 2.687595712098009e-07,
      "logits/chosen": -2.784428119659424,
      "logits/rejected": -2.7339842319488525,
      "logps/chosen": -281.16998291015625,
      "logps/rejected": -290.2821960449219,
      "loss": 0.0869,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.5696399211883545,
      "rewards/margins": 4.695879936218262,
      "rewards/rejected": -5.265519618988037,
      "step": 750
    },
    {
      "epoch": 1.57,
      "learning_rate": 2.649310872894334e-07,
      "logits/chosen": -2.7697200775146484,
      "logits/rejected": -2.751024007797241,
      "logps/chosen": -261.14117431640625,
      "logps/rejected": -278.02984619140625,
      "loss": 0.1014,
      "rewards/accuracies": 0.940625011920929,
      "rewards/chosen": -0.4582810401916504,
      "rewards/margins": 4.3716230392456055,
      "rewards/rejected": -4.829904079437256,
      "step": 760
    },
    {
      "epoch": 1.59,
      "learning_rate": 2.6110260336906586e-07,
      "logits/chosen": -2.8378894329071045,
      "logits/rejected": -2.810879945755005,
      "logps/chosen": -284.22113037109375,
      "logps/rejected": -285.4598693847656,
      "loss": 0.1108,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": -0.3764747977256775,
      "rewards/margins": 4.466374397277832,
      "rewards/rejected": -4.842849254608154,
      "step": 770
    },
    {
      "epoch": 1.61,
      "learning_rate": 2.572741194486983e-07,
      "logits/chosen": -2.8060693740844727,
      "logits/rejected": -2.7883379459381104,
      "logps/chosen": -276.88592529296875,
      "logps/rejected": -285.6861267089844,
      "loss": 0.1146,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.31279510259628296,
      "rewards/margins": 4.379072189331055,
      "rewards/rejected": -4.691866874694824,
      "step": 780
    },
    {
      "epoch": 1.63,
      "learning_rate": 2.5344563552833074e-07,
      "logits/chosen": -2.8530561923980713,
      "logits/rejected": -2.8231146335601807,
      "logps/chosen": -288.106201171875,
      "logps/rejected": -296.70770263671875,
      "loss": 0.0955,
      "rewards/accuracies": 0.9593750238418579,
      "rewards/chosen": -0.5869344472885132,
      "rewards/margins": 4.610665321350098,
      "rewards/rejected": -5.197599411010742,
      "step": 790
    },
    {
      "epoch": 1.65,
      "learning_rate": 2.496171516079632e-07,
      "logits/chosen": -2.804647922515869,
      "logits/rejected": -2.762570381164551,
      "logps/chosen": -298.3683166503906,
      "logps/rejected": -296.1361999511719,
      "loss": 0.1017,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.4437989294528961,
      "rewards/margins": 4.659768104553223,
      "rewards/rejected": -5.103568077087402,
      "step": 800
    },
    {
      "epoch": 1.65,
      "eval_logits/chosen": -2.8233227729797363,
      "eval_logits/rejected": -2.7970263957977295,
      "eval_logps/chosen": -289.2352294921875,
      "eval_logps/rejected": -271.4718933105469,
      "eval_loss": 0.49377888441085815,
      "eval_rewards/accuracies": 0.777999997138977,
      "eval_rewards/chosen": -1.5009784698486328,
      "eval_rewards/margins": 2.06701922416687,
      "eval_rewards/rejected": -3.567997694015503,
      "eval_runtime": 351.5261,
      "eval_samples_per_second": 5.689,
      "eval_steps_per_second": 0.356,
      "step": 800
    },
    {
      "epoch": 1.67,
      "learning_rate": 2.457886676875957e-07,
      "logits/chosen": -2.8444576263427734,
      "logits/rejected": -2.8208250999450684,
      "logps/chosen": -284.90496826171875,
      "logps/rejected": -292.76397705078125,
      "loss": 0.1013,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.47984448075294495,
      "rewards/margins": 4.4324140548706055,
      "rewards/rejected": -4.912258148193359,
      "step": 810
    },
    {
      "epoch": 1.69,
      "learning_rate": 2.4196018376722816e-07,
      "logits/chosen": -2.7602972984313965,
      "logits/rejected": -2.7448487281799316,
      "logps/chosen": -270.0898132324219,
      "logps/rejected": -292.3755798339844,
      "loss": 0.0969,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.5020721554756165,
      "rewards/margins": 4.625714302062988,
      "rewards/rejected": -5.127787113189697,
      "step": 820
    },
    {
      "epoch": 1.71,
      "learning_rate": 2.3813169984686063e-07,
      "logits/chosen": -2.786911725997925,
      "logits/rejected": -2.7684903144836426,
      "logps/chosen": -265.9609680175781,
      "logps/rejected": -286.37445068359375,
      "loss": 0.1054,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.666102409362793,
      "rewards/margins": 4.600396156311035,
      "rewards/rejected": -5.266499042510986,
      "step": 830
    },
    {
      "epoch": 1.73,
      "learning_rate": 2.343032159264931e-07,
      "logits/chosen": -2.8092284202575684,
      "logits/rejected": -2.7773725986480713,
      "logps/chosen": -282.3887634277344,
      "logps/rejected": -291.4471435546875,
      "loss": 0.1062,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.47004085779190063,
      "rewards/margins": 4.290153980255127,
      "rewards/rejected": -4.760195255279541,
      "step": 840
    },
    {
      "epoch": 1.76,
      "learning_rate": 2.3047473200612557e-07,
      "logits/chosen": -2.8055591583251953,
      "logits/rejected": -2.7884533405303955,
      "logps/chosen": -294.50274658203125,
      "logps/rejected": -300.5123596191406,
      "loss": 0.0936,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.5539692640304565,
      "rewards/margins": 4.763749122619629,
      "rewards/rejected": -5.317718982696533,
      "step": 850
    },
    {
      "epoch": 1.78,
      "learning_rate": 2.26646248085758e-07,
      "logits/chosen": -2.7786107063293457,
      "logits/rejected": -2.757014751434326,
      "logps/chosen": -276.4443054199219,
      "logps/rejected": -310.6683044433594,
      "loss": 0.0994,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.7117125988006592,
      "rewards/margins": 4.650470733642578,
      "rewards/rejected": -5.362184047698975,
      "step": 860
    },
    {
      "epoch": 1.8,
      "learning_rate": 2.228177641653905e-07,
      "logits/chosen": -2.779181957244873,
      "logits/rejected": -2.7437307834625244,
      "logps/chosen": -302.57928466796875,
      "logps/rejected": -294.8531188964844,
      "loss": 0.1062,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.5076280236244202,
      "rewards/margins": 4.748777389526367,
      "rewards/rejected": -5.256405830383301,
      "step": 870
    },
    {
      "epoch": 1.82,
      "learning_rate": 2.1898928024502298e-07,
      "logits/chosen": -2.8428640365600586,
      "logits/rejected": -2.823258638381958,
      "logps/chosen": -291.81280517578125,
      "logps/rejected": -321.46697998046875,
      "loss": 0.1001,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.5858921408653259,
      "rewards/margins": 4.868914604187012,
      "rewards/rejected": -5.454807281494141,
      "step": 880
    },
    {
      "epoch": 1.84,
      "learning_rate": 2.1516079632465542e-07,
      "logits/chosen": -2.7857911586761475,
      "logits/rejected": -2.796041250228882,
      "logps/chosen": -260.4945373535156,
      "logps/rejected": -292.18280029296875,
      "loss": 0.1017,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.8166616559028625,
      "rewards/margins": 4.301018714904785,
      "rewards/rejected": -5.117680549621582,
      "step": 890
    },
    {
      "epoch": 1.86,
      "learning_rate": 2.113323124042879e-07,
      "logits/chosen": -2.795987606048584,
      "logits/rejected": -2.7470993995666504,
      "logps/chosen": -275.99658203125,
      "logps/rejected": -275.63287353515625,
      "loss": 0.0853,
      "rewards/accuracies": 0.971875011920929,
      "rewards/chosen": -0.8849824666976929,
      "rewards/margins": 4.627337455749512,
      "rewards/rejected": -5.512320041656494,
      "step": 900
    },
    {
      "epoch": 1.86,
      "eval_logits/chosen": -2.795854330062866,
      "eval_logits/rejected": -2.766737222671509,
      "eval_logps/chosen": -295.14727783203125,
      "eval_logps/rejected": -278.9565124511719,
      "eval_loss": 0.5149641036987305,
      "eval_rewards/accuracies": 0.7839999794960022,
      "eval_rewards/chosen": -1.9916783571243286,
      "eval_rewards/margins": 2.197537899017334,
      "eval_rewards/rejected": -4.189216136932373,
      "eval_runtime": 349.9647,
      "eval_samples_per_second": 5.715,
      "eval_steps_per_second": 0.357,
      "step": 900
    },
    {
      "epoch": 1.88,
      "learning_rate": 2.0750382848392037e-07,
      "logits/chosen": -2.7703442573547363,
      "logits/rejected": -2.7275280952453613,
      "logps/chosen": -294.9524841308594,
      "logps/rejected": -296.74359130859375,
      "loss": 0.1016,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.758061408996582,
      "rewards/margins": 4.951012134552002,
      "rewards/rejected": -5.709073066711426,
      "step": 910
    },
    {
      "epoch": 1.9,
      "learning_rate": 2.036753445635528e-07,
      "logits/chosen": -2.8267745971679688,
      "logits/rejected": -2.7849204540252686,
      "logps/chosen": -311.01727294921875,
      "logps/rejected": -302.3856506347656,
      "loss": 0.0919,
      "rewards/accuracies": 0.965624988079071,
      "rewards/chosen": -0.8452297449111938,
      "rewards/margins": 4.621523380279541,
      "rewards/rejected": -5.4667534828186035,
      "step": 920
    },
    {
      "epoch": 1.92,
      "learning_rate": 1.9984686064318528e-07,
      "logits/chosen": -2.8363468647003174,
      "logits/rejected": -2.8089370727539062,
      "logps/chosen": -297.74957275390625,
      "logps/rejected": -295.922119140625,
      "loss": 0.0974,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.7593555450439453,
      "rewards/margins": 4.662358283996582,
      "rewards/rejected": -5.421713829040527,
      "step": 930
    },
    {
      "epoch": 1.94,
      "learning_rate": 1.9601837672281775e-07,
      "logits/chosen": -2.799447536468506,
      "logits/rejected": -2.7984437942504883,
      "logps/chosen": -262.63702392578125,
      "logps/rejected": -293.9930725097656,
      "loss": 0.0919,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.6781708598136902,
      "rewards/margins": 4.636082172393799,
      "rewards/rejected": -5.314253330230713,
      "step": 940
    },
    {
      "epoch": 1.96,
      "learning_rate": 1.9218989280245022e-07,
      "logits/chosen": -2.8430511951446533,
      "logits/rejected": -2.829907178878784,
      "logps/chosen": -283.31317138671875,
      "logps/rejected": -303.9375305175781,
      "loss": 0.1138,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.9054574966430664,
      "rewards/margins": 4.864491939544678,
      "rewards/rejected": -5.769949436187744,
      "step": 950
    },
    {
      "epoch": 1.98,
      "learning_rate": 1.883614088820827e-07,
      "logits/chosen": -2.8085925579071045,
      "logits/rejected": -2.807694435119629,
      "logps/chosen": -278.26263427734375,
      "logps/rejected": -279.7634582519531,
      "loss": 0.1018,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.054297685623169,
      "rewards/margins": 4.191025733947754,
      "rewards/rejected": -5.24532413482666,
      "step": 960
    },
    {
      "epoch": 2.0,
      "learning_rate": 1.8453292496171516e-07,
      "logits/chosen": -2.8521769046783447,
      "logits/rejected": -2.8313467502593994,
      "logps/chosen": -275.1976013183594,
      "logps/rejected": -290.14068603515625,
      "loss": 0.1008,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.8346074223518372,
      "rewards/margins": 4.5843939781188965,
      "rewards/rejected": -5.41900110244751,
      "step": 970
    },
    {
      "epoch": 2.02,
      "learning_rate": 1.807044410413476e-07,
      "logits/chosen": -2.764686107635498,
      "logits/rejected": -2.756455898284912,
      "logps/chosen": -274.3100280761719,
      "logps/rejected": -328.5483093261719,
      "loss": 0.0254,
      "rewards/accuracies": 0.9906250238418579,
      "rewards/chosen": -0.7804474234580994,
      "rewards/margins": 5.736757755279541,
      "rewards/rejected": -6.517205238342285,
      "step": 980
    },
    {
      "epoch": 2.04,
      "learning_rate": 1.7687595712098008e-07,
      "logits/chosen": -2.8040549755096436,
      "logits/rejected": -2.7453818321228027,
      "logps/chosen": -284.9351806640625,
      "logps/rejected": -301.5528564453125,
      "loss": 0.0272,
      "rewards/accuracies": 0.9906250238418579,
      "rewards/chosen": -1.0040773153305054,
      "rewards/margins": 5.6107378005981445,
      "rewards/rejected": -6.614814758300781,
      "step": 990
    },
    {
      "epoch": 2.07,
      "learning_rate": 1.7304747320061255e-07,
      "logits/chosen": -2.7672786712646484,
      "logits/rejected": -2.734074115753174,
      "logps/chosen": -283.12847900390625,
      "logps/rejected": -314.47149658203125,
      "loss": 0.0214,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.4745429754257202,
      "rewards/margins": 5.9901838302612305,
      "rewards/rejected": -7.46472692489624,
      "step": 1000
    },
    {
      "epoch": 2.07,
      "eval_logits/chosen": -2.7811498641967773,
      "eval_logits/rejected": -2.7488815784454346,
      "eval_logps/chosen": -306.46063232421875,
      "eval_logps/rejected": -295.9729309082031,
      "eval_loss": 0.5712993144989014,
      "eval_rewards/accuracies": 0.7739999890327454,
      "eval_rewards/chosen": -2.9306883811950684,
      "eval_rewards/margins": 2.670891523361206,
      "eval_rewards/rejected": -5.601580619812012,
      "eval_runtime": 350.286,
      "eval_samples_per_second": 5.71,
      "eval_steps_per_second": 0.357,
      "step": 1000
    },
    {
      "epoch": 2.09,
      "learning_rate": 1.6921898928024502e-07,
      "logits/chosen": -2.77101993560791,
      "logits/rejected": -2.737403392791748,
      "logps/chosen": -296.3531799316406,
      "logps/rejected": -322.0106506347656,
      "loss": 0.0179,
      "rewards/accuracies": 0.996874988079071,
      "rewards/chosen": -1.2550556659698486,
      "rewards/margins": 6.442480564117432,
      "rewards/rejected": -7.697535514831543,
      "step": 1010
    },
    {
      "epoch": 2.11,
      "learning_rate": 1.6539050535987747e-07,
      "logits/chosen": -2.7734529972076416,
      "logits/rejected": -2.7330188751220703,
      "logps/chosen": -279.7518615722656,
      "logps/rejected": -326.99029541015625,
      "loss": 0.0205,
      "rewards/accuracies": 0.9906250238418579,
      "rewards/chosen": -1.3698816299438477,
      "rewards/margins": 6.544535160064697,
      "rewards/rejected": -7.914417266845703,
      "step": 1020
    },
    {
      "epoch": 2.13,
      "learning_rate": 1.6156202143950994e-07,
      "logits/chosen": -2.8035759925842285,
      "logits/rejected": -2.767061948776245,
      "logps/chosen": -302.5785217285156,
      "logps/rejected": -332.73675537109375,
      "loss": 0.0183,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3139612674713135,
      "rewards/margins": 6.302945137023926,
      "rewards/rejected": -7.61690616607666,
      "step": 1030
    },
    {
      "epoch": 2.15,
      "learning_rate": 1.5773353751914243e-07,
      "logits/chosen": -2.81292462348938,
      "logits/rejected": -2.762890338897705,
      "logps/chosen": -292.2890625,
      "logps/rejected": -339.5328063964844,
      "loss": 0.0196,
      "rewards/accuracies": 0.9906250238418579,
      "rewards/chosen": -1.1018422842025757,
      "rewards/margins": 6.696122169494629,
      "rewards/rejected": -7.797965049743652,
      "step": 1040
    },
    {
      "epoch": 2.17,
      "learning_rate": 1.5390505359877488e-07,
      "logits/chosen": -2.7605996131896973,
      "logits/rejected": -2.735153913497925,
      "logps/chosen": -282.97198486328125,
      "logps/rejected": -319.02996826171875,
      "loss": 0.0203,
      "rewards/accuracies": 0.996874988079071,
      "rewards/chosen": -1.696224570274353,
      "rewards/margins": 6.712541103363037,
      "rewards/rejected": -8.40876579284668,
      "step": 1050
    },
    {
      "epoch": 2.19,
      "learning_rate": 1.5007656967840735e-07,
      "logits/chosen": -2.776733875274658,
      "logits/rejected": -2.747337579727173,
      "logps/chosen": -303.0177917480469,
      "logps/rejected": -326.96246337890625,
      "loss": 0.0211,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.5432361364364624,
      "rewards/margins": 6.363909721374512,
      "rewards/rejected": -7.907146453857422,
      "step": 1060
    },
    {
      "epoch": 2.21,
      "learning_rate": 1.4624808575803982e-07,
      "logits/chosen": -2.7601959705352783,
      "logits/rejected": -2.7408816814422607,
      "logps/chosen": -265.16192626953125,
      "logps/rejected": -306.35955810546875,
      "loss": 0.0198,
      "rewards/accuracies": 0.996874988079071,
      "rewards/chosen": -1.5904560089111328,
      "rewards/margins": 6.512013912200928,
      "rewards/rejected": -8.102469444274902,
      "step": 1070
    },
    {
      "epoch": 2.23,
      "learning_rate": 1.4241960183767226e-07,
      "logits/chosen": -2.767143964767456,
      "logits/rejected": -2.739062786102295,
      "logps/chosen": -314.4750061035156,
      "logps/rejected": -345.097412109375,
      "loss": 0.0215,
      "rewards/accuracies": 0.9906250238418579,
      "rewards/chosen": -1.3406659364700317,
      "rewards/margins": 6.835707664489746,
      "rewards/rejected": -8.176374435424805,
      "step": 1080
    },
    {
      "epoch": 2.25,
      "learning_rate": 1.3859111791730473e-07,
      "logits/chosen": -2.7684121131896973,
      "logits/rejected": -2.7613167762756348,
      "logps/chosen": -308.7982482910156,
      "logps/rejected": -335.6389465332031,
      "loss": 0.0217,
      "rewards/accuracies": 0.9906250238418579,
      "rewards/chosen": -1.6674400568008423,
      "rewards/margins": 6.796760559082031,
      "rewards/rejected": -8.464200973510742,
      "step": 1090
    },
    {
      "epoch": 2.27,
      "learning_rate": 1.347626339969372e-07,
      "logits/chosen": -2.7207558155059814,
      "logits/rejected": -2.7273590564727783,
      "logps/chosen": -266.62744140625,
      "logps/rejected": -325.07366943359375,
      "loss": 0.0169,
      "rewards/accuracies": 0.996874988079071,
      "rewards/chosen": -1.993160605430603,
      "rewards/margins": 6.693910121917725,
      "rewards/rejected": -8.687070846557617,
      "step": 1100
    },
    {
      "epoch": 2.27,
      "eval_logits/chosen": -2.7449355125427246,
      "eval_logits/rejected": -2.7095649242401123,
      "eval_logps/chosen": -314.7624816894531,
      "eval_logps/rejected": -308.3752746582031,
      "eval_loss": 0.6225754618644714,
      "eval_rewards/accuracies": 0.7760000228881836,
      "eval_rewards/chosen": -3.619741916656494,
      "eval_rewards/margins": 3.0112340450286865,
      "eval_rewards/rejected": -6.63097620010376,
      "eval_runtime": 351.0248,
      "eval_samples_per_second": 5.698,
      "eval_steps_per_second": 0.356,
      "step": 1100
    },
    {
      "epoch": 2.29,
      "learning_rate": 1.3093415007656965e-07,
      "logits/chosen": -2.744692325592041,
      "logits/rejected": -2.6791775226593018,
      "logps/chosen": -321.56939697265625,
      "logps/rejected": -335.9093017578125,
      "loss": 0.0187,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.579837679862976,
      "rewards/margins": 7.068323612213135,
      "rewards/rejected": -8.648160934448242,
      "step": 1110
    },
    {
      "epoch": 2.31,
      "learning_rate": 1.2710566615620215e-07,
      "logits/chosen": -2.745988368988037,
      "logits/rejected": -2.7055306434631348,
      "logps/chosen": -302.10040283203125,
      "logps/rejected": -344.9727478027344,
      "loss": 0.0153,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9112125635147095,
      "rewards/margins": 7.005129814147949,
      "rewards/rejected": -8.916341781616211,
      "step": 1120
    },
    {
      "epoch": 2.33,
      "learning_rate": 1.2327718223583462e-07,
      "logits/chosen": -2.785207748413086,
      "logits/rejected": -2.767353057861328,
      "logps/chosen": -298.5472106933594,
      "logps/rejected": -337.4159240722656,
      "loss": 0.02,
      "rewards/accuracies": 0.9906250238418579,
      "rewards/chosen": -1.8807827234268188,
      "rewards/margins": 6.743574619293213,
      "rewards/rejected": -8.624357223510742,
      "step": 1130
    },
    {
      "epoch": 2.35,
      "learning_rate": 1.1944869831546706e-07,
      "logits/chosen": -2.746328592300415,
      "logits/rejected": -2.737269401550293,
      "logps/chosen": -305.3216857910156,
      "logps/rejected": -338.6804504394531,
      "loss": 0.0153,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.9345585107803345,
      "rewards/margins": 6.828926086425781,
      "rewards/rejected": -8.763484954833984,
      "step": 1140
    },
    {
      "epoch": 2.37,
      "learning_rate": 1.1562021439509953e-07,
      "logits/chosen": -2.741163730621338,
      "logits/rejected": -2.7296128273010254,
      "logps/chosen": -302.2476501464844,
      "logps/rejected": -322.75311279296875,
      "loss": 0.02,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.8639028072357178,
      "rewards/margins": 6.6066741943359375,
      "rewards/rejected": -8.470577239990234,
      "step": 1150
    },
    {
      "epoch": 2.4,
      "learning_rate": 1.11791730474732e-07,
      "logits/chosen": -2.717231273651123,
      "logits/rejected": -2.702437400817871,
      "logps/chosen": -277.90960693359375,
      "logps/rejected": -338.58660888671875,
      "loss": 0.022,
      "rewards/accuracies": 0.9906250238418579,
      "rewards/chosen": -2.1362576484680176,
      "rewards/margins": 6.782662868499756,
      "rewards/rejected": -8.918919563293457,
      "step": 1160
    },
    {
      "epoch": 2.42,
      "learning_rate": 1.0796324655436446e-07,
      "logits/chosen": -2.663057565689087,
      "logits/rejected": -2.6754283905029297,
      "logps/chosen": -272.3413391113281,
      "logps/rejected": -350.61431884765625,
      "loss": 0.0182,
      "rewards/accuracies": 0.996874988079071,
      "rewards/chosen": -2.2835097312927246,
      "rewards/margins": 6.981085300445557,
      "rewards/rejected": -9.264595031738281,
      "step": 1170
    },
    {
      "epoch": 2.44,
      "learning_rate": 1.0413476263399694e-07,
      "logits/chosen": -2.7029130458831787,
      "logits/rejected": -2.7080864906311035,
      "logps/chosen": -305.60406494140625,
      "logps/rejected": -352.0567321777344,
      "loss": 0.0178,
      "rewards/accuracies": 0.9906250238418579,
      "rewards/chosen": -2.22029447555542,
      "rewards/margins": 7.394560813903809,
      "rewards/rejected": -9.61485481262207,
      "step": 1180
    },
    {
      "epoch": 2.46,
      "learning_rate": 1.003062787136294e-07,
      "logits/chosen": -2.637941360473633,
      "logits/rejected": -2.619065046310425,
      "logps/chosen": -301.44287109375,
      "logps/rejected": -333.2865295410156,
      "loss": 0.017,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.1894443035125732,
      "rewards/margins": 7.116552829742432,
      "rewards/rejected": -9.305997848510742,
      "step": 1190
    },
    {
      "epoch": 2.48,
      "learning_rate": 9.647779479326186e-08,
      "logits/chosen": -2.7164902687072754,
      "logits/rejected": -2.649604320526123,
      "logps/chosen": -313.3140563964844,
      "logps/rejected": -349.67327880859375,
      "loss": 0.0192,
      "rewards/accuracies": 0.9906250238418579,
      "rewards/chosen": -2.3402724266052246,
      "rewards/margins": 7.368611812591553,
      "rewards/rejected": -9.708883285522461,
      "step": 1200
    },
    {
      "epoch": 2.48,
      "eval_logits/chosen": -2.714653730392456,
      "eval_logits/rejected": -2.6779398918151855,
      "eval_logps/chosen": -317.4913024902344,
      "eval_logps/rejected": -311.97998046875,
      "eval_loss": 0.6315363645553589,
      "eval_rewards/accuracies": 0.7760000228881836,
      "eval_rewards/chosen": -3.8462328910827637,
      "eval_rewards/margins": 3.083934783935547,
      "eval_rewards/rejected": -6.9301676750183105,
      "eval_runtime": 350.524,
      "eval_samples_per_second": 5.706,
      "eval_steps_per_second": 0.357,
      "step": 1200
    },
    {
      "epoch": 2.5,
      "learning_rate": 9.264931087289433e-08,
      "logits/chosen": -2.7216286659240723,
      "logits/rejected": -2.6908698081970215,
      "logps/chosen": -293.97943115234375,
      "logps/rejected": -333.76556396484375,
      "loss": 0.0221,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.8224798440933228,
      "rewards/margins": 6.719883918762207,
      "rewards/rejected": -8.542363166809082,
      "step": 1210
    },
    {
      "epoch": 2.52,
      "learning_rate": 8.88208269525268e-08,
      "logits/chosen": -2.7265164852142334,
      "logits/rejected": -2.72367525100708,
      "logps/chosen": -285.72625732421875,
      "logps/rejected": -368.04705810546875,
      "loss": 0.0198,
      "rewards/accuracies": 0.9906250238418579,
      "rewards/chosen": -1.9060132503509521,
      "rewards/margins": 7.0853447914123535,
      "rewards/rejected": -8.991357803344727,
      "step": 1220
    },
    {
      "epoch": 2.54,
      "learning_rate": 8.499234303215926e-08,
      "logits/chosen": -2.7041897773742676,
      "logits/rejected": -2.6765353679656982,
      "logps/chosen": -294.8292236328125,
      "logps/rejected": -342.88531494140625,
      "loss": 0.0204,
      "rewards/accuracies": 0.9906250238418579,
      "rewards/chosen": -1.9344571828842163,
      "rewards/margins": 6.991697788238525,
      "rewards/rejected": -8.926154136657715,
      "step": 1230
    },
    {
      "epoch": 2.56,
      "learning_rate": 8.116385911179173e-08,
      "logits/chosen": -2.7612404823303223,
      "logits/rejected": -2.719346523284912,
      "logps/chosen": -303.2325134277344,
      "logps/rejected": -338.2950134277344,
      "loss": 0.0185,
      "rewards/accuracies": 0.9906250238418579,
      "rewards/chosen": -1.6432809829711914,
      "rewards/margins": 6.880975246429443,
      "rewards/rejected": -8.524255752563477,
      "step": 1240
    },
    {
      "epoch": 2.58,
      "learning_rate": 7.733537519142419e-08,
      "logits/chosen": -2.730088710784912,
      "logits/rejected": -2.7266387939453125,
      "logps/chosen": -298.4267883300781,
      "logps/rejected": -344.01129150390625,
      "loss": 0.0149,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.1269819736480713,
      "rewards/margins": 7.208932399749756,
      "rewards/rejected": -9.33591365814209,
      "step": 1250
    },
    {
      "epoch": 2.6,
      "learning_rate": 7.350689127105667e-08,
      "logits/chosen": -2.7158589363098145,
      "logits/rejected": -2.6916403770446777,
      "logps/chosen": -307.059326171875,
      "logps/rejected": -336.71807861328125,
      "loss": 0.016,
      "rewards/accuracies": 0.996874988079071,
      "rewards/chosen": -2.355757236480713,
      "rewards/margins": 7.1834564208984375,
      "rewards/rejected": -9.539213180541992,
      "step": 1260
    },
    {
      "epoch": 2.62,
      "learning_rate": 6.967840735068913e-08,
      "logits/chosen": -2.6886634826660156,
      "logits/rejected": -2.6580770015716553,
      "logps/chosen": -303.42169189453125,
      "logps/rejected": -342.952880859375,
      "loss": 0.0182,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.1419014930725098,
      "rewards/margins": 7.385179042816162,
      "rewards/rejected": -9.527081489562988,
      "step": 1270
    },
    {
      "epoch": 2.64,
      "learning_rate": 6.584992343032159e-08,
      "logits/chosen": -2.6542093753814697,
      "logits/rejected": -2.6138882637023926,
      "logps/chosen": -288.07281494140625,
      "logps/rejected": -357.1811828613281,
      "loss": 0.0221,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.1404623985290527,
      "rewards/margins": 7.747374534606934,
      "rewards/rejected": -9.887838363647461,
      "step": 1280
    },
    {
      "epoch": 2.66,
      "learning_rate": 6.202143950995406e-08,
      "logits/chosen": -2.72222638130188,
      "logits/rejected": -2.697075366973877,
      "logps/chosen": -301.9345703125,
      "logps/rejected": -350.2940368652344,
      "loss": 0.018,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.0181188583374023,
      "rewards/margins": 6.994577884674072,
      "rewards/rejected": -9.012696266174316,
      "step": 1290
    },
    {
      "epoch": 2.68,
      "learning_rate": 5.819295558958652e-08,
      "logits/chosen": -2.749208688735962,
      "logits/rejected": -2.7095093727111816,
      "logps/chosen": -312.4373474121094,
      "logps/rejected": -341.3883361816406,
      "loss": 0.0157,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.050492525100708,
      "rewards/margins": 7.114993095397949,
      "rewards/rejected": -9.165484428405762,
      "step": 1300
    },
    {
      "epoch": 2.68,
      "eval_logits/chosen": -2.7170982360839844,
      "eval_logits/rejected": -2.682952404022217,
      "eval_logps/chosen": -320.8811950683594,
      "eval_logps/rejected": -317.50311279296875,
      "eval_loss": 0.6585144400596619,
      "eval_rewards/accuracies": 0.7760000228881836,
      "eval_rewards/chosen": -4.127589702606201,
      "eval_rewards/margins": 3.2609987258911133,
      "eval_rewards/rejected": -7.388588905334473,
      "eval_runtime": 349.0268,
      "eval_samples_per_second": 5.73,
      "eval_steps_per_second": 0.358,
      "step": 1300
    },
    {
      "epoch": 2.71,
      "learning_rate": 5.436447166921899e-08,
      "logits/chosen": -2.729393720626831,
      "logits/rejected": -2.7150323390960693,
      "logps/chosen": -283.3392028808594,
      "logps/rejected": -338.7149658203125,
      "loss": 0.0147,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.0723025798797607,
      "rewards/margins": 7.32839822769165,
      "rewards/rejected": -9.400700569152832,
      "step": 1310
    },
    {
      "epoch": 2.73,
      "learning_rate": 5.0535987748851455e-08,
      "logits/chosen": -2.6937530040740967,
      "logits/rejected": -2.6434850692749023,
      "logps/chosen": -299.10302734375,
      "logps/rejected": -340.21295166015625,
      "loss": 0.0152,
      "rewards/accuracies": 0.9906250238418579,
      "rewards/chosen": -2.281100273132324,
      "rewards/margins": 7.282484531402588,
      "rewards/rejected": -9.56358528137207,
      "step": 1320
    },
    {
      "epoch": 2.75,
      "learning_rate": 4.670750382848391e-08,
      "logits/chosen": -2.7323451042175293,
      "logits/rejected": -2.6806094646453857,
      "logps/chosen": -287.5840148925781,
      "logps/rejected": -361.5538024902344,
      "loss": 0.0167,
      "rewards/accuracies": 0.9906250238418579,
      "rewards/chosen": -2.500239133834839,
      "rewards/margins": 7.510396480560303,
      "rewards/rejected": -10.010635375976562,
      "step": 1330
    },
    {
      "epoch": 2.77,
      "learning_rate": 4.287901990811638e-08,
      "logits/chosen": -2.73530650138855,
      "logits/rejected": -2.715400218963623,
      "logps/chosen": -298.9871826171875,
      "logps/rejected": -349.3652648925781,
      "loss": 0.0183,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.9337704181671143,
      "rewards/margins": 7.583467960357666,
      "rewards/rejected": -9.517239570617676,
      "step": 1340
    },
    {
      "epoch": 2.79,
      "learning_rate": 3.905053598774885e-08,
      "logits/chosen": -2.654972553253174,
      "logits/rejected": -2.653392791748047,
      "logps/chosen": -283.0529479980469,
      "logps/rejected": -336.0701599121094,
      "loss": 0.0127,
      "rewards/accuracies": 0.9906250238418579,
      "rewards/chosen": -2.131284475326538,
      "rewards/margins": 7.301896095275879,
      "rewards/rejected": -9.43317985534668,
      "step": 1350
    },
    {
      "epoch": 2.81,
      "learning_rate": 3.522205206738132e-08,
      "logits/chosen": -2.7224278450012207,
      "logits/rejected": -2.681025505065918,
      "logps/chosen": -302.8782653808594,
      "logps/rejected": -354.0222473144531,
      "loss": 0.0149,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.070284366607666,
      "rewards/margins": 7.328897953033447,
      "rewards/rejected": -9.399182319641113,
      "step": 1360
    },
    {
      "epoch": 2.83,
      "learning_rate": 3.1393568147013776e-08,
      "logits/chosen": -2.7233169078826904,
      "logits/rejected": -2.7085554599761963,
      "logps/chosen": -307.65399169921875,
      "logps/rejected": -372.25201416015625,
      "loss": 0.0142,
      "rewards/accuracies": 0.996874988079071,
      "rewards/chosen": -2.249838352203369,
      "rewards/margins": 7.7565412521362305,
      "rewards/rejected": -10.006380081176758,
      "step": 1370
    },
    {
      "epoch": 2.85,
      "learning_rate": 2.7565084226646246e-08,
      "logits/chosen": -2.6606523990631104,
      "logits/rejected": -2.616541862487793,
      "logps/chosen": -287.24774169921875,
      "logps/rejected": -342.07501220703125,
      "loss": 0.0179,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -2.1292500495910645,
      "rewards/margins": 7.465462684631348,
      "rewards/rejected": -9.59471321105957,
      "step": 1380
    },
    {
      "epoch": 2.87,
      "learning_rate": 2.3736600306278714e-08,
      "logits/chosen": -2.7015137672424316,
      "logits/rejected": -2.6835923194885254,
      "logps/chosen": -313.46142578125,
      "logps/rejected": -367.58331298828125,
      "loss": 0.0144,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.3003878593444824,
      "rewards/margins": 7.391711235046387,
      "rewards/rejected": -9.692098617553711,
      "step": 1390
    },
    {
      "epoch": 2.89,
      "learning_rate": 1.9908116385911178e-08,
      "logits/chosen": -2.686868667602539,
      "logits/rejected": -2.652298927307129,
      "logps/chosen": -301.98089599609375,
      "logps/rejected": -334.4278259277344,
      "loss": 0.018,
      "rewards/accuracies": 0.9906250238418579,
      "rewards/chosen": -2.0941269397735596,
      "rewards/margins": 7.590572357177734,
      "rewards/rejected": -9.684700012207031,
      "step": 1400
    },
    {
      "epoch": 2.89,
      "eval_logits/chosen": -2.7057137489318848,
      "eval_logits/rejected": -2.6702585220336914,
      "eval_logps/chosen": -322.8377990722656,
      "eval_logps/rejected": -320.3536376953125,
      "eval_loss": 0.6695687174797058,
      "eval_rewards/accuracies": 0.7739999890327454,
      "eval_rewards/chosen": -4.2899932861328125,
      "eval_rewards/margins": 3.3351902961730957,
      "eval_rewards/rejected": -7.625182628631592,
      "eval_runtime": 353.9926,
      "eval_samples_per_second": 5.65,
      "eval_steps_per_second": 0.353,
      "step": 1400
    },
    {
      "epoch": 2.91,
      "learning_rate": 1.6079632465543645e-08,
      "logits/chosen": -2.6916399002075195,
      "logits/rejected": -2.6415302753448486,
      "logps/chosen": -301.39239501953125,
      "logps/rejected": -356.01104736328125,
      "loss": 0.0189,
      "rewards/accuracies": 0.9906250238418579,
      "rewards/chosen": -2.3104169368743896,
      "rewards/margins": 7.3361358642578125,
      "rewards/rejected": -9.646553993225098,
      "step": 1410
    },
    {
      "epoch": 2.93,
      "learning_rate": 1.225114854517611e-08,
      "logits/chosen": -2.7237906455993652,
      "logits/rejected": -2.6929593086242676,
      "logps/chosen": -298.5174560546875,
      "logps/rejected": -339.3135681152344,
      "loss": 0.0171,
      "rewards/accuracies": 0.9906250238418579,
      "rewards/chosen": -2.355860471725464,
      "rewards/margins": 7.252816677093506,
      "rewards/rejected": -9.608677864074707,
      "step": 1420
    },
    {
      "epoch": 2.95,
      "learning_rate": 8.422664624808575e-09,
      "logits/chosen": -2.660881519317627,
      "logits/rejected": -2.6167492866516113,
      "logps/chosen": -297.0005798339844,
      "logps/rejected": -351.19940185546875,
      "loss": 0.0159,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.330714225769043,
      "rewards/margins": 7.545656681060791,
      "rewards/rejected": -9.876371383666992,
      "step": 1430
    },
    {
      "epoch": 2.97,
      "learning_rate": 4.594180704441042e-09,
      "logits/chosen": -2.6811654567718506,
      "logits/rejected": -2.6512813568115234,
      "logps/chosen": -295.09124755859375,
      "logps/rejected": -335.70428466796875,
      "loss": 0.0179,
      "rewards/accuracies": 0.996874988079071,
      "rewards/chosen": -2.044374704360962,
      "rewards/margins": 7.269686222076416,
      "rewards/rejected": -9.31406021118164,
      "step": 1440
    },
    {
      "epoch": 2.99,
      "learning_rate": 7.656967840735068e-10,
      "logits/chosen": -2.7477376461029053,
      "logits/rejected": -2.719231605529785,
      "logps/chosen": -300.9207763671875,
      "logps/rejected": -350.79449462890625,
      "loss": 0.0186,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.297039270401001,
      "rewards/margins": 7.076939582824707,
      "rewards/rejected": -9.373978614807129,
      "step": 1450
    },
    {
      "epoch": 3.0,
      "step": 1452,
      "total_flos": 0.0,
      "train_loss": 0.20900526624326862,
      "train_runtime": 65325.2512,
      "train_samples_per_second": 2.846,
      "train_steps_per_second": 0.022
    }
  ],
  "logging_steps": 10,
  "max_steps": 1452,
  "num_train_epochs": 3,
  "save_steps": 500,
  "total_flos": 0.0,
  "trial_name": null,
  "trial_params": null
}