{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9995320542817033,
  "eval_steps": 500,
  "global_step": 1068,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.009358914365933552,
      "grad_norm": 36.0,
      "learning_rate": 4.672897196261682e-08,
      "log_odds_chosen": 0.027360862120985985,
      "log_odds_ratio": -0.7125600576400757,
      "logits/chosen": -2.177670955657959,
      "logits/rejected": -2.1592280864715576,
      "logps/chosen": -0.49792352318763733,
      "logps/rejected": -0.5029939413070679,
      "loss": 1.2707,
      "nll_loss": 1.1897716522216797,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.04979235306382179,
      "rewards/margins": 0.0005070472252555192,
      "rewards/rejected": -0.05029939487576485,
      "step": 10
    },
    {
      "epoch": 0.018717828731867104,
      "grad_norm": 33.75,
      "learning_rate": 9.345794392523364e-08,
      "log_odds_chosen": 0.10758718103170395,
      "log_odds_ratio": -0.683829128742218,
      "logits/chosen": -2.206817865371704,
      "logits/rejected": -2.195065975189209,
      "logps/chosen": -0.5090934038162231,
      "logps/rejected": -0.5577604174613953,
      "loss": 1.3409,
      "nll_loss": 1.2595471143722534,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.050909340381622314,
      "rewards/margins": 0.0048667010851204395,
      "rewards/rejected": -0.05577604100108147,
      "step": 20
    },
    {
      "epoch": 0.028076743097800654,
      "grad_norm": 28.375,
      "learning_rate": 1.4018691588785045e-07,
      "log_odds_chosen": -0.06396888941526413,
      "log_odds_ratio": -0.7616688013076782,
      "logits/chosen": -2.1569924354553223,
      "logits/rejected": -2.136850595474243,
      "logps/chosen": -0.5089480876922607,
      "logps/rejected": -0.48712658882141113,
      "loss": 1.2937,
      "nll_loss": 1.2157827615737915,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.05089481547474861,
      "rewards/margins": -0.0021821539849042892,
      "rewards/rejected": -0.04871265962719917,
      "step": 30
    },
    {
      "epoch": 0.03743565746373421,
      "grad_norm": 33.5,
      "learning_rate": 1.8691588785046729e-07,
      "log_odds_chosen": 0.027819359675049782,
      "log_odds_ratio": -0.7156058549880981,
      "logits/chosen": -2.2016537189483643,
      "logits/rejected": -2.1811304092407227,
      "logps/chosen": -0.47956228256225586,
      "logps/rejected": -0.48842015862464905,
      "loss": 1.2882,
      "nll_loss": 1.2497555017471313,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.047956231981515884,
      "rewards/margins": 0.0008857922512106597,
      "rewards/rejected": -0.048842012882232666,
      "step": 40
    },
    {
      "epoch": 0.04679457182966776,
      "grad_norm": 26.125,
      "learning_rate": 2.336448598130841e-07,
      "log_odds_chosen": 0.07888893783092499,
      "log_odds_ratio": -0.7044392824172974,
      "logits/chosen": -2.106088161468506,
      "logits/rejected": -2.102090358734131,
      "logps/chosen": -0.4838789999485016,
      "logps/rejected": -0.5239019989967346,
      "loss": 1.2274,
      "nll_loss": 1.1539485454559326,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.04838789626955986,
      "rewards/margins": 0.004002300091087818,
      "rewards/rejected": -0.0523901991546154,
      "step": 50
    },
    {
      "epoch": 0.05615348619560131,
      "grad_norm": 29.625,
      "learning_rate": 2.803738317757009e-07,
      "log_odds_chosen": 0.0740145593881607,
      "log_odds_ratio": -0.6914735436439514,
      "logits/chosen": -2.1816258430480957,
      "logits/rejected": -2.178445339202881,
      "logps/chosen": -0.4614052176475525,
      "logps/rejected": -0.4912230968475342,
      "loss": 1.2751,
      "nll_loss": 1.1571671962738037,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.046140529215335846,
      "rewards/margins": 0.002981787081807852,
      "rewards/rejected": -0.04912231117486954,
      "step": 60
    },
    {
      "epoch": 0.06551240056153486,
      "grad_norm": 25.875,
      "learning_rate": 3.271028037383177e-07,
      "log_odds_chosen": -0.02471914514899254,
      "log_odds_ratio": -0.741605818271637,
      "logits/chosen": -2.2288427352905273,
      "logits/rejected": -2.2079877853393555,
      "logps/chosen": -0.48365911841392517,
      "logps/rejected": -0.4822389483451843,
      "loss": 1.3016,
      "nll_loss": 1.2900102138519287,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -0.048365913331508636,
      "rewards/margins": -0.00014201310114003718,
      "rewards/rejected": -0.04822390154004097,
      "step": 70
    },
    {
      "epoch": 0.07487131492746842,
      "grad_norm": 26.625,
      "learning_rate": 3.7383177570093457e-07,
      "log_odds_chosen": 0.11039517819881439,
      "log_odds_ratio": -0.6913446187973022,
      "logits/chosen": -2.2124581336975098,
      "logits/rejected": -2.194650411605835,
      "logps/chosen": -0.4804244935512543,
      "logps/rejected": -0.5464648008346558,
      "loss": 1.1872,
      "nll_loss": 1.0913323163986206,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.04804245010018349,
      "rewards/margins": 0.006604035850614309,
      "rewards/rejected": -0.054646484553813934,
      "step": 80
    },
    {
      "epoch": 0.08423022929340196,
      "grad_norm": 21.75,
      "learning_rate": 4.205607476635514e-07,
      "log_odds_chosen": -0.002868836745619774,
      "log_odds_ratio": -0.7235916256904602,
      "logits/chosen": -2.246835470199585,
      "logits/rejected": -2.2278454303741455,
      "logps/chosen": -0.4966287612915039,
      "logps/rejected": -0.49866318702697754,
      "loss": 1.2336,
      "nll_loss": 1.1712753772735596,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.04966288059949875,
      "rewards/margins": 0.00020344238146208227,
      "rewards/rejected": -0.04986632615327835,
      "step": 90
    },
    {
      "epoch": 0.09358914365933552,
      "grad_norm": 21.625,
      "learning_rate": 4.672897196261682e-07,
      "log_odds_chosen": 0.006027285009622574,
      "log_odds_ratio": -0.7175697088241577,
      "logits/chosen": -2.194366216659546,
      "logits/rejected": -2.1875548362731934,
      "logps/chosen": -0.4724898934364319,
      "logps/rejected": -0.46772247552871704,
      "loss": 1.1875,
      "nll_loss": 1.1505517959594727,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.04724898934364319,
      "rewards/margins": -0.00047674038796685636,
      "rewards/rejected": -0.046772249042987823,
      "step": 100
    },
    {
      "epoch": 0.10294805802526907,
      "grad_norm": 16.625,
      "learning_rate": 4.999879772999679e-07,
      "log_odds_chosen": 0.061622392386198044,
      "log_odds_ratio": -0.6916904449462891,
      "logits/chosen": -2.169373035430908,
      "logits/rejected": -2.1729514598846436,
      "logps/chosen": -0.4589506685733795,
      "logps/rejected": -0.4915087819099426,
      "loss": 1.1308,
      "nll_loss": 1.0613527297973633,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04589506983757019,
      "rewards/margins": 0.003255808260291815,
      "rewards/rejected": -0.04915087670087814,
      "step": 110
    },
    {
      "epoch": 0.11230697239120262,
      "grad_norm": 15.75,
      "learning_rate": 4.997742725777528e-07,
      "log_odds_chosen": 0.06405100971460342,
      "log_odds_ratio": -0.6904009580612183,
      "logits/chosen": -2.2223846912384033,
      "logits/rejected": -2.203917980194092,
      "logps/chosen": -0.4754715859889984,
      "logps/rejected": -0.5003005862236023,
      "loss": 1.0866,
      "nll_loss": 1.0029579401016235,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.04754715785384178,
      "rewards/margins": 0.0024829022586345673,
      "rewards/rejected": -0.05003006383776665,
      "step": 120
    },
    {
      "epoch": 0.12166588675713617,
      "grad_norm": 14.5625,
      "learning_rate": 4.992936596071119e-07,
      "log_odds_chosen": 0.08829396218061447,
      "log_odds_ratio": -0.6891029477119446,
      "logits/chosen": -2.222391128540039,
      "logits/rejected": -2.2042291164398193,
      "logps/chosen": -0.46681737899780273,
      "logps/rejected": -0.5088747143745422,
      "loss": 1.1225,
      "nll_loss": 1.0937663316726685,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.04668173938989639,
      "rewards/margins": 0.004205727018415928,
      "rewards/rejected": -0.05088747292757034,
      "step": 130
    },
    {
      "epoch": 0.13102480112306972,
      "grad_norm": 15.3125,
      "learning_rate": 4.985466519700276e-07,
      "log_odds_chosen": 0.06562065333127975,
      "log_odds_ratio": -0.6990588903427124,
      "logits/chosen": -2.2474286556243896,
      "logits/rejected": -2.2379136085510254,
      "logps/chosen": -0.47886237502098083,
      "logps/rejected": -0.5128284096717834,
      "loss": 1.1017,
      "nll_loss": 1.040492296218872,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.047886233776807785,
      "rewards/margins": 0.0033966104965656996,
      "rewards/rejected": -0.051282841712236404,
      "step": 140
    },
    {
      "epoch": 0.14038371548900327,
      "grad_norm": 15.625,
      "learning_rate": 4.975340479172484e-07,
      "log_odds_chosen": 0.03810061141848564,
      "log_odds_ratio": -0.7092625498771667,
      "logits/chosen": -2.2372212409973145,
      "logits/rejected": -2.206622362136841,
      "logps/chosen": -0.4353889524936676,
      "logps/rejected": -0.44863614439964294,
      "loss": 0.9927,
      "nll_loss": 0.9188264608383179,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.0435388945043087,
      "rewards/margins": 0.0013247155584394932,
      "rewards/rejected": -0.04486361891031265,
      "step": 150
    },
    {
      "epoch": 0.14974262985493683,
      "grad_norm": 16.0,
      "learning_rate": 4.962569295152789e-07,
      "log_odds_chosen": 0.1309119015932083,
      "log_odds_ratio": -0.666577935218811,
      "logits/chosen": -2.1617026329040527,
      "logits/rejected": -2.1651737689971924,
      "logps/chosen": -0.46459126472473145,
      "logps/rejected": -0.5068212747573853,
      "loss": 1.1044,
      "nll_loss": 1.0174779891967773,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.046459127217531204,
      "rewards/margins": 0.004223003052175045,
      "rewards/rejected": -0.050682127475738525,
      "step": 160
    },
    {
      "epoch": 0.1591015442208704,
      "grad_norm": 16.0,
      "learning_rate": 4.947166614900862e-07,
      "log_odds_chosen": 0.05984800308942795,
      "log_odds_ratio": -0.6976341009140015,
      "logits/chosen": -2.2195987701416016,
      "logits/rejected": -2.2141668796539307,
      "logps/chosen": -0.4686553478240967,
      "logps/rejected": -0.4932120740413666,
      "loss": 0.9915,
      "nll_loss": 0.9667671918869019,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.04686553403735161,
      "rewards/margins": 0.0024556724820286036,
      "rewards/rejected": -0.04932120442390442,
      "step": 170
    },
    {
      "epoch": 0.16846045858680392,
      "grad_norm": 15.75,
      "learning_rate": 4.929148897687566e-07,
      "log_odds_chosen": 0.2192097008228302,
      "log_odds_ratio": -0.6804882884025574,
      "logits/chosen": -2.2159080505371094,
      "logits/rejected": -2.1967945098876953,
      "logps/chosen": -0.45083823800086975,
      "logps/rejected": -0.5973262786865234,
      "loss": 0.9957,
      "nll_loss": 0.9682543873786926,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.04508382827043533,
      "rewards/margins": 0.0146488007158041,
      "rewards/rejected": -0.059732627123594284,
      "step": 180
    },
    {
      "epoch": 0.17781937295273748,
      "grad_norm": 13.5625,
      "learning_rate": 4.908535397206616e-07,
      "log_odds_chosen": 0.1358589231967926,
      "log_odds_ratio": -0.6608278751373291,
      "logits/chosen": -2.1806671619415283,
      "logits/rejected": -2.1832306385040283,
      "logps/chosen": -0.42772555351257324,
      "logps/rejected": -0.4777906835079193,
      "loss": 1.0472,
      "nll_loss": 0.9567255973815918,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0427725613117218,
      "rewards/margins": 0.005006510764360428,
      "rewards/rejected": -0.04777907580137253,
      "step": 190
    },
    {
      "epoch": 0.18717828731867103,
      "grad_norm": 13.9375,
      "learning_rate": 4.885348141000122e-07,
      "log_odds_chosen": 0.11417017132043839,
      "log_odds_ratio": -0.6762548089027405,
      "logits/chosen": -2.184654712677002,
      "logits/rejected": -2.170086145401001,
      "logps/chosen": -0.4621838629245758,
      "logps/rejected": -0.518031120300293,
      "loss": 1.0153,
      "nll_loss": 0.9638531804084778,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.0462183877825737,
      "rewards/margins": 0.00558472191914916,
      "rewards/rejected": -0.0518031120300293,
      "step": 200
    },
    {
      "epoch": 0.1965372016846046,
      "grad_norm": 14.3125,
      "learning_rate": 4.859611906920007e-07,
      "log_odds_chosen": 0.18621578812599182,
      "log_odds_ratio": -0.6375190019607544,
      "logits/chosen": -2.171351909637451,
      "logits/rejected": -2.1605920791625977,
      "logps/chosen": -0.43598541617393494,
      "logps/rejected": -0.5046157240867615,
      "loss": 1.0046,
      "nll_loss": 0.995398998260498,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.04359854385256767,
      "rewards/margins": 0.006863036658614874,
      "rewards/rejected": -0.050461579114198685,
      "step": 210
    },
    {
      "epoch": 0.20589611605053815,
      "grad_norm": 14.9375,
      "learning_rate": 4.831354196650446e-07,
      "log_odds_chosen": 0.1494245082139969,
      "log_odds_ratio": -0.6539293527603149,
      "logits/chosen": -2.2000341415405273,
      "logits/rejected": -2.1807079315185547,
      "logps/chosen": -0.4382123053073883,
      "logps/rejected": -0.4922669529914856,
      "loss": 1.0051,
      "nll_loss": 0.9585235714912415,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.04382122680544853,
      "rewards/margins": 0.005405460484325886,
      "rewards/rejected": -0.04922669380903244,
      "step": 220
    },
    {
      "epoch": 0.21525503041647168,
      "grad_norm": 14.25,
      "learning_rate": 4.800605206319624e-07,
      "log_odds_chosen": 0.19924063980579376,
      "log_odds_ratio": -0.6384560465812683,
      "logits/chosen": -2.1733033657073975,
      "logits/rejected": -2.1657328605651855,
      "logps/chosen": -0.40226203203201294,
      "logps/rejected": -0.4800339639186859,
      "loss": 0.9612,
      "nll_loss": 0.872852623462677,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04022620618343353,
      "rewards/margins": 0.007777197752147913,
      "rewards/rejected": -0.04800339788198471,
      "step": 230
    },
    {
      "epoch": 0.22461394478240523,
      "grad_norm": 14.4375,
      "learning_rate": 4.767397794232225e-07,
      "log_odds_chosen": 0.14606961607933044,
      "log_odds_ratio": -0.652389645576477,
      "logits/chosen": -2.217345714569092,
      "logits/rejected": -2.193671703338623,
      "logps/chosen": -0.4080810546875,
      "logps/rejected": -0.459284245967865,
      "loss": 0.9276,
      "nll_loss": 0.856499195098877,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04080810770392418,
      "rewards/margins": 0.005120322573930025,
      "rewards/rejected": -0.045928433537483215,
      "step": 240
    },
    {
      "epoch": 0.2339728591483388,
      "grad_norm": 14.0,
      "learning_rate": 4.731767445757111e-07,
      "log_odds_chosen": 0.1912066489458084,
      "log_odds_ratio": -0.6586419343948364,
      "logits/chosen": -2.1863677501678467,
      "logits/rejected": -2.1660654544830322,
      "logps/chosen": -0.4087969660758972,
      "logps/rejected": -0.5012443661689758,
      "loss": 1.0828,
      "nll_loss": 0.9812431335449219,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.04087969660758972,
      "rewards/margins": 0.009244738146662712,
      "rewards/rejected": -0.05012443661689758,
      "step": 250
    },
    {
      "epoch": 0.24333177351427235,
      "grad_norm": 13.8125,
      "learning_rate": 4.6937522354077397e-07,
      "log_odds_chosen": 0.07517801970243454,
      "log_odds_ratio": -0.6879104375839233,
      "logits/chosen": -2.2382235527038574,
      "logits/rejected": -2.2303104400634766,
      "logps/chosen": -0.4272983968257904,
      "logps/rejected": -0.45664095878601074,
      "loss": 1.0369,
      "nll_loss": 0.9628226161003113,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.04272983968257904,
      "rewards/margins": 0.0029342600610107183,
      "rewards/rejected": -0.04566409811377525,
      "step": 260
    },
    {
      "epoch": 0.2526906878802059,
      "grad_norm": 14.375,
      "learning_rate": 4.6533927861558166e-07,
      "log_odds_chosen": 0.20296251773834229,
      "log_odds_ratio": -0.6265792846679688,
      "logits/chosen": -2.2084856033325195,
      "logits/rejected": -2.191228151321411,
      "logps/chosen": -0.4132423400878906,
      "logps/rejected": -0.4871252179145813,
      "loss": 1.0579,
      "nll_loss": 0.9442154765129089,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.04132423549890518,
      "rewards/margins": 0.007388289086520672,
      "rewards/rejected": -0.04871252924203873,
      "step": 270
    },
    {
      "epoch": 0.26204960224613943,
      "grad_norm": 17.375,
      "learning_rate": 4.6107322260216787e-07,
      "log_odds_chosen": 0.19398412108421326,
      "log_odds_ratio": -0.6390923261642456,
      "logits/chosen": -2.160511016845703,
      "logits/rejected": -2.1479556560516357,
      "logps/chosen": -0.3960379362106323,
      "logps/rejected": -0.47004491090774536,
      "loss": 0.9542,
      "nll_loss": 0.8770105242729187,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.03960379213094711,
      "rewards/margins": 0.007400700356811285,
      "rewards/rejected": -0.047004491090774536,
      "step": 280
    },
    {
      "epoch": 0.271408516612073,
      "grad_norm": 15.25,
      "learning_rate": 4.565816141987782e-07,
      "log_odds_chosen": 0.27176782488822937,
      "log_odds_ratio": -0.6160404682159424,
      "logits/chosen": -2.181396961212158,
      "logits/rejected": -2.1620774269104004,
      "logps/chosen": -0.3963298499584198,
      "logps/rejected": -0.5069735646247864,
      "loss": 0.9935,
      "nll_loss": 0.932470977306366,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.03963298350572586,
      "rewards/margins": 0.01106436736881733,
      "rewards/rejected": -0.05069734901189804,
      "step": 290
    },
    {
      "epoch": 0.28076743097800655,
      "grad_norm": 16.125,
      "learning_rate": 4.518692531284555e-07,
      "log_odds_chosen": 0.1647796332836151,
      "log_odds_ratio": -0.6567476391792297,
      "logits/chosen": -2.1729674339294434,
      "logits/rejected": -2.177335023880005,
      "logps/chosen": -0.43337172269821167,
      "logps/rejected": -0.5012940168380737,
      "loss": 1.0111,
      "nll_loss": 0.9405239820480347,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.043337173759937286,
      "rewards/margins": 0.006792229600250721,
      "rewards/rejected": -0.05012940242886543,
      "step": 300
    },
    {
      "epoch": 0.2901263453439401,
      "grad_norm": 15.875,
      "learning_rate": 4.469411750100657e-07,
      "log_odds_chosen": 0.20056188106536865,
      "log_odds_ratio": -0.6289246678352356,
      "logits/chosen": -2.1089398860931396,
      "logits/rejected": -2.097630023956299,
      "logps/chosen": -0.419585645198822,
      "logps/rejected": -0.4954373240470886,
      "loss": 1.0291,
      "nll_loss": 0.9756892323493958,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04195856675505638,
      "rewards/margins": 0.007585165090858936,
      "rewards/rejected": -0.04954373463988304,
      "step": 310
    },
    {
      "epoch": 0.29948525970987366,
      "grad_norm": 14.9375,
      "learning_rate": 4.418026459772465e-07,
      "log_odds_chosen": 0.2775348424911499,
      "log_odds_ratio": -0.6147114038467407,
      "logits/chosen": -2.151935577392578,
      "logits/rejected": -2.149029016494751,
      "logps/chosen": -0.39582258462905884,
      "logps/rejected": -0.5231629610061646,
      "loss": 0.9973,
      "nll_loss": 0.8973016738891602,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.039582256227731705,
      "rewards/margins": 0.012734037823975086,
      "rewards/rejected": -0.052316296845674515,
      "step": 320
    },
    {
      "epoch": 0.3088441740758072,
      "grad_norm": 13.125,
      "learning_rate": 4.3645915705102875e-07,
      "log_odds_chosen": 0.1660947948694229,
      "log_odds_ratio": -0.650874674320221,
      "logits/chosen": -2.193951368331909,
      "logits/rejected": -2.201650619506836,
      "logps/chosen": -0.4254066050052643,
      "logps/rejected": -0.4931983947753906,
      "loss": 1.0248,
      "nll_loss": 0.9826960563659668,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.04254066199064255,
      "rewards/margins": 0.006779176648706198,
      "rewards/rejected": -0.04931983724236488,
      "step": 330
    },
    {
      "epoch": 0.3182030884417408,
      "grad_norm": 16.875,
      "learning_rate": 4.3091641827214255e-07,
      "log_odds_chosen": 0.23508663475513458,
      "log_odds_ratio": -0.6352331042289734,
      "logits/chosen": -2.1884193420410156,
      "logits/rejected": -2.194756269454956,
      "logps/chosen": -0.3882560729980469,
      "logps/rejected": -0.4830912947654724,
      "loss": 0.9394,
      "nll_loss": 0.8814057111740112,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.03882560878992081,
      "rewards/margins": 0.0094835190102458,
      "rewards/rejected": -0.04830912500619888,
      "step": 340
    },
    {
      "epoch": 0.32756200280767434,
      "grad_norm": 14.375,
      "learning_rate": 4.251803525992806e-07,
      "log_odds_chosen": 0.16706052422523499,
      "log_odds_ratio": -0.6470171213150024,
      "logits/chosen": -2.176159620285034,
      "logits/rejected": -2.174161911010742,
      "logps/chosen": -0.4145967960357666,
      "logps/rejected": -0.486347496509552,
      "loss": 0.9638,
      "nll_loss": 0.9122942686080933,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.04145968705415726,
      "rewards/margins": 0.00717506418004632,
      "rewards/rejected": -0.04863474518060684,
      "step": 350
    },
    {
      "epoch": 0.33692091717360784,
      "grad_norm": 16.5,
      "learning_rate": 4.192570895798369e-07,
      "log_odds_chosen": 0.23265504837036133,
      "log_odds_ratio": -0.6187289357185364,
      "logits/chosen": -2.1640992164611816,
      "logits/rejected": -2.168822765350342,
      "logps/chosen": -0.40140947699546814,
      "logps/rejected": -0.4813784658908844,
      "loss": 0.9937,
      "nll_loss": 0.8817814588546753,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04014094918966293,
      "rewards/margins": 0.007996905595064163,
      "rewards/rejected": -0.0481378547847271,
      "step": 360
    },
    {
      "epoch": 0.3462798315395414,
      "grad_norm": 16.75,
      "learning_rate": 4.1315295879988603e-07,
      "log_odds_chosen": 0.1851670742034912,
      "log_odds_ratio": -0.641275942325592,
      "logits/chosen": -2.17275333404541,
      "logits/rejected": -2.1673264503479004,
      "logps/chosen": -0.40128177404403687,
      "logps/rejected": -0.4769595265388489,
      "loss": 0.9637,
      "nll_loss": 0.9042321443557739,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.040128178894519806,
      "rewards/margins": 0.007567767985165119,
      "rewards/rejected": -0.04769594967365265,
      "step": 370
    },
    {
      "epoch": 0.35563874590547495,
      "grad_norm": 15.25,
      "learning_rate": 4.06874483120401e-07,
      "log_odds_chosen": 0.22965331375598907,
      "log_odds_ratio": -0.6257021427154541,
      "logits/chosen": -2.2078561782836914,
      "logits/rejected": -2.190913677215576,
      "logps/chosen": -0.39686036109924316,
      "logps/rejected": -0.4888846278190613,
      "loss": 1.0607,
      "nll_loss": 0.9836522340774536,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.03968603163957596,
      "rewards/margins": 0.009202429093420506,
      "rewards/rejected": -0.04888846352696419,
      "step": 380
    },
    {
      "epoch": 0.3649976602714085,
      "grad_norm": 15.25,
      "learning_rate": 4.00428371706938e-07,
      "log_odds_chosen": 0.2075185775756836,
      "log_odds_ratio": -0.624189019203186,
      "logits/chosen": -2.219852924346924,
      "logits/rejected": -2.1979775428771973,
      "logps/chosen": -0.4044666886329651,
      "logps/rejected": -0.4880141317844391,
      "loss": 0.9671,
      "nll_loss": 0.9435387849807739,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.04044666886329651,
      "rewards/margins": 0.008354747667908669,
      "rewards/rejected": -0.04880141466856003,
      "step": 390
    },
    {
      "epoch": 0.37435657463734207,
      "grad_norm": 16.25,
      "learning_rate": 3.9382151286023644e-07,
      "log_odds_chosen": 0.18440015614032745,
      "log_odds_ratio": -0.6386502981185913,
      "logits/chosen": -2.2246038913726807,
      "logits/rejected": -2.222909450531006,
      "logps/chosen": -0.4041818082332611,
      "logps/rejected": -0.47141432762145996,
      "loss": 0.915,
      "nll_loss": 0.8205062747001648,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.04041817784309387,
      "rewards/margins": 0.006723250262439251,
      "rewards/rejected": -0.0471414290368557,
      "step": 400
    },
    {
      "epoch": 0.3837154890032756,
      "grad_norm": 16.0,
      "learning_rate": 3.8706096665539617e-07,
      "log_odds_chosen": 0.27819472551345825,
      "log_odds_ratio": -0.628924548625946,
      "logits/chosen": -2.2116692066192627,
      "logits/rejected": -2.1926889419555664,
      "logps/chosen": -0.39858338236808777,
      "logps/rejected": -0.5448154211044312,
      "loss": 1.0069,
      "nll_loss": 0.9505244493484497,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.039858341217041016,
      "rewards/margins": 0.014623197726905346,
      "rewards/rejected": -0.054481543600559235,
      "step": 410
    },
    {
      "epoch": 0.3930744033692092,
      "grad_norm": 14.125,
      "learning_rate": 3.801539573974959e-07,
      "log_odds_chosen": 0.3876788318157196,
      "log_odds_ratio": -0.6021304130554199,
      "logits/chosen": -2.2393152713775635,
      "logits/rejected": -2.2127761840820312,
      "logps/chosen": -0.3716931939125061,
      "logps/rejected": -0.5833589434623718,
      "loss": 0.9809,
      "nll_loss": 0.9177610278129578,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.03716932237148285,
      "rewards/margins": 0.021166574209928513,
      "rewards/rejected": -0.05833590030670166,
      "step": 420
    },
    {
      "epoch": 0.40243331773514274,
      "grad_norm": 14.0,
      "learning_rate": 3.7310786590171683e-07,
      "log_odds_chosen": 0.18535861372947693,
      "log_odds_ratio": -0.6408358812332153,
      "logits/chosen": -2.23064923286438,
      "logits/rejected": -2.2105820178985596,
      "logps/chosen": -0.40638476610183716,
      "logps/rejected": -0.4804498255252838,
      "loss": 1.0212,
      "nll_loss": 0.9341148138046265,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.04063846915960312,
      "rewards/margins": 0.007406515069305897,
      "rewards/rejected": -0.04804498702287674,
      "step": 430
    },
    {
      "epoch": 0.4117922321010763,
      "grad_norm": 23.625,
      "learning_rate": 3.659302216062191e-07,
      "log_odds_chosen": 0.2502005398273468,
      "log_odds_ratio": -0.6098745465278625,
      "logits/chosen": -2.180509567260742,
      "logits/rejected": -2.174722671508789,
      "logps/chosen": -0.39229241013526917,
      "logps/rejected": -0.48466572165489197,
      "loss": 0.9713,
      "nll_loss": 0.9427000284194946,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.03922923654317856,
      "rewards/margins": 0.009237336926162243,
      "rewards/rejected": -0.048466578125953674,
      "step": 440
    },
    {
      "epoch": 0.42115114646700985,
      "grad_norm": 13.5625,
      "learning_rate": 3.586286945262007e-07,
      "log_odds_chosen": 0.2504832148551941,
      "log_odds_ratio": -0.6089121103286743,
      "logits/chosen": -2.0650932788848877,
      "logits/rejected": -2.0757880210876465,
      "logps/chosen": -0.3794267177581787,
      "logps/rejected": -0.4683166444301605,
      "loss": 0.9343,
      "nll_loss": 0.8940143585205078,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.03794267028570175,
      "rewards/margins": 0.00888899713754654,
      "rewards/rejected": -0.04683166742324829,
      "step": 450
    },
    {
      "epoch": 0.43051006083294335,
      "grad_norm": 15.25,
      "learning_rate": 3.512110870577357e-07,
      "log_odds_chosen": 0.23200464248657227,
      "log_odds_ratio": -0.6252235770225525,
      "logits/chosen": -2.1500890254974365,
      "logits/rejected": -2.1427481174468994,
      "logps/chosen": -0.39273735880851746,
      "logps/rejected": -0.4773070812225342,
      "loss": 1.0269,
      "nll_loss": 0.9092708826065063,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.039273738861083984,
      "rewards/margins": 0.008456969633698463,
      "rewards/rejected": -0.0477307066321373,
      "step": 460
    },
    {
      "epoch": 0.4398689751988769,
      "grad_norm": 11.9375,
      "learning_rate": 3.436853256401506e-07,
      "log_odds_chosen": 0.23003438115119934,
      "log_odds_ratio": -0.6194323301315308,
      "logits/chosen": -2.1569676399230957,
      "logits/rejected": -2.1662819385528564,
      "logps/chosen": -0.3882262110710144,
      "logps/rejected": -0.4661303162574768,
      "loss": 0.9625,
      "nll_loss": 0.8705393671989441,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.03882262483239174,
      "rewards/margins": 0.007790408097207546,
      "rewards/rejected": -0.04661303386092186,
      "step": 470
    },
    {
      "epoch": 0.44922788956481047,
      "grad_norm": 15.5,
      "learning_rate": 3.3605945228584866e-07,
      "log_odds_chosen": 0.1848507523536682,
      "log_odds_ratio": -0.637083113193512,
      "logits/chosen": -2.208573579788208,
      "logits/rejected": -2.1971964836120605,
      "logps/chosen": -0.4037590026855469,
      "logps/rejected": -0.473149836063385,
      "loss": 0.9891,
      "nll_loss": 0.9415995478630066,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.04037589952349663,
      "rewards/margins": 0.006939084734767675,
      "rewards/rejected": -0.04731498286128044,
      "step": 480
    },
    {
      "epoch": 0.458586803930744,
      "grad_norm": 13.0625,
      "learning_rate": 3.283416159866321e-07,
      "log_odds_chosen": 0.24159303307533264,
      "log_odds_ratio": -0.6120076179504395,
      "logits/chosen": -2.184213161468506,
      "logits/rejected": -2.1987740993499756,
      "logps/chosen": -0.39372795820236206,
      "logps/rejected": -0.48281997442245483,
      "loss": 0.9589,
      "nll_loss": 0.8757813572883606,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.039372798055410385,
      "rewards/margins": 0.00890920776873827,
      "rewards/rejected": -0.04828200489282608,
      "step": 490
    },
    {
      "epoch": 0.4679457182966776,
      "grad_norm": 16.375,
      "learning_rate": 3.205400640057081e-07,
      "log_odds_chosen": 0.133102148771286,
      "log_odds_ratio": -0.6767079830169678,
      "logits/chosen": -2.183706760406494,
      "logits/rejected": -2.1818044185638428,
      "logps/chosen": -0.4044373631477356,
      "logps/rejected": -0.4518662095069885,
      "loss": 1.0155,
      "nll_loss": 0.9624871015548706,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.04044373333454132,
      "rewards/margins": 0.004742881283164024,
      "rewards/rejected": -0.045186616480350494,
      "step": 500
    },
    {
      "epoch": 0.47730463266261114,
      "grad_norm": 15.5625,
      "learning_rate": 3.126631330646801e-07,
      "log_odds_chosen": 0.2868886888027191,
      "log_odds_ratio": -0.6187784671783447,
      "logits/chosen": -2.2198147773742676,
      "logits/rejected": -2.224412441253662,
      "logps/chosen": -0.3866862952709198,
      "logps/rejected": -0.5035656094551086,
      "loss": 0.9648,
      "nll_loss": 0.8982638120651245,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.03866862505674362,
      "rewards/margins": 0.01168792974203825,
      "rewards/rejected": -0.050356559455394745,
      "step": 510
    },
    {
      "epoch": 0.4866635470285447,
      "grad_norm": 14.125,
      "learning_rate": 3.0471924043494595e-07,
      "log_odds_chosen": 0.24677081406116486,
      "log_odds_ratio": -0.618061363697052,
      "logits/chosen": -2.144286870956421,
      "logits/rejected": -2.1702799797058105,
      "logps/chosen": -0.3944970667362213,
      "logps/rejected": -0.49615901708602905,
      "loss": 0.9674,
      "nll_loss": 0.8812870979309082,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.03944971039891243,
      "rewards/margins": 0.010166196152567863,
      "rewards/rejected": -0.049615900963544846,
      "step": 520
    },
    {
      "epoch": 0.49602246139447825,
      "grad_norm": 13.0,
      "learning_rate": 2.967168749430191e-07,
      "log_odds_chosen": 0.25857609510421753,
      "log_odds_ratio": -0.6192947030067444,
      "logits/chosen": -2.184696674346924,
      "logits/rejected": -2.1700081825256348,
      "logps/chosen": -0.3798629939556122,
      "logps/rejected": -0.49370065331459045,
      "loss": 0.9305,
      "nll_loss": 0.8286808133125305,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.03798630088567734,
      "rewards/margins": 0.011383764445781708,
      "rewards/rejected": -0.04937007278203964,
      "step": 530
    },
    {
      "epoch": 0.5053813757604118,
      "grad_norm": 15.3125,
      "learning_rate": 2.8866458789938774e-07,
      "log_odds_chosen": 0.3041766285896301,
      "log_odds_ratio": -0.6016188859939575,
      "logits/chosen": -2.1878724098205566,
      "logits/rejected": -2.197441577911377,
      "logps/chosen": -0.3805080056190491,
      "logps/rejected": -0.5073161721229553,
      "loss": 0.9574,
      "nll_loss": 0.8998135328292847,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.038050804287195206,
      "rewards/margins": 0.012680811807513237,
      "rewards/rejected": -0.050731610506772995,
      "step": 540
    },
    {
      "epoch": 0.5147402901263454,
      "grad_norm": 13.875,
      "learning_rate": 2.8057098396060196e-07,
      "log_odds_chosen": 0.13988998532295227,
      "log_odds_ratio": -0.665267825126648,
      "logits/chosen": -2.1989827156066895,
      "logits/rejected": -2.184476852416992,
      "logps/chosen": -0.3995954096317291,
      "logps/rejected": -0.4471829831600189,
      "loss": 0.9386,
      "nll_loss": 0.8581770062446594,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.039959538727998734,
      "rewards/margins": 0.004758755676448345,
      "rewards/rejected": -0.04471829533576965,
      "step": 550
    },
    {
      "epoch": 0.5240992044922789,
      "grad_norm": 16.375,
      "learning_rate": 2.724447119343572e-07,
      "log_odds_chosen": 0.209095761179924,
      "log_odds_ratio": -0.6507378816604614,
      "logits/chosen": -2.19320011138916,
      "logits/rejected": -2.1836113929748535,
      "logps/chosen": -0.39416733384132385,
      "logps/rejected": -0.4946066439151764,
      "loss": 0.9738,
      "nll_loss": 0.9462787508964539,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.039416730403900146,
      "rewards/margins": 0.010043924674391747,
      "rewards/rejected": -0.04946065694093704,
      "step": 560
    },
    {
      "epoch": 0.5334581188582125,
      "grad_norm": 14.5625,
      "learning_rate": 2.642944555373965e-07,
      "log_odds_chosen": 0.20115435123443604,
      "log_odds_ratio": -0.6333853006362915,
      "logits/chosen": -2.2073943614959717,
      "logits/rejected": -2.212639093399048,
      "logps/chosen": -0.39787840843200684,
      "logps/rejected": -0.47424665093421936,
      "loss": 0.9612,
      "nll_loss": 0.9760659337043762,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.03978784382343292,
      "rewards/margins": 0.007636821363121271,
      "rewards/rejected": -0.04742466285824776,
      "step": 570
    },
    {
      "epoch": 0.542817033224146,
      "grad_norm": 15.625,
      "learning_rate": 2.561289241161095e-07,
      "log_odds_chosen": 0.2170640528202057,
      "log_odds_ratio": -0.6275693774223328,
      "logits/chosen": -2.196486234664917,
      "logits/rejected": -2.1990580558776855,
      "logps/chosen": -0.3897285461425781,
      "logps/rejected": -0.4585072100162506,
      "loss": 1.0314,
      "nll_loss": 0.9672943949699402,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.03897285833954811,
      "rewards/margins": 0.006877865642309189,
      "rewards/rejected": -0.0458507239818573,
      "step": 580
    },
    {
      "epoch": 0.5521759475900796,
      "grad_norm": 14.0625,
      "learning_rate": 2.479568433397441e-07,
      "log_odds_chosen": 0.19771938025951385,
      "log_odds_ratio": -0.6418076753616333,
      "logits/chosen": -2.196673631668091,
      "logits/rejected": -2.1924281120300293,
      "logps/chosen": -0.3822599947452545,
      "logps/rejected": -0.4602930545806885,
      "loss": 0.9235,
      "nll_loss": 0.8624385595321655,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.03822600096464157,
      "rewards/margins": 0.007803303189575672,
      "rewards/rejected": -0.04602930322289467,
      "step": 590
    },
    {
      "epoch": 0.5615348619560131,
      "grad_norm": 16.375,
      "learning_rate": 2.3978694587617473e-07,
      "log_odds_chosen": 0.24181723594665527,
      "log_odds_ratio": -0.6263023614883423,
      "logits/chosen": -2.1748032569885254,
      "logits/rejected": -2.196155309677124,
      "logps/chosen": -0.3913511335849762,
      "logps/rejected": -0.47442513704299927,
      "loss": 0.9472,
      "nll_loss": 0.8652606010437012,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.03913511335849762,
      "rewards/margins": 0.008307400159537792,
      "rewards/rejected": -0.047442514449357986,
      "step": 600
    },
    {
      "epoch": 0.5708937763219466,
      "grad_norm": 15.6875,
      "learning_rate": 2.3162796206019266e-07,
      "log_odds_chosen": 0.25003498792648315,
      "log_odds_ratio": -0.6166588068008423,
      "logits/chosen": -2.122239351272583,
      "logits/rejected": -2.120157241821289,
      "logps/chosen": -0.39171507954597473,
      "logps/rejected": -0.4818199574947357,
      "loss": 0.9285,
      "nll_loss": 0.812226414680481,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.039171505719423294,
      "rewards/margins": 0.009010489098727703,
      "rewards/rejected": -0.04818199574947357,
      "step": 610
    },
    {
      "epoch": 0.5802526906878802,
      "grad_norm": 17.25,
      "learning_rate": 2.2348861056428868e-07,
      "log_odds_chosen": 0.23789414763450623,
      "log_odds_ratio": -0.6213194131851196,
      "logits/chosen": -2.2460482120513916,
      "logits/rejected": -2.226989269256592,
      "logps/chosen": -0.3984091281890869,
      "logps/rejected": -0.5115697979927063,
      "loss": 0.9339,
      "nll_loss": 0.8749570846557617,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.03984091058373451,
      "rewards/margins": 0.011316070333123207,
      "rewards/rejected": -0.05115697905421257,
      "step": 620
    },
    {
      "epoch": 0.5896116050538137,
      "grad_norm": 17.0,
      "learning_rate": 2.153775890818989e-07,
      "log_odds_chosen": 0.22108840942382812,
      "log_odds_ratio": -0.6147779822349548,
      "logits/chosen": -2.239152431488037,
      "logits/rejected": -2.2272543907165527,
      "logps/chosen": -0.39232850074768066,
      "logps/rejected": -0.46175652742385864,
      "loss": 0.9649,
      "nll_loss": 0.9323352575302124,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.039232850074768066,
      "rewards/margins": 0.006942803505808115,
      "rewards/rejected": -0.046175651252269745,
      "step": 630
    },
    {
      "epoch": 0.5989705194197473,
      "grad_norm": 15.0,
      "learning_rate": 2.0730356503306806e-07,
      "log_odds_chosen": 0.12334088981151581,
      "log_odds_ratio": -0.6655514240264893,
      "logits/chosen": -2.1896536350250244,
      "logits/rejected": -2.1869468688964844,
      "logps/chosen": -0.3938687741756439,
      "logps/rejected": -0.4362581670284271,
      "loss": 0.9508,
      "nll_loss": 0.8674702644348145,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.03938687965273857,
      "rewards/margins": 0.0042389435693621635,
      "rewards/rejected": -0.04362582042813301,
      "step": 640
    },
    {
      "epoch": 0.6083294337856808,
      "grad_norm": 16.625,
      "learning_rate": 1.9927516630246335e-07,
      "log_odds_chosen": 0.20489105582237244,
      "log_odds_ratio": -0.6253317594528198,
      "logits/chosen": -2.2503440380096436,
      "logits/rejected": -2.2342848777770996,
      "logps/chosen": -0.394917756319046,
      "logps/rejected": -0.46595415472984314,
      "loss": 0.9924,
      "nll_loss": 0.8516713976860046,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.03949177637696266,
      "rewards/margins": 0.0071036433801054955,
      "rewards/rejected": -0.04659541696310043,
      "step": 650
    },
    {
      "epoch": 0.6176883481516144,
      "grad_norm": 13.3125,
      "learning_rate": 1.9130097201963545e-07,
      "log_odds_chosen": 0.3604734539985657,
      "log_odds_ratio": -0.5889347195625305,
      "logits/chosen": -2.1605143547058105,
      "logits/rejected": -2.1294660568237305,
      "logps/chosen": -0.3886405825614929,
      "logps/rejected": -0.5594497919082642,
      "loss": 0.9053,
      "nll_loss": 0.8217363357543945,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.03886405751109123,
      "rewards/margins": 0.017080921679735184,
      "rewards/rejected": -0.055944979190826416,
      "step": 660
    },
    {
      "epoch": 0.6270472625175479,
      "grad_norm": 15.125,
      "learning_rate": 1.833895033913789e-07,
      "log_odds_chosen": 0.3233751654624939,
      "log_odds_ratio": -0.5963612794876099,
      "logits/chosen": -2.1719839572906494,
      "logits/rejected": -2.190833568572998,
      "logps/chosen": -0.3922678530216217,
      "logps/rejected": -0.535617470741272,
      "loss": 0.9524,
      "nll_loss": 0.9001427888870239,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.03922678157687187,
      "rewards/margins": 0.014334957115352154,
      "rewards/rejected": -0.0535617396235466,
      "step": 670
    },
    {
      "epoch": 0.6364061768834816,
      "grad_norm": 15.875,
      "learning_rate": 1.755492145959896e-07,
      "log_odds_chosen": 0.31382274627685547,
      "log_odds_ratio": -0.5895714163780212,
      "logits/chosen": -2.211315155029297,
      "logits/rejected": -2.233793020248413,
      "logps/chosen": -0.3794475197792053,
      "logps/rejected": -0.5004490613937378,
      "loss": 0.9666,
      "nll_loss": 0.919101893901825,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.03794475272297859,
      "rewards/margins": 0.012100161984562874,
      "rewards/rejected": -0.050044916570186615,
      "step": 680
    },
    {
      "epoch": 0.6457650912494151,
      "grad_norm": 14.25,
      "learning_rate": 1.6778848374914728e-07,
      "log_odds_chosen": 0.22027654945850372,
      "log_odds_ratio": -0.6377453207969666,
      "logits/chosen": -2.2121682167053223,
      "logits/rejected": -2.1823742389678955,
      "logps/chosen": -0.3827424645423889,
      "logps/rejected": -0.4631470739841461,
      "loss": 0.9201,
      "nll_loss": 0.8197261691093445,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.03827425092458725,
      "rewards/margins": 0.00804046168923378,
      "rewards/rejected": -0.04631470888853073,
      "step": 690
    },
    {
      "epoch": 0.6551240056153487,
      "grad_norm": 14.1875,
      "learning_rate": 1.6011560395107998e-07,
      "log_odds_chosen": 0.2072620838880539,
      "log_odds_ratio": -0.6260396838188171,
      "logits/chosen": -2.1534218788146973,
      "logits/rejected": -2.135467052459717,
      "logps/chosen": -0.4003871977329254,
      "logps/rejected": -0.48006024956703186,
      "loss": 0.9605,
      "nll_loss": 0.9064604043960571,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.04003872722387314,
      "rewards/margins": 0.007967302575707436,
      "rewards/rejected": -0.04800602048635483,
      "step": 700
    },
    {
      "epoch": 0.6644829199812822,
      "grad_norm": 15.0625,
      "learning_rate": 1.5253877442457446e-07,
      "log_odds_chosen": 0.2182326316833496,
      "log_odds_ratio": -0.6246551275253296,
      "logits/chosen": -2.1990370750427246,
      "logits/rejected": -2.187234878540039,
      "logps/chosen": -0.39677727222442627,
      "logps/rejected": -0.4731627404689789,
      "loss": 0.9649,
      "nll_loss": 0.8980569839477539,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.03967772796750069,
      "rewards/margins": 0.007638547569513321,
      "rewards/rejected": -0.04731627553701401,
      "step": 710
    },
    {
      "epoch": 0.6738418343472157,
      "grad_norm": 14.625,
      "learning_rate": 1.450660917533048e-07,
      "log_odds_chosen": 0.2397657334804535,
      "log_odds_ratio": -0.6196783185005188,
      "logits/chosen": -2.1318721771240234,
      "logits/rejected": -2.14440655708313,
      "logps/chosen": -0.39654839038848877,
      "logps/rejected": -0.49670910835266113,
      "loss": 0.9984,
      "nll_loss": 0.9670102000236511,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.039654843509197235,
      "rewards/margins": 0.010016070678830147,
      "rewards/rejected": -0.049670904874801636,
      "step": 720
    },
    {
      "epoch": 0.6832007487131493,
      "grad_norm": 15.875,
      "learning_rate": 1.377055412298402e-07,
      "log_odds_chosen": 0.2634773552417755,
      "log_odds_ratio": -0.6008420586585999,
      "logits/chosen": -2.2238945960998535,
      "logits/rejected": -2.2294325828552246,
      "logps/chosen": -0.3819832503795624,
      "logps/rejected": -0.48257875442504883,
      "loss": 0.9467,
      "nll_loss": 0.8946998715400696,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.038198329508304596,
      "rewards/margins": 0.010059546679258347,
      "rewards/rejected": -0.04825787618756294,
      "step": 730
    },
    {
      "epoch": 0.6925596630790828,
      "grad_norm": 13.625,
      "learning_rate": 1.3046498832257924e-07,
      "log_odds_chosen": 0.37252184748649597,
      "log_odds_ratio": -0.5736531019210815,
      "logits/chosen": -2.1820709705352783,
      "logits/rejected": -2.193258047103882,
      "logps/chosen": -0.38064131140708923,
      "logps/rejected": -0.529471218585968,
      "loss": 0.9604,
      "nll_loss": 0.8576000332832336,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.038064129650592804,
      "rewards/margins": 0.014882983639836311,
      "rewards/rejected": -0.052947115153074265,
      "step": 740
    },
    {
      "epoch": 0.7019185774450164,
      "grad_norm": 18.25,
      "learning_rate": 1.233521702707264e-07,
      "log_odds_chosen": 0.20452013611793518,
      "log_odds_ratio": -0.6333576440811157,
      "logits/chosen": -2.2299623489379883,
      "logits/rejected": -2.227400064468384,
      "logps/chosen": -0.4053714871406555,
      "logps/rejected": -0.4876191020011902,
      "loss": 0.9534,
      "nll_loss": 0.9184444546699524,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.04053715243935585,
      "rewards/margins": 0.008224757388234138,
      "rewards/rejected": -0.04876190796494484,
      "step": 750
    },
    {
      "epoch": 0.7112774918109499,
      "grad_norm": 12.0625,
      "learning_rate": 1.1637468781629567e-07,
      "log_odds_chosen": 0.2872607111930847,
      "log_odds_ratio": -0.6115022897720337,
      "logits/chosen": -2.1839165687561035,
      "logits/rejected": -2.183589220046997,
      "logps/chosen": -0.38220152258872986,
      "logps/rejected": -0.5047668218612671,
      "loss": 0.9557,
      "nll_loss": 0.8811558485031128,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.03822015970945358,
      "rewards/margins": 0.012256531044840813,
      "rewards/rejected": -0.050476688891649246,
      "step": 760
    },
    {
      "epoch": 0.7206364061768835,
      "grad_norm": 13.375,
      "learning_rate": 1.0953999708197404e-07,
      "log_odds_chosen": 0.2062472403049469,
      "log_odds_ratio": -0.6398475766181946,
      "logits/chosen": -2.2169864177703857,
      "logits/rejected": -2.220945119857788,
      "logps/chosen": -0.3971841633319855,
      "logps/rejected": -0.48631978034973145,
      "loss": 0.9464,
      "nll_loss": 0.9025236368179321,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.03971841558814049,
      "rewards/margins": 0.00891356635838747,
      "rewards/rejected": -0.048631977289915085,
      "step": 770
    },
    {
      "epoch": 0.729995320542817,
      "grad_norm": 13.25,
      "learning_rate": 1.0285540160352404e-07,
      "log_odds_chosen": 0.18557827174663544,
      "log_odds_ratio": -0.6399238705635071,
      "logits/chosen": -2.2538866996765137,
      "logits/rejected": -2.241267442703247,
      "logps/chosen": -0.4030405879020691,
      "logps/rejected": -0.47014960646629333,
      "loss": 0.9506,
      "nll_loss": 0.9130373001098633,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.04030405730009079,
      "rewards/margins": 0.006710907910019159,
      "rewards/rejected": -0.04701497033238411,
      "step": 780
    },
    {
      "epoch": 0.7393542349087506,
      "grad_norm": 13.375,
      "learning_rate": 9.632804452524256e-08,
      "log_odds_chosen": 0.2168750762939453,
      "log_odds_ratio": -0.6271758675575256,
      "logits/chosen": -2.145925760269165,
      "logits/rejected": -2.147688865661621,
      "logps/chosen": -0.40305909514427185,
      "logps/rejected": -0.48798856139183044,
      "loss": 0.8831,
      "nll_loss": 0.8791316747665405,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.040305912494659424,
      "rewards/margins": 0.008492941036820412,
      "rewards/rejected": -0.048798851668834686,
      "step": 790
    },
    {
      "epoch": 0.7487131492746841,
      "grad_norm": 13.6875,
      "learning_rate": 8.996490096681109e-08,
      "log_odds_chosen": 0.19949845969676971,
      "log_odds_ratio": -0.6497266888618469,
      "logits/chosen": -2.161623954772949,
      "logits/rejected": -2.14546275138855,
      "logps/chosen": -0.41016340255737305,
      "logps/rejected": -0.49314770102500916,
      "loss": 0.9495,
      "nll_loss": 0.9498335123062134,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.0410163477063179,
      "rewards/margins": 0.00829843245446682,
      "rewards/rejected": -0.049314774572849274,
      "step": 800
    },
    {
      "epoch": 0.7580720636406177,
      "grad_norm": 13.875,
      "learning_rate": 8.377277056969842e-08,
      "log_odds_chosen": 0.2793668508529663,
      "log_odds_ratio": -0.6106365323066711,
      "logits/chosen": -2.225426435470581,
      "logits/rejected": -2.216858386993408,
      "logps/chosen": -0.40902504324913025,
      "logps/rejected": -0.5241647958755493,
      "loss": 0.9711,
      "nll_loss": 0.9040560722351074,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.040902502834796906,
      "rewards/margins": 0.01151397917419672,
      "rewards/rejected": -0.05241648107767105,
      "step": 810
    },
    {
      "epoch": 0.7674309780065512,
      "grad_norm": 16.125,
      "learning_rate": 7.775827023107834e-08,
      "log_odds_chosen": 0.29139918088912964,
      "log_odds_ratio": -0.5888545513153076,
      "logits/chosen": -2.2389183044433594,
      "logits/rejected": -2.2176356315612793,
      "logps/chosen": -0.39003506302833557,
      "logps/rejected": -0.4982146620750427,
      "loss": 0.9545,
      "nll_loss": 0.9721220135688782,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.03900350257754326,
      "rewards/margins": 0.01081795897334814,
      "rewards/rejected": -0.04982146620750427,
      "step": 820
    },
    {
      "epoch": 0.7767898923724847,
      "grad_norm": 17.25,
      "learning_rate": 7.192782703302785e-08,
      "log_odds_chosen": 0.3136988878250122,
      "log_odds_ratio": -0.5841724276542664,
      "logits/chosen": -2.1989166736602783,
      "logits/rejected": -2.1767630577087402,
      "logps/chosen": -0.38346153497695923,
      "logps/rejected": -0.49425944685935974,
      "loss": 0.9185,
      "nll_loss": 0.8561797142028809,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.03834615275263786,
      "rewards/margins": 0.01107979379594326,
      "rewards/rejected": -0.04942594841122627,
      "step": 830
    },
    {
      "epoch": 0.7861488067384184,
      "grad_norm": 12.5625,
      "learning_rate": 6.628767137456067e-08,
      "log_odds_chosen": 0.20183363556861877,
      "log_odds_ratio": -0.6364968419075012,
      "logits/chosen": -2.163003444671631,
      "logits/rejected": -2.169506788253784,
      "logps/chosen": -0.4223906099796295,
      "logps/rejected": -0.4997871518135071,
      "loss": 0.9932,
      "nll_loss": 0.9328519701957703,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.04223906248807907,
      "rewards/margins": 0.007739657070487738,
      "rewards/rejected": -0.04997871816158295,
      "step": 840
    },
    {
      "epoch": 0.7955077211043519,
      "grad_norm": 16.25,
      "learning_rate": 6.08438303138365e-08,
      "log_odds_chosen": 0.15886886417865753,
      "log_odds_ratio": -0.6480736136436462,
      "logits/chosen": -2.1696417331695557,
      "logits/rejected": -2.1737618446350098,
      "logps/chosen": -0.38803622126579285,
      "logps/rejected": -0.44714784622192383,
      "loss": 0.971,
      "nll_loss": 0.917407214641571,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.038803622126579285,
      "rewards/margins": 0.005911166779696941,
      "rewards/rejected": -0.0447147861123085,
      "step": 850
    },
    {
      "epoch": 0.8048666354702855,
      "grad_norm": 13.125,
      "learning_rate": 5.560212112766011e-08,
      "log_odds_chosen": 0.19609014689922333,
      "log_odds_ratio": -0.6415227651596069,
      "logits/chosen": -2.193361759185791,
      "logits/rejected": -2.1855015754699707,
      "logps/chosen": -0.3944851756095886,
      "logps/rejected": -0.473043292760849,
      "loss": 0.9432,
      "nll_loss": 0.9007611274719238,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.03944852203130722,
      "rewards/margins": 0.007855813950300217,
      "rewards/rejected": -0.04730432480573654,
      "step": 860
    },
    {
      "epoch": 0.814225549836219,
      "grad_norm": 15.5625,
      "learning_rate": 5.056814509515092e-08,
      "log_odds_chosen": 0.27595824003219604,
      "log_odds_ratio": -0.618240237236023,
      "logits/chosen": -2.1721434593200684,
      "logits/rejected": -2.2051827907562256,
      "logps/chosen": -0.38330593705177307,
      "logps/rejected": -0.4931607246398926,
      "loss": 0.9415,
      "nll_loss": 0.831408679485321,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.03833059221506119,
      "rewards/margins": 0.010985485278069973,
      "rewards/rejected": -0.049316078424453735,
      "step": 870
    },
    {
      "epoch": 0.8235844642021526,
      "grad_norm": 15.6875,
      "learning_rate": 4.57472815122294e-08,
      "log_odds_chosen": 0.1982923001050949,
      "log_odds_ratio": -0.6450881958007812,
      "logits/chosen": -2.2192018032073975,
      "logits/rejected": -2.205094337463379,
      "logps/chosen": -0.40160322189331055,
      "logps/rejected": -0.4759804308414459,
      "loss": 0.9264,
      "nll_loss": 0.87147057056427,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.040160324424505234,
      "rewards/margins": 0.007437719497829676,
      "rewards/rejected": -0.04759804531931877,
      "step": 880
    },
    {
      "epoch": 0.8329433785680861,
      "grad_norm": 14.5625,
      "learning_rate": 4.1144681943312135e-08,
      "log_odds_chosen": 0.26552319526672363,
      "log_odds_ratio": -0.6161493062973022,
      "logits/chosen": -2.134911298751831,
      "logits/rejected": -2.146639347076416,
      "logps/chosen": -0.3881653845310211,
      "logps/rejected": -0.5064771771430969,
      "loss": 0.9764,
      "nll_loss": 0.9315429925918579,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.03881654143333435,
      "rewards/margins": 0.011831178329885006,
      "rewards/rejected": -0.05064771696925163,
      "step": 890
    },
    {
      "epoch": 0.8423022929340197,
      "grad_norm": 14.0,
      "learning_rate": 3.676526471636168e-08,
      "log_odds_chosen": 0.2872588634490967,
      "log_odds_ratio": -0.6009818315505981,
      "logits/chosen": -2.1846354007720947,
      "logits/rejected": -2.1831088066101074,
      "logps/chosen": -0.39365488290786743,
      "logps/rejected": -0.5202240943908691,
      "loss": 0.9021,
      "nll_loss": 0.7989016771316528,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.0393654927611351,
      "rewards/margins": 0.012656916864216328,
      "rewards/rejected": -0.052022408694028854,
      "step": 900
    },
    {
      "epoch": 0.8516612072999532,
      "grad_norm": 14.5,
      "learning_rate": 3.2613709667171893e-08,
      "log_odds_chosen": 0.22882740199565887,
      "log_odds_ratio": -0.625227153301239,
      "logits/chosen": -2.161898612976074,
      "logits/rejected": -2.1723926067352295,
      "logps/chosen": -0.39593061804771423,
      "logps/rejected": -0.48067528009414673,
      "loss": 0.9656,
      "nll_loss": 0.8755291104316711,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.03959306702017784,
      "rewards/margins": 0.008474460802972317,
      "rewards/rejected": -0.048067521303892136,
      "step": 910
    },
    {
      "epoch": 0.8610201216658867,
      "grad_norm": 16.75,
      "learning_rate": 2.8694453138505296e-08,
      "log_odds_chosen": 0.2396305501461029,
      "log_odds_ratio": -0.6223834753036499,
      "logits/chosen": -2.132664442062378,
      "logits/rejected": -2.13667893409729,
      "logps/chosen": -0.3883790373802185,
      "logps/rejected": -0.4780302941799164,
      "loss": 0.9318,
      "nll_loss": 0.9020143747329712,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.03883790224790573,
      "rewards/margins": 0.008965130895376205,
      "rewards/rejected": -0.04780303314328194,
      "step": 920
    },
    {
      "epoch": 0.8703790360318203,
      "grad_norm": 15.625,
      "learning_rate": 2.5011683239426847e-08,
      "log_odds_chosen": 0.2851574420928955,
      "log_odds_ratio": -0.5965331792831421,
      "logits/chosen": -2.1351237297058105,
      "logits/rejected": -2.108013153076172,
      "logps/chosen": -0.4045374393463135,
      "logps/rejected": -0.5147396922111511,
      "loss": 1.0019,
      "nll_loss": 0.9257608652114868,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.040453746914863586,
      "rewards/margins": 0.011020222678780556,
      "rewards/rejected": -0.05147396773099899,
      "step": 930
    },
    {
      "epoch": 0.8797379503977538,
      "grad_norm": 15.125,
      "learning_rate": 2.1569335369899884e-08,
      "log_odds_chosen": 0.28318914771080017,
      "log_odds_ratio": -0.5989499092102051,
      "logits/chosen": -2.1903491020202637,
      "logits/rejected": -2.1783673763275146,
      "logps/chosen": -0.376907616853714,
      "logps/rejected": -0.48921266198158264,
      "loss": 0.9765,
      "nll_loss": 0.953335165977478,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.03769076615571976,
      "rewards/margins": 0.011230507865548134,
      "rewards/rejected": -0.04892126843333244,
      "step": 940
    },
    {
      "epoch": 0.8890968647636874,
      "grad_norm": 12.875,
      "learning_rate": 1.837108801542589e-08,
      "log_odds_chosen": 0.3352145552635193,
      "log_odds_ratio": -0.5945736169815063,
      "logits/chosen": -2.170628786087036,
      "logits/rejected": -2.1475720405578613,
      "logps/chosen": -0.37348341941833496,
      "logps/rejected": -0.5254369974136353,
      "loss": 0.9728,
      "nll_loss": 0.9157112240791321,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.03734833747148514,
      "rewards/margins": 0.01519535668194294,
      "rewards/rejected": -0.052543699741363525,
      "step": 950
    },
    {
      "epoch": 0.8984557791296209,
      "grad_norm": 16.125,
      "learning_rate": 1.5420358816223e-08,
      "log_odds_chosen": 0.2994464635848999,
      "log_odds_ratio": -0.6000243425369263,
      "logits/chosen": -2.1644070148468018,
      "logits/rejected": -2.1604244709014893,
      "logps/chosen": -0.4016776978969574,
      "logps/rejected": -0.527344286441803,
      "loss": 0.9684,
      "nll_loss": 0.9093497395515442,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.04016777127981186,
      "rewards/margins": 0.012566653080284595,
      "rewards/rejected": -0.05273442342877388,
      "step": 960
    },
    {
      "epoch": 0.9078146934955545,
      "grad_norm": 13.5,
      "learning_rate": 1.2720300915142978e-08,
      "log_odds_chosen": 0.25296837091445923,
      "log_odds_ratio": -0.606758713722229,
      "logits/chosen": -2.21331787109375,
      "logits/rejected": -2.1903586387634277,
      "logps/chosen": -0.3899661600589752,
      "logps/rejected": -0.4821712076663971,
      "loss": 1.0186,
      "nll_loss": 0.8984857797622681,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.0389966145157814,
      "rewards/margins": 0.009220501407980919,
      "rewards/rejected": -0.04821711406111717,
      "step": 970
    },
    {
      "epoch": 0.917173607861488,
      "grad_norm": 12.4375,
      "learning_rate": 1.0273799588229659e-08,
      "log_odds_chosen": 0.19899612665176392,
      "log_odds_ratio": -0.6300643682479858,
      "logits/chosen": -2.1826329231262207,
      "logits/rejected": -2.171332836151123,
      "logps/chosen": -0.3943658471107483,
      "logps/rejected": -0.46531087160110474,
      "loss": 1.0421,
      "nll_loss": 0.9711662530899048,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.03943658620119095,
      "rewards/margins": 0.007094503380358219,
      "rewards/rejected": -0.046531084924936295,
      "step": 980
    },
    {
      "epoch": 0.9265325222274217,
      "grad_norm": 14.5,
      "learning_rate": 8.08346916151903e-09,
      "log_odds_chosen": 0.24496571719646454,
      "log_odds_ratio": -0.6311475038528442,
      "logits/chosen": -2.199125051498413,
      "logits/rejected": -2.1684861183166504,
      "logps/chosen": -0.399278849363327,
      "logps/rejected": -0.5040075182914734,
      "loss": 0.9278,
      "nll_loss": 0.855401873588562,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.0399278849363327,
      "rewards/margins": 0.010472869500517845,
      "rewards/rejected": -0.0504007525742054,
      "step": 990
    },
    {
      "epoch": 0.9358914365933552,
      "grad_norm": 13.8125,
      "learning_rate": 6.151650217376347e-09,
      "log_odds_chosen": 0.26680201292037964,
      "log_odds_ratio": -0.5990436673164368,
      "logits/chosen": -2.1688525676727295,
      "logits/rejected": -2.1509175300598145,
      "logps/chosen": -0.39877527952194214,
      "logps/rejected": -0.5044213533401489,
      "loss": 0.9671,
      "nll_loss": 0.9201105237007141,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.03987753018736839,
      "rewards/margins": 0.01056460477411747,
      "rewards/rejected": -0.05044213682413101,
      "step": 1000
    },
    {
      "epoch": 0.9452503509592888,
      "grad_norm": 13.625,
      "learning_rate": 4.480407093354566e-09,
      "log_odds_chosen": 0.1911187767982483,
      "log_odds_ratio": -0.6375032663345337,
      "logits/chosen": -2.192441463470459,
      "logits/rejected": -2.178623676300049,
      "logps/chosen": -0.4046042561531067,
      "logps/rejected": -0.46979087591171265,
      "loss": 0.9988,
      "nll_loss": 0.8718518018722534,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.04046043008565903,
      "rewards/margins": 0.006518671754747629,
      "rewards/rejected": -0.04697910323739052,
      "step": 1010
    },
    {
      "epoch": 0.9546092653252223,
      "grad_norm": 11.8125,
      "learning_rate": 3.0715256762478825e-09,
      "log_odds_chosen": 0.31750017404556274,
      "log_odds_ratio": -0.5878022909164429,
      "logits/chosen": -2.161133289337158,
      "logits/rejected": -2.1433868408203125,
      "logps/chosen": -0.3968185782432556,
      "logps/rejected": -0.5128824710845947,
      "loss": 0.9572,
      "nll_loss": 0.9017167091369629,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.03968185931444168,
      "rewards/margins": 0.01160639338195324,
      "rewards/rejected": -0.05128825455904007,
      "step": 1020
    },
    {
      "epoch": 0.9639681796911558,
      "grad_norm": 16.75,
      "learning_rate": 1.926511493696936e-09,
      "log_odds_chosen": 0.1701984405517578,
      "log_odds_ratio": -0.6456406712532043,
      "logits/chosen": -2.196162700653076,
      "logits/rejected": -2.202619791030884,
      "logps/chosen": -0.40211939811706543,
      "logps/rejected": -0.4752349853515625,
      "loss": 0.9981,
      "nll_loss": 0.8766402006149292,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.040211938321590424,
      "rewards/margins": 0.007311551831662655,
      "rewards/rejected": -0.04752349108457565,
      "step": 1030
    },
    {
      "epoch": 0.9733270940570894,
      "grad_norm": 14.75,
      "learning_rate": 1.04658810538516e-09,
      "log_odds_chosen": 0.23622456192970276,
      "log_odds_ratio": -0.6107891798019409,
      "logits/chosen": -2.175441265106201,
      "logits/rejected": -2.1792852878570557,
      "logps/chosen": -0.3729916214942932,
      "logps/rejected": -0.45000284910202026,
      "loss": 0.926,
      "nll_loss": 0.8379205465316772,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.03729915991425514,
      "rewards/margins": 0.0077011240646243095,
      "rewards/rejected": -0.045000284910202026,
      "step": 1040
    },
    {
      "epoch": 0.9826860084230229,
      "grad_norm": 15.0625,
      "learning_rate": 4.3269579554558455e-10,
      "log_odds_chosen": 0.19562143087387085,
      "log_odds_ratio": -0.6367918848991394,
      "logits/chosen": -2.2001359462738037,
      "logits/rejected": -2.183256149291992,
      "logps/chosen": -0.4047182500362396,
      "logps/rejected": -0.48164814710617065,
      "loss": 0.9974,
      "nll_loss": 1.0015695095062256,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.04047182947397232,
      "rewards/margins": 0.007692983839660883,
      "rewards/rejected": -0.04816481098532677,
      "step": 1050
    },
    {
      "epoch": 0.9920449227889565,
      "grad_norm": 14.4375,
      "learning_rate": 8.549056817513944e-11,
      "log_odds_chosen": 0.29663318395614624,
      "log_odds_ratio": -0.6037989854812622,
      "logits/chosen": -2.197646379470825,
      "logits/rejected": -2.1741132736206055,
      "logps/chosen": -0.3990115523338318,
      "logps/rejected": -0.5252410173416138,
      "loss": 0.9644,
      "nll_loss": 0.9385454058647156,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.03990115597844124,
      "rewards/margins": 0.012622940354049206,
      "rewards/rejected": -0.05252409726381302,
      "step": 1060
    },
    {
      "epoch": 0.9995320542817033,
      "step": 1068,
      "total_flos": 0.0,
      "train_loss": 1.0056083492571941,
      "train_runtime": 16376.6673,
      "train_samples_per_second": 2.088,
      "train_steps_per_second": 0.065
    }
  ],
  "logging_steps": 10,
  "max_steps": 1068,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}