{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9982859101816935,
  "eval_steps": 0,
  "global_step": 182,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.005485087418580734,
      "grad_norm": 5.038481473898129,
      "learning_rate": 5.263157894736843e-07,
      "logits/chosen": -0.3854110836982727,
      "logits/rejected": -0.38843637704849243,
      "logps/chosen": -0.5867404937744141,
      "logps/rejected": -0.7349259853363037,
      "loss": 0.8549,
      "odds_ratio_loss": 8.495767593383789,
      "rewards/accuracies": 0.328125,
      "rewards/chosen": -0.07349259406328201,
      "rewards/margins": -0.014818555675446987,
      "rewards/rejected": -0.05867404490709305,
      "sft_loss": 0.00530597660690546,
      "step": 1
    },
    {
      "epoch": 0.010970174837161468,
      "grad_norm": 3.8692149987333275,
      "learning_rate": 1.0526315789473685e-06,
      "logits/chosen": -0.4200110137462616,
      "logits/rejected": -0.4337027370929718,
      "logps/chosen": -0.5888247489929199,
      "logps/rejected": -0.7141146659851074,
      "loss": 0.8261,
      "odds_ratio_loss": 8.218369483947754,
      "rewards/accuracies": 0.3671875,
      "rewards/chosen": -0.07141146808862686,
      "rewards/margins": -0.01252899132668972,
      "rewards/rejected": -0.05888247489929199,
      "sft_loss": 0.004305172245949507,
      "step": 2
    },
    {
      "epoch": 0.0164552622557422,
      "grad_norm": 3.4352579088349455,
      "learning_rate": 1.5789473684210526e-06,
      "logits/chosen": -0.39038556814193726,
      "logits/rejected": -0.3625536262989044,
      "logps/chosen": -0.6846470236778259,
      "logps/rejected": -0.6906213760375977,
      "loss": 0.8054,
      "odds_ratio_loss": 8.022075653076172,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.06906213611364365,
      "rewards/margins": -0.0005974352825433016,
      "rewards/rejected": -0.06846471130847931,
      "sft_loss": 0.0031916582956910133,
      "step": 3
    },
    {
      "epoch": 0.021940349674322936,
      "grad_norm": 6.165427271910815,
      "learning_rate": 2.105263157894737e-06,
      "logits/chosen": -0.4156845211982727,
      "logits/rejected": -0.4445635974407196,
      "logps/chosen": -0.6091395020484924,
      "logps/rejected": -0.7165651321411133,
      "loss": 0.8329,
      "odds_ratio_loss": 8.273240089416504,
      "rewards/accuracies": 0.4140625,
      "rewards/chosen": -0.07165651768445969,
      "rewards/margins": -0.010742560029029846,
      "rewards/rejected": -0.06091395765542984,
      "sft_loss": 0.005578206852078438,
      "step": 4
    },
    {
      "epoch": 0.027425437092903668,
      "grad_norm": 3.719991962975026,
      "learning_rate": 2.631578947368421e-06,
      "logits/chosen": -0.4787546396255493,
      "logits/rejected": -0.4570210874080658,
      "logps/chosen": -0.5717373490333557,
      "logps/rejected": -0.6768375039100647,
      "loss": 0.7966,
      "odds_ratio_loss": 7.933419227600098,
      "rewards/accuracies": 0.3359375,
      "rewards/chosen": -0.06768374890089035,
      "rewards/margins": -0.010510011576116085,
      "rewards/rejected": -0.05717373639345169,
      "sft_loss": 0.0032929559238255024,
      "step": 5
    },
    {
      "epoch": 0.0329105245114844,
      "grad_norm": 3.623178718641896,
      "learning_rate": 3.157894736842105e-06,
      "logits/chosen": -0.34492218494415283,
      "logits/rejected": -0.42876753211021423,
      "logps/chosen": -0.5948079824447632,
      "logps/rejected": -0.7103158235549927,
      "loss": 0.8222,
      "odds_ratio_loss": 8.177949905395508,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0710315853357315,
      "rewards/margins": -0.01155078411102295,
      "rewards/rejected": -0.05948079749941826,
      "sft_loss": 0.004363520070910454,
      "step": 6
    },
    {
      "epoch": 0.03839561193006513,
      "grad_norm": 3.845464498598408,
      "learning_rate": 3.6842105263157896e-06,
      "logits/chosen": -0.465349018573761,
      "logits/rejected": -0.5269383788108826,
      "logps/chosen": -0.5850939154624939,
      "logps/rejected": -0.6428964734077454,
      "loss": 0.7481,
      "odds_ratio_loss": 7.433981418609619,
      "rewards/accuracies": 0.4140625,
      "rewards/chosen": -0.0642896518111229,
      "rewards/margins": -0.00578026007860899,
      "rewards/rejected": -0.05850938707590103,
      "sft_loss": 0.004695751238614321,
      "step": 7
    },
    {
      "epoch": 0.04388069934864587,
      "grad_norm": 3.534994725436119,
      "learning_rate": 4.210526315789474e-06,
      "logits/chosen": -0.47682783007621765,
      "logits/rejected": -0.5616908669471741,
      "logps/chosen": -0.5679696798324585,
      "logps/rejected": -0.6700828671455383,
      "loss": 0.7817,
      "odds_ratio_loss": 7.7718095779418945,
      "rewards/accuracies": 0.390625,
      "rewards/chosen": -0.06700828671455383,
      "rewards/margins": -0.010211320593953133,
      "rewards/rejected": -0.05679696798324585,
      "sft_loss": 0.004522908478975296,
      "step": 8
    },
    {
      "epoch": 0.049365786767226603,
      "grad_norm": 3.0620529937519723,
      "learning_rate": 4.736842105263158e-06,
      "logits/chosen": -0.5015044808387756,
      "logits/rejected": -0.5388280153274536,
      "logps/chosen": -0.4832647740840912,
      "logps/rejected": -0.7363763451576233,
      "loss": 0.8647,
      "odds_ratio_loss": 8.59887981414795,
      "rewards/accuracies": 0.265625,
      "rewards/chosen": -0.07363763451576233,
      "rewards/margins": -0.02531115524470806,
      "rewards/rejected": -0.04832647740840912,
      "sft_loss": 0.004782060626894236,
      "step": 9
    },
    {
      "epoch": 0.054850874185807336,
      "grad_norm": 3.1485369964141117,
      "learning_rate": 5.263157894736842e-06,
      "logits/chosen": -0.5910385251045227,
      "logits/rejected": -0.5793564319610596,
      "logps/chosen": -0.5233993530273438,
      "logps/rejected": -0.7609111666679382,
      "loss": 0.8793,
      "odds_ratio_loss": 8.740400314331055,
      "rewards/accuracies": 0.28125,
      "rewards/chosen": -0.07609111815690994,
      "rewards/margins": -0.02375117875635624,
      "rewards/rejected": -0.052339933812618256,
      "sft_loss": 0.00524852704256773,
      "step": 10
    },
    {
      "epoch": 0.06033596160438807,
      "grad_norm": 4.044788224688044,
      "learning_rate": 5.789473684210527e-06,
      "logits/chosen": -0.63301682472229,
      "logits/rejected": -0.6308025121688843,
      "logps/chosen": -0.5794235467910767,
      "logps/rejected": -0.7120978832244873,
      "loss": 0.8159,
      "odds_ratio_loss": 8.079412460327148,
      "rewards/accuracies": 0.3359375,
      "rewards/chosen": -0.07120979577302933,
      "rewards/margins": -0.013267431408166885,
      "rewards/rejected": -0.057942356914281845,
      "sft_loss": 0.007982916198670864,
      "step": 11
    },
    {
      "epoch": 0.0658210490229688,
      "grad_norm": 2.667877748800269,
      "learning_rate": 6.31578947368421e-06,
      "logits/chosen": -0.6804186701774597,
      "logits/rejected": -0.669500470161438,
      "logps/chosen": -0.6842315196990967,
      "logps/rejected": -0.7194308638572693,
      "loss": 0.8272,
      "odds_ratio_loss": 8.232641220092773,
      "rewards/accuracies": 0.296875,
      "rewards/chosen": -0.07194308936595917,
      "rewards/margins": -0.0035199355334043503,
      "rewards/rejected": -0.06842315196990967,
      "sft_loss": 0.0039048779290169477,
      "step": 12
    },
    {
      "epoch": 0.07130613644154954,
      "grad_norm": 3.0798648484182354,
      "learning_rate": 6.842105263157896e-06,
      "logits/chosen": -0.6367232203483582,
      "logits/rejected": -0.6232490539550781,
      "logps/chosen": -0.5626485347747803,
      "logps/rejected": -0.6783699989318848,
      "loss": 0.7821,
      "odds_ratio_loss": 7.7734174728393555,
      "rewards/accuracies": 0.34375,
      "rewards/chosen": -0.06783699989318848,
      "rewards/margins": -0.011572152376174927,
      "rewards/rejected": -0.05626484751701355,
      "sft_loss": 0.0047150785103440285,
      "step": 13
    },
    {
      "epoch": 0.07679122386013026,
      "grad_norm": 2.5628005096722,
      "learning_rate": 7.368421052631579e-06,
      "logits/chosen": -0.6715413928031921,
      "logits/rejected": -0.6445600390434265,
      "logps/chosen": -0.7547533512115479,
      "logps/rejected": -0.7515701055526733,
      "loss": 0.8562,
      "odds_ratio_loss": 8.51838207244873,
      "rewards/accuracies": 0.3515625,
      "rewards/chosen": -0.07515701651573181,
      "rewards/margins": 0.0003183241933584213,
      "rewards/rejected": -0.07547533512115479,
      "sft_loss": 0.004331222269684076,
      "step": 14
    },
    {
      "epoch": 0.082276311278711,
      "grad_norm": 3.6153906624666243,
      "learning_rate": 7.894736842105265e-06,
      "logits/chosen": -0.6177899241447449,
      "logits/rejected": -0.5948590040206909,
      "logps/chosen": -0.733797550201416,
      "logps/rejected": -0.7167279124259949,
      "loss": 0.8106,
      "odds_ratio_loss": 8.030726432800293,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.07167279720306396,
      "rewards/margins": 0.0017069588648155332,
      "rewards/rejected": -0.0733797550201416,
      "sft_loss": 0.007539147045463324,
      "step": 15
    },
    {
      "epoch": 0.08776139869729174,
      "grad_norm": 2.9810395185494576,
      "learning_rate": 8.421052631578948e-06,
      "logits/chosen": -0.6613759994506836,
      "logits/rejected": -0.6372989416122437,
      "logps/chosen": -0.6857063174247742,
      "logps/rejected": -0.7137706875801086,
      "loss": 0.8113,
      "odds_ratio_loss": 8.056015968322754,
      "rewards/accuracies": 0.359375,
      "rewards/chosen": -0.07137707620859146,
      "rewards/margins": -0.0028064418584108353,
      "rewards/rejected": -0.06857062876224518,
      "sft_loss": 0.005733313038945198,
      "step": 16
    },
    {
      "epoch": 0.09324648611587247,
      "grad_norm": 2.6891547946091565,
      "learning_rate": 8.947368421052632e-06,
      "logits/chosen": -0.6225110292434692,
      "logits/rejected": -0.6161482930183411,
      "logps/chosen": -0.7698332667350769,
      "logps/rejected": -0.7336711287498474,
      "loss": 0.8323,
      "odds_ratio_loss": 8.280593872070312,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.07336711138486862,
      "rewards/margins": 0.0036162168253213167,
      "rewards/rejected": -0.07698333263397217,
      "sft_loss": 0.004251881968230009,
      "step": 17
    },
    {
      "epoch": 0.09873157353445321,
      "grad_norm": 3.1701971147920314,
      "learning_rate": 9.473684210526315e-06,
      "logits/chosen": -0.6359448432922363,
      "logits/rejected": -0.5702680349349976,
      "logps/chosen": -0.77215576171875,
      "logps/rejected": -0.7223557829856873,
      "loss": 0.8107,
      "odds_ratio_loss": 8.051921844482422,
      "rewards/accuracies": 0.453125,
      "rewards/chosen": -0.0722355842590332,
      "rewards/margins": 0.004979996010661125,
      "rewards/rejected": -0.07721558213233948,
      "sft_loss": 0.005473591852933168,
      "step": 18
    },
    {
      "epoch": 0.10421666095303393,
      "grad_norm": 5.701499338439259,
      "learning_rate": 1e-05,
      "logits/chosen": -0.6352053284645081,
      "logits/rejected": -0.6071786880493164,
      "logps/chosen": -0.6723276376724243,
      "logps/rejected": -0.6906519532203674,
      "loss": 0.7743,
      "odds_ratio_loss": 7.4815754890441895,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.06906520575284958,
      "rewards/margins": -0.001832429552450776,
      "rewards/rejected": -0.06723276525735855,
      "sft_loss": 0.026183124631643295,
      "step": 19
    },
    {
      "epoch": 0.10970174837161467,
      "grad_norm": 2.32222317741005,
      "learning_rate": 9.999071352056676e-06,
      "logits/chosen": -0.6049666404724121,
      "logits/rejected": -0.5781897306442261,
      "logps/chosen": -0.5833567976951599,
      "logps/rejected": -0.688912034034729,
      "loss": 0.7844,
      "odds_ratio_loss": 7.809441566467285,
      "rewards/accuracies": 0.359375,
      "rewards/chosen": -0.06889119744300842,
      "rewards/margins": -0.01055552251636982,
      "rewards/rejected": -0.05833568051457405,
      "sft_loss": 0.0034232870675623417,
      "step": 20
    },
    {
      "epoch": 0.11518683579019541,
      "grad_norm": 2.5009890128022625,
      "learning_rate": 9.996285753181499e-06,
      "logits/chosen": -0.6923652291297913,
      "logits/rejected": -0.622920036315918,
      "logps/chosen": -0.5788823366165161,
      "logps/rejected": -0.7099403738975525,
      "loss": 0.8091,
      "odds_ratio_loss": 8.047269821166992,
      "rewards/accuracies": 0.3359375,
      "rewards/chosen": -0.07099403440952301,
      "rewards/margins": -0.013105800375342369,
      "rewards/rejected": -0.05788823589682579,
      "sft_loss": 0.004356673918664455,
      "step": 21
    },
    {
      "epoch": 0.12067192320877614,
      "grad_norm": 2.504946922805322,
      "learning_rate": 9.991644238110741e-06,
      "logits/chosen": -0.7856975197792053,
      "logits/rejected": -0.6942786574363708,
      "logps/chosen": -0.8530550003051758,
      "logps/rejected": -0.6810007691383362,
      "loss": 0.7769,
      "odds_ratio_loss": 7.728884696960449,
      "rewards/accuracies": 0.328125,
      "rewards/chosen": -0.06810007989406586,
      "rewards/margins": 0.017205417156219482,
      "rewards/rejected": -0.08530549705028534,
      "sft_loss": 0.003965733572840691,
      "step": 22
    },
    {
      "epoch": 0.12615701062735687,
      "grad_norm": 2.3381968971086358,
      "learning_rate": 9.985148530977767e-06,
      "logits/chosen": -0.7372395992279053,
      "logits/rejected": -0.6523022651672363,
      "logps/chosen": -0.5776190161705017,
      "logps/rejected": -0.6639264822006226,
      "loss": 0.7573,
      "odds_ratio_loss": 7.541070938110352,
      "rewards/accuracies": 0.3203125,
      "rewards/chosen": -0.06639264523983002,
      "rewards/margins": -0.008630745112895966,
      "rewards/rejected": -0.05776190012693405,
      "sft_loss": 0.0031744332518428564,
      "step": 23
    },
    {
      "epoch": 0.1316420980459376,
      "grad_norm": 2.210859888036035,
      "learning_rate": 9.976801044672608e-06,
      "logits/chosen": -0.7757277488708496,
      "logits/rejected": -0.6656205654144287,
      "logps/chosen": -0.5713773965835571,
      "logps/rejected": -0.6431125402450562,
      "loss": 0.7376,
      "odds_ratio_loss": 7.344100475311279,
      "rewards/accuracies": 0.3984375,
      "rewards/chosen": -0.06431125849485397,
      "rewards/margins": -0.007173515390604734,
      "rewards/rejected": -0.057137735188007355,
      "sft_loss": 0.0031648115254938602,
      "step": 24
    },
    {
      "epoch": 0.13712718546451835,
      "grad_norm": 2.5797643977876326,
      "learning_rate": 9.966604879945659e-06,
      "logits/chosen": -0.8338367342948914,
      "logits/rejected": -0.7324843406677246,
      "logps/chosen": -0.5937628149986267,
      "logps/rejected": -0.6450057625770569,
      "loss": 0.735,
      "odds_ratio_loss": 7.3058085441589355,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.06450057774782181,
      "rewards/margins": -0.005124296993017197,
      "rewards/rejected": -0.05937628448009491,
      "sft_loss": 0.004404762759804726,
      "step": 25
    },
    {
      "epoch": 0.14261227288309908,
      "grad_norm": 2.285207611169885,
      "learning_rate": 9.954563824255879e-06,
      "logits/chosen": -0.7936639785766602,
      "logits/rejected": -0.7150090932846069,
      "logps/chosen": -0.5255985856056213,
      "logps/rejected": -0.6223771572113037,
      "loss": 0.7183,
      "odds_ratio_loss": 7.146646499633789,
      "rewards/accuracies": 0.4296875,
      "rewards/chosen": -0.062237709760665894,
      "rewards/margins": -0.009677855297923088,
      "rewards/rejected": -0.052559856325387955,
      "sft_loss": 0.0036687427200376987,
      "step": 26
    },
    {
      "epoch": 0.1480973603016798,
      "grad_norm": 2.323478414221198,
      "learning_rate": 9.940682350363913e-06,
      "logits/chosen": -0.7588051557540894,
      "logits/rejected": -0.7106639742851257,
      "logps/chosen": -0.5572831630706787,
      "logps/rejected": -0.6621856689453125,
      "loss": 0.7592,
      "odds_ratio_loss": 7.553915977478027,
      "rewards/accuracies": 0.328125,
      "rewards/chosen": -0.06621856987476349,
      "rewards/margins": -0.010490254499018192,
      "rewards/rejected": -0.05572831630706787,
      "sft_loss": 0.0038430241402238607,
      "step": 27
    },
    {
      "epoch": 0.15358244772026053,
      "grad_norm": 2.2864612903822175,
      "learning_rate": 9.924965614670629e-06,
      "logits/chosen": -0.7962155342102051,
      "logits/rejected": -0.7206164598464966,
      "logps/chosen": -0.6034549474716187,
      "logps/rejected": -0.6673721075057983,
      "loss": 0.7587,
      "odds_ratio_loss": 7.547135829925537,
      "rewards/accuracies": 0.3828125,
      "rewards/chosen": -0.06673721969127655,
      "rewards/margins": -0.006391720846295357,
      "rewards/rejected": -0.060345496982336044,
      "sft_loss": 0.00402811961248517,
      "step": 28
    },
    {
      "epoch": 0.15906753513884128,
      "grad_norm": 5.4563944483821025,
      "learning_rate": 9.90741945530174e-06,
      "logits/chosen": -0.7886516451835632,
      "logits/rejected": -0.7214576005935669,
      "logps/chosen": -0.6277047991752625,
      "logps/rejected": -0.774495542049408,
      "loss": 0.873,
      "odds_ratio_loss": 8.486030578613281,
      "rewards/accuracies": 0.2890625,
      "rewards/chosen": -0.07744955271482468,
      "rewards/margins": -0.0146790761500597,
      "rewards/rejected": -0.06277047842741013,
      "sft_loss": 0.02440182864665985,
      "step": 29
    },
    {
      "epoch": 0.164552622557422,
      "grad_norm": 2.5755938714187216,
      "learning_rate": 9.888050389939172e-06,
      "logits/chosen": -0.8029124736785889,
      "logits/rejected": -0.7149651050567627,
      "logps/chosen": -0.6227287650108337,
      "logps/rejected": -0.7041884660720825,
      "loss": 0.796,
      "odds_ratio_loss": 7.916073799133301,
      "rewards/accuracies": 0.4140625,
      "rewards/chosen": -0.07041884958744049,
      "rewards/margins": -0.008145971223711967,
      "rewards/rejected": -0.06227288395166397,
      "sft_loss": 0.004434916190803051,
      "step": 30
    },
    {
      "epoch": 0.17003770997600273,
      "grad_norm": 2.738052763928764,
      "learning_rate": 9.866865613400008e-06,
      "logits/chosen": -0.745469331741333,
      "logits/rejected": -0.7005943059921265,
      "logps/chosen": -0.635623574256897,
      "logps/rejected": -0.706161379814148,
      "loss": 0.8005,
      "odds_ratio_loss": 7.957401275634766,
      "rewards/accuracies": 0.3828125,
      "rewards/chosen": -0.07061614096164703,
      "rewards/margins": -0.007053782232105732,
      "rewards/rejected": -0.06356236338615417,
      "sft_loss": 0.0047803232446312904,
      "step": 31
    },
    {
      "epoch": 0.17552279739458349,
      "grad_norm": 2.110256032046137,
      "learning_rate": 9.843872994963912e-06,
      "logits/chosen": -0.8181653618812561,
      "logits/rejected": -0.7452071309089661,
      "logps/chosen": -0.5545619130134583,
      "logps/rejected": -0.6596910357475281,
      "loss": 0.754,
      "odds_ratio_loss": 7.508852958679199,
      "rewards/accuracies": 0.34375,
      "rewards/chosen": -0.06596910208463669,
      "rewards/margins": -0.010512909851968288,
      "rewards/rejected": -0.05545618757605553,
      "sft_loss": 0.0030794725753366947,
      "step": 32
    },
    {
      "epoch": 0.1810078848131642,
      "grad_norm": 3.010643924076955,
      "learning_rate": 9.819081075450014e-06,
      "logits/chosen": -0.7885746359825134,
      "logits/rejected": -0.7216675877571106,
      "logps/chosen": -0.7158695459365845,
      "logps/rejected": -0.6520407199859619,
      "loss": 0.7394,
      "odds_ratio_loss": 7.32637882232666,
      "rewards/accuracies": 0.4453125,
      "rewards/chosen": -0.06520406901836395,
      "rewards/margins": 0.006382887717336416,
      "rewards/rejected": -0.07158695161342621,
      "sft_loss": 0.006795875262469053,
      "step": 33
    },
    {
      "epoch": 0.18649297223174494,
      "grad_norm": 2.476605715920486,
      "learning_rate": 9.792499064044343e-06,
      "logits/chosen": -0.8410877585411072,
      "logits/rejected": -0.7374798059463501,
      "logps/chosen": -0.6041279435157776,
      "logps/rejected": -0.6848863959312439,
      "loss": 0.7765,
      "odds_ratio_loss": 7.719598770141602,
      "rewards/accuracies": 0.3671875,
      "rewards/chosen": -0.06848862767219543,
      "rewards/margins": -0.008075837977230549,
      "rewards/rejected": -0.060412801802158356,
      "sft_loss": 0.00449481513351202,
      "step": 34
    },
    {
      "epoch": 0.1919780596503257,
      "grad_norm": 2.5393725876466284,
      "learning_rate": 9.764136834878987e-06,
      "logits/chosen": -0.8155717253684998,
      "logits/rejected": -0.7687007188796997,
      "logps/chosen": -0.6000927686691284,
      "logps/rejected": -0.6661568880081177,
      "loss": 0.7545,
      "odds_ratio_loss": 7.500143051147461,
      "rewards/accuracies": 0.390625,
      "rewards/chosen": -0.06661568582057953,
      "rewards/margins": -0.006606410723179579,
      "rewards/rejected": -0.06000927463173866,
      "sft_loss": 0.004530766978859901,
      "step": 35
    },
    {
      "epoch": 0.19746314706890641,
      "grad_norm": 2.5736785638504127,
      "learning_rate": 9.734004923364258e-06,
      "logits/chosen": -0.7973151803016663,
      "logits/rejected": -0.777863085269928,
      "logps/chosen": -0.6574283838272095,
      "logps/rejected": -0.7205468416213989,
      "loss": 0.8155,
      "odds_ratio_loss": 8.100136756896973,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0720546767115593,
      "rewards/margins": -0.006311837118119001,
      "rewards/rejected": -0.06574283540248871,
      "sft_loss": 0.0054591940715909,
      "step": 36
    },
    {
      "epoch": 0.20294823448748714,
      "grad_norm": 2.670773258722001,
      "learning_rate": 9.702114522275216e-06,
      "logits/chosen": -0.7815201878547668,
      "logits/rejected": -0.7440929412841797,
      "logps/chosen": -0.6075029373168945,
      "logps/rejected": -0.7015030384063721,
      "loss": 0.7927,
      "odds_ratio_loss": 7.880993843078613,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.07015030831098557,
      "rewards/margins": -0.009400014765560627,
      "rewards/rejected": -0.06075029447674751,
      "sft_loss": 0.004569103941321373,
      "step": 37
    },
    {
      "epoch": 0.20843332190606786,
      "grad_norm": 4.330660536492607,
      "learning_rate": 9.668477477594021e-06,
      "logits/chosen": -0.7350383996963501,
      "logits/rejected": -0.6684775352478027,
      "logps/chosen": -0.6681689023971558,
      "logps/rejected": -0.6945660710334778,
      "loss": 0.7763,
      "odds_ratio_loss": 7.6398186683654785,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.06945660710334778,
      "rewards/margins": -0.0026397148612886667,
      "rewards/rejected": -0.06681689620018005,
      "sft_loss": 0.012337452732026577,
      "step": 38
    },
    {
      "epoch": 0.21391840932464862,
      "grad_norm": 2.0952393942157537,
      "learning_rate": 9.633106284109612e-06,
      "logits/chosen": -0.71222984790802,
      "logits/rejected": -0.6697849035263062,
      "logps/chosen": -0.5882755517959595,
      "logps/rejected": -0.6613237261772156,
      "loss": 0.7526,
      "odds_ratio_loss": 7.493914604187012,
      "rewards/accuracies": 0.3515625,
      "rewards/chosen": -0.06613237410783768,
      "rewards/margins": -0.007304816506803036,
      "rewards/rejected": -0.05882755666971207,
      "sft_loss": 0.003193965647369623,
      "step": 39
    },
    {
      "epoch": 0.21940349674322934,
      "grad_norm": 2.5760849251786686,
      "learning_rate": 9.596014080776424e-06,
      "logits/chosen": -0.7155317068099976,
      "logits/rejected": -0.701492428779602,
      "logps/chosen": -0.6026738882064819,
      "logps/rejected": -0.7281025648117065,
      "loss": 0.8228,
      "odds_ratio_loss": 8.181192398071289,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.07281024754047394,
      "rewards/margins": -0.01254285965114832,
      "rewards/rejected": -0.06026739254593849,
      "sft_loss": 0.004728448111563921,
      "step": 40
    },
    {
      "epoch": 0.22488858416181007,
      "grad_norm": 3.204343350485489,
      "learning_rate": 9.557214645833792e-06,
      "logits/chosen": -0.815459668636322,
      "logits/rejected": -0.756503701210022,
      "logps/chosen": -0.5707991719245911,
      "logps/rejected": -0.6623473763465881,
      "loss": 0.7528,
      "odds_ratio_loss": 7.451156139373779,
      "rewards/accuracies": 0.3515625,
      "rewards/chosen": -0.06623473763465881,
      "rewards/margins": -0.009154818020761013,
      "rewards/rejected": -0.057079918682575226,
      "sft_loss": 0.0077085429802536964,
      "step": 41
    },
    {
      "epoch": 0.23037367158039082,
      "grad_norm": 2.5773834426506927,
      "learning_rate": 9.516722391687903e-06,
      "logits/chosen": -0.6963569521903992,
      "logits/rejected": -0.6759909391403198,
      "logps/chosen": -0.6941713094711304,
      "logps/rejected": -0.7426818013191223,
      "loss": 0.846,
      "odds_ratio_loss": 8.418367385864258,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.07426817715167999,
      "rewards/margins": -0.004851049743592739,
      "rewards/rejected": -0.06941712647676468,
      "sft_loss": 0.004144534468650818,
      "step": 42
    },
    {
      "epoch": 0.23585875899897155,
      "grad_norm": 2.2138593444070183,
      "learning_rate": 9.474552359558167e-06,
      "logits/chosen": -0.7346150279045105,
      "logits/rejected": -0.6656796932220459,
      "logps/chosen": -0.6170094609260559,
      "logps/rejected": -0.6504194736480713,
      "loss": 0.736,
      "odds_ratio_loss": 7.326251983642578,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.06504195928573608,
      "rewards/margins": -0.0033410112373530865,
      "rewards/rejected": -0.06170094013214111,
      "sft_loss": 0.0034048811066895723,
      "step": 43
    },
    {
      "epoch": 0.24134384641755227,
      "grad_norm": 2.4455061811366283,
      "learning_rate": 9.43072021389003e-06,
      "logits/chosen": -0.7304015159606934,
      "logits/rejected": -0.7045088410377502,
      "logps/chosen": -0.6213870644569397,
      "logps/rejected": -0.6929490566253662,
      "loss": 0.7783,
      "odds_ratio_loss": 7.74072265625,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.06929488480091095,
      "rewards/margins": -0.0071561927907168865,
      "rewards/rejected": -0.06213871017098427,
      "sft_loss": 0.004237705375999212,
      "step": 44
    },
    {
      "epoch": 0.24682893383613302,
      "grad_norm": 2.2689944416738097,
      "learning_rate": 9.385242236536259e-06,
      "logits/chosen": -0.7978068590164185,
      "logits/rejected": -0.7774004936218262,
      "logps/chosen": -0.7401710748672485,
      "logps/rejected": -0.6718741655349731,
      "loss": 0.7556,
      "odds_ratio_loss": 7.514792442321777,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.06718742102384567,
      "rewards/margins": 0.006829693913459778,
      "rewards/rejected": -0.07401710748672485,
      "sft_loss": 0.004071537870913744,
      "step": 45
    },
    {
      "epoch": 0.25231402125471375,
      "grad_norm": 2.2496277356274805,
      "learning_rate": 9.338135320708912e-06,
      "logits/chosen": -0.7474158406257629,
      "logits/rejected": -0.6932293176651001,
      "logps/chosen": -0.6186713576316833,
      "logps/rejected": -0.6508045196533203,
      "loss": 0.7338,
      "odds_ratio_loss": 7.302546501159668,
      "rewards/accuracies": 0.359375,
      "rewards/chosen": -0.06508044898509979,
      "rewards/margins": -0.003213312244042754,
      "rewards/rejected": -0.06186714768409729,
      "sft_loss": 0.003513358999043703,
      "step": 46
    },
    {
      "epoch": 0.2577991086732945,
      "grad_norm": 2.229413899291416,
      "learning_rate": 9.289416964704186e-06,
      "logits/chosen": -0.7597657442092896,
      "logits/rejected": -0.7157298922538757,
      "logps/chosen": -0.5537360906600952,
      "logps/rejected": -0.6344780921936035,
      "loss": 0.7248,
      "odds_ratio_loss": 7.21159029006958,
      "rewards/accuracies": 0.3359375,
      "rewards/chosen": -0.06344781070947647,
      "rewards/margins": -0.008074202574789524,
      "rewards/rejected": -0.05537361279129982,
      "sft_loss": 0.0036083455197513103,
      "step": 47
    },
    {
      "epoch": 0.2632841960918752,
      "grad_norm": 1.8803176092666092,
      "learning_rate": 9.239105265402525e-06,
      "logits/chosen": -0.727577269077301,
      "logits/rejected": -0.70409095287323,
      "logps/chosen": -0.5894535779953003,
      "logps/rejected": -0.6098184585571289,
      "loss": 0.692,
      "odds_ratio_loss": 6.8936448097229,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.06098184362053871,
      "rewards/margins": -0.002036482561379671,
      "rewards/rejected": -0.058945365250110626,
      "sft_loss": 0.0026457542553544044,
      "step": 48
    },
    {
      "epoch": 0.2687692835104559,
      "grad_norm": 2.3520901683998137,
      "learning_rate": 9.187218911546363e-06,
      "logits/chosen": -0.7061797976493835,
      "logits/rejected": -0.6750433444976807,
      "logps/chosen": -0.6185547709465027,
      "logps/rejected": -0.6763335466384888,
      "loss": 0.7632,
      "odds_ratio_loss": 7.58837890625,
      "rewards/accuracies": 0.34375,
      "rewards/chosen": -0.06763336062431335,
      "rewards/margins": -0.005777883809059858,
      "rewards/rejected": -0.06185547634959221,
      "sft_loss": 0.004371694289147854,
      "step": 49
    },
    {
      "epoch": 0.2742543709290367,
      "grad_norm": 2.3579251994795682,
      "learning_rate": 9.133777176798013e-06,
      "logits/chosen": -0.6374361515045166,
      "logits/rejected": -0.6806906461715698,
      "logps/chosen": -0.6513757705688477,
      "logps/rejected": -0.6520736813545227,
      "loss": 0.7371,
      "odds_ratio_loss": 7.324197292327881,
      "rewards/accuracies": 0.4453125,
      "rewards/chosen": -0.06520736962556839,
      "rewards/margins": -6.9792615249753e-05,
      "rewards/rejected": -0.065137580037117,
      "sft_loss": 0.004632241558283567,
      "step": 50
    },
    {
      "epoch": 0.27973945834761743,
      "grad_norm": 2.2191064607544746,
      "learning_rate": 9.078799912580305e-06,
      "logits/chosen": -0.6874204277992249,
      "logits/rejected": -0.6352756023406982,
      "logps/chosen": -0.6136583685874939,
      "logps/rejected": -0.5992326736450195,
      "loss": 0.6819,
      "odds_ratio_loss": 6.7846455574035645,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.059923265129327774,
      "rewards/margins": 0.0014425672125071287,
      "rewards/rejected": -0.06136583536863327,
      "sft_loss": 0.003475453471764922,
      "step": 51
    },
    {
      "epoch": 0.28522454576619816,
      "grad_norm": 2.3179034735535966,
      "learning_rate": 9.022307540702576e-06,
      "logits/chosen": -0.7218315005302429,
      "logits/rejected": -0.6824548840522766,
      "logps/chosen": -0.6579533815383911,
      "logps/rejected": -0.688935399055481,
      "loss": 0.7754,
      "odds_ratio_loss": 7.713038444519043,
      "rewards/accuracies": 0.34375,
      "rewards/chosen": -0.06889353692531586,
      "rewards/margins": -0.0030981944873929024,
      "rewards/rejected": -0.06579533964395523,
      "sft_loss": 0.004128783475607634,
      "step": 52
    },
    {
      "epoch": 0.2907096331847789,
      "grad_norm": 2.422506021035563,
      "learning_rate": 8.964321045774808e-06,
      "logits/chosen": -0.6635921597480774,
      "logits/rejected": -0.6691629886627197,
      "logps/chosen": -0.6667566895484924,
      "logps/rejected": -0.6598675847053528,
      "loss": 0.7415,
      "odds_ratio_loss": 7.367385387420654,
      "rewards/accuracies": 0.4140625,
      "rewards/chosen": -0.0659867525100708,
      "rewards/margins": 0.0006889166543260217,
      "rewards/rejected": -0.06667566299438477,
      "sft_loss": 0.004781263880431652,
      "step": 53
    },
    {
      "epoch": 0.2961947206033596,
      "grad_norm": 2.3663856409710604,
      "learning_rate": 8.904861967412702e-06,
      "logits/chosen": -0.729520857334137,
      "logits/rejected": -0.721049427986145,
      "logps/chosen": -0.6047177314758301,
      "logps/rejected": -0.6680710315704346,
      "loss": 0.7525,
      "odds_ratio_loss": 7.480541706085205,
      "rewards/accuracies": 0.390625,
      "rewards/chosen": -0.0668071061372757,
      "rewards/margins": -0.006335328333079815,
      "rewards/rejected": -0.060471780598163605,
      "sft_loss": 0.004433046095073223,
      "step": 54
    },
    {
      "epoch": 0.30167980802194033,
      "grad_norm": 5.907064022641608,
      "learning_rate": 8.843952392236595e-06,
      "logits/chosen": -0.7405853867530823,
      "logits/rejected": -0.6213741302490234,
      "logps/chosen": -0.6097532510757446,
      "logps/rejected": -0.7615051865577698,
      "loss": 0.8548,
      "odds_ratio_loss": 7.123908042907715,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.07615052163600922,
      "rewards/margins": -0.015175196342170238,
      "rewards/rejected": -0.060975320637226105,
      "sft_loss": 0.1424168199300766,
      "step": 55
    },
    {
      "epoch": 0.30716489544052106,
      "grad_norm": 2.3801144562859733,
      "learning_rate": 8.78161494566717e-06,
      "logits/chosen": -0.7353604435920715,
      "logits/rejected": -0.6483398675918579,
      "logps/chosen": -0.628172755241394,
      "logps/rejected": -0.6272903084754944,
      "loss": 0.7044,
      "odds_ratio_loss": 7.003999710083008,
      "rewards/accuracies": 0.421875,
      "rewards/chosen": -0.06272903829813004,
      "rewards/margins": 8.82376916706562e-05,
      "rewards/rejected": -0.0628172755241394,
      "sft_loss": 0.004041117150336504,
      "step": 56
    },
    {
      "epoch": 0.31264998285910184,
      "grad_norm": 2.0180451466126073,
      "learning_rate": 8.717872783521048e-06,
      "logits/chosen": -0.7183849215507507,
      "logits/rejected": -0.6541566848754883,
      "logps/chosen": -0.7877294421195984,
      "logps/rejected": -0.6478090882301331,
      "loss": 0.7258,
      "odds_ratio_loss": 7.223598480224609,
      "rewards/accuracies": 0.453125,
      "rewards/chosen": -0.06478091329336166,
      "rewards/margins": 0.01399203110486269,
      "rewards/rejected": -0.07877294719219208,
      "sft_loss": 0.0034678103402256966,
      "step": 57
    },
    {
      "epoch": 0.31813507027768256,
      "grad_norm": 2.2432673044491187,
      "learning_rate": 8.65274958340934e-06,
      "logits/chosen": -0.7320058345794678,
      "logits/rejected": -0.6547486186027527,
      "logps/chosen": -0.6831153631210327,
      "logps/rejected": -0.6445227265357971,
      "loss": 0.7223,
      "odds_ratio_loss": 7.180177688598633,
      "rewards/accuracies": 0.4453125,
      "rewards/chosen": -0.06445227563381195,
      "rewards/margins": 0.0038592619821429253,
      "rewards/rejected": -0.06831153482198715,
      "sft_loss": 0.004313957877457142,
      "step": 58
    },
    {
      "epoch": 0.3236201576962633,
      "grad_norm": 2.253712626261103,
      "learning_rate": 8.586269535942386e-06,
      "logits/chosen": -0.7257508039474487,
      "logits/rejected": -0.6928269267082214,
      "logps/chosen": -0.6556897759437561,
      "logps/rejected": -0.6581557393074036,
      "loss": 0.7387,
      "odds_ratio_loss": 7.349256992340088,
      "rewards/accuracies": 0.4140625,
      "rewards/chosen": -0.06581557542085648,
      "rewards/margins": -0.00024658930487930775,
      "rewards/rejected": -0.06556899100542068,
      "sft_loss": 0.003765275003388524,
      "step": 59
    },
    {
      "epoch": 0.329105245114844,
      "grad_norm": 2.2071752316866995,
      "learning_rate": 8.518457335743927e-06,
      "logits/chosen": -0.679678201675415,
      "logits/rejected": -0.664280891418457,
      "logps/chosen": -0.7143180966377258,
      "logps/rejected": -0.6450636982917786,
      "loss": 0.7221,
      "odds_ratio_loss": 7.182507514953613,
      "rewards/accuracies": 0.4609375,
      "rewards/chosen": -0.06450636684894562,
      "rewards/margins": 0.0069254375994205475,
      "rewards/rejected": -0.07143180072307587,
      "sft_loss": 0.0038413461297750473,
      "step": 60
    },
    {
      "epoch": 0.33459033253342474,
      "grad_norm": 2.592752168909039,
      "learning_rate": 8.44933817227806e-06,
      "logits/chosen": -0.7278729677200317,
      "logits/rejected": -0.6981578469276428,
      "logps/chosen": -0.7912625074386597,
      "logps/rejected": -0.7018301486968994,
      "loss": 0.7851,
      "odds_ratio_loss": 7.783860206604004,
      "rewards/accuracies": 0.3828125,
      "rewards/chosen": -0.07018300890922546,
      "rewards/margins": 0.008943240158259869,
      "rewards/rejected": -0.0791262537240982,
      "sft_loss": 0.0066674333065748215,
      "step": 61
    },
    {
      "epoch": 0.34007541995200546,
      "grad_norm": 5.454393119042178,
      "learning_rate": 8.378937720492384e-06,
      "logits/chosen": -0.7215204238891602,
      "logits/rejected": -0.6841076016426086,
      "logps/chosen": -0.7616227865219116,
      "logps/rejected": -0.6432775259017944,
      "loss": 0.7209,
      "odds_ratio_loss": 7.020868301391602,
      "rewards/accuracies": 0.4609375,
      "rewards/chosen": -0.06432775408029556,
      "rewards/margins": 0.0118345245718956,
      "rewards/rejected": -0.07616227120161057,
      "sft_loss": 0.01877610757946968,
      "step": 62
    },
    {
      "epoch": 0.34556050737058625,
      "grad_norm": 2.0687414102448085,
      "learning_rate": 8.307282131280805e-06,
      "logits/chosen": -0.7056500315666199,
      "logits/rejected": -0.6676835417747498,
      "logps/chosen": -0.6483266353607178,
      "logps/rejected": -0.626646876335144,
      "loss": 0.7043,
      "odds_ratio_loss": 7.007047176361084,
      "rewards/accuracies": 0.3984375,
      "rewards/chosen": -0.062664695084095,
      "rewards/margins": 0.0021679732017219067,
      "rewards/rejected": -0.0648326650261879,
      "sft_loss": 0.003634248161688447,
      "step": 63
    },
    {
      "epoch": 0.35104559478916697,
      "grad_norm": 2.3475998338820365,
      "learning_rate": 8.234398021769541e-06,
      "logits/chosen": -0.743563175201416,
      "logits/rejected": -0.7020170092582703,
      "logps/chosen": -0.7933846712112427,
      "logps/rejected": -0.6398557424545288,
      "loss": 0.718,
      "odds_ratio_loss": 7.136807441711426,
      "rewards/accuracies": 0.4609375,
      "rewards/chosen": -0.06398558616638184,
      "rewards/margins": 0.015352879650890827,
      "rewards/rejected": -0.07933846116065979,
      "sft_loss": 0.004295012913644314,
      "step": 64
    },
    {
      "epoch": 0.3565306822077477,
      "grad_norm": 2.240900663083349,
      "learning_rate": 8.160312465429952e-06,
      "logits/chosen": -0.7589367628097534,
      "logits/rejected": -0.6660048365592957,
      "logps/chosen": -0.8276284337043762,
      "logps/rejected": -0.6442070603370667,
      "loss": 0.7207,
      "odds_ratio_loss": 7.165534973144531,
      "rewards/accuracies": 0.4609375,
      "rewards/chosen": -0.06442070752382278,
      "rewards/margins": 0.018342135474085808,
      "rewards/rejected": -0.08276285231113434,
      "sft_loss": 0.004189603962004185,
      "step": 65
    },
    {
      "epoch": 0.3620157696263284,
      "grad_norm": 2.2431116846423977,
      "learning_rate": 8.085052982021849e-06,
      "logits/chosen": -0.7091330289840698,
      "logits/rejected": -0.6447920799255371,
      "logps/chosen": -0.6476763486862183,
      "logps/rejected": -0.6205025315284729,
      "loss": 0.7019,
      "odds_ratio_loss": 6.978389739990234,
      "rewards/accuracies": 0.3984375,
      "rewards/chosen": -0.06205025315284729,
      "rewards/margins": 0.0027173825073987246,
      "rewards/rejected": -0.06476763635873795,
      "sft_loss": 0.004105943720787764,
      "step": 66
    },
    {
      "epoch": 0.36750085704490915,
      "grad_norm": 2.4189727594993613,
      "learning_rate": 8.008647527371022e-06,
      "logits/chosen": -0.7610785961151123,
      "logits/rejected": -0.7081894874572754,
      "logps/chosen": -0.8307157754898071,
      "logps/rejected": -0.6853421330451965,
      "loss": 0.7664,
      "odds_ratio_loss": 7.616702079772949,
      "rewards/accuracies": 0.4609375,
      "rewards/chosen": -0.06853421032428741,
      "rewards/margins": 0.014537353999912739,
      "rewards/rejected": -0.08307155966758728,
      "sft_loss": 0.004689488559961319,
      "step": 67
    },
    {
      "epoch": 0.37298594446348987,
      "grad_norm": 2.1080406296134604,
      "learning_rate": 7.931124482984802e-06,
      "logits/chosen": -0.7000724077224731,
      "logits/rejected": -0.648707926273346,
      "logps/chosen": -0.7306436896324158,
      "logps/rejected": -0.6893948912620544,
      "loss": 0.7721,
      "odds_ratio_loss": 7.686524391174316,
      "rewards/accuracies": 0.359375,
      "rewards/chosen": -0.06893948465585709,
      "rewards/margins": 0.004124888684600592,
      "rewards/rejected": -0.07306437194347382,
      "sft_loss": 0.0034583115484565496,
      "step": 68
    },
    {
      "epoch": 0.3784710318820706,
      "grad_norm": 2.279048108347312,
      "learning_rate": 7.85251264550948e-06,
      "logits/chosen": -0.7173571586608887,
      "logits/rejected": -0.6854091882705688,
      "logps/chosen": -0.8085934519767761,
      "logps/rejected": -0.6753490567207336,
      "loss": 0.7495,
      "odds_ratio_loss": 7.4520111083984375,
      "rewards/accuracies": 0.4609375,
      "rewards/chosen": -0.067534901201725,
      "rewards/margins": 0.013324443250894547,
      "rewards/rejected": -0.08085935562849045,
      "sft_loss": 0.004327339120209217,
      "step": 69
    },
    {
      "epoch": 0.3839561193006514,
      "grad_norm": 2.2929493653439623,
      "learning_rate": 7.772841216033534e-06,
      "logits/chosen": -0.7415894865989685,
      "logits/rejected": -0.7029559016227722,
      "logps/chosen": -0.7581257224082947,
      "logps/rejected": -0.6466966867446899,
      "loss": 0.7216,
      "odds_ratio_loss": 7.17643404006958,
      "rewards/accuracies": 0.4609375,
      "rewards/chosen": -0.06466967612504959,
      "rewards/margins": 0.011142895556986332,
      "rewards/rejected": -0.07581256330013275,
      "sft_loss": 0.003946883603930473,
      "step": 70
    },
    {
      "epoch": 0.3894412067192321,
      "grad_norm": 2.4115483219556104,
      "learning_rate": 7.692139789240611e-06,
      "logits/chosen": -0.7181853652000427,
      "logits/rejected": -0.7305399775505066,
      "logps/chosen": -0.5790694952011108,
      "logps/rejected": -0.6291408538818359,
      "loss": 0.7136,
      "odds_ratio_loss": 7.071855068206787,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.06291408091783524,
      "rewards/margins": -0.005007130093872547,
      "rewards/rejected": -0.057906948029994965,
      "sft_loss": 0.00638266047462821,
      "step": 71
    },
    {
      "epoch": 0.39492629413781283,
      "grad_norm": 2.585528390215209,
      "learning_rate": 7.61043834241632e-06,
      "logits/chosen": -0.7213830947875977,
      "logits/rejected": -0.6931593418121338,
      "logps/chosen": -0.6970454454421997,
      "logps/rejected": -0.6629546880722046,
      "loss": 0.7414,
      "odds_ratio_loss": 7.3666510581970215,
      "rewards/accuracies": 0.421875,
      "rewards/chosen": -0.06629546731710434,
      "rewards/margins": 0.003409074852243066,
      "rewards/rejected": -0.06970454007387161,
      "sft_loss": 0.004775386769324541,
      "step": 72
    },
    {
      "epoch": 0.40041138155639355,
      "grad_norm": 2.5206082590938657,
      "learning_rate": 7.527767224312883e-06,
      "logits/chosen": -0.6767313480377197,
      "logits/rejected": -0.6348490118980408,
      "logps/chosen": -0.7730945348739624,
      "logps/rejected": -0.6511149406433105,
      "loss": 0.7309,
      "odds_ratio_loss": 7.25616455078125,
      "rewards/accuracies": 0.4296875,
      "rewards/chosen": -0.06511148810386658,
      "rewards/margins": 0.012197963893413544,
      "rewards/rejected": -0.07730945199728012,
      "sft_loss": 0.005264699459075928,
      "step": 73
    },
    {
      "epoch": 0.4058964689749743,
      "grad_norm": 2.342735729722048,
      "learning_rate": 7.44415714387582e-06,
      "logits/chosen": -0.7068739533424377,
      "logits/rejected": -0.6443241834640503,
      "logps/chosen": -0.6506850123405457,
      "logps/rejected": -0.660474419593811,
      "loss": 0.7423,
      "odds_ratio_loss": 7.371833801269531,
      "rewards/accuracies": 0.390625,
      "rewards/chosen": -0.06604744493961334,
      "rewards/margins": -0.0009789422620087862,
      "rewards/rejected": -0.06506850570440292,
      "sft_loss": 0.005163977388292551,
      "step": 74
    },
    {
      "epoch": 0.411381556393555,
      "grad_norm": 2.244946483469262,
      "learning_rate": 7.359639158836828e-06,
      "logits/chosen": -0.6814281940460205,
      "logits/rejected": -0.6734283566474915,
      "logps/chosen": -0.7911446690559387,
      "logps/rejected": -0.6680716276168823,
      "loss": 0.7437,
      "odds_ratio_loss": 7.399670124053955,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.06680716574192047,
      "rewards/margins": 0.012307303957641125,
      "rewards/rejected": -0.07911446690559387,
      "sft_loss": 0.0037362282164394855,
      "step": 75
    },
    {
      "epoch": 0.41686664381213573,
      "grad_norm": 1.9776378551597844,
      "learning_rate": 7.2742446641770985e-06,
      "logits/chosen": -0.6895711421966553,
      "logits/rejected": -0.6461458802223206,
      "logps/chosen": -0.7058255076408386,
      "logps/rejected": -0.6571383476257324,
      "loss": 0.7396,
      "odds_ratio_loss": 7.361542224884033,
      "rewards/accuracies": 0.3359375,
      "rewards/chosen": -0.06571383774280548,
      "rewards/margins": 0.004868712741881609,
      "rewards/rejected": -0.0705825537443161,
      "sft_loss": 0.0034039851743727922,
      "step": 76
    },
    {
      "epoch": 0.4223517312307165,
      "grad_norm": 2.5525880372678342,
      "learning_rate": 7.188005380465365e-06,
      "logits/chosen": -0.672836184501648,
      "logits/rejected": -0.6136391162872314,
      "logps/chosen": -0.7278799414634705,
      "logps/rejected": -0.6120200753211975,
      "loss": 0.6869,
      "odds_ratio_loss": 6.825428485870361,
      "rewards/accuracies": 0.4765625,
      "rewards/chosen": -0.061201997101306915,
      "rewards/margins": 0.011585984379053116,
      "rewards/rejected": -0.07278798520565033,
      "sft_loss": 0.004372562747448683,
      "step": 77
    },
    {
      "epoch": 0.42783681864929723,
      "grad_norm": 2.2583507815548707,
      "learning_rate": 7.10095334207501e-06,
      "logits/chosen": -0.6909606456756592,
      "logits/rejected": -0.6990565061569214,
      "logps/chosen": -0.8577082753181458,
      "logps/rejected": -0.6745601296424866,
      "loss": 0.7559,
      "odds_ratio_loss": 7.520651340484619,
      "rewards/accuracies": 0.390625,
      "rewards/chosen": -0.06745601445436478,
      "rewards/margins": 0.018314823508262634,
      "rewards/rejected": -0.08577083051204681,
      "sft_loss": 0.0038547185249626637,
      "step": 78
    },
    {
      "epoch": 0.43332190606787796,
      "grad_norm": 2.1348804607525684,
      "learning_rate": 7.013120885284599e-06,
      "logits/chosen": -0.8005620241165161,
      "logits/rejected": -0.745074987411499,
      "logps/chosen": -0.7438629865646362,
      "logps/rejected": -0.6111059784889221,
      "loss": 0.6881,
      "odds_ratio_loss": 6.849056720733643,
      "rewards/accuracies": 0.453125,
      "rewards/chosen": -0.06111060082912445,
      "rewards/margins": 0.013275692239403725,
      "rewards/rejected": -0.07438629865646362,
      "sft_loss": 0.0032314579002559185,
      "step": 79
    },
    {
      "epoch": 0.4388069934864587,
      "grad_norm": 2.2851255790088536,
      "learning_rate": 6.924540636266272e-06,
      "logits/chosen": -0.8250284194946289,
      "logits/rejected": -0.7828257083892822,
      "logps/chosen": -0.6604968905448914,
      "logps/rejected": -0.6587929129600525,
      "loss": 0.7421,
      "odds_ratio_loss": 7.378921985626221,
      "rewards/accuracies": 0.3515625,
      "rewards/chosen": -0.06587929278612137,
      "rewards/margins": 0.0001703993184491992,
      "rewards/rejected": -0.06604968756437302,
      "sft_loss": 0.004175678826868534,
      "step": 80
    },
    {
      "epoch": 0.4442920809050394,
      "grad_norm": 2.2595985001445165,
      "learning_rate": 6.835245498966461e-06,
      "logits/chosen": -0.7141305208206177,
      "logits/rejected": -0.6828845739364624,
      "logps/chosen": -0.8394588828086853,
      "logps/rejected": -0.6575326919555664,
      "loss": 0.7341,
      "odds_ratio_loss": 7.295228958129883,
      "rewards/accuracies": 0.4140625,
      "rewards/chosen": -0.0657532662153244,
      "rewards/margins": 0.018192624673247337,
      "rewards/rejected": -0.08394590020179749,
      "sft_loss": 0.004541095811873674,
      "step": 81
    },
    {
      "epoch": 0.44977716832362014,
      "grad_norm": 2.2335414228759576,
      "learning_rate": 6.7452686428834045e-06,
      "logits/chosen": -0.7194411754608154,
      "logits/rejected": -0.6863387227058411,
      "logps/chosen": -0.8470317721366882,
      "logps/rejected": -0.6532705426216125,
      "loss": 0.73,
      "odds_ratio_loss": 7.253733158111572,
      "rewards/accuracies": 0.4140625,
      "rewards/chosen": -0.06532706320285797,
      "rewards/margins": 0.019376114010810852,
      "rewards/rejected": -0.08470318466424942,
      "sft_loss": 0.00458897277712822,
      "step": 82
    },
    {
      "epoch": 0.4552622557422009,
      "grad_norm": 2.054108081993998,
      "learning_rate": 6.654643490746042e-06,
      "logits/chosen": -0.7527655959129333,
      "logits/rejected": -0.7424032092094421,
      "logps/chosen": -0.676724374294281,
      "logps/rejected": -0.6338033080101013,
      "loss": 0.7136,
      "odds_ratio_loss": 7.100609302520752,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.06338033080101013,
      "rewards/margins": 0.004292105324566364,
      "rewards/rejected": -0.06767243146896362,
      "sft_loss": 0.003586169332265854,
      "step": 83
    },
    {
      "epoch": 0.46074734316078164,
      "grad_norm": 2.0252693365171623,
      "learning_rate": 6.563403706098833e-06,
      "logits/chosen": -0.7776511907577515,
      "logits/rejected": -0.7408439517021179,
      "logps/chosen": -0.7689855098724365,
      "logps/rejected": -0.6419221758842468,
      "loss": 0.7196,
      "odds_ratio_loss": 7.161562919616699,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.06419222801923752,
      "rewards/margins": 0.012706330046057701,
      "rewards/rejected": -0.07689855247735977,
      "sft_loss": 0.0034365493338555098,
      "step": 84
    },
    {
      "epoch": 0.46623243057936237,
      "grad_norm": 2.405711155989815,
      "learning_rate": 6.471583180797121e-06,
      "logits/chosen": -0.7478362321853638,
      "logits/rejected": -0.7417604327201843,
      "logps/chosen": -0.8892000317573547,
      "logps/rejected": -0.6463872790336609,
      "loss": 0.726,
      "odds_ratio_loss": 7.2149338722229,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.06463872641324997,
      "rewards/margins": 0.024281270802021027,
      "rewards/rejected": -0.088919997215271,
      "sft_loss": 0.004515405744314194,
      "step": 85
    },
    {
      "epoch": 0.4717175179979431,
      "grad_norm": 2.327436937255859,
      "learning_rate": 6.379216022417695e-06,
      "logits/chosen": -0.713384211063385,
      "logits/rejected": -0.7074405550956726,
      "logps/chosen": -0.8071316480636597,
      "logps/rejected": -0.6722535490989685,
      "loss": 0.7491,
      "odds_ratio_loss": 7.439596652984619,
      "rewards/accuracies": 0.453125,
      "rewards/chosen": -0.06722535192966461,
      "rewards/margins": 0.013487813994288445,
      "rewards/rejected": -0.0807131677865982,
      "sft_loss": 0.005136145744472742,
      "step": 86
    },
    {
      "epoch": 0.4772026054165238,
      "grad_norm": 4.597101480060567,
      "learning_rate": 6.286336541589224e-06,
      "logits/chosen": -0.8244719505310059,
      "logits/rejected": -0.793204128742218,
      "logps/chosen": -0.9169348478317261,
      "logps/rejected": -0.6403506398200989,
      "loss": 0.7141,
      "odds_ratio_loss": 6.974415302276611,
      "rewards/accuracies": 0.4765625,
      "rewards/chosen": -0.06403505802154541,
      "rewards/margins": 0.027658430859446526,
      "rewards/rejected": -0.09169349074363708,
      "sft_loss": 0.0166785828769207,
      "step": 87
    },
    {
      "epoch": 0.48268769283510454,
      "grad_norm": 2.98534441586508,
      "learning_rate": 6.192979239247243e-06,
      "logits/chosen": -0.7573586106300354,
      "logits/rejected": -0.7331765294075012,
      "logps/chosen": -0.7313320636749268,
      "logps/rejected": -0.6538473963737488,
      "loss": 0.7336,
      "odds_ratio_loss": 7.25970983505249,
      "rewards/accuracies": 0.390625,
      "rewards/chosen": -0.06538473814725876,
      "rewards/margins": 0.007748466916382313,
      "rewards/rejected": -0.0731332078576088,
      "sft_loss": 0.007589638698846102,
      "step": 88
    },
    {
      "epoch": 0.48817278025368527,
      "grad_norm": 3.060607573744663,
      "learning_rate": 6.099178793818479e-06,
      "logits/chosen": -0.8188400268554688,
      "logits/rejected": -0.7003345489501953,
      "logps/chosen": -0.7988659143447876,
      "logps/rejected": -0.6332603096961975,
      "loss": 0.7111,
      "odds_ratio_loss": 7.054601192474365,
      "rewards/accuracies": 0.5078125,
      "rewards/chosen": -0.06332603842020035,
      "rewards/margins": 0.01656055450439453,
      "rewards/rejected": -0.07988659292459488,
      "sft_loss": 0.005684853531420231,
      "step": 89
    },
    {
      "epoch": 0.49365786767226605,
      "grad_norm": 3.426360021441316,
      "learning_rate": 6.0049700483392256e-06,
      "logits/chosen": -0.7684093713760376,
      "logits/rejected": -0.7432892322540283,
      "logps/chosen": -0.6437249183654785,
      "logps/rejected": -0.6601754426956177,
      "loss": 0.7411,
      "odds_ratio_loss": 7.306463241577148,
      "rewards/accuracies": 0.390625,
      "rewards/chosen": -0.06601753830909729,
      "rewards/margins": -0.0016450518742203712,
      "rewards/rejected": -0.06437249481678009,
      "sft_loss": 0.01042965892702341,
      "step": 90
    },
    {
      "epoch": 0.4991429550908468,
      "grad_norm": 2.462338863296407,
      "learning_rate": 5.910387997512573e-06,
      "logits/chosen": -0.7122192978858948,
      "logits/rejected": -0.7019898295402527,
      "logps/chosen": -0.7473130822181702,
      "logps/rejected": -0.68088299036026,
      "loss": 0.7631,
      "odds_ratio_loss": 7.561764240264893,
      "rewards/accuracies": 0.3984375,
      "rewards/chosen": -0.06808829307556152,
      "rewards/margins": 0.006643014028668404,
      "rewards/rejected": -0.07473130524158478,
      "sft_loss": 0.0068762716837227345,
      "step": 91
    },
    {
      "epoch": 0.5046280425094275,
      "grad_norm": 2.0658511683882184,
      "learning_rate": 5.815467774709314e-06,
      "logits/chosen": -0.8243841528892517,
      "logits/rejected": -0.7515370845794678,
      "logps/chosen": -0.8632618188858032,
      "logps/rejected": -0.6042035222053528,
      "loss": 0.6749,
      "odds_ratio_loss": 6.714731693267822,
      "rewards/accuracies": 0.484375,
      "rewards/chosen": -0.06042035296559334,
      "rewards/margins": 0.025905832648277283,
      "rewards/rejected": -0.08632618188858032,
      "sft_loss": 0.0033876379020512104,
      "step": 92
    },
    {
      "epoch": 0.5101131299280083,
      "grad_norm": 2.387632371765397,
      "learning_rate": 5.7202446389173225e-06,
      "logits/chosen": -0.7121865749359131,
      "logits/rejected": -0.6870692372322083,
      "logps/chosen": -0.8353071212768555,
      "logps/rejected": -0.6251768469810486,
      "loss": 0.6945,
      "odds_ratio_loss": 6.89461612701416,
      "rewards/accuracies": 0.5078125,
      "rewards/chosen": -0.0625176802277565,
      "rewards/margins": 0.021013032644987106,
      "rewards/rejected": -0.0835307165980339,
      "sft_loss": 0.0050766305066645145,
      "step": 93
    },
    {
      "epoch": 0.515598217346589,
      "grad_norm": 2.4212503643030865,
      "learning_rate": 5.624753961644281e-06,
      "logits/chosen": -0.7463563084602356,
      "logits/rejected": -0.7374889850616455,
      "logps/chosen": -0.6804830431938171,
      "logps/rejected": -0.6773659586906433,
      "loss": 0.7635,
      "odds_ratio_loss": 7.582502841949463,
      "rewards/accuracies": 0.3359375,
      "rewards/chosen": -0.06773659586906433,
      "rewards/margins": 0.00031171052251011133,
      "rewards/rejected": -0.06804830580949783,
      "sft_loss": 0.005270948633551598,
      "step": 94
    },
    {
      "epoch": 0.5210833047651697,
      "grad_norm": 2.1125220078129914,
      "learning_rate": 5.529031213778615e-06,
      "logits/chosen": -0.7246003150939941,
      "logits/rejected": -0.7458670139312744,
      "logps/chosen": -0.6863371133804321,
      "logps/rejected": -0.642282247543335,
      "loss": 0.7268,
      "odds_ratio_loss": 7.228869438171387,
      "rewards/accuracies": 0.3984375,
      "rewards/chosen": -0.06422823667526245,
      "rewards/margins": 0.0044054812751710415,
      "rewards/rejected": -0.06863371282815933,
      "sft_loss": 0.0038752383552491665,
      "step": 95
    },
    {
      "epoch": 0.5265683921837504,
      "grad_norm": 1.9888533547489935,
      "learning_rate": 5.433111952413496e-06,
      "logits/chosen": -0.8252724409103394,
      "logits/rejected": -0.7656226754188538,
      "logps/chosen": -0.698495090007782,
      "logps/rejected": -0.6140046715736389,
      "loss": 0.6895,
      "odds_ratio_loss": 6.8552350997924805,
      "rewards/accuracies": 0.5234375,
      "rewards/chosen": -0.06140046939253807,
      "rewards/margins": 0.008449044078588486,
      "rewards/rejected": -0.06984951347112656,
      "sft_loss": 0.003965131007134914,
      "step": 96
    },
    {
      "epoch": 0.5320534796023312,
      "grad_norm": 2.5827652079178165,
      "learning_rate": 5.3370318076388405e-06,
      "logits/chosen": -0.7367005348205566,
      "logits/rejected": -0.7038895487785339,
      "logps/chosen": -0.8228276968002319,
      "logps/rejected": -0.6827890872955322,
      "loss": 0.7607,
      "odds_ratio_loss": 7.550937175750732,
      "rewards/accuracies": 0.390625,
      "rewards/chosen": -0.06827891618013382,
      "rewards/margins": 0.014003856107592583,
      "rewards/rejected": -0.08228276669979095,
      "sft_loss": 0.0056315576657652855,
      "step": 97
    },
    {
      "epoch": 0.5375385670209119,
      "grad_norm": 2.0893113762601585,
      "learning_rate": 5.240826469306187e-06,
      "logits/chosen": -0.8139528632164001,
      "logits/rejected": -0.7607035636901855,
      "logps/chosen": -0.889180064201355,
      "logps/rejected": -0.6422348618507385,
      "loss": 0.7188,
      "odds_ratio_loss": 7.150365829467773,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.06422348320484161,
      "rewards/margins": 0.0246945321559906,
      "rewards/rejected": -0.08891801536083221,
      "sft_loss": 0.003753484459593892,
      "step": 98
    },
    {
      "epoch": 0.5430236544394926,
      "grad_norm": 2.5109300003185537,
      "learning_rate": 5.144531673771364e-06,
      "logits/chosen": -0.8394501209259033,
      "logits/rejected": -0.7594123482704163,
      "logps/chosen": -1.1132104396820068,
      "logps/rejected": -0.678527295589447,
      "loss": 0.7596,
      "odds_ratio_loss": 7.536545753479004,
      "rewards/accuracies": 0.3984375,
      "rewards/chosen": -0.06785273551940918,
      "rewards/margins": 0.04346831887960434,
      "rewards/rejected": -0.11132105439901352,
      "sft_loss": 0.005897625349462032,
      "step": 99
    },
    {
      "epoch": 0.5485087418580734,
      "grad_norm": 2.363027336814075,
      "learning_rate": 5.048183190619904e-06,
      "logits/chosen": -0.776432454586029,
      "logits/rejected": -0.7214481830596924,
      "logps/chosen": -0.8866347074508667,
      "logps/rejected": -0.6367273330688477,
      "loss": 0.7085,
      "odds_ratio_loss": 7.037672996520996,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06367272883653641,
      "rewards/margins": 0.024990737438201904,
      "rewards/rejected": -0.08866347372531891,
      "sft_loss": 0.004776695277541876,
      "step": 100
    },
    {
      "epoch": 0.5539938292766541,
      "grad_norm": 1.935622563669197,
      "learning_rate": 4.951816809380098e-06,
      "logits/chosen": -0.7725133895874023,
      "logits/rejected": -0.7367417812347412,
      "logps/chosen": -0.7350077629089355,
      "logps/rejected": -0.5911646485328674,
      "loss": 0.6656,
      "odds_ratio_loss": 6.6231279373168945,
      "rewards/accuracies": 0.4765625,
      "rewards/chosen": -0.059116464108228683,
      "rewards/margins": 0.014384310692548752,
      "rewards/rejected": -0.07350076735019684,
      "sft_loss": 0.003327850252389908,
      "step": 101
    },
    {
      "epoch": 0.5594789166952349,
      "grad_norm": 2.2992228974600817,
      "learning_rate": 4.855468326228638e-06,
      "logits/chosen": -0.8631143569946289,
      "logits/rejected": -0.8134572505950928,
      "logps/chosen": -0.7315545082092285,
      "logps/rejected": -0.6632150411605835,
      "loss": 0.7458,
      "odds_ratio_loss": 7.4111127853393555,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.06632150709629059,
      "rewards/margins": 0.006833940744400024,
      "rewards/rejected": -0.07315544784069061,
      "sft_loss": 0.004706574138253927,
      "step": 102
    },
    {
      "epoch": 0.5649640041138155,
      "grad_norm": 2.0889788103776255,
      "learning_rate": 4.7591735306938144e-06,
      "logits/chosen": -0.790824294090271,
      "logits/rejected": -0.7726211547851562,
      "logps/chosen": -0.6989539861679077,
      "logps/rejected": -0.6077806949615479,
      "loss": 0.6848,
      "odds_ratio_loss": 6.813911437988281,
      "rewards/accuracies": 0.4140625,
      "rewards/chosen": -0.06077807396650314,
      "rewards/margins": 0.00911732017993927,
      "rewards/rejected": -0.06989538669586182,
      "sft_loss": 0.0033795726485550404,
      "step": 103
    },
    {
      "epoch": 0.5704490915323963,
      "grad_norm": 2.0996097283080295,
      "learning_rate": 4.662968192361161e-06,
      "logits/chosen": -0.8079996705055237,
      "logits/rejected": -0.7932396531105042,
      "logps/chosen": -0.7709510326385498,
      "logps/rejected": -0.6129697561264038,
      "loss": 0.6869,
      "odds_ratio_loss": 6.833165645599365,
      "rewards/accuracies": 0.4453125,
      "rewards/chosen": -0.0612969771027565,
      "rewards/margins": 0.015798134729266167,
      "rewards/rejected": -0.07709510624408722,
      "sft_loss": 0.0036084074527025223,
      "step": 104
    },
    {
      "epoch": 0.575934178950977,
      "grad_norm": 2.392066071877973,
      "learning_rate": 4.5668880475865074e-06,
      "logits/chosen": -0.8237022161483765,
      "logits/rejected": -0.7995460629463196,
      "logps/chosen": -0.6559365391731262,
      "logps/rejected": -0.5953893065452576,
      "loss": 0.6716,
      "odds_ratio_loss": 6.677926063537598,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.05953892320394516,
      "rewards/margins": 0.006054719910025597,
      "rewards/rejected": -0.0655936449766159,
      "sft_loss": 0.003847929183393717,
      "step": 105
    },
    {
      "epoch": 0.5814192663695578,
      "grad_norm": 2.2088045238777636,
      "learning_rate": 4.4709687862213866e-06,
      "logits/chosen": -0.7528365254402161,
      "logits/rejected": -0.6947425603866577,
      "logps/chosen": -0.9411884546279907,
      "logps/rejected": -0.6525102853775024,
      "loss": 0.7281,
      "odds_ratio_loss": 7.2357940673828125,
      "rewards/accuracies": 0.4765625,
      "rewards/chosen": -0.06525103747844696,
      "rewards/margins": 0.028867818415164948,
      "rewards/rejected": -0.0941188633441925,
      "sft_loss": 0.0045068394392728806,
      "step": 106
    },
    {
      "epoch": 0.5869043537881385,
      "grad_norm": 3.1924453094544942,
      "learning_rate": 4.3752460383557195e-06,
      "logits/chosen": -0.8217104077339172,
      "logits/rejected": -0.7772889733314514,
      "logps/chosen": -0.7546268701553345,
      "logps/rejected": -0.6607457399368286,
      "loss": 0.7412,
      "odds_ratio_loss": 7.280969619750977,
      "rewards/accuracies": 0.390625,
      "rewards/chosen": -0.06607457995414734,
      "rewards/margins": 0.009388094767928123,
      "rewards/rejected": -0.07546267658472061,
      "sft_loss": 0.013106751255691051,
      "step": 107
    },
    {
      "epoch": 0.5923894412067192,
      "grad_norm": 2.246014553224009,
      "learning_rate": 4.27975536108268e-06,
      "logits/chosen": -0.7625182867050171,
      "logits/rejected": -0.7424900531768799,
      "logps/chosen": -0.7097453474998474,
      "logps/rejected": -0.5825679302215576,
      "loss": 0.6546,
      "odds_ratio_loss": 6.507444381713867,
      "rewards/accuracies": 0.453125,
      "rewards/chosen": -0.05825679004192352,
      "rewards/margins": 0.012717743404209614,
      "rewards/rejected": -0.07097454369068146,
      "sft_loss": 0.003896431066095829,
      "step": 108
    },
    {
      "epoch": 0.5978745286253,
      "grad_norm": 2.107300230600946,
      "learning_rate": 4.184532225290687e-06,
      "logits/chosen": -0.7371281385421753,
      "logits/rejected": -0.7014196515083313,
      "logps/chosen": -0.9409867525100708,
      "logps/rejected": -0.6328235268592834,
      "loss": 0.7067,
      "odds_ratio_loss": 7.03104305267334,
      "rewards/accuracies": 0.4921875,
      "rewards/chosen": -0.06328234821557999,
      "rewards/margins": 0.030816327780485153,
      "rewards/rejected": -0.09409867972135544,
      "sft_loss": 0.003552825888618827,
      "step": 109
    },
    {
      "epoch": 0.6033596160438807,
      "grad_norm": 2.32750784917777,
      "learning_rate": 4.089612002487428e-06,
      "logits/chosen": -0.7617780566215515,
      "logits/rejected": -0.6975070834159851,
      "logps/chosen": -0.8203774690628052,
      "logps/rejected": -0.616916298866272,
      "loss": 0.6908,
      "odds_ratio_loss": 6.856823921203613,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": -0.061691634356975555,
      "rewards/margins": 0.020346110686659813,
      "rewards/rejected": -0.08203774690628052,
      "sft_loss": 0.005122015718370676,
      "step": 110
    },
    {
      "epoch": 0.6088447034624614,
      "grad_norm": 3.687888221085956,
      "learning_rate": 3.995029951660777e-06,
      "logits/chosen": -0.7304787039756775,
      "logits/rejected": -0.6660119295120239,
      "logps/chosen": -0.8241205215454102,
      "logps/rejected": -0.684959888458252,
      "loss": 0.7648,
      "odds_ratio_loss": 7.607235908508301,
      "rewards/accuracies": 0.3984375,
      "rewards/chosen": -0.0684959888458252,
      "rewards/margins": 0.01391606591641903,
      "rewards/rejected": -0.08241204917430878,
      "sft_loss": 0.004041440784931183,
      "step": 111
    },
    {
      "epoch": 0.6143297908810421,
      "grad_norm": 3.815048531172436,
      "learning_rate": 3.900821206181521e-06,
      "logits/chosen": -0.7075001001358032,
      "logits/rejected": -0.7053488492965698,
      "logps/chosen": -0.7641057372093201,
      "logps/rejected": -0.6342787146568298,
      "loss": 0.7087,
      "odds_ratio_loss": 7.047255516052246,
      "rewards/accuracies": 0.5078125,
      "rewards/chosen": -0.0634278729557991,
      "rewards/margins": 0.012982700951397419,
      "rewards/rejected": -0.07641058415174484,
      "sft_loss": 0.0039539458230137825,
      "step": 112
    },
    {
      "epoch": 0.6198148782996229,
      "grad_norm": 2.6437128083822654,
      "learning_rate": 3.8070207607527587e-06,
      "logits/chosen": -0.751364529132843,
      "logits/rejected": -0.7497612237930298,
      "logps/chosen": -0.707420289516449,
      "logps/rejected": -0.6882792711257935,
      "loss": 0.7676,
      "odds_ratio_loss": 7.612734794616699,
      "rewards/accuracies": 0.390625,
      "rewards/chosen": -0.06882793456315994,
      "rewards/margins": 0.001914096181280911,
      "rewards/rejected": -0.07074202597141266,
      "sft_loss": 0.0063278162851929665,
      "step": 113
    },
    {
      "epoch": 0.6252999657182037,
      "grad_norm": 2.249087144528014,
      "learning_rate": 3.7136634584107787e-06,
      "logits/chosen": -0.7711942791938782,
      "logits/rejected": -0.7532557249069214,
      "logps/chosen": -0.6726529598236084,
      "logps/rejected": -0.6081224679946899,
      "loss": 0.6863,
      "odds_ratio_loss": 6.813708305358887,
      "rewards/accuracies": 0.453125,
      "rewards/chosen": -0.060812242329120636,
      "rewards/margins": 0.0064530507661402225,
      "rewards/rejected": -0.06726529449224472,
      "sft_loss": 0.004954230505973101,
      "step": 114
    },
    {
      "epoch": 0.6307850531367843,
      "grad_norm": 2.235141918994794,
      "learning_rate": 3.620783977582305e-06,
      "logits/chosen": -0.7548055648803711,
      "logits/rejected": -0.7235039472579956,
      "logps/chosen": -0.7916484475135803,
      "logps/rejected": -0.6184461116790771,
      "loss": 0.6943,
      "odds_ratio_loss": 6.894530773162842,
      "rewards/accuracies": 0.421875,
      "rewards/chosen": -0.061844613403081894,
      "rewards/margins": 0.017320234328508377,
      "rewards/rejected": -0.07916485518217087,
      "sft_loss": 0.004836176987737417,
      "step": 115
    },
    {
      "epoch": 0.6362701405553651,
      "grad_norm": 2.1938466749292114,
      "learning_rate": 3.528416819202881e-06,
      "logits/chosen": -0.7192294597625732,
      "logits/rejected": -0.695332944393158,
      "logps/chosen": -0.7696835994720459,
      "logps/rejected": -0.6214407086372375,
      "loss": 0.7016,
      "odds_ratio_loss": 6.972049236297607,
      "rewards/accuracies": 0.390625,
      "rewards/chosen": -0.06214407458901405,
      "rewards/margins": 0.014824289828538895,
      "rewards/rejected": -0.07696835696697235,
      "sft_loss": 0.004408481530845165,
      "step": 116
    },
    {
      "epoch": 0.6417552279739458,
      "grad_norm": 3.6541286655490524,
      "learning_rate": 3.43659629390117e-06,
      "logits/chosen": -0.7324870824813843,
      "logits/rejected": -0.688734769821167,
      "logps/chosen": -0.7166416645050049,
      "logps/rejected": -0.6708381175994873,
      "loss": 0.7539,
      "odds_ratio_loss": 7.414391040802002,
      "rewards/accuracies": 0.4140625,
      "rewards/chosen": -0.06708382070064545,
      "rewards/margins": 0.004580352455377579,
      "rewards/rejected": -0.07166417688131332,
      "sft_loss": 0.01244704145938158,
      "step": 117
    },
    {
      "epoch": 0.6472403153925266,
      "grad_norm": 2.2808659065754777,
      "learning_rate": 3.3453565092539586e-06,
      "logits/chosen": -0.7768323421478271,
      "logits/rejected": -0.7245185971260071,
      "logps/chosen": -0.7158573269844055,
      "logps/rejected": -0.616123616695404,
      "loss": 0.694,
      "odds_ratio_loss": 6.893760681152344,
      "rewards/accuracies": 0.4296875,
      "rewards/chosen": -0.06161235645413399,
      "rewards/margins": 0.009973364882171154,
      "rewards/rejected": -0.07158571481704712,
      "sft_loss": 0.0046532335691154,
      "step": 118
    },
    {
      "epoch": 0.6527254028111072,
      "grad_norm": 2.0368022051271844,
      "learning_rate": 3.254731357116597e-06,
      "logits/chosen": -0.7800723910331726,
      "logits/rejected": -0.725759744644165,
      "logps/chosen": -0.7847654819488525,
      "logps/rejected": -0.6137481331825256,
      "loss": 0.6878,
      "odds_ratio_loss": 6.837745189666748,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.06137481704354286,
      "rewards/margins": 0.017101740464568138,
      "rewards/rejected": -0.07847655564546585,
      "sft_loss": 0.003991706296801567,
      "step": 119
    },
    {
      "epoch": 0.658210490229688,
      "grad_norm": 1.9900402701993793,
      "learning_rate": 3.16475450103354e-06,
      "logits/chosen": -0.7510137557983398,
      "logits/rejected": -0.7143795490264893,
      "logps/chosen": -0.6110856533050537,
      "logps/rejected": -0.5404046773910522,
      "loss": 0.614,
      "odds_ratio_loss": 6.107670783996582,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.05404047295451164,
      "rewards/margins": 0.007068089675158262,
      "rewards/rejected": -0.061108559370040894,
      "sft_loss": 0.003193741664290428,
      "step": 120
    },
    {
      "epoch": 0.6636955776482688,
      "grad_norm": 1.9532423296861858,
      "learning_rate": 3.0754593637337276e-06,
      "logits/chosen": -0.7384210824966431,
      "logits/rejected": -0.7613973021507263,
      "logps/chosen": -0.5634211897850037,
      "logps/rejected": -0.6090326309204102,
      "loss": 0.6938,
      "odds_ratio_loss": 6.90548038482666,
      "rewards/accuracies": 0.3828125,
      "rewards/chosen": -0.06090325862169266,
      "rewards/margins": -0.004561137408018112,
      "rewards/rejected": -0.056342121213674545,
      "sft_loss": 0.003293172223493457,
      "step": 121
    },
    {
      "epoch": 0.6691806650668495,
      "grad_norm": 2.314587585640712,
      "learning_rate": 2.986879114715403e-06,
      "logits/chosen": -0.7613508701324463,
      "logits/rejected": -0.7241955995559692,
      "logps/chosen": -0.6129331588745117,
      "logps/rejected": -0.6036895513534546,
      "loss": 0.6794,
      "odds_ratio_loss": 6.749566555023193,
      "rewards/accuracies": 0.453125,
      "rewards/chosen": -0.06036894768476486,
      "rewards/margins": 0.0009243742097169161,
      "rewards/rejected": -0.06129331886768341,
      "sft_loss": 0.004476083908230066,
      "step": 122
    },
    {
      "epoch": 0.6746657524854303,
      "grad_norm": 1.9868042893647633,
      "learning_rate": 2.899046657924992e-06,
      "logits/chosen": -0.721281886100769,
      "logits/rejected": -0.6971719264984131,
      "logps/chosen": -0.6944752931594849,
      "logps/rejected": -0.609091579914093,
      "loss": 0.6844,
      "odds_ratio_loss": 6.81266975402832,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.06090915948152542,
      "rewards/margins": 0.008538373745977879,
      "rewards/rejected": -0.06944753229618073,
      "sft_loss": 0.003112279111519456,
      "step": 123
    },
    {
      "epoch": 0.6801508399040109,
      "grad_norm": 2.489835822157132,
      "learning_rate": 2.8119946195346375e-06,
      "logits/chosen": -0.8069707155227661,
      "logits/rejected": -0.7617440223693848,
      "logps/chosen": -0.7581583857536316,
      "logps/rejected": -0.6261847019195557,
      "loss": 0.6998,
      "odds_ratio_loss": 6.9503560066223145,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": -0.06261847168207169,
      "rewards/margins": 0.013197370804846287,
      "rewards/rejected": -0.0758158341050148,
      "sft_loss": 0.004782428033649921,
      "step": 124
    },
    {
      "epoch": 0.6856359273225917,
      "grad_norm": 2.1133326129430134,
      "learning_rate": 2.725755335822903e-06,
      "logits/chosen": -0.7975918054580688,
      "logits/rejected": -0.7674388885498047,
      "logps/chosen": -0.9080885648727417,
      "logps/rejected": -0.6341651678085327,
      "loss": 0.7058,
      "odds_ratio_loss": 7.018553256988525,
      "rewards/accuracies": 0.484375,
      "rewards/chosen": -0.0634165108203888,
      "rewards/margins": 0.027392340824007988,
      "rewards/rejected": -0.09080885350704193,
      "sft_loss": 0.003976397681981325,
      "step": 125
    },
    {
      "epoch": 0.6911210147411725,
      "grad_norm": 2.858149017181916,
      "learning_rate": 2.6403608411631744e-06,
      "logits/chosen": -0.7425932288169861,
      "logits/rejected": -0.7346398234367371,
      "logps/chosen": -0.7898511290550232,
      "logps/rejected": -0.6491058468818665,
      "loss": 0.735,
      "odds_ratio_loss": 7.295699119567871,
      "rewards/accuracies": 0.3671875,
      "rewards/chosen": -0.06491059064865112,
      "rewards/margins": 0.014074533246457577,
      "rewards/rejected": -0.07898511737585068,
      "sft_loss": 0.005448752082884312,
      "step": 126
    },
    {
      "epoch": 0.6966061021597532,
      "grad_norm": 2.150148574063965,
      "learning_rate": 2.555842856124182e-06,
      "logits/chosen": -0.7330852150917053,
      "logits/rejected": -0.6947073340415955,
      "logps/chosen": -0.7349799871444702,
      "logps/rejected": -0.6045773029327393,
      "loss": 0.678,
      "odds_ratio_loss": 6.737186908721924,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.060457728803157806,
      "rewards/margins": 0.013040272518992424,
      "rewards/rejected": -0.07349800318479538,
      "sft_loss": 0.004280973691493273,
      "step": 127
    },
    {
      "epoch": 0.7020911895783339,
      "grad_norm": 2.0636310647720912,
      "learning_rate": 2.472232775687119e-06,
      "logits/chosen": -0.7301775217056274,
      "logits/rejected": -0.6499804258346558,
      "logps/chosen": -0.7413673400878906,
      "logps/rejected": -0.6203609108924866,
      "loss": 0.6958,
      "odds_ratio_loss": 6.925926208496094,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.06203608959913254,
      "rewards/margins": 0.012100642547011375,
      "rewards/rejected": -0.07413673400878906,
      "sft_loss": 0.0032169800251722336,
      "step": 128
    },
    {
      "epoch": 0.7075762769969146,
      "grad_norm": 2.458823451021782,
      "learning_rate": 2.389561657583681e-06,
      "logits/chosen": -0.738516092300415,
      "logits/rejected": -0.6830987334251404,
      "logps/chosen": -0.9752374887466431,
      "logps/rejected": -0.6493728756904602,
      "loss": 0.7323,
      "odds_ratio_loss": 7.261032581329346,
      "rewards/accuracies": 0.3828125,
      "rewards/chosen": -0.0649372935295105,
      "rewards/margins": 0.03258645534515381,
      "rewards/rejected": -0.09752374142408371,
      "sft_loss": 0.006236766930669546,
      "step": 129
    },
    {
      "epoch": 0.7130613644154954,
      "grad_norm": 2.4560062596870877,
      "learning_rate": 2.30786021075939e-06,
      "logits/chosen": -0.6550124883651733,
      "logits/rejected": -0.7046679258346558,
      "logps/chosen": -0.6320035457611084,
      "logps/rejected": -0.6258726119995117,
      "loss": 0.7064,
      "odds_ratio_loss": 7.010786533355713,
      "rewards/accuracies": 0.4453125,
      "rewards/chosen": -0.06258726119995117,
      "rewards/margins": 0.0006130932597443461,
      "rewards/rejected": -0.06320035457611084,
      "sft_loss": 0.005346982274204493,
      "step": 130
    },
    {
      "epoch": 0.7185464518340761,
      "grad_norm": 2.5029419648282083,
      "learning_rate": 2.2271587839664673e-06,
      "logits/chosen": -0.688187837600708,
      "logits/rejected": -0.6578387022018433,
      "logps/chosen": -0.8063658475875854,
      "logps/rejected": -0.6298286318778992,
      "loss": 0.7054,
      "odds_ratio_loss": 6.999924182891846,
      "rewards/accuracies": 0.453125,
      "rewards/chosen": -0.06298286467790604,
      "rewards/margins": 0.01765371486544609,
      "rewards/rejected": -0.08063658326864243,
      "sft_loss": 0.005409896373748779,
      "step": 131
    },
    {
      "epoch": 0.7240315392526568,
      "grad_norm": 3.1547535885726976,
      "learning_rate": 2.1474873544905204e-06,
      "logits/chosen": -0.6956002712249756,
      "logits/rejected": -0.6422963738441467,
      "logps/chosen": -0.7219193577766418,
      "logps/rejected": -0.6385369300842285,
      "loss": 0.7147,
      "odds_ratio_loss": 7.056730270385742,
      "rewards/accuracies": 0.4765625,
      "rewards/chosen": -0.06385369598865509,
      "rewards/margins": 0.008338242769241333,
      "rewards/rejected": -0.07219193875789642,
      "sft_loss": 0.008979414589703083,
      "step": 132
    },
    {
      "epoch": 0.7295166266712376,
      "grad_norm": 2.0510077565138185,
      "learning_rate": 2.0688755170152e-06,
      "logits/chosen": -0.6617864370346069,
      "logits/rejected": -0.6148036122322083,
      "logps/chosen": -0.6251884698867798,
      "logps/rejected": -0.5985109210014343,
      "loss": 0.6755,
      "odds_ratio_loss": 6.721668243408203,
      "rewards/accuracies": 0.453125,
      "rewards/chosen": -0.05985109508037567,
      "rewards/margins": 0.0026677525602281094,
      "rewards/rejected": -0.06251884996891022,
      "sft_loss": 0.003360216738656163,
      "step": 133
    },
    {
      "epoch": 0.7350017140898183,
      "grad_norm": 2.2969756511886286,
      "learning_rate": 1.9913524726289784e-06,
      "logits/chosen": -0.6497606039047241,
      "logits/rejected": -0.6395284533500671,
      "logps/chosen": -0.7028053402900696,
      "logps/rejected": -0.5877602100372314,
      "loss": 0.6632,
      "odds_ratio_loss": 6.588471412658691,
      "rewards/accuracies": 0.484375,
      "rewards/chosen": -0.058776021003723145,
      "rewards/margins": 0.011504517868161201,
      "rewards/rejected": -0.0702805295586586,
      "sft_loss": 0.004390079528093338,
      "step": 134
    },
    {
      "epoch": 0.7404868015083991,
      "grad_norm": 2.213983061915376,
      "learning_rate": 1.914947017978153e-06,
      "logits/chosen": -0.6579415202140808,
      "logits/rejected": -0.6358848214149475,
      "logps/chosen": -0.7505279183387756,
      "logps/rejected": -0.6064987182617188,
      "loss": 0.6827,
      "odds_ratio_loss": 6.784019470214844,
      "rewards/accuracies": 0.453125,
      "rewards/chosen": -0.06064987555146217,
      "rewards/margins": 0.014402923174202442,
      "rewards/rejected": -0.07505279779434204,
      "sft_loss": 0.004277239087969065,
      "step": 135
    },
    {
      "epoch": 0.7459718889269797,
      "grad_norm": 2.2270439093795993,
      "learning_rate": 1.8396875345700498e-06,
      "logits/chosen": -0.7001329064369202,
      "logits/rejected": -0.6527349948883057,
      "logps/chosen": -0.744738757610321,
      "logps/rejected": -0.6307080388069153,
      "loss": 0.7068,
      "odds_ratio_loss": 7.029452323913574,
      "rewards/accuracies": 0.453125,
      "rewards/chosen": -0.06307081133127213,
      "rewards/margins": 0.011403076350688934,
      "rewards/rejected": -0.07447388023138046,
      "sft_loss": 0.003887307131662965,
      "step": 136
    },
    {
      "epoch": 0.7514569763455605,
      "grad_norm": 2.256827786206518,
      "learning_rate": 1.7656019782304602e-06,
      "logits/chosen": -0.7055901288986206,
      "logits/rejected": -0.6784175038337708,
      "logps/chosen": -0.6831467151641846,
      "logps/rejected": -0.5970955491065979,
      "loss": 0.675,
      "odds_ratio_loss": 6.704926490783691,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.05970955267548561,
      "rewards/margins": 0.008605118840932846,
      "rewards/rejected": -0.06831467151641846,
      "sft_loss": 0.004465795587748289,
      "step": 137
    },
    {
      "epoch": 0.7569420637641412,
      "grad_norm": 2.1013122838813394,
      "learning_rate": 1.6927178687191953e-06,
      "logits/chosen": -0.7317599058151245,
      "logits/rejected": -0.7018448114395142,
      "logps/chosen": -0.6241502165794373,
      "logps/rejected": -0.5972384214401245,
      "loss": 0.6753,
      "odds_ratio_loss": 6.7175421714782715,
      "rewards/accuracies": 0.4140625,
      "rewards/chosen": -0.05972383916378021,
      "rewards/margins": 0.00269118370488286,
      "rewards/rejected": -0.062415026128292084,
      "sft_loss": 0.0035422425717115402,
      "step": 138
    },
    {
      "epoch": 0.762427151182722,
      "grad_norm": 2.042262448509973,
      "learning_rate": 1.621062279507617e-06,
      "logits/chosen": -0.7065898180007935,
      "logits/rejected": -0.6461969614028931,
      "logps/chosen": -0.7283148765563965,
      "logps/rejected": -0.594868004322052,
      "loss": 0.6671,
      "odds_ratio_loss": 6.63283634185791,
      "rewards/accuracies": 0.4921875,
      "rewards/chosen": -0.05948680266737938,
      "rewards/margins": 0.01334468089044094,
      "rewards/rejected": -0.07283148169517517,
      "sft_loss": 0.0038246996700763702,
      "step": 139
    },
    {
      "epoch": 0.7679122386013028,
      "grad_norm": 2.1324897242067022,
      "learning_rate": 1.550661827721941e-06,
      "logits/chosen": -0.7135088443756104,
      "logits/rejected": -0.6793374419212341,
      "logps/chosen": -0.811589241027832,
      "logps/rejected": -0.6043570041656494,
      "loss": 0.6734,
      "odds_ratio_loss": 6.6996307373046875,
      "rewards/accuracies": 0.484375,
      "rewards/chosen": -0.0604357086122036,
      "rewards/margins": 0.020723219960927963,
      "rewards/rejected": -0.08115892857313156,
      "sft_loss": 0.0034452907275408506,
      "step": 140
    },
    {
      "epoch": 0.7733973260198834,
      "grad_norm": 1.8928274566112053,
      "learning_rate": 1.4815426642560753e-06,
      "logits/chosen": -0.7378033399581909,
      "logits/rejected": -0.6366161108016968,
      "logps/chosen": -0.766107976436615,
      "logps/rejected": -0.573859453201294,
      "loss": 0.6475,
      "odds_ratio_loss": 6.4447150230407715,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.057385943830013275,
      "rewards/margins": 0.01922486163675785,
      "rewards/rejected": -0.07661080360412598,
      "sft_loss": 0.0030438436660915613,
      "step": 141
    },
    {
      "epoch": 0.7788824134384642,
      "grad_norm": 1.9538303248376696,
      "learning_rate": 1.4137304640576161e-06,
      "logits/chosen": -0.662543535232544,
      "logits/rejected": -0.619470477104187,
      "logps/chosen": -0.6948879361152649,
      "logps/rejected": -0.6252713799476624,
      "loss": 0.704,
      "odds_ratio_loss": 7.005824089050293,
      "rewards/accuracies": 0.4453125,
      "rewards/chosen": -0.062527135014534,
      "rewards/margins": 0.006961653009057045,
      "rewards/rejected": -0.06948879361152649,
      "sft_loss": 0.003440008033066988,
      "step": 142
    },
    {
      "epoch": 0.7843675008570449,
      "grad_norm": 1.8890336186417962,
      "learning_rate": 1.3472504165906614e-06,
      "logits/chosen": -0.6532924771308899,
      "logits/rejected": -0.6276803016662598,
      "logps/chosen": -0.6293469071388245,
      "logps/rejected": -0.589890718460083,
      "loss": 0.6721,
      "odds_ratio_loss": 6.69063138961792,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.05898907035589218,
      "rewards/margins": 0.003945623058825731,
      "rewards/rejected": -0.06293469667434692,
      "sft_loss": 0.0030280579812824726,
      "step": 143
    },
    {
      "epoch": 0.7898525882756257,
      "grad_norm": 1.8750003224288405,
      "learning_rate": 1.2821272164789544e-06,
      "logits/chosen": -0.7127547264099121,
      "logits/rejected": -0.6667823791503906,
      "logps/chosen": -0.813082754611969,
      "logps/rejected": -0.6018574237823486,
      "loss": 0.6788,
      "odds_ratio_loss": 6.758955478668213,
      "rewards/accuracies": 0.453125,
      "rewards/chosen": -0.0601857453584671,
      "rewards/margins": 0.021122539415955544,
      "rewards/rejected": -0.0813082829117775,
      "sft_loss": 0.0029439451172947884,
      "step": 144
    },
    {
      "epoch": 0.7953376756942063,
      "grad_norm": 2.1968606793011936,
      "learning_rate": 1.2183850543328313e-06,
      "logits/chosen": -0.708900511264801,
      "logits/rejected": -0.689564049243927,
      "logps/chosen": -0.673328697681427,
      "logps/rejected": -0.5980782508850098,
      "loss": 0.6739,
      "odds_ratio_loss": 6.683194160461426,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.059807829558849335,
      "rewards/margins": 0.007525048218667507,
      "rewards/rejected": -0.06733287870883942,
      "sft_loss": 0.005550979636609554,
      "step": 145
    },
    {
      "epoch": 0.8008227631127871,
      "grad_norm": 2.3780917622435966,
      "learning_rate": 1.156047607763407e-06,
      "logits/chosen": -0.7008949518203735,
      "logits/rejected": -0.6693176031112671,
      "logps/chosen": -0.8738444447517395,
      "logps/rejected": -0.6471556425094604,
      "loss": 0.7249,
      "odds_ratio_loss": 7.202227592468262,
      "rewards/accuracies": 0.4140625,
      "rewards/chosen": -0.06471556425094604,
      "rewards/margins": 0.022668881341814995,
      "rewards/rejected": -0.08738444745540619,
      "sft_loss": 0.004629730246961117,
      "step": 146
    },
    {
      "epoch": 0.8063078505313679,
      "grad_norm": 2.0196260017903396,
      "learning_rate": 1.095138032587298e-06,
      "logits/chosen": -0.6812174916267395,
      "logits/rejected": -0.6691566705703735,
      "logps/chosen": -0.8704717755317688,
      "logps/rejected": -0.5937949419021606,
      "loss": 0.6703,
      "odds_ratio_loss": 6.671693801879883,
      "rewards/accuracies": 0.4453125,
      "rewards/chosen": -0.059379495680332184,
      "rewards/margins": 0.027667677029967308,
      "rewards/rejected": -0.08704717457294464,
      "sft_loss": 0.0031060208566486835,
      "step": 147
    },
    {
      "epoch": 0.8117929379499486,
      "grad_norm": 3.415410597307275,
      "learning_rate": 1.0356789542251939e-06,
      "logits/chosen": -0.6848728656768799,
      "logits/rejected": -0.6237005591392517,
      "logps/chosen": -0.8248082995414734,
      "logps/rejected": -0.6273688077926636,
      "loss": 0.6999,
      "odds_ratio_loss": 6.8439483642578125,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": -0.0627368837594986,
      "rewards/margins": 0.019743941724300385,
      "rewards/rejected": -0.08248083293437958,
      "sft_loss": 0.015508507378399372,
      "step": 148
    },
    {
      "epoch": 0.8172780253685293,
      "grad_norm": 2.294183923518428,
      "learning_rate": 9.776924592974257e-07,
      "logits/chosen": -0.6827746033668518,
      "logits/rejected": -0.6851654052734375,
      "logps/chosen": -0.6488504409790039,
      "logps/rejected": -0.6348989605903625,
      "loss": 0.7172,
      "odds_ratio_loss": 7.13347053527832,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.0634898990392685,
      "rewards/margins": 0.0013951496221125126,
      "rewards/rejected": -0.06488504260778427,
      "sft_loss": 0.003804245265200734,
      "step": 149
    },
    {
      "epoch": 0.82276311278711,
      "grad_norm": 2.2280214785837256,
      "learning_rate": 9.212000874196953e-07,
      "logits/chosen": -0.6501573920249939,
      "logits/rejected": -0.6508501768112183,
      "logps/chosen": -0.9070018529891968,
      "logps/rejected": -0.5991308093070984,
      "loss": 0.6697,
      "odds_ratio_loss": 6.647359848022461,
      "rewards/accuracies": 0.5078125,
      "rewards/chosen": -0.05991308391094208,
      "rewards/margins": 0.030787091702222824,
      "rewards/rejected": -0.0907001718878746,
      "sft_loss": 0.004916863515973091,
      "step": 150
    },
    {
      "epoch": 0.8282482002056908,
      "grad_norm": 2.1857741774575277,
      "learning_rate": 8.662228232019876e-07,
      "logits/chosen": -0.6549192667007446,
      "logits/rejected": -0.6354272365570068,
      "logps/chosen": -0.8838653564453125,
      "logps/rejected": -0.6490474939346313,
      "loss": 0.7277,
      "odds_ratio_loss": 7.240221977233887,
      "rewards/accuracies": 0.3984375,
      "rewards/chosen": -0.06490474939346313,
      "rewards/margins": 0.023481788113713264,
      "rewards/rejected": -0.08838653564453125,
      "sft_loss": 0.003684336319565773,
      "step": 151
    },
    {
      "epoch": 0.8337332876242715,
      "grad_norm": 1.987517923655104,
      "learning_rate": 8.127810884536402e-07,
      "logits/chosen": -0.6637884378433228,
      "logits/rejected": -0.6511223912239075,
      "logps/chosen": -0.7829376459121704,
      "logps/rejected": -0.6296254992485046,
      "loss": 0.7088,
      "odds_ratio_loss": 7.052569389343262,
      "rewards/accuracies": 0.421875,
      "rewards/chosen": -0.06296255439519882,
      "rewards/margins": 0.015331214293837547,
      "rewards/rejected": -0.07829376310110092,
      "sft_loss": 0.003527900902554393,
      "step": 152
    },
    {
      "epoch": 0.8392183750428522,
      "grad_norm": 2.378533291363515,
      "learning_rate": 7.60894734597476e-07,
      "logits/chosen": -0.7232801914215088,
      "logits/rejected": -0.6934952139854431,
      "logps/chosen": -0.673111081123352,
      "logps/rejected": -0.6153658628463745,
      "loss": 0.6995,
      "odds_ratio_loss": 6.946869850158691,
      "rewards/accuracies": 0.3671875,
      "rewards/chosen": -0.06153658404946327,
      "rewards/margins": 0.005774518009275198,
      "rewards/rejected": -0.06731110066175461,
      "sft_loss": 0.00481205340474844,
      "step": 153
    },
    {
      "epoch": 0.844703462461433,
      "grad_norm": 2.105440987324829,
      "learning_rate": 7.105830352958143e-07,
      "logits/chosen": -0.6993785500526428,
      "logits/rejected": -0.6673418879508972,
      "logps/chosen": -0.8246726989746094,
      "logps/rejected": -0.6575340032577515,
      "loss": 0.7351,
      "odds_ratio_loss": 7.309325695037842,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.06575340777635574,
      "rewards/margins": 0.016713865101337433,
      "rewards/rejected": -0.08246727287769318,
      "sft_loss": 0.004190374165773392,
      "step": 154
    },
    {
      "epoch": 0.8501885498800137,
      "grad_norm": 2.0837906888947284,
      "learning_rate": 6.618646792910893e-07,
      "logits/chosen": -0.6814290881156921,
      "logits/rejected": -0.6457995772361755,
      "logps/chosen": -0.7051143050193787,
      "logps/rejected": -0.5800508856773376,
      "loss": 0.6505,
      "odds_ratio_loss": 6.475788593292236,
      "rewards/accuracies": 0.5234375,
      "rewards/chosen": -0.058005087077617645,
      "rewards/margins": 0.012506341561675072,
      "rewards/rejected": -0.07051143050193787,
      "sft_loss": 0.002879747888073325,
      "step": 155
    },
    {
      "epoch": 0.8556736372985945,
      "grad_norm": 2.3480039986855235,
      "learning_rate": 6.147577634637413e-07,
      "logits/chosen": -0.709331214427948,
      "logits/rejected": -0.6663718223571777,
      "logps/chosen": -0.7591659426689148,
      "logps/rejected": -0.6053435802459717,
      "loss": 0.6828,
      "odds_ratio_loss": 6.782125949859619,
      "rewards/accuracies": 0.4296875,
      "rewards/chosen": -0.060534361749887466,
      "rewards/margins": 0.015382234007120132,
      "rewards/rejected": -0.0759166032075882,
      "sft_loss": 0.004561963025480509,
      "step": 156
    },
    {
      "epoch": 0.8611587247171751,
      "grad_norm": 2.0141939824459105,
      "learning_rate": 5.692797861099719e-07,
      "logits/chosen": -0.6299980282783508,
      "logits/rejected": -0.6576442122459412,
      "logps/chosen": -0.6320803761482239,
      "logps/rejected": -0.5932613611221313,
      "loss": 0.6692,
      "odds_ratio_loss": 6.658140659332275,
      "rewards/accuracies": 0.4453125,
      "rewards/chosen": -0.059326138347387314,
      "rewards/margins": 0.0038819059263914824,
      "rewards/rejected": -0.06320804357528687,
      "sft_loss": 0.0034193547908216715,
      "step": 157
    },
    {
      "epoch": 0.8666438121357559,
      "grad_norm": 2.117743395165982,
      "learning_rate": 5.254476404418341e-07,
      "logits/chosen": -0.68840092420578,
      "logits/rejected": -0.661196231842041,
      "logps/chosen": -0.740635871887207,
      "logps/rejected": -0.5854509472846985,
      "loss": 0.6559,
      "odds_ratio_loss": 6.521589756011963,
      "rewards/accuracies": 0.4921875,
      "rewards/chosen": -0.05854509398341179,
      "rewards/margins": 0.015518485568463802,
      "rewards/rejected": -0.07406358420848846,
      "sft_loss": 0.003763651242479682,
      "step": 158
    },
    {
      "epoch": 0.8721288995543367,
      "grad_norm": 2.0431260947506273,
      "learning_rate": 4.832776083120983e-07,
      "logits/chosen": -0.6590278148651123,
      "logits/rejected": -0.5882821679115295,
      "logps/chosen": -0.6959192752838135,
      "logps/rejected": -0.5968501567840576,
      "loss": 0.6729,
      "odds_ratio_loss": 6.690352916717529,
      "rewards/accuracies": 0.484375,
      "rewards/chosen": -0.05968501418828964,
      "rewards/margins": 0.009906908497214317,
      "rewards/rejected": -0.06959191709756851,
      "sft_loss": 0.0038192402571439743,
      "step": 159
    },
    {
      "epoch": 0.8776139869729174,
      "grad_norm": 2.06927462291822,
      "learning_rate": 4.4278535416620914e-07,
      "logits/chosen": -0.7277648448944092,
      "logits/rejected": -0.6717500686645508,
      "logps/chosen": -0.6366525888442993,
      "logps/rejected": -0.5973187685012817,
      "loss": 0.6734,
      "odds_ratio_loss": 6.694884300231934,
      "rewards/accuracies": 0.4296875,
      "rewards/chosen": -0.059731870889663696,
      "rewards/margins": 0.003933383151888847,
      "rewards/rejected": -0.06366526335477829,
      "sft_loss": 0.003877094015479088,
      "step": 160
    },
    {
      "epoch": 0.8830990743914982,
      "grad_norm": 2.5991781469777684,
      "learning_rate": 4.0398591922357787e-07,
      "logits/chosen": -0.6706148386001587,
      "logits/rejected": -0.6340373158454895,
      "logps/chosen": -0.7028053402900696,
      "logps/rejected": -0.5866550207138062,
      "loss": 0.6609,
      "odds_ratio_loss": 6.551386833190918,
      "rewards/accuracies": 0.484375,
      "rewards/chosen": -0.05866550654172897,
      "rewards/margins": 0.011615024879574776,
      "rewards/rejected": -0.0702805295586586,
      "sft_loss": 0.005807496141642332,
      "step": 161
    },
    {
      "epoch": 0.8885841618100788,
      "grad_norm": 1.9365443243739644,
      "learning_rate": 3.6689371589039013e-07,
      "logits/chosen": -0.66963130235672,
      "logits/rejected": -0.5979880690574646,
      "logps/chosen": -0.7448896169662476,
      "logps/rejected": -0.6330960392951965,
      "loss": 0.7074,
      "odds_ratio_loss": 7.0415358543396,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.06330960988998413,
      "rewards/margins": 0.011179355904459953,
      "rewards/rejected": -0.07448896020650864,
      "sft_loss": 0.00325138121843338,
      "step": 162
    },
    {
      "epoch": 0.8940692492286596,
      "grad_norm": 2.121200239152531,
      "learning_rate": 3.315225224059809e-07,
      "logits/chosen": -0.6617680191993713,
      "logits/rejected": -0.6018663644790649,
      "logps/chosen": -0.8706423044204712,
      "logps/rejected": -0.6283361911773682,
      "loss": 0.7038,
      "odds_ratio_loss": 6.99859094619751,
      "rewards/accuracies": 0.453125,
      "rewards/chosen": -0.06283361464738846,
      "rewards/margins": 0.024230612441897392,
      "rewards/rejected": -0.087064228951931,
      "sft_loss": 0.003959609195590019,
      "step": 163
    },
    {
      "epoch": 0.8995543366472403,
      "grad_norm": 2.148488564632459,
      "learning_rate": 2.9788547772478416e-07,
      "logits/chosen": -0.6609708666801453,
      "logits/rejected": -0.6172842979431152,
      "logps/chosen": -0.7381229400634766,
      "logps/rejected": -0.6342223882675171,
      "loss": 0.7124,
      "odds_ratio_loss": 7.068850040435791,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.06342223286628723,
      "rewards/margins": 0.010390058159828186,
      "rewards/rejected": -0.07381229847669601,
      "sft_loss": 0.005511356517672539,
      "step": 164
    },
    {
      "epoch": 0.905039424065821,
      "grad_norm": 2.1106479655528796,
      "learning_rate": 2.6599507663574387e-07,
      "logits/chosen": -0.6930041313171387,
      "logits/rejected": -0.666252851486206,
      "logps/chosen": -0.6971046328544617,
      "logps/rejected": -0.6307591199874878,
      "loss": 0.7082,
      "odds_ratio_loss": 7.04301643371582,
      "rewards/accuracies": 0.4453125,
      "rewards/chosen": -0.06307591497898102,
      "rewards/margins": 0.006634547375142574,
      "rewards/rejected": -0.06971046328544617,
      "sft_loss": 0.003914778120815754,
      "step": 165
    },
    {
      "epoch": 0.9105245114844018,
      "grad_norm": 2.12980458488719,
      "learning_rate": 2.3586316512101416e-07,
      "logits/chosen": -0.6665065288543701,
      "logits/rejected": -0.6230036616325378,
      "logps/chosen": -0.7909663915634155,
      "logps/rejected": -0.5812108516693115,
      "loss": 0.6543,
      "odds_ratio_loss": 6.501043796539307,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.05812108889222145,
      "rewards/margins": 0.02097555249929428,
      "rewards/rejected": -0.07909663766622543,
      "sft_loss": 0.004211761057376862,
      "step": 166
    },
    {
      "epoch": 0.9160095989029825,
      "grad_norm": 1.9925848439582086,
      "learning_rate": 2.0750093595565735e-07,
      "logits/chosen": -0.6866216659545898,
      "logits/rejected": -0.6430922150611877,
      "logps/chosen": -0.7136643528938293,
      "logps/rejected": -0.6055830121040344,
      "loss": 0.681,
      "odds_ratio_loss": 6.776293754577637,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.06055830419063568,
      "rewards/margins": 0.010808136314153671,
      "rewards/rejected": -0.07136643677949905,
      "sft_loss": 0.0033255324233323336,
      "step": 167
    },
    {
      "epoch": 0.9214946863215633,
      "grad_norm": 1.7865099751357647,
      "learning_rate": 1.8091892454998595e-07,
      "logits/chosen": -0.7047858238220215,
      "logits/rejected": -0.6222947835922241,
      "logps/chosen": -0.8674630522727966,
      "logps/rejected": -0.635217547416687,
      "loss": 0.7125,
      "odds_ratio_loss": 7.095528602600098,
      "rewards/accuracies": 0.4140625,
      "rewards/chosen": -0.06352175027132034,
      "rewards/margins": 0.023224547505378723,
      "rewards/rejected": -0.08674629777669907,
      "sft_loss": 0.0029704533517360687,
      "step": 168
    },
    {
      "epoch": 0.926979773740144,
      "grad_norm": 2.4300450960067157,
      "learning_rate": 1.561270050360897e-07,
      "logits/chosen": -0.6807259321212769,
      "logits/rejected": -0.6181145310401917,
      "logps/chosen": -0.939436674118042,
      "logps/rejected": -0.6324220895767212,
      "loss": 0.7055,
      "odds_ratio_loss": 7.014206886291504,
      "rewards/accuracies": 0.5078125,
      "rewards/chosen": -0.06324220448732376,
      "rewards/margins": 0.03070145845413208,
      "rewards/rejected": -0.09394367039203644,
      "sft_loss": 0.0040901475585997105,
      "step": 169
    },
    {
      "epoch": 0.9324648611587247,
      "grad_norm": 1.9641250789279827,
      "learning_rate": 1.33134386599994e-07,
      "logits/chosen": -0.7125731110572815,
      "logits/rejected": -0.673245370388031,
      "logps/chosen": -0.9093432426452637,
      "logps/rejected": -0.6548852324485779,
      "loss": 0.7325,
      "odds_ratio_loss": 7.290490627288818,
      "rewards/accuracies": 0.3984375,
      "rewards/chosen": -0.06548852473497391,
      "rewards/margins": 0.025445803999900818,
      "rewards/rejected": -0.09093432873487473,
      "sft_loss": 0.003472922369837761,
      "step": 170
    },
    {
      "epoch": 0.9379499485773054,
      "grad_norm": 2.2978574983286086,
      "learning_rate": 1.1194961006082972e-07,
      "logits/chosen": -0.6797432899475098,
      "logits/rejected": -0.6302465200424194,
      "logps/chosen": -0.826444685459137,
      "logps/rejected": -0.6436266899108887,
      "loss": 0.7225,
      "odds_ratio_loss": 7.156667709350586,
      "rewards/accuracies": 0.453125,
      "rewards/chosen": -0.06436267495155334,
      "rewards/margins": 0.01828179880976677,
      "rewards/rejected": -0.08264447748661041,
      "sft_loss": 0.006842226721346378,
      "step": 171
    },
    {
      "epoch": 0.9434350359958862,
      "grad_norm": 2.3466230771755594,
      "learning_rate": 9.258054469825972e-08,
      "logits/chosen": -0.7173478007316589,
      "logits/rejected": -0.6696880459785461,
      "logps/chosen": -0.6279891133308411,
      "logps/rejected": -0.6100993752479553,
      "loss": 0.6933,
      "odds_ratio_loss": 6.895969390869141,
      "rewards/accuracies": 0.4296875,
      "rewards/chosen": -0.06100994348526001,
      "rewards/margins": 0.0017889718292281032,
      "rewards/rejected": -0.06279890984296799,
      "sft_loss": 0.0036887084133923054,
      "step": 172
    },
    {
      "epoch": 0.948920123414467,
      "grad_norm": 2.2629683724373226,
      "learning_rate": 7.503438532937169e-08,
      "logits/chosen": -0.6861296892166138,
      "logits/rejected": -0.6311776041984558,
      "logps/chosen": -0.6609523296356201,
      "logps/rejected": -0.6280564069747925,
      "loss": 0.7091,
      "odds_ratio_loss": 7.0425190925598145,
      "rewards/accuracies": 0.3984375,
      "rewards/chosen": -0.0628056451678276,
      "rewards/margins": 0.003289599437266588,
      "rewards/rejected": -0.06609524041414261,
      "sft_loss": 0.00483954232186079,
      "step": 173
    },
    {
      "epoch": 0.9544052108330476,
      "grad_norm": 2.3036615374451106,
      "learning_rate": 5.9317649636088656e-08,
      "logits/chosen": -0.6945533156394958,
      "logits/rejected": -0.6203831434249878,
      "logps/chosen": -0.8298579454421997,
      "logps/rejected": -0.6409691572189331,
      "loss": 0.7149,
      "odds_ratio_loss": 7.09760856628418,
      "rewards/accuracies": 0.4765625,
      "rewards/chosen": -0.06409691274166107,
      "rewards/margins": 0.018888890743255615,
      "rewards/rejected": -0.08298580348491669,
      "sft_loss": 0.005160990636795759,
      "step": 174
    },
    {
      "epoch": 0.9598902982516284,
      "grad_norm": 2.450600909672264,
      "learning_rate": 4.543617574412185e-08,
      "logits/chosen": -0.7120048403739929,
      "logits/rejected": -0.6609210968017578,
      "logps/chosen": -0.8656575679779053,
      "logps/rejected": -0.6917891502380371,
      "loss": 0.7722,
      "odds_ratio_loss": 7.664754390716553,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.06917892396450043,
      "rewards/margins": 0.01738683693110943,
      "rewards/rejected": -0.08656575530767441,
      "sft_loss": 0.0056978315114974976,
      "step": 175
    },
    {
      "epoch": 0.9653753856702091,
      "grad_norm": 2.0318496856067942,
      "learning_rate": 3.339512005434309e-08,
      "logits/chosen": -0.6582708954811096,
      "logits/rejected": -0.5941011905670166,
      "logps/chosen": -0.7900950908660889,
      "logps/rejected": -0.5978555083274841,
      "loss": 0.6728,
      "odds_ratio_loss": 6.6846771240234375,
      "rewards/accuracies": 0.484375,
      "rewards/chosen": -0.05978555232286453,
      "rewards/margins": 0.019223961979150772,
      "rewards/rejected": -0.079009510576725,
      "sft_loss": 0.004282682668417692,
      "step": 176
    },
    {
      "epoch": 0.9708604730887899,
      "grad_norm": 2.101347721071758,
      "learning_rate": 2.319895532739369e-08,
      "logits/chosen": -0.6375358700752258,
      "logits/rejected": -0.6700392961502075,
      "logps/chosen": -0.7632755637168884,
      "logps/rejected": -0.653388261795044,
      "loss": 0.7326,
      "odds_ratio_loss": 7.288682460784912,
      "rewards/accuracies": 0.4296875,
      "rewards/chosen": -0.06533882021903992,
      "rewards/margins": 0.010988726280629635,
      "rewards/rejected": -0.07632754743099213,
      "sft_loss": 0.003733065677806735,
      "step": 177
    },
    {
      "epoch": 0.9763455605073705,
      "grad_norm": 2.1469363578193033,
      "learning_rate": 1.4851469022234e-08,
      "logits/chosen": -0.6644502878189087,
      "logits/rejected": -0.6307498216629028,
      "logps/chosen": -0.6507246494293213,
      "logps/rejected": -0.5957509875297546,
      "loss": 0.6743,
      "odds_ratio_loss": 6.702306270599365,
      "rewards/accuracies": 0.421875,
      "rewards/chosen": -0.059575099498033524,
      "rewards/margins": 0.00549736525863409,
      "rewards/rejected": -0.06507246941328049,
      "sft_loss": 0.004071300383657217,
      "step": 178
    },
    {
      "epoch": 0.9818306479259513,
      "grad_norm": 2.2611714428995313,
      "learning_rate": 8.35576188926046e-09,
      "logits/chosen": -0.685587465763092,
      "logits/rejected": -0.6573466658592224,
      "logps/chosen": -0.7864770889282227,
      "logps/rejected": -0.5809304714202881,
      "loss": 0.6574,
      "odds_ratio_loss": 6.511574745178223,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.058093056082725525,
      "rewards/margins": 0.020554661750793457,
      "rewards/rejected": -0.07864771038293839,
      "sft_loss": 0.006258727982640266,
      "step": 179
    },
    {
      "epoch": 0.9873157353445321,
      "grad_norm": 2.10857188318165,
      "learning_rate": 3.71424681850141e-09,
      "logits/chosen": -0.7009637355804443,
      "logits/rejected": -0.7090278267860413,
      "logps/chosen": -0.7139046788215637,
      "logps/rejected": -0.6147310137748718,
      "loss": 0.6931,
      "odds_ratio_loss": 6.891450881958008,
      "rewards/accuracies": 0.4453125,
      "rewards/chosen": -0.06147310137748718,
      "rewards/margins": 0.009917369112372398,
      "rewards/rejected": -0.07139047980308533,
      "sft_loss": 0.0039411671459674835,
      "step": 180
    },
    {
      "epoch": 0.9928008227631128,
      "grad_norm": 2.18736540958863,
      "learning_rate": 9.286479433257e-10,
      "logits/chosen": -0.6564709544181824,
      "logits/rejected": -0.5686830878257751,
      "logps/chosen": -0.8073873519897461,
      "logps/rejected": -0.573136031627655,
      "loss": 0.6412,
      "odds_ratio_loss": 6.370337009429932,
      "rewards/accuracies": 0.546875,
      "rewards/chosen": -0.05731360614299774,
      "rewards/margins": 0.023425137624144554,
      "rewards/rejected": -0.08073873817920685,
      "sft_loss": 0.004156089387834072,
      "step": 181
    },
    {
      "epoch": 0.9982859101816935,
      "grad_norm": 2.121915449515644,
      "learning_rate": 0.0,
      "logits/chosen": -0.7368127703666687,
      "logits/rejected": -0.6576196551322937,
      "logps/chosen": -1.124916672706604,
      "logps/rejected": -0.5984194278717041,
      "loss": 0.6649,
      "odds_ratio_loss": 6.61328125,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": -0.05984194204211235,
      "rewards/margins": 0.05264972895383835,
      "rewards/rejected": -0.1124916821718216,
      "sft_loss": 0.0035864519886672497,
      "step": 182
    },
    {
      "epoch": 0.9982859101816935,
      "step": 182,
      "total_flos": 58779245903872.0,
      "train_loss": 0.7302135016236987,
      "train_runtime": 13700.9712,
      "train_samples_per_second": 1.703,
      "train_steps_per_second": 0.013
    }
  ],
  "logging_steps": 1,
  "max_steps": 182,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 182,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 58779245903872.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}